From 3f6eddd5c20f1fa3df6ba1cc208efbe3e2fffb43 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Wed, 2 Sep 2026 19:23:46 -0700 Subject: [PATCH] Add compressed 171-byte OTA transport --- docs/ota_protocol.md | 59 +- platformio.ini | 4 + src/helpers/ota/MotaSeederProto.h | 13 +- src/helpers/ota/MotaSourceSerial.cpp | 33 +- src/helpers/ota/MotaSourceSerial.h | 2 + src/helpers/ota/OtaContext.h | 5 + src/helpers/ota/OtaDeflate.cpp | 28 + src/helpers/ota/OtaDeflate.h | 16 + src/helpers/ota/OtaManager.cpp | 306 +++++++++- src/helpers/ota/OtaManager.h | 66 ++- src/helpers/ota/OtaProtocol.h | 69 ++- src/helpers/ota/OtaSource.h | 15 + src/helpers/ota/OtaTinf.c | 17 + src/helpers/ota/tinf/LICENSE | 22 + src/helpers/ota/tinf/README.meshcore.txt | 23 + src/helpers/ota/tinf/tinf.h | 154 +++++ src/helpers/ota/tinf/tinflate.c | 666 +++++++++++++++++++++ test/test_ota/test_ota_core.cpp | 705 +++++++++++++++++++++++ tools/ble_mota/ble_mota_seeder.py | 48 ++ tools/ble_mota/test_ble_mota_seeder.py | 68 +++ 20 files changed, 2273 insertions(+), 46 deletions(-) create mode 100644 src/helpers/ota/OtaDeflate.cpp create mode 100644 src/helpers/ota/OtaDeflate.h create mode 100644 src/helpers/ota/OtaTinf.c create mode 100644 src/helpers/ota/tinf/LICENSE create mode 100644 src/helpers/ota/tinf/README.meshcore.txt create mode 100644 src/helpers/ota/tinf/tinf.h create mode 100644 src/helpers/ota/tinf/tinflate.c diff --git a/docs/ota_protocol.md b/docs/ota_protocol.md index 5bf8bf4e..e6314931 100644 --- a/docs/ota_protocol.md +++ b/docs/ota_protocol.md @@ -544,8 +544,13 @@ All offsets after the 1-byte type. Encoders/decoders in `OtaProtocol.cpp`; const OTA_GET_MANIFEST: manifest_id[4] want_mask(uint16) # bit k = send manifest fragment k; 0xFFFF = all OTA_MANIFEST: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up to OTA_MF_FRAG=176 B/frag OTA_REQ: manifest_id[4] { block_idx(uint16) want_mask(uint16) }[1..4] - # one or more rows; bit k = send fragment k of that block -OTA_DATA: manifest_id[4] block_idx(uint16) frag_off(uint16) data[] # up to OTA_FRAG_DATA=160 B + # legacy: bit k requests its 160-byte fragment + # v2: bit15=marker, bit14=allow transport DEFLATE, bits0..12=fragment bitmap +OTA_DATA legacy: manifest_id[4] block_idx(uint16) frag_off(uint16) data[<=160] +OTA_DATA v2: manifest_id[4] block_idx(uint16) descriptor(uint16) + stream_id[4] data[<=171] + # descriptor: bit15=marker, bit14=DEFLATE, bits13..10=fragment, + # bits9..0=complete encoded length minus one OTA_REQ_PROOF: manifest_id[4] block_idx(uint16) OTA_PROOF: manifest_id[4] block_idx(uint16) n_proof(1) proof[] # n_proof x 4 bytes OTA_GET_LEAVES: manifest_id[4] want_mask(uint16) # bit k = send leaves fragment k; 0xFFFF = all @@ -565,10 +570,30 @@ OTA_LEAVES: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up fetch. **Normal P2P nodes never use this** - they target only the blocks they want; the only always-on part is answering `OTA_GET_LEAVES` with leaves the node already holds, so any node's firmware can be captured. -- **Block <-> fragments:** a 1 KB block remains split into self-describing `OTA_DATA` fragments. `frag_off` is the - byte offset of `data` within the block, so the global position is `block_idx*block_size + frag_off` - - a fragment is self-placing when returned by the source. The fetcher tracks a - per-block slice bitmap and reassembles before requesting the proof. +- **Negotiated 171-byte fragments:** the deployed profile remains unchanged: `frag_off` is a byte offset and + `data[]` carries at most 160 bytes. A new fetcher marks an `OTA_REQ` row as v2 and includes all seven legacy + bits on its first request. Old sources ignore/mask the high flags and can therefore return a complete legacy + block immediately; receipt of an untagged `OTA_DATA` switches that fetch session to the old geometry. If no + v2 data appears by the first adaptive deadline, the fetcher retries with an ordinary legacy mask. New + sources answer v2 with a packed descriptor, a repeated 4-byte representation ID, and exactly 171 data bytes + except the final fragment. A raw 1 KiB block therefore falls from seven data packets to six. Message type + IDs do not change, so multi-hop relays continue to forward request/data/proof packets opaquely with the same + priority. +- **Transport-only DEFLATE:** a fetcher sets the v2 DEFLATE-permission bit only when its application includes + and configures the decoder. A source may encode each logical block as an independent raw RFC 1951 stream + with full stored, fixed-Huffman, and dynamic-Huffman support (`BTYPE=0/1/2`). Compression is used only when + strictly smaller, otherwise the source returns raw v2 data. The + receiver inflates to the manifest-derived block length, then performs the unchanged Merkle proof and writes + the original bytes to staging. Thus existing `.mota` containers, signatures, resume markers, and legacy + bootloader apply remain unchanged. The 4-byte `stream_id` is SHA-256:4 of the complete raw or compressed + representation and appears in every fragment. A receiver locks `{encoding, encoded length, stream_id}` for + a block, preventing fragments from independently encoded seeders from being mixed. After one sparse retry, + a second stalled interval clears the full in-flight window and switches the session to legacy geometry, so + a vanished v2 seeder cannot prevent an older source from taking over; proof verification remains unchanged. + Every `ENABLE_OTA` MeshCore application registers the vendored tinf 1.2.1 full raw decoder; its wrapper + bounds output to the exact logical block length and rejects truncation, malformed streams, and trailing + whole bytes. Builds without OTA do not link the decoder. This application capability is independent of the + bootloader because transport data is inflated before the unchanged staged container is written. - **Adaptive flight size is not signed block size.** The container continues to use 1 KiB Merkle leaves and each slot is one existing 1 KiB block. A clean link changes how many of those blocks one `OTA_REQ` names: 1, then 2, then 3, then the compiled cap. The manifest stores `block_size_log2`, so 3 KiB is not a valid @@ -589,7 +614,8 @@ OTA_LEAVES: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up half-duplex radios: re-requesting a whole multi-fragment burst let the periodic retry (a transmit) collide with the tail of the in-flight burst and drop the same fragment forever - a hang. Requesting only the hole removes the burst, so there is nothing to collide with. The block/manifest mask matches the 16-bit - reassembly bitmap (<=16 fragments/block; 1 KB blocks = 7). `OTA_PROOF` is a single packet and needs no mask. + reassembly bitmap (<=16 fragments/block; legacy 1 KB blocks = 7, raw v2 = 6). `OTA_PROOF` is a single packet + and needs no mask. - **Data and proof remain separate packets, without a normal extra round trip.** A server retains requested blocks in a bounded descriptor queue, admits at most one response per main-loop pass, and sends one `OTA_PROOF` after each block's requested fragments. Before admitting that proactive proof, the source leaves @@ -605,7 +631,8 @@ OTA_LEAVES: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up | message | fixed overhead | payload/packet | |---|---|---| -| `OTA_DATA` | 9 B (type+mid4+idx2+off2) | `OTA_FRAG_DATA = 160` -> 7 frags per 1 KB block | +| `OTA_DATA` legacy | 9 B (type+mid4+idx2+off2) | `OTA_FRAG_DATA = 160` -> 7 frags per 1 KB block | +| `OTA_DATA` v2 | 13 B (legacy header + stream ID) | `OTA_FRAG_DATA_V2 = 171` -> 6 raw frags per 1 KB block; fewer when compressed | | `OTA_MANIFEST` | 7 B | `OTA_MF_FRAG = 176` -> signed manifest ~ 2 frags | | `OTA_HAVE` | 12 B | 10 rows x 16 B per fragment | | `OTA_PROOF` | 8 B | up to ~44 sibling digests (>> any real tree) | @@ -715,12 +742,15 @@ drives USB-serial, BLE, a WiFi URL list, an NFS/samba mount, etc. - only `read() ```cpp struct MotaDesc { // catalog metadata + region offsets (no whole image in RAM) uint8_t mid[4]; uint32_t target_id, fw_version; uint8_t codec_id, flags; + uint8_t block_size_log2, source_caps; uint32_t total_size, leaves_off, block_count, payload_off, payload_size; }; class MotaSource { virtual uint8_t count(); // # mOTAs offered virtual bool describe(uint8_t idx, MotaDesc& out); // metadata + offsets virtual bool read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len); // random-access bytes + virtual bool read_deflated_block(uint8_t idx, uint16_t block, + uint8_t* buf, uint16_t cap, uint16_t* len); // optional raw RFC 1951 }; ``` @@ -745,9 +775,11 @@ response (host -> device): 'm' 's' op(1) status(1) payload... xsum(1 = XO OP_COUNT 0x01 args: - -> payload: count(1) OP_DESCRIBE 0x02 args: idx(1) -> payload: MotaDesc wire (38 B) OP_READ 0x03 args: idx(1) off(4) len(2) -> payload: len bytes +OP_DEFLATE_BLOCK 0x09 args: idx(1) block(2) off(2) len(2) + -> payload: total_encoded_len(2) + requested bytes MotaDesc wire (38 B): mid[4] target_id(4) fw_version(4) codec(1) flags(1) total_size(4) leaves_off(4) block_count(4) payload_off(4) payload_size(4) - block_size_log2(1) reserved(3) + block_size_log2(1) source_caps(1) reserved(2) status: 0 = OK, non-zero = error (out of range / past EOF). ``` @@ -758,6 +790,15 @@ CDC/UART receive rings even though the host successfully wrote the complete reply. Chunking is internal to the transport and does not change `OP_READ` or the `MotaSource` random-access contract. +`OP_DEFLATE_BLOCK` lets a host-folder source perform the optional transport compression without linking an +encoder into the embedded seeder. `len=0, off=0` queries the exact encoded length; subsequent chunks are at +most 190 bytes, keeping the total response payload at 192 bytes. The host independently encodes each manifest +payload block as ordinary raw RFC 1951 at level 9. It returns an error for an invalid range, an unsupported +operation, or a result that is not smaller than the raw block; `SerialMotaSource` then serves the ordinary raw +v2 representation. A supporting host sets `source_caps bit 0` in every descriptor. Older hosts leave that +formerly-reserved byte zero, so upgraded firmware skips the optional request instead of waiting for an old +daemon that silently ignores unknown operations. + Manifest fragments are retained as bounded response jobs and admitted one at a time. Their source-side gap follows the active maximum packet airtime and dispatcher duty spacing, clamped to 100-1000 ms. The 100 ms floor protects diff --git a/platformio.ini b/platformio.ini index 3a37d47d..4fe2deea 100644 --- a/platformio.ini +++ b/platformio.ini @@ -49,6 +49,7 @@ build_flags = -w -DNDEBUG -DRADIOLIB_STATIC_ONLY=1 -DRADIOLIB_GODMODE=1 build_src_filter = +<*.cpp> + + + ; MQTT-only sources (need Timezone/NTPClient); observer envs add them back. - - @@ -355,6 +356,7 @@ build_flags = -std=c++17 -I src -I test/mocks -D OTA_FETCH_PIPELINE=4 + -D OTA_TRANSPORT_DEFLATE_TEST=1 test_build_src = yes test_ignore = test_kiss_modem build_src_filter = @@ -374,6 +376,8 @@ build_src_filter = +<../src/helpers/ota/FolderMotaStore.cpp> +<../src/helpers/ota/OtaProtocol.cpp> +<../src/helpers/ota/OtaManager.cpp> + +<../src/helpers/ota/OtaDeflate.cpp> + +<../src/helpers/ota/OtaTinf.c> +<../src/helpers/ota/detools/detools.c> +<../src/helpers/UserGpio.cpp> +<../src/helpers/ConfigSerializer.cpp> diff --git a/src/helpers/ota/MotaSeederProto.h b/src/helpers/ota/MotaSeederProto.h index 9291637d..06d28b9f 100644 --- a/src/helpers/ota/MotaSeederProto.h +++ b/src/helpers/ota/MotaSeederProto.h @@ -17,10 +17,17 @@ // OP_COUNT 0x01 args: - resp payload: count(1) // OP_DESCRIBE 0x02 args: idx(1) resp payload: MotaDesc wire (38 B, see below) [status OK] // OP_READ 0x03 args: idx(1) off(4) len(2) resp payload: len bytes [status OK] +// OP_DEFLATE_BLOCK 0x09 args: idx(1) block(2) off(2) len(2) +// resp payload: total_encoded_len(2) + len encoded bytes [status OK] +// `len=0` queries the size. Chunks are <=190 bytes so the full response payload remains +// <=192. The host returns ERR unless independent raw-RFC1951 compression is smaller than +// the original logical block; callers then use ordinary OP_READ/raw OTA_DATA v2. // // MotaDesc wire (38 B): mid[4] target_id(4) fw_version(4) codec(1) flags(1) total_size(4) // leaves_off(4) block_count(4) payload_off(4) payload_size(4) -// block_size_log2(1) reserved(3) +// block_size_log2(1) source_caps(1) reserved(2) +// source_caps bit 0 = OP_DEFLATE_BLOCK is implemented. Deployed hosts send zero, so a new node never +// waits on an unsupported operation during a rolling host upgrade. // // --- STORAGE ops (device -> host WRITE): "pull to folder". The device is fetching a `.mota` off the mesh // (e.g. a neighbour's self-served firmware it has no local copy of) and streams it into the host folder as @@ -52,14 +59,16 @@ static const uint8_t MS_OP_BEGIN = 0x05; // storage: create/truncate static const uint8_t MS_OP_WRITE = 0x06; // storage: write bytes at offset static const uint8_t MS_OP_SREAD = 0x07; // storage: read bytes back (resume: recompute missing blocks) static const uint8_t MS_OP_FIN = 0x08; // storage: transfer complete - validate + make servable +static const uint8_t MS_OP_DEFLATE_BLOCK = 0x09; // source: independently compressed payload block/chunk static const uint8_t MS_STATUS_OK = 0x00; static const uint8_t MS_STATUS_ERR = 0x01; -static const uint16_t MOTA_DESC_WIRE = 38; // existing reserved bytes carry block geometry; wire size is unchanged +static const uint16_t MOTA_DESC_WIRE = 38; // existing reserved bytes carry geometry/caps; wire size is unchanged // Keep host-to-device replies below common 256-byte USB CDC/UART receive rings. Manifest leaves can exceed // that ring, and payload reads are normally 1 KB; splitting them here prevents silent overrun/checksum loss. static const uint16_t MOTA_SEEDER_READ_MAX = 192; +static const uint16_t MOTA_SEEDER_DEFLATE_CHUNK_MAX = 190; // plus uint16 total length = 192-byte response payload static const uint16_t MOTA_SEEDER_WRITE_MAX = 512; // max data bytes per OP_WRITE/OP_SREAD (bounds frames) } // namespace ota diff --git a/src/helpers/ota/MotaSourceSerial.cpp b/src/helpers/ota/MotaSourceSerial.cpp index d806c0c6..fba687d9 100644 --- a/src/helpers/ota/MotaSourceSerial.cpp +++ b/src/helpers/ota/MotaSourceSerial.cpp @@ -86,7 +86,8 @@ bool SerialMotaSource::describe(uint8_t idx, MotaDesc& out) { out.payload_off = rd_u32le(w + 26); out.payload_size = rd_u32le(w + 30); out.block_size_log2 = w[34]; - // bytes [35,38) remain reserved (zero) for forward compatibility. + out.source_caps = w[35]; + // bytes [36,38) remain reserved (zero) for forward compatibility. return true; } @@ -107,5 +108,35 @@ bool SerialMotaSource::read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t le return true; } +bool SerialMotaSource::read_deflated_block(uint8_t idx, uint16_t block, uint8_t* buf, + uint16_t cap, uint16_t* len) { + if (!buf || !len || cap == 0) return false; + *len = 0; + + uint8_t args[7] = { idx }; + args[1] = (uint8_t)(block & 0xFF); args[2] = (uint8_t)(block >> 8); + // Query the independently compressed representation's exact size first. The old host returns an error for + // the unknown operation, which is the intentional raw fallback during a rolling deployment. + uint8_t total_wire[2]; + if (!txn(MS_OP_DEFLATE_BLOCK, args, sizeof(args), total_wire, sizeof(total_wire))) return false; + const uint16_t total = rd_u16le(total_wire); + if (total == 0 || total > cap) return false; + + uint16_t done = 0; + while (done < total) { + const uint16_t chunk = (uint16_t)((total - done > MOTA_SEEDER_DEFLATE_CHUNK_MAX) + ? MOTA_SEEDER_DEFLATE_CHUNK_MAX : total - done); + args[3] = (uint8_t)(done & 0xFF); args[4] = (uint8_t)(done >> 8); + args[5] = (uint8_t)(chunk & 0xFF); args[6] = (uint8_t)(chunk >> 8); + uint8_t response[2 + MOTA_SEEDER_DEFLATE_CHUNK_MAX]; + if (!txn(MS_OP_DEFLATE_BLOCK, args, sizeof(args), response, (uint16_t)(2 + chunk)) || + rd_u16le(response) != total) return false; + memcpy(buf + done, response + 2, chunk); + done = (uint16_t)(done + chunk); + } + *len = total; + return true; +} + } // namespace ota } // namespace mesh diff --git a/src/helpers/ota/MotaSourceSerial.h b/src/helpers/ota/MotaSourceSerial.h index ae56d758..3b8daf80 100644 --- a/src/helpers/ota/MotaSourceSerial.h +++ b/src/helpers/ota/MotaSourceSerial.h @@ -22,6 +22,8 @@ public: uint8_t count() override; bool describe(uint8_t idx, MotaDesc& out) override; bool read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len) override; + bool read_deflated_block(uint8_t idx, uint16_t block, uint8_t* buf, + uint16_t cap, uint16_t* len) override; private: // Send a request (op+args) and read its response header; on OK, `payload` (if non-null) receives diff --git a/src/helpers/ota/OtaContext.h b/src/helpers/ota/OtaContext.h index c5ffe526..017667a3 100644 --- a/src/helpers/ota/OtaContext.h +++ b/src/helpers/ota/OtaContext.h @@ -4,6 +4,7 @@ #include // malloc/free (lazy ESP32 manual-stage buffer) #include // strncmp/strncpy (hw_id) #include "OtaManager.h" +#include "OtaDeflate.h" #include "OtaStore.h" #include "SignerAllowlist.h" #include "OtaApply.h" @@ -562,6 +563,10 @@ struct OtaContext { target_id = 0; #endif manager.begin(target_id, send, ctx); + // Every OTA-capable application can receive ordinary raw RFC1951 transport blocks. OtaManager advertises + // that permission only because this exact, bounded decoder is registered; non-OTA builds do not compile + // OtaContext or link the decoder implementation. + manager.set_transport_deflate_decoder(ota_transport_inflate); manager.set_auto_version_floor(_fi.valid ? _fi.fw_version : 0, true); fetch_to_folder = false; if (hw) { strncpy(hw_id, hw, sizeof(hw_id) - 1); hw_id[sizeof(hw_id) - 1] = 0; } diff --git a/src/helpers/ota/OtaDeflate.cpp b/src/helpers/ota/OtaDeflate.cpp new file mode 100644 index 00000000..e0b310e9 --- /dev/null +++ b/src/helpers/ota/OtaDeflate.cpp @@ -0,0 +1,28 @@ +#include "OtaDeflate.h" + +#if defined(ENABLE_OTA) || defined(OTA_TRANSPORT_DEFLATE_TEST) + +extern "C" { +#include "tinf/tinf.h" +} + +namespace mesh { +namespace ota { + +bool ota_transport_inflate(void* context, const uint8_t* src, uint16_t src_len, + uint8_t* dst, uint16_t dst_cap, uint16_t* dst_len) { + (void)context; + if (dst_len) *dst_len = 0; + if (!src || src_len == 0 || !dst || dst_cap == 0 || !dst_len) return false; + + unsigned int produced = dst_cap; + const int result = tinf_uncompress_exact(dst, &produced, src, src_len); + if (result != TINF_OK || produced != dst_cap) return false; + *dst_len = (uint16_t)produced; + return true; +} + +} // namespace ota +} // namespace mesh + +#endif // ENABLE_OTA || OTA_TRANSPORT_DEFLATE_TEST diff --git a/src/helpers/ota/OtaDeflate.h b/src/helpers/ota/OtaDeflate.h new file mode 100644 index 00000000..5cbadd41 --- /dev/null +++ b/src/helpers/ota/OtaDeflate.h @@ -0,0 +1,16 @@ +#pragma once + +#include + +namespace mesh { +namespace ota { + +// Full raw RFC1951 transport decoder (stored, fixed-Huffman, and dynamic-Huffman blocks). +// `dst_cap` is the exact expected logical block length, not merely spare capacity. Success +// requires exact output and exact whole-byte input consumption; malformed/truncated streams, +// output overflow, and trailing bytes fail closed. +bool ota_transport_inflate(void* context, const uint8_t* src, uint16_t src_len, + uint8_t* dst, uint16_t dst_cap, uint16_t* dst_len); + +} // namespace ota +} // namespace mesh diff --git a/src/helpers/ota/OtaManager.cpp b/src/helpers/ota/OtaManager.cpp index 64d67e89..2150864e 100644 --- a/src/helpers/ota/OtaManager.cpp +++ b/src/helpers/ota/OtaManager.cpp @@ -69,6 +69,7 @@ bool OtaManager::serve(const uint8_t* mota, uint32_t len) { if (!mota_parse(mota, len, _view0.m)) return false; _view0.mfl = (uint16_t)(_view0.m.leaves - _view0.m.manifest_start); // contiguous container _view0.read = nullptr; _view0.read_ctx = nullptr; // payload is contiguous _view0.m.payload + _view0.read_deflated = nullptr; _view0.deflate_ctx = nullptr; _view0.scratch = scratch; _view0.scratch_sz = OTA_PROOFGEN_SCRATCH; // <=1024 blocks (RAM .mota is small) _view0.valid = true; clearPendingEgress(); @@ -86,6 +87,7 @@ bool OtaManager::serve_self(const uint8_t* manifest, uint16_t mfl, const uint8_t _view0.m.payload = nullptr; // read on demand via `read` _view0.m.block_count = block_count; _view0.mfl = mfl; _view0.read = read; _view0.read_ctx = ctx; + _view0.read_deflated = nullptr; _view0.deflate_ctx = nullptr; _view0.scratch = proof_scratch; _view0.scratch_sz = proof_scratch_sz; // sized for our (large) image _view0.valid = true; clearPendingEgress(); @@ -259,6 +261,9 @@ bool OtaManager::loadSource(const ServeEntry& e) { _srcv_rdctx.src = e.src; _srcv_rdctx.idx = e.src_idx; _srcv_rdctx.payload_off = d.payload_off; _srcv.read = srcReadTramp; _srcv.read_ctx = &_srcv_rdctx; + _srcv.read_deflated = (d.source_caps & MOTA_SOURCE_CAP_DEFLATE_BLOCK) + ? srcDeflateReadTramp : nullptr; + _srcv.deflate_ctx = _srcv.read_deflated ? &_srcv_rdctx : nullptr; _srcv.scratch = scratch; _srcv.scratch_sz = OTA_PROOFGEN_SCRATCH; memcpy(_srcv_mid, d.mid, 4); _srcv.valid = true; @@ -271,6 +276,12 @@ bool OtaManager::srcReadTramp(void* c, uint32_t off, uint8_t* buf, uint32_t len) return x->src->read(x->idx, x->payload_off + off, buf, len); } +bool OtaManager::srcDeflateReadTramp(void* c, uint16_t block, uint8_t* buf, + uint16_t cap, uint16_t* len) { + SrcReadCtx* x = (SrcReadCtx*)c; + return x->src->read_deflated_block(x->idx, block, buf, cap, len); +} + // sha2-256:4 over the SORTED set of mids we serve - peers use it to tell if our offering changed. Sorting // makes it canonical across nodes regardless of insert order; for a single mota it is mh4(mid) (unchanged). void OtaManager::setDigest(uint8_t out[4]) const { @@ -414,10 +425,15 @@ static inline uint16_t frag_full_mask(uint32_t nf) { // Retain a bounded descriptor instead of allocating every DATA packet inline. Repeated requests that arrive // while a block is already queued merge only fragments not yet admitted to the radio queue; a later retry can // enqueue the block again if one of those admitted packets was actually lost over the air. -bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want_mask) { +bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want_mask, + bool wire_v2, bool allow_deflate) { for (uint8_t i = 0; i < _n_serve_jobs; i++) { ServeJob& job = _serve_jobs[i]; if (job.block != block || memcmp(job.mid, mid, 4) != 0) continue; + // A proof-only request can complete whichever representation is already queued. DATA requests with + // different geometries remain distinct because their bitmap positions do not describe the same bytes. + if (want_mask != 0 && (job.wire_v2 != wire_v2 || + (wire_v2 && job.allow_deflate != allow_deflate))) continue; job.pending_mask |= (uint16_t)(want_mask & ~job.emitted_mask); if (want_mask == 0) job.proof_requested = true; return true; @@ -430,6 +446,8 @@ bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want job.emitted_mask = 0; job.proof_ready_at = 0; job.proof_requested = want_mask == 0; + job.wire_v2 = wire_v2; + job.allow_deflate = allow_deflate; return true; } @@ -475,6 +493,10 @@ void OtaManager::clearPendingEgress() { _n_serve_jobs = 0; _serve_block_len = 0; _serve_block_loaded = false; + _serve_wire_block = nullptr; + _serve_wire_len = 0; + _serve_wire_deflated = false; + memset(_serve_wire_id, 0, sizeof(_serve_wire_id)); } void OtaManager::popManifestJob() { @@ -546,6 +568,10 @@ void OtaManager::popServeJob() { _n_serve_jobs--; _serve_block_len = 0; _serve_block_loaded = false; + _serve_wire_block = nullptr; + _serve_wire_len = 0; + _serve_wire_deflated = false; + memset(_serve_wire_id, 0, sizeof(_serve_wire_id)); } bool OtaManager::loadActiveServeBlock() { @@ -563,6 +589,30 @@ bool OtaManager::loadActiveServeBlock() { memcpy(_serve_block, v->m.payload + off, blen); } _serve_block_len = (uint16_t)blen; + _serve_wire_block = _serve_block; + _serve_wire_len = _serve_block_len; + _serve_wire_deflated = false; + + // A host-backed MotaSource can provide a pre-compressed block without putting an encoder in firmware. + // Other sources may opt into a caller-supplied encoder. Both write into the proof scratch, which is safe + // until every DATA fragment has left; proof generation reuses it only after that point. + if (job.wire_v2 && job.allow_deflate && v->scratch && v->scratch_sz >= blen) { + uint16_t encoded_len = 0; + bool encoded = false; + if (v->read_deflated) { + encoded = v->read_deflated(v->deflate_ctx, job.block, v->scratch, + (uint16_t)blen, &encoded_len); + } else if (_deflate_encode) { + encoded = _deflate_encode(_deflate_encode_ctx, _serve_block, (uint16_t)blen, + v->scratch, (uint16_t)blen, &encoded_len); + } + if (encoded && encoded_len > 0 && encoded_len < blen) { + _serve_wire_block = v->scratch; + _serve_wire_len = encoded_len; + _serve_wire_deflated = true; + } + } + mh4(_serve_wire_id, _serve_wire_block, _serve_wire_len); _serve_block_loaded = true; return true; } @@ -580,9 +630,17 @@ bool OtaManager::handleReq(const uint8_t* m, uint16_t n) { uint32_t blen = block_size; const uint32_t off = idx * block_size; if (off + blen > v->m.payload_size) blen = v->m.payload_size - off; - const uint16_t valid_mask = frag_full_mask((blen + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA); - const uint16_t want = (uint16_t)(rq.items[i].want_mask & valid_mask); - if (want != 0) accepted |= queueServeJob(v->m.merkle_root, (uint16_t)idx, want); + const bool wire_v2 = ota_req_is_v2(rq.items[i].want_mask); + const bool allow_deflate = wire_v2 && + (rq.items[i].want_mask & OTA_REQ_V2_ALLOW_DEFLATE) != 0; + const uint16_t valid_mask = frag_full_mask((blen + + (wire_v2 ? OTA_FRAG_DATA_V2 : OTA_FRAG_DATA) - 1) / + (wire_v2 ? OTA_FRAG_DATA_V2 : OTA_FRAG_DATA)); + const uint16_t requested = wire_v2 + ? ota_req_v2_fragments(rq.items[i].want_mask) : rq.items[i].want_mask; + const uint16_t want = (uint16_t)(requested & valid_mask); + if (want != 0) accepted |= queueServeJob(v->m.merkle_root, (uint16_t)idx, want, + wire_v2, allow_deflate); } return accepted; } @@ -630,21 +688,41 @@ void OtaManager::serviceEgress() { popServeJob(); return; } + const uint16_t fragment_data = job.wire_v2 ? OTA_FRAG_DATA_V2 : OTA_FRAG_DATA; + const uint16_t valid_mask = frag_full_mask( + (_serve_wire_len + fragment_data - 1u) / fragment_data); + job.pending_mask &= valid_mask; + if (job.pending_mask == 0) { + job.proof_ready_at = _now_ms + proofEgressGapMs(); + return; + } uint8_t fragment = 0; while (fragment < 16 && !(job.pending_mask & (1u << fragment))) fragment++; - uint32_t frag_off = (uint32_t)fragment * OTA_FRAG_DATA; - if (fragment >= 16 || frag_off >= _serve_block_len) { + uint32_t frag_off = (uint32_t)fragment * fragment_data; + if (fragment >= 16 || frag_off >= _serve_wire_len) { job.pending_mask = 0; return; } - uint32_t frag_len = _serve_block_len - frag_off; - if (frag_len > OTA_FRAG_DATA) frag_len = OTA_FRAG_DATA; + uint32_t frag_len = _serve_wire_len - frag_off; + if (frag_len > fragment_data) frag_len = fragment_data; DataMsg dm; memcpy(dm.manifest_id, job.mid, 4); dm.block_idx = job.block; - dm.frag_off = (uint16_t)frag_off; - dm.data = _serve_block + frag_off; - dm.data_len = (uint16_t)frag_len; + uint8_t v2_data[OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2]; + if (job.wire_v2) { + if (!ota_data_v2_pack(fragment, _serve_wire_len, _serve_wire_deflated, dm.frag_off)) { + popServeJob(); + return; + } + memcpy(v2_data, _serve_wire_id, OTA_DATA_V2_STREAM_ID_BYTES); + memcpy(v2_data + OTA_DATA_V2_STREAM_ID_BYTES, _serve_wire_block + frag_off, frag_len); + dm.data = v2_data; + dm.data_len = (uint16_t)(OTA_DATA_V2_STREAM_ID_BYTES + frag_len); + } else { + dm.frag_off = (uint16_t)frag_off; + dm.data = _serve_block + frag_off; + dm.data_len = (uint16_t)frag_len; + } uint8_t b[MAX_PACKET_PAYLOAD]; if (emit(b, encode_data(b, sizeof(b), dm), false)) { const uint16_t bit = (uint16_t)(1u << fragment); @@ -944,6 +1022,11 @@ void OtaManager::clearReassemblySlot(uint8_t slot) { _reasm[slot].need = 0; _reasm[slot].awaiting_proof = false; _reasm[slot].proof_request_at = 0; + _reasm[slot].encoded_len = 0; + _reasm[slot].wire_v2 = false; + _reasm[slot].deflated = false; + memset(_reasm[slot].stream_id, 0, sizeof(_reasm[slot].stream_id)); + _reasm[slot].wire_stalls = 0; } // Forget all blocks currently being reassembled or awaiting proofs. @@ -956,6 +1039,35 @@ void OtaManager::clearReassembly() { noteFetchActivity(); } +bool OtaManager::anyWireDataReceived() const { + for (uint8_t slot = 0; slot < OTA_FETCH_PIPELINE; slot++) { + if (_reasm[slot].block != NO_BLOCK && _reasm[slot].mask != 0) return true; + } + return false; +} + +void OtaManager::downgradeWireToLegacy() { + if (!_wire_v2_session) return; + _wire_v2_session = false; + _wire_v2_confirmed = false; + _wire_allow_deflate = false; + _wire_empty_stalls = 0; + for (uint8_t i = 0; i < OTA_FETCH_PIPELINE; i++) { + ReassemblySlot& slot = _reasm[i]; + if (slot.block == NO_BLOCK) continue; + slot.mask = 0; + slot.need = frag_full_mask((blockLen(slot.block) + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA); + slot.awaiting_proof = false; + slot.proof_request_at = 0; + slot.encoded_len = 0; + slot.wire_v2 = false; + slot.deflated = false; + memset(slot.stream_id, 0, sizeof(slot.stream_id)); + slot.wire_stalls = 0; + } + OTA_DBG("OTA: source uses legacy 160-byte DATA geometry\n"); +} + int OtaManager::findReassemblySlot(uint32_t block) const { for (uint8_t slot = 0; slot < OTA_FETCH_PIPELINE; slot++) { if (_reasm[slot].block == block) return slot; @@ -1007,7 +1119,8 @@ uint32_t OtaManager::fetchRetryTimeoutMs() const { : (uint32_t)count_mask_bits((uint16_t)(slot.need & ~slot.mask)) + 1u; } if (active == 0) { - const uint32_t fragments = (_fbs + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA; + const uint32_t fragment_data = _wire_v2_session ? OTA_FRAG_DATA_V2 : OTA_FRAG_DATA; + const uint32_t fragments = (_fbs + fragment_data - 1) / fragment_data; packets += (fragments + 1) * _pipeline_width; } uint32_t path_transmissions = _observed_path_transmissions; @@ -1129,6 +1242,10 @@ OtaManager::PullResult OtaManager::startFetch(const uint8_t* mid, uint32_t targe if (!_fetch) return PULL_NO_STORE; if (fetchActive()) return PULL_BUSY; _fetch_error = FETCH_ERROR_NONE; + _wire_v2_session = _wire_v2_enabled; + _wire_v2_confirmed = false; + _wire_allow_deflate = _wire_v2_session && _deflate_decode != nullptr; + _wire_empty_stalls = 0; _fexpected_target = target; _validate = validate; // motatool folder-capture warm-start (seed leaf-diff) // A validate pull is a FRESH seed capture, not a resume: the store already holds the seed's payload (not a @@ -1149,6 +1266,10 @@ OtaManager::PullResult OtaManager::startFetch(const uint8_t* mid, uint32_t targe bool OtaManager::resumeStagedExplicit(const uint8_t* want_mid, uint32_t expected_target) { if (!want_mid || !_fetch || fetchActive()) return false; _fetch_error = FETCH_ERROR_NONE; + _wire_v2_session = _wire_v2_enabled; + _wire_v2_confirmed = false; + _wire_allow_deflate = _wire_v2_session && _deflate_decode != nullptr; + _wire_empty_stalls = 0; _validate = false; _archive_fetch = false; _desired_target = expected_target; @@ -1350,6 +1471,10 @@ bool OtaManager::resumeStaged(const uint8_t* want_mid) { || _fstate == WANT_LEAVES || _fstate == VERIFYING_STAGED) { return false; } + _wire_v2_session = _wire_v2_enabled; + _wire_v2_confirmed = false; + _wire_allow_deflate = _wire_v2_session && _deflate_decode != nullptr; + _wire_empty_stalls = 0; if (!_fetch->reopen()) return false; // nothing persisted in the store uint32_t total = _fetch->staged_size(); uint8_t hdr[8]; @@ -1509,19 +1634,100 @@ bool OtaManager::handleData(const uint8_t* m, uint16_t n) { if (slot_index < 0) return false; // not one of this node's requested blocks ReassemblySlot& slot = _reasm[slot_index]; uint32_t blen = blockLen(dm.block_idx); - if (dm.frag_off % OTA_FRAG_DATA != 0) return false; // canonical FRAG_DATA-aligned slices only - if ((uint32_t)dm.frag_off + dm.data_len > blen) return false; // slice out of the block - uint32_t expected_len = blen - dm.frag_off; - if (expected_len > OTA_FRAG_DATA) expected_len = OTA_FRAG_DATA; - if (dm.data_len != expected_len) return false; // do not mark a short slice as complete - uint32_t kf = dm.frag_off / OTA_FRAG_DATA; - if (kf >= 16) return false; - const uint16_t bit = (uint16_t)(1u << kf); - if (slot.mask & bit) return true; // duplicate flood/retry copy is terminal too - memcpy(slot.buf + dm.frag_off, dm.data, dm.data_len); - slot.mask |= bit; + + if ((dm.frag_off & OTA_DATA_V2_MARK) == 0) { + if (dm.frag_off % OTA_FRAG_DATA != 0) return false; // canonical legacy-aligned slices only + if ((uint32_t)dm.frag_off + dm.data_len > blen) return false; + uint32_t expected_len = blen - dm.frag_off; + if (expected_len > OTA_FRAG_DATA) expected_len = OTA_FRAG_DATA; + if (dm.data_len != expected_len) return false; // do not mark a short slice as complete + uint32_t kf = dm.frag_off / OTA_FRAG_DATA; + if (kf >= 16) return false; + // A deployed source ignores the v2 request bits, masks them against its seven-fragment geometry, and + // responds here. Only a fully canonical legacy slice may select that profile; malformed matching-MID + // traffic must not force a session-wide downgrade. Never mix it with already accepted v2 bytes. + if (_wire_v2_session) { + if (_wire_v2_confirmed || anyWireDataReceived()) return false; + downgradeWireToLegacy(); + } + const uint16_t bit = (uint16_t)(1u << kf); + if (slot.mask & bit) return true; // duplicate flood/retry copy is terminal too + memcpy(slot.buf + dm.frag_off, dm.data, dm.data_len); + slot.mask |= bit; + } else { + if (!_wire_v2_session || dm.data_len <= OTA_DATA_V2_STREAM_ID_BYTES) return false; + uint8_t fragment = 0; + uint16_t encoded_len = 0; + bool deflated = false; + if (!ota_data_v2_unpack(dm.frag_off, fragment, encoded_len, deflated)) return false; + if (encoded_len > blen || (deflated ? encoded_len >= blen : encoded_len != blen)) return false; + const uint16_t fragment_len = (uint16_t)(dm.data_len - OTA_DATA_V2_STREAM_ID_BYTES); + const uint32_t fragment_off = (uint32_t)fragment * OTA_FRAG_DATA_V2; + if (fragment_off >= encoded_len || fragment_off + fragment_len > encoded_len) return false; + uint32_t expected_len = encoded_len - fragment_off; + if (expected_len > OTA_FRAG_DATA_V2) expected_len = OTA_FRAG_DATA_V2; + if (fragment_len != expected_len) return false; + _wire_v2_confirmed = true; + + const uint8_t* stream_id = dm.data; + if (slot.encoded_len == 0) { + slot.encoded_len = encoded_len; + slot.wire_v2 = true; + slot.deflated = deflated; + memcpy(slot.stream_id, stream_id, sizeof(slot.stream_id)); + slot.mask = 0; + slot.need = frag_full_mask((encoded_len + OTA_FRAG_DATA_V2 - 1) / OTA_FRAG_DATA_V2); + } else if (!slot.wire_v2 || slot.encoded_len != encoded_len || slot.deflated != deflated || + memcmp(slot.stream_id, stream_id, sizeof(slot.stream_id)) != 0) { + return false; // another seeder's representation; never mix + } + _wire_empty_stalls = 0; // a valid v2 source is still serving this flight + const uint16_t bit = (uint16_t)(1u << fragment); + if (!(slot.need & bit)) return false; + if (slot.mask & bit) return true; + memcpy(slot.buf + fragment_off, dm.data + OTA_DATA_V2_STREAM_ID_BYTES, fragment_len); + slot.mask |= bit; + slot.wire_stalls = 0; + } noteFetchActivity(); if (slot.mask != slot.need || slot.awaiting_proof) return true; + + if (slot.wire_v2) { + uint8_t stream_id[4]; + mh4(stream_id, slot.buf, slot.encoded_len); + if (memcmp(stream_id, slot.stream_id, sizeof(stream_id)) != 0) { + slot.mask = 0; // complete bytes do not match their representation id + slot.encoded_len = 0; + slot.wire_v2 = false; + slot.deflated = false; + memset(slot.stream_id, 0, sizeof(slot.stream_id)); + notePipelineStall(); + return true; + } + if (slot.deflated) { + uint16_t decoded_len = 0; + bool decoded = _deflate_decode && _deflate_decode( + _deflate_decode_ctx, slot.buf, slot.encoded_len, + _serve_block, (uint16_t)blen, &decoded_len); + // _serve_block is also the outbound server cache. Invalidate it even on failure so a simultaneous + // serve job reloads its own authenticated block rather than transmitting decoder workspace. + _serve_block_loaded = false; + _serve_wire_block = nullptr; + _serve_wire_len = 0; + if (!decoded || decoded_len != blen) { + _wire_allow_deflate = false; // retry this/future blocks as raw v2 + slot.mask = 0; + slot.need = frag_full_mask((blen + OTA_FRAG_DATA_V2 - 1) / OTA_FRAG_DATA_V2); + slot.encoded_len = 0; + slot.wire_v2 = false; + slot.deflated = false; + memset(slot.stream_id, 0, sizeof(slot.stream_id)); + notePipelineStall(); + return true; + } + memcpy(slot.buf, _serve_block, blen); + } + } // The paced server sends a proof immediately after this block's requested DATA. Wait briefly so that proof // can arrive without another request/response turn. An older server, or a lost proof, falls back through // serviceEgress() to the existing REQ_PROOF wire message. @@ -1579,6 +1785,22 @@ bool OtaManager::handleProof(const uint8_t* m, uint16_t n) { return true; } +uint16_t OtaManager::wireRequestMask(uint32_t block, uint16_t need, uint16_t received, + uint16_t encoded_len) const { + uint16_t want = (uint16_t)(need & ~received); + if (_wire_v2_session) { + // Until a v2 representation answers, include every legacy fragment bit. A deployed source ignores the + // high profile flags and can therefore return a complete seven-fragment block on this very first request. + if (encoded_len == 0) { + want = frag_full_mask((blockLen(block) + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA); + } + if (want == 0) want = need; + return ota_req_make_v2(want, _wire_allow_deflate); + } + if (want == 0) want = need; + return want; +} + bool OtaManager::requestSlot(uint8_t slot_index) { if (_fstate != FETCHING || slot_index >= OTA_FETCH_PIPELINE) return false; ReassemblySlot& slot = _reasm[slot_index]; @@ -1597,8 +1819,7 @@ bool OtaManager::requestSlot(uint8_t slot_index) { } return sent; } - uint16_t want = (uint16_t)(slot.need & ~slot.mask); - if (want == 0) want = slot.need; // safety: never send an empty request + uint16_t want = wireRequestMask(slot.block, slot.need, slot.mask, slot.encoded_len); ReqMsg rq; memcpy(rq.manifest_id, _fid, 4); rq.block_idx = (uint16_t)slot.block; rq.want_mask = want; uint8_t b[16]; @@ -1619,8 +1840,7 @@ bool OtaManager::requestFlight() { if (slot.block == NO_BLOCK || slot.awaiting_proof) continue; ReqItem& item = request.items[request.n_items++]; item.block_idx = (uint16_t)slot.block; - item.want_mask = (uint16_t)(slot.need & ~slot.mask); - if (item.want_mask == 0) item.want_mask = slot.need; + item.want_mask = wireRequestMask(slot.block, slot.need, slot.mask, slot.encoded_len); } if (request.n_items == 0) return false; _req_start = request.items[0].block_idx; @@ -1645,7 +1865,8 @@ bool OtaManager::fillPipeline() { ReassemblySlot& slot = _reasm[slot_index]; slot.block = block; slot.mask = 0; - slot.need = frag_full_mask((blockLen(block) + OTA_FRAG_DATA - 1) / OTA_FRAG_DATA); + const uint16_t fragment_data = _wire_v2_session ? OTA_FRAG_DATA_V2 : OTA_FRAG_DATA; + slot.need = frag_full_mask((blockLen(block) + fragment_data - 1) / fragment_data); slot.awaiting_proof = false; slot.proof_request_at = 0; assigned++; @@ -1660,13 +1881,42 @@ void OtaManager::requestMissing() { if (_fstate != FETCHING) return; if (activePipelineSlots() == 0) { fillPipeline(); return; } + // No v2 DATA at all means the request may have reached only deployed sources which ignored its profile + // bits (or the first response burst was lost). Retry once in the universal legacy profile instead of + // pinning the session to a feature it has not observed on air. + if (_wire_v2_session && !_wire_v2_confirmed && !anyWireDataReceived()) { + downgradeWireToLegacy(); + notePipelineStall(); + requestFlight(); + return; + } + // The airtime-aware deadline expired. Retry one slot's holes per deadline, round-robin, so a weak block // cannot trigger a simultaneous multi-block re-burst on a half-duplex channel. Recovery dirties the // flight; its successor is reduced (4->2, 3->2, 2->1) after all current slots finish. notePipelineStall(); + + // Once v2 has been confirmed, the source can still disappear cleanly between blocks (or after a bad + // representation is discarded), leaving every active slot empty. Give the v2 flight one sparse retry, + // then clear the whole flight and ask again in the universal legacy profile so an old seeder can take over. + if (_wire_v2_session && _wire_v2_confirmed && !anyWireDataReceived() && + ++_wire_empty_stalls >= 2) { + downgradeWireToLegacy(); + requestFlight(); + return; + } + for (uint8_t offset = 0; offset < OTA_FETCH_PIPELINE; offset++) { uint8_t slot = (uint8_t)((_retry_slot + offset) % OTA_FETCH_PIPELINE); if (_reasm[slot].block == NO_BLOCK) continue; + ReassemblySlot& reassembly = _reasm[slot]; + if (_wire_v2_session && reassembly.wire_v2 && reassembly.mask != 0 && + !reassembly.awaiting_proof && ++reassembly.wire_stalls >= 2) { + // A v2 seeder may disappear after establishing a representation while only deployed legacy seeders + // remain. Keep one sparse retry, then fall back the whole flight to the universal profile. Clearing + // every slot prevents geometry/representation mixing and guarantees that old sources can take over. + downgradeWireToLegacy(); + } if (requestSlot(slot)) _retry_slot = (uint8_t)((slot + 1) % OTA_FETCH_PIPELINE); return; } diff --git a/src/helpers/ota/OtaManager.h b/src/helpers/ota/OtaManager.h index b29e02bf..d913bb2b 100644 --- a/src/helpers/ota/OtaManager.h +++ b/src/helpers/ota/OtaManager.h @@ -31,6 +31,22 @@ typedef bool (*OtaSend)(void* ctx, const uint8_t* msg, uint16_t len, bool flood) // error. nullptr means the payload is a contiguous RAM buffer (the staged `.mota`). typedef bool (*ServeReadFn)(void* ctx, uint32_t off, uint8_t* buf, uint32_t len); +// Optional full raw-RFC1951 transport codec (stored, fixed-Huffman, and dynamic-Huffman blocks). Compression +// is a wire optimization only: the receiver inflates exactly one authenticated logical block before Merkle +// verification and writes the original `.mota` bytes to its store. Returning false, an empty result, or a +// result no smaller than the source selects raw v2 DATA. A decoder must reject output overflow, truncated +// streams, and trailing compressed input; OtaManager also checks that it produced exactly the manifest-derived +// block length before accepting a proof. +typedef bool (*OtaDeflateEncodeFn)(void* ctx, const uint8_t* src, uint16_t src_len, + uint8_t* dst, uint16_t dst_cap, uint16_t* dst_len); +typedef bool (*OtaDeflateDecodeFn)(void* ctx, const uint8_t* src, uint16_t src_len, + uint8_t* dst, uint16_t dst_cap, uint16_t* dst_len); +// Optional source-side fast path for a host/archive that already has an independently compressed logical +// block. It avoids carrying a DEFLATE encoder on the embedded seeder. The result has the same strict bounds +// and deterministic-per-block contract as OtaDeflateEncodeFn. +typedef bool (*ServeDeflateReadFn)(void* ctx, uint16_t block, uint8_t* dst, + uint16_t dst_cap, uint16_t* dst_len); + #ifndef OTA_PROOFGEN_SCRATCH #if defined(OTA_SD_STORE) #define OTA_PROOFGEN_SCRATCH 8192 // SD archive can seed <=2048 blocks (about 2 MB at 1 KB/block) @@ -147,7 +163,10 @@ typedef bool (*ServeReadFn)(void* ctx, uint32_t off, uint8_t* buf, uint32_t len) #define OTA_SERVE_SUPPRESS_MS 1500 // don't re-serve a block whose DATA we just overheard another holder send #endif // (so multiple sources of the same mota don't duplicate-broadcast it) #ifndef OTA_FRAG_DATA -#define OTA_FRAG_DATA 160 // data bytes per DATA fragment (<= MAX_PACKET_PAYLOAD - 9-byte header) +#define OTA_FRAG_DATA 160 // deployed legacy DATA geometry; never change in place +#endif +#ifndef OTA_FRAG_DATA_V2 +#define OTA_FRAG_DATA_V2 171 // negotiated DATA geometry (13-byte overhead => 184-byte packet payload) #endif #ifndef OTA_SERVE_QUEUE #define OTA_SERVE_QUEUE 4 // requested blocks retained without allocating one packet per fragment @@ -245,6 +264,8 @@ public: uint16_t mfl = 0; // manifest-minus-leaves length (the OTA_MANIFEST payload) ServeReadFn read = nullptr; // payload reader (nullptr => m.payload is contiguous in RAM) void* read_ctx = nullptr; + ServeDeflateReadFn read_deflated = nullptr; // optional host/source-generated raw-DEFLATE block + void* deflate_ctx = nullptr; uint8_t* scratch = nullptr; // proof-gen working buffer (>= block_count*4) uint32_t scratch_sz = 0; }; @@ -302,6 +323,18 @@ public: // --- fetch --- Provide the staging store; fetching starts on a matching OTA_ADV. void set_fetch_store(OtaStore* s) { _fetch = s; } + // Install either side of the optional per-block transport codec. A receiver with no decoder still uses + // negotiated 171-byte raw DATA; a source with no encoder automatically returns raw DATA. Context lifetimes + // are caller-owned and independent because a relay/seeder may fetch and serve at the same time. + void set_transport_deflate_encoder(OtaDeflateEncodeFn fn, void* ctx = nullptr) { + _deflate_encode = fn; _deflate_encode_ctx = ctx; + } + void set_transport_deflate_decoder(OtaDeflateDecodeFn fn, void* ctx = nullptr) { + _deflate_decode = fn; _deflate_decode_ctx = ctx; + } + // Defaults on for moving-forward 171-byte transfers. This switch exists for qualification and emergency + // rollback; disabling it produces byte-for-byte legacy OTA_REQ/OTA_DATA behavior. + void set_wire_v2_enabled(bool on) { _wire_v2_enabled = on; } // Resume a fetch from a container already persisted in the store (after a reboot). want_mid=nullptr is // automatic adoption and rechecks current autofetch/target/version policy; a non-null MID is an explicit @@ -441,6 +474,7 @@ public: _lv_retries = 0; _loop_last_lvmask = 0; _resume_verify_idx = 0; _resume_invalidated = false; _resume_merkle.reset(); + _wire_empty_stalls = 0; } FetchState fetchState() const { return _fstate; } @@ -525,7 +559,10 @@ private: bool loadSource(const ServeEntry& e); // load an external mota into _srcv (head+leaves) void registerSelfEntry(); // (re)build entry[0] from view0 static bool srcReadTramp(void* c, uint32_t off, uint8_t* buf, uint32_t len); // source payload reader - bool queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want_mask); + static bool srcDeflateReadTramp(void* c, uint16_t block, uint8_t* buf, + uint16_t cap, uint16_t* len); + bool queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want_mask, + bool wire_v2 = false, bool allow_deflate = false); bool queueManifestJob(const uint8_t* mid, uint16_t want_mask); uint32_t manifestEgressGapMs() const; uint32_t proofEgressGapMs() const; @@ -556,6 +593,10 @@ private: bool requestFlight(); // send all newly assigned blocks in one OTA_REQ bool fillPipeline(); // assign + send a new flight only when the old one ended void requestMissing(); // start a flight, or recover one stalled slot + uint16_t wireRequestMask(uint32_t block, uint16_t need, uint16_t received, + uint16_t encoded_len) const; + void downgradeWireToLegacy(); // old source answered/ignored the v2 request profile + bool anyWireDataReceived() const; uint32_t blockLen(uint32_t i) const; uint32_t _target = 0; @@ -593,6 +634,8 @@ private: uint16_t emitted_mask = 0; uint32_t proof_ready_at = 0; bool proof_requested = false; + bool wire_v2 = false; + bool allow_deflate = false; }; ServeJob _serve_jobs[OTA_SERVE_QUEUE]; uint8_t _n_serve_jobs = 0; @@ -607,6 +650,15 @@ private: uint8_t _serve_block[OTA_MAX_BLOCK]; uint16_t _serve_block_len = 0; bool _serve_block_loaded = false; + const uint8_t* _serve_wire_block = nullptr; // raw _serve_block or DEFLATE bytes in ServeView::scratch + uint16_t _serve_wire_len = 0; + bool _serve_wire_deflated = false; + uint8_t _serve_wire_id[4] = {0}; // SHA-256:4 of this exact encoded representation + + OtaDeflateEncodeFn _deflate_encode = nullptr; + void* _deflate_encode_ctx = nullptr; + OtaDeflateDecodeFn _deflate_decode = nullptr; + void* _deflate_decode_ctx = nullptr; // fetch OtaStore* _fetch = nullptr; @@ -638,6 +690,11 @@ private: uint16_t _advert_mins = OTA_ADVERT_INTERVAL_MINS; // beacon re-advertise cadence, minutes; 0=off (persisted) uint8_t _max_hops = OTA_HOP_LIMIT_DEFAULT; // OTA flood reach in hops; 0=direct only (persisted) uint8_t _fflags = 0; // flags of the manifest currently being fetched + bool _wire_v2_enabled = true; // advertise v2 in OTA_REQ while retaining transparent fallback + bool _wire_v2_session = true; // false after an old source responds or the first request stalls + bool _wire_v2_confirmed = false; // at least one structurally valid v2 DATA packet was observed + bool _wire_allow_deflate = false; // session may request compressed blocks only with a decoder + uint8_t _wire_empty_stalls = 0; // confirmed v2 source may vanish between otherwise empty flights // Bounded adaptive multi-block client flight. Each slot independently reassembles DATA and awaits its // Merkle proof. One append-only OTA_REQ packet opens the whole flight; the receiver then stays quiet until // every slot completes or an airtime-aware deadline expires. Clean flights grow 1->2->3->4; any recovery @@ -648,6 +705,11 @@ private: uint16_t need = 0; // full bitmap for this block bool awaiting_proof = false; uint32_t proof_request_at = 0; // proactive-proof grace deadline; 0 after fallback is sent + uint16_t encoded_len = 0; // v2 wire length; zero until the first DATA fragment establishes it + bool wire_v2 = false; + bool deflated = false; + uint8_t stream_id[4] = {0}; // locks fragments to one encoded representation across seeders + uint8_t wire_stalls = 0; // eventually unlock a vanished representation and retry cleanly uint8_t buf[OTA_MAX_BLOCK]; }; ReassemblySlot _reasm[OTA_FETCH_PIPELINE]; diff --git a/src/helpers/ota/OtaProtocol.h b/src/helpers/ota/OtaProtocol.h index ccfd2d4d..b4ffc211 100644 --- a/src/helpers/ota/OtaProtocol.h +++ b/src/helpers/ota/OtaProtocol.h @@ -69,8 +69,11 @@ struct ManifestMsg { // source queues every row. This lets a fetcher open one adaptive flight with one packet, then remain silent // while the source and relays return the requested DATA/PROOF train. // -// want_mask bit k = "send fragment k" (the slice at byte offset k*OTA_FRAG_DATA). A recovery request uses -// only the missing bits, so one lost fragment costs one fragment to re-send rather than the whole block. +// Legacy want_mask bit k = "send fragment k" (the slice at byte offset k*OTA_FRAG_DATA). A recovery request +// uses only the missing bits, so one lost fragment costs one fragment to re-send rather than the whole block. +// New receivers reserve bits 15..14 as an in-band request profile: bit 15 asks for the v2 171-byte geometry +// and bit 14 permits per-block raw DEFLATE. An old source masks those bits away and returns the ordinary +// 160-byte DATA train; the new receiver detects that response and downgrades without stranding old fleets. struct ReqMsg { uint8_t manifest_id[4]; uint16_t block_idx; uint16_t want_mask; }; struct ReqItem { uint16_t block_idx; uint16_t want_mask; }; #ifndef OTA_REQ_MAX_ITEMS @@ -84,8 +87,13 @@ struct ReqWindowMsg { // ---- OTA_DATA: one self-describing fragment of a block's data (proof stays in a separate packet) ---- // body: manifest_id(4) block_idx(2) frag_off(2) data[] -// `frag_off` is the byte offset of `data` within block `block_idx` (global position = block_idx*block_size -// + frag_off), so a fragment is self-placing when received from the selected source. +// In the legacy profile, `frag_off` is the byte offset of `data` within block `block_idx` (global position = +// block_idx*block_size + frag_off), so a fragment is self-placing when received from the selected source. +// In the negotiated v2 profile its high bit is set and the field packs the per-block wire encoding, fragment +// index, and total encoded size; data[] starts with SHA-256:4 of the complete encoded representation, followed +// by up to 171 bytes. Repeating that id in every fragment prevents different seeders' valid-but-different +// DEFLATE streams from being mixed. Keeping the existing OTA_DATA type means deployed opaque relays retain the +// same transfer priority. Old endpoints reject the tagged offset and new endpoints only request it explicitly. struct DataMsg { uint8_t manifest_id[4]; uint16_t block_idx; @@ -93,6 +101,59 @@ struct DataMsg { const uint8_t* data; uint16_t data_len; }; +// OTA_REQ want_mask extension. The v2 marker and DEFLATE permission are deliberately outside every valid +// fragment bit for the fixed <=1 KiB blocks used by this protocol. A v2 request includes all seven legacy +// fragment bits on its first attempt, so a deployed source which ignores these flags can answer completely. +static const uint16_t OTA_REQ_V2_MARK = 0x8000u; +static const uint16_t OTA_REQ_V2_ALLOW_DEFLATE = 0x4000u; +static const uint16_t OTA_REQ_V2_RESERVED = 0x2000u; +static const uint16_t OTA_REQ_V2_FRAGMENT_MASK = 0x1FFFu; + +// OTA_DATA v2 frag_off packing: +// bit 15 v2 marker +// bit 14 data[] is one raw-RFC1951-DEFLATE stream fragment (clear = raw block bytes) +// bits 13..10 fragment index (0..15; byte offset = index * OTA_FRAG_DATA_V2) +// bits 9..0 total encoded block length minus one (1..1024 bytes) +static const uint16_t OTA_DATA_V2_MARK = 0x8000u; +static const uint16_t OTA_DATA_V2_DEFLATED = 0x4000u; +static const uint16_t OTA_DATA_V2_FRAGMENT_BITS = 0x3C00u; +static const uint16_t OTA_DATA_V2_LENGTH_BITS = 0x03FFu; +static const uint8_t OTA_DATA_V2_FRAGMENT_SHIFT = 10; +static const uint16_t OTA_DATA_V2_MAX_ENCODED = 1024; +static const uint8_t OTA_DATA_V2_STREAM_ID_BYTES = 4; + +inline bool ota_req_is_v2(uint16_t want_mask) { + return (want_mask & OTA_REQ_V2_MARK) != 0 && (want_mask & OTA_REQ_V2_RESERVED) == 0; +} + +inline uint16_t ota_req_v2_fragments(uint16_t want_mask) { + return (uint16_t)(want_mask & OTA_REQ_V2_FRAGMENT_MASK); +} + +inline uint16_t ota_req_make_v2(uint16_t fragments, bool allow_deflate) { + return (uint16_t)((fragments & OTA_REQ_V2_FRAGMENT_MASK) | OTA_REQ_V2_MARK | + (allow_deflate ? OTA_REQ_V2_ALLOW_DEFLATE : 0)); +} + +inline bool ota_data_v2_pack(uint8_t fragment, uint16_t encoded_len, bool deflated, + uint16_t& packed) { + if (fragment >= 16 || encoded_len == 0 || encoded_len > OTA_DATA_V2_MAX_ENCODED) return false; + packed = (uint16_t)(OTA_DATA_V2_MARK | + (deflated ? OTA_DATA_V2_DEFLATED : 0) | + ((uint16_t)fragment << OTA_DATA_V2_FRAGMENT_SHIFT) | + (encoded_len - 1u)); + return true; +} + +inline bool ota_data_v2_unpack(uint16_t packed, uint8_t& fragment, uint16_t& encoded_len, + bool& deflated) { + if ((packed & OTA_DATA_V2_MARK) == 0) return false; + fragment = (uint8_t)((packed & OTA_DATA_V2_FRAGMENT_BITS) >> OTA_DATA_V2_FRAGMENT_SHIFT); + encoded_len = (uint16_t)((packed & OTA_DATA_V2_LENGTH_BITS) + 1u); + deflated = (packed & OTA_DATA_V2_DEFLATED) != 0; + return true; +} + // ---- OTA_GET_LEAVES: request the target's merkle leaves[] in bulk (direct; motatool warm-start only) ---- // body: manifest_id(4) want_mask(2). want_mask bit k = "send leaves fragment k"; 0xFFFF = "send all" on the // first ask (before frag_total is known); a retry requests only the still-missing fragments (anti-burst, diff --git a/src/helpers/ota/OtaSource.h b/src/helpers/ota/OtaSource.h index e9dec2e6..daf0d104 100644 --- a/src/helpers/ota/OtaSource.h +++ b/src/helpers/ota/OtaSource.h @@ -18,6 +18,11 @@ namespace mesh { namespace ota { +// Optional operations exposed by a MotaSource. Serial/TCP sources carry these +// bits in byte 35 of their unchanged 38-byte descriptor; deployed hosts leave +// the formerly-reserved byte zero. +static const uint8_t MOTA_SOURCE_CAP_DEFLATE_BLOCK = 0x01; + // A parsed top-level descriptor of one `.mota` a source provides: enough to advertise it in the catalog // AND to locate every region for serving, WITHOUT holding the whole image in RAM. Offsets are absolute // byte positions within the `.mota` container (which always begins MAGIC(4) total(4) manifest...). @@ -28,6 +33,7 @@ struct MotaDesc { uint8_t codec_id = 0; uint8_t flags = 0; uint8_t block_size_log2 = 0; // serve geometry; 0 means a legacy descriptor that must be inspected + uint8_t source_caps = 0; // MOTA_SOURCE_CAP_*; zero preserves deployed source behavior uint32_t total_size = 0; // full `.mota` length (bytes) uint32_t leaves_off = 0; // byte offset of the merkle leaves[] (manifest-minus-leaves = [8, leaves_off)) uint32_t block_count = 0; // == number of leaves (== number of payload blocks) @@ -50,6 +56,15 @@ public: // exactly `len` bytes were produced. May block on the transport (serial round-trip); implementations // should keep active-transfer reads prompt because requested OTA traffic is primary traffic. virtual bool read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len) = 0; + // Optional transport-only raw-DEFLATE representation of one logical payload block. Implementations which + // can generate/cache it off-device return a stream strictly smaller than the raw block. The default keeps + // every existing source source-compatible and makes OtaManager fall back to raw 171-byte DATA. + virtual bool read_deflated_block(uint8_t idx, uint16_t block, uint8_t* buf, + uint16_t cap, uint16_t* len) { + (void)idx; (void)block; (void)buf; (void)cap; + if (len) *len = 0; + return false; + } }; } // namespace ota diff --git a/src/helpers/ota/OtaTinf.c b/src/helpers/ota/OtaTinf.c new file mode 100644 index 00000000..deea6328 --- /dev/null +++ b/src/helpers/ota/OtaTinf.c @@ -0,0 +1,17 @@ +/* + * Compile the vendored tinf core as C. Building the same source as C++ costs + * several extra kilobytes with the embedded toolchains used by MeshCore. + */ +#if defined(ENABLE_OTA) || defined(OTA_TRANSPORT_DEFLATE_TEST) +#if defined(__GNUC__) && !defined(__clang__) +/* nRF52's Arduino recipe defaults to -Ofast, which more than doubles this + * decoder. Keep the size policy local to tinf instead of changing MeshCore. */ +#pragma GCC push_options +#pragma GCC optimize ("Os") +#endif +#define MESHCORE_TINF_IMPLEMENTATION 1 +#include "tinf/tinflate.c" +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC pop_options +#endif +#endif diff --git a/src/helpers/ota/tinf/LICENSE b/src/helpers/ota/tinf/LICENSE new file mode 100644 index 00000000..c90a73c5 --- /dev/null +++ b/src/helpers/ota/tinf/LICENSE @@ -0,0 +1,22 @@ +The zlib License (Zlib) + +Copyright (c) 2003-2019 Joergen Ibsen + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. diff --git a/src/helpers/ota/tinf/README.meshcore.txt b/src/helpers/ota/tinf/README.meshcore.txt new file mode 100644 index 00000000..ecc41dc9 --- /dev/null +++ b/src/helpers/ota/tinf/README.meshcore.txt @@ -0,0 +1,23 @@ +tinf 1.2.1 raw-DEFLATE decoder +================================ + +Upstream: https://github.com/jibsen/tinf +Revision: 57ffa1f1d5e3dde19011b2127bd26d01689b694b + +Vendored files: + LICENSE + tinf.h + tinflate.c + +MeshCore changes are marked in tinf.h and tinflate.c. They add +tinf_uncompress_exact(), which accepts legal padding bits in the byte containing +the final end-of-block code but rejects any trailing whole input byte. The C +implementation is included by ../OtaTinf.c only for ENABLE_OTA application +builds (and the native OTA test profile). ../OtaDeflate.cpp is the small C++ +callback adapter. Keeping the decoder in a true C translation unit avoids the +several-kilobyte penalty produced by the embedded C++ toolchains; non-OTA +firmware links no decoder code or persistent RAM. + +Only the raw RFC 1951 decoder is integrated. The zlib/gzip wrappers and checksum +sources are intentionally omitted. The original zlib license is preserved in +LICENSE and in the source headers. diff --git a/src/helpers/ota/tinf/tinf.h b/src/helpers/ota/tinf/tinf.h new file mode 100644 index 00000000..5d6288f1 --- /dev/null +++ b/src/helpers/ota/tinf/tinf.h @@ -0,0 +1,154 @@ +/* + * tinf - tiny inflate library (inflate, gzip, zlib) + * + * Copyright (c) 2003-2019 Joergen Ibsen + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must + * not claim that you wrote the original software. If you use this + * software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and must + * not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ + +/* + * MeshCore modification (2026): declare tinf_uncompress_exact(), a raw-DEFLATE + * entry point which rejects trailing whole input bytes. The original API and + * behavior remain available as tinf_uncompress(). + */ + +#ifndef TINF_H_INCLUDED +#define TINF_H_INCLUDED + +#ifdef __cplusplus +extern "C" { +#endif + +#define TINF_VER_MAJOR 1 /**< Major version number */ +#define TINF_VER_MINOR 2 /**< Minor version number */ +#define TINF_VER_PATCH 1 /**< Patch version number */ +#define TINF_VER_STRING "1.2.1" /**< Version number as a string */ + +#ifndef TINFCC +# ifdef __WATCOMC__ +# define TINFCC __cdecl +# else +# define TINFCC +# endif +#endif + +/** + * Status codes returned. + * + * @see tinf_uncompress, tinf_gzip_uncompress, tinf_zlib_uncompress + */ +typedef enum { + TINF_OK = 0, /**< Success */ + TINF_DATA_ERROR = -3, /**< Input error */ + TINF_BUF_ERROR = -5 /**< Not enough room for output */ +} tinf_error_code; + +/** + * Initialize global data used by tinf. + * + * @deprecated No longer required, may be removed in a future version. + */ +void TINFCC tinf_init(void); + +/** + * Decompress `sourceLen` bytes of deflate data from `source` to `dest`. + * + * The variable `destLen` points to must contain the size of `dest` on entry, + * and will be set to the size of the decompressed data on success. + * + * Reads at most `sourceLen` bytes from `source`. + * Writes at most `*destLen` bytes to `dest`. + * + * @param dest pointer to where to place decompressed data + * @param destLen pointer to variable containing size of `dest` + * @param source pointer to compressed data + * @param sourceLen size of compressed data + * @return `TINF_OK` on success, error code on error + */ +int TINFCC tinf_uncompress(void *dest, unsigned int *destLen, + const void *source, unsigned int sourceLen); + +/** + * Decompress exactly one raw DEFLATE stream and reject trailing whole bytes. + * Padding bits in the byte containing the final end-of-block code are allowed. + * Output bounds and `destLen` semantics match tinf_uncompress(). + */ +int TINFCC tinf_uncompress_exact(void *dest, unsigned int *destLen, + const void *source, unsigned int sourceLen); + +/** + * Decompress `sourceLen` bytes of gzip data from `source` to `dest`. + * + * The variable `destLen` points to must contain the size of `dest` on entry, + * and will be set to the size of the decompressed data on success. + * + * Reads at most `sourceLen` bytes from `source`. + * Writes at most `*destLen` bytes to `dest`. + * + * @param dest pointer to where to place decompressed data + * @param destLen pointer to variable containing size of `dest` + * @param source pointer to compressed data + * @param sourceLen size of compressed data + * @return `TINF_OK` on success, error code on error + */ +int TINFCC tinf_gzip_uncompress(void *dest, unsigned int *destLen, + const void *source, unsigned int sourceLen); + +/** + * Decompress `sourceLen` bytes of zlib data from `source` to `dest`. + * + * The variable `destLen` points to must contain the size of `dest` on entry, + * and will be set to the size of the decompressed data on success. + * + * Reads at most `sourceLen` bytes from `source`. + * Writes at most `*destLen` bytes to `dest`. + * + * @param dest pointer to where to place decompressed data + * @param destLen pointer to variable containing size of `dest` + * @param source pointer to compressed data + * @param sourceLen size of compressed data + * @return `TINF_OK` on success, error code on error + */ +int TINFCC tinf_zlib_uncompress(void *dest, unsigned int *destLen, + const void *source, unsigned int sourceLen); + +/** + * Compute Adler-32 checksum of `length` bytes starting at `data`. + * + * @param data pointer to data + * @param length size of data + * @return Adler-32 checksum + */ +unsigned int TINFCC tinf_adler32(const void *data, unsigned int length); + +/** + * Compute CRC32 checksum of `length` bytes starting at `data`. + * + * @param data pointer to data + * @param length size of data + * @return CRC32 checksum + */ +unsigned int TINFCC tinf_crc32(const void *data, unsigned int length); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* TINF_H_INCLUDED */ diff --git a/src/helpers/ota/tinf/tinflate.c b/src/helpers/ota/tinf/tinflate.c new file mode 100644 index 00000000..6b7c58dc --- /dev/null +++ b/src/helpers/ota/tinf/tinflate.c @@ -0,0 +1,666 @@ +/* + * tinflate - tiny inflate + * + * Copyright (c) 2003-2019 Joergen Ibsen + * + * This software is provided 'as-is', without any express or implied + * warranty. In no event will the authors be held liable for any damages + * arising from the use of this software. + * + * Permission is granted to anyone to use this software for any purpose, + * including commercial applications, and to alter it and redistribute it + * freely, subject to the following restrictions: + * + * 1. The origin of this software must not be misrepresented; you must + * not claim that you wrote the original software. If you use this + * software in a product, an acknowledgment in the product + * documentation would be appreciated but is not required. + * + * 2. Altered source versions must be plainly marked as such, and must + * not be misrepresented as being the original software. + * + * 3. This notice may not be removed or altered from any source + * distribution. + */ + +/* + * MeshCore modifications (2026): + * - expose tinf_uncompress_exact() to reject trailing whole input bytes; + * - compile this vendored implementation through OtaTinf.c only, keeping + * it out of builds which do not define ENABLE_OTA. + * See README.meshcore.txt for the upstream revision and integration details. + */ + +#if defined(MESHCORE_TINF_IMPLEMENTATION) + +#include "tinf.h" + +#include +#include + +#if defined(UINT_MAX) && (UINT_MAX) < 0xFFFFFFFFUL +# error "tinf requires unsigned int to be at least 32-bit" +#endif + +/* -- Internal data structures -- */ + +struct tinf_tree { + unsigned short counts[16]; /* Number of codes with a given length */ + unsigned short symbols[288]; /* Symbols sorted by code */ + int max_sym; +}; + +struct tinf_data { + const unsigned char *source; + const unsigned char *source_end; + unsigned int tag; + int bitcount; + int overflow; + + unsigned char *dest_start; + unsigned char *dest; + unsigned char *dest_end; + + struct tinf_tree ltree; /* Literal/length tree */ + struct tinf_tree dtree; /* Distance tree */ +}; + +/* -- Utility functions -- */ + +static unsigned int read_le16(const unsigned char *p) +{ + return ((unsigned int) p[0]) + | ((unsigned int) p[1] << 8); +} + +/* Build fixed Huffman trees */ +static void tinf_build_fixed_trees(struct tinf_tree *lt, struct tinf_tree *dt) +{ + int i; + + /* Build fixed literal/length tree */ + for (i = 0; i < 16; ++i) { + lt->counts[i] = 0; + } + + lt->counts[7] = 24; + lt->counts[8] = 152; + lt->counts[9] = 112; + + for (i = 0; i < 24; ++i) { + lt->symbols[i] = 256 + i; + } + for (i = 0; i < 144; ++i) { + lt->symbols[24 + i] = i; + } + for (i = 0; i < 8; ++i) { + lt->symbols[24 + 144 + i] = 280 + i; + } + for (i = 0; i < 112; ++i) { + lt->symbols[24 + 144 + 8 + i] = 144 + i; + } + + lt->max_sym = 285; + + /* Build fixed distance tree */ + for (i = 0; i < 16; ++i) { + dt->counts[i] = 0; + } + + dt->counts[5] = 32; + + for (i = 0; i < 32; ++i) { + dt->symbols[i] = i; + } + + dt->max_sym = 29; +} + +/* Given an array of code lengths, build a tree */ +static int tinf_build_tree(struct tinf_tree *t, const unsigned char *lengths, + unsigned int num) +{ + unsigned short offs[16]; + unsigned int i, num_codes, available; + + assert(num <= 288); + + for (i = 0; i < 16; ++i) { + t->counts[i] = 0; + } + + t->max_sym = -1; + + /* Count number of codes for each non-zero length */ + for (i = 0; i < num; ++i) { + assert(lengths[i] <= 15); + + if (lengths[i]) { + t->max_sym = i; + t->counts[lengths[i]]++; + } + } + + /* Compute offset table for distribution sort */ + for (available = 1, num_codes = 0, i = 0; i < 16; ++i) { + unsigned int used = t->counts[i]; + + /* Check length contains no more codes than available */ + if (used > available) { + return TINF_DATA_ERROR; + } + available = 2 * (available - used); + + offs[i] = num_codes; + num_codes += used; + } + + /* + * Check all codes were used, or for the special case of only one + * code that it has length 1 + */ + if ((num_codes > 1 && available > 0) + || (num_codes == 1 && t->counts[1] != 1)) { + return TINF_DATA_ERROR; + } + + /* Fill in symbols sorted by code */ + for (i = 0; i < num; ++i) { + if (lengths[i]) { + t->symbols[offs[lengths[i]]++] = i; + } + } + + /* + * For the special case of only one code (which will be 0) add a + * code 1 which results in a symbol that is too large + */ + if (num_codes == 1) { + t->counts[1] = 2; + t->symbols[1] = t->max_sym + 1; + } + + return TINF_OK; +} + +/* -- Decode functions -- */ + +static void tinf_refill(struct tinf_data *d, int num) +{ + assert(num >= 0 && num <= 32); + + /* Read bytes until at least num bits available */ + while (d->bitcount < num) { + if (d->source != d->source_end) { + d->tag |= (unsigned int) *d->source++ << d->bitcount; + } + else { + d->overflow = 1; + } + d->bitcount += 8; + } + + assert(d->bitcount <= 32); +} + +static unsigned int tinf_getbits_no_refill(struct tinf_data *d, int num) +{ + unsigned int bits; + + assert(num >= 0 && num <= d->bitcount); + + /* Get bits from tag */ + bits = d->tag & ((1UL << num) - 1); + + /* Remove bits from tag */ + d->tag >>= num; + d->bitcount -= num; + + return bits; +} + +/* Get num bits from source stream */ +static unsigned int tinf_getbits(struct tinf_data *d, int num) +{ + tinf_refill(d, num); + return tinf_getbits_no_refill(d, num); +} + +/* Read a num bit value from stream and add base */ +static unsigned int tinf_getbits_base(struct tinf_data *d, int num, int base) +{ + return base + (num ? tinf_getbits(d, num) : 0); +} + +/* Given a data stream and a tree, decode a symbol */ +static int tinf_decode_symbol(struct tinf_data *d, const struct tinf_tree *t) +{ + int base = 0, offs = 0; + int len; + + /* + * Get more bits while code index is above number of codes + * + * Rather than the actual code, we are computing the position of the + * code in the sorted order of codes, which is the index of the + * corresponding symbol. + * + * Conceptually, for each code length (level in the tree), there are + * counts[len] leaves on the left and internal nodes on the right. + * The index we have decoded so far is base + offs, and if that + * falls within the leaves we are done. Otherwise we adjust the range + * of offs and add one more bit to it. + */ + for (len = 1; ; ++len) { + offs = 2 * offs + tinf_getbits(d, 1); + + assert(len <= 15); + + if (offs < t->counts[len]) { + break; + } + + base += t->counts[len]; + offs -= t->counts[len]; + } + + assert(base + offs >= 0 && base + offs < 288); + + return t->symbols[base + offs]; +} + +/* Given a data stream, decode dynamic trees from it */ +static int tinf_decode_trees(struct tinf_data *d, struct tinf_tree *lt, + struct tinf_tree *dt) +{ + unsigned char lengths[288 + 32]; + + /* Special ordering of code length codes */ + static const unsigned char clcidx[19] = { + 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, + 11, 4, 12, 3, 13, 2, 14, 1, 15 + }; + + unsigned int hlit, hdist, hclen; + unsigned int i, num, length; + int res; + + /* Get 5 bits HLIT (257-286) */ + hlit = tinf_getbits_base(d, 5, 257); + + /* Get 5 bits HDIST (1-32) */ + hdist = tinf_getbits_base(d, 5, 1); + + /* Get 4 bits HCLEN (4-19) */ + hclen = tinf_getbits_base(d, 4, 4); + + /* + * The RFC limits the range of HLIT to 286, but lists HDIST as range + * 1-32, even though distance codes 30 and 31 have no meaning. While + * we could allow the full range of HLIT and HDIST to make it possible + * to decode the fixed trees with this function, we consider it an + * error here. + * + * See also: https://github.com/madler/zlib/issues/82 + */ + if (hlit > 286 || hdist > 30) { + return TINF_DATA_ERROR; + } + + for (i = 0; i < 19; ++i) { + lengths[i] = 0; + } + + /* Read code lengths for code length alphabet */ + for (i = 0; i < hclen; ++i) { + /* Get 3 bits code length (0-7) */ + unsigned int clen = tinf_getbits(d, 3); + + lengths[clcidx[i]] = clen; + } + + /* Build code length tree (in literal/length tree to save space) */ + res = tinf_build_tree(lt, lengths, 19); + + if (res != TINF_OK) { + return res; + } + + /* Check code length tree is not empty */ + if (lt->max_sym == -1) { + return TINF_DATA_ERROR; + } + + /* Decode code lengths for the dynamic trees */ + for (num = 0; num < hlit + hdist; ) { + int sym = tinf_decode_symbol(d, lt); + + if (sym > lt->max_sym) { + return TINF_DATA_ERROR; + } + + switch (sym) { + case 16: + /* Copy previous code length 3-6 times (read 2 bits) */ + if (num == 0) { + return TINF_DATA_ERROR; + } + sym = lengths[num - 1]; + length = tinf_getbits_base(d, 2, 3); + break; + case 17: + /* Repeat code length 0 for 3-10 times (read 3 bits) */ + sym = 0; + length = tinf_getbits_base(d, 3, 3); + break; + case 18: + /* Repeat code length 0 for 11-138 times (read 7 bits) */ + sym = 0; + length = tinf_getbits_base(d, 7, 11); + break; + default: + /* Values 0-15 represent the actual code lengths */ + length = 1; + break; + } + + if (length > hlit + hdist - num) { + return TINF_DATA_ERROR; + } + + while (length--) { + lengths[num++] = sym; + } + } + + /* Check EOB symbol is present */ + if (lengths[256] == 0) { + return TINF_DATA_ERROR; + } + + /* Build dynamic trees */ + res = tinf_build_tree(lt, lengths, hlit); + + if (res != TINF_OK) { + return res; + } + + res = tinf_build_tree(dt, lengths + hlit, hdist); + + if (res != TINF_OK) { + return res; + } + + return TINF_OK; +} + +/* -- Block inflate functions -- */ + +/* Given a stream and two trees, inflate a block of data */ +static int tinf_inflate_block_data(struct tinf_data *d, struct tinf_tree *lt, + struct tinf_tree *dt) +{ + /* Extra bits and base tables for length codes */ + static const unsigned char length_bits[30] = { + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, + 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 5, 5, 5, 5, 0, 127 + }; + + static const unsigned short length_base[30] = { + 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, + 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, + 67, 83, 99, 115, 131, 163, 195, 227, 258, 0 + }; + + /* Extra bits and base tables for distance codes */ + static const unsigned char dist_bits[30] = { + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 + }; + + static const unsigned short dist_base[30] = { + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, + 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, + 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577 + }; + + for (;;) { + int sym = tinf_decode_symbol(d, lt); + + /* Check for overflow in bit reader */ + if (d->overflow) { + return TINF_DATA_ERROR; + } + + if (sym < 256) { + if (d->dest == d->dest_end) { + return TINF_BUF_ERROR; + } + *d->dest++ = sym; + } + else { + int length, dist, offs; + int i; + + /* Check for end of block */ + if (sym == 256) { + return TINF_OK; + } + + /* Check sym is within range and distance tree is not empty */ + if (sym > lt->max_sym || sym - 257 > 28 || dt->max_sym == -1) { + return TINF_DATA_ERROR; + } + + sym -= 257; + + /* Possibly get more bits from length code */ + length = tinf_getbits_base(d, length_bits[sym], + length_base[sym]); + + dist = tinf_decode_symbol(d, dt); + + /* Check dist is within range */ + if (dist > dt->max_sym || dist > 29) { + return TINF_DATA_ERROR; + } + + /* Possibly get more bits from distance code */ + offs = tinf_getbits_base(d, dist_bits[dist], + dist_base[dist]); + + if (offs > d->dest - d->dest_start) { + return TINF_DATA_ERROR; + } + + if (d->dest_end - d->dest < length) { + return TINF_BUF_ERROR; + } + + /* Copy match */ + for (i = 0; i < length; ++i) { + d->dest[i] = d->dest[i - offs]; + } + + d->dest += length; + } + } +} + +/* Inflate an uncompressed block of data */ +static int tinf_inflate_uncompressed_block(struct tinf_data *d) +{ + unsigned int length, invlength; + + if (d->source_end - d->source < 4) { + return TINF_DATA_ERROR; + } + + /* Get length */ + length = read_le16(d->source); + + /* Get one's complement of length */ + invlength = read_le16(d->source + 2); + + /* Check length */ + if (length != (~invlength & 0x0000FFFF)) { + return TINF_DATA_ERROR; + } + + d->source += 4; + + if (d->source_end - d->source < length) { + return TINF_DATA_ERROR; + } + + if (d->dest_end - d->dest < length) { + return TINF_BUF_ERROR; + } + + /* Copy block */ + while (length--) { + *d->dest++ = *d->source++; + } + + /* Make sure we start next block on a byte boundary */ + d->tag = 0; + d->bitcount = 0; + + return TINF_OK; +} + +/* Inflate a block of data compressed with fixed Huffman trees */ +static int tinf_inflate_fixed_block(struct tinf_data *d) +{ + /* Build fixed Huffman trees */ + tinf_build_fixed_trees(&d->ltree, &d->dtree); + + /* Decode block using fixed trees */ + return tinf_inflate_block_data(d, &d->ltree, &d->dtree); +} + +/* Inflate a block of data compressed with dynamic Huffman trees */ +static int tinf_inflate_dynamic_block(struct tinf_data *d) +{ + /* Decode trees from stream */ + int res = tinf_decode_trees(d, &d->ltree, &d->dtree); + + if (res != TINF_OK) { + return res; + } + + /* Decode block using decoded trees */ + return tinf_inflate_block_data(d, &d->ltree, &d->dtree); +} + +/* -- Public functions -- */ + +/* Initialize global (static) data */ +void tinf_init(void) +{ + return; +} + +/* Inflate stream from source to dest. Optionally require the final DEFLATE block to consume all source bytes. */ +static int tinf_uncompress_internal(void *dest, unsigned int *destLen, + const void *source, unsigned int sourceLen, + int require_exact_source) +{ + struct tinf_data d; + int bfinal; + + /* Initialise data */ + d.source = (const unsigned char *) source; + d.source_end = d.source + sourceLen; + d.tag = 0; + d.bitcount = 0; + d.overflow = 0; + + d.dest = (unsigned char *) dest; + d.dest_start = d.dest; + d.dest_end = d.dest + *destLen; + + do { + unsigned int btype; + int res; + + /* Read final block flag */ + bfinal = tinf_getbits(&d, 1); + + /* Read block type (2 bits) */ + btype = tinf_getbits(&d, 2); + + /* Decompress block */ + switch (btype) { + case 0: + /* Decompress uncompressed block */ + res = tinf_inflate_uncompressed_block(&d); + break; + case 1: + /* Decompress block with fixed Huffman trees */ + res = tinf_inflate_fixed_block(&d); + break; + case 2: + /* Decompress block with dynamic Huffman trees */ + res = tinf_inflate_dynamic_block(&d); + break; + default: + res = TINF_DATA_ERROR; + break; + } + + if (res != TINF_OK) { + return res; + } + } while (!bfinal); + + /* Check for overflow in bit reader */ + if (d.overflow) { + return TINF_DATA_ERROR; + } + + /* d.source includes the byte containing any legal final padding bits, but not a following whole byte. */ + if (require_exact_source && d.source != d.source_end) { + return TINF_DATA_ERROR; + } + + *destLen = d.dest - d.dest_start; + + return TINF_OK; +} + +int tinf_uncompress(void *dest, unsigned int *destLen, + const void *source, unsigned int sourceLen) +{ + return tinf_uncompress_internal(dest, destLen, source, sourceLen, 0); +} + +int tinf_uncompress_exact(void *dest, unsigned int *destLen, + const void *source, unsigned int sourceLen) +{ + return tinf_uncompress_internal(dest, destLen, source, sourceLen, 1); +} + +/* clang -g -O1 -fsanitize=fuzzer,address -DTINF_FUZZING tinflate.c */ +#if defined(TINF_FUZZING) +#include +#include +#include +#include +#include + +unsigned char depacked[64 * 1024]; + +extern int +LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + if (size > UINT_MAX / 2) { return 0; } + unsigned int destLen = sizeof(depacked); + tinf_uncompress(depacked, &destLen, data, size); + return 0; +} +#endif + +#endif /* MESHCORE_TINF_IMPLEMENTATION */ diff --git a/test/test_ota/test_ota_core.cpp b/test/test_ota/test_ota_core.cpp index 9894952f..ebb55972 100644 --- a/test/test_ota/test_ota_core.cpp +++ b/test/test_ota/test_ota_core.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -16,6 +17,7 @@ #include "helpers/ota/OtaStore.h" #include "helpers/ota/OtaProtocol.h" #include "helpers/ota/OtaManager.h" +#include "helpers/ota/OtaDeflate.h" #include "helpers/ota/OtaApply.h" #include "helpers/ota/OtaBlInfo.h" #include "helpers/ota/OtaFlashLayout_nrf52.h" @@ -773,6 +775,71 @@ private: size_t response_pos = 0; }; +class FakeDeflateMotaSeederStream : public Stream { +public: + using Stream::write; + + FakeDeflateMotaSeederStream() { + encoded.resize(421); + for (size_t i = 0; i < encoded.size(); i++) encoded[i] = (uint8_t)(i * 17u + 3u); + } + + size_t write(const uint8_t* request, size_t len) override { + request_valid = len == 11 && request[0] == MOTA_SEEDER_REQ_MAGIC0 + && request[1] == MOTA_SEEDER_REQ_MAGIC1 && request[2] == MS_OP_DEFLATE_BLOCK; + uint8_t checksum = request_valid ? request[2] : 0; + for (size_t i = 3; request_valid && i + 1 < len; i++) checksum ^= request[i]; + request_valid = request_valid && checksum == request[len - 1]; + if (!request_valid) return len; + + const uint16_t block = rd_u16le(request + 4); + const uint16_t offset = rd_u16le(request + 6); + const uint16_t read_len = rd_u16le(request + 8); + blocks.push_back(block); + offsets.push_back(offset); + lengths.push_back(read_len); + + response.clear(); + response_pos = 0; + response.push_back(MOTA_SEEDER_RSP_MAGIC0); + response.push_back(MOTA_SEEDER_RSP_MAGIC1); + response.push_back(MS_OP_DEFLATE_BLOCK); + const bool valid_range = offset <= encoded.size() && read_len <= encoded.size() - offset; + const uint8_t status = reject || !valid_range ? MS_STATUS_ERR : MS_STATUS_OK; + response.push_back(status); + if (status == MS_STATUS_OK) { + response.push_back((uint8_t)(encoded.size() & 0xFF)); + response.push_back((uint8_t)(encoded.size() >> 8)); + response.insert(response.end(), encoded.begin() + offset, + encoded.begin() + offset + read_len); + } + uint8_t response_checksum = 0; + for (uint8_t byte : response) response_checksum ^= byte; + response.push_back(response_checksum); + return len; + } + + int read() override { + if (response_pos < response.size()) return response[response_pos++]; + ++g_mock_millis; + return -1; + } + + void flush() override { ++flush_calls; } + + std::vector encoded; + std::vector blocks; + std::vector offsets; + std::vector lengths; + bool request_valid = true; + bool reject = false; + size_t flush_calls = 0; + +private: + std::vector response; + size_t response_pos = 0; +}; + static_assert(!std::is_constructible::value, "mOTA source callers must choose a stream write policy"); static_assert(!std::is_constructible::value, @@ -837,6 +904,37 @@ TEST(MotaSourceSerial, WrongFlushPolicyDemonstratesNetworkReplyLoss) { EXPECT_EQ(stream.flush_calls, 1U); } +TEST(MotaSourceSerial, ReadsIndependentDeflateBlockInBoundedChunks) { + resetArduinoMock(); + FakeDeflateMotaSeederStream stream; + SerialMotaSource source(stream, MotaStreamWritePolicy::FlushTransmit); + std::array data{}; + uint16_t encoded_len = 0; + + ASSERT_TRUE(source.read_deflated_block(2, 37, data.data(), data.size(), &encoded_len)); + EXPECT_TRUE(stream.request_valid); + EXPECT_EQ(encoded_len, stream.encoded.size()); + EXPECT_EQ(stream.blocks, (std::vector{37, 37, 37, 37})); + EXPECT_EQ(stream.offsets, (std::vector{0, 0, 190, 380})); + EXPECT_EQ(stream.lengths, (std::vector{0, 190, 190, 41})); + EXPECT_EQ(stream.flush_calls, 4u); + EXPECT_EQ(0, memcmp(data.data(), stream.encoded.data(), encoded_len)); +} + +TEST(MotaSourceSerial, DeflateExtensionFailureLeavesRawFallbackAvailable) { + resetArduinoMock(); + FakeDeflateMotaSeederStream stream; + stream.reject = true; // deployed host does not implement operation 0x09 + SerialMotaSource source(stream, MotaStreamWritePolicy::FlushTransmit); + std::array data{}; + uint16_t encoded_len = 999; + + EXPECT_FALSE(source.read_deflated_block(0, 0, data.data(), data.size(), &encoded_len)); + EXPECT_EQ(encoded_len, 0u); + EXPECT_EQ(stream.offsets, (std::vector{0})); + EXPECT_EQ(stream.lengths, (std::vector{0})); +} + class FakeFolderMotaSeederStream : public Stream { public: using Stream::write; @@ -1329,6 +1427,104 @@ TEST(OtaStoreDiscardTest, ConservativeDefaultCannotClaimDurableInvalidation) { EXPECT_TRUE(store.cleared); } +TEST(OtaDeflate, AcceptsStoredFixedDynamicAndMultiBlockRawStreams) { + static const uint8_t stored[] = { + 0x01,0x40,0x00,0xBF,0xFF,0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09, + 0x0A,0x0B,0x0C,0x0D,0x0E,0x0F,0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18, + 0x19,0x1A,0x1B,0x1C,0x1D,0x1E,0x1F,0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27, + 0x28,0x29,0x2A,0x2B,0x2C,0x2D,0x2E,0x2F,0x30,0x31,0x32,0x33,0x34,0x35,0x36, + 0x37,0x38,0x39,0x3A,0x3B,0x3C,0x3D,0x3E,0x3F, + }; + static const uint8_t fixed[] = { + 0xF3,0x4D,0x2D,0xCE,0x70,0xCE,0x2F,0x4A,0x55,0x28,0x29,0x4A,0xCC,0x2B,0x2E,0xC8, + 0x2F,0x2A,0x51,0x48,0xCB,0xAC,0x48,0x4D,0x51,0xF0,0x28,0x4D,0x4B,0xCB,0x4D,0xCC, + 0x53,0x28,0x49,0x2D,0x2E,0xD1,0x53,0xF0,0x1D,0x55,0x46,0x1B,0x65,0x00, + }; + static const uint8_t dynamic[] = { + 0xED,0xCF,0xD9,0x0D,0x83,0x30,0x14,0x44,0xD1,0x0B,0xC1,0x2C,0xC1,0x2C,0xC1,0x2C, + 0xB1,0x81,0xF4,0x5F,0x66,0x9E,0x26,0x05,0xA4,0x01,0x7F,0x5F,0x1D,0x69,0x86,0xA2, + 0x7C,0x54,0xAE,0xA6,0xED,0x9E,0xBD,0x1F,0x98,0xE6,0xD7,0x12,0x56,0xF6,0xE3,0x1D, + 0xD3,0xC9,0xFD,0xC1,0x32,0xAE,0x6E,0x2C,0xE3,0x87,0xD1,0x32,0x61,0xDD,0x2C,0x93, + 0xCE,0xCB,0x32,0xD2,0x0D,0xD2,0x23,0xD2,0x1B,0xD2,0x17,0xD2,0x15,0xD2,0x3D,0xD2, + 0x0B,0xD2,0x11,0xE9,0x02,0xE9,0x16,0xE9,0x09,0xE9,0x1D,0xE9,0x1B,0x69,0x87,0xB4, + 0x47,0x3A,0x20,0x9D,0x90,0x2E,0x91,0xEE,0x90,0x9E,0x91,0x3E,0x88,0xBF,0x69,0xF9, + 0x58,0x3E,0x96,0x8F,0xFD,0x3B,0xF6,0x05, + }; + static const uint8_t multi_block[] = { + 0x4A,0xCB,0x2C,0x2A,0x2E,0xD1,0x4D,0xCA,0xC9,0x4F,0xCE,0xD6,0x4D,0x1B,0x66,0x6C, + 0x00,0x00,0x00,0x00,0xFF,0xFF,0x2B,0x4E,0x4D,0xCE,0xCF,0x4B,0x81,0x0A,0x8C,0x72, + 0xF0,0x72,0x00, + }; + + EXPECT_EQ((stored[0] >> 1) & 3u, 0u); + EXPECT_EQ((fixed[0] >> 1) & 3u, 1u); + EXPECT_EQ((dynamic[0] >> 1) & 3u, 2u); + + std::array output{}; + uint16_t output_len = 0; + ASSERT_TRUE(ota_transport_inflate(nullptr, stored, sizeof(stored), output.data(), 64, &output_len)); + ASSERT_EQ(output_len, 64u); + for (uint16_t i = 0; i < output_len; i++) EXPECT_EQ(output[i], (uint8_t)i); + + static const char fixed_phrase[] = "MeshCore transport fixed Huffman test. "; + ASSERT_TRUE(ota_transport_inflate(nullptr, fixed, sizeof(fixed), output.data(), 384, &output_len)); + ASSERT_EQ(output_len, 384u); + for (uint16_t i = 0; i < output_len; i++) + EXPECT_EQ(output[i], (uint8_t)fixed_phrase[i % (sizeof(fixed_phrase) - 1)]); + + ASSERT_TRUE(ota_transport_inflate(nullptr, dynamic, sizeof(dynamic), output.data(), 1024, &output_len)); + ASSERT_EQ(output_len, 1024u); + for (uint16_t i = 0; i < output_len; i++) + EXPECT_EQ(output[i], (uint8_t)((i % 7u) ? i % 31u : 0u)); + + ASSERT_TRUE(ota_transport_inflate(nullptr, multi_block, sizeof(multi_block), output.data(), 520, + &output_len)); + ASSERT_EQ(output_len, 520u); + static const char first_phrase[] = "first-block-"; + static const char second_phrase[] = "second-block-"; + for (uint16_t i = 0; i < 220; i++) + EXPECT_EQ(output[i], (uint8_t)first_phrase[i % (sizeof(first_phrase) - 1)]); + for (uint16_t i = 220; i < output_len; i++) + EXPECT_EQ(output[i], (uint8_t)second_phrase[(i - 220) % (sizeof(second_phrase) - 1)]); +} + +TEST(OtaDeflate, RejectsTruncationBoundsWrongLengthTrailingBytesAndMalformedTrees) { + static const uint8_t dynamic[] = { + 0xED,0xCF,0xD9,0x0D,0x83,0x30,0x14,0x44,0xD1,0x0B,0xC1,0x2C,0xC1,0x2C,0xC1,0x2C, + 0xB1,0x81,0xF4,0x5F,0x66,0x9E,0x26,0x05,0xA4,0x01,0x7F,0x5F,0x1D,0x69,0x86,0xA2, + 0x7C,0x54,0xAE,0xA6,0xED,0x9E,0xBD,0x1F,0x98,0xE6,0xD7,0x12,0x56,0xF6,0xE3,0x1D, + 0xD3,0xC9,0xFD,0xC1,0x32,0xAE,0x6E,0x2C,0xE3,0x87,0xD1,0x32,0x61,0xDD,0x2C,0x93, + 0xCE,0xCB,0x32,0xD2,0x0D,0xD2,0x23,0xD2,0x1B,0xD2,0x17,0xD2,0x15,0xD2,0x3D,0xD2, + 0x0B,0xD2,0x11,0xE9,0x02,0xE9,0x16,0xE9,0x09,0xE9,0x1D,0xE9,0x1B,0x69,0x87,0xB4, + 0x47,0x3A,0x20,0x9D,0x90,0x2E,0x91,0xEE,0x90,0x9E,0x91,0x3E,0x88,0xBF,0x69,0xF9, + 0x58,0x3E,0x96,0x8F,0xFD,0x3B,0xF6,0x05, + }; + static const uint8_t invalid_distance[] = {0x63,0x00,0x3E,0x00}; + static const uint8_t invalid_tree[] = { + 0x0D,0xCE,0x81,0x00,0x00,0x00,0x00,0x80,0xA0,0xFD,0xA9,0xBB,0x1F,0xA0,0x01, + }; + std::array output{}; + uint16_t output_len = 999; + + EXPECT_FALSE(ota_transport_inflate(nullptr, dynamic, sizeof(dynamic) - 1, + output.data(), 1024, &output_len)); + EXPECT_EQ(output_len, 0u); + EXPECT_FALSE(ota_transport_inflate(nullptr, dynamic, sizeof(dynamic), + output.data(), 1023, &output_len)); + EXPECT_FALSE(ota_transport_inflate(nullptr, dynamic, sizeof(dynamic), + output.data(), 1025, &output_len)); + + std::array trailing{}; + memcpy(trailing.data(), dynamic, sizeof(dynamic)); + trailing.back() = 0x00; + EXPECT_FALSE(ota_transport_inflate(nullptr, trailing.data(), trailing.size(), + output.data(), 1024, &output_len)); + EXPECT_FALSE(ota_transport_inflate(nullptr, invalid_distance, sizeof(invalid_distance), + output.data(), 4, &output_len)); + EXPECT_FALSE(ota_transport_inflate(nullptr, invalid_tree, sizeof(invalid_tree), + output.data(), 4, &output_len)); +} + // --- merkle proof GENERATION (server side) matches the Python oracle --------------------------- TEST(OtaMerkle, GenProofMatchesPythonAndVerifies) { @@ -1484,6 +1680,40 @@ TEST(OtaProtocol, CodecRoundTrips) { EXPECT_EQ(0, memcmp(pm2.proof, proof, 12)); } +TEST(OtaProtocol, V2ProfilePacks171ByteFragmentsInsideExistingMessages) { + const uint16_t request = ota_req_make_v2(0x007Fu, true); + EXPECT_TRUE(ota_req_is_v2(request)); + EXPECT_EQ(ota_req_v2_fragments(request), 0x007Fu); + EXPECT_NE(request & OTA_REQ_V2_ALLOW_DEFLATE, 0u); + EXPECT_FALSE(ota_req_is_v2((uint16_t)(request | OTA_REQ_V2_RESERVED))); + EXPECT_FALSE(ota_req_is_v2(0x007Fu)); + + uint16_t descriptor = 0; + ASSERT_TRUE(ota_data_v2_pack(5, 1024, true, descriptor)); + uint8_t fragment = 0; + uint16_t encoded_len = 0; + bool deflated = false; + ASSERT_TRUE(ota_data_v2_unpack(descriptor, fragment, encoded_len, deflated)); + EXPECT_EQ(fragment, 5u); + EXPECT_EQ(encoded_len, 1024u); + EXPECT_TRUE(deflated); + EXPECT_FALSE(ota_data_v2_pack(16, 1024, false, descriptor)); + EXPECT_FALSE(ota_data_v2_pack(0, 0, false, descriptor)); + EXPECT_FALSE(ota_data_v2_pack(0, 1025, false, descriptor)); + + std::array payload{}; + ASSERT_TRUE(ota_data_v2_pack(0, 1024, false, descriptor)); + DataMsg message{{1, 2, 3, 4}, 9, descriptor, payload.data(), + (uint16_t)payload.size()}; + uint8_t wire[MAX_PACKET_PAYLOAD]; + const uint16_t wire_len = encode_data(wire, sizeof(wire), message); + EXPECT_EQ(wire_len, MAX_PACKET_PAYLOAD); + DataMsg decoded{}; + ASSERT_TRUE(decode_data(wire, wire_len, decoded)); + EXPECT_EQ(decoded.data_len, + (uint16_t)(OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2)); +} + // --- full transfer simulation between two OtaManagers (P4b) ------------------------------------ namespace { @@ -1510,6 +1740,60 @@ static bool gated_capture_send(void* ctx, const uint8_t* msg, uint16_t len, bool return true; } +// Tiny reversible stand-in for the separately tested RFC1951 implementation. These callbacks isolate the +// OtaManager negotiation/reassembly path: each payload block is represented by its authenticated block index, +// then expanded from the immutable test manifest before the normal Merkle proof is checked. +struct TestWireCodec { + const MotaManifest* manifest = nullptr; + uint32_t calls = 0; +}; + +static bool test_wire_encode(void* context, const uint8_t* source, uint16_t source_len, + uint8_t* destination, uint16_t capacity, uint16_t* encoded_len) { + TestWireCodec* codec = (TestWireCodec*)context; + if (!codec || !codec->manifest || !source || !destination || !encoded_len || capacity < 2) return false; + const MotaManifest& manifest = *codec->manifest; + const uint32_t block_size = manifest.block_size(); + for (uint32_t block = 0; block < manifest.block_count && block <= 0xFFu; block++) { + const uint32_t offset = block * block_size; + const uint32_t length = std::min(block_size, manifest.payload_size - offset); + if (length == source_len && memcmp(source, manifest.payload + offset, length) == 0) { + destination[0] = 0xD1; + destination[1] = (uint8_t)block; + *encoded_len = 2; + codec->calls++; + return true; + } + } + return false; +} + +static bool test_wire_decode(void* context, const uint8_t* source, uint16_t source_len, + uint8_t* destination, uint16_t capacity, uint16_t* decoded_len) { + TestWireCodec* codec = (TestWireCodec*)context; + if (!codec || !codec->manifest || !source || !destination || !decoded_len || source_len != 2 || + source[0] != 0xD1 || source[1] >= codec->manifest->block_count) return false; + const MotaManifest& manifest = *codec->manifest; + const uint32_t offset = (uint32_t)source[1] * manifest.block_size(); + const uint32_t length = std::min(manifest.block_size(), manifest.payload_size - offset); + if (length != capacity) return false; + memcpy(destination, manifest.payload + offset, length); + *decoded_len = (uint16_t)length; + codec->calls++; + return true; +} + +static bool test_representation_decode(void* context, const uint8_t* source, uint16_t source_len, + uint8_t* destination, uint16_t capacity, + uint16_t* decoded_len) { + const MotaManifest* manifest = (const MotaManifest*)context; + if (!manifest || !source || !destination || !decoded_len || source_len != 300 || + (source[0] != 0xA1 && source[0] != 0xB2) || capacity != manifest->block_size()) return false; + memcpy(destination, manifest->payload, capacity); + *decoded_len = capacity; + return true; +} + TEST(OtaMetrics, CountsOnlyPacketsAcceptedByTheRadioAdapter) { OtaManager manager; GatedCapture sent; @@ -1585,6 +1869,8 @@ private: class RamMotaSource : public mesh::ota::MotaSource { public: void add(const uint8_t* buf, uint32_t len) { if (_n < 8) { _buf[_n] = buf; _len[_n] = len; _n++; } } + void setSourceCaps(uint8_t caps) { _source_caps = caps; } + uint32_t deflateCalls() const { return _deflate_calls; } uint8_t count() override { return _n; } bool describe(uint8_t idx, mesh::ota::MotaDesc& d) override { if (idx >= _n) return false; @@ -1594,6 +1880,7 @@ public: d.target_id = m.target_id; d.fw_version = m.fw_version; d.codec_id = m.codec_id; d.flags = m.flags; d.block_size_log2 = m.block_size_log2; + d.source_caps = _source_caps; d.total_size = _len[idx]; d.leaves_off = (uint32_t)(m.leaves - _buf[idx]); d.block_count = m.block_count; @@ -1606,8 +1893,15 @@ public: std::memcpy(out, _buf[idx] + off, len); return true; } + bool read_deflated_block(uint8_t, uint16_t, uint8_t*, uint16_t, uint16_t* len) override { + _deflate_calls++; + if (len) *len = 0; + return false; + } private: const uint8_t* _buf[8] = {nullptr}; uint32_t _len[8] = {0}; uint8_t _n = 0; + uint8_t _source_caps = 0; + uint32_t _deflate_calls = 0; }; class SyntheticCatalogSource : public mesh::ota::MotaSource { @@ -1706,6 +2000,36 @@ TEST(OtaTransfer, TwoManagersFullTransfer) { EXPECT_TRUE(mota_check_image_hash_full(m)); } +TEST(OtaTransfer, TransportCodecInflatesBeforeProofAndStagesOriginalContainer) { + g_q.clear(); + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); + OtaManager server, client; + OtaStoreRam<4096> store; + SendTo to_client{&client}, to_server{&server}; + TestWireCodec encoder; + TestWireCodec decoder; + encoder.manifest = &manifest; + decoder.manifest = &manifest; + + server.begin(0, sim_send, &to_client); + server.set_transport_deflate_encoder(test_wire_encode, &encoder); + client.begin(SIM_TARGET_ID, sim_send, &to_server); + client.set_transport_deflate_decoder(test_wire_decode, &decoder); + client.set_fetch_store(&store); + client.set_autofetch(OtaManager::AUTOFETCH_ANY); + ASSERT_TRUE(server.serve(SIM_MOTA_1K, SIM_MOTA_1K_LEN)); + + server.announce(); + pump(client, &server); + + EXPECT_EQ(client.fetchState(), OtaManager::COMPLETE); + EXPECT_GT(encoder.calls, 0u); + EXPECT_EQ(decoder.calls, manifest.block_count); + ASSERT_EQ(store.staged_size(), SIM_MOTA_1K_LEN); + EXPECT_EQ(0, memcmp(store.data(), SIM_MOTA_1K, SIM_MOTA_1K_LEN)); +} + static void deliver_manifest_fragment(OtaManager& client, const uint8_t mid[4], uint8_t frag_idx, const uint8_t* bytes, uint16_t len) { uint8_t wire[MAX_PACKET_PAYLOAD]; @@ -1758,6 +2082,48 @@ static void deliver_verified_block(OtaManager& client, const MotaManifest& manif ASSERT_TRUE(client.on_message(wire, wire_len)); } +static void deliver_verified_v2_raw_block(OtaManager& client, const MotaManifest& manifest, + uint32_t block) { + ASSERT_LT(block, manifest.block_count); + const uint32_t block_size = manifest.block_size(); + const uint32_t block_off = block * block_size; + const uint16_t block_len = (uint16_t)(block_off + block_size <= manifest.payload_size + ? block_size : manifest.payload_size - block_off); + const uint32_t fragments = (block_len + OTA_FRAG_DATA_V2 - 1) / OTA_FRAG_DATA_V2; + uint8_t stream_id[OTA_DATA_V2_STREAM_ID_BYTES]; + mh4(stream_id, manifest.payload + block_off, block_len); + uint8_t wire[MAX_PACKET_PAYLOAD]; + for (uint32_t fragment = 0; fragment < fragments; fragment++) { + const uint32_t offset = fragment * OTA_FRAG_DATA_V2; + uint16_t length = (uint16_t)(block_len - offset); + if (length > OTA_FRAG_DATA_V2) length = OTA_FRAG_DATA_V2; + uint8_t body[OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2]; + memcpy(body, stream_id, sizeof(stream_id)); + memcpy(body + sizeof(stream_id), manifest.payload + block_off + offset, length); + DataMsg data{}; + memcpy(data.manifest_id, manifest.merkle_root, 4); + data.block_idx = (uint16_t)block; + ASSERT_TRUE(ota_data_v2_pack((uint8_t)fragment, block_len, false, data.frag_off)); + data.data = body; + data.data_len = (uint16_t)(sizeof(stream_id) + length); + const uint16_t wire_len = encode_data(wire, sizeof(wire), data); + ASSERT_GT(wire_len, 0); + ASSERT_TRUE(client.on_message(wire, wire_len)); + } + std::vector scratch(manifest.block_count * 4); + uint8_t siblings[32 * 4]; + const uint8_t sibling_count = merkle_gen_proof( + manifest.leaves, manifest.block_count, block, scratch.data(), siblings); + ProofMsg proof{}; + memcpy(proof.manifest_id, manifest.merkle_root, 4); + proof.block_idx = (uint16_t)block; + proof.n_proof = sibling_count; + proof.proof = siblings; + const uint16_t wire_len = encode_proof(wire, sizeof(wire), proof); + ASSERT_GT(wire_len, 0); + ASSERT_TRUE(client.on_message(wire, wire_len)); +} + TEST(OtaTransfer, ServerPacesOneKilobyteBlockAndProactiveProofWithBackpressure) { MotaManifest manifest; ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); @@ -1811,6 +2177,58 @@ TEST(OtaTransfer, ServerPacesOneKilobyteBlockAndProactiveProofWithBackpressure) EXPECT_EQ(server.pendingServeJobs(), 0u); } +TEST(OtaTransfer, NegotiatedV2Carries171RawBytesAndRepeatsRepresentationId) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); + ASSERT_EQ(manifest.block_size(), 1024u); + + OtaManager server; + CapturedMessages sent; + server.begin(0, capture_send, &sent); + ASSERT_TRUE(server.serve(SIM_MOTA_1K, SIM_MOTA_1K_LEN)); + + ReqMsg request; + memcpy(request.manifest_id, manifest.merkle_root, 4); + request.block_idx = 0; + request.want_mask = ota_req_make_v2(0x007Fu, true); // no encoder installed: source must fall back to raw + uint8_t wire[MAX_PACKET_PAYLOAD]; + const uint16_t request_len = encode_req(wire, sizeof(wire), request); + ASSERT_GT(request_len, 0); + ASSERT_TRUE(server.on_message(wire, request_len)); + + const uint32_t fragment_count = + (manifest.block_size() + OTA_FRAG_DATA_V2 - 1) / OTA_FRAG_DATA_V2; + ASSERT_EQ(fragment_count, 6u); + std::array reassembled{}; + uint8_t expected_stream_id[4]; + mh4(expected_stream_id, manifest.payload, manifest.block_size()); + + for (uint32_t i = 0; i < fragment_count; i++) { + server.serviceEgress(); + ASSERT_EQ(sent.items.size(), i + 1); + DataMsg data{}; + ASSERT_TRUE(decode_data(sent.items.back().data(), + (uint16_t)sent.items.back().size(), data)); + uint8_t fragment = 0; + uint16_t encoded_len = 0; + bool deflated = true; + ASSERT_TRUE(ota_data_v2_unpack(data.frag_off, fragment, encoded_len, deflated)); + EXPECT_EQ(fragment, i); + EXPECT_EQ(encoded_len, manifest.block_size()); + EXPECT_FALSE(deflated); + ASSERT_GE(data.data_len, OTA_DATA_V2_STREAM_ID_BYTES); + EXPECT_EQ(0, memcmp(data.data, expected_stream_id, sizeof(expected_stream_id))); + const uint16_t slice_len = + (uint16_t)(data.data_len - OTA_DATA_V2_STREAM_ID_BYTES); + const uint32_t offset = i * OTA_FRAG_DATA_V2; + EXPECT_EQ(slice_len, (uint16_t)std::min(OTA_FRAG_DATA_V2, + manifest.block_size() - offset)); + if (i + 1 < fragment_count) EXPECT_EQ(sent.items.back().size(), MAX_PACKET_PAYLOAD); + memcpy(reassembled.data() + offset, data.data + OTA_DATA_V2_STREAM_ID_BYTES, slice_len); + } + EXPECT_EQ(0, memcmp(reassembled.data(), manifest.payload, manifest.block_size())); +} + TEST(OtaTransfer, LegacyProofRequestBypassesProactiveProofTurnaround) { MotaManifest manifest; ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); @@ -2046,6 +2464,265 @@ TEST(OtaTransfer, ClientUsesQuietBatchedFlightsAndAcceptsOutOfOrderBlocks) { EXPECT_EQ(sent.items.size(), 2u); // no continuous-refill request was emitted } +TEST(OtaTransfer, NewClientRequestsV2ThenRetriesLegacyWhenNoV2DataArrives) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); + + OtaManager client; + OtaStoreRam<4096> store; + CapturedMessages sent; + client.begin(SIM_TARGET_ID, capture_send, &sent); + client.set_fetch_store(&store); + client.set_clock(100); + client.pull(manifest.merkle_root, manifest.target_id); + sent.items.clear(); + deliver_manifest_fragment(client, manifest.merkle_root, 0, + manifest.manifest_start, OTA_MF_FRAG); + deliver_manifest_fragment(client, manifest.merkle_root, 1, + manifest.manifest_start + OTA_MF_FRAG, + (uint16_t)(MOTA_MFL - OTA_MF_FRAG)); + ASSERT_EQ(sent.items.size(), 1u); + ReqMsg first{}; + ASSERT_TRUE(decode_req(sent.items[0].data(), (uint16_t)sent.items[0].size(), first)); + EXPECT_TRUE(ota_req_is_v2(first.want_mask)); + EXPECT_EQ(ota_req_v2_fragments(first.want_mask), 0x007Fu); // remains a complete old-source request + EXPECT_EQ(first.want_mask & OTA_REQ_V2_ALLOW_DEFLATE, 0u); // no decoder was linked/configured + + const uint32_t timeout = client.fetchRetryTimeoutMs(); + client.set_clock(100 + timeout); + client.loop(); + ASSERT_EQ(sent.items.size(), 2u); + ReqMsg fallback{}; + ASSERT_TRUE(decode_req(sent.items[1].data(), (uint16_t)sent.items[1].size(), fallback)); + EXPECT_FALSE(ota_req_is_v2(fallback.want_mask)); + EXPECT_EQ(fallback.want_mask, 0x007Fu); +} + +TEST(OtaTransfer, MalformedLegacyDataCannotForceV2SessionDowngrade) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); + OtaManager client; + OtaStoreRam<4096> store; + CapturedMessages sent; + client.begin(SIM_TARGET_ID, capture_send, &sent); + client.set_fetch_store(&store); + client.pull(manifest.merkle_root, manifest.target_id); + sent.items.clear(); + deliver_manifest_fragment(client, manifest.merkle_root, 0, + manifest.manifest_start, OTA_MF_FRAG); + deliver_manifest_fragment(client, manifest.merkle_root, 1, + manifest.manifest_start + OTA_MF_FRAG, + (uint16_t)(MOTA_MFL - OTA_MF_FRAG)); + + uint8_t wire[MAX_PACKET_PAYLOAD]; + DataMsg malformed; + memcpy(malformed.manifest_id, manifest.merkle_root, 4); + malformed.block_idx = 0; + malformed.frag_off = 1; // not canonical 160-byte legacy alignment + malformed.data = manifest.payload; + malformed.data_len = OTA_FRAG_DATA; + uint16_t wire_len = encode_data(wire, sizeof(wire), malformed); + ASSERT_GT(wire_len, 0); + EXPECT_FALSE(client.on_message(wire, wire_len)); + + uint8_t body[OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2]; + mh4(body, manifest.payload, manifest.block_size()); + memcpy(body + OTA_DATA_V2_STREAM_ID_BYTES, manifest.payload, OTA_FRAG_DATA_V2); + DataMsg valid; + memcpy(valid.manifest_id, manifest.merkle_root, 4); + valid.block_idx = 0; + ASSERT_TRUE(ota_data_v2_pack(0, (uint16_t)manifest.block_size(), false, valid.frag_off)); + valid.data = body; + valid.data_len = sizeof(body); + wire_len = encode_data(wire, sizeof(wire), valid); + ASSERT_EQ(wire_len, MAX_PACKET_PAYLOAD); + EXPECT_TRUE(client.on_message(wire, wire_len)); // malformed legacy packet left v2 negotiation intact +} + +TEST(OtaTransfer, PartialV2SeederCanDisappearAndLegacySeederTakesOver) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); + OtaManager client; + OtaStoreRam<4096> store; + CapturedMessages sent; + client.begin(SIM_TARGET_ID, capture_send, &sent); + client.set_transport_deflate_decoder(ota_transport_inflate); + client.set_fetch_store(&store); + client.set_clock(100); + client.pull(manifest.merkle_root, manifest.target_id); + sent.items.clear(); + deliver_manifest_fragment(client, manifest.merkle_root, 0, + manifest.manifest_start, OTA_MF_FRAG); + deliver_manifest_fragment(client, manifest.merkle_root, 1, + manifest.manifest_start + OTA_MF_FRAG, + (uint16_t)(MOTA_MFL - OTA_MF_FRAG)); + + uint8_t body[OTA_DATA_V2_STREAM_ID_BYTES + OTA_FRAG_DATA_V2]; + mh4(body, manifest.payload, manifest.block_size()); + memcpy(body + OTA_DATA_V2_STREAM_ID_BYTES, manifest.payload, OTA_FRAG_DATA_V2); + DataMsg partial{}; + memcpy(partial.manifest_id, manifest.merkle_root, 4); + partial.block_idx = 0; + ASSERT_TRUE(ota_data_v2_pack(0, (uint16_t)manifest.block_size(), false, partial.frag_off)); + partial.data = body; + partial.data_len = sizeof(body); + uint8_t wire[MAX_PACKET_PAYLOAD]; + uint16_t wire_len = encode_data(wire, sizeof(wire), partial); + ASSERT_EQ(wire_len, MAX_PACKET_PAYLOAD); + ASSERT_TRUE(client.on_message(wire, wire_len)); + + sent.items.clear(); + uint32_t now = 100; + uint32_t timeout = client.fetchRetryTimeoutMs(); + now += timeout; + client.set_clock(now); + client.loop(); // one sparse v2 retry + ASSERT_FALSE(sent.items.empty()); + ReqMsg retry{}; + ASSERT_TRUE(decode_req(sent.items.back().data(), (uint16_t)sent.items.back().size(), retry)); + EXPECT_TRUE(ota_req_is_v2(retry.want_mask)); + + sent.items.clear(); + timeout = client.fetchRetryTimeoutMs(); + now += timeout; + client.set_clock(now); + client.loop(); // bounded whole-session legacy fallback + ASSERT_FALSE(sent.items.empty()); + ReqMsg fallback{}; + ASSERT_TRUE(decode_req(sent.items.back().data(), (uint16_t)sent.items.back().size(), fallback)); + EXPECT_FALSE(ota_req_is_v2(fallback.want_mask)); + EXPECT_EQ(fallback.want_mask, 0x007Fu); + + deliver_verified_block(client, manifest, 0); // an old seeder can now make forward progress + EXPECT_EQ(client.blocksHave(), 1u); +} + +TEST(OtaTransfer, ConfirmedV2SeederCanDisappearBetweenBlocksAndLegacySeederTakesOver) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); + ASSERT_GE(manifest.block_count, 2u); + OtaManager client; + OtaStoreRam<4096> store; + CapturedMessages sent; + client.begin(SIM_TARGET_ID, capture_send, &sent); + client.set_fetch_store(&store); + client.set_clock(100); + client.pull(manifest.merkle_root, manifest.target_id); + sent.items.clear(); + deliver_manifest_fragment(client, manifest.merkle_root, 0, + manifest.manifest_start, OTA_MF_FRAG); + deliver_manifest_fragment(client, manifest.merkle_root, 1, + manifest.manifest_start + OTA_MF_FRAG, + (uint16_t)(MOTA_MFL - OTA_MF_FRAG)); + + deliver_verified_v2_raw_block(client, manifest, 0); + ASSERT_EQ(client.blocksHave(), 1u); + sent.items.clear(); // v2 source vanishes before the next flight answers + + uint32_t now = 100; + uint32_t timeout = client.fetchRetryTimeoutMs(); + now += timeout; + client.set_clock(now); + client.loop(); // one sparse v2 retry + ASSERT_EQ(sent.items.size(), 1u); + ReqWindowMsg retry{}; + ASSERT_TRUE(decode_req_window(sent.items.back().data(), + (uint16_t)sent.items.back().size(), retry)); + ASSERT_EQ(retry.n_items, 1u); + EXPECT_TRUE(ota_req_is_v2(retry.items[0].want_mask)); + + sent.items.clear(); + timeout = client.fetchRetryTimeoutMs(); + now += timeout; + client.set_clock(now); + client.loop(); // whole empty flight falls back to legacy + ASSERT_EQ(sent.items.size(), 1u); + ReqWindowMsg fallback{}; + ASSERT_TRUE(decode_req_window(sent.items.back().data(), + (uint16_t)sent.items.back().size(), fallback)); + ASSERT_GE(fallback.n_items, 1u); + for (uint8_t i = 0; i < fallback.n_items; i++) { + EXPECT_FALSE(ota_req_is_v2(fallback.items[i].want_mask)); + } + + deliver_verified_block(client, manifest, 1); // deployed source resumes the same session + EXPECT_EQ(client.blocksHave(), 2u); +} + +TEST(OtaTransfer, V2NeverMixesDifferentCompressedRepresentationsAcrossSeeders) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); + + OtaManager client; + OtaStoreRam<4096> store; + CapturedMessages sent; + client.begin(SIM_TARGET_ID, capture_send, &sent); + client.set_transport_deflate_decoder(test_representation_decode, &manifest); + client.set_fetch_store(&store); + client.pull(manifest.merkle_root, manifest.target_id); + sent.items.clear(); + deliver_manifest_fragment(client, manifest.merkle_root, 0, + manifest.manifest_start, OTA_MF_FRAG); + deliver_manifest_fragment(client, manifest.merkle_root, 1, + manifest.manifest_start + OTA_MF_FRAG, + (uint16_t)(MOTA_MFL - OTA_MF_FRAG)); + ASSERT_EQ(sent.items.size(), 1u); + ReqMsg request{}; + ASSERT_TRUE(decode_req(sent.items[0].data(), (uint16_t)sent.items[0].size(), request)); + EXPECT_NE(request.want_mask & OTA_REQ_V2_ALLOW_DEFLATE, 0u); + + std::array representation_a{}; + std::array representation_b{}; + representation_a.fill(0xA1); + representation_b.fill(0xB2); + uint8_t id_a[4], id_b[4]; + mh4(id_a, representation_a.data(), representation_a.size()); + mh4(id_b, representation_b.data(), representation_b.size()); + ASSERT_NE(0, memcmp(id_a, id_b, sizeof(id_a))); + + auto deliver_fragment = [&](const std::array& representation, + const uint8_t id[4], uint8_t fragment) { + const uint32_t offset = (uint32_t)fragment * OTA_FRAG_DATA_V2; + const uint16_t length = (uint16_t)std::min(OTA_FRAG_DATA_V2, + representation.size() - offset); + std::array body{}; + memcpy(body.data(), id, OTA_DATA_V2_STREAM_ID_BYTES); + memcpy(body.data() + OTA_DATA_V2_STREAM_ID_BYTES, representation.data() + offset, length); + uint16_t descriptor = 0; + if (!ota_data_v2_pack(fragment, (uint16_t)representation.size(), true, descriptor)) return false; + DataMsg data; + memcpy(data.manifest_id, manifest.merkle_root, 4); + data.block_idx = 0; + data.frag_off = descriptor; + data.data = body.data(); + data.data_len = (uint16_t)(OTA_DATA_V2_STREAM_ID_BYTES + length); + uint8_t wire[MAX_PACKET_PAYLOAD]; + const uint16_t wire_len = encode_data(wire, sizeof(wire), data); + return wire_len != 0 && client.on_message(wire, wire_len); + }; + + ASSERT_TRUE(deliver_fragment(representation_a, id_a, 0)); + EXPECT_FALSE(deliver_fragment(representation_b, id_b, 1)); // valid stream, wrong representation id + ASSERT_TRUE(deliver_fragment(representation_a, id_a, 1)); + + std::vector scratch(manifest.block_count * 4); + uint8_t siblings[32 * 4]; + const uint8_t sibling_count = merkle_gen_proof( + manifest.leaves, manifest.block_count, 0, scratch.data(), siblings); + ProofMsg proof; + memcpy(proof.manifest_id, manifest.merkle_root, 4); + proof.block_idx = 0; + proof.n_proof = sibling_count; + proof.proof = siblings; + uint8_t proof_wire[MAX_PACKET_PAYLOAD]; + const uint16_t proof_len = encode_proof(proof_wire, sizeof(proof_wire), proof); + ASSERT_GT(proof_len, 0); + ASSERT_TRUE(client.on_message(proof_wire, proof_len)); + EXPECT_EQ(client.blocksHave(), 1u); + const uint32_t payload_offset = (uint32_t)(manifest.payload - SIM_MOTA_1K); + EXPECT_EQ(0, memcmp(store.data() + payload_offset, manifest.payload, manifest.block_size())); +} + TEST(OtaTransfer, NewClientFallsBackWhenLegacySourceServesOnlyFirstWindowRow) { MotaManifest manifest; ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); @@ -2430,6 +3107,34 @@ TEST(OtaFolder, ServesSelfPlusFolderAndFetchesExternal) { EXPECT_TRUE(mota_check_image_hash_full(got)); } +TEST(OtaFolder, OldHostCapabilityZeroSkipsUnsupportedDeflateOperation) { + g_q.clear(); + OtaManager server, client; + OtaStoreRam<4096> store; + SendTo to_client{&client}, to_server{&server}; + server.begin(0, sim_send, &to_client); + client.begin(SIM_TARGET_ID, sim_send, &to_server); + client.set_transport_deflate_decoder(ota_transport_inflate); + client.set_fetch_store(&store); + + RamMotaSource old_host; + old_host.add(SIM_MOTA_1K, SIM_MOTA_1K_LEN); + old_host.setSourceCaps(0); // deployed descriptor's reserved byte + ASSERT_TRUE(server.add_source(&old_host)); + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, manifest)); + + server.announce(); + pump(client, &server); + client.pull(manifest.merkle_root, manifest.target_id); + pump(client, &server); + + EXPECT_EQ(client.fetchState(), OtaManager::COMPLETE); + EXPECT_EQ(old_host.deflateCalls(), 0u); // no per-block timeout on unknown op 0x09 + ASSERT_EQ(store.staged_size(), SIM_MOTA_1K_LEN); + EXPECT_EQ(0, memcmp(store.data(), SIM_MOTA_1K, SIM_MOTA_1K_LEN)); +} + TEST(OtaCatalog, QueryCanRequestOnlyMissingHaveFragment) { OtaManager server; CapturedMessages sent; diff --git a/tools/ble_mota/ble_mota_seeder.py b/tools/ble_mota/ble_mota_seeder.py index f0acf16a..33e5eb7d 100755 --- a/tools/ble_mota/ble_mota_seeder.py +++ b/tools/ble_mota/ble_mota_seeder.py @@ -10,11 +10,13 @@ from __future__ import annotations import argparse import asyncio import dataclasses +import functools import shutil import signal import struct import subprocess import sys +import zlib from pathlib import Path BLEAK_IMPORT_ERROR: ImportError | None = None @@ -50,10 +52,13 @@ MOTA_FLAG_ANOTHER_LINK_ACTIVE = 0x04 OP_COUNT = 0x01 OP_DESCRIBE = 0x02 OP_READ = 0x03 +OP_DEFLATE_BLOCK = 0x09 STATUS_OK = 0 STATUS_ERR = 1 MOTA_READ_MAX = 192 +MOTA_DEFLATE_CHUNK_MAX = 190 MOTA_DESC_WIRE = 38 +MOTA_SOURCE_CAP_DEFLATE_BLOCK = 0x01 MOTA_HEADER_LEN = 8 MOTA_MANIFEST_LEN = 197 MOTA_TRAILER = b"vk496" @@ -72,6 +77,28 @@ class MotaFile: size: int descriptor: bytes + @functools.lru_cache(maxsize=16) + def deflated_block(self, block_index: int) -> bytes | None: + """Return one smaller raw-DEFLATE payload block, cached for chunk reads.""" + block_count = struct.unpack_from("= block_count: + return None + raw_offset = block_index * block_size + raw_length = min(block_size, payload_size - raw_offset) + if raw_length <= 0 or raw_length > 1024 or self.path.stat().st_size != self.size: + return None + with self.path.open("rb") as stream: + stream.seek(payload_offset + raw_offset) + raw = stream.read(raw_length) + if len(raw) != raw_length: + return None + compressor = zlib.compressobj(level=9, method=zlib.DEFLATED, wbits=-10) + encoded = compressor.compress(raw) + compressor.flush() + return encoded if 0 < len(encoded) < len(raw) else None + @staticmethod def load(path: Path) -> "MotaFile": size = path.stat().st_size @@ -117,6 +144,7 @@ class MotaFile: struct.pack_into("= offset and end <= len(encoded) + and (length != 0 or offset == 0)): + payload = struct.pack(" bytes: + return b"MS" + bytes((op,)) + args + bytes((seeder.xor_bytes(args, op),)) + + +def response_payload(response: bytes, op: int) -> tuple[int, bytes]: + assert response[:2] == b"ms" + assert response[2] == op + assert response[-1] == seeder.xor_bytes(response[:-1]) + return response[3], response[4:-1] + + +class TransportDeflateTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.path = Path(self.directory.name) / "test.mota" + # This distribution makes zlib level 9 choose a dynamic-Huffman block, proving the + # host is not silently constrained to the bootloader's separate fixed-only profile. + self.raw = bytes(i % 31 if i % 7 else 0 for i in range(1024)) + payload_offset = 256 + self.path.write_bytes(bytes(payload_offset) + self.raw) + descriptor = bytearray(seeder.MOTA_DESC_WIRE) + struct.pack_into(" tuple[int, bytes]: + args = struct.pack("> 1) & 0x03, 2) # dynamic Huffman (BTYPE=2) + self.assertEqual(zlib.decompress(bytes(encoded), wbits=-10), self.raw) + + def test_rejects_bad_block_range_and_oversized_chunk(self): + self.assertEqual(self.exchange(1, 0, 0)[0], seeder.STATUS_ERR) + self.assertEqual( + self.exchange(0, 0, seeder.MOTA_DEFLATE_CHUNK_MAX + 1)[0], + seeder.STATUS_ERR, + ) + + class SourceStatusTests(unittest.TestCase): def test_parses_current_status_with_packet_count(self): response = bytes.fromhex("0001030200020078563412")