From eed6581299e21d141ebdf12017385a0864412cf7 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Tue, 4 Aug 2026 17:33:17 -0700 Subject: [PATCH] Add persistent SD OTA archive support --- docs/ota_meshtower_v2_sdcard.md | 233 +++++++++++++++ docs/ota_protocol.md | 43 ++- docs/ota_user_guide.md | 15 +- src/Mesh.cpp | 3 + src/helpers/CommonCLI.cpp | 227 +++++++++++++++ src/helpers/CommonCLI.h | 10 + src/helpers/ota/MotaSeederProto.h | 3 +- src/helpers/ota/MotaSourceSerial.cpp | 3 +- src/helpers/ota/OtaCacheSdNrf52.cpp | 408 +++++++++++++++++++++++++++ src/helpers/ota/OtaCacheSdNrf52.h | 90 ++++++ src/helpers/ota/OtaCli.cpp | 119 +++++++- src/helpers/ota/OtaContext.h | 147 +++++++++- src/helpers/ota/OtaManager.cpp | 259 +++++++++++++---- src/helpers/ota/OtaManager.h | 80 +++++- src/helpers/ota/OtaProtocol.cpp | 16 +- src/helpers/ota/OtaProtocol.h | 6 +- src/helpers/ota/OtaSource.h | 1 + src/helpers/ota/OtaStoreSdNrf52.cpp | 254 +++++++++++++++++ src/helpers/ota/OtaStoreSdNrf52.h | 12 + test/test_ota/test_ota_core.cpp | 199 ++++++++++++- 20 files changed, 2034 insertions(+), 94 deletions(-) create mode 100644 src/helpers/ota/OtaCacheSdNrf52.cpp create mode 100644 src/helpers/ota/OtaCacheSdNrf52.h diff --git a/docs/ota_meshtower_v2_sdcard.md b/docs/ota_meshtower_v2_sdcard.md index 24c965fde..5b0485c05 100644 --- a/docs/ota_meshtower_v2_sdcard.md +++ b/docs/ota_meshtower_v2_sdcard.md @@ -69,3 +69,236 @@ The SD-aware bootloader is mandatory. `ota install` refuses to reboot if the bootloader capability marker does not advertise SD staging and the selected codec. Existing `Heltec_tower_v2_repeater` firmware continues to use the internal-flash delta path and is unchanged. + +## SD card CLI + +The SD-backed target provides these CLI commands: + +```text +set sdcard format [--force] +set sdcard erase [--force] +get sdcard +get sdcard * +get sdcard format +get sdcard erase +get sdcard free +get sdcard ls +get sdcard ls 2 +get sdcard dir 3 +``` + +`format` creates a new FAT16, FAT32, or exFAT filesystem according to card +size. `erase` first uses the card's raw media erase command and then formats it, +so a successful erase finishes with a usable filesystem. Both operations +destroy all data on the card and cancel any staged OTA download. + +The firmware records successful format and erase completion times in RAM. +Repeating the same operation within five minutes is rejected unless `--force` +is present. Format and erase have independent cooldowns. Because erase also +formats the card, a successful erase updates both timestamps. The timestamps +reset when the device reboots. The `get sdcard` age queries report how long ago +each operation completed. `get sdcard free` reports used and free filesystem +space in human-readable binary units. + +`get sdcard ls` and `get sdcard dir` recursively list files on the card, four +files per page. A bare command shows page 1; append a positive page number to +move through the remaining results. Each row includes the path and a compact +file size. The header reports the selected page, total pages, and total files. + +## Persistent OTA archive and seeder + +On the SD-backed target, automatic OTA archiving is on by default. While the +temporary OTA radio is active, the node requests full catalogs from seeders and +saves every complete mOTA it discovers, including firmware for other hardware +targets and codecs that this node cannot install. Archive downloads use the +same per-block Merkle proof checks as an install download, but archived images +are never selected for local installation. + +Completed containers are stored as `/mota/.mota`. An interrupted +download remains `/mota/.part` and resumes when that mOTA is seen +again. Completed files survive reboot, are enumerated on the first archive +access or TempRadio window, and are advertised and served directly from SD. +The SD target supports the protocol maximum served set: its own running +firmware plus up to 254 archived mOTAs. + +Automatic capture preserves an 8 MiB free-space reserve for manual +`/meshcore-ota.mota` installation staging. It stops starting new archive files +when the next file would cross that reserve. Archive allocation first tries the +fast contiguous path and then falls back to an ordinary fragmented FAT file; +only the bootloader staging file requires contiguous sectors. + +### Preload many mOTAs from a computer + +You can populate the archive much faster on a computer than over LoRa. Use only +complete, verified `.mota` containers. Do not copy firmware `.bin`, `.hex`, +`.zip`, or `.part` files into the archive. + +Install the standalone [`motatool`](https://github.com/vk496/motatool) first if +it is not already available: + +```bash +git clone https://github.com/vk496/motatool.git +cargo install --path ./motatool +``` + +The on-card filename is part of the archive index and has a strict format: + +```text +/mota/.mota +``` + +`` is the eight-hex-digit value printed by `motatool inspect`. It +is also the mOTA's four-byte manifest ID. The extension must be lowercase, the +file must be directly inside `/mota`, and descriptive release filenames are +not indexed. For example, if inspection reports: + +```text +merkle_root : ABCD1234 +``` + +copy that container to: + +```text +/mota/abcd1234.mota +``` + +If two containers have the same Merkle root, they have the same protocol ID +and cannot both be present. Keep only the intended one. The current SD seeder +can index up to 254 archived files. A served file must use a logical block size +of at most 1024 bytes and contain at most 2048 blocks; check `block_size` and +`block_count` in `motatool inspect` when importing unusually large images. +Files outside that serve geometry are not counted or advertised. If a malformed +`/mota/.mota` conflicts with a newly discovered valid image, the repeater +preserves the malformed file as `.bad` through `.bad9` and downloads a +clean replacement instead of treating path existence as a valid cache hit. + +To prepare and load the card: + +1. Format it as described under [Card requirements](#card-requirements). The + easiest way to guarantee the expected layout is to insert it in the + repeater, run `set sdcard format`, power the repeater off, and then move the + card to the computer. Formatting destroys the existing card contents. +2. Mount the card on the computer and create a directory named `mota` at the + filesystem root. Do not use a nested directory such as `/firmware/mota`. +3. Run `motatool verify FILE.mota` for every source file. Do not copy a file + that reports `FAIL`. +4. Run `motatool inspect FILE.mota`, read its `merkle_root`, and copy the file + to `/mota/.mota` on the card. +5. Flush pending writes, safely eject the card, power the repeater off, insert + the card, and boot + `Heltec_tower_v2_sdcard_repeater_lora_ota_no_external_sensors`. + +On Linux or macOS, this Bash example verifies and imports every `.mota` from +`./motas`. Replace the example mount path before running it: + +```bash +card_mount=/media/YOU/MESHCORE +mkdir -p "$card_mount/mota" + +shopt -s nullglob +for image in ./motas/*.mota; do + motatool verify "$image" || exit 1 + mid=$(motatool inspect "$image" | + awk '$1 == "merkle_root" { print tolower($3) }') + if [[ ! $mid =~ ^[0-9a-f]{8}$ ]]; then + echo "Could not read the Merkle root from: $image" >&2 + exit 1 + fi + + destination="$card_mount/mota/$mid.mota" + if [[ -e $destination ]]; then + cmp -s "$image" "$destination" || { + echo "Different containers have the same ID: $mid" >&2 + exit 1 + } + else + cp "$image" "$destination" + fi +done +sync +``` + +On Windows, create `E:\mota`, inspect each source with `motatool inspect`, and +rename it to the reported lowercase root in the same way. Safely eject the +drive after all copies finish. + +After boot, scan the archive and confirm the card contents from the repeater +console: + +```text +get sdcard ls +get sdcard ls 2 +ota cache +ota folder +``` + +`ota cache` should report the imported count. `ota folder` reports the current +served count; once OTA serving starts in TempRadio, that set also includes the +repeater's running firmware. If the files appear in `get sdcard ls` but not in +the served count, check their exact names, run `motatool verify` again, and +inspect their block geometry. It is fine to run `ota cache off` for a curated, +read-mostly archive: that disables capture of new mOTAs but continues serving +every valid file already on the card. + +### Serve the preloaded archive over TempRadio + +LoRa OTA traffic exists only during an active temporary-radio window. The SD +repeater, each receiving node, and every intermediate repeater in the path need +overlapping windows on the same temporary channel. Use a frequency permitted +for the node's configured region. This North American example uses the +recommended fast OTA settings and a 120-minute window: + +```text +tempradio 909.950,250,5,5,120 +``` + +Start the farthest receiving node first, then intermediate repeaters, and the +SD source last so their windows overlap for as long as possible. Before or +during the source window, `ota cache` makes the source scan and attach the SD +archive. Entry into TempRadio automatically triggers an OTA advertisement +burst; `ota announce` can send another advertisement immediately. + +On a receiving OTA-capable node, allow a few seconds for catalog exchange, +then run: + +```text +ota ls +ota ls 2 +ota get 1 flash +ota status +``` + +Use `ota ls 2`, `ota ls 3`, and so on when the source advertises more than the +two rows that fit in one remote CLI reply. The update numbers are global across +pages, so select the displayed number for the desired target instead of assuming +it is always `1`. A receiver retains the complete protocol catalog and verifies +every transferred block. It still applies its normal target, hardware, codec, +signature, and installation checks. The SD +repeater may advertise images for many hardware families; it never installs +those archive files merely because it serves them. + +When a temporary window expires or a node reboots, it returns to its saved +radio settings and OTA transfer stops. The archive remains on the card. Start +another set of overlapping `tempradio` windows to resume an interrupted +download, or use synchronized `tempradioat` entries for a scheduled window. + +Archive capture is lower priority than operator work. An explicit `ota pull` +to install storage or a host folder immediately takes the single receive slot; +the archive partial is checkpointed and resumes later. Existing cached files +can still answer peer requests while another archive file is downloading. + +Use these commands to inspect or control automatic capture: + +```text +ota cache +ota cache on +ota cache off +ota config cache on +ota config cache off +``` + +The on/off choice is saved on the SD card. Turning capture off stops new +downloads but keeps serving files already cached. Formatting or erasing the +card removes both the archive and its off marker, so a newly formatted card +returns to the default-on setting. `ota config sdseed on|off` is also accepted +as an alias. diff --git a/docs/ota_protocol.md b/docs/ota_protocol.md index ba40b1b7c..b3ceae380 100644 --- a/docs/ota_protocol.md +++ b/docs/ota_protocol.md @@ -399,7 +399,8 @@ the beacon (steady state is query-free). For a single served mota, `set_digest = **Tier 2 - `OTA_QUERY` -> `OTA_HAVE`** (on interest only): ``` -OTA_QUERY (flood): seeder_id[4] set_digest[4] filter_target(uint32) # filter_target 0 = everything +OTA_QUERY (flood): seeder_id[4] set_digest[4] filter_target(uint32) want_fragments(uint32) + # filter_target 0 = everything; want_fragments 0 = every fragment OTA_HAVE (flood): seeder_id[4] set_digest[4] frag_idx(1) frag_total(1) n_rows(1) rows[] HaveRow (16 bytes, OTA_HAVE_ROW_BYTES): mid[4] target_id(4) fw_version(4) codec_id(1) flags(1) have_count(2) ``` @@ -408,9 +409,17 @@ OTA_HAVE (flood): seeder_id[4] set_digest[4] frag_idx(1) frag_total(1) n_ro Receivers do not advertise partial or completed downloads as new sources. A node interested in a source's offering schedules a QUERY; the source replies with its full catalog as -`OTA_HAVE` rows (fragmented if they exceed one packet - up to 12 rows per fragment). The heavy manifest is +`OTA_HAVE` rows (fragmented if they exceed one packet - 10 rows per fragment). A receiver marks the catalog +complete only after all `frag_total` fragments arrive. If any are missing after the recovery timeout, it sends +another QUERY whose `want_fragments` bitmap names only the holes. `want_fragments` is an append-only extension: +an original 13-byte QUERY is still accepted and means "send every fragment." The heavy manifest is fetched per-mid only on commit (Section 8.3). +Fragment numbers are canonical pages of the complete catalog sorted by `manifest_id`. `filter_target` may +remove rows from a requested page (and can therefore produce an empty fragment), but it never renumbers pages +or changes `frag_total`. This keeps missing-fragment recovery unambiguous when filtered and unfiltered queries +for the same `{seeder, set_digest}` are overheard together. + ### 8.2 Anti-storm (mandatory at mesh scale) If 50 neighbours all queried a new beacon at once, the mesh would collapse. Mitigations (gossip/mDNS @@ -420,8 +429,11 @@ pattern), all in `OtaManager`: (keyed by `{seeder, set_digest}`) - no query of its own needed. - **Jittered query:** a peer needing a catalog schedules its `OTA_QUERY` after a random delay `OTA_QUERY_MIN_MS (300) + rand(OTA_QUERY_SPREAD_MS (4000))`, derived from `id +/ digest +/ self`. -- **Overhear suppression:** during the jitter window, overhearing *another* QUERY **or** a HAVE for the same - `{seeder, set_digest}` CANCELS the pending query. +- **Overhear suppression:** during the jitter window, overhearing another QUERY that covers the same scope, + or completing the HAVE fragment set for the same `{seeder, set_digest}`, cancels the pending query. +- **Per-source recovery:** each seeder has independent query/retry state. One source cannot overwrite another + source's timer, and a partial reply requests only missing fragments after 15 seconds (five bounded retries, + then another source ADV or explicit `ota ls` can start a fresh series). Net effect: a digest change costs ~1 query + ~1 HAVE flood mesh-wide; a stable mesh is query-free. @@ -474,15 +486,16 @@ OTA_LEAVES: manifest_id[4] frag_idx(1) frag_total(1) bytes[] # up byte offset of `data` within the block, so the global position is `block_idx*block_size + frag_off` - a fragment is self-placing when returned by the source. The fetcher tracks a per-block slice bitmap and reassembles before requesting the proof. -- **Fragment-level requests (anti-deadlock + anti-congestion):** both `OTA_REQ` and `OTA_GET_MANIFEST` carry - a `want_mask` - bit *k* asks for fragment *k*. A fetcher requests the **full** mask on the first ask +- **Fragment-level requests (anti-deadlock + anti-congestion):** `OTA_REQ`, `OTA_GET_MANIFEST`, and `OTA_QUERY` + carry fragment masks. For catalog discovery, `want_fragments` is a 32-bit bitmap and covers the protocol + maximum 255-row catalog (26 fragments at the current packet size). For block and manifest transfer, the + `want_mask` is 16 bits. A fetcher requests the full set on the first ask (`(1< 7 frags per 1 KB block | | `OTA_MANIFEST` | 7 B | `OTA_MF_FRAG = 176` -> signed manifest ~ 2 frags | -| `OTA_HAVE` | 12 B | 12 rows x 14 B per fragment | +| `OTA_HAVE` | 12 B | 10 rows x 16 B per fragment | | `OTA_PROOF` | 8 B | up to ~44 sibling digests (>> any real tree) | A served mota supports up to `OTA_MAX_BLOCK/4` leaves in the default 4 KB proof scratch (<=1024 blocks ~ 1 MB @@ -502,8 +515,11 @@ payload); larger self-images pass a bigger scratch buffer. OTA packets may cross normal mesh relay hops, but each participating node processes or relays them only while its `tempradio` window is actually running. A receiver requests missing blocks in serial order from the offered -firmware source. It never serves partial blocks and never re-advertises a completed download. This keeps each -update as one transmitter and one receiver while still allowing active temporary-radio repeaters between them. +firmware source. It never serves partial blocks. A normal install receiver never re-advertises its completed +download. An SD archive node is the deliberate exception: after a fully proof-verified container is published +to its persistent archive, it registers that complete file as a MotaSource and advertises it as a new seeder. +This keeps each active transfer as one transmitter and one receiver while still allowing active temporary-radio +repeaters between them and persistent archive nodes to improve future availability. --- @@ -598,6 +614,7 @@ OP_DESCRIBE 0x02 args: idx(1) -> payload: MotaDesc wire (38 B) OP_READ 0x03 args: idx(1) off(4) len(2) -> payload: len bytes MotaDesc wire (38 B): mid[4] target_id(4) fw_version(4) codec(1) flags(1) total_size(4) leaves_off(4) block_count(4) payload_off(4) payload_size(4) + block_size_log2(1) reserved(3) status: 0 = OK, non-zero = error (out of range / past EOF). ``` @@ -645,7 +662,7 @@ the recommended user-facing forms. Output is plain-language (a user-facing guide ``` ota help | ? | h list the commands ota status | st (or bare `ota`) plain-language: running fw, the one fetch session (state/%/id), serving, keys -ota ls | neighbors | nbrs | updates | n discovered updates (queries sources; rows arrive async via OTA_HAVE) +ota ls | neighbors | nbrs | updates | n [page] paged updates (queries sources; rows arrive async via OTA_HAVE) ota get | pull | download <#|mid8> fetch a chosen mOTA (manual; works regardless of autofetch) ota install | apply | applydelta verify + approve + (ESP32) apply / (nRF52) reboot-to-bootloader ota cancel | drop | stop drop the current fetch session (frees the slot) diff --git a/docs/ota_user_guide.md b/docs/ota_user_guide.md index 18fe49b01..a32ee9069 100644 --- a/docs/ota_user_guide.md +++ b/docs/ota_user_guide.md @@ -92,6 +92,7 @@ from the app's repeater command screen, or the WiFi/serial OTA console. ``` ota ls +ota ls 2 # page 2 when more than two updates are available ``` Your node asks around and lists the firmware updates other nodes nearby are offering, in plain words - @@ -99,7 +100,7 @@ each with a **number**, its version, whether it's a full image or a small delta, and how recently it was seen. For example: ``` -Updates nearby (2 src) - `ota get <#>` to download: +Updates 1/1 (2 src) - `ota get <#>`: 1) v1.2.3 delta [yours] 3n 5s 2) v1.2.0 full [other hw] 1n 12s [downloading] ``` @@ -112,7 +113,9 @@ long ago it was seen. The fit marker: - **[?]** - can't tell (a build with no target id set, e.g. a bare IDE build rather than a release build). Run it again after a few seconds - discovery happens in the background, so the list fills in. Nothing is -downloaded yet; this is just looking around. (`ota neighbors` / `ota updates` also work.) +downloaded yet; this is just looking around. Two updates fit in each remote CLI reply; use `ota ls 2`, +`ota ls 3`, and so on for later pages. The displayed update numbers remain global across pages. +(`ota neighbors` / `ota updates` also work.) ### 3. Download an update @@ -208,6 +211,14 @@ ota config # show the current settings Recommended for most people: leave both **off** and update by hand. Use `autoinstall trusted` only once you've added the signer's key (next section) and you trust them to push updates unattended. +The MeshTower V2 SD OTA target has a separate, default-on **archive** policy. It saves all mOTAs it sees +to the SD card so the repeater can seed them later; this does not install them and does not change the +install-oriented `autofetch` default above. Use `ota cache` for status and `ota cache off` or +`ota config cache off` to stop new archive captures. Already cached files remain available to peers. +Manual `ota pull` commands take priority and an interrupted archive capture resumes later. +See [Preload many mOTAs from a computer](ota_meshtower_v2_sdcard.md#preload-many-motas-from-a-computer) +for the required `/mota/.mota` filenames and the complete TempRadio seeding workflow. + --- ## Optional: only trust updates from specific people diff --git a/src/Mesh.cpp b/src/Mesh.cpp index 1bde2c93f..0ccf9cb1b 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -314,6 +314,9 @@ void __attribute__((noinline)) Mesh::serviceLoopMaintenance() { } ota::ota_ctx().manager.set_clock(_ms->getMillis()); // for discovery jitter/ages + the pending-query timer ota::ota_ctx().manager.loop(); // re-request still-missing OTA blocks + fire scheduled queries +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + ota::ota_ctx().serviceSdCache(_ms->getMillis()); // archive unseen mOTAs only while the receive slot is idle +#endif _next_ota_tick = futureMillis(3000); } if (millisHasNowPassed(_next_ota_announce)) { // auto-advertise so peers discover us (tiny beacon) diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 01454737a..4c910354e 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -63,6 +63,60 @@ static void resetToUf2Bootloader() { #define RECENT_REPEATER_PREFIX_MAX_BYTES 3 +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) +static const uint32_t SDCARD_ACTION_COOLDOWN_MILLIS = 5UL * 60UL * 1000UL; + +static void formatSdCardElapsed(char* out, size_t out_len, bool recorded, + uint32_t ran_at, uint32_t now) { + if (!recorded) { + snprintf(out, out_len, "never"); + return; + } + + const uint32_t elapsed_secs = (now - ran_at) / 1000UL; + if (elapsed_secs < 60UL) { + snprintf(out, out_len, "%lus ago", (unsigned long)elapsed_secs); + } else if (elapsed_secs < 60UL * 60UL) { + snprintf(out, out_len, "%lum %lus ago", + (unsigned long)(elapsed_secs / 60UL), + (unsigned long)(elapsed_secs % 60UL)); + } else if (elapsed_secs < 24UL * 60UL * 60UL) { + snprintf(out, out_len, "%luh %lum ago", + (unsigned long)(elapsed_secs / 3600UL), + (unsigned long)((elapsed_secs / 60UL) % 60UL)); + } else { + snprintf(out, out_len, "%lud %luh ago", + (unsigned long)(elapsed_secs / 86400UL), + (unsigned long)((elapsed_secs / 3600UL) % 24UL)); + } +} + +static void formatSdCardBytes(char* out, size_t out_len, uint64_t bytes) { + static const char* const units[] = {"B", "KiB", "MiB", "GiB", "TiB"}; + uint64_t unit_size = 1; + uint8_t unit = 0; + while (unit < 4 && bytes >= unit_size * 1024ULL) { + unit_size *= 1024ULL; + unit++; + } + + if (unit == 0) { + snprintf(out, out_len, "%lu B", (unsigned long)bytes); + return; + } + + uint64_t whole = bytes / unit_size; + uint32_t tenths = (uint32_t)(((bytes % unit_size) * 10ULL + unit_size / 2ULL) / + unit_size); + if (tenths == 10) { + whole++; + tenths = 0; + } + snprintf(out, out_len, "%lu.%lu %s", (unsigned long)whole, + (unsigned long)tenths, units[unit]); +} +#endif + // Believe it or not, this std C function is busted on some platforms! static uint32_t _atoi(const char* sp) { uint32_t n = 0; @@ -3029,6 +3083,173 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re #endif } +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) +bool CommonCLI::handleSdCardSetCmd(const char* config, char* reply) { + if (strncmp(config, "sdcard", 6) != 0 || + (config[6] != 0 && config[6] != ' ')) { + return false; + } + + const char* action_args = skipSpacesConst(config + 6); + bool is_format; + bool force = false; + if (strcmp(action_args, "format") == 0) { + is_format = true; + } else if (strcmp(action_args, "format --force") == 0) { + is_format = true; + force = true; + } else if (strcmp(action_args, "erase") == 0) { + is_format = false; + } else if (strcmp(action_args, "erase --force") == 0) { + is_format = false; + force = true; + } else { + strcpy(reply, "Error: use set sdcard format|erase [--force]"); + return true; + } + + const bool recorded = is_format ? _sdcard_format_recorded : _sdcard_erase_recorded; + const uint32_t ran_at = is_format ? _sdcard_format_at : _sdcard_erase_at; + const char* action = is_format ? "format" : "erase"; + const uint32_t now = millis(); + if (!force && recorded && now - ran_at < SDCARD_ACTION_COOLDOWN_MILLIS) { + char age[32]; + formatSdCardElapsed(age, sizeof(age), true, ran_at, now); + snprintf(reply, 160, + "Error: SD card %s ran %s; use set sdcard %s --force", + action, age, action); + return true; + } + + mesh::ota::OtaContext& context = mesh::ota::ota_ctx(); + if (context.apply_pending) { + strcpy(reply, "Error: OTA apply is pending"); + return true; + } + + // A card-wide destructive operation invalidates any staged fetch. First let + // an archive capture checkpoint and close its shared card handles. + context.prepareSdCardReset(); + context.manager.reset_session(); + context.manager.want(0); + context.manager.want_mid(nullptr); + context.session_started_ms = 0; + context.prev_fstate = mesh::ota::OtaManager::IDLE; + + if (is_format) { + if (!context.fetch_store.formatCard(*_board)) { + char error[80]; + strncpy(error, context.fetch_store.last_error(), sizeof(error) - 1); + error[sizeof(error) - 1] = 0; + context.finishSdCardReset(millis()); + snprintf(reply, 160, "Error: SD card format failed: %s", + error); + return true; + } + context.finishSdCardReset(millis()); + _sdcard_format_recorded = true; + _sdcard_format_at = millis(); + strcpy(reply, "OK - SD card format complete"); + return true; + } + + if (!context.fetch_store.eraseCard(*_board)) { + char error[80]; + strncpy(error, context.fetch_store.last_error(), sizeof(error) - 1); + error[sizeof(error) - 1] = 0; + context.finishSdCardReset(millis()); + snprintf(reply, 160, "Error: SD card erase failed: %s", + error); + return true; + } + _sdcard_erase_recorded = true; + _sdcard_erase_at = millis(); + + if (!context.fetch_store.formatCard(*_board)) { + char error[80]; + strncpy(error, context.fetch_store.last_error(), sizeof(error) - 1); + error[sizeof(error) - 1] = 0; + context.finishSdCardReset(millis()); + snprintf(reply, 160, "Error: SD card erased but format failed: %s", + error); + return true; + } + _sdcard_format_recorded = true; + _sdcard_format_at = millis(); + context.finishSdCardReset(millis()); + strcpy(reply, "OK - SD card erase and format complete"); + return true; +} + +bool CommonCLI::handleSdCardGetCmd(const char* config, char* reply) { + if (strncmp(config, "sdcard", 6) != 0 || + (config[6] != 0 && config[6] != ' ')) { + return false; + } + + const char* query = skipSpacesConst(config + 6); + const uint32_t now = millis(); + if (*query == 0 || strcmp(query, "*") == 0) { + char format_age[32]; + char erase_age[32]; + formatSdCardElapsed(format_age, sizeof(format_age), + _sdcard_format_recorded, _sdcard_format_at, now); + formatSdCardElapsed(erase_age, sizeof(erase_age), + _sdcard_erase_recorded, _sdcard_erase_at, now); + snprintf(reply, 160, "> format: %s, erase: %s", format_age, erase_age); + } else if (strcmp(query, "format") == 0) { + char age[32]; + formatSdCardElapsed(age, sizeof(age), _sdcard_format_recorded, + _sdcard_format_at, now); + snprintf(reply, 160, "> format: %s", age); + } else if (strcmp(query, "erase") == 0) { + char age[32]; + formatSdCardElapsed(age, sizeof(age), _sdcard_erase_recorded, + _sdcard_erase_at, now); + snprintf(reply, 160, "> erase: %s", age); + } else if (strcmp(query, "free") == 0) { + uint64_t used_bytes = 0; + uint64_t free_bytes = 0; + mesh::ota::OtaStoreSdNrf52& store = mesh::ota::ota_ctx().fetch_store; + if (!store.getSpace(*_board, used_bytes, free_bytes)) { + snprintf(reply, 160, "Error: SD card space query failed: %s", + store.last_error()); + } else { + char used[24]; + char free[24]; + formatSdCardBytes(used, sizeof(used), used_bytes); + formatSdCardBytes(free, sizeof(free), free_bytes); + snprintf(reply, 160, "> used: %s, free: %s", used, free); + } + } else if (strncmp(query, "ls", 2) == 0 || strncmp(query, "dir", 3) == 0) { + const bool is_ls = strncmp(query, "ls", 2) == 0; + const size_t word_len = is_ls ? 2 : 3; + if (query[word_len] != 0 && query[word_len] != ' ') { + strcpy(reply, "Error: use get sdcard ls|dir [page]"); + return true; + } + const char* page_arg = skipSpacesConst(query + word_len); + uint32_t page = 1; + if (*page_arg) { + char* end = nullptr; + unsigned long parsed = strtoul(page_arg, &end, 10); + if (parsed == 0 || parsed > 65535 || !end || *skipSpacesConst(end) != 0) { + strcpy(reply, "Error: use get sdcard ls|dir [page]"); + return true; + } + page = parsed; + } + mesh::ota::OtaStoreSdNrf52& store = mesh::ota::ota_ctx().fetch_store; + if (!store.listFiles(*_board, (uint16_t)page, reply, 160)) { + snprintf(reply, 160, "Error: SD card list failed: %s", store.last_error()); + } + } else { + strcpy(reply, "Error: use get sdcard [format|erase|free|ls|dir [page]|*]"); + } + return true; +} +#endif + void CommonCLI::handleSetCmd(uint32_t sender_timestamp, char* command, char* reply) { const char* config = &command[4]; #if defined(ESP32_PLATFORM) || defined(USER_GPIO_CONTROL) @@ -3074,6 +3295,9 @@ void CommonCLI::handleSetCmd(uint32_t sender_timestamp, char* command, char* rep #endif // Observer/MQTT/WiFi/timezone/alert/SNMP commands live in CommonCLI_Observer.cpp. if (handleObserverSetCmd(sender_timestamp, config, reply)) return; +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + if (handleSdCardSetCmd(config, reply)) return; +#endif #if defined(ESP_PLATFORM) && defined(ADMIN_PASSWORD) && \ !defined(WEBCONFIG_DISABLED) && !defined(WITH_MQTT_BRIDGE) const char* wifi_value = nullptr; @@ -4187,6 +4411,9 @@ void CommonCLI::handleGetCmd(uint32_t sender_timestamp, char* command, char* rep #endif // Observer/MQTT/WiFi/timezone/alert/SNMP commands live in CommonCLI_Observer.cpp. if (handleObserverGetCmd(sender_timestamp, config, reply)) return; +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + if (handleSdCardGetCmd(config, reply)) return; +#endif #if defined(ESP_PLATFORM) && defined(ADMIN_PASSWORD) && !defined(WEBCONFIG_DISABLED) // Non-MQTT FULL repeater/room-server builds keep WiFi only for WebConfig. // Their credentials live in the standalone WebConfig NVS namespace rather diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index 98b1f9f4b..4e8cacd4f 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -514,6 +514,12 @@ class CommonCLI { RegionMap* _region_map; ClientACL* _acl; char tmp[COMMON_CLI_TMP_LEN]; +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + bool _sdcard_format_recorded = false; + bool _sdcard_erase_recorded = false; + uint32_t _sdcard_format_at = 0; + uint32_t _sdcard_erase_at = 0; +#endif #ifdef WITH_MQTT_BRIDGE MQTTPrefs _mqtt_prefs; LegacyObserverTail _legacy_tail; @@ -540,6 +546,10 @@ class CommonCLI { void handleGetCmd(uint32_t sender_timestamp, char* command, char* reply); void handleSetCmd(uint32_t sender_timestamp, char* command, char* reply); void handleDelCmd(char* command, char* reply); +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + bool handleSdCardGetCmd(const char* config, char* reply); + bool handleSdCardSetCmd(const char* config, char* reply); +#endif // Observer/MQTT/WiFi/timezone/alert/SNMP CLI handling lives in the fork-owned // CommonCLI_Observer.cpp to keep these branches out of the upstream-tracked diff --git a/src/helpers/ota/MotaSeederProto.h b/src/helpers/ota/MotaSeederProto.h index a5d365939..93e74a83d 100644 --- a/src/helpers/ota/MotaSeederProto.h +++ b/src/helpers/ota/MotaSeederProto.h @@ -20,6 +20,7 @@ // // MotaDesc wire (38 B): mid[4] target_id(4) fw_version(4) codec(1) flags(1) total_size(4) // leaves_off(4) block_count(4) payload_off(4) payload_size(4) +// block_size_log2(1) reserved(3) // // --- STORAGE ops (device -> host WRITE): "pull to folder". The device is fetching a `.mota` off the mesh // (e.g. a neighbour's self-served firmware it has no local copy of) and streams it into the host folder as @@ -55,7 +56,7 @@ static const uint8_t MS_OP_FIN = 0x08; // storage: transfer complete - v static const uint8_t MS_STATUS_OK = 0x00; static const uint8_t MS_STATUS_ERR = 0x01; -static const uint16_t MOTA_DESC_WIRE = 38; // bytes of a MotaDesc on the wire (see layout above) +static const uint16_t MOTA_DESC_WIRE = 38; // existing reserved bytes carry block geometry; wire size is unchanged static const uint16_t MOTA_SEEDER_WRITE_MAX = 512; // max data bytes per OP_WRITE/OP_SREAD (bounds frames) } // namespace ota diff --git a/src/helpers/ota/MotaSourceSerial.cpp b/src/helpers/ota/MotaSourceSerial.cpp index b1ef7a736..0ceea474e 100644 --- a/src/helpers/ota/MotaSourceSerial.cpp +++ b/src/helpers/ota/MotaSourceSerial.cpp @@ -79,7 +79,8 @@ bool SerialMotaSource::describe(uint8_t idx, MotaDesc& out) { out.block_count = rd_u32le(w + 22); out.payload_off = rd_u32le(w + 26); out.payload_size = rd_u32le(w + 30); - // bytes [34,38) reserved (zero) - kept for forward compat without changing MOTA_DESC_WIRE + out.block_size_log2 = w[34]; + // bytes [35,38) remain reserved (zero) for forward compatibility. return true; } diff --git a/src/helpers/ota/OtaCacheSdNrf52.cpp b/src/helpers/ota/OtaCacheSdNrf52.cpp new file mode 100644 index 000000000..ad4fd722b --- /dev/null +++ b/src/helpers/ota/OtaCacheSdNrf52.cpp @@ -0,0 +1,408 @@ +#include "OtaCacheSdNrf52.h" + +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + +#include +#include +#include +#include +#include "MotaContainer.h" +#include "OtaByteIO.h" +#include "OtaFormat.h" +#include "OtaManager.h" +#include "OtaStoreSdNrf52.h" + +namespace mesh { +namespace ota { + +const char* const OtaCacheSdNrf52::DIR = "/mota"; +const char* const OtaCacheSdNrf52::DISABLED_PATH = "/mota/cache.off"; + +OtaCacheSdNrf52::OtaCacheSdNrf52() : _file(new (std::nothrow) FsFile()) {} + +OtaCacheSdNrf52::~OtaCacheSdNrf52() { + if (_file) { _file->close(); delete _file; } +} + +void OtaCacheSdNrf52::fail(const char* message) { + strncpy(_error, message ? message : "SD archive error", sizeof(_error) - 1); + _error[sizeof(_error) - 1] = 0; +} + +void OtaCacheSdNrf52::attach(OtaStoreSdNrf52& owner) { + if (_owner == &owner) return; + resetMedia(); + _owner = &owner; +} + +void OtaCacheSdNrf52::resetMedia() { + if (_file && *_file) _file->close(); + _count = 0; + _total = 0; + _initialized = false; + _enabled = true; + _error[0] = 0; +} + +bool OtaCacheSdNrf52::ready() { + if (!_owner || !_file) { fail("SD archive is not attached"); return false; } + if (!_owner->mount()) { fail(_owner->last_error()); return false; } + return true; +} + +bool OtaCacheSdNrf52::ensureDir() { + if (_owner->_sd->exists(DIR)) return true; + if (_owner->_sd->mkdir(DIR)) return true; + fail("SD archive directory create failed"); + return false; +} + +bool OtaCacheSdNrf52::initialize() { + if (_initialized) return true; + _error[0] = 0; + if (!ready() || !ensureDir()) return false; + _enabled = !_owner->_sd->exists(DISABLED_PATH); + return rescan(); +} + +static int cache_hex_nibble(char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; +} + +bool OtaCacheSdNrf52::describeFile(FsFile& file, MotaDesc& out) const { + uint64_t size64 = file.fileSize(); + if (size64 < 8 + MOTA_MFL + 5 || size64 > UINT32_MAX) return false; + const uint32_t total = (uint32_t)size64; + uint8_t hdr[8], manifest[MOTA_MFL], trailer[5]; + if (!file.seekSet(0) || file.read(hdr, sizeof(hdr)) != (int)sizeof(hdr) || + memcmp(hdr, MOTA_MAGIC, 4) != 0 || rd_u32le(hdr + 4) != total || + !file.seekSet(8) || file.read(manifest, sizeof(manifest)) != (int)sizeof(manifest) || + !file.seekSet(total - sizeof(trailer)) || + file.read(trailer, sizeof(trailer)) != (int)sizeof(trailer) || + memcmp(trailer, MOTA_TRAILER, sizeof(trailer)) != 0) return false; + + MotaManifest m; + if (!mota_parse_manifest(manifest, sizeof(manifest), m)) return false; + if (m.hash_algo != HASH_ALGO_SHA256 || m.block_size() == 0 || m.block_size() > OTA_MAX_BLOCK || + (uint64_t)m.block_count * 4 > OTA_PROOFGEN_SCRATCH) return false; + uint64_t leaves_off = 8 + MOTA_MFL; + uint64_t payload_off = leaves_off + (uint64_t)m.block_count * 4; + if (payload_off > UINT32_MAX || payload_off + m.payload_size + 5 != total) return false; + + memcpy(out.mid, m.merkle_root, 4); + out.target_id = m.target_id; + out.fw_version = m.fw_version; + out.codec_id = m.codec_id; + out.flags = m.flags; + out.block_size_log2 = m.block_size_log2; + out.total_size = total; + out.leaves_off = (uint32_t)leaves_off; + out.block_count = m.block_count; + out.payload_off = (uint32_t)payload_off; + out.payload_size = m.payload_size; + return true; +} + +bool OtaCacheSdNrf52::rescan() { + _initialized = false; + _count = 0; + if (!ready() || !ensureDir()) return false; + + FsFile dir = _owner->_sd->open(DIR, O_RDONLY); + if (!dir || !dir.isDir()) { fail("SD archive directory open failed"); return false; } + FsFile file; + while (_count < OTA_SD_CACHE_MAX && file.openNext(&dir, O_RDONLY)) { + if (!file.isDir()) { + char name[24] = {0}; + file.getName(name, sizeof(name)); + size_t len = strlen(name); + if (len == 13 && strcmp(name + 8, ".mota") == 0) { + uint8_t name_mid[4]; + bool name_ok = true; + for (uint8_t i = 0; i < 4; i++) { + int hi = cache_hex_nibble(name[i * 2]); + int lo = cache_hex_nibble(name[i * 2 + 1]); + if (hi < 0 || lo < 0) { name_ok = false; break; } + name_mid[i] = (uint8_t)((hi << 4) | lo); + } + MotaDesc d; + if (name_ok && describeFile(file, d) && memcmp(name_mid, d.mid, 4) == 0) + _entries[_count++] = d; + } + } + file.close(); + } + dir.close(); + _initialized = true; + _error[0] = 0; + return true; +} + +void OtaCacheSdNrf52::pathFor(const uint8_t mid[4], const char* ext, + char* out, size_t cap) const { + snprintf(out, cap, "%s/%02x%02x%02x%02x.%s", DIR, + mid[0], mid[1], mid[2], mid[3], ext); +} + +bool OtaCacheSdNrf52::contains(const uint8_t mid[4]) { + if (!mid || !initialize()) return false; + for (uint8_t i = 0; i < _count; i++) + if (memcmp(_entries[i].mid, mid, 4) == 0) return true; + + // Recover a valid file that was published just before a card-sync/rescan failure. Existence alone is + // never enough: malformed files must remain eligible for quarantine and a clean re-download. + char path[32]; + pathFor(mid, "mota", path, sizeof(path)); + FsFile file = _owner->_sd->open(path, O_RDONLY); + MotaDesc d; + bool valid = file && describeFile(file, d) && memcmp(d.mid, mid, 4) == 0; + file.close(); + if (!valid) return false; + if (_count < OTA_SD_CACHE_MAX) _entries[_count++] = d; + return true; +} + +bool OtaCacheSdNrf52::quarantineFinal(const uint8_t mid[4]) { + char final_path[32]; + pathFor(mid, "mota", final_path, sizeof(final_path)); + if (!_owner->_sd->exists(final_path)) return true; + for (uint8_t i = 0; i < 10; i++) { + char ext[8], bad_path[32]; + if (i == 0) strcpy(ext, "bad"); + else snprintf(ext, sizeof(ext), "bad%u", (unsigned)i); + pathFor(mid, ext, bad_path, sizeof(bad_path)); + if (!_owner->_sd->exists(bad_path) && _owner->_sd->rename(final_path, bad_path)) return true; + } + fail("SD archive invalid file quarantine failed"); + return false; +} + +bool OtaCacheSdNrf52::setAutoCaptureEnabled(bool enabled) { + if (!initialize()) return false; + if (enabled) { + if (_owner->_sd->exists(DISABLED_PATH) && !_owner->_sd->remove(DISABLED_PATH)) { + fail("SD archive enable marker remove failed"); + return false; + } + } else { + FsFile marker = _owner->_sd->open(DISABLED_PATH, O_WRONLY | O_CREAT | O_TRUNC); + static const uint8_t off[] = {'o', 'f', 'f', '\n'}; + if (!marker || marker.write(off, sizeof(off)) != sizeof(off) || !marker.sync()) { + marker.close(); + fail("SD archive disable marker write failed"); + return false; + } + marker.close(); + } + _enabled = enabled; // runtime mirrors the marker even if the final device sync reports late + if (_owner->_sd->card() && !_owner->_sd->card()->syncDevice()) { + fail("SD archive setting sync failed"); + return false; + } + _error[0] = 0; + return true; +} + +void OtaCacheSdNrf52::set_mid(const uint8_t mid[4]) { + if (mid) memcpy(_mid, mid, 4); + else memset(_mid, 0, sizeof(_mid)); +} + +bool OtaCacheSdNrf52::begin(uint32_t total_size) { + _total = 0; + if (total_size < 8 + MOTA_MFL + 5 || !initialize()) return false; + if ((_mid[0] | _mid[1] | _mid[2] | _mid[3]) == 0) { fail("SD archive mid is unset"); return false; } + char final_path[32], part_path[32]; + pathFor(_mid, "mota", final_path, sizeof(final_path)); + pathFor(_mid, "part", part_path, sizeof(part_path)); + if (contains(_mid)) { fail("SD archive already contains this mOTA"); return false; } + if (_owner->_sd->exists(final_path) && !quarantineFinal(_mid)) return false; + // begin() means resumeStaged() already rejected any old partial. Drop it before measuring space so a + // corrupt/obsolete .part is not double-counted against the new allocation and the install reserve. + if (*_file) _file->close(); + if (_owner->_sd->exists(part_path) && !_owner->_sd->remove(part_path)) { + fail("SD archive obsolete partial remove failed"); + return false; + } + const uint32_t cluster_count = _owner->_sd->clusterCount(); + const uint32_t free_clusters = _owner->_sd->freeClusterCount(); + const uint32_t bytes_per_cluster = _owner->_sd->bytesPerCluster(); + const uint64_t free_bytes = (uint64_t)free_clusters * bytes_per_cluster; + if (cluster_count == 0 || bytes_per_cluster == 0 || free_clusters > cluster_count) { + fail("SD archive free-space query failed"); + return false; + } + if (free_bytes < (uint64_t)total_size + OTA_SD_CACHE_RESERVE_BYTES) { + fail("SD archive free-space reserve reached"); + return false; + } + if (!_file->open(part_path, O_RDWR | O_CREAT | O_EXCL)) { + fail("SD archive partial create failed"); + return false; + } + if (!_file->preAllocate(total_size)) { + // preAllocate() requires one contiguous cluster run. Retry with normal sequential writes so a + // fragmented card with enough total free space remains usable for archive files (unlike boot staging, + // archive files are read through FAT and do not need contiguous sectors). + _file->close(); + _owner->_sd->remove(part_path); + if (!_file->open(part_path, O_RDWR | O_CREAT | O_EXCL)) { + fail("SD archive fragmented allocation create failed"); + return false; + } + uint8_t erased[512]; + memset(erased, 0xFF, sizeof(erased)); + uint32_t left = total_size; + while (left) { + uint32_t n = left > sizeof(erased) ? sizeof(erased) : left; + if (_file->write(erased, n) != n) { + fail("SD archive lacks allocatable space"); + _file->close(); + _owner->_sd->remove(part_path); + return false; + } + left -= n; + } + if (!_file->sync()) { + fail("SD archive fragmented allocation sync failed"); + _file->close(); + _owner->_sd->remove(part_path); + return false; + } + } + _total = total_size; + return true; +} + +bool OtaCacheSdNrf52::set_meta_size(uint32_t meta_bytes) { + if (!_total || meta_bytes > _total - 5) return false; + uint8_t erased[512]; + memset(erased, 0xFF, sizeof(erased)); + for (uint32_t off = 0; off < meta_bytes; ) { + uint32_t n = meta_bytes - off; + if (n > sizeof(erased)) n = sizeof(erased); + if (!write(off, erased, n)) return false; + off += n; + } + return true; +} + +bool OtaCacheSdNrf52::write(uint32_t off, const uint8_t* data, uint32_t len) { + if (!_file || !*_file || (uint64_t)off + len > _total || !_file->seekSet(off) || + _file->write(data, len) != len) { + fail("SD archive write failed"); + return false; + } + return true; +} + +bool OtaCacheSdNrf52::read(uint32_t off, uint8_t* buf, uint32_t len) const { + return _file && *_file && (uint64_t)off + len <= _total && _file->seekSet(off) && + _file->read(buf, len) == (int)len; +} + +uint32_t OtaCacheSdNrf52::capacity() const { + if (!_owner || !_owner->_mounted || !_owner->_sd || !_owner->_sd->card()) return 0; + uint64_t bytes = (uint64_t)_owner->_sd->card()->sectorCount() * 512; + return bytes > UINT32_MAX ? UINT32_MAX : (uint32_t)bytes; +} + +void OtaCacheSdNrf52::checkpoint() { + if (_file && *_file) { + _file->sync(); + if (_owner && _owner->_sd->card()) _owner->_sd->card()->syncDevice(); + } +} + +bool OtaCacheSdNrf52::finalize() { + if (!_file || !*_file || !_total || !_file->sync()) { fail("SD archive sync failed"); return false; } + if (_owner->_sd->card() && !_owner->_sd->card()->syncDevice()) { + fail("SD archive card sync failed"); + return false; + } + char final_path[32], part_path[32]; + pathFor(_mid, "mota", final_path, sizeof(final_path)); + pathFor(_mid, "part", part_path, sizeof(part_path)); + _file->close(); + + FsFile check = _owner->_sd->open(part_path, O_RDONLY); + MotaDesc d; + bool valid = check && describeFile(check, d) && memcmp(d.mid, _mid, 4) == 0; + check.close(); + if (!valid) { fail("SD archive final container invalid"); return false; } + + if (_owner->_sd->exists(final_path)) { + if (!_owner->_sd->remove(part_path)) { + fail("SD archive duplicate cleanup failed"); + return false; + } + } else if (!_owner->_sd->rename(part_path, final_path)) { + fail("SD archive publish rename failed"); + return false; + } + bool device_synced = !_owner->_sd->card() || _owner->_sd->card()->syncDevice(); + bool scanned = rescan(); // register a valid renamed file even if device sync reported late + if (!device_synced) { fail("SD archive publish sync failed"); return false; } + return scanned; +} + +bool OtaCacheSdNrf52::reopen() { + _total = 0; + if (!initialize()) return false; + char part_path[32]; + pathFor(_mid, "part", part_path, sizeof(part_path)); + if (*_file) _file->close(); + if (!_file->open(part_path, O_RDWR)) return false; + uint8_t hdr[8], manifest[MOTA_MFL]; + if (_file->fileSize() < 8 + MOTA_MFL + 5 || !_file->seekSet(0) || + _file->read(hdr, sizeof(hdr)) != (int)sizeof(hdr) || + memcmp(hdr, MOTA_MAGIC, 4) != 0 || !_file->seekSet(8) || + _file->read(manifest, sizeof(manifest)) != (int)sizeof(manifest)) { + _file->close(); + return false; + } + uint32_t total = rd_u32le(hdr + 4); + MotaManifest m; + if (total != _file->fileSize() || !mota_parse_manifest(manifest, sizeof(manifest), m) || + memcmp(m.merkle_root, _mid, 4) != 0) { + _file->close(); + return false; + } + _total = total; + return true; +} + +void OtaCacheSdNrf52::clear() { + if (_file && *_file) _file->close(); + if (_owner && ready()) { + char part_path[32]; + pathFor(_mid, "part", part_path, sizeof(part_path)); + _owner->_sd->remove(part_path); + } + _total = 0; +} + +bool OtaCacheSdNrf52::describe(uint8_t idx, MotaDesc& out) { + if (!_initialized || idx >= _count) return false; + out = _entries[idx]; + return true; +} + +bool OtaCacheSdNrf52::read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len) { + if (!_initialized || idx >= _count || (uint64_t)off + len > _entries[idx].total_size || !ready()) + return false; + char path[32]; + pathFor(_entries[idx].mid, "mota", path, sizeof(path)); + FsFile file = _owner->_sd->open(path, O_RDONLY); + bool ok = file && file.seekSet(off) && file.read(buf, len) == (int)len; + file.close(); + return ok; +} + +} // namespace ota +} // namespace mesh + +#endif diff --git a/src/helpers/ota/OtaCacheSdNrf52.h b/src/helpers/ota/OtaCacheSdNrf52.h new file mode 100644 index 000000000..9181c80a1 --- /dev/null +++ b/src/helpers/ota/OtaCacheSdNrf52.h @@ -0,0 +1,90 @@ +#pragma once + +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + +#include "OtaSource.h" +#include "OtaStore.h" + +class FsFile; + +namespace mesh { +namespace ota { + +class OtaStoreSdNrf52; + +// The wire format has an 8-bit advertised count. Reserve one slot for this node's running firmware and +// use every remaining slot for persistent SD-backed mOTAs. +#ifndef OTA_SD_CACHE_MAX +#define OTA_SD_CACHE_MAX 254 +#endif +#ifndef OTA_SD_CACHE_RESERVE_BYTES +#define OTA_SD_CACHE_RESERVE_BYTES (8UL * 1024UL * 1024UL) // preserve room for manual install staging +#endif + +// Persistent capture destination and MotaSource for the SD OTA archive. Completed containers live as +// /mota/.mota; an in-progress capture uses /mota/.part and can resume after interruption. +// This object shares the staging store's mounted SdFs volume so FAT metadata never has two independent +// caches for the same card. +class OtaCacheSdNrf52 : public OtaStore, public MotaSource { +public: + OtaCacheSdNrf52(); + ~OtaCacheSdNrf52() override; + + void attach(OtaStoreSdNrf52& owner); + bool initialize(); + void resetMedia(); + bool rescan(); + + bool autoCaptureEnabled() const { return _enabled; } + bool setAutoCaptureEnabled(bool enabled); + bool contains(const uint8_t mid[4]); + uint8_t capturedCount() const { return _count; } + bool initialized() const { return _initialized; } + const char* last_error() const { return _error; } + + void set_mid(const uint8_t mid[4]); + + // OtaStore: one resumable .part download at a time. + bool begin(uint32_t total_size) override; + bool write(uint32_t off, const uint8_t* data, uint32_t len) override; + bool read(uint32_t off, uint8_t* buf, uint32_t len) const override; + uint32_t capacity() const override; + uint32_t staged_size() const override { return _total; } + void clear() override; + bool set_meta_size(uint32_t meta_bytes) override; + bool finalize() override; + void checkpoint() override; + bool reopen() override; + bool plan_layout(bool, uint32_t, uint32_t, uint32_t) override { return true; } + + // MotaSource: completed files are advertised and read directly from SD on demand. + uint8_t count() override { return _initialized ? _count : 0; } + bool describe(uint8_t idx, MotaDesc& out) override; + bool read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len) override; + +private: + static const char* const DIR; + static const char* const DISABLED_PATH; + + bool ready(); + bool ensureDir(); + bool describeFile(FsFile& file, MotaDesc& out) const; + bool quarantineFinal(const uint8_t mid[4]); + void pathFor(const uint8_t mid[4], const char* ext, char* out, size_t cap) const; + void fail(const char* message); + + OtaStoreSdNrf52* _owner = nullptr; + FsFile* _file = nullptr; + MotaDesc _entries[OTA_SD_CACHE_MAX]; + uint8_t _count = 0; + uint8_t _mid[4] = {0}; + uint32_t _total = 0; + bool _initialized = false; + bool _enabled = true; + char _error[80] = {0}; +}; + +} // namespace ota +} // namespace mesh + +#endif diff --git a/src/helpers/ota/OtaCli.cpp b/src/helpers/ota/OtaCli.cpp index 612dc9c09..e7774a8c2 100644 --- a/src/helpers/ota/OtaCli.cpp +++ b/src/helpers/ota/OtaCli.cpp @@ -23,6 +23,21 @@ static uint32_t parse_u32(const char* s) { return n; } +static bool parse_page(const char* s, uint16_t& page) { + while (*s == ' ') s++; + if (*s == 0) { page = 1; return true; } + uint32_t n = 0; + const char* p = s; + while (*p >= '0' && *p <= '9') { + n = n * 10 + (uint32_t)(*p++ - '0'); + if (n > 255) return false; + } + while (*p == ' ') p++; + if (*p != 0 || n == 0) return false; + page = (uint16_t)n; + return true; +} + static char fstate_char(OtaManager::FetchState s) { switch (s) { case OtaManager::IDLE: return 'I'; @@ -111,7 +126,7 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board if (is_cmd(a, "help|?|h", &rest)) { snprintf(reply, 160, "OTA: status | stats=admin ids/hashes | ls=find updates | get <#>=download | install | cancel | " - "announce | self | folder | config | key. Try `ota ls`."); + "announce | self | folder | cache | config | key. Use `ota ls [page]`."); // ---- inventory dashboard: running fw (self), the one fetch session, serving state ---- } else if (*a == 0 || is_cmd(a, "status|st", &rest)) { @@ -187,18 +202,29 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board } else if (is_cmd(a, "neighbors|nbrs|updates|ls|n", &rest)) { // Kick a fresh round of catalog queries (async - rows arrive over the next seconds); render what we // have now in plain words. The reply buffer is 160 B (serial / one LoRa packet for remote-admin), so - // writes are bounded and extra rows collapse to "+N more". + // the retained protocol catalog is rendered two rows at a time. + uint16_t page; + if (!parse_page(rest, page)) { strcpy(reply, "ERR usage: ota ls [page]"); return true; } c.manager.queryAll(); const int CAP = 160; - int n = snprintf(reply, CAP, "Updates nearby (%u src) - `ota get <#>` to download:", - (unsigned)c.manager.sourceCount()); + static const uint8_t PAGE_SIZE = 2; + uint16_t count = c.manager.catalogCount(); + uint16_t pages = count ? (uint16_t)((count + PAGE_SIZE - 1) / PAGE_SIZE) : 1; + if (page > pages) { + snprintf(reply, CAP, "ERR update page %u out of range (1-%u)", (unsigned)page, (unsigned)pages); + return true; + } + int n = snprintf(reply, CAP, "Updates %u/%u (%u src) - `ota get <#>`:", + (unsigned)page, (unsigned)pages, (unsigned)c.manager.sourceCount()); OtaManager::FetchState fs = c.manager.fetchState(); const uint8_t* cur = (fs != OtaManager::IDLE) ? c.manager.fetchManifestId() : nullptr; uint32_t myt = c.manager.target(); // effective target (EndF identity if present, else build flag) - uint32_t now = millis(); int shown = 0, more = 0; - for (uint8_t i = 0; i < c.manager.catalogCount(); i++) { + uint32_t now = millis(); int shown = 0; + uint16_t first = (uint16_t)(page - 1) * PAGE_SIZE; + uint16_t last = first + PAGE_SIZE; if (last > count) last = count; + for (uint16_t i = first; i < last; i++) { const OtaManager::CatRow* h = c.manager.catalogRow(i); - if (CAP - n < 48) { more++; continue; } + if (!h || CAP - n < 24) break; bool on = cur && memcmp(cur, h->mid, 4) == 0; // Tag the active session by real fetch state (not always "downloading" - COMPLETE is ready). const char* tag = ""; @@ -220,11 +246,10 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board else if (env) fit = env; else if (h->target_id == 0) fit = "?"; else { snprintf(hwbuf, sizeof hwbuf, "hw %08X", (unsigned)h->target_id); fit = hwbuf; } - n += snprintf(reply + n, CAP - n, "\n %d) %s %s [%s] %un %us%s", shown + 1, ver, + n += snprintf(reply + n, CAP - n, "\n %u) %s %s [%s] %un %us%s", (unsigned)(i + 1), ver, codec_kind(h->codec), fit, (unsigned)h->n_seeders, (unsigned)age, tag); shown++; } - if (more && n < CAP) snprintf(reply + n, CAP - n, "\n +%d more", more); if (shown == 0) strcpy(reply, "No updates seen yet - re-run `ota ls` in a few seconds (just asked around)."); // ---- start fetching a specific catalogued mOTA (by list index or manifest_id) ---- @@ -267,12 +292,18 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board uint8_t selmid[4]; uint32_t seltgt = sel->target_id; memcpy(selmid, sel->mid, 4); // sel may move on reset OtaStore* store; const char* dname; if (strncmp(dst, "flash", 5) == 0) { +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + c.stopSdCacheFetch(); // manual install download takes priority over archiving +#endif store = &c.fetch_store; c.fetch_store.clear(); dname = "flash"; validate = false; // seed lives in the folder #if defined(NRF52_PLATFORM) && !defined(OTA_SD_STORE) c.manager.set_accept_full(false); // nRF52 flash can install only in-place deltas #endif } else if (strncmp(dst, "folder", 6) == 0) { if (!c.folder_dest) { strcpy(reply, "ERR no folder connected (run motatool serve --tcp/--serial)"); return true; } +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + c.stopSdCacheFetch(); // an explicit host capture also takes priority +#endif c.folder_dest->set_mid(selmid); store = c.folder_dest; dname = validate ? "folder+validate" : "folder"; #if defined(NRF52_PLATFORM) c.manager.set_accept_full(true); // capture can store a full image; it is not applied @@ -289,6 +320,9 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board OtaManager::FetchState fs = c.manager.fetchState(); char midhx[9]; strcpy(midhx, "-"); if (fs != OtaManager::IDLE) mesh::Utils::toHex(midhx, c.manager.fetchManifestId(), 4); +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + c.stopSdCacheFetch(); +#endif c.manager.reset_session(); c.manager.want(0); c.manager.want_mid(nullptr); c.manager.set_fetch_store(&c.fetch_store); // revert to the default flash store (a folder pull switched it) #if defined(NRF52_PLATFORM) && !defined(OTA_SD_STORE) @@ -329,6 +363,12 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board // yes" round-trip - unreliable over LoRa): refuse unless the fetch is COMPLETE, then the apply path // validates in order (payload hash -> built-for-this-firmware -> signature/trust) and returns the // FIRST failing gate, so the operator knows exactly why it refused; it proceeds only if all pass. +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + if (c.sdCacheFetching()) { + strcpy(reply, "ERR SD archive capture is active; use `ota cancel` before installing"); + return true; + } +#endif if (c.manager.fetchState() != OtaManager::COMPLETE || c.fetch_store.staged_size() == 0) { sprintf(reply, "ERR no complete update fetched (fetch=%c %u/%u)", fstate_char(c.manager.fetchState()), (unsigned)c.manager.blocksHave(), @@ -367,9 +407,56 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board } } + // ---- SD OTA archive: default-on capture of every advertised mOTA, retained and served after reboot. ---- + } else if (is_cmd(a, "cache|archive|seed", &rest)) { +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + const char* p = rest; + if (strcmp(p, "on") == 0 || strcmp(p, "off") == 0) { + bool enabled = p[1] == 'n'; + if (!c.setSdCacheEnabled(enabled)) { + snprintf(reply, 160, "ERR SD OTA archive setting failed: %s", c.sd_cache.last_error()); + } else { + snprintf(reply, 160, "OK SD OTA archive capture %s (saved on card; cached files still seed)", + enabled ? "on" : "off"); + } + } else if (*p != 0) { + strcpy(reply, "ERR usage: ota cache [on|off]"); + } else if (!c.ensureSdCache()) { + snprintf(reply, 160, "SD OTA archive unavailable: %s", c.sd_cache.last_error()); + } else { + OtaManager::FetchState fs = c.manager.fetchState(); + if (c.sdCacheFetching()) { + char midhx[9]; mesh::Utils::toHex(midhx, c.manager.fetchManifestId(), 4); + unsigned have = (unsigned)c.manager.blocksHave(), total = (unsigned)c.manager.blocksTotal(); + snprintf(reply, 160, "SD OTA archive: capture=%s cached=%u, saving %s %u/%u", + c.sd_cache.autoCaptureEnabled() ? "on" : "off", + (unsigned)c.sd_cache.capturedCount(), midhx, have, total); + } else { + snprintf(reply, 160, "SD OTA archive: capture=%s cached=%u, %s; `ota cache off` stops new saves", + c.sd_cache.autoCaptureEnabled() ? "on" : "off", + (unsigned)c.sd_cache.capturedCount(), state_short(fs)); + } + } +#else + strcpy(reply, "ERR this target has no SD-backed OTA archive"); +#endif + // ---- policy config (persisted via NodePrefs). conservative defaults: autofetch/autoinstall off ---- } else if (is_cmd(a, "config|cfg|set", &rest)) { const char* p = rest; +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + if (strncmp(p, "cache ", 6) == 0 || strncmp(p, "sdseed ", 7) == 0) { + const char* v = p + (p[0] == 'c' ? 6 : 7); + bool enabled; + if (strcmp(v, "on") == 0) enabled = true; + else if (strcmp(v, "off") == 0) enabled = false; + else { strcpy(reply, "ERR usage: ota config cache "); return true; } + if (!c.setSdCacheEnabled(enabled)) + snprintf(reply, 160, "ERR SD OTA archive setting failed: %s", c.sd_cache.last_error()); + else + snprintf(reply, 160, "OK SD OTA archive capture %s (saved on card)", enabled ? "on" : "off"); + } else +#endif if (strncmp(p, "autofetch ", 10) == 0) { const char* v = p + 10; uint8_t pol = strncmp(v, "any", 3) == 0 ? OtaManager::AUTOFETCH_ANY @@ -400,11 +487,22 @@ bool handle_ota_command(const char* command, char* reply, mesh::MainBoard& board sprintf(reply, "OK OTA reach = %ld hop%s (saved)%s", h, h == 1 ? "" : "s", h == 0 ? " - direct only" : ""); } else { // show current policy uint8_t af = c.manager.autofetch(); +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + bool cache_ready = c.ensureSdCache(); + sprintf(reply, "ota config: cache=%s/%u autofetch=%s autoinstall=%s checkpoint=%u advert=%umin hops=%u keys=%u", + cache_ready ? (c.sd_cache.autoCaptureEnabled() ? "on" : "off") : "unavailable", + cache_ready ? (unsigned)c.sd_cache.capturedCount() : 0, + af == OtaManager::AUTOFETCH_ANY ? "any" : af == OtaManager::AUTOFETCH_SIGNED ? "signed" : "off", + c.autoinstall == OtaContext::AUTOINSTALL_TRUSTED ? "trusted" : "off", + (unsigned)c.manager.checkpoint_blocks(), (unsigned)c.manager.advert_mins(), + (unsigned)c.manager.max_hops(), (unsigned)c.allow.count()); +#else sprintf(reply, "ota config: autofetch=%s autoinstall=%s checkpoint=%u advert=%umin hops=%u keys=%u (persisted)", af == OtaManager::AUTOFETCH_ANY ? "any" : af == OtaManager::AUTOFETCH_SIGNED ? "signed" : "off", c.autoinstall == OtaContext::AUTOINSTALL_TRUSTED ? "trusted" : "off", (unsigned)c.manager.checkpoint_blocks(), (unsigned)c.manager.advert_mins(), (unsigned)c.manager.max_hops(), (unsigned)c.allow.count()); +#endif } // ---- trusted signer allowlist (security config; persisted): `ota key add|rm ` / `ota key` lists ---- @@ -534,6 +632,9 @@ static bool handle_dev(const char* d, char* reply, OtaContext& c) { } } else if (strncmp(d, "clear", 5) == 0) { +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + c.stopSdCacheFetch(); +#endif c.manager.clear_primary(); c.serve_expected = 0; c.serving = false; c.releaseServeBuffer(); c.fetch_store.clear(); c.manager.reset_session(); diff --git a/src/helpers/ota/OtaContext.h b/src/helpers/ota/OtaContext.h index 2062df8d3..965390d9d 100644 --- a/src/helpers/ota/OtaContext.h +++ b/src/helpers/ota/OtaContext.h @@ -12,6 +12,7 @@ #include "OtaBlInfo.h" // bootloader OTA-apply capability marker (nRF52); cached after first read #if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) #include "OtaStoreSdNrf52.h" + #include "OtaCacheSdNrf52.h" #elif defined(NRF52_PLATFORM) && defined(OTA_FLASH_STORE) #include "OtaStoreFlashNrf52.h" #elif defined(ESP32_PLATFORM) && defined(OTA_FLASH_STORE) @@ -59,6 +60,7 @@ struct OtaContext { OtaManager manager; #if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) OtaStoreSdNrf52 fetch_store; // MeshTower V2: persistent SD staging, full + delta + OtaCacheSdNrf52 sd_cache; // persistent capture + source for every OTA container heard #elif defined(NRF52_PLATFORM) && defined(OTA_FLASH_STORE) OtaStoreFlashNrf52 fetch_store; // persistent flash staging (survives reboot; large deltas) #elif defined(ESP32_PLATFORM) && defined(OTA_FLASH_STORE) @@ -129,6 +131,13 @@ struct OtaContext { // so the deferred-reboot path (mesh loop) takes over. Caller ensures the fetch is COMPLETE. Shared by // manual `ota applydelta` and the auto-install path. bool apply_fetched(char* msg) { +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + if (sdCacheFetching()) { + strncpy(msg, "refused: SD archive capture owns the OTA receive slot", 96); + msg[95] = 0; + return false; + } +#endif // hardware-compatibility gate (brick-safety) - refuse a .mota whose hw_id is for different hardware, // independent of signature; covers a manual cross-target `ota dev want` onto an incompatible board. { @@ -202,7 +211,18 @@ struct OtaContext { _folder_link == FOLDER_LINK_TCP ? "tcp" : "serial"); return false; } - manager.clear_sources(); + if (folder_active && _folder_source == source) { + manager.refresh_sources(); + snprintf(msg, cap, "OK folder refreshed (%s) - serving %u mOTA total", + label ? label : "external", (unsigned)manager.servedCount()); + return true; + } + if (folder_active && _folder_source) { + manager.remove_source(_folder_source); // replace this link without detaching the SD archive + folder_active = false; + _folder_link = FOLDER_LINK_NONE; + _folder_source = nullptr; + } if (!manager.add_source(source)) { strncpy(msg, "ERR no free source slot", cap); msg[cap - 1] = 0; @@ -210,6 +230,7 @@ struct OtaContext { } folder_active = true; _folder_link = link; + _folder_source = source; snprintf(msg, cap, "OK folder attached (%s) - serving %u mOTA total (own fw + folder)", label ? label : "external", (unsigned)manager.servedCount()); return true; @@ -227,11 +248,121 @@ struct OtaContext { } #endif void detach_folder() { - manager.clear_sources(); + if (_folder_source) manager.remove_source(_folder_source); folder_active = false; _folder_link = FOLDER_LINK_NONE; + _folder_source = nullptr; } +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + // Initialize and attach the persistent source lazily. This lets Mesh::begin finish quickly when no card + // is inserted, while archive interest is already enabled so early OTA advertisements still get queried. + bool ensureSdCache() { + sd_cache.attach(fetch_store); + if (!sd_cache.initialize()) return false; + if (!_sd_cache_source_attached) { + if (!manager.add_source(&sd_cache)) return false; + _sd_cache_source_attached = true; + } + manager.set_archive_interest(sd_cache.autoCaptureEnabled()); + return true; + } + + bool setSdCacheEnabled(bool enabled) { + if (!ensureSdCache()) return false; + bool saved = sd_cache.setAutoCaptureEnabled(enabled); + // The marker create/remove can succeed even if the card's final sync reports late. Keep the live + // manager consistent with the cache object's actual state while still reporting the persistence error. + bool active = sd_cache.autoCaptureEnabled(); + manager.set_archive_interest(active); + if (!active) stopSdCacheFetch(); + else if (saved) manager.queryAll(); + return saved; + } + + bool sdCacheFetching() const { return _sd_cache_fetching; } + + // Give install-oriented/manual work immediate ownership of OtaManager's single receive slot. The partial + // archive remains on SD and is resumed later instead of being discarded. + void stopSdCacheFetch() { + if (!_sd_cache_fetching) return; + sd_cache.checkpoint(); + manager.reset_session(); + manager.want(0); + manager.want_mid(nullptr); + manager.set_fetch_store(&fetch_store); + _sd_cache_fetching = false; + } + + // Close every archive handle before card-wide format/erase, then rebuild the source view afterwards. + void prepareSdCardReset() { + stopSdCacheFetch(); + if (_sd_cache_source_attached) manager.remove_source(&sd_cache); + _sd_cache_source_attached = false; + sd_cache.resetMedia(); + manager.set_fetch_store(&fetch_store); + manager.set_archive_interest(true); // a freshly formatted card defaults to capture on + _sd_cache_init_retry_at = 0; + } + + void finishSdCardReset(uint32_t now) { + sd_cache.resetMedia(); + _sd_cache_init_retry_at = now; + ensureSdCache(); + } + + void serviceSdCache(uint32_t now) { + if (!_sd_cache_source_attached) { + if (_sd_cache_init_retry_at && (int32_t)(now - _sd_cache_init_retry_at) < 0) return; + if (!ensureSdCache()) { + _sd_cache_init_retry_at = now + 60000UL; + return; + } + _sd_cache_init_retry_at = 0; + } + + if (_sd_cache_fetching) { + OtaManager::FetchState state = manager.fetchState(); + if (state == OtaManager::COMPLETE) { + _sd_cache_fetching = false; // cache store already finalized + published the file + manager.reset_session(); + manager.want(0); + manager.want_mid(nullptr); + manager.set_fetch_store(&fetch_store); + manager.refresh_sources(); + manager.announce(); // peers learn the expanded persistent served set immediately + _sd_cache_next_at = now + 3000UL; + } else if (state == OtaManager::FAILED || state == OtaManager::PAUSED) { + manager.deferCatalog(_sd_cache_mid, now + 300000UL); + stopSdCacheFetch(); // retain .part; a later pass resumes its verified blocks + _sd_cache_next_at = now + 3000UL; + } else if (state == OtaManager::IDLE) { + // A CLI/reset path may have reset the manager directly. Never leave the archive ownership flag + // wedged merely because the shared manager reached IDLE outside this service function. + stopSdCacheFetch(); + _sd_cache_next_at = now + 3000UL; + } + return; + } + + if (!sd_cache.autoCaptureEnabled() || apply_pending || manager.fetchState() != OtaManager::IDLE || + (_sd_cache_next_at && (int32_t)(now - _sd_cache_next_at) < 0)) return; + + for (uint8_t i = 0; i < manager.catalogCount(); i++) { + const OtaManager::CatRow* row = manager.catalogRow(i); + if (!row || row->have_max == 0 || sd_cache.contains(row->mid)) continue; + if (!manager.catalogReady(row, now)) continue; + + memcpy(_sd_cache_mid, row->mid, 4); + sd_cache.set_mid(row->mid); + manager.set_fetch_store(&sd_cache); + _sd_cache_fetching = true; + manager.pull_archive(row->mid, row->target_id); + return; + } + } +#endif + void track_session(uint8_t fstate, uint32_t now) { // stamp the session start (age display) if (fstate != prev_fstate) { if (prev_fstate == OtaManager::IDLE && fstate != OtaManager::IDLE) session_started_ms = now; @@ -263,10 +394,22 @@ struct OtaContext { manager.set_apply_codec2(CODEC_DETOOLS_INPLACE); // also accepted -> a single in-place .mota fits both #endif manager.set_fetch_store(&fetch_store); +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + sd_cache.attach(fetch_store); + manager.set_archive_interest(true); // default on; the SD marker can turn it off at first mount +#endif } private: FolderLink _folder_link = FOLDER_LINK_NONE; + MotaSource* _folder_source = nullptr; +#if defined(NRF52_PLATFORM) && defined(OTA_SD_STORE) + bool _sd_cache_source_attached = false; + bool _sd_cache_fetching = false; + uint8_t _sd_cache_mid[4] = {0}; + uint32_t _sd_cache_init_retry_at = 0; + uint32_t _sd_cache_next_at = 0; +#endif }; OtaContext& ota_ctx(); // process-wide singleton diff --git a/src/helpers/ota/OtaManager.cpp b/src/helpers/ota/OtaManager.cpp index ad10bcf4e..723418411 100644 --- a/src/helpers/ota/OtaManager.cpp +++ b/src/helpers/ota/OtaManager.cpp @@ -32,6 +32,7 @@ void OtaManager::begin(uint32_t my_target_id, OtaSend send, void* ctx) { _target = my_target_id; _send = send; _ctx = ctx; _fstate = IDLE; _have = 0; _fbc = 0; _n_serve = 0; _n_src_obj = 0; _view0.valid = false; _srcv.valid = false; + _n_src = 0; _n_cat = 0; } // ---------------- serve (multi-mota registry) ---------------- @@ -88,12 +89,28 @@ void OtaManager::registerSelfEntry() { } bool OtaManager::add_source(MotaSource* src) { - if (!src || _n_src_obj >= OTA_MAX_SOURCE_OBJ) return false; + if (!src) return false; + for (uint8_t i = 0; i < _n_src_obj; i++) { + if (_src_list[i] == src) { refresh_sources(); return true; } + } + if (_n_src_obj >= OTA_MAX_SOURCE_OBJ) return false; _src_list[_n_src_obj++] = src; refresh_sources(); return true; } +bool OtaManager::remove_source(MotaSource* src) { + if (!src) return false; + for (uint8_t i = 0; i < _n_src_obj; i++) { + if (_src_list[i] != src) continue; + for (uint8_t j = i + 1; j < _n_src_obj; j++) _src_list[j - 1] = _src_list[j]; + _src_list[--_n_src_obj] = nullptr; + refresh_sources(); + return true; + } + return false; +} + void OtaManager::refresh_sources() { _n_serve = 0; if (_view0.valid) registerSelfEntry(); @@ -104,6 +121,21 @@ void OtaManager::refresh_sources() { for (uint8_t i = 0; i < cnt && _n_serve < OTA_MAX_SERVE; i++) { MotaDesc d; if (!src->describe(i, d)) continue; + if (d.block_count == 0 || (uint64_t)d.block_count * 4 > OTA_PROOFGEN_SCRATCH) continue; + // A legacy serial descriptor left the geometry byte zero. Inspect its fixed manifest once so it is + // either proven servable before advertisement or excluded from the registry. + if (d.block_size_log2 == 0) { + uint8_t mf[MOTA_MFL]; + MotaManifest parsed; + if (!src->read(i, 8, mf, sizeof(mf)) || !mota_parse_manifest(mf, sizeof(mf), parsed) || + memcmp(parsed.merkle_root, d.mid, 4) != 0 || parsed.block_count != d.block_count || + parsed.payload_size != d.payload_size) continue; + d.block_size_log2 = parsed.block_size_log2; + } + if (d.block_size_log2 >= 32 || (1UL << d.block_size_log2) > OTA_MAX_BLOCK) continue; + if (d.leaves_off != 8 + MOTA_MFL || + (uint64_t)d.leaves_off + (uint64_t)d.block_count * 4 != d.payload_off || + (uint64_t)d.payload_off + d.payload_size + 5 != d.total_size) continue; if (serveEntryIndex(d.mid) >= 0) continue; // already offered (e.g. our own fw in the folder) ServeEntry& e = _serve[_n_serve++]; memcpy(e.mid, d.mid, 4); @@ -148,7 +180,8 @@ OtaManager::ServeView* OtaManager::resolve(const uint8_t* mid) { // Load an external mota into the on-demand _srcv: read its manifest-minus-leaves + leaves[] from the // source into RAM, parse, and wire a payload reader that streams blocks from the source on REQ. (The -// payload itself is NOT held in RAM - only the small head + the leaves, <=4 KB for <=1024 blocks.) +// payload itself is NOT held in RAM - only the small head and the leaves, bounded by the configured +// proof-generation scratch size.) bool OtaManager::loadSource(const ServeEntry& e) { const MotaDesc& d = e.desc; if (d.leaves_off < 8) return false; @@ -161,6 +194,7 @@ bool OtaManager::loadSource(const ServeEntry& e) { if (!src_leaves || !scratch) return false; bool ok = e.src->read(e.src_idx, 8, _src_manifest, mfl); if (!ok || !mota_parse_manifest(_src_manifest, mfl, _srcv.m)) return false; + if (_srcv.m.block_size() == 0 || _srcv.m.block_size() > OTA_MAX_BLOCK) return false; if (memcmp(_srcv.m.merkle_root, d.mid, 4) != 0) return false; // descriptor/bytes disagree if (_srcv.m.block_count != d.block_count) return false; ok = e.src->read(e.src_idx, d.leaves_off, src_leaves, d.block_count * 4); @@ -190,7 +224,7 @@ void OtaManager::setDigest(uint8_t out[4]) const { if (_n_serve == 0) { memset(out, 0, 4); return; } uint8_t order[OTA_MAX_SERVE]; for (uint8_t i = 0; i < _n_serve; i++) order[i] = i; - for (uint8_t i = 1; i < _n_serve; i++) { // insertion sort by mid (n <= 12) + for (uint8_t i = 1; i < _n_serve; i++) { // insertion sort by mid uint8_t v = order[i]; int j = (int)i - 1; while (j >= 0 && memcmp(_serve[order[j]].mid, _serve[v].mid, 4) > 0) { order[j+1] = order[j]; j--; } order[j+1] = v; @@ -216,31 +250,60 @@ void OtaManager::announce() { // tiny per-node beacon (constant size, inde void OtaManager::handleQuery(const uint8_t* m, uint16_t n) { QueryMsg q; if (!decode_query(m, n, q)) return; - if (_pq_active && memcmp(_pq_seeder, q.seeder_id, 4) == 0 && memcmp(_pq_digest, q.set_digest, 4) == 0) - _pq_active = false; // (1) suppress our own pending query + // Suppress only a pending query whose scope is covered by the overheard request. A target-filtered or + // fragment-only query cannot stand in for our unfiltered initial request. + for (uint8_t i = 0; i < _n_src; i++) { + Source& s = _sources[i]; + if (!s.query_pending || memcmp(s.seeder, q.seeder_id, 4) != 0 || + memcmp(s.digest, q.set_digest, 4) != 0 || q.filter_target != 0) continue; + uint32_t missing = 0; + if (s.have_total) { + uint32_t full = s.have_total >= 32 ? UINT32_MAX : ((1UL << s.have_total) - 1); + missing = full & ~s.have_mask; + } + if (q.want_fragments == 0 || (missing != 0 && (q.want_fragments & missing) == missing)) { + s.query_pending = false; + s.query_retry_at = _now_ms + OTA_CATALOG_RETRY_MS; + } + } if (_n_serve == 0 || memcmp(q.seeder_id, _seeder_id, 4) != 0) return; // (2) only WE answer queries to us uint8_t dg[4]; setDigest(dg); - uint8_t rowbuf[OTA_MAX_SERVE * OTA_HAVE_ROW_BYTES]; - uint8_t nm = 0; - for (uint8_t i = 0; i < _n_serve; i++) { - const ServeEntry& e = _serve[i]; - if (q.filter_target != 0 && q.filter_target != e.target_id) continue; - uint8_t* row = rowbuf + (uint32_t)nm * OTA_HAVE_ROW_BYTES; - memcpy(row, e.mid, 4); - wr_u32le(row + 4, e.target_id); wr_u32le(row + 8, e.fw_version); - row[12] = e.codec_id; row[13] = e.flags; - uint32_t hc = e.have_count > 0xFFFFu ? 0xFFFFu : e.have_count; // blocks we hold (awareness for fetchers) - row[14] = (uint8_t)(hc & 0xFF); row[15] = (uint8_t)(hc >> 8); - nm++; - } const uint8_t per = (uint8_t)((MAX_PACKET_PAYLOAD - 12) / OTA_HAVE_ROW_BYTES); // rows per HAVE fragment - uint8_t ftotal = (uint8_t)((nm + per - 1) / per); if (ftotal == 0) ftotal = 1; + uint8_t ftotal = (uint8_t)((_n_serve + per - 1) / per); if (ftotal == 0) ftotal = 1; + if (ftotal > OTA_HAVE_MAX_FRAGMENTS) return; // QueryMsg's recovery bitmap cannot represent it + + // Fragment positions are canonical across rescans: digest-stable sets sort into the same pages even if + // FAT/folder enumeration order changed. filter_target removes rows from those pages but never renumbers + // fragments, so filtered and unfiltered HAVE floods cannot corrupt each other's completion bitmap. + uint8_t order[OTA_MAX_SERVE]; + for (uint8_t i = 0; i < _n_serve; i++) order[i] = i; + for (uint8_t i = 1; i < _n_serve; i++) { + uint8_t v = order[i]; int j = (int)i - 1; + while (j >= 0 && memcmp(_serve[order[j]].mid, _serve[v].mid, 4) > 0) { + order[j + 1] = order[j]; j--; + } + order[j + 1] = v; + } for (uint8_t fi = 0; fi < ftotal; fi++) { - uint8_t bse = (uint8_t)(fi * per); - uint8_t cnt = (uint8_t)((nm - bse > per) ? per : (nm - bse)); + if (q.want_fragments != 0 && !(q.want_fragments & (1UL << fi))) continue; + const uint8_t first = (uint8_t)(fi * per); + uint8_t rowbuf[MAX_PACKET_PAYLOAD]; + uint8_t cnt = 0; + uint16_t end = (uint16_t)first + per; if (end > _n_serve) end = _n_serve; + for (uint16_t pos = first; pos < end; pos++) { + const ServeEntry& e = _serve[order[pos]]; + if (q.filter_target != 0 && q.filter_target != e.target_id) continue; + uint8_t* row = rowbuf + (uint32_t)cnt * OTA_HAVE_ROW_BYTES; + memcpy(row, e.mid, 4); + wr_u32le(row + 4, e.target_id); wr_u32le(row + 8, e.fw_version); + row[12] = e.codec_id; row[13] = e.flags; + uint32_t hc = e.have_count > 0xFFFFu ? 0xFFFFu : e.have_count; + row[14] = (uint8_t)(hc & 0xFF); row[15] = (uint8_t)(hc >> 8); + cnt++; + } HaveMsg hv; memcpy(hv.seeder_id, _seeder_id, 4); memcpy(hv.set_digest, dg, 4); hv.frag_idx = fi; hv.frag_total = ftotal; hv.n_rows = cnt; - hv.rows = cnt ? (rowbuf + (uint32_t)bse * OTA_HAVE_ROW_BYTES) : nullptr; + hv.rows = cnt ? rowbuf : nullptr; uint8_t b[MAX_PACKET_PAYLOAD]; emit(b, encode_have(b, sizeof(b), hv), true); // broadcast: all neighbours cache it } @@ -372,46 +435,104 @@ void OtaManager::handleAdv(const uint8_t* m, uint16_t n) { bool changed = fresh || memcmp(s.digest, a.set_digest, 4) != 0; memcpy(s.seeder, a.seeder_id, 4); memcpy(s.digest, a.set_digest, 4); s.n_motas = a.n_motas; s.last_ms = _now_ms; - if (changed) s.have_catalog = false; + if (changed) { + s.have_catalog = false; + s.have_total = 0; + s.have_mask = 0; + s.query_pending = false; + s.query_owned = false; + s.query_retries = 0; + s.query_retry_at = 0; + } // interested = auto-fetch enabled, or a manual pull/want is pending. (Browsing queries via queryAll().) - bool interested = (_autofetch != AUTOFETCH_OFF) || _have_desired_mid || _desired_target; - if (interested && !s.have_catalog) scheduleQuery(a.seeder_id, a.set_digest); // jittered + suppressible + bool interested = _archive_interest || (_autofetch != AUTOFETCH_OFF) || + _have_desired_mid || _desired_target; + if (interested && !s.have_catalog) { + // A fresh ADV is also a new opportunity after the bounded retry series was exhausted. + if (s.query_retries >= OTA_CATALOG_MAX_RETRY) { + s.query_retries = 0; + s.query_retry_at = 0; + s.query_owned = false; + } + scheduleQuery(a.seeder_id, a.set_digest); // jittered + suppressible + } } // Schedule a catalog query after a random jitter (id +/ digest, so neighbours pick different delays). The // node with the shortest jitter sends; the rest overhear that QUERY (or the broadcast HAVE) and suppress. void OtaManager::scheduleQuery(const uint8_t* seeder, const uint8_t* digest) { - if (_pq_active && memcmp(_pq_seeder, seeder, 4) == 0 && memcmp(_pq_digest, digest, 4) == 0) return; // already pending - memcpy(_pq_seeder, seeder, 4); memcpy(_pq_digest, digest, 4); - uint32_t j = (rd_u32le(seeder) ^ rd_u32le(digest) ^ rd_u32le(_seeder_id)) % OTA_QUERY_SPREAD_MS; - _pq_at = _now_ms + OTA_QUERY_MIN_MS + j; - _pq_active = true; + for (uint8_t i = 0; i < _n_src; i++) { + Source& s = _sources[i]; + if (memcmp(s.seeder, seeder, 4) != 0 || memcmp(s.digest, digest, 4) != 0) continue; + if (s.query_pending || s.have_catalog || + (s.query_owned && s.query_retry_at && (int32_t)(_now_ms - s.query_retry_at) < 0)) return; + uint32_t j = (rd_u32le(seeder) ^ rd_u32le(digest) ^ rd_u32le(_seeder_id)) % OTA_QUERY_SPREAD_MS; + s.query_at = _now_ms + OTA_QUERY_MIN_MS + j; + s.query_pending = true; + s.query_owned = true; + return; + } } -void OtaManager::sendQuery(const uint8_t* seeder, const uint8_t* digest, uint32_t filter_target) { - QueryMsg q; memcpy(q.seeder_id, seeder, 4); memcpy(q.set_digest, digest, 4); q.filter_target = filter_target; - uint8_t b[16]; +void OtaManager::sendQuery(const uint8_t* seeder, const uint8_t* digest, uint32_t filter_target, + uint32_t want_fragments) { + QueryMsg q; + memcpy(q.seeder_id, seeder, 4); memcpy(q.set_digest, digest, 4); + q.filter_target = filter_target; q.want_fragments = want_fragments; + uint8_t b[24]; emit(b, encode_query(b, sizeof(b), q), true); // FLOODED so neighbours overhear it and suppress } -// User-initiated browse (`ota neighbors`): ask every known source now (no jitter - infrequent + explicit). -void OtaManager::queryAll() { for (uint8_t i = 0; i < _n_src; i++) sendQuery(_sources[i].seeder, _sources[i].digest, 0); } +// User-initiated browse (`ota neighbors`): immediately ask every incomplete/changed source. A complete +// digest-tagged catalog is already current, so paging through a 255-row list must not re-flood it each time. +void OtaManager::queryAll() { + for (uint8_t i = 0; i < _n_src; i++) { + Source& s = _sources[i]; + if (s.have_catalog) continue; + s.query_pending = false; + s.query_owned = true; + s.query_retries = 0; + s.query_retry_at = _now_ms + OTA_CATALOG_RETRY_MS; + uint32_t want = 0; + if (s.have_total) { + uint32_t full = s.have_total >= 32 ? UINT32_MAX : ((1UL << s.have_total) - 1); + want = full & ~s.have_mask; + } + sendQuery(s.seeder, s.digest, 0, want); + } +} // A catalog reply: record each mOTA (deduped by mid; distinct-source count for the UI), and if a row // matches our fetch interest (auto-fetch own-target, or a pending pull/want), begin fetching it. void OtaManager::handleHave(const uint8_t* m, uint16_t n) { HaveMsg hv; if (!decode_have(m, n, hv)) return; + if (hv.frag_total == 0 || hv.frag_total > OTA_HAVE_MAX_FRAGMENTS || hv.frag_idx >= hv.frag_total) return; bool have_sid = (_seeder_id[0] | _seeder_id[1] | _seeder_id[2] | _seeder_id[3]) != 0; if (have_sid && memcmp(hv.seeder_id, _seeder_id, 4) == 0) return; // our own catalog - // PASSIVE: any overheard HAVE marks its source catalogued + cancels a pending query for it (storm - // suppression) - every node caches the rows below, even one that never queried. - for (uint8_t i = 0; i < _n_src; i++) - if (memcmp(_sources[i].seeder, hv.seeder_id, 4) == 0 && memcmp(_sources[i].digest, hv.set_digest, 4) == 0) - _sources[i].have_catalog = true; - if (_pq_active && memcmp(_pq_seeder, hv.seeder_id, 4) == 0 && memcmp(_pq_digest, hv.set_digest, 4) == 0) - _pq_active = false; + // PASSIVE: every node caches rows it overhears. A source is catalogued only after EVERY advertised + // fragment arrived; otherwise a timed recovery QUERY asks for just the missing bitmap. + for (uint8_t i = 0; i < _n_src; i++) { + Source& s = _sources[i]; + if (memcmp(s.seeder, hv.seeder_id, 4) != 0 || memcmp(s.digest, hv.set_digest, 4) != 0) continue; + if (s.have_total != hv.frag_total) { + s.have_total = hv.frag_total; + s.have_mask = 0; + s.have_catalog = false; + } + s.have_mask |= 1UL << hv.frag_idx; + uint32_t full = hv.frag_total >= 32 ? UINT32_MAX : ((1UL << hv.frag_total) - 1); + s.have_catalog = s.have_mask == full; + if (s.have_catalog) { + s.query_pending = false; + s.query_owned = false; + s.query_retries = 0; + s.query_retry_at = 0; + } else { + s.query_retry_at = _now_ms + OTA_CATALOG_RETRY_MS; + } + } for (uint8_t r = 0; r < hv.n_rows && hv.rows; r++) { const uint8_t* row = hv.rows + (uint32_t)r * OTA_HAVE_ROW_BYTES; const uint8_t* mid = row; @@ -443,10 +564,20 @@ void OtaManager::handleHave(const uint8_t* m, uint16_t n) { } } +void OtaManager::deferCatalog(const uint8_t mid[4], uint32_t until_ms) { + if (!mid) return; + for (uint8_t i = 0; i < _n_cat; i++) { + if (memcmp(_catalog[i].mid, mid, 4) == 0) { + _catalog[i].retry_after_ms = until_ms; + return; + } + } +} + bool OtaManager::wantRow(const uint8_t* mid, uint32_t target, uint8_t codec, uint8_t flags) const { if (!_fetch || _fstate == FETCHING || _fstate == WANT_MANIFEST || _fstate == PAUSED) return false; // busy if (_fstate == COMPLETE && memcmp(mid, _fid, 4) == 0) return false; // already have it - if (!codecOk(codec)) return false; // can't apply this codec + if (!_archive_fetch && !codecOk(codec)) return false; // can't apply this codec if (_have_desired_mid) // manual pull of a specific mid return memcmp(mid, _desired_mid, 4) == 0 && (_desired_target == 0 || target == _desired_target); if (_desired_target) return target == _desired_target; // cross-target want (role switch) @@ -509,7 +640,8 @@ void OtaManager::handleManifest(const uint8_t* m, uint16_t n) { const uint8_t* mf = _mf_buf; // fully reassembled manifest-minus-leaves uint32_t mfl = _mf_len; if (mfl != MOTA_MFL) { _fstate = FAILED; return; } // manifest-minus-leaves is a fixed 197 bytes - if (!codecOk(mf[56])) { _fstate = IDLE; return; } // codec we can't apply (lying/stale ADV) - abort + if (mf[2] != HASH_ALGO_SHA256) { _fstate = FAILED; return; } // this implementation supports sha2-256 only + if (!_archive_fetch && !codecOk(mf[56])) { _fstate = IDLE; return; } // incompatible install codec uint32_t payload_size = rd_u32le(mf + 15); uint8_t bsl = mf[19]; if (bsl >= 32) { _fstate = FAILED; return; } @@ -518,11 +650,16 @@ void OtaManager::handleManifest(const uint8_t* m, uint16_t n) { if (bs == 0 || bs > OTA_MAX_BLOCK || payload_size == 0) { _fstate = FAILED; return; } uint32_t bc = (payload_size + bs - 1) / bs; if (bc > 0xFFFFu) { _fstate = FAILED; return; } // block_idx is uint16 on the wire - can't address more + if (_archive_fetch && (uint64_t)bc * 4 > OTA_PROOFGEN_SCRATCH) { + _fstate = FAILED; return; // retaining an image we cannot subsequently seed is useless + } memcpy(_froot, mf + 20, 4); uint32_t leaves_off = 8 + mfl; uint32_t payload_off = leaves_off + bc * 4; - uint32_t total = payload_off + payload_size + 5; + uint64_t total64 = (uint64_t)payload_off + payload_size + 5; + if (total64 > UINT32_MAX) { _fstate = FAILED; return; } + uint32_t total = (uint32_t)total64; // Hand the store the parsed layout BEFORE begin(), so a partition-backed store (ESP32) can choose // placement and refuse an unfittable fetch up front: a FULL payload streams to the inactive slot, @@ -642,11 +779,13 @@ bool OtaManager::resumeStaged(const uint8_t* want_mid) { MotaManifest m; if (!_fetch->read(8, mbuf, mread) || !mota_parse_manifest(mbuf, mread, m)) return false; if (want_mid && memcmp(m.merkle_root, want_mid, 4) != 0) return false; // a different fw is staged - if (!codecOk(m.codec_id)) return false; + if (m.hash_algo != HASH_ALGO_SHA256) return false; + if (!_archive_fetch && !codecOk(m.codec_id)) return false; uint32_t mfl = (uint32_t)(m.approval - m.manifest_start) + 4; // manifest-minus-leaves length uint32_t bs = m.block_size(); if (bs == 0 || bs > OTA_MAX_BLOCK) return false; uint32_t bc = m.block_count; + if (_archive_fetch && (uint64_t)bc * 4 > OTA_PROOFGEN_SCRATCH) return false; uint32_t leaves_off = 8 + mfl; uint32_t payload_off = leaves_off + bc * 4; if ((uint64_t)payload_off + m.payload_size + 5 != total) return false; // geometry must match the header @@ -795,10 +934,30 @@ uint32_t OtaManager::pickMissingBlock() { } void OtaManager::loop() { - // fire a scheduled catalog query once its jitter has elapsed (unless overhearing already suppressed it) - if (_pq_active && (int32_t)(_now_ms - _pq_at) >= 0) { - _pq_active = false; - sendQuery(_pq_seeder, _pq_digest, 0); // unfiltered: one broadcast HAVE serves everyone + // Each source owns its own pending/retry state, so a burst of advertisements cannot overwrite another + // seeder's query. Initial requests ask for all; recovery asks only for missing HAVE fragments. + for (uint8_t i = 0; i < _n_src; i++) { + Source& s = _sources[i]; + if (s.query_pending && (int32_t)(_now_ms - s.query_at) >= 0) { + s.query_pending = false; + uint32_t want = 0; + if (s.have_total) { + uint32_t full = s.have_total >= 32 ? UINT32_MAX : ((1UL << s.have_total) - 1); + want = full & ~s.have_mask; + } + sendQuery(s.seeder, s.digest, 0, want); + s.query_retry_at = _now_ms + OTA_CATALOG_RETRY_MS; + } else if (s.query_owned && !s.have_catalog && !s.query_pending && s.query_retry_at && + (int32_t)(_now_ms - s.query_retry_at) >= 0 && s.query_retries < OTA_CATALOG_MAX_RETRY) { + uint32_t want = 0; + if (s.have_total) { + uint32_t full = s.have_total >= 32 ? UINT32_MAX : ((1UL << s.have_total) - 1); + want = full & ~s.have_mask; + } + s.query_retries++; + sendQuery(s.seeder, s.digest, 0, want); + s.query_retry_at = _now_ms + OTA_CATALOG_RETRY_MS; + } } if (_fstate == WANT_MANIFEST) { // Retry GET_MANIFEST ONLY when a tick passed with no new fragment - re-bursting every tick would congest diff --git a/src/helpers/ota/OtaManager.h b/src/helpers/ota/OtaManager.h index 8028af0fd..c26c2e915 100644 --- a/src/helpers/ota/OtaManager.h +++ b/src/helpers/ota/OtaManager.h @@ -28,7 +28,11 @@ typedef void (*OtaSend)(void* ctx, const uint8_t* msg, uint16_t len, bool flood) typedef bool (*ServeReadFn)(void* ctx, uint32_t off, uint8_t* buf, uint32_t len); #ifndef OTA_PROOFGEN_SCRATCH -#define OTA_PROOFGEN_SCRATCH 4096 // server proof-gen working buffer (supports up to 1024 blocks) + #if defined(OTA_SD_STORE) + #define OTA_PROOFGEN_SCRATCH 8192 // SD archive can seed <=2048 blocks (about 2 MB at 1 KB/block) + #else + #define OTA_PROOFGEN_SCRATCH 4096 // server proof-gen working buffer (supports up to 1024 blocks) + #endif #endif #ifndef OTA_MAX_BLOCK @@ -74,7 +78,11 @@ typedef bool (*ServeReadFn)(void* ctx, uint32_t off, uint8_t* buf, uint32_t len) #define OTA_MAX_SOURCES 12 // heard OTA sources (beacon senders) tracked (LRU); ~12 B each #endif #ifndef OTA_MAX_SERVE -#define OTA_MAX_SERVE 12 // mOTAs THIS node offers (own fw + external folder); == one HAVE fragment + #if defined(OTA_SD_STORE) + #define OTA_MAX_SERVE 255 // protocol maximum: own fw plus up to 254 persistent SD archive entries + #else + #define OTA_MAX_SERVE 12 // mOTAs THIS node offers (own fw + external folder); == one HAVE fragment + #endif #endif #ifndef OTA_MAX_SOURCE_OBJ #define OTA_MAX_SOURCE_OBJ 4 // external MotaSource objects (folders/transports) attached at once @@ -83,7 +91,7 @@ typedef bool (*ServeReadFn)(void* ctx, uint32_t off, uint8_t* buf, uint32_t len) #define OTA_SRC_MANIFEST_MAX 256 // manifest-minus-leaves buffer for the loaded source mota (head+sig+approval) #endif #ifndef OTA_MAX_CATALOG -#define OTA_MAX_CATALOG 12 // distinct mOTAs catalogued from OTA_HAVE replies (LRU) +#define OTA_MAX_CATALOG 255 // protocol maximum; large SD seeders must remain browsable remotely #endif #ifndef OTA_QUERY_MIN_MS #define OTA_QUERY_MIN_MS 300 // min delay before sending a catalog query (overhear-suppression window) @@ -91,6 +99,12 @@ typedef bool (*ServeReadFn)(void* ctx, uint32_t off, uint8_t* buf, uint32_t len) #ifndef OTA_QUERY_SPREAD_MS #define OTA_QUERY_SPREAD_MS 4000 // random jitter span so 50 neighbours don't all query at once (storm) #endif +#ifndef OTA_CATALOG_RETRY_MS +#define OTA_CATALOG_RETRY_MS 15000 // wait for a fragmented HAVE burst before requesting only its holes +#endif +#ifndef OTA_CATALOG_MAX_RETRY +#define OTA_CATALOG_MAX_RETRY 5 // bound retries until the source's next ADV or an explicit `ota ls` +#endif #ifndef OTA_REQ_SPREAD_MS #define OTA_REQ_SPREAD_MS 3000 // initial random hold before a fetch's first REQ (de-sync N fetchers) #endif @@ -164,6 +178,9 @@ public: // Re-enumerates the source into the serve registry. Returns false if no slots remain. (Trustless: the // fetcher verifies merkle+signature, so the source is never trusted - see OtaSource.h.) bool add_source(MotaSource* src); + // Detach one source without disturbing other attached sources (for example, keep the persistent SD + // archive attached while a temporary host-folder link comes and goes). + bool remove_source(MotaSource* src); // Re-read every attached source's catalog (call when the folder's contents change). Rebuilds entries // [1..] from the sources; entry 0 (our own fw) is preserved. void refresh_sources(); @@ -213,8 +230,15 @@ public: // build already staged in the destination, and pull DATA only for the differing blocks. Ignored unless a // seed is present (a plain fetch just re-transfers everything). Normal P2P pulls pass false. void pull(const uint8_t* mid, uint32_t target, bool validate = false) { + _archive_fetch = false; want(target); want_mid(mid); startFetch(mid, target, validate); } + // Capture an advertised container for relaying, not installation. Archive pulls accept every codec and + // target because the receiver only verifies and stores bytes; it never tries to apply the result locally. + void pull_archive(const uint8_t* mid, uint32_t target) { + _archive_fetch = true; + want(target); want_mid(mid); startFetch(mid, target, false); + } // Ask every known source for its catalog (populates `ota neighbors`). Async - rows arrive via OTA_HAVE. void queryAll(); // Coarse clock for source/catalog ages + LRU (the Mesh adapter feeds millis; 0 in host tests is fine). @@ -237,6 +261,11 @@ public: static const uint8_t AUTOFETCH_OFF = 0, AUTOFETCH_ANY = 1, AUTOFETCH_SIGNED = 2; void set_autofetch(uint8_t p) { _autofetch = p; reDiscover(); } uint8_t autofetch() const { return _autofetch; } + // A persistent archive needs full catalogs even when install-oriented autofetch is off. + void set_archive_interest(bool on) { + if (_archive_interest != on) { _archive_interest = on; reDiscover(); } + } + bool archive_interest() const { return _archive_interest; } // Resume checkpoint cadence (runtime-tunable, persisted in NodePrefs): persist progress every N // committed blocks. 0 = never (resume only from a finalized container). Default OTA_CHECKPOINT_BLOCKS. @@ -268,7 +297,8 @@ public: clearReassembly(); _loop_last_have = 0; _loop_last_mask = 0; _mf_total = 0; _mf_mask = 0; _mf_len = 0; _loop_last_mfmask = 0; - freeLeaves(); _validate = false; _lv_retries = 0; _loop_last_lvmask = 0; + freeLeaves(); _validate = false; _archive_fetch = false; + _lv_retries = 0; _loop_last_lvmask = 0; } FetchState fetchState() const { return _fstate; } @@ -286,10 +316,15 @@ public: uint8_t n_seeders; // count of the above (capped at OTA_CAT_SEEDERS) - "N+ nodes have it" uint32_t have_max; // best block-count any source reported (== total when a full copy exists) uint32_t last_ms; + uint32_t retry_after_ms; // per-image archive retry deadline (0 = eligible) }; uint8_t catalogCount() const { return _n_cat; } const CatRow* catalogRow(uint8_t i) const { return i < _n_cat ? &_catalog[i] : nullptr; } uint8_t sourceCount() const { return _n_src; } // distinct OTA sources (beacon senders) heard + void deferCatalog(const uint8_t mid[4], uint32_t until_ms); + bool catalogReady(const CatRow* row, uint32_t now_ms) const { + return row && (row->retry_after_ms == 0 || (int32_t)(now_ms - row->retry_after_ms) >= 0); + } private: void emit(const uint8_t* b, uint16_t n, bool flood) { if (_send && n) _send(_ctx, b, n, flood); } @@ -318,9 +353,20 @@ private: static bool srcReadTramp(void* c, uint32_t off, uint8_t* buf, uint32_t len); // source payload reader void emitBlockData(const uint8_t* mid, uint32_t idx, const uint8_t* data, uint32_t blen, uint16_t want_mask); // emit only the requested DATA fragments of one block - void sendQuery(const uint8_t* seeder, const uint8_t* digest, uint32_t filter_target); // ask a source for its catalog + void sendQuery(const uint8_t* seeder, const uint8_t* digest, uint32_t filter_target, + uint32_t want_fragments); // ask a source for all/selected catalog fragments void scheduleQuery(const uint8_t* seeder, const uint8_t* digest); // jittered + suppressible - void reDiscover() { for (uint8_t i = 0; i < _n_src; i++) _sources[i].have_catalog = false; _pq_active = false; } + void reDiscover() { + for (uint8_t i = 0; i < _n_src; i++) { + _sources[i].have_catalog = false; + _sources[i].have_total = 0; + _sources[i].have_mask = 0; + _sources[i].query_pending = false; + _sources[i].query_owned = false; + _sources[i].query_retries = 0; + _sources[i].query_retry_at = 0; + } + } void setDigest(uint8_t out[4]) const; // sha2-256:4 over our served mids bool blockPresent(uint32_t i) const; void requestMissing(); @@ -365,6 +411,8 @@ private: uint8_t _apply_codec = CODEC_DETOOLS_SEQUENTIAL; // platform's delta codec (OtaContext sets it) uint8_t _apply_codec2 = 0xFF; // optional 2nd accepted delta codec (ESP32: in-place) bool _accept_full = true; // false on nRF52 flash (single-slot cannot apply full) + bool _archive_fetch = false; // capture-only pull: accept any target/codec + bool _archive_interest = false; // query full catalogs for the persistent archive uint8_t _seeder_id[4] = {0,0,0,0}; // our node id (pubkey[0:4]) for advert seeder counting uint8_t _autofetch = AUTOFETCH_OFF; // auto-fetch policy (persisted in NodePrefs) uint16_t _checkpoint_blocks = OTA_CHECKPOINT_BLOCKS; // resume checkpoint cadence (persisted) @@ -400,17 +448,25 @@ private: uint8_t* ensureScratch(); // discovery: heard sources (beacon senders) + the catalog assembled from their OTA_HAVE replies - struct Source { uint8_t seeder[4]; uint8_t digest[4]; uint8_t n_motas; uint32_t last_ms; bool have_catalog; }; + struct Source { + uint8_t seeder[4]; + uint8_t digest[4]; + uint8_t n_motas; + uint32_t last_ms; + bool have_catalog; + uint8_t have_total; + uint32_t have_mask; + bool query_pending; + bool query_owned; + uint8_t query_retries; + uint32_t query_at; + uint32_t query_retry_at; + }; Source _sources[OTA_MAX_SOURCES]; uint8_t _n_src = 0; CatRow _catalog[OTA_MAX_CATALOG]; uint8_t _n_cat = 0; uint32_t _now_ms = 0; // coarse clock (fed by set_clock; for ages/LRU/jitter) - // pending catalog query (jittered + suppressed on overhearing a matching QUERY/HAVE - anti-storm) - bool _pq_active = false; - uint8_t _pq_seeder[4] = {0}; - uint8_t _pq_digest[4] = {0}; - uint32_t _pq_at = 0; // _now_ms deadline to actually send the query }; } // namespace ota diff --git a/src/helpers/ota/OtaProtocol.cpp b/src/helpers/ota/OtaProtocol.cpp index 28091cc67..6a5d7afc3 100644 --- a/src/helpers/ota/OtaProtocol.cpp +++ b/src/helpers/ota/OtaProtocol.cpp @@ -12,7 +12,11 @@ struct W { void u8(uint8_t v) { if (n + 1 > cap) { ok = false; return; } p[n++] = v; } void u16(uint16_t v){ u8(v & 0xFF); u8(v >> 8); } void u32(uint32_t v){ u8(v); u8(v >> 8); u8(v >> 16); u8(v >> 24); } - void raw(const uint8_t* d, uint16_t l) { if (n + l > cap) { ok = false; return; } memcpy(p + n, d, l); n += l; } + void raw(const uint8_t* d, uint16_t l) { + if (n + l > cap) { ok = false; return; } + if (l) memcpy(p + n, d, l); + n += l; + } }; struct R { const uint8_t* p; uint16_t len; uint16_t n; bool ok; @@ -38,7 +42,8 @@ bool decode_adv(const uint8_t* buf, uint16_t len, AdvMsg& m) { } uint16_t encode_query(uint8_t* buf, uint16_t cap, const QueryMsg& m) { - W w(buf, cap); w.u8(OTA_QUERY); w.raw(m.seeder_id, 4); w.raw(m.set_digest, 4); w.u32(m.filter_target); + W w(buf, cap); w.u8(OTA_QUERY); w.raw(m.seeder_id, 4); w.raw(m.set_digest, 4); + w.u32(m.filter_target); w.u32(m.want_fragments); return w.ok ? w.n : 0; } bool decode_query(const uint8_t* buf, uint16_t len, QueryMsg& m) { @@ -46,6 +51,13 @@ bool decode_query(const uint8_t* buf, uint16_t len, QueryMsg& m) { const uint8_t* sid = r.raw(4); if (sid) memcpy(m.seeder_id, sid, 4); const uint8_t* dg = r.raw(4); if (dg) memcpy(m.set_digest, dg, 4); m.filter_target = r.u32(); + // The fragment mask is an append-only protocol extension. Accept the original 13-byte QUERY as + // "all fragments" so new and old firmware remain interoperable during a rolling deployment. + m.want_fragments = 0; + if (r.remaining() != 0) { + if (r.remaining() < 4) return false; + m.want_fragments = r.u32(); + } return r.ok; } diff --git a/src/helpers/ota/OtaProtocol.h b/src/helpers/ota/OtaProtocol.h index 623ca222e..330c69dd2 100644 --- a/src/helpers/ota/OtaProtocol.h +++ b/src/helpers/ota/OtaProtocol.h @@ -24,11 +24,14 @@ struct AdvMsg { // ---- OTA_QUERY: "list what you serve" - addressed to a source by seeder_id, FLOODED so neighbours // overhear it (storm suppression). set_digest identifies the offering being asked about (so an overhearer -// can suppress its own pending query for the same {source,digest}). filter_target=0 = everything. ---- +// can suppress its own pending query for the same {source,digest}). filter_target=0 = everything. +// want_fragments=0 asks for every fragment; otherwise bit k asks only for HAVE fragment k. The latter +// lets a receiver recover catalog packet loss without causing another large all-fragment burst. ---- struct QueryMsg { uint8_t seeder_id[4]; // which source this query is for (the source matches its own id) uint8_t set_digest[4]; // the offering digest we're asking about (for overhear-suppression) uint32_t filter_target; // 0 = all (the scalable default); else only mOTAs for this target_id + uint32_t want_fragments; // 0 = all; otherwise bit k requests OTA_HAVE fragment k }; // ---- OTA_HAVE: the compact catalog (source -> mesh), FLOODED + tagged with set_digest so EVERY node @@ -44,6 +47,7 @@ struct HaveMsg { const uint8_t* rows; // points into buf: n_rows * OTA_HAVE_ROW_BYTES }; static const uint8_t OTA_HAVE_ROW_BYTES = 16; // mid4 + target4 + fwver4 + codec1 + flags1 + have_count2 +static const uint8_t OTA_HAVE_MAX_FRAGMENTS = 32; // QueryMsg::want_fragments bitmap width // ---- OTA_GET_MANIFEST: request the manifest for a content id (direct) ---- // body: manifest_id(4) want_mask(2). want_mask bit k = "send manifest fragment k"; 0xFFFF = "send all" diff --git a/src/helpers/ota/OtaSource.h b/src/helpers/ota/OtaSource.h index 88731a69f..27666d376 100644 --- a/src/helpers/ota/OtaSource.h +++ b/src/helpers/ota/OtaSource.h @@ -27,6 +27,7 @@ struct MotaDesc { uint32_t fw_version = 0; uint8_t codec_id = 0; uint8_t flags = 0; + uint8_t block_size_log2 = 0; // serve geometry; 0 means a legacy descriptor that must be inspected uint32_t total_size = 0; // full `.mota` length (bytes) uint32_t leaves_off = 0; // byte offset of the merkle leaves[] (manifest-minus-leaves = [8, leaves_off)) uint32_t block_count = 0; // == number of leaves (== number of payload blocks) diff --git a/src/helpers/ota/OtaStoreSdNrf52.cpp b/src/helpers/ota/OtaStoreSdNrf52.cpp index f2286460e..38cb9dbc8 100644 --- a/src/helpers/ota/OtaStoreSdNrf52.cpp +++ b/src/helpers/ota/OtaStoreSdNrf52.cpp @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include "MeshCore.h" #include "OtaByteIO.h" #include "OtaFlashLayout_nrf52.h" #include "OtaSdHandoff.h" @@ -36,6 +38,29 @@ void OtaStoreSdNrf52::fail(const char* message) { _error[sizeof(_error) - 1] = 0; } +void OtaStoreSdNrf52::resetStoreState() { + _total = 0; + _first_sector = 0; + _allocated_sectors = 0; + _partition_start = 0; + _partition_end = 0; + if (_file && *_file) _file->close(); + if (_sd) _sd->end(); + _mounted = false; +} + +bool OtaStoreSdNrf52::beginCardOnly() { + _error[0] = 0; + resetStoreState(); + if (!_sd || !_file || + !_sd->cardBegin(SdSpiConfig(OTA_SD_CS_PIN, DEDICATED_SPI, + SD_SCK_MHZ(OTA_SD_SCK_MHZ), &SPI1))) { + fail("SD card init failed"); + return false; + } + return true; +} + bool OtaStoreSdNrf52::mount() { if (_mounted) return true; _error[0] = 0; @@ -53,6 +78,235 @@ bool OtaStoreSdNrf52::mount() { return true; } +bool OtaStoreSdNrf52::formatCard(MainBoard& board) { + if (!beginCardOnly()) return false; + + board.serviceWatchdog(); + if (!_sd->format()) { + fail("SD format failed"); + _sd->end(); + return false; + } + board.serviceWatchdog(); + if (!_sd->volumeBegin()) { + fail("SD remount after format failed"); + _sd->end(); + return false; + } + + _mounted = true; + if (!inspect_mbr()) { + _sd->end(); + _mounted = false; + return false; + } + if (!invalidate_handoff()) { + fail("SD handoff clear failed"); + _sd->end(); + _mounted = false; + return false; + } + board.serviceWatchdog(); + return true; +} + +bool OtaStoreSdNrf52::eraseCard(MainBoard& board) { + if (!beginCardOnly()) return false; + + SdCard* card = _sd->card(); + const uint32_t sector_count = card ? card->sectorCount() : 0; + if (!card || sector_count == 0) { + fail("SD sector count failed"); + _sd->end(); + return false; + } + + // Match SdFat's formatter example: bounded erase ranges avoid cards that + // reject or time out on a single whole-device erase command. Servicing the + // board watchdog between ranges keeps the intentionally long operation from + // looking like a wedged main loop. + const uint32_t erase_sectors = 262144UL; + uint32_t first_sector = 0; + while (first_sector < sector_count) { + const uint32_t remaining = sector_count - first_sector; + const uint32_t count = remaining < erase_sectors ? remaining : erase_sectors; + const uint32_t last_sector = first_sector + count - 1; + board.serviceWatchdog(); + if (!card->erase(first_sector, last_sector)) { + fail("SD erase failed"); + _sd->end(); + return false; + } + first_sector += count; + } + + board.serviceWatchdog(); + if (!card->syncDevice()) { + fail("SD erase sync failed"); + _sd->end(); + return false; + } + _sd->end(); + return true; +} + +bool OtaStoreSdNrf52::getSpace(MainBoard& board, uint64_t& used_bytes, + uint64_t& free_bytes) { + used_bytes = 0; + free_bytes = 0; + _error[0] = 0; + if (!mount()) return false; + + board.serviceWatchdog(); + const uint32_t cluster_count = _sd->clusterCount(); + const uint32_t free_clusters = _sd->freeClusterCount(); + const uint32_t bytes_per_cluster = _sd->bytesPerCluster(); + board.serviceWatchdog(); + if (cluster_count == 0 || bytes_per_cluster == 0 || + free_clusters > cluster_count) { + fail("SD space query failed"); + return false; + } + + free_bytes = (uint64_t)free_clusters * bytes_per_cluster; + used_bytes = (uint64_t)(cluster_count - free_clusters) * bytes_per_cluster; + return true; +} + +namespace { + +static const uint16_t SD_LIST_PAGE_SIZE = 2; +static const size_t SD_LIST_NAME_CAP = 256; // FAT/exFAT long-name maximum plus NUL +static const size_t SD_LIST_SHOWN_PATH = 50; // bounded CLI line; long paths get an explicit leading "..." + +struct SdListState { + SdListState(MainBoard& b, uint32_t f) : board(b), first(f) {} + MainBoard& board; + uint32_t first; + uint32_t total = 0; + char body[132] = {0}; + size_t used = 0; + char* path = nullptr; + size_t path_len = 0; + size_t path_cap = 0; + bool failed = false; +}; + +static void compact_file_size(char* out, size_t cap, uint64_t bytes) { + static const char units[] = {'B', 'K', 'M', 'G', 'T'}; + uint8_t unit = 0; + uint64_t whole = bytes; + while (whole >= 1024 && unit < sizeof(units) - 1) { whole /= 1024; unit++; } + snprintf(out, cap, "%lu%c", (unsigned long)whole, units[unit]); +} + +static bool ensure_list_path(SdListState& state, size_t need) { + if (need <= state.path_cap) return true; + size_t cap = state.path_cap ? state.path_cap : 256; + while (cap < need) { + if (cap > SIZE_MAX / 2) { state.failed = true; return false; } + cap *= 2; + } + char* grown = static_cast(realloc(state.path, cap)); + if (!grown) { state.failed = true; return false; } + state.path = grown; + state.path_cap = cap; + return true; +} + +static bool append_sd_list_entry(SdListState& state, const char* path, uint64_t bytes) { + char size[12]; + compact_file_size(size, sizeof(size), bytes); + const size_t path_len = strlen(path); + const char* shown = path; + char shortened[SD_LIST_SHOWN_PATH + 1]; + if (path_len > SD_LIST_SHOWN_PATH) { + memcpy(shortened, "...", 3); + memcpy(shortened + 3, path + path_len - (SD_LIST_SHOWN_PATH - 3), SD_LIST_SHOWN_PATH - 3); + shortened[SD_LIST_SHOWN_PATH] = 0; + shown = shortened; + } + size_t remain = sizeof(state.body) - state.used; + int n = snprintf(state.body + state.used, remain, "\n %s %s", shown, size); + if (n <= 0 || (size_t)n >= remain) { state.failed = true; return false; } + state.used += (size_t)n; + return true; +} + +static bool scan_sd_files(FsFile& dir, SdListState& state) { + FsFile file; + while (file.openNext(&dir, O_RDONLY)) { + state.board.serviceWatchdog(); + const size_t parent_len = state.path_len; + if (parent_len > SIZE_MAX - SD_LIST_NAME_CAP - 2 || + !ensure_list_path(state, parent_len + SD_LIST_NAME_CAP + 2)) { + file.close(); + return false; + } + size_t name_at = parent_len; + if (name_at == 0 || state.path[name_at - 1] != '/') state.path[name_at++] = '/'; + state.path[name_at] = 0; + file.getName(state.path + name_at, SD_LIST_NAME_CAP); + size_t name_len = strlen(state.path + name_at); + if (name_len == 0) { state.failed = true; file.close(); return false; } + state.path_len = name_at + name_len; + + if (file.isDir()) { + if (!scan_sd_files(file, state)) { + state.path_len = parent_len; + state.path[parent_len] = 0; + file.close(); + return false; + } + } else { + uint32_t index = state.total++; + if (index >= state.first && index < state.first + SD_LIST_PAGE_SIZE) { + if (!append_sd_list_entry(state, state.path, file.fileSize())) { + state.path_len = parent_len; + state.path[parent_len] = 0; + file.close(); + return false; + } + } + } + state.path_len = parent_len; + state.path[parent_len] = 0; + file.close(); + } + return true; +} + +} // namespace + +bool OtaStoreSdNrf52::listFiles(MainBoard& board, uint16_t page, + char* reply, size_t cap) { + if (!reply || cap == 0) return false; + if (page == 0) page = 1; + _error[0] = 0; + if (!mount()) return false; + + FsFile root = _sd->open("/", O_RDONLY); + if (!root || !root.isDir()) { fail("SD root directory open failed"); return false; } + SdListState state(board, (uint32_t)(page - 1) * SD_LIST_PAGE_SIZE); + bool scanned = ensure_list_path(state, 2) && scan_sd_files(root, state); + root.close(); + free(state.path); + if (!scanned || state.failed) { fail("SD file listing failed"); return false; } + + uint32_t pages = (state.total + SD_LIST_PAGE_SIZE - 1) / SD_LIST_PAGE_SIZE; + if (pages == 0) { + snprintf(reply, cap, "> SD files: empty"); + } else if (page > pages) { + snprintf(reply, cap, "Error: SD file page %u out of range (1-%lu)", + (unsigned)page, (unsigned long)pages); + } else { + snprintf(reply, cap, "> SD files %u/%lu (%lu):%s", + (unsigned)page, (unsigned long)pages, + (unsigned long)state.total, state.body); + } + return true; +} + bool OtaStoreSdNrf52::inspect_mbr() { uint8_t sector[MOTA_SD_SECTOR_SIZE]; if (!_sd->card() || !_sd->card()->readSector(0, sector)) { diff --git a/src/helpers/ota/OtaStoreSdNrf52.h b/src/helpers/ota/OtaStoreSdNrf52.h index 46312b99e..71cf4ff48 100644 --- a/src/helpers/ota/OtaStoreSdNrf52.h +++ b/src/helpers/ota/OtaStoreSdNrf52.h @@ -8,8 +8,13 @@ class SdFs; class FsFile; namespace mesh { + +class MainBoard; + namespace ota { +class OtaCacheSdNrf52; + // Persistent nRF52840 OTA store backed by the MeshTower V2 microSD socket. // The .mota remains a normal file, but is preallocated contiguously so the // bootloader can read it by raw sectors without embedding a FAT implementation. @@ -33,15 +38,22 @@ public: // Called only after the app has verified payload, base, signature and trust. // Writes APRV into the file, then publishes the raw-sector handoff record. bool approve_for_bootloader(); + bool formatCard(MainBoard& board); + bool eraseCard(MainBoard& board); + bool getSpace(MainBoard& board, uint64_t& used_bytes, uint64_t& free_bytes); + bool listFiles(MainBoard& board, uint16_t page, char* reply, size_t cap); const char* last_error() const { return _error; } private: + friend class OtaCacheSdNrf52; static const char* const PATH; bool mount(); + bool beginCardOnly(); bool inspect_mbr(); bool locate_file(); bool invalidate_handoff(); + void resetStoreState(); void fail(const char* message); SdFs* _sd = nullptr; diff --git a/test/test_ota/test_ota_core.cpp b/test/test_ota/test_ota_core.cpp index 0358d0ce2..0121e4aa6 100644 --- a/test/test_ota/test_ota_core.cpp +++ b/test/test_ota/test_ota_core.cpp @@ -385,13 +385,18 @@ TEST(OtaProtocol, CodecRoundTrips) { EXPECT_EQ(0, memcmp(a2.set_digest, adv.set_digest, 4)); // OTA_QUERY: ask a source (by seeder_id) for the offering set_digest, optionally filtered to a target - QueryMsg qy{{0x29,0x17,0xe4,0xf7}, {0xd1,0xd2,0xd3,0xd4}, 0x11223344}; + QueryMsg qy{{0x29,0x17,0xe4,0xf7}, {0xd1,0xd2,0xd3,0xd4}, 0x11223344, 0xA5A50002}; n = encode_query(buf, sizeof(buf), qy); ASSERT_GT(n, 0); EXPECT_EQ(ota_msg_type(buf, n), OTA_QUERY); QueryMsg q2; ASSERT_TRUE(decode_query(buf, n, q2)); EXPECT_EQ(0, memcmp(q2.seeder_id, qy.seeder_id, 4)); EXPECT_EQ(0, memcmp(q2.set_digest, qy.set_digest, 4)); EXPECT_EQ(q2.filter_target, 0x11223344u); + EXPECT_EQ(q2.want_fragments, 0xA5A50002u); + QueryMsg qlegacy; + ASSERT_TRUE(decode_query(buf, n - 4, qlegacy)); // original QUERY ended after filter_target + EXPECT_EQ(qlegacy.filter_target, 0x11223344u); + EXPECT_EQ(qlegacy.want_fragments, 0u); // old sender means "all fragments" // OTA_HAVE: a 2-row catalog (mid, target, fwver, codec, flags per row) tagged with the offering digest uint8_t rows[2 * OTA_HAVE_ROW_BYTES]; @@ -480,6 +485,10 @@ struct SendTo { OtaManager* dest; }; static void sim_send(void* ctx, const uint8_t* msg, uint16_t len, bool /*flood*/) { g_q.push_back({((SendTo*)ctx)->dest, std::vector(msg, msg + len)}); } +struct CapturedMessages { std::vector> items; }; +static void capture_send(void* ctx, const uint8_t* msg, uint16_t len, bool /*flood*/) { + ((CapturedMessages*)ctx)->items.emplace_back(msg, msg + len); +} // Drive the bus to quiescence: deliver queued messages; when idle, advance the client's clock (monotonic // across calls, so a jittered query scheduled in a prior pump still comes due) and call loop() (fires the // scheduled catalog query / block re-requests). Two idle ticks in a row = quiescent. @@ -513,6 +522,7 @@ public: std::memcpy(d.mid, m.merkle_root, 4); d.target_id = m.target_id; d.fw_version = m.fw_version; d.codec_id = m.codec_id; d.flags = m.flags; + d.block_size_log2 = m.block_size_log2; d.total_size = _len[idx]; d.leaves_off = (uint32_t)(m.leaves - _buf[idx]); d.block_count = m.block_count; @@ -528,6 +538,27 @@ public: private: const uint8_t* _buf[8] = {nullptr}; uint32_t _len[8] = {0}; uint8_t _n = 0; }; + +class SyntheticCatalogSource : public mesh::ota::MotaSource { +public: + explicit SyntheticCatalogSource(uint8_t count, uint8_t block_log2 = 10) + : _count(count), _block_log2(block_log2) {} + uint8_t count() override { return _count; } + bool describe(uint8_t idx, mesh::ota::MotaDesc& d) override { + if (idx >= _count) return false; + d = mesh::ota::MotaDesc{}; + d.mid[0] = (uint8_t)(idx + 1); d.mid[1] = 0xA5; d.mid[2] = 0x5A; d.mid[3] = 0xC3; + d.target_id = SIM_TARGET_ID; d.fw_version = 0x01000000u + idx; + d.codec_id = CODEC_FULL; d.flags = MFLAG_FULL; d.block_size_log2 = _block_log2; + d.leaves_off = 8 + MOTA_MFL; d.block_count = 1; + d.payload_off = d.leaves_off + 4; d.payload_size = 1; d.total_size = d.payload_off + 1 + 5; + return true; + } + bool read(uint8_t, uint32_t, uint8_t*, uint32_t) override { return false; } +private: + uint8_t _count; + uint8_t _block_log2; +}; } TEST(OtaServe, ClearPrimaryInvalidatesCallerOwnedView) { @@ -550,6 +581,12 @@ TEST(OtaServe, ClearPrimaryInvalidatesCallerOwnedView) { ASSERT_EQ(manager.servedCount(), 2); EXPECT_TRUE(manager.servedEntry(0)->is_self); EXPECT_FALSE(manager.servedEntry(1)->is_self); + + // Detaching one source must preserve the primary view (and, on device, any other source such as SD). + EXPECT_TRUE(manager.remove_source(&folder)); + ASSERT_EQ(manager.servedCount(), 1); + EXPECT_TRUE(manager.servedEntry(0)->is_self); + EXPECT_FALSE(manager.remove_source(&folder)); } TEST(OtaTransfer, TwoManagersFullTransfer) { @@ -642,6 +679,28 @@ TEST(OtaTransfer, RejectsManifestBlockExponentBeforeShift) { EXPECT_EQ(client.fetchState(), OtaManager::FAILED); } +TEST(OtaTransfer, RejectsUnsupportedManifestHashAlgorithm) { + g_q.clear(); + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA, SIM_MOTA_LEN, manifest)); + std::array bytes; + memcpy(bytes.data(), manifest.manifest_start, bytes.size()); + bytes[2] = HASH_ALGO_SHA256 + 1; + + OtaManager client; + OtaStoreRam<4096> store; + SendTo to_none{&client}; + client.begin(SIM_TARGET_ID, sim_send, &to_none); + client.set_fetch_store(&store); + client.pull(manifest.merkle_root, manifest.target_id); + g_q.clear(); + + deliver_manifest_fragment(client, manifest.merkle_root, 0, bytes.data(), OTA_MF_FRAG); + deliver_manifest_fragment(client, manifest.merkle_root, 1, bytes.data() + OTA_MF_FRAG, + (uint16_t)(MOTA_MFL - OTA_MF_FRAG)); + EXPECT_EQ(client.fetchState(), OtaManager::FAILED); +} + // Same end-to-end transfer, but with 1 KB logical blocks: each block is delivered as several // self-describing DATA fragments (frag_off), reassembled by the client, then its merkle PROOF is // requested + verified separately before the block is committed. Exercises the multi-fragment path. @@ -722,6 +781,120 @@ TEST(OtaFolder, ServesSelfPlusFolderAndFetchesExternal) { EXPECT_TRUE(mota_check_image_hash_full(got)); } +TEST(OtaCatalog, QueryCanRequestOnlyMissingHaveFragment) { + OtaManager server; + CapturedMessages sent; + uint8_t sid[4] = {0x10, 0x20, 0x30, 0x40}; + server.begin(0, capture_send, &sent); + server.set_seeder_id(sid); + SyntheticCatalogSource source(12); // 10 rows/fragment => two HAVE fragments + ASSERT_TRUE(server.add_source(&source)); + ASSERT_EQ(server.servedCount(), 12); + + QueryMsg query{}; + memcpy(query.seeder_id, sid, 4); + query.want_fragments = 1UL << 1; // recover only fragment 1 + uint8_t wire[32]; + uint16_t n = encode_query(wire, sizeof(wire), query); + ASSERT_GT(n, 0); + server.on_message(wire, n); + + ASSERT_EQ(sent.items.size(), 1u); + HaveMsg have; + ASSERT_TRUE(decode_have(sent.items[0].data(), (uint16_t)sent.items[0].size(), have)); + EXPECT_EQ(have.frag_idx, 1); + EXPECT_EQ(have.frag_total, 2); + EXPECT_EQ(have.n_rows, 2); +} + +TEST(OtaCatalog, IncompleteCatalogRetriesOnlyMissingFragment) { + OtaManager client; + CapturedMessages sent; + client.begin(SIM_TARGET_ID, capture_send, &sent); + client.set_archive_interest(true); + uint8_t sid[4] = {0x44, 0x33, 0x22, 0x11}; + uint8_t digest[4] = {0xDE, 0xAD, 0xBE, 0xEF}; + AdvMsg adv{}; memcpy(adv.seeder_id, sid, 4); memcpy(adv.set_digest, digest, 4); adv.n_motas = 3; + uint8_t wire[MAX_PACKET_PAYLOAD]; + uint16_t n = encode_adv(wire, sizeof(wire), adv); + client.set_clock(100); + client.on_message(wire, n); + client.set_clock(5000); // past every possible discovery jitter + client.loop(); + ASSERT_EQ(sent.items.size(), 1u); + QueryMsg first; + ASSERT_TRUE(decode_query(sent.items[0].data(), (uint16_t)sent.items[0].size(), first)); + EXPECT_EQ(first.want_fragments, 0u); + sent.items.clear(); + + auto deliver_have = [&](uint8_t frag) { + uint8_t row[OTA_HAVE_ROW_BYTES] = {0}; + row[0] = (uint8_t)(frag + 1); row[1] = 0x77; + wr_u32le(row + 4, SIM_TARGET_ID); wr_u32le(row + 8, 0x01000000u + frag); + row[12] = CODEC_FULL; row[13] = MFLAG_FULL; row[14] = 1; + HaveMsg have{}; + memcpy(have.seeder_id, sid, 4); memcpy(have.set_digest, digest, 4); + have.frag_idx = frag; have.frag_total = 3; have.n_rows = 1; have.rows = row; + uint16_t have_len = encode_have(wire, sizeof(wire), have); + ASSERT_GT(have_len, 0); + client.on_message(wire, have_len); + }; + + deliver_have(0); + deliver_have(2); // fragment 1 is lost + client.set_clock(5000 + OTA_CATALOG_RETRY_MS + 1); + client.loop(); + ASSERT_EQ(sent.items.size(), 1u); + QueryMsg retry; + ASSERT_TRUE(decode_query(sent.items[0].data(), (uint16_t)sent.items[0].size(), retry)); + EXPECT_EQ(retry.want_fragments, 1UL << 1); + sent.items.clear(); + + deliver_have(1); + EXPECT_EQ(client.catalogCount(), 3); + client.set_clock(5000 + OTA_CATALOG_RETRY_MS * 2 + 2); + client.loop(); + EXPECT_TRUE(sent.items.empty()); // complete means no further catalog retries +} + +TEST(OtaCatalog, RetainsProtocolMaximumRows) { + OtaManager client; + client.begin(SIM_TARGET_ID, nullptr, nullptr); + const uint8_t per = (uint8_t)((MAX_PACKET_PAYLOAD - 12) / OTA_HAVE_ROW_BYTES); + const uint8_t total = (uint8_t)((255 + per - 1) / per); + uint8_t sid[4] = {1, 2, 3, 4}, digest[4] = {5, 6, 7, 8}; + uint16_t index = 0; + for (uint8_t frag = 0; frag < total; frag++) { + uint8_t rows[MAX_PACKET_PAYLOAD] = {0}; + uint8_t count = 0; + while (count < per && index < 255) { + uint8_t* row = rows + (uint16_t)count * OTA_HAVE_ROW_BYTES; + row[0] = (uint8_t)index; row[1] = 0xC1; row[2] = 0xD2; row[3] = 0xE3; + wr_u32le(row + 4, SIM_TARGET_ID); wr_u32le(row + 8, 0x01000000u + index); + row[12] = CODEC_FULL; row[13] = MFLAG_FULL; row[14] = 1; + count++; index++; + } + HaveMsg have{}; + memcpy(have.seeder_id, sid, 4); memcpy(have.set_digest, digest, 4); + have.frag_idx = frag; have.frag_total = total; have.n_rows = count; have.rows = rows; + uint8_t wire[MAX_PACKET_PAYLOAD]; + uint16_t n = encode_have(wire, sizeof(wire), have); + ASSERT_GT(n, 0); + client.on_message(wire, n); + } + ASSERT_EQ(client.catalogCount(), 255); + ASSERT_NE(client.catalogRow(254), nullptr); + EXPECT_EQ(client.catalogRow(254)->mid[0], 254); +} + +TEST(OtaCatalog, RejectsAdvertisedSourceWithOversizedBlocks) { + OtaManager server; + server.begin(0, nullptr, nullptr); + SyntheticCatalogSource source(1, 11); // 2048-byte blocks exceed OTA_MAX_BLOCK + ASSERT_TRUE(server.add_source(&source)); + EXPECT_EQ(server.servedCount(), 0); +} + // Fetch-resume across a reboot: a client commits some blocks, "reboots" (a fresh OtaManager on the SAME // persisted store), and resumeStaged() re-adopts the partial container and finishes the remaining blocks - // without re-fetching the manifest or the blocks already present. @@ -859,6 +1032,30 @@ TEST(OtaTransfer, RejectsIncompatibleCodec) { g_q.clear(); } +// An archive capture is not an install. It must retain cross-target and otherwise unsupported containers +// byte-for-byte so this node can relay them to hardware that does understand their codec. +TEST(OtaTransfer, ArchivePullAcceptsCrossTargetUnsupportedCodec) { + g_q.clear(); + OtaManager server, client; + OtaStoreRam<4096> store; + SendTo to_client{&client}, to_server{&server}; + server.begin(0, sim_send, &to_client); + client.begin(SIM_TARGET_ID ^ 0x55AAu, sim_send, &to_server); + client.set_fetch_store(&store); + client.set_apply_codec(CODEC_DETOOLS_INPLACE); + client.set_accept_full(false); // install path cannot accept SIM_MOTA (full image) + ASSERT_TRUE(server.serve(SIM_MOTA, SIM_MOTA_LEN)); + + MotaManifest m; + ASSERT_TRUE(mota_parse(SIM_MOTA, SIM_MOTA_LEN, m)); + client.pull_archive(m.merkle_root, m.target_id); // capture ignores local target + install codec + pump(client); + + EXPECT_EQ(client.fetchState(), OtaManager::COMPLETE); + ASSERT_EQ(store.staged_size(), SIM_MOTA_LEN); + EXPECT_EQ(0, std::memcmp(store.data(), SIM_MOTA, SIM_MOTA_LEN)); +} + // Encode a 1-row OTA_HAVE from a specific seeder, carrying have_count (Phase-2 awareness). static uint16_t make_have_row(uint8_t* buf, uint16_t cap, const uint8_t mid[4], uint32_t target, uint32_t fwver, uint8_t codec, uint8_t flags,