Fix OTA recovery, radio2 replies, and preference persistence

Keep each deferred OTA response tied to the request's radio profile and generation. Separate requests from different radio sessions, discard stale queued responses, and retain the existing companion and infrastructure reply policies.

Reject staged captures with invalid file framing before adopting their state. Recover fixed internal-flash trailer bytes after interrupted checkpoints and handle small ESP32 delta trailers that share metadata sectors. Fail closed when install preflight cannot read a valid header, so hardware and automatic-update policy checks cannot be skipped.

Replace Companion preferences through verified file transactions and quarantine writes after incomplete loads on all platforms. Initialize serializer runtime fields before preferences snapshots copy them.

Add production-code fault regressions to CI for deferred routing, store recovery, install preflight, and preferences persistence. Validation: 1,510 native tests, 60 host checks, and six firmware targets across ESP32, nRF52, STM32, and RP2040.
This commit is contained in:
mikecarper committed 2026-09-15 10:45:04 -07:00
1 parent 39c3f7b25c
commit 016a1ed2ea
20 files changed
+1010 -44

No files matched your search

+25 -2
View File
@@ -202,6 +202,18 @@ bool Mesh::otaSendAdapter(void* ctx, const uint8_t* msg, uint16_t len, bool /*fl
Packet* p = m->createOtaPacket(msg, len);
if (!p) return false;
p->radio_reply = len && ota::ota_is_response_message(msg[0]);
if (p->radio_reply) {
if (const auto* context = ota::ota_context_if_active()) {
const auto route = context->manager.replyRoute();
if (route.profile < 2) {
// DATA, PROOF and MANIFEST are emitted after the RX call stack ends.
// Restore origin only; normal reply policy still decides one/both TX.
p->radio_profile = p->radio_origin = route.profile;
p->radio_generation = p->radio_origin_generation = route.generation;
p->radio_local = false;
}
}
}
const uint8_t mask = m->getTransmitProfileMask(p);
const int copies = (mask & 1 ? 1 : 0) + (mask & 2 ? 1 : 0);
// p already occupies one slot. Reserve the other copy as well, keeping the
@@ -461,7 +473,15 @@ void __attribute__((noinline)) Mesh::serviceLoopMaintenance() {
_next_ota_tick = futureMillis(ota::scaleDelay(OTA_RETRY_TICK_MS, ota_speed < 1.0f ? ota_speed : 1.0f));
}
syncOtaTiming(ota::ota_ctx().manager);
ota::ota_ctx().manager.serviceEgress();
ota::ota_ctx().manager.serviceEgress([](void* ctx, const ota::OtaReplyRoute& route) {
if (route.profile == 0xFF) return true;
const auto* profiles = static_cast<Mesh*>(ctx)->_radio->profiles();
// A completed/reconfigured temp session must not leak its retained replies
// onto the replacement channel, or pin a stale descriptor behind backpressure.
return route.profile < 2 && (!profiles ? route.profile == 0
: (route.profile == 0 || profiles->enabled())
&& route.generation == profiles->generation[route.profile]);
}, this);
const uint32_t ota_loop_interval = ota::ota_ctx().manager.loopIntervalMs(OTA_RETRY_TICK_MS);
if ((int32_t)(_next_ota_tick - _ms->getMillis()) > (int32_t)ota_loop_interval) {
// Entering local verification must not inherit a long radio retry wait.
@@ -1275,7 +1295,10 @@ DispatcherAction Mesh::onRecvPacket(Packet* pkt) {
if (ota::OtaContext* context = ota::ota_context_if_active()) {
syncOtaTiming(context->manager);
context->manager.note_rx_path_hops(n);
terminal_ota = context->manager.on_message(pkt->payload, pkt->payload_len);
ota::OtaReplyRoute route;
route.profile = pkt->radio_profile;
route.generation = pkt->radio_generation;
terminal_ota = context->manager.on_message(pkt->payload, pkt->payload_len, route);
context->track_session(context->manager.fetchState(), _ms->getMillis());
onOtaRecv(pkt);
}
+2 -2
View File
@@ -15,8 +15,8 @@
#endif
class ConfigSerializer {
bool _first;
int8_t _depth;
bool _first = false;
int8_t _depth = 0;
bool _dirty = false;
protected:
+13 -6
View File
@@ -9,6 +9,7 @@
#include "SignerAllowlist.h"
#include "OtaApply.h"
#include "OtaFormat.h"
#include "OtaByteIO.h"
#include "OtaSelf.h" // ota_self_firmware() - prefer self-describing EndF identity at begin()
#include "OtaBlInfo.h" // bootloader OTA-apply capability marker (nRF52); cached after first read
@@ -262,12 +263,18 @@ struct OtaContext {
// hardware-compatibility gate (brick-safety) - refuse a .mota whose hw_id is for different hardware,
// independent of signature; covers a manual cross-target `ota dev want` onto an incompatible board.
{
uint8_t hdr[8], mb[256];
uint8_t hdr[8], mb[MOTA_MFL];
uint32_t total = fetch_store.staged_size();
if (total >= 13 && fetch_store.read(0, hdr, 8) && memcmp(hdr, MOTA_MAGIC, 4) == 0) {
uint32_t mr = total - 8; if (mr > sizeof(mb)) mr = sizeof(mb);
// A failed first read must not bypass these policy gates and reach an
// applier whose later read happens to succeed.
if (total < 8 + MOTA_MFL + 5 || !fetch_store.read(0, hdr, sizeof(hdr))
|| memcmp(hdr, MOTA_MAGIC, sizeof(MOTA_MAGIC)) != 0
|| rd_u32le(hdr + 4) != total) {
strncpy(msg, "refused: invalid or unreadable staged header", 96); msg[95] = 0; return false;
}
{
MotaManifest mm;
if (!fetch_store.read(8, mb, mr) || !mota_parse_manifest(mb, mr, mm)) {
if (!fetch_store.read(8, mb, sizeof(mb)) || !mota_parse_manifest(mb, sizeof(mb), mm)) {
strncpy(msg, "refused: invalid staged manifest", 96); msg[95] = 0; return false;
}
if (mm.is_bootloader()) {
@@ -283,7 +290,7 @@ struct OtaContext {
if (!ota_self_firmware(self) || !self.valid ||
!ota_trusted_auto_version_allows(self.fw_version, mm.fw_version)) {
snprintf(msg, 96,
"refused: trusted auto-install is forward-only (running=%08lX candidate=%08lX); use ota install to override",
"refused: auto-install needs newer firmware (%08lX -> %08lX); use ota install",
(unsigned long)(self.valid ? self.fw_version : 0),
(unsigned long)mm.fw_version);
return false;
@@ -291,7 +298,7 @@ struct OtaContext {
}
if (!hwMatches(mm.hw_id)) {
char want[33] = {0}; memcpy(want, mm.hw_id, 32);
snprintf(msg, 96, "refused: .mota hw_id '%.32s' != this device '%s' (incompatible hardware)", want, hw_id);
snprintf(msg, 96, "refused: incompatible hardware (%.24s / %.24s)", want, hw_id);
return false;
}
}
+38 -17
View File
@@ -483,7 +483,7 @@ bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want
bool wire_v2, bool allow_deflate, bool extended_length) {
for (uint8_t i = 0; i < _n_serve_jobs; i++) {
ServeJob& job = _serve_jobs[i];
if (job.block != block || memcmp(job.mid, mid, 4) != 0) continue;
if (job.block != block || memcmp(job.mid, mid, 4) != 0 || !(job.route == _request_route)) continue;
// A proof-only request can complete whichever representation is already queued. DATA requests with
// different geometries remain distinct because their bitmap positions do not describe the same bytes.
if (want_mask != 0 && (job.wire_v2 != wire_v2 ||
@@ -495,6 +495,7 @@ bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want
}
if (_n_serve_jobs >= OTA_SERVE_QUEUE) return false;
ServeJob& job = _serve_jobs[_n_serve_jobs++];
job.route = _request_route;
memcpy(job.mid, mid, 4);
job.block = block;
job.pending_mask = want_mask;
@@ -510,12 +511,13 @@ bool OtaManager::queueServeJob(const uint8_t* mid, uint16_t block, uint16_t want
bool OtaManager::queueManifestJob(const uint8_t* mid, uint16_t want_mask) {
for (uint8_t i = 0; i < _n_manifest_jobs; i++) {
ManifestServeJob& job = _manifest_jobs[i];
if (memcmp(job.mid, mid, 4) != 0) continue;
if (memcmp(job.mid, mid, 4) != 0 || !(job.route == _request_route)) continue;
job.pending_mask |= (uint16_t)(want_mask & ~job.emitted_mask);
return true;
}
if (_n_manifest_jobs >= OTA_MANIFEST_SERVE_QUEUE) return false;
ManifestServeJob& job = _manifest_jobs[_n_manifest_jobs++];
job.route = _request_route;
memcpy(job.mid, mid, 4);
job.pending_mask = want_mask;
job.emitted_mask = 0;
@@ -563,9 +565,13 @@ void OtaManager::popManifestJob() {
_n_manifest_jobs--;
}
bool OtaManager::serviceManifestEgress() {
bool OtaManager::serviceManifestEgress(OtaReplyRouteValid route_valid, void* route_ctx) {
if (_n_manifest_jobs == 0) return false;
ManifestServeJob& job = _manifest_jobs[0];
if (route_valid && !route_valid(route_ctx, job.route)) {
popManifestJob();
return true;
}
if ((int32_t)(_now_ms - job.ready_at) < 0) {
return true;
}
@@ -603,7 +609,7 @@ bool OtaManager::serviceManifestEgress() {
message.len = (uint16_t)len;
uint8_t wire[MAX_PACKET_PAYLOAD];
const uint16_t wire_len = encode_manifest(wire, sizeof(wire), message);
if (emit(wire, wire_len, false)) {
if (emit(wire, wire_len, false, &job.route)) {
OTA_DBG("OTA: MANIFEST tx frag=%u/%u len=%u\n",
(unsigned)fragment, (unsigned)ftotal, (unsigned)len);
const uint16_t bit = (uint16_t)(1u << fragment);
@@ -716,10 +722,10 @@ bool OtaManager::handleReqProof(const uint8_t* m, uint16_t n) {
return queueServeJob(v->m.merkle_root, rp.block_idx, 0); // proof-only, or merge with queued DATA
}
void OtaManager::serviceEgress() {
void OtaManager::serviceEgress(OtaReplyRouteValid route_valid, void* route_ctx) {
// Metadata comes first so an older receiver gets every manifest fragment before its retry horizon. The
// send callback supplies radio-queue backpressure, and a rejected fragment remains in this descriptor.
if (serviceManifestEgress()) return;
if (serviceManifestEgress(route_valid, route_ctx)) return;
// A proactive proof normally follows the final DATA fragment. If it was lost, or the source is older and
// never sent one, issue the legacy proof request after a short grace. Stay RX-silent while another slot
@@ -738,6 +744,10 @@ void OtaManager::serviceEgress() {
if (_n_serve_jobs == 0) return;
ServeJob& job = _serve_jobs[0];
if (route_valid && !route_valid(route_ctx, job.route)) {
popServeJob();
return;
}
ServeView* v = resolve(job.mid);
if (!v || job.block >= v->m.block_count
|| (uint64_t)v->m.block_count * 4 > v->scratch_sz) {
@@ -789,7 +799,7 @@ void OtaManager::serviceEgress() {
dm.data_len = (uint16_t)frag_len;
}
uint8_t b[MAX_PACKET_PAYLOAD];
if (emit(b, encode_data(b, sizeof(b), dm), false)) {
if (emit(b, encode_data(b, sizeof(b), dm), false, &job.route)) {
const uint16_t bit = (uint16_t)(1u << fragment);
job.pending_mask &= (uint16_t)~bit;
job.emitted_mask |= bit;
@@ -818,7 +828,7 @@ void OtaManager::serviceEgress() {
pm.n_proof = np;
pm.proof = proof;
uint8_t b[MAX_PACKET_PAYLOAD];
if (emit(b, encode_proof(b, sizeof(b), pm), false)) popServeJob();
if (emit(b, encode_proof(b, sizeof(b), pm), false, &job.route)) popServeJob();
}
// ---------------- fetch ----------------
@@ -1585,19 +1595,15 @@ bool OtaManager::resumeStaged(const uint8_t* want_mid) {
|| _fstate == WANT_LEAVES || _fstate == VERIFYING_STAGED) {
return false;
}
_wire_v2_session = _wire_v2_enabled;
_wire_v2_confirmed = false;
_wire_allow_deflate = _wire_v2_session && _deflate_decode != nullptr;
_wire_empty_stalls = 0;
if (!_fetch->reopen()) return false; // nothing persisted in the store
uint32_t total = _fetch->staged_size();
uint8_t hdr[8];
if (total < 13 || !_fetch->read(0, hdr, 8) || memcmp(hdr, MOTA_MAGIC, 4) != 0) return false;
if (total < 8u + MOTA_MFL + 5u || !_fetch->read(0, hdr, sizeof(hdr))
|| memcmp(hdr, MOTA_MAGIC, 4) != 0 || rd_u32le(hdr + 4) != total) return false;
// read + parse the stored manifest (everything before leaves[]) to recompute the geometry
uint8_t mbuf[256];
uint32_t mread = total - 8; if (mread > sizeof(mbuf)) mread = sizeof(mbuf);
uint8_t mbuf[MOTA_MFL];
MotaManifest m;
if (!_fetch->read(8, mbuf, mread) || !mota_parse_manifest(mbuf, mread, m)) return false;
if (!_fetch->read(8, mbuf, sizeof(mbuf)) || !mota_parse_manifest(mbuf, sizeof(mbuf), m)) return false;
if (want_mid && memcmp(m.merkle_root, want_mid, 4) != 0) return false; // a different fw is staged
const bool automatic_resume = want_mid == nullptr;
// A boot-time resume is an automatic fetch decision and therefore belongs
@@ -1625,7 +1631,14 @@ bool OtaManager::resumeStaged(const uint8_t* want_mid) {
uint32_t leaves_off = 8 + mfl;
uint32_t payload_off = leaves_off + bc * 4;
if ((uint64_t)payload_off + m.payload_size + 5 != total) return false; // geometry must match the header
uint8_t trailer[5];
if (!_fetch->read(total - sizeof(trailer), trailer, sizeof(trailer))
|| memcmp(trailer, MOTA_TRAILER, sizeof(trailer)) != 0) return false;
_wire_v2_session = _wire_v2_enabled;
_wire_v2_confirmed = false;
_wire_allow_deflate = _wire_v2_session && _deflate_decode != nullptr;
_wire_empty_stalls = 0;
memcpy(_fid, m.merkle_root, 4);
memcpy(_froot, m.merkle_root, 4);
_fflags = m.flags;
@@ -2147,7 +2160,15 @@ void OtaManager::loop() {
// ---------------- dispatch ----------------
bool OtaManager::on_message(const uint8_t* msg, uint16_t len) {
bool OtaManager::on_message(const uint8_t* msg, uint16_t len, OtaReplyRoute route) {
const OtaReplyRoute previous = _request_route;
_request_route = route;
const bool consumed = dispatchMessage(msg, len);
_request_route = previous;
return consumed;
}
bool OtaManager::dispatchMessage(const uint8_t* msg, uint16_t len) {
switch (ota_msg_type(msg, len)) {
case OTA_ADV: handleAdv(msg, len); return false;
case OTA_QUERY: handleQuery(msg, len); return false;
+27 -4
View File
@@ -30,6 +30,17 @@ namespace ota {
// replies. False applies backpressure: the manager retains paced DATA/PROOF egress and tries it again.
typedef bool (*OtaSend)(void* ctx, const uint8_t* msg, uint16_t len, bool flood);
// Deferred replies retain the request's radio session, without changing the
// portable send callback or putting radio-routing metadata on the OTA wire.
struct OtaReplyRoute {
uint32_t generation = 0;
uint8_t profile = 0xFF; // no radio affinity (portable callers)
bool operator==(const OtaReplyRoute& other) const {
return profile == other.profile && generation == other.generation;
}
};
typedef bool (*OtaReplyRouteValid)(void* ctx, const OtaReplyRoute& route);
// Read `len` payload bytes at offset `off` from the serve source (flash-backed self-serve); false on
// error. nullptr means the payload is a contiguous RAM buffer (the staged `.mota`).
typedef bool (*ServeReadFn)(void* ctx, uint32_t off, uint8_t* buf, uint32_t len);
@@ -477,11 +488,15 @@ public:
// Feed one received OTA message. True means this node terminally consumed a
// bulk request/response, so Mesh must not echo it as if this node were an
// intermediate relay.
bool on_message(const uint8_t* msg, uint16_t len);
bool on_message(const uint8_t* msg, uint16_t len, OtaReplyRoute route = {});
void loop(); // drive fetch (re-request missing blocks)
// Fast, non-blocking service called from the main radio loop. It admits at most one retained server
// response or proof fallback per call; the send callback provides packet-pool/queue backpressure.
void serviceEgress();
// An optional validator discards expired request sessions before reading or sending a response.
void serviceEgress(OtaReplyRouteValid route_valid = nullptr, void* route_ctx = nullptr);
// Available only inside the send callback; returns a value, never a retained
// pointer into a queue that may be compacted after the callback returns.
OtaReplyRoute replyRoute() const { return _reply_route ? *_reply_route : OtaReplyRoute{}; }
void clearPendingEgress();
uint8_t pendingServeJobs() const { return _n_serve_jobs; }
uint8_t pendingManifestJobs() const { return _n_manifest_jobs; }
@@ -542,8 +557,11 @@ private:
uint16_t catalogCapacity() const { return _catalog_heap ? OTA_MAX_CATALOG : OTA_INLINE_CATALOG; }
bool expandCatalog();
bool emit(const uint8_t* b, uint16_t n, bool flood) {
bool emit(const uint8_t* b, uint16_t n, bool flood, const OtaReplyRoute* route = nullptr) {
const OtaReplyRoute* previous = _reply_route;
_reply_route = route;
const bool sent = _send && n && _send(_ctx, b, n, flood);
_reply_route = previous;
if (sent) _packets_sent++;
return sent;
}
@@ -593,7 +611,8 @@ private:
bool queueManifestJob(const uint8_t* mid, uint16_t want_mask);
uint32_t manifestEgressGapMs() const;
uint32_t proofEgressGapMs() const;
bool serviceManifestEgress();
bool serviceManifestEgress(OtaReplyRouteValid route_valid, void* route_ctx);
bool dispatchMessage(const uint8_t* msg, uint16_t len);
void popManifestJob();
bool loadActiveServeBlock();
void popServeJob();
@@ -655,6 +674,7 @@ private:
// Server-side response descriptors are tiny. The active logical block has its own buffer so proof generation
// or a simultaneous fetch cannot overwrite DATA retained behind radio-queue backpressure.
struct ServeJob {
OtaReplyRoute route;
uint8_t mid[4] = {0};
uint16_t block = 0;
uint16_t pending_mask = 0;
@@ -668,6 +688,7 @@ private:
ServeJob _serve_jobs[OTA_SERVE_QUEUE];
uint8_t _n_serve_jobs = 0;
struct ManifestServeJob {
OtaReplyRoute route;
uint8_t mid[4];
uint16_t pending_mask;
uint16_t emitted_mask;
@@ -675,6 +696,8 @@ private:
};
ManifestServeJob _manifest_jobs[OTA_MANIFEST_SERVE_QUEUE];
uint8_t _n_manifest_jobs = 0;
OtaReplyRoute _request_route;
const OtaReplyRoute* _reply_route = nullptr;
uint8_t _serve_block[OTA_MAX_BLOCK];
uint16_t _serve_block_len = 0;
bool _serve_block_loaded = false;
+21 -2
View File
@@ -210,7 +210,11 @@ const uint8_t* OtaStoreFlashEsp32::meta_slot_c(uint32_t L) const {
// Bytes from `pos` that stay in one region, and (for payload) one flash sector.
uint32_t OtaStoreFlashEsp32::run(uint32_t pos, uint32_t remain) const {
if (pos >= _total - 5) return remain; // trailer (<=5, one buffer)
if (pos < _meta_span) { uint32_t c = _meta_span - pos; return remain < c ? remain : c; }
if (pos < _meta_span) {
uint32_t end = _meta_span < _total - 5u ? _meta_span : _total - 5u;
uint32_t c = end - pos;
return remain < c ? remain : c; // never consume the separate trailer through metadata
}
uint32_t poff = pay_part(pos);
uint32_t to_sec = SEC - (poff % SEC);
uint32_t to_end = (_total - 5) - pos; // don't cross into the trailer
@@ -286,6 +290,16 @@ bool OtaStoreFlashEsp32::finalize() {
// trailer sits right after the payload; this also covers the rare case it spills into a fresh sector).
uint32_t tpoff = _write_start + (_total - 5);
for (uint32_t off = 0; off < 5; ) {
const uint32_t logical = _total - 5u + off;
if (logical < _meta_span) {
// A small delta's tail can share the pinned metadata sector. Put those
// bytes in the buffer that will be flushed last, including a split tail.
uint32_t n = _meta_span - logical;
if (n > 5u - off) n = 5u - off;
memcpy(_meta + logical, _trailer + off, n);
off += n;
continue;
}
uint32_t sec = (tpoff + off) / SEC;
if (!_pay_open || sec != _pay_sec) open_pay(sec);
uint32_t in = SEC - ((tpoff + off) % SEC); if (in > 5 - off) in = 5 - off;
@@ -332,7 +346,12 @@ bool OtaStoreFlashEsp32::reopen() {
if (esp_partition_read(_part, _meta_part, _meta, _meta_flush) != ESP_OK) {
free(_meta); _meta = nullptr; _total = 0; return false;
}
memset(_trailer, 0xFF, sizeof(_trailer)); // delta trailer (re-written at finalize); full reads it from _meta
// Raw staging defers the fixed trailer; an unvisited sector may still
// contain old firmware bytes after an early checkpoint. Restore framing
// from the validated layout, not that stale tail. The manager separately
// revalidates the selected image and every present payload block.
memcpy(_trailer, MOTA_TRAILER, sizeof(_trailer));
if (_full) memcpy(_meta + _meta_bytes, MOTA_TRAILER, sizeof(_trailer));
_pay_open = false; _pay_sec = 0; _flushed = false; _io_ok = true;
_pay_max_sec = (_pay_part0 + _pay_size + SEC) / SEC; // treat all payload sectors as seen -> RMW preserves committed blocks
OTA_DBG("OTA esp32: reopen %s total=%u meta_part=%u\n", _full ? "FULL" : "DELTA", (unsigned)_total, (unsigned)o);
+4 -1
View File
@@ -599,7 +599,10 @@ bool OtaStoreFlashNrf52::reopen() {
_total = total;
_hybrid = false;
memcpy(_meta_page, p, PG); // load page 0 (header+manifest+leaves) into RAM to continue
memcpy(_trailer, p + (total - 5), 5); // recover the trailer tail (flushed at last finalize, if any)
// The tail is deferred until finalize and can still hold old flash bytes.
// Reconstruct only fixed framing; the manager must verify the manifest and
// rehash every present payload block before the container becomes complete.
memcpy(_trailer, MOTA_TRAILER, sizeof(_trailer));
_pay_idx = 0;
_flushed = false;
_io_ok = true;