From f844d3036153f36bb2f19b4670e84c84c37fe4b5 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Tue, 15 Sep 2026 09:27:37 -0700 Subject: [PATCH] Preserve OTA serving state across rejected images and source failures Validate primary images before publishing their views, invalidate the shared source cache before reads that can partially overwrite it, and revoke self-serving references before replacing caller-owned buffers. Keep the CLI serving flag aligned with a surviving primary image after a rejected replacement. Regression tests reproduce rejected replacements, partial source reads, and self-refresh allocation/read failures. Verify queued manifests, recovered payloads, Merkle proofs, and sanitizer-checked buffer lifetimes. --- src/helpers/ota/OtaCli.cpp | 6 +- src/helpers/ota/OtaManager.cpp | 20 +++- src/helpers/ota/OtaSelf.cpp | 11 ++- test/test_ota/test_ota_core.cpp | 167 ++++++++++++++++++++++++++++++++ test/test_ota_heap_context.py | 134 ++++++++++++++++++++++--- 5 files changed, 314 insertions(+), 24 deletions(-) diff --git a/src/helpers/ota/OtaCli.cpp b/src/helpers/ota/OtaCli.cpp index f18d75e5e..e99443c5c 100644 --- a/src/helpers/ota/OtaCli.cpp +++ b/src/helpers/ota/OtaCli.cpp @@ -1097,8 +1097,10 @@ static bool handle_dev(const char* d, char* reply, OtaContext& c) { strcpy(reply, "ERR nothing staged"); return true; } - c.serving = c.manager.serve(c.serve_buf, c.serve_expected); - if (!c.serving) { strcpy(reply, "ERR serve (bad .mota)"); return true; } + if (!c.manager.serve(c.serve_buf, c.serve_expected)) { + strcpy(reply, "ERR serve (bad .mota)"); return true; + } + c.serving = true; VerifyResult r = ota_verify(c.serve_buf, c.serve_expected, c.allow); sprintf(reply, "OK serving | root=%d payload=%d img=%d sig=%d trust=%d", r.root_ok, r.payload_ok, r.image_ok, r.sig_ok, r.trusted); diff --git a/src/helpers/ota/OtaManager.cpp b/src/helpers/ota/OtaManager.cpp index c4a9e1ca5..5a3c5d2ea 100644 --- a/src/helpers/ota/OtaManager.cpp +++ b/src/helpers/ota/OtaManager.cpp @@ -102,10 +102,15 @@ void OtaManager::begin(uint32_t my_target_id, OtaSend send, void* ctx) { // we advertise / answer OTA_QUERY with) is the lightweight _serve[] registry. bool OtaManager::serve(const uint8_t* mota, uint32_t len) { + MotaManifest parsed; + if (!mota || !mota_parse(mota, len, parsed)) return false; + if (parsed.block_size() == 0 || parsed.block_size() > OTA_MAX_BLOCK || + (uint64_t)parsed.block_count * 4 > OTA_PROOFGEN_SCRATCH) return false; uint8_t* scratch = ensureScratch(); if (!scratch) return false; - if (!mota_parse(mota, len, _view0.m)) return false; - if (_view0.m.block_size() == 0 || _view0.m.block_size() > OTA_MAX_BLOCK) return false; + // Parsing can fail after writing fields and pointers. Publish the replacement + // only after all checks pass, preserving the old image and its queued replies. + _view0.m = parsed; _view0.mfl = (uint16_t)(_view0.m.leaves - _view0.m.manifest_start); // contiguous container _view0.read = nullptr; _view0.read_ctx = nullptr; // payload is contiguous _view0.m.payload _view0.read_deflated = nullptr; _view0.deflate_ctx = nullptr; @@ -119,9 +124,13 @@ bool OtaManager::serve(const uint8_t* mota, uint32_t len) { bool OtaManager::serve_self(const uint8_t* manifest, uint16_t mfl, const uint8_t* leaves, uint32_t block_count, uint8_t* proof_scratch, uint32_t proof_scratch_sz, ServeReadFn read, void* ctx) { + if (!manifest || mfl != MOTA_MFL || !leaves || !proof_scratch || !read) return false; if (proof_scratch_sz < (uint64_t)block_count * 4) return false; // proof-gen needs count*4 working bytes - if (!mota_parse_manifest(manifest, mfl, _view0.m)) return false; // fixed fields: root, image_hash, sizes - if (_view0.m.block_size() == 0 || _view0.m.block_size() > OTA_MAX_BLOCK) return false; + MotaManifest parsed; + if (!mota_parse_manifest(manifest, mfl, parsed)) return false; // fixed fields: root, image_hash, sizes + if (parsed.block_size() == 0 || parsed.block_size() > OTA_MAX_BLOCK || + parsed.block_count != block_count) return false; + _view0.m = parsed; _view0.m.manifest_start = manifest; _view0.m.leaves = leaves; // pre-computed, caller-owned (heap) _view0.m.payload = nullptr; // read on demand via `read` @@ -287,6 +296,9 @@ bool OtaManager::loadSource(const ServeEntry& e) { uint8_t* src_leaves = ensureSourceLeaves(); uint8_t* scratch = ensureScratch(); if (!src_leaves || !scratch) return false; + // Both images share these buffers. Even a failed read can overwrite part of + // the old manifest/leaves, so its cached MID must stop resolving immediately. + _srcv.valid = false; bool ok = e.src->read(e.src_idx, 8, _src_manifest, mfl); if (!ok || !mota_parse_manifest(_src_manifest, mfl, _srcv.m)) return false; if (_srcv.m.block_size() == 0 || _srcv.m.block_size() > OTA_MAX_BLOCK) return false; diff --git a/src/helpers/ota/OtaSelf.cpp b/src/helpers/ota/OtaSelf.cpp index 446d372b2..47132b583 100644 --- a/src/helpers/ota/OtaSelf.cpp +++ b/src/helpers/ota/OtaSelf.cpp @@ -151,6 +151,12 @@ bool ota_serve_self(OtaContext& c, uint32_t fw_version) { const uint32_t bc = image_size / BS + (image_size % BS != 0 ? 1u : 0u); if ((uint64_t)bc * 4 > OTA_SELF_LEAVES_MAX) return false; + // A repeated `serve self` replaces caller-owned buffers. Revoke the old + // view and queued replies before freeing them; OOM/read failure must leave + // serving disabled rather than advertise dangling pointers. Avoid holding + // two images' metadata at once on boards with little free heap. + c.manager.clear_primary(); + c.serving = false; free(c.serve_self_leaves); free(c.serve_self_proof); c.serve_self_leaves = (uint8_t*)malloc((size_t)bc * 4); c.serve_self_proof = (uint8_t*)malloc((size_t)bc * 4); // proof-gen working buffer (sized to OUR image) @@ -193,8 +199,9 @@ bool ota_serve_self(OtaContext& c, uint32_t fw_version) { m[56] = CODEC_FULL; memcpy(m + 57, out_hw, strlen(out_hw) < 32 ? strlen(out_hw) : 32); // hw_id[32] (NUL-padded by memset) memcpy(m + MOTA_OFF_APPROVAL, APPROVAL_NOT, 4); // approval (fetching device's apply-gate handles it) - return c.manager.serve_self(m, MOTA_MFL, c.serve_self_leaves, bc, - c.serve_self_proof, (size_t)bc * 4, self_read_cb, nullptr); + c.serving = c.manager.serve_self(m, MOTA_MFL, c.serve_self_leaves, bc, + c.serve_self_proof, (size_t)bc * 4, self_read_cb, nullptr); + return c.serving; } #endif diff --git a/test/test_ota/test_ota_core.cpp b/test/test_ota/test_ota_core.cpp index 3fee375d3..3b07cadaf 100644 --- a/test/test_ota/test_ota_core.cpp +++ b/test/test_ota/test_ota_core.cpp @@ -2200,6 +2200,173 @@ private: uint8_t _count; uint8_t _block_log2; }; + +class FaultingMotaSource : public RamMotaSource { +public: + uint32_t fail_offset = UINT32_MAX; + bool partial_read = false; + bool read(uint8_t idx, uint32_t off, uint8_t* out, uint32_t len) override { + if (idx == 1 && off == fail_offset) { + if (partial_read && len > 1) RamMotaSource::read(idx, off, out, len / 2); + return false; + } + return RamMotaSource::read(idx, off, out, len); + } +}; + +static bool request_manifest(OtaManager& manager, const uint8_t* mid) { + GetManifestMsg request{}; + memcpy(request.manifest_id, mid, 4); + request.want_mask = 0xFFFF; + uint8_t wire[MAX_PACKET_PAYLOAD]; + const uint16_t length = encode_get_manifest(wire, sizeof(wire), request); + return length && manager.on_message(wire, length); +} + +static void expect_queued_manifest(OtaManager& manager, CapturedMessages& sent, + const MotaManifest& expected) { + for (uint32_t tick = 1; tick <= 16 && manager.pendingManifestJobs(); ++tick) { + manager.set_clock(tick * 10000u); + manager.serviceEgress(); + } + ASSERT_EQ(manager.pendingManifestJobs(), 0u); + std::vector bytes; + for (const auto& wire : sent.items) { + ManifestMsg fragment{}; + ASSERT_TRUE(decode_manifest(wire.data(), (uint16_t)wire.size(), fragment)); + EXPECT_EQ(memcmp(fragment.manifest_id, expected.merkle_root, 4), 0); + EXPECT_EQ(fragment.frag_idx * OTA_MF_FRAG, bytes.size()); + bytes.insert(bytes.end(), fragment.bytes, fragment.bytes + fragment.len); + } + ASSERT_EQ(bytes.size(), MOTA_MFL); + EXPECT_EQ(memcmp(bytes.data(), expected.manifest_start, MOTA_MFL), 0); +} + +static void expect_served_block(OtaManager& manager, CapturedMessages& sent, + const MotaManifest& expected) { + sent.items.clear(); + ReqMsg request{}; + memcpy(request.manifest_id, expected.merkle_root, 4); + request.want_mask = 0xFFFF; + uint8_t wire[MAX_PACKET_PAYLOAD]; + const uint16_t length = encode_req(wire, sizeof(wire), request); + ASSERT_TRUE(manager.on_message(wire, length)); + for (uint32_t tick = 1; tick <= 32 && manager.pendingServeJobs(); ++tick) { + manager.set_clock(200000u + tick * 10000u); + manager.serviceEgress(); + } + ASSERT_EQ(manager.pendingServeJobs(), 0u); + std::vector bytes; + bool verified = false; + for (const auto& packet : sent.items) { + DataMsg data{}; + ProofMsg proof{}; + if (decode_data(packet.data(), (uint16_t)packet.size(), data)) { + EXPECT_EQ(memcmp(data.manifest_id, expected.merkle_root, 4), 0); + ASSERT_EQ(data.frag_off, bytes.size()); + bytes.insert(bytes.end(), data.data, data.data + data.data_len); + } else { + ASSERT_TRUE(decode_proof(packet.data(), (uint16_t)packet.size(), proof)); + EXPECT_EQ(memcmp(proof.manifest_id, expected.merkle_root, 4), 0); + verified = merkle_verify(bytes.data(), (uint32_t)bytes.size(), 0, + proof.proof, proof.n_proof, expected.merkle_root, expected.block_count); + } + } + ASSERT_EQ(bytes.size(), std::min(expected.block_size(), expected.payload_size)); + EXPECT_EQ(memcmp(bytes.data(), expected.payload, bytes.size()), 0); + EXPECT_TRUE(verified); +} +} + +TEST(OtaServe, RejectedPrimaryReplacementPreservesQueuedManifest) { + MotaManifest original; + ASSERT_TRUE(mota_parse(SIM_MOTA, SIM_MOTA_LEN, original)); + for (bool streaming : {false, true}) { + SCOPED_TRACE(streaming ? "serve_self" : "serve"); + OtaManager manager; + CapturedMessages sent; + manager.begin(0, capture_send, &sent); + ASSERT_TRUE(manager.serve(SIM_MOTA, SIM_MOTA_LEN)); + ASSERT_TRUE(request_manifest(manager, original.merkle_root)); + // Parsing gets far enough to overwrite pointers/geometry before rejecting + // this replacement. It must never mutate the live, advertised image. + std::vector bad(SIM_MOTA_1K, SIM_MOTA_1K + SIM_MOTA_1K_LEN); + bad[8 + 19] = 25; + uint8_t scratch[OTA_PROOFGEN_SCRATCH]; + if (streaming) { + EXPECT_FALSE(manager.serve_self(bad.data() + 8, MOTA_MFL, + original.leaves, original.block_count, scratch, sizeof(scratch), + [](void*, uint32_t, uint8_t*, uint32_t) { return false; }, nullptr)); + } else { + EXPECT_FALSE(manager.serve(bad.data(), (uint32_t)bad.size())); + } + expect_queued_manifest(manager, sent, original); + expect_served_block(manager, sent, original); + } +} + +TEST(OtaServe, InvalidPrimaryBackingDoesNotReplaceLiveImage) { + MotaManifest original; + ASSERT_TRUE(mota_parse(SIM_MOTA, SIM_MOTA_LEN, original)); + uint8_t scratch[OTA_PROOFGEN_SCRATCH]; + const auto read = [](void*, uint32_t, uint8_t*, uint32_t) { return false; }; + for (unsigned failure = 0; failure < 9; ++failure) { + SCOPED_TRACE(failure); + OtaManager manager; + CapturedMessages sent; + manager.begin(0, capture_send, &sent); + ASSERT_TRUE(manager.serve(SIM_MOTA, SIM_MOTA_LEN)); + if (failure < 2) { + EXPECT_FALSE(manager.serve(failure == 0 ? nullptr : SIM_MOTA, + failure == 0 ? SIM_MOTA_LEN : 0)); + } else { + EXPECT_FALSE(manager.serve_self( + failure == 2 ? nullptr : original.manifest_start, + failure == 3 ? MOTA_MFL - 1 : MOTA_MFL, + failure == 4 ? nullptr : original.leaves, + failure == 5 ? original.block_count + 1 : original.block_count, + failure == 6 ? nullptr : scratch, + failure == 7 ? 0 : sizeof(scratch), + failure == 8 ? nullptr : +read, nullptr)); + } + ASSERT_TRUE(request_manifest(manager, original.merkle_root)); + expect_queued_manifest(manager, sent, original); + expect_served_block(manager, sent, original); + } +} + +TEST(OtaServe, FailedSourceSwitchReloadsPreviousImageBeforeQueuedReply) { + MotaManifest original; + ASSERT_TRUE(mota_parse(SIM_MOTA, SIM_MOTA_LEN, original)); + MotaManifest other; + ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, other)); + for (uint32_t failure : {8u, 8u + MOTA_MFL}) { + for (bool partial : {false, true}) { + SCOPED_TRACE(failure); + SCOPED_TRACE(partial); + OtaManager manager; + CapturedMessages sent; + FaultingMotaSource folder; + folder.add(SIM_MOTA, SIM_MOTA_LEN); + folder.add(SIM_MOTA_1K, SIM_MOTA_1K_LEN); + manager.begin(0, capture_send, &sent); + ASSERT_TRUE(manager.add_source(&folder)); + ASSERT_TRUE(request_manifest(manager, original.merkle_root)); + folder.fail_offset = failure; + folder.partial_read = partial; + EXPECT_FALSE(request_manifest(manager, other.merkle_root)); + expect_queued_manifest(manager, sent, original); + expect_served_block(manager, sent, original); + // A transient source failure must also recover when the new image is + // requested again, with the original source still registered. + sent.items.clear(); + manager.set_clock(0); + folder.fail_offset = UINT32_MAX; + ASSERT_TRUE(request_manifest(manager, other.merkle_root)); + expect_queued_manifest(manager, sent, other); + expect_served_block(manager, sent, other); + } + } } TEST(OtaServe, ClearPrimaryInvalidatesCallerOwnedView) { diff --git a/test/test_ota_heap_context.py b/test/test_ota_heap_context.py index a1620671a..8d27dcdfd 100644 --- a/test/test_ota_heap_context.py +++ b/test/test_ota_heap_context.py @@ -117,22 +117,124 @@ int main() { assert(!ota_context_if_active()); } ''') - flags = [] if os.name == "nt" else ["-fsanitize=address,undefined"] - tinf = path / "tinf.o" - subprocess.run([shutil.which("cc") or "gcc", "-DENABLE_OTA=1", *flags, "-c", - str(ROOT / "src/helpers/ota/OtaTinf.c"), "-o", str(tinf)], check=True) - sources = ["OtaContext.cpp", "OtaManager.cpp", "OtaProtocol.cpp", - "MotaContainer.cpp", "MerkleTree.cpp", "OtaDeflate.cpp"] - binary = path / "heap.exe" - result = subprocess.run([ - "c++", "-std=c++17", *flags, "-DENABLE_OTA=1", - "-DOTA_HEAP_CONTEXT=1", "-DESP32_PLATFORM=1", "-DOTA_SEEDER_ONLY=1", - "-I", str(ROOT / "src"), "-I", str(ROOT / "test/mocks"), - str(source), *[str(ROOT / "src/helpers/ota" / name) for name in sources], - str(ROOT / "src/Utils.cpp"), str(tinf), "-o", str(binary), - ], capture_output=True, text=True) - self.assertEqual(result.returncode, 0, result.stderr) - subprocess.run([str(binary)], check=True) + self.compile_and_run(path, source) + + def test_self_refresh_revokes_view_before_freeing_buffers(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) + source = path / "test.cpp" + (path / "self_serve.h").write_text(method( + (ROOT / "src/helpers/ota/OtaSelf.cpp").read_text(), + "bool ota_serve_self(")) + serve_command = method((ROOT / "src/helpers/ota/OtaCli.cpp").read_text(), + 'else if (strncmp(d, "serve", 5) == 0)') + # Run the production admission/status path; the verification report + # following it is unrelated to a rejected replacement. + serve_command = serve_command.removeprefix("else ").split("VerifyResult r", 1)[0] + (path / "serve_command.h").write_text( + 'static bool serve_command(OtaContext& c, char* reply) {\n' + 'const char* d = "serve";\n' + serve_command + '} return true; }\n') + source.write_text(r''' +#include +#include +#include +#include +#define OTA_SELF_LEAVES_MAX 65536u +using namespace mesh::ota; +static int allocation_countdown = -1; +static bool fail_read = false; +static void* self_malloc(size_t bytes) { + if (allocation_countdown == 0) return nullptr; + if (allocation_countdown > 0) --allocation_countdown; + return malloc(bytes); +} +namespace mesh { namespace ota { +bool ota_self_firmware(SelfFwInfo& info) { + info = SelfFwInfo(); info.valid = true; info.image_len = 3000; + info.target_id = 123; info.fw_version = 1; + return true; +} +bool ota_self_read(uint32_t off, uint8_t* buf, uint32_t len) { + if (fail_read) return false; + for (uint32_t i = 0; i < len; ++i) buf[i] = (uint8_t)(off + i); + return true; +} +static bool self_read_cb(void*, uint32_t off, uint8_t* buf, uint32_t len) { + return ota_self_read(off, buf, len); +} +#define malloc self_malloc +#include "self_serve.h" +#undef malloc +} } +static bool send(void*, const uint8_t*, uint16_t, bool) { return true; } +#include "serve_command.h" +int main(int argc, char** argv) { + OtaContext context; + context.begin(123, send, nullptr, "test"); + assert(ota_serve_self(context, 1)); + context.serving = true; + assert(context.manager.servedCount() == 1); + // A rejected staged replacement must preserve the CLI's serving flag too. + assert(context.ensureServeBuffer()); + context.serve_expected = 1; + char reply[160] = {}; + assert(serve_command(context, reply)); + assert(strncmp(reply, "ERR serve", 9) == 0); + assert(context.serving && context.manager.servedCount() == 1); + context.serve_expected = 0; + GetManifestMsg manifest{}; + ReqMsg block{}; + memcpy(manifest.manifest_id, context.serve_self_manifest + 20, 4); + memcpy(block.manifest_id, manifest.manifest_id, 4); + manifest.want_mask = block.want_mask = 0xFFFF; + uint8_t wire[MAX_PACKET_PAYLOAD]; + auto length = encode_get_manifest(wire, sizeof(wire), manifest); + assert(context.manager.on_message(wire, length)); + length = encode_req(wire, sizeof(wire), block); + assert(context.manager.on_message(wire, length)); + assert(context.manager.pendingManifestJobs() == 1); + assert(context.manager.pendingServeJobs() == 1); + + const int failure = argc > 1 ? atoi(argv[1]) : 0; + allocation_countdown = failure < 2 ? failure : -1; + fail_read = failure == 2; + assert(!ota_serve_self(context, 2)); + assert(context.manager.servedCount() == 0); + assert(!context.serving); + assert(!context.serve_self_leaves && !context.serve_self_proof); + assert(context.manager.pendingManifestJobs() == 0); + assert(context.manager.pendingServeJobs() == 0); + context.manager.serviceEgress(); + assert(!context.manager.on_message(wire, length)); + + // Reconnect/retry is still possible after either allocation or flash-read failure. + allocation_countdown = -1; fail_read = false; + assert(ota_serve_self(context, 3)); + assert(context.manager.servedCount() == 1); + assert(context.serving); +} +''') + self.compile_and_run(path, source, [("0",), ("1",), ("2",)]) + + def compile_and_run(self, path, source, scenarios=((),)): + flags = [] if os.name == "nt" else ["-fsanitize=address,undefined"] + tinf = path / "tinf.o" + subprocess.run([shutil.which("cc") or "gcc", "-DENABLE_OTA=1", *flags, "-c", + str(ROOT / "src/helpers/ota/OtaTinf.c"), "-o", str(tinf)], check=True) + sources = ["OtaContext.cpp", "OtaManager.cpp", "OtaProtocol.cpp", + "MotaContainer.cpp", "MerkleTree.cpp", "OtaDeflate.cpp"] + binary = path / "heap.exe" + result = subprocess.run([ + "c++", "-std=c++17", *flags, "-DENABLE_OTA=1", + "-DOTA_HEAP_CONTEXT=1", "-DESP32_PLATFORM=1", "-DOTA_SEEDER_ONLY=1", + "-I", str(ROOT / "src"), "-I", str(ROOT / "test/mocks"), + str(source), *[str(ROOT / "src/helpers/ota" / name) for name in sources], + str(ROOT / "src/Utils.cpp"), str(tinf), "-o", str(binary), + ], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + for scenario in scenarios: + with self.subTest(scenario=scenario): + subprocess.run([str(binary), *scenario], check=True) if __name__ == "__main__":