Preserve OTA serving state across rejected images and source failures

Validate primary images before publishing their views, invalidate the shared source cache before reads that can partially overwrite it, and revoke self-serving references before replacing caller-owned buffers. Keep the CLI serving flag aligned with a surviving primary image after a rejected replacement.

Regression tests reproduce rejected replacements, partial source reads, and self-refresh allocation/read failures. Verify queued manifests, recovered payloads, Merkle proofs, and sanitizer-checked buffer lifetimes.
This commit is contained in:
mikecarper committed 2026-09-15 09:27:37 -07:00
1 parent a2a7d3b168
commit f844d30361
5 files changed
+314 -24

No files matched your search

+4 -2
View File
@@ -1097,8 +1097,10 @@ static bool handle_dev(const char* d, char* reply, OtaContext& c) {
strcpy(reply, "ERR nothing staged");
return true;
}
c.serving = c.manager.serve(c.serve_buf, c.serve_expected);
if (!c.serving) { strcpy(reply, "ERR serve (bad .mota)"); return true; }
if (!c.manager.serve(c.serve_buf, c.serve_expected)) {
strcpy(reply, "ERR serve (bad .mota)"); return true;
}
c.serving = true;
VerifyResult r = ota_verify(c.serve_buf, c.serve_expected, c.allow);
sprintf(reply, "OK serving | root=%d payload=%d img=%d sig=%d trust=%d",
r.root_ok, r.payload_ok, r.image_ok, r.sig_ok, r.trusted);
+16 -4
View File
@@ -102,10 +102,15 @@ void OtaManager::begin(uint32_t my_target_id, OtaSend send, void* ctx) {
// we advertise / answer OTA_QUERY with) is the lightweight _serve[] registry.
bool OtaManager::serve(const uint8_t* mota, uint32_t len) {
MotaManifest parsed;
if (!mota || !mota_parse(mota, len, parsed)) return false;
if (parsed.block_size() == 0 || parsed.block_size() > OTA_MAX_BLOCK ||
(uint64_t)parsed.block_count * 4 > OTA_PROOFGEN_SCRATCH) return false;
uint8_t* scratch = ensureScratch();
if (!scratch) return false;
if (!mota_parse(mota, len, _view0.m)) return false;
if (_view0.m.block_size() == 0 || _view0.m.block_size() > OTA_MAX_BLOCK) return false;
// Parsing can fail after writing fields and pointers. Publish the replacement
// only after all checks pass, preserving the old image and its queued replies.
_view0.m = parsed;
_view0.mfl = (uint16_t)(_view0.m.leaves - _view0.m.manifest_start); // contiguous container
_view0.read = nullptr; _view0.read_ctx = nullptr; // payload is contiguous _view0.m.payload
_view0.read_deflated = nullptr; _view0.deflate_ctx = nullptr;
@@ -119,9 +124,13 @@ bool OtaManager::serve(const uint8_t* mota, uint32_t len) {
bool OtaManager::serve_self(const uint8_t* manifest, uint16_t mfl, const uint8_t* leaves,
uint32_t block_count, uint8_t* proof_scratch, uint32_t proof_scratch_sz,
ServeReadFn read, void* ctx) {
if (!manifest || mfl != MOTA_MFL || !leaves || !proof_scratch || !read) return false;
if (proof_scratch_sz < (uint64_t)block_count * 4) return false; // proof-gen needs count*4 working bytes
if (!mota_parse_manifest(manifest, mfl, _view0.m)) return false; // fixed fields: root, image_hash, sizes
if (_view0.m.block_size() == 0 || _view0.m.block_size() > OTA_MAX_BLOCK) return false;
MotaManifest parsed;
if (!mota_parse_manifest(manifest, mfl, parsed)) return false; // fixed fields: root, image_hash, sizes
if (parsed.block_size() == 0 || parsed.block_size() > OTA_MAX_BLOCK ||
parsed.block_count != block_count) return false;
_view0.m = parsed;
_view0.m.manifest_start = manifest;
_view0.m.leaves = leaves; // pre-computed, caller-owned (heap)
_view0.m.payload = nullptr; // read on demand via `read`
@@ -287,6 +296,9 @@ bool OtaManager::loadSource(const ServeEntry& e) {
uint8_t* src_leaves = ensureSourceLeaves();
uint8_t* scratch = ensureScratch();
if (!src_leaves || !scratch) return false;
// Both images share these buffers. Even a failed read can overwrite part of
// the old manifest/leaves, so its cached MID must stop resolving immediately.
_srcv.valid = false;
bool ok = e.src->read(e.src_idx, 8, _src_manifest, mfl);
if (!ok || !mota_parse_manifest(_src_manifest, mfl, _srcv.m)) return false;
if (_srcv.m.block_size() == 0 || _srcv.m.block_size() > OTA_MAX_BLOCK) return false;
+9 -2
View File
@@ -151,6 +151,12 @@ bool ota_serve_self(OtaContext& c, uint32_t fw_version) {
const uint32_t bc = image_size / BS + (image_size % BS != 0 ? 1u : 0u);
if ((uint64_t)bc * 4 > OTA_SELF_LEAVES_MAX) return false;
// A repeated `serve self` replaces caller-owned buffers. Revoke the old
// view and queued replies before freeing them; OOM/read failure must leave
// serving disabled rather than advertise dangling pointers. Avoid holding
// two images' metadata at once on boards with little free heap.
c.manager.clear_primary();
c.serving = false;
free(c.serve_self_leaves); free(c.serve_self_proof);
c.serve_self_leaves = (uint8_t*)malloc((size_t)bc * 4);
c.serve_self_proof = (uint8_t*)malloc((size_t)bc * 4); // proof-gen working buffer (sized to OUR image)
@@ -193,8 +199,9 @@ bool ota_serve_self(OtaContext& c, uint32_t fw_version) {
m[56] = CODEC_FULL;
memcpy(m + 57, out_hw, strlen(out_hw) < 32 ? strlen(out_hw) : 32); // hw_id[32] (NUL-padded by memset)
memcpy(m + MOTA_OFF_APPROVAL, APPROVAL_NOT, 4); // approval (fetching device's apply-gate handles it)
return c.manager.serve_self(m, MOTA_MFL, c.serve_self_leaves, bc,
c.serve_self_proof, (size_t)bc * 4, self_read_cb, nullptr);
c.serving = c.manager.serve_self(m, MOTA_MFL, c.serve_self_leaves, bc,
c.serve_self_proof, (size_t)bc * 4, self_read_cb, nullptr);
return c.serving;
}
#endif
+167
View File
@@ -2200,6 +2200,173 @@ private:
uint8_t _count;
uint8_t _block_log2;
};
class FaultingMotaSource : public RamMotaSource {
public:
uint32_t fail_offset = UINT32_MAX;
bool partial_read = false;
bool read(uint8_t idx, uint32_t off, uint8_t* out, uint32_t len) override {
if (idx == 1 && off == fail_offset) {
if (partial_read && len > 1) RamMotaSource::read(idx, off, out, len / 2);
return false;
}
return RamMotaSource::read(idx, off, out, len);
}
};
static bool request_manifest(OtaManager& manager, const uint8_t* mid) {
GetManifestMsg request{};
memcpy(request.manifest_id, mid, 4);
request.want_mask = 0xFFFF;
uint8_t wire[MAX_PACKET_PAYLOAD];
const uint16_t length = encode_get_manifest(wire, sizeof(wire), request);
return length && manager.on_message(wire, length);
}
static void expect_queued_manifest(OtaManager& manager, CapturedMessages& sent,
const MotaManifest& expected) {
for (uint32_t tick = 1; tick <= 16 && manager.pendingManifestJobs(); ++tick) {
manager.set_clock(tick * 10000u);
manager.serviceEgress();
}
ASSERT_EQ(manager.pendingManifestJobs(), 0u);
std::vector<uint8_t> bytes;
for (const auto& wire : sent.items) {
ManifestMsg fragment{};
ASSERT_TRUE(decode_manifest(wire.data(), (uint16_t)wire.size(), fragment));
EXPECT_EQ(memcmp(fragment.manifest_id, expected.merkle_root, 4), 0);
EXPECT_EQ(fragment.frag_idx * OTA_MF_FRAG, bytes.size());
bytes.insert(bytes.end(), fragment.bytes, fragment.bytes + fragment.len);
}
ASSERT_EQ(bytes.size(), MOTA_MFL);
EXPECT_EQ(memcmp(bytes.data(), expected.manifest_start, MOTA_MFL), 0);
}
static void expect_served_block(OtaManager& manager, CapturedMessages& sent,
const MotaManifest& expected) {
sent.items.clear();
ReqMsg request{};
memcpy(request.manifest_id, expected.merkle_root, 4);
request.want_mask = 0xFFFF;
uint8_t wire[MAX_PACKET_PAYLOAD];
const uint16_t length = encode_req(wire, sizeof(wire), request);
ASSERT_TRUE(manager.on_message(wire, length));
for (uint32_t tick = 1; tick <= 32 && manager.pendingServeJobs(); ++tick) {
manager.set_clock(200000u + tick * 10000u);
manager.serviceEgress();
}
ASSERT_EQ(manager.pendingServeJobs(), 0u);
std::vector<uint8_t> bytes;
bool verified = false;
for (const auto& packet : sent.items) {
DataMsg data{};
ProofMsg proof{};
if (decode_data(packet.data(), (uint16_t)packet.size(), data)) {
EXPECT_EQ(memcmp(data.manifest_id, expected.merkle_root, 4), 0);
ASSERT_EQ(data.frag_off, bytes.size());
bytes.insert(bytes.end(), data.data, data.data + data.data_len);
} else {
ASSERT_TRUE(decode_proof(packet.data(), (uint16_t)packet.size(), proof));
EXPECT_EQ(memcmp(proof.manifest_id, expected.merkle_root, 4), 0);
verified = merkle_verify(bytes.data(), (uint32_t)bytes.size(), 0,
proof.proof, proof.n_proof, expected.merkle_root, expected.block_count);
}
}
ASSERT_EQ(bytes.size(), std::min(expected.block_size(), expected.payload_size));
EXPECT_EQ(memcmp(bytes.data(), expected.payload, bytes.size()), 0);
EXPECT_TRUE(verified);
}
}
TEST(OtaServe, RejectedPrimaryReplacementPreservesQueuedManifest) {
MotaManifest original;
ASSERT_TRUE(mota_parse(SIM_MOTA, SIM_MOTA_LEN, original));
for (bool streaming : {false, true}) {
SCOPED_TRACE(streaming ? "serve_self" : "serve");
OtaManager manager;
CapturedMessages sent;
manager.begin(0, capture_send, &sent);
ASSERT_TRUE(manager.serve(SIM_MOTA, SIM_MOTA_LEN));
ASSERT_TRUE(request_manifest(manager, original.merkle_root));
// Parsing gets far enough to overwrite pointers/geometry before rejecting
// this replacement. It must never mutate the live, advertised image.
std::vector<uint8_t> bad(SIM_MOTA_1K, SIM_MOTA_1K + SIM_MOTA_1K_LEN);
bad[8 + 19] = 25;
uint8_t scratch[OTA_PROOFGEN_SCRATCH];
if (streaming) {
EXPECT_FALSE(manager.serve_self(bad.data() + 8, MOTA_MFL,
original.leaves, original.block_count, scratch, sizeof(scratch),
[](void*, uint32_t, uint8_t*, uint32_t) { return false; }, nullptr));
} else {
EXPECT_FALSE(manager.serve(bad.data(), (uint32_t)bad.size()));
}
expect_queued_manifest(manager, sent, original);
expect_served_block(manager, sent, original);
}
}
TEST(OtaServe, InvalidPrimaryBackingDoesNotReplaceLiveImage) {
MotaManifest original;
ASSERT_TRUE(mota_parse(SIM_MOTA, SIM_MOTA_LEN, original));
uint8_t scratch[OTA_PROOFGEN_SCRATCH];
const auto read = [](void*, uint32_t, uint8_t*, uint32_t) { return false; };
for (unsigned failure = 0; failure < 9; ++failure) {
SCOPED_TRACE(failure);
OtaManager manager;
CapturedMessages sent;
manager.begin(0, capture_send, &sent);
ASSERT_TRUE(manager.serve(SIM_MOTA, SIM_MOTA_LEN));
if (failure < 2) {
EXPECT_FALSE(manager.serve(failure == 0 ? nullptr : SIM_MOTA,
failure == 0 ? SIM_MOTA_LEN : 0));
} else {
EXPECT_FALSE(manager.serve_self(
failure == 2 ? nullptr : original.manifest_start,
failure == 3 ? MOTA_MFL - 1 : MOTA_MFL,
failure == 4 ? nullptr : original.leaves,
failure == 5 ? original.block_count + 1 : original.block_count,
failure == 6 ? nullptr : scratch,
failure == 7 ? 0 : sizeof(scratch),
failure == 8 ? nullptr : +read, nullptr));
}
ASSERT_TRUE(request_manifest(manager, original.merkle_root));
expect_queued_manifest(manager, sent, original);
expect_served_block(manager, sent, original);
}
}
TEST(OtaServe, FailedSourceSwitchReloadsPreviousImageBeforeQueuedReply) {
MotaManifest original;
ASSERT_TRUE(mota_parse(SIM_MOTA, SIM_MOTA_LEN, original));
MotaManifest other;
ASSERT_TRUE(mota_parse(SIM_MOTA_1K, SIM_MOTA_1K_LEN, other));
for (uint32_t failure : {8u, 8u + MOTA_MFL}) {
for (bool partial : {false, true}) {
SCOPED_TRACE(failure);
SCOPED_TRACE(partial);
OtaManager manager;
CapturedMessages sent;
FaultingMotaSource folder;
folder.add(SIM_MOTA, SIM_MOTA_LEN);
folder.add(SIM_MOTA_1K, SIM_MOTA_1K_LEN);
manager.begin(0, capture_send, &sent);
ASSERT_TRUE(manager.add_source(&folder));
ASSERT_TRUE(request_manifest(manager, original.merkle_root));
folder.fail_offset = failure;
folder.partial_read = partial;
EXPECT_FALSE(request_manifest(manager, other.merkle_root));
expect_queued_manifest(manager, sent, original);
expect_served_block(manager, sent, original);
// A transient source failure must also recover when the new image is
// requested again, with the original source still registered.
sent.items.clear();
manager.set_clock(0);
folder.fail_offset = UINT32_MAX;
ASSERT_TRUE(request_manifest(manager, other.merkle_root));
expect_queued_manifest(manager, sent, other);
expect_served_block(manager, sent, other);
}
}
}
TEST(OtaServe, ClearPrimaryInvalidatesCallerOwnedView) {
+118 -16
View File
@@ -117,22 +117,124 @@ int main() {
assert(!ota_context_if_active());
}
''')
flags = [] if os.name == "nt" else ["-fsanitize=address,undefined"]
tinf = path / "tinf.o"
subprocess.run([shutil.which("cc") or "gcc", "-DENABLE_OTA=1", *flags, "-c",
str(ROOT / "src/helpers/ota/OtaTinf.c"), "-o", str(tinf)], check=True)
sources = ["OtaContext.cpp", "OtaManager.cpp", "OtaProtocol.cpp",
"MotaContainer.cpp", "MerkleTree.cpp", "OtaDeflate.cpp"]
binary = path / "heap.exe"
result = subprocess.run([
"c++", "-std=c++17", *flags, "-DENABLE_OTA=1",
"-DOTA_HEAP_CONTEXT=1", "-DESP32_PLATFORM=1", "-DOTA_SEEDER_ONLY=1",
"-I", str(ROOT / "src"), "-I", str(ROOT / "test/mocks"),
str(source), *[str(ROOT / "src/helpers/ota" / name) for name in sources],
str(ROOT / "src/Utils.cpp"), str(tinf), "-o", str(binary),
], capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
subprocess.run([str(binary)], check=True)
self.compile_and_run(path, source)
def test_self_refresh_revokes_view_before_freeing_buffers(self):
with tempfile.TemporaryDirectory() as temp:
path = Path(temp)
source = path / "test.cpp"
(path / "self_serve.h").write_text(method(
(ROOT / "src/helpers/ota/OtaSelf.cpp").read_text(),
"bool ota_serve_self("))
serve_command = method((ROOT / "src/helpers/ota/OtaCli.cpp").read_text(),
'else if (strncmp(d, "serve", 5) == 0)')
# Run the production admission/status path; the verification report
# following it is unrelated to a rejected replacement.
serve_command = serve_command.removeprefix("else ").split("VerifyResult r", 1)[0]
(path / "serve_command.h").write_text(
'static bool serve_command(OtaContext& c, char* reply) {\n'
'const char* d = "serve";\n' + serve_command + '} return true; }\n')
source.write_text(r'''
#include <helpers/ota/OtaContext.h>
#include <helpers/ota/OtaByteIO.h>
#include <SHA256.h>
#include <cassert>
#define OTA_SELF_LEAVES_MAX 65536u
using namespace mesh::ota;
static int allocation_countdown = -1;
static bool fail_read = false;
static void* self_malloc(size_t bytes) {
if (allocation_countdown == 0) return nullptr;
if (allocation_countdown > 0) --allocation_countdown;
return malloc(bytes);
}
namespace mesh { namespace ota {
bool ota_self_firmware(SelfFwInfo& info) {
info = SelfFwInfo(); info.valid = true; info.image_len = 3000;
info.target_id = 123; info.fw_version = 1;
return true;
}
bool ota_self_read(uint32_t off, uint8_t* buf, uint32_t len) {
if (fail_read) return false;
for (uint32_t i = 0; i < len; ++i) buf[i] = (uint8_t)(off + i);
return true;
}
static bool self_read_cb(void*, uint32_t off, uint8_t* buf, uint32_t len) {
return ota_self_read(off, buf, len);
}
#define malloc self_malloc
#include "self_serve.h"
#undef malloc
} }
static bool send(void*, const uint8_t*, uint16_t, bool) { return true; }
#include "serve_command.h"
int main(int argc, char** argv) {
OtaContext context;
context.begin(123, send, nullptr, "test");
assert(ota_serve_self(context, 1));
context.serving = true;
assert(context.manager.servedCount() == 1);
// A rejected staged replacement must preserve the CLI's serving flag too.
assert(context.ensureServeBuffer());
context.serve_expected = 1;
char reply[160] = {};
assert(serve_command(context, reply));
assert(strncmp(reply, "ERR serve", 9) == 0);
assert(context.serving && context.manager.servedCount() == 1);
context.serve_expected = 0;
GetManifestMsg manifest{};
ReqMsg block{};
memcpy(manifest.manifest_id, context.serve_self_manifest + 20, 4);
memcpy(block.manifest_id, manifest.manifest_id, 4);
manifest.want_mask = block.want_mask = 0xFFFF;
uint8_t wire[MAX_PACKET_PAYLOAD];
auto length = encode_get_manifest(wire, sizeof(wire), manifest);
assert(context.manager.on_message(wire, length));
length = encode_req(wire, sizeof(wire), block);
assert(context.manager.on_message(wire, length));
assert(context.manager.pendingManifestJobs() == 1);
assert(context.manager.pendingServeJobs() == 1);
const int failure = argc > 1 ? atoi(argv[1]) : 0;
allocation_countdown = failure < 2 ? failure : -1;
fail_read = failure == 2;
assert(!ota_serve_self(context, 2));
assert(context.manager.servedCount() == 0);
assert(!context.serving);
assert(!context.serve_self_leaves && !context.serve_self_proof);
assert(context.manager.pendingManifestJobs() == 0);
assert(context.manager.pendingServeJobs() == 0);
context.manager.serviceEgress();
assert(!context.manager.on_message(wire, length));
// Reconnect/retry is still possible after either allocation or flash-read failure.
allocation_countdown = -1; fail_read = false;
assert(ota_serve_self(context, 3));
assert(context.manager.servedCount() == 1);
assert(context.serving);
}
''')
self.compile_and_run(path, source, [("0",), ("1",), ("2",)])
def compile_and_run(self, path, source, scenarios=((),)):
flags = [] if os.name == "nt" else ["-fsanitize=address,undefined"]
tinf = path / "tinf.o"
subprocess.run([shutil.which("cc") or "gcc", "-DENABLE_OTA=1", *flags, "-c",
str(ROOT / "src/helpers/ota/OtaTinf.c"), "-o", str(tinf)], check=True)
sources = ["OtaContext.cpp", "OtaManager.cpp", "OtaProtocol.cpp",
"MotaContainer.cpp", "MerkleTree.cpp", "OtaDeflate.cpp"]
binary = path / "heap.exe"
result = subprocess.run([
"c++", "-std=c++17", *flags, "-DENABLE_OTA=1",
"-DOTA_HEAP_CONTEXT=1", "-DESP32_PLATFORM=1", "-DOTA_SEEDER_ONLY=1",
"-I", str(ROOT / "src"), "-I", str(ROOT / "test/mocks"),
str(source), *[str(ROOT / "src/helpers/ota" / name) for name in sources],
str(ROOT / "src/Utils.cpp"), str(tinf), "-o", str(binary),
], capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
for scenario in scenarios:
with self.subTest(scenario=scenario):
subprocess.run([str(binary), *scenario], check=True)
if __name__ == "__main__":