Fit V3 Full with NimBLE and bounded OTA source proofs

This commit is contained in:
mikecarper committed 2026-10-19 02:47:13 -07:00
1 parent 025eadb39f
commit a3f858f6f0
14 files changed
+803 -13

No files matched your search

+10
View File
@@ -209,6 +209,16 @@ uint16_t SerialMotaSource::responsePayloadCapacity() const {
return (uint16_t)(frame - 5u);
}
#if OTA_BOUNDED_SOURCE_PROOFS
bool SerialMotaSource::read_metadata(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len) {
if (!buf || !len || len > responsePayloadCapacity() || len > UINT32_MAX - off) return false;
uint8_t args[7];
args[0] = idx; wr_u32le(args + 1, off);
args[5] = (uint8_t)len; args[6] = (uint8_t)(len >> 8);
return txn(MS_OP_READ, args, sizeof(args), buf, len);
}
#endif
bool SerialMotaSource::read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len) {
if (!buf && len != 0) return false;
if (len > UINT32_MAX - off) return false;
+4
View File
@@ -24,6 +24,10 @@ public:
uint8_t count() override;
bool describe(uint8_t idx, MotaDesc& out) override;
bool read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len) override;
#if OTA_BOUNDED_SOURCE_PROOFS
uint16_t read_quantum() const override { return responsePayloadCapacity(); }
bool read_metadata(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len) override;
#endif
bool read_deflated_block(uint8_t idx, uint16_t block, uint8_t* buf,
uint16_t cap, uint16_t* len) override;
+98 -10
View File
@@ -6,6 +6,9 @@
#include "OtaDebug.h"
#include <string.h>
#include <stdlib.h> // malloc/free for the (transient) leaf-diff buffer
#if OTA_BOUNDED_SOURCE_PROOFS
#include <new>
#endif
namespace mesh {
namespace ota {
@@ -28,7 +31,12 @@ void OtaManager::invalidateDirectTransport() {
OtaManager::~OtaManager() {
free(_catalog_heap);
free(_leaves_buf);
#if OTA_BOUNDED_SOURCE_PROOFS
if (_source_proofs) _source_proofs->~BoundedSourceProof();
free(_source_proofs);
#else
free(_src_leaves);
#endif
free(_scratch);
}
@@ -89,6 +97,15 @@ void OtaManager::releaseSmallCatalog() {
_catalog_heap = nullptr;
}
#if OTA_BOUNDED_SOURCE_PROOFS
OtaManager::BoundedSourceProof* OtaManager::ensureSourceProofs() {
if (!_source_proofs) {
void* memory = malloc(sizeof(BoundedSourceProof));
if (memory) _source_proofs = new (memory) BoundedSourceProof;
}
return _source_proofs;
}
#else
uint8_t* OtaManager::ensureSourceLeaves(uint32_t bytes) {
if (!bytes || bytes > OTA_PROOFGEN_SCRATCH) return nullptr;
if (_src_leaves_capacity < bytes) {
@@ -102,6 +119,7 @@ uint8_t* OtaManager::ensureSourceLeaves(uint32_t bytes) {
}
return _src_leaves;
}
#endif
uint8_t* OtaManager::ensureScratch() {
if (!_scratch) {
@@ -125,15 +143,27 @@ void OtaManager::releaseColdBuffers() {
_serve_block_loaded = false;
_serve_wire_block = nullptr;
_serve_wire_len = 0;
#if OTA_BOUNDED_SOURCE_PROOFS
if (_source_proofs) _source_proofs->~BoundedSourceProof();
free(_source_proofs);
_source_proofs = nullptr;
#else
free(_src_leaves);
_src_leaves = nullptr;
_src_leaves_capacity = 0;
#endif
free(_scratch);
_scratch = nullptr;
}
void OtaManager::releaseIdleColdBuffers() {
if ((_src_leaves || _scratch) &&
if ((
#if OTA_BOUNDED_SOURCE_PROOFS
_source_proofs ||
#else
_src_leaves ||
#endif
_scratch) &&
(uint32_t)(_now_ms - _source_buffer_used_at) >= OTA_SOURCE_BUFFER_IDLE_MS) {
releaseColdBuffers();
}
@@ -359,10 +389,9 @@ OtaManager::ServeView* OtaManager::resolve(const uint8_t* mid) {
return loadSource(_serve[i]) ? &_srcv : nullptr;
}
// Load an external mota into the on-demand _srcv: read its manifest-minus-leaves + leaves[] from the
// source into RAM, parse, and wire a payload reader that streams blocks from the source on REQ. (The
// payload itself is NOT held in RAM - only the small head and the leaves, bounded by the configured
// proof-generation scratch size.)
// Load an external source's fixed manifest and wire its payload reader. Legacy
// builds also cache leaves here; bounded builds acquire proof metadata through
// cooperative egress steps, without delaying request admission on a full table.
bool OtaManager::loadSource(const ServeEntry& e) {
const MotaDesc& d = e.desc;
if (d.leaves_off < 8) return false;
@@ -373,18 +402,25 @@ bool OtaManager::loadSource(const ServeEntry& e) {
// Both images share these buffers. Even a failed read can overwrite part of
// the old manifest/leaves, so its cached MID must stop resolving immediately.
_srcv.valid = false;
#if !OTA_BOUNDED_SOURCE_PROOFS
uint8_t* src_leaves = ensureSourceLeaves(d.block_count * 4u);
if (!src_leaves) return false;
#endif
_source_buffer_used_at = _now_ms;
bool ok = e.src->read(e.src_idx, 8, _src_manifest, mfl);
if (!ok || !mota_parse_manifest(_src_manifest, mfl, _srcv.m)) return false;
if (_srcv.m.block_size() == 0 || _srcv.m.block_size() > OTA_MAX_BLOCK) return false;
if (memcmp(_srcv.m.merkle_root, d.mid, 4) != 0) return false; // descriptor/bytes disagree
if (_srcv.m.block_count != d.block_count) return false;
_srcv.m.manifest_start = _src_manifest;
#if OTA_BOUNDED_SOURCE_PROOFS
_srcv.m.leaves = nullptr;
_src_leaves_off = d.leaves_off;
#else
ok = e.src->read(e.src_idx, d.leaves_off, src_leaves, d.block_count * 4);
if (!ok) return false;
_srcv.m.manifest_start = _src_manifest;
_srcv.m.leaves = src_leaves;
#endif
_srcv.m.payload = nullptr;
_srcv.mfl = mfl;
_srcv_rdctx.src = e.src; _srcv_rdctx.idx = e.src_idx;
@@ -529,7 +565,11 @@ bool OtaManager::handleGetLeaves(const uint8_t* m, uint16_t n) {
GetLeavesMsg gl;
if (!decode_get_leaves(m, n, gl) || gl.want_mask == 0) return false;
ServeView* v = resolve(gl.manifest_id);
if (!v || !v->m.leaves || v->m.block_count == 0 ||
if (!v || (!v->m.leaves
#if OTA_BOUNDED_SOURCE_PROOFS
&& v != &_srcv
#endif
) || v->m.block_count == 0 ||
v->m.block_count > OTA_DIFF_MAX_BLOCKS) return false;
const uint32_t total = (v->m.block_count * 4 + OTA_LEAVES_FRAG - 1) / OTA_LEAVES_FRAG;
if (total == 0 || total > OTA_LEAVES_MAXFRAG) return false;
@@ -660,7 +700,11 @@ bool OtaManager::serviceManifestEgress(OtaReplyRouteValid route_valid, void* rou
const uint32_t mfl = job.leaves ? v->m.block_count * 4u : v->mfl;
const uint16_t fragment_size = job.leaves ? OTA_LEAVES_FRAG : OTA_MF_FRAG;
const uint32_t total = (mfl + fragment_size - 1u) / fragment_size;
if ((job.leaves && (!v->m.leaves || total > OTA_LEAVES_MAXFRAG)) || total > 16u) {
if ((job.leaves && ((!v->m.leaves
#if OTA_BOUNDED_SOURCE_PROOFS
&& v != &_srcv
#endif
) || total > OTA_LEAVES_MAXFRAG)) || total > 16u) {
popManifestJob();
return true;
}
@@ -689,7 +733,19 @@ bool OtaManager::serviceManifestEgress(OtaReplyRouteValid route_valid, void* rou
LeavesMsg message;
memcpy(message.manifest_id, v->m.merkle_root, 4);
message.frag_idx = fragment; message.frag_total = ftotal;
message.bytes = v->m.leaves + off; message.len = (uint16_t)len;
message.bytes = v->m.leaves ? v->m.leaves + off : nullptr;
#if OTA_BOUNDED_SOURCE_PROOFS
if (!message.bytes) {
BoundedSourceProof* workspace = ensureSourceProofs();
if (!workspace) { popManifestJob(); return true; }
_source_buffer_used_at = _now_ms;
const auto step = workspace->fragment(_srcv_rdctx.src, _srcv_rdctx.idx,
_src_leaves_off + off, (uint16_t)len, message.bytes);
if (step == BoundedSourceProof::Step::Pending) return true;
if (step == BoundedSourceProof::Step::Failed) { popManifestJob(); return true; }
}
#endif
message.len = (uint16_t)len;
wire_len = encode_leaves(wire, sizeof(wire), message);
} else {
ManifestMsg message;
@@ -699,6 +755,9 @@ bool OtaManager::serviceManifestEgress(OtaReplyRouteValid route_valid, void* rou
wire_len = encode_manifest(wire, sizeof(wire), message);
}
if (emit(wire, wire_len, false, &job.route)) {
#if OTA_BOUNDED_SOURCE_PROOFS
if (job.leaves && _source_proofs) _source_proofs->releaseFragment();
#endif
OTA_DBG("OTA: %s tx frag=%u/%u len=%u\n", job.leaves ? "LEAVES" : "MANIFEST",
(unsigned)fragment, (unsigned)ftotal, (unsigned)len);
const uint16_t bit = (uint16_t)(1u << fragment);
@@ -908,10 +967,25 @@ void OtaManager::serviceEgress(OtaReplyRouteValid route_valid, void* route_ctx)
}
if (job.pending_mask != 0) {
const bool was_loaded = _serve_block_loaded;
if (!_serve_block_loaded && !loadActiveServeBlock()) {
popServeJob();
return;
}
#if OTA_BOUNDED_SOURCE_PROOFS
// Warm proof metadata while DATA waits behind radio backpressure. Never
// combine this new read with the existing payload load in the same pass.
if (was_loaded && v == &_srcv) {
BoundedSourceProof* workspace = ensureSourceProofs();
if (!workspace || !workspace->bind(_srcv_rdctx.src, _srcv_rdctx.idx,
_src_leaves_off, v->m.block_count, v->m.merkle_root) ||
workspace->prepare() == BoundedSourceProof::Step::Failed) {
popServeJob(); return;
}
}
#else
(void)was_loaded;
#endif
const uint16_t fragment_data = job.route.direct ? OTA_DIRECT_FRAG_DATA :
(job.wire_v2 ? OTA_FRAG_DATA_V2 : OTA_FRAG_DATA);
const uint16_t valid_mask = frag_full_mask(
@@ -974,7 +1048,21 @@ void OtaManager::serviceEgress(OtaReplyRouteValid route_valid, void* route_ctx)
&& (int32_t)(_now_ms - job.proof_ready_at) < 0) return;
uint8_t proof[32 * 4];
uint8_t np = merkle_gen_proof(v->m.leaves, v->m.block_count, job.block, v->scratch, proof);
uint8_t np = 0;
#if OTA_BOUNDED_SOURCE_PROOFS
if (v == &_srcv) {
BoundedSourceProof* workspace = ensureSourceProofs();
if (!workspace || !workspace->bind(_srcv_rdctx.src, _srcv_rdctx.idx,
_src_leaves_off, v->m.block_count, v->m.merkle_root)) { popServeJob(); return; }
_source_buffer_used_at = _now_ms;
const auto step = workspace->proof(job.block, proof, np);
if (step == BoundedSourceProof::Step::Pending) return;
if (step == BoundedSourceProof::Step::Failed) { popServeJob(); return; }
} else
#endif
{
np = merkle_gen_proof(v->m.leaves, v->m.block_count, job.block, v->scratch, proof);
}
ProofMsg pm;
memcpy(pm.manifest_id, job.mid, 4);
pm.block_idx = job.block;
+16
View File
@@ -12,6 +12,9 @@
#include "MotaContainer.h"
#include "OtaSource.h"
#include "MerkleTree.h"
#if OTA_BOUNDED_SOURCE_PROOFS
#include "SourceMerkleProof.h"
#endif
// Transport-agnostic OTA session engine (docs/ota_protocol.md Section 5/Section 8). It SERVES a complete `.mota`
// (answering GET_MANIFEST / REQ) and/or FETCHES one into an OtaStore (verifying every block against
@@ -751,8 +754,17 @@ private:
// capacity limit. Device-encoder output needs at most one logical block;
// host-encoded output uses _serve_block. Cold buffers stay pinned until
// queued egress drains, then return to the heap at idle.
#if OTA_BOUNDED_SOURCE_PROOFS
using BoundedSourceProof = SourceMerkleProof<OTA_PROOFGEN_SCRATCH / 4, OTA_LEAVES_FRAG>;
static_assert(OTA_PROOFGEN_SCRATCH >= 4 && OTA_PROOFGEN_SCRATCH <= 16384 &&
sizeof(BoundedSourceProof) <= 1536,
"Update the bounded source workspace RAM budget in check_firmware_ram.py");
BoundedSourceProof* _source_proofs = nullptr;
uint32_t _src_leaves_off = 0;
#else
uint8_t* _src_leaves = nullptr;
uint32_t _src_leaves_capacity = 0;
#endif
uint8_t* _scratch = nullptr;
uint32_t _source_buffer_used_at = 0;
@@ -900,7 +912,11 @@ private:
bool _diffing = false; // leaves are in; diffing the seed a batch per tick
uint32_t _diff_idx = 0; // next block index to diff against the seed
#if OTA_BOUNDED_SOURCE_PROOFS
BoundedSourceProof* ensureSourceProofs();
#else
uint8_t* ensureSourceLeaves(uint32_t bytes);
#endif
uint8_t* ensureScratch();
void releaseColdBuffers();
void releaseIdleColdBuffers();
+20
View File
@@ -1,5 +1,25 @@
#pragma once
// Non-PSRAM Full Companions retain the complete source geometry without
// retaining every Merkle leaf in internal RAM. An explicit 0 keeps the legacy
// cache for matched qualification builds.
#ifndef OTA_BOUNDED_SOURCE_PROOFS
#if defined(ESP32_PLATFORM) && defined(COMPANION_RADIO_FULL) && !defined(BOARD_HAS_PSRAM) && \
(!defined(OTA_PROOFGEN_SCRATCH) || \
(OTA_PROOFGEN_SCRATCH >= 4 && OTA_PROOFGEN_SCRATCH <= 16384))
#define OTA_BOUNDED_SOURCE_PROOFS 1
#else
#define OTA_BOUNDED_SOURCE_PROOFS 0
#endif
#endif
#if OTA_BOUNDED_SOURCE_PROOFS != 0 && OTA_BOUNDED_SOURCE_PROOFS != 1
#error "OTA_BOUNDED_SOURCE_PROOFS must be 0 or 1"
#endif
#if OTA_BOUNDED_SOURCE_PROOFS && defined(OTA_PROOFGEN_SCRATCH) && \
(OTA_PROOFGEN_SCRATCH < 4 || OTA_PROOFGEN_SCRATCH > 16384)
#error "Bounded source proofs require 4..16384 leaf bytes; use legacy mode for another capacity"
#endif
// Qualified Full Companions stream host-provided firmware and need no permanent
// mOTA workspace. Apply the same policy to build.sh and direct PlatformIO builds.
#if defined(OTA_SEEDER_ONLY) && defined(COMPANION_RADIO_FULL) && \
+12
View File
@@ -3,6 +3,7 @@
#include <stdint.h>
#include <stddef.h>
#include "OtaFormat.h"
#include "OtaMemoryPolicy.h"
// Transport-agnostic "folder of firmware" abstraction (docs/ota_protocol.md Section 9). A node can RELAY
// `.mota` images it does not hold in flash - a user drops several `.mota` (different architectures) into
@@ -56,6 +57,17 @@ public:
// exactly `len` bytes were produced. May block on the transport (serial round-trip); implementations
// should keep active-transfer reads prompt because requested OTA traffic is primary traffic.
virtual bool read(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len) = 0;
#if OTA_BOUNDED_SOURCE_PROOFS
// Maximum bytes for one cooperative metadata step. Framed transports expose
// their negotiated single-response capacity so proof work never batches
// several timeout-bearing transactions into one mesh-loop pass.
virtual uint16_t read_quantum() const { return 128; }
// A framed source must reject a chunk that no longer fits after a negotiated
// capacity change, rather than silently splitting it into several requests.
virtual bool read_metadata(uint8_t idx, uint32_t off, uint8_t* buf, uint32_t len) {
return read(idx, off, buf, len);
}
#endif
// Optional transport-only raw-DEFLATE representation of one logical payload block. Implementations which
// can generate/cache it off-device return a stream strictly smaller than the raw block. The default keeps
// every existing source source-compatible and makes OtaManager fall back to raw 171-byte DATA.
+154
View File
@@ -0,0 +1,154 @@
#pragma once
#include "MerkleTree.h"
#include "OtaSource.h"
#include <string.h>
namespace mesh { namespace ota {
// A power-of-two group preserves the original tree after five reductions,
// including promotion of the final partial group. Retain group roots instead
// of the complete leaf table; a block's wire proof is its local proof followed
// by the group-root proof. No source/transport pointer owns storage here.
template <uint32_t MaximumLeaves, uint16_t FragmentCapacity>
class SourceMerkleProof {
public:
enum class Step : uint8_t { Pending, Ready, Failed };
static constexpr uint32_t GroupLeaves = 32;
static constexpr uint32_t GroupBytes = GroupLeaves * 4;
static constexpr uint32_t MaximumGroups = (MaximumLeaves + GroupLeaves - 1) / GroupLeaves;
static_assert(MaximumLeaves > 0 && MaximumLeaves <= UINT32_MAX / 4,
"invalid source proof capacity");
void reset() {
_source = nullptr;
_roots_ready = false;
_fragment_source = nullptr;
_fragment_used = 0;
}
bool bind(MotaSource* source, uint8_t index, uint32_t offset,
uint32_t count, const uint8_t root[4]) {
if (!source || !root || !count || count > MaximumLeaves ||
uint64_t(offset) + uint64_t(count) * 4 > UINT32_MAX) return false;
if (_source == source && _index == index && _offset == offset &&
_count == count && !memcmp(_root, root, 4)) return true;
_source = source; _index = index; _offset = offset; _count = count;
memcpy(_root, root, 4);
_groups = (count + GroupLeaves - 1) / GroupLeaves;
_warm_group = 0; _group_used = 0; _selected_group = UINT32_MAX;
_roots_ready = false;
return true;
}
// At most one source read, at most 32 leaves reduced after that read. The
// final root check is bounded by MaximumGroups (128 on ESP32).
Step prepare() {
if (!_source) return Step::Failed;
if (_roots_ready) return Step::Ready;
const uint32_t bytes = groupBytes(_warm_group);
if (!readPart(_source, _index, _offset + _warm_group * GroupBytes,
_group, bytes, _group_used)) return failProof();
if (_group_used != bytes) return Step::Pending;
if (!groupRoot(bytes, _roots + _warm_group * 4)) return failProof();
++_warm_group;
_group_used = 0;
if (_warm_group == _groups) {
_accumulator.reset();
for (uint32_t i = 0; i < _groups; ++i)
if (!_accumulator.add(_roots + i * 4)) return failProof();
uint8_t root[4];
if (!_accumulator.finish(root) || memcmp(root, _root, 4)) return failProof();
_roots_ready = true;
}
// Even the final warm read yields before reading the requested group.
return Step::Pending;
}
Step proof(uint32_t block, uint8_t output[128], uint8_t& length) {
length = 0;
if (!output || !_source || block >= _count) return Step::Failed;
const Step prepared = prepare();
if (prepared != Step::Ready) return prepared;
const uint32_t group = block / GroupLeaves;
if (_selected_group != group) {
_selected_group = group;
_group_used = 0;
}
const uint32_t bytes = groupBytes(group);
if (_group_used != bytes) {
if (!readPart(_source, _index, _offset + group * GroupBytes,
_group, bytes, _group_used)) return failProof();
if (_group_used != bytes) return Step::Pending;
uint8_t root[4];
if (!groupRoot(bytes, root) || memcmp(root, _roots + group * 4, 4))
return failProof();
}
length = merkle_gen_proof(_group, bytes / 4, block % GroupLeaves, nullptr, output);
length += merkle_gen_proof(_roots, _groups, group, nullptr, output + length * 4);
return Step::Ready;
}
// Getter fragments have an independent binding and buffer. A request for
// another MID cannot reset a partially warmed proof cache, and a rejected
// radio emit retains the exact bytes without another source transaction.
Step fragment(MotaSource* source, uint8_t index, uint32_t offset,
uint16_t bytes, const uint8_t*& output) {
output = nullptr;
if (!source || !bytes || bytes > FragmentCapacity ||
uint64_t(offset) + bytes > UINT32_MAX) return Step::Failed;
if (_fragment_source != source || _fragment_index != index ||
_fragment_offset != offset || _fragment_bytes != bytes) {
_fragment_source = source; _fragment_index = index;
_fragment_offset = offset; _fragment_bytes = bytes; _fragment_used = 0;
}
if (_fragment_used != bytes &&
!readPart(source, index, offset, _fragment, bytes, _fragment_used)) {
_fragment_source = nullptr;
return Step::Failed;
}
if (_fragment_used != bytes) return Step::Pending;
output = _fragment;
return Step::Ready;
}
void releaseFragment() { _fragment_source = nullptr; _fragment_used = 0; }
private:
static bool readPart(MotaSource* source, uint8_t index, uint32_t offset,
uint8_t* output, uint32_t bytes, uint32_t& used) {
uint32_t chunk = source->read_quantum();
if (!chunk) return false;
if (chunk > GroupBytes) chunk = GroupBytes;
if (chunk > bytes - used) chunk = bytes - used;
if (!source->read_metadata(index, offset + used, output + used, chunk)) return false;
used += chunk;
return true;
}
uint32_t groupBytes(uint32_t group) const {
const uint32_t remaining = _count - group * GroupLeaves;
return (remaining < GroupLeaves ? remaining : GroupLeaves) * 4;
}
bool groupRoot(uint32_t bytes, uint8_t root[4]) {
_accumulator.reset();
for (uint32_t i = 0; i < bytes; i += 4)
if (!_accumulator.add(_group + i)) return false;
return _accumulator.finish(root);
}
Step failProof() { _source = nullptr; _roots_ready = false; return Step::Failed; }
uint8_t _roots[MaximumGroups * 4];
uint8_t _group[GroupBytes];
uint8_t _fragment[FragmentCapacity];
MerkleAccumulator _accumulator;
MotaSource* _source = nullptr;
MotaSource* _fragment_source = nullptr;
uint32_t _offset = 0, _count = 0, _groups = 0;
uint32_t _warm_group = 0, _group_used = 0, _selected_group = UINT32_MAX;
uint32_t _fragment_offset = 0, _fragment_used = 0;
uint16_t _fragment_bytes = 0;
uint8_t _root[4] = {};
uint8_t _index = 0, _fragment_index = 0;
bool _roots_ready = false;
};
}} // namespace mesh::ota