Fix OTA recovery, radio2 replies, and preference persistence

Keep each deferred OTA response tied to the request's radio profile and generation. Separate requests from different radio sessions, discard stale queued responses, and retain the existing companion and infrastructure reply policies.

Reject staged captures with invalid file framing before adopting their state. Recover fixed internal-flash trailer bytes after interrupted checkpoints and handle small ESP32 delta trailers that share metadata sectors. Fail closed when install preflight cannot read a valid header, so hardware and automatic-update policy checks cannot be skipped.

Replace Companion preferences through verified file transactions and quarantine writes after incomplete loads on all platforms. Initialize serializer runtime fields before preferences snapshots copy them.

Add production-code fault regressions to CI for deferred routing, store recovery, install preflight, and preferences persistence. Validation: 1,510 native tests, 60 host checks, and six firmware targets across ESP32, nRF52, STM32, and RP2040.
This commit is contained in:
mikecarper committed 2026-09-15 10:45:04 -07:00
1 parent 39c3f7b25c
commit 016a1ed2ea
20 files changed
+1010 -44

No files matched your search

@@ -235,6 +235,57 @@ static void hybrid_state_is_one_shot_and_not_reopened_after_restart() {
CHECK(after_restart.staged_size() == 0u);
}
static void checkpointed_flash_resume_restores_deferred_trailer() {
using namespace mesh::ota;
reset_target_memory();
const uint32_t total = 3072u;
auto bytes = patterned_container(total);
std::memcpy(bytes.data() + total - 5u, MOTA_TRAILER, 5u);
{
OtaStoreFlashNrf52 store;
CHECK(store.plan_layout(false, 0x80000u, 205u, total - 210u, false));
CHECK(store.begin(total));
CHECK(!store.is_hybrid());
CHECK(store.write(0, bytes.data(), total));
store.checkpoint();
}
OtaStoreFlashNrf52 resumed;
CHECK(resumed.reopen());
CHECK(resumed.finalize());
std::vector<uint8_t> output(total);
CHECK(resumed.read(0, output.data(), total));
CHECK(output == bytes);
CHECK(std::memcmp(resumed.data(), bytes.data(), total) == 0);
}
static void dirty_flash_partial_checkpoint_finishes(bool reboot) {
using namespace mesh::ota;
reset_target_memory();
std::memset(reinterpret_cast<void*>(FLASH_MAP_START), 0xA5, FLASH_MAP_SIZE);
install_bootloader_capabilities();
const uint32_t total = 3u * MOTA_NRF52_FLASH_PAGE;
auto bytes = patterned_container(total);
std::memcpy(bytes.data() + total - 5u, MOTA_TRAILER, 5u);
OtaStoreFlashNrf52 original, reopened;
// Exercise the ordinary pure-flash backend rather than this build's optional
// preplanned volatile hybrid placement.
CHECK(original.begin(total));
CHECK(original.write(0, bytes.data(), 1024u));
CHECK(original.write(total - 5u, MOTA_TRAILER, 5u));
original.checkpoint();
OtaStoreFlashNrf52& active = reboot ? reopened : original;
if (reboot) CHECK(active.reopen());
uint8_t first[1024];
CHECK(active.read(0, first, sizeof(first)));
CHECK(std::memcmp(first, bytes.data(), sizeof(first)) == 0);
CHECK(active.write(1024u, bytes.data() + 1024u, total - 1024u - 5u));
CHECK(active.finalize());
std::vector<uint8_t> output(total);
CHECK(active.read(0, output.data(), total));
CHECK(output == bytes);
CHECK(std::memcmp(active.data(), bytes.data(), total) == 0);
}
int main() {
map_target_region(FLASH_MAP_START, FLASH_MAP_SIZE);
map_target_region(RAM_MAP_START, RAM_MAP_SIZE);
@@ -246,6 +297,12 @@ int main() {
std::puts("PASS: persistence failure stays closed");
hybrid_state_is_one_shot_and_not_reopened_after_restart();
std::puts("PASS: restart cannot reopen volatile suffix");
std::puts("4 OtaStoreFlashNrf52 hybrid lifecycle checks passed");
checkpointed_flash_resume_restores_deferred_trailer();
std::puts("PASS: checkpointed flash resume restores deferred trailer");
dirty_flash_partial_checkpoint_finishes(true);
std::puts("PASS: dirty flash partial checkpoint resumes and finishes");
dirty_flash_partial_checkpoint_finishes(false);
std::puts("PASS: dirty flash partial checkpoint continues without reboot");
std::puts("7 OtaStoreFlashNrf52 hybrid lifecycle checks passed");
return 0;
}
+3
View File
@@ -0,0 +1,3 @@
#pragma once
#include "esp_partition.h"
const esp_partition_t* esp_ota_get_next_update_partition(const esp_partition_t*);
+9
View File
@@ -0,0 +1,9 @@
#pragma once
#include <stddef.h>
#include <stdint.h>
typedef int esp_err_t;
constexpr esp_err_t ESP_OK = 0;
struct esp_partition_t { uint32_t size; };
esp_err_t esp_partition_read(const esp_partition_t*, size_t, void*, size_t);
esp_err_t esp_partition_write(const esp_partition_t*, size_t, const void*, size_t);
esp_err_t esp_partition_erase_range(const esp_partition_t*, size_t, size_t);
+178
View File
@@ -0,0 +1,178 @@
#include <helpers/ota/OtaManager.h>
#include <helpers/ota/OtaStoreFlashEsp32.h>
#include <helpers/ota/OtaByteIO.h>
#include "../../test_ota/mota_vectors.h"
#include <cassert>
#include <cstring>
#include <vector>
using namespace mesh::ota;
static std::vector<uint8_t> flash(65536, 0xFF);
static const esp_partition_t partition{65536};
const esp_partition_t* esp_ota_get_next_update_partition(const esp_partition_t*) {
return &partition;
}
esp_err_t esp_partition_read(const esp_partition_t*, size_t off, void* data, size_t n) {
if (off > flash.size() || n > flash.size() - off) return -1;
memcpy(data, flash.data() + off, n);
return ESP_OK;
}
esp_err_t esp_partition_write(const esp_partition_t*, size_t off, const void* data, size_t n) {
if (off > flash.size() || n > flash.size() - off) return -1;
const auto* bytes = static_cast<const uint8_t*>(data);
for (size_t i = 0; i < n; ++i) {
assert((flash[off + i] & bytes[i]) == bytes[i]);
flash[off + i] &= bytes[i];
}
return ESP_OK;
}
esp_err_t esp_partition_erase_range(const esp_partition_t*, size_t off, size_t n) {
if (off > flash.size() || n > flash.size() - off || off % 4096 || n % 4096) return -1;
memset(flash.data() + off, 0xFF, n);
return ESP_OK;
}
static bool send(void*, const uint8_t*, uint16_t, bool) { return true; }
// Folder/SD store size comes from the backing file, independently of its header.
struct StatSizeStore : OtaStoreRam<4096> {
bool fail_trailer_read = false;
bool reopen() override { return staged_size() != 0; }
bool read(uint32_t off, uint8_t* bytes, uint32_t n) const override {
return !(fail_trailer_read && off == staged_size() - 5)
&& OtaStoreRam<4096>::read(off, bytes, n);
}
};
static void resume_checks_file_envelope(int corruption) {
StatSizeStore store;
std::vector<uint8_t> bytes(SIM_MOTA_1K, SIM_MOTA_1K + SIM_MOTA_1K_LEN);
MotaManifest manifest;
assert(mota_parse(bytes.data(), bytes.size(), manifest));
uint8_t mid[4]; memcpy(mid, manifest.merkle_root, sizeof(mid));
if (corruption == 1) wr_u32le(bytes.data() + 4, bytes.size() + 1);
if (corruption == 2) bytes.back() ^= 1;
if (corruption == 3) store.fail_trailer_read = true;
if (corruption == 4) memset(bytes.data() + bytes.size() - 5, 0xFF, 5);
assert(store.begin(bytes.size()));
assert(store.write(0, bytes.data(), bytes.size()));
OtaManager receiver;
receiver.begin(SIM_TARGET_ID, send, nullptr);
receiver.set_fetch_store(&store);
const bool adopted = receiver.resumeStagedExplicit(mid, SIM_TARGET_ID);
assert(adopted == (corruption == 0));
if (adopted) {
for (unsigned guard = 0; guard < 100 && receiver.fetchState() == OtaManager::VERIFYING_STAGED; ++guard)
receiver.loop();
assert(receiver.fetchState() == OtaManager::COMPLETE);
}
}
static std::vector<uint8_t> delta_container(uint32_t total) {
uint32_t blocks = (total - 210u + 1023u) / 1024u;
const uint32_t payload_size = total - 210u - blocks * 4u;
assert((payload_size + 1023u) / 1024u == blocks);
std::vector<uint8_t> bytes(total);
memcpy(bytes.data(), MOTA_VEC, 8u + MOTA_MFL);
wr_u32le(bytes.data() + 4, total);
bytes[8 + 1] = 0;
bytes[8 + 56] = CODEC_DETOOLS_SEQUENTIAL;
wr_u32le(bytes.data() + 8 + 15, payload_size);
uint8_t* leaves = bytes.data() + 8u + MOTA_MFL;
uint8_t* payload = leaves + blocks * 4u;
for (uint32_t i = 0; i < payload_size; ++i) payload[i] = static_cast<uint8_t>(i * 31u);
for (uint32_t i = 0; i < blocks; ++i) {
const uint32_t n = std::min(1024u, payload_size - i * 1024u);
merkle_leaf(leaves + i * 4u, payload + i * 1024u, n);
}
merkle_root(bytes.data() + 8 + 20, leaves, blocks);
memcpy(bytes.data() + total - 5, MOTA_TRAILER, 5);
return bytes;
}
static void esp32_resume_preserves_container_trailer(bool finalized, bool full, uint32_t small_size = 0) {
std::fill(flash.begin(), flash.end(), 0xFF);
std::vector<uint8_t> bytes = small_size ? delta_container(small_size)
: std::vector<uint8_t>(MOTA_VEC, MOTA_VEC + MOTA_VEC_LEN);
if (!full) {
bytes[8 + 1] = 0;
bytes[8 + 56] = CODEC_DETOOLS_SEQUENTIAL;
}
MotaManifest manifest;
assert(mota_parse(bytes.data(), bytes.size(), manifest));
uint8_t mid[4]; memcpy(mid, manifest.merkle_root, sizeof(mid));
{
OtaStoreFlashEsp32 store;
assert(store.plan_layout(full, manifest.image_size,
manifest.payload - bytes.data(), manifest.payload_size, false));
assert(store.begin(bytes.size()));
// The manager writes the trailer separately from the metadata/payload.
assert(store.write(0, bytes.data(), bytes.size() - 5));
assert(store.write(bytes.size() - 5, bytes.data() + bytes.size() - 5, 5));
if (finalized) assert(store.finalize());
else store.checkpoint();
}
OtaStoreFlashEsp32 resumed;
OtaManager receiver;
receiver.begin(EXP_TARGET_ID, send, nullptr);
receiver.set_fetch_store(&resumed);
assert(receiver.resumeStagedExplicit(mid, EXP_TARGET_ID));
for (unsigned guard = 0; guard < 100 && receiver.fetchState() == OtaManager::VERIFYING_STAGED; ++guard)
receiver.loop();
assert(receiver.fetchState() == OtaManager::COMPLETE);
std::vector<uint8_t> output(bytes.size());
assert(resumed.read(0, output.data(), output.size()));
assert(output == bytes);
MotaManifest parsed;
assert(mota_parse(output.data(), output.size(), parsed));
assert(mota_check_payload(parsed));
}
static void dirty_partial_checkpoint_can_finish(bool reboot) {
std::fill(flash.begin(), flash.end(), 0xA5);
auto bytes = delta_container(5547);
MotaManifest manifest;
assert(mota_parse(bytes.data(), bytes.size(), manifest));
const uint32_t payload_off = manifest.payload - bytes.data();
const uint32_t leaves_off = manifest.leaves - bytes.data();
uint8_t mid[4]; memcpy(mid, manifest.merkle_root, sizeof(mid));
OtaStoreFlashEsp32 original, reopened;
assert(original.plan_layout(false, manifest.image_size, payload_off, manifest.payload_size, false));
assert(original.begin(bytes.size()));
assert(original.write(0, bytes.data(), leaves_off));
assert(original.write(payload_off, manifest.payload, 1024));
assert(original.write(leaves_off, manifest.leaves, 4));
assert(original.write(bytes.size() - 5, MOTA_TRAILER, 5));
original.checkpoint();
OtaStoreFlashEsp32& active = reboot ? reopened : original;
OtaManager receiver;
receiver.begin(EXP_TARGET_ID, send, nullptr);
receiver.set_fetch_store(&active);
if (reboot) {
assert(receiver.resumeStagedExplicit(mid, EXP_TARGET_ID));
receiver.loop();
assert(receiver.fetchState() == OtaManager::FETCHING);
assert(receiver.blocksHave() == 1);
}
for (uint32_t i = 1; i < manifest.block_count; ++i) {
const uint32_t n = std::min(1024u, manifest.payload_size - i * 1024u);
assert(active.write(payload_off + i * 1024u, manifest.payload + i * 1024u, n));
assert(active.write(leaves_off + i * 4u, manifest.leaves + i * 4u, 4));
}
assert(active.finalize());
receiver.reset_session();
assert(receiver.resumeStagedExplicit(mid, EXP_TARGET_ID));
receiver.loop();
assert(receiver.fetchState() == OtaManager::COMPLETE);
std::vector<uint8_t> output(bytes.size());
assert(active.read(0, output.data(), output.size()));
assert(output == bytes);
}
int main(int argc, char** argv) {
const int scenario = argc > 1 ? atoi(argv[1]) : 0;
if (scenario < 5) resume_checks_file_envelope(scenario);
else if (scenario < 9) esp32_resume_preserves_container_trailer((scenario & 1) == 0, scenario >= 7);
else if (scenario < 13) esp32_resume_preserves_container_trailer((scenario & 1) == 0, false,
scenario < 11 ? 2278 : 4098);
else dirty_partial_checkpoint_can_finish(scenario == 13);
}
@@ -29,6 +29,7 @@ class MemoryFS {
public:
std::map<std::string, std::vector<uint8_t>> files;
bool fail_write = false;
int fail_write_after = -1;
bool fail_read_open = false;
int fail_read_after = -1;
bool fail_remove = false;
@@ -71,6 +72,10 @@ inline int File::read(uint8_t* data, size_t size) {
inline size_t File::write(const uint8_t* data, size_t size) {
if (!fs_ || fs_->fail_write) return 0;
auto& bytes = fs_->files[path_];
if (fs_->fail_write_after >= 0) {
if (bytes.size() >= static_cast<size_t>(fs_->fail_write_after)) return 0;
size = std::min(size, static_cast<size_t>(fs_->fail_write_after) - bytes.size());
}
bytes.insert(bytes.end(), data, data + size); return size;
}
#define FILESYSTEM MemoryFS
@@ -0,0 +1,192 @@
"""Exercise real Companion preferences saves and startup recovery with I/O faults."""
from pathlib import Path
import subprocess
import tempfile
import unittest
from test_radio_receive_contract import method
ROOT = Path(__file__).resolve().parents[1]
HARNESS = r'''
#include <cassert>
#include <cstdlib>
#include <cstring>
#include <new>
#include <helpers/IdentityStore.h>
#include "ContactFileTransaction.h"
#if defined(STM32_PLATFORM) || defined(NRF52_PLATFORM)
#define ATOMIC_FILE_WRITER_IMPLEMENTATION
#include <helpers/AtomicFileWriter.h>
#endif
#include "examples/companion_radio/NodePrefs.h"
#define MESH_DEBUG_PRINTLN(...) ((void)0)
struct DataStore {
MemoryFS fs;
FILESYSTEM* _fs=&fs;
bool _prefs_load_incomplete=false, _primary_storage_unavailable=false;
bool _secondary_authority_unknown=false;
DataStore(){
#if defined(STM32_PLATFORM) || defined(NRF52_PLATFORM)
fs.rename_replaces=true;
#endif
}
bool loadPrefs(CompanionNodePrefs&, double&, double&);
bool loadPrefsInt(const char*, CompanionNodePrefs&, double&, double&);
bool savePrefs(const CompanionNodePrefs&, double, double);
};
File openRead(FILESYSTEM* fs,const char* path){return fs->open(path,"r");}
File openWrite(FILESYSTEM* fs,const char* path){return fs->open(path,"w");}
bool contactPathPresence(FILESYSTEM* fs,const char* path,bool& present){
present=fs->exists(path);return true;
}
@METHODS@
int main(int argc,char** argv){
const int scenario=argc>1?atoi(argv[1]):0;
DataStore store;CompanionNodePrefs original;
strcpy(original.node_name,"durable node");original.freq=910.525f;
original.ble_pin=876543;original.bluetooth_stealth_mode=2;
original.gps_interval=123;original.sf=7;original.cr=7;original.bw=62.5f;
assert(store.savePrefs(original,47.1,-122.2));
const auto disk=store.fs.files["/new_prefs"];
CompanionNodePrefs changed=original;changed.freq=910.1f;
if(scenario==0){
store.fs.fail_write=true;
assert(!store.savePrefs(changed,42.3,-121.2));
assert(store.fs.files["/new_prefs"]==disk);
store.fs.fail_write=false;store.fs.fail_write_after=17;
assert(!store.savePrefs(changed,42.3,-121.2));
assert(store.fs.files["/new_prefs"]==disk);
store.fs.fail_write_after=-1;
#if defined(STM32_PLATFORM) || defined(NRF52_PLATFORM)
const int rename_steps[]={1};
#else
const int rename_steps[]={1,2};
#endif
for(int fail_step : rename_steps){
store.fs.fail_rename=fail_step;
assert(!store.savePrefs(changed,42.3,-121.2));
assert(store.fs.files["/new_prefs"]==disk);
}
assert(store.savePrefs(changed,42.3,-121.2));
CompanionNodePrefs loaded;double lat=0,lon=0;
assert(store.loadPrefs(loaded,lat,lon));
assert(loaded.freq==changed.freq&&loaded.ble_pin==original.ble_pin);
assert(lat==42.3&&lon==-121.2);
} else if(scenario==1){
for(int fault : {0,1,2,3,4}){
for(const char* path : {"/new_prefs","/node_prefs"}){
DataStore boot;boot.fs.files[path]=disk;
CompanionNodePrefs live;double lat=1,lon=2;
if(fault==0)boot.fs.fail_read_open=true;
if(fault==1)boot.fs.fail_read_after=0;
if(fault==2)boot.fs.fail_read_after=84;
if(fault==3)boot.fs.fail_read_after=214;
if(fault==4)boot.fs.files[path].resize(87);
const auto durable=boot.fs.files[path];
assert(!boot.loadPrefs(live,lat,lon));
assert(live.node_name[0]==0&&lat==1&&lon==2);
boot.fs.fail_read_after=-1;boot.fs.fail_read_open=false;
assert(!boot.loadPrefs(live,lat,lon)); // quarantine lasts this boot
assert(!boot.savePrefs(live,lat,lon));
assert(boot.fs.files[path]==durable);
}
}
DataStore fresh;CompanionNodePrefs defaults;double lat=0,lon=0;
assert(fresh.loadPrefs(defaults,lat,lon));
assert(fresh.savePrefs(defaults,lat,lon));
} else if(scenario==2){
#if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM)
store.fs.fail_rename_from={"/new_prefs.tmp","/new_prefs.bak"};
assert(!store.savePrefs(changed,42.3,-121.2));
assert(!store.fs.exists("/new_prefs"));
assert(store.fs.files["/new_prefs.bak"]==disk);
DataStore failed_boot;failed_boot.fs=store.fs;
CompanionNodePrefs loaded;double lat=0,lon=0;
assert(!failed_boot.loadPrefs(loaded,lat,lon));
failed_boot.fs.fail_rename_from.clear();
assert(!failed_boot.loadPrefs(loaded,lat,lon));
assert(!failed_boot.savePrefs(changed,42.3,-121.2));
assert(failed_boot.fs.files["/new_prefs.bak"]==disk);
DataStore reboot;reboot.fs=failed_boot.fs;
assert(reboot.loadPrefs(loaded,lat,lon));
assert(loaded.freq==original.freq&&loaded.ble_pin==original.ble_pin);
assert(lat==47.1&&lon==-122.2);
#endif
} else if(scenario==3){
// Copy the real adapter-bearing preferences from dirty storage. This
// catches reads of uninitialized serializer bools under UBSan reliably.
alignas(CompanionNodePrefs) unsigned char memory[sizeof(CompanionNodePrefs)];
memset(memory,0xa5,sizeof(memory));
auto* prefs=new(memory)CompanionNodePrefs();
CompanionNodePrefs snapshot=*prefs;
assert(!snapshot.isDirty());
prefs->~CompanionNodePrefs();
} else if(scenario==4){
DataStore legacy;legacy.fs.files["/node_prefs"]=disk;
legacy.fs.fail_write_after=17;
CompanionNodePrefs loaded;double lat=0,lon=0;
assert(legacy.loadPrefs(loaded,lat,lon));
assert(loaded.freq==original.freq);
assert(legacy.fs.files["/node_prefs"]==disk&&!legacy.fs.exists("/new_prefs"));
legacy.fs.fail_write_after=-1;
assert(legacy.loadPrefs(loaded,lat,lon));
assert(!legacy.fs.exists("/node_prefs")&&legacy.fs.files["/new_prefs"]==disk);
}
}
'''
class CompanionPreferencesTransactionTests(unittest.TestCase):
def test_real_preferences_io_failures(self):
source = (ROOT / 'examples/companion_radio/DataStore.cpp').read_text()
methods = '\n'.join(method(source, signature) for signature in (
'bool DataStore::loadPrefs(', 'bool DataStore::loadPrefsInt(',
'bool DataStore::savePrefs('))
with tempfile.TemporaryDirectory() as directory:
work = Path(directory)
# Binary preference persistence never invokes JSON blob conversion.
# Satisfy those unrelated serializer links without crypto libraries.
(work / 'Utils.h').write_text('''#pragma once
#include <Arduino.h>
#include <cassert>
namespace mesh { struct Utils {
static void printHex(Stream&,const uint8_t*,size_t){assert(false);}
static void fromHex(uint8_t*,size_t,const char*){assert(false);}
}; }
''')
(work / 'platform_shim.h').write_text('''#include <cstdlib>
#include <cstdio>
inline char* utoa(unsigned int value,char* output,int base){
if(base!=10)abort();sprintf(output,"%u",value);return output;
}
''')
transaction = (ROOT / 'src/helpers/ContactFileTransaction.h').read_text()
(work / 'ContactFileTransaction.h').write_text(transaction.replace(
'#include "IdentityStore.h"', '#include <helpers/IdentityStore.h>'))
(work / 'test.cpp').write_text(HARNESS.replace('@METHODS@', methods))
for platform in ('ESP32_PLATFORM', 'RP2040_PLATFORM', 'STM32_PLATFORM', 'NRF52_PLATFORM'):
with self.subTest(platform=platform):
binary = work / 'test'
build = subprocess.run(['g++', '-std=c++17',
'-D'+platform+'=1', '-fsanitize=address,undefined', '-fno-sanitize-recover=all',
'-fno-pie', '-no-pie', '-include', str(work / 'platform_shim.h'),
'-I', str(work),
'-I', str(ROOT / 'test/fixtures/radio_profiles/mocks'),
'-I', str(ROOT / 'test/mocks'), '-I', str(ROOT / 'src'),
'-I', str(ROOT / 'src/helpers'), '-I', str(ROOT),
str(work / 'test.cpp'),
str(ROOT / 'src/helpers/ConfigSerializer.cpp'),
str(ROOT / 'src/helpers/DynamicConfigSerializer.cpp'),
str(ROOT / 'src/helpers/CommonRadioPrefs.cpp'),
str(ROOT / 'src/helpers/TxtDataHelpers.cpp'),
'-o', str(binary)], capture_output=True, text=True)
self.assertEqual(build.returncode, 0, build.stdout + build.stderr)
for scenario in range(5):
with self.subTest(scenario=scenario):
run = subprocess.run([str(binary), str(scenario)], capture_output=True, text=True)
self.assertEqual(run.returncode, 0, run.stdout + run.stderr)
if __name__ == '__main__':
unittest.main()
+144
View File
@@ -0,0 +1,144 @@
"""Exercise production install policy before handing a store to the applier."""
from pathlib import Path
import os
import subprocess
import tempfile
import unittest
from test_t096_full_memory import method
ROOT = Path(__file__).resolve().parents[1]
class OtaInstallPreflightTest(unittest.TestCase):
def test_storage_failures_cannot_skip_hardware_and_auto_install_policy(self):
context = (ROOT / "src/helpers/ota/OtaContext.h").read_text()
production = method(context, "bool hwMatches(") + "\n" + method(
context, "bool apply_fetched_impl(")
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
(path / "preflight.h").write_text(production)
source = path / "test.cpp"
source.write_text(r'''
#include <helpers/ota/MotaContainer.h>
#include <helpers/ota/OtaByteIO.h>
#include <helpers/ota/FirmwareInfo.h>
#include <cassert>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <vector>
#include "test_ota/mota_vectors.h"
using namespace mesh::ota;
struct Store {
std::vector<uint8_t> bytes{MOTA_VEC, MOTA_VEC + MOTA_VEC_LEN};
mutable int fail_at = -1;
uint32_t staged_size() const { return bytes.size(); }
bool read(uint32_t off, uint8_t* out, uint32_t len) const {
if (int(off) == fail_at) { fail_at = -1; return false; }
if (off > bytes.size() || len > bytes.size() - off) return false;
memcpy(out, bytes.data() + off, len); return true;
}
const uint8_t* data() const { return bytes.data(); }
};
static int apply_calls = 0;
static bool apply_ok = true, self_ok = true;
static uint32_t running_version = EXP_FW_VERSION - 1;
static bool ota_self_firmware(SelfFwInfo& self) {
self.valid = self_ok; self.fw_version = running_version; return self_ok;
}
// Hardware apply is the boundary under test: a failed preflight must never
// reach it, regardless of whether a second store read might now succeed.
static bool ota_apply_detools_mota(Store&, int, int&, char*) {
++apply_calls; return apply_ok;
}
static bool ota_apply_mota_nrf52(Store&, int, int&, char*) {
++apply_calls; return apply_ok;
}
struct Context {
Store fetch_store;
int allow = 0, apply_st = 0;
bool fetch_to_folder = false, apply_pending = false, bootloader_apply_pending = false;
char hw_id[33] = "TESTHW";
#include "preflight.h"
};
int main(int argc, char** argv) {
assert(argc == 2);
const int scenario = atoi(argv[1]);
Context context;
char reply[160] = {};
bool trusted_auto = false;
bool expected = false;
if (scenario == 0) {
strcpy(context.hw_id, "WRONG_BOARD"); context.fetch_store.fail_at = 0;
} else if (scenario == 1) {
context.fetch_store.fail_at = 0; trusted_auto = true; // unsigned package
} else if (scenario == 2) {
context.fetch_store.bytes[9] |= MFLAG_SIGNED;
running_version = EXP_FW_VERSION; trusted_auto = true; context.fetch_store.fail_at = 0;
} else if (scenario == 3) {
context.fetch_store.bytes.resize(8);
} else if (scenario == 4) {
context.fetch_store.bytes[0] ^= 1;
} else if (scenario == 5) {
context.fetch_store.bytes[4] ^= 1;
} else if (scenario == 6) {
context.fetch_store.fail_at = 8;
} else if (scenario == 7) {
strcpy(context.hw_id, "WRONG_BOARD");
} else if (scenario == 8) {
trusted_auto = true; // unsigned is never eligible for auto-install
} else if (scenario == 9) {
context.fetch_store.bytes[9] |= MFLAG_SIGNED;
trusted_auto = true; running_version = EXP_FW_VERSION;
} else if (scenario == 10) {
context.fetch_store.bytes[9] |= MFLAG_SIGNED;
trusted_auto = true; self_ok = false;
} else if (scenario == 11) {
context.fetch_store.bytes[9] |= MFLAG_SIGNED;
trusted_auto = true; expected = true;
} else if (scenario == 12) {
running_version = EXP_FW_VERSION + 1; expected = true; // manual override retained
} else if (scenario == 13) {
context.fetch_to_folder = true;
} else if (scenario == 14) {
context.hw_id[0] = 0; expected = true; // existing unknown-hardware policy
} else if (scenario == 15) {
apply_ok = false;
}
assert(context.apply_fetched_impl(nullptr, trusted_auto, reply) == expected);
assert(context.apply_pending == expected);
assert(!context.bootloader_apply_pending);
assert(apply_calls == (expected || scenario == 15 ? 1 : 0));
if (scenario == 0) {
// Once I/O recovers, retry still evaluates hardware compatibility.
assert(!context.apply_fetched_impl(nullptr, false, reply));
assert(apply_calls == 0 && strstr(reply, "incompatible hardware"));
strcpy(context.hw_id, "TESTHW");
assert(context.apply_fetched_impl(nullptr, false, reply));
assert(apply_calls == 1 && context.apply_pending);
}
}
''')
for platform in ("esp32", "nrf52_qspi"):
defines = (["-DESP32_PLATFORM", "-DOTA_FLASH_STORE"]
if platform == "esp32" else ["-DNRF52_PLATFORM", "-DOTA_QSPI_STORE"])
binary = path / platform
flags = [] if os.name == "nt" else ["-fsanitize=address,undefined"]
result = subprocess.run([
"c++", "-std=c++17", "-ffunction-sections", "-fdata-sections",
"-Wl,--gc-sections", *flags, *defines,
"-I", str(ROOT / "src"), "-I", str(ROOT / "test"),
"-I", str(ROOT / "test/mocks"),
str(source), str(ROOT / "src/helpers/ota/MotaContainer.cpp"),
"-o", str(binary),
], text=True, capture_output=True)
self.assertEqual(result.returncode, 0, result.stderr)
for scenario in range(16):
with self.subTest(platform=platform, scenario=scenario):
result = subprocess.run([str(binary), str(scenario)], text=True, capture_output=True)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
if __name__ == "__main__":
unittest.main()
+2 -2
View File
@@ -51,10 +51,10 @@ class OtaStoreFlashNrf52HybridTest(unittest.TestCase):
)
self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr)
self.assertIn(
"4 OtaStoreFlashNrf52 hybrid lifecycle checks passed",
"7 OtaStoreFlashNrf52 hybrid lifecycle checks passed",
checked.stdout,
)
self.assertEqual(checked.stdout.count("PASS:"), 4)
self.assertEqual(checked.stdout.count("PASS:"), 7)
if __name__ == "__main__":
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""Run OTA staged-file validation and real ESP32 flash resume on the host."""
from pathlib import Path
import os
import shutil
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
FIXTURE = ROOT / "test/fixtures/ota_store_resume"
class OtaStoreResumeTest(unittest.TestCase):
def test_container_envelope_and_flash_resume(self):
with tempfile.TemporaryDirectory(prefix="ota-resume-") as directory:
path = Path(directory)
flags = [] if os.name == "nt" else ["-fsanitize=address,undefined"]
tinf = path / "tinf.o"
subprocess.run([shutil.which("cc") or "gcc", "-DENABLE_OTA=1", *flags,
"-c", str(ROOT / "src/helpers/ota/OtaTinf.c"),
"-o", str(tinf)], check=True)
sources = ["OtaManager.cpp", "OtaProtocol.cpp", "MotaContainer.cpp",
"MerkleTree.cpp", "OtaDeflate.cpp", "OtaStoreFlashEsp32.cpp"]
binary = path / "resume.exe"
result = subprocess.run([
"c++", "-std=c++17", *flags, "-DENABLE_OTA=1", "-DESP32_PLATFORM=1",
"-DOTA_FLASH_STORE=1", "-I", str(FIXTURE / "mocks"),
"-I", str(ROOT / "src"), "-I", str(ROOT / "test/mocks"),
str(FIXTURE / "test.cpp"),
*[str(ROOT / "src/helpers/ota" / name) for name in sources],
str(ROOT / "src/Utils.cpp"), str(tinf), "-o", str(binary),
], capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
for scenario in range(15):
with self.subTest(scenario=scenario):
subprocess.run([str(binary), str(scenario)], check=True, timeout=10)
if __name__ == "__main__":
unittest.main()
+227 -1
View File
@@ -12,6 +12,223 @@ ROOT = Path(__file__).resolve().parents[1]
class ReplyTxIntegrationTest(unittest.TestCase):
def test_deferred_responses_keep_the_request_profile(self):
mesh_source = (ROOT / 'src/Mesh.cpp').read_text()
dispatcher = (ROOT / 'src/Dispatcher.cpp').read_text()
harness = r'''
#include <cassert>
#include <vector>
#include <RadioProfiles.h>
#include <Packet.h>
#include <helpers/ota/OtaManager.h>
#include <helpers/ota/OtaProtocol.h>
#include "mota_vectors.h"
namespace mesh {
namespace ota {
struct OtaContext { OtaManager& manager; };
static OtaContext* active;
OtaContext* ota_context_if_active() { return active; }
OtaContext& ota_ctx() { return *active; }
}
struct Manager {
int free=40;
int getFreeCount() { return free; }
};
static int queuedPacedOtaResponses(Manager*) { return 0; }
static constexpr int OTA_EGRESS_QUEUE_CREDIT=4, OTA_EGRESS_MIN_FREE=4;
@PACED@
struct Radio {
RadioProfiles config;
const RadioProfiles* profiles() const { return &config; }
};
struct Mesh {
Manager manager; Manager* _mgr=&manager;
Radio radio; Radio* _radio=&radio;
ota::OtaManager ota_manager;
ota::OtaContext context{ota_manager};
Packet packet;
std::vector<uint8_t> masks;
uint8_t receive_profile=0;
bool receiving=false;
Mesh() { ota::active=&context; }
bool isAnyTempRadioActive() { return true; }
Packet* createOtaPacket(const uint8_t*,uint16_t) {
packet=Packet(); packet.header=PAYLOAD_TYPE_OTA << PH_TYPE_SHIFT;
packet.radio_profile=receive_profile; packet.radio_local=!receiving;
packet.radio_generation=receiving ? radio.config.generation[receive_profile] : 0;
return &packet;
}
void releasePacket(Packet*) {}
bool sendOtaFlood(Packet* p) { masks.push_back(getTransmitProfileMask(p)); return true; }
uint8_t getTransmitProfileMask(const Packet*) const;
static bool otaSendAdapter(void*,const uint8_t*,uint16_t,bool);
void service() { @SERVICE@ }
};
@MASK@
@ADAPTER@
}
int main() {
mesh::Mesh node;
node.radio.config.primary_temporary=true;
node.radio.config.secondary_temporary=true;
node.radio.config.secondary.mode=mesh::RadioProfileMode::RxTx;
node.radio.config.cross=mesh::RadioCrossMode::Off;
node.ota_manager.begin(0,node.otaSendAdapter,&node);
assert(node.ota_manager.serve(MOTA_VEC,sizeof(MOTA_VEC)));
mesh::ota::ReqMsg request{};
memcpy(request.manifest_id,EXP_MERKLE_ROOT,4);
request.block_idx=0; request.want_mask=1;
uint8_t wire[MAX_PACKET_PAYLOAD];
const auto length=mesh::ota::encode_req(wire,sizeof(wire),request);
node.receiving=true; node.receive_profile=1;
mesh::ota::OtaReplyRoute route;
route.profile=1; route.generation=node.radio.config.generation[1];
assert(node.ota_manager.on_message(wire,length,route));
assert(node.masks.empty()); // DATA is deliberately deferred until later
node.receiving=false; node.receive_profile=0;
node.manager.free=4; node.service(); // backpressure must retain origin
assert(node.ota_manager.pendingServeJobs()==1 && node.masks.empty());
node.manager.free=40;
node.service();
assert(node.masks.size()==1 && node.masks[0]==2);
assert(node.packet.radio_profile==1 && !node.packet.radio_local);
assert(node.packet.radio_generation==route.generation);
assert(node.ota_manager.replyRoute().profile==0xFF);
// Infrastructure reply-both still overrides AUTO origin selection, including
// a receive-only secondary when the configured reply policy forces it.
node.ota_manager.clearPendingEgress(); node.masks.clear();
node.radio.config.reply_tx=mesh::RADIO_TX_BOTH;
node.radio.config.reply_force=true;
node.radio.config.secondary.mode=mesh::RadioProfileMode::Rx;
assert(node.ota_manager.on_message(wire,length,route));
node.service();
assert(node.masks.size()==1 && node.masks[0]==3);
node.radio.config.reply_tx=mesh::RADIO_TX_AUTO;
node.radio.config.reply_force=false;
node.radio.config.secondary.mode=mesh::RadioProfileMode::RxTx;
// Identical block requests heard on separate radio domains must not merge.
node.ota_manager.clearPendingEgress(); node.masks.clear();
assert(node.ota_manager.on_message(wire,length,route));
auto primary=route; primary.profile=0;
assert(node.ota_manager.on_message(wire,length,primary));
assert(node.ota_manager.pendingServeJobs()==2);
node.service();
mesh::ota::ReqProofMsg proof{};
memcpy(proof.manifest_id,EXP_MERKLE_ROOT,4); proof.block_idx=0;
uint8_t proof_wire[MAX_PACKET_PAYLOAD];
const auto proof_len=mesh::ota::encode_req_proof(proof_wire,sizeof(proof_wire),proof);
assert(node.ota_manager.on_message(proof_wire,proof_len,primary));
assert(node.ota_manager.pendingServeJobs()==2); // primary proof merges only with primary DATA
assert(node.ota_manager.on_message(proof_wire,proof_len,route));
node.service(); // secondary proof
node.service(); // primary DATA
node.service(); // primary proof
assert(node.ota_manager.pendingServeJobs()==0);
assert((node.masks==std::vector<uint8_t>{2,2,1,1}));
// Reconfiguration retires the stale job even when it was waiting on proof.
node.masks.clear();
assert(node.ota_manager.on_message(wire,length,route));
node.service();
const auto sent=node.ota_manager.packetsSent();
++node.radio.config.generation[1];
auto replacement=route; ++replacement.generation;
assert(node.ota_manager.on_message(wire,length,replacement));
assert(node.ota_manager.pendingServeJobs()==2);
node.service();
assert(node.ota_manager.pendingServeJobs()==1);
assert(node.ota_manager.packetsSent()==sent); // discarded is not reported as sent
node.service();
assert(node.masks.size()==2 && node.packet.radio_generation==replacement.generation);
node.radio.config.secondary.mode=mesh::RadioProfileMode::Off;
node.service();
assert(node.ota_manager.pendingServeJobs()==0);
node.radio.config.secondary.mode=mesh::RadioProfileMode::RxTx;
// The paced manifest queue also separates origins and expires stale sessions.
node.masks.clear();
mesh::ota::GetManifestMsg manifest{};
memcpy(manifest.manifest_id,EXP_MERKLE_ROOT,4); manifest.want_mask=1;
uint8_t manifest_wire[MAX_PACKET_PAYLOAD];
const auto manifest_len=mesh::ota::encode_get_manifest(manifest_wire,sizeof(manifest_wire),manifest);
assert(node.ota_manager.on_message(manifest_wire,manifest_len,route));
assert(node.ota_manager.on_message(manifest_wire,manifest_len,replacement));
assert(node.ota_manager.on_message(manifest_wire,manifest_len,primary));
assert(node.ota_manager.pendingManifestJobs()==3);
node.service(); // stale route is discarded even before its pacing deadline
assert(node.ota_manager.pendingManifestJobs()==2 && node.masks.empty());
node.ota_manager.set_clock(1000000);
node.service(); node.service();
assert((node.masks==std::vector<uint8_t>{2,1}));
assert(node.ota_manager.pendingManifestJobs()==0);
// Changing primary also retires its old request before any fragment is sent.
node.masks.clear();
assert(node.ota_manager.on_message(wire,length,primary));
++node.radio.config.generation[0];
++primary.generation;
assert(node.ota_manager.on_message(wire,length,primary));
node.service();
assert(node.masks.empty() && node.ota_manager.pendingServeJobs()==1);
node.service();
assert(node.masks.size()==1 && node.masks[0]==1);
assert(node.packet.radio_generation==primary.generation);
node.ota_manager.clearPendingEgress();
// LEAVES remains an immediate response and inherits the live receive scope.
// Deferred affinity cannot bleed into that callback or a later local send.
node.masks.clear();
mesh::ota::GetLeavesMsg leaves{};
memcpy(leaves.manifest_id,EXP_MERKLE_ROOT,4); leaves.want_mask=1;
uint8_t leaves_wire[MAX_PACKET_PAYLOAD];
const auto leaves_len=mesh::ota::encode_get_leaves(leaves_wire,sizeof(leaves_wire),leaves);
node.receiving=true; node.receive_profile=1;
node.ota_manager.on_message(leaves_wire,leaves_len,replacement);
assert(node.masks.size()==1 && node.masks[0]==2);
assert(node.packet.radio_reply && !node.packet.radio_local);
node.receiving=false; node.receive_profile=0;
const uint8_t data=mesh::ota::OTA_DATA;
assert(node.otaSendAdapter(&node,&data,1,false));
assert(node.masks.size()==2 && node.masks[1]==1 && node.packet.radio_local);
// A regular primary and temporary secondary stay isolated unless cross is on.
node.radio.config.primary_temporary=false;
node.masks.clear();
assert(node.ota_manager.on_message(wire,length,replacement));
node.service();
assert(node.masks.size()==1 && node.masks[0]==2);
node.ota_manager.clearPendingEgress();
node.radio.config.cross=mesh::RadioCrossMode::On;
assert(node.ota_manager.on_message(wire,length,replacement));
node.service();
assert(node.masks.size()==2 && node.masks[1]==3);
}
'''
harness = harness.replace('@PACED@', method(mesh_source, 'static bool isPacedOtaResponse('))
harness = harness.replace('@ADAPTER@', method(mesh_source, 'bool Mesh::otaSendAdapter('))
harness = harness.replace('@MASK@', method(dispatcher, 'uint8_t Dispatcher::getTransmitProfileMask(')
.replace('Dispatcher::', 'Mesh::'))
start = mesh_source.index('ota::ota_ctx().manager.serviceEgress(')
service = mesh_source[start:mesh_source.index('}, this);', start) + len('}, this);')]
harness = harness.replace('@SERVICE@', service)
self.assertIn('context->manager.on_message(pkt->payload, pkt->payload_len, route)', mesh_source)
with tempfile.TemporaryDirectory() as directory:
path = Path(directory)
source = path / 'routing.cpp'; source.write_text(harness)
binary = path / 'routing'
sources = ['OtaManager.cpp', 'OtaProtocol.cpp', 'MotaContainer.cpp', 'MerkleTree.cpp']
result = subprocess.run([os.environ.get('CXX', 'g++'), '-std=c++17',
'-I', str(ROOT / 'src'), '-I', str(ROOT / 'test/mocks'),
'-I', str(ROOT / 'test/test_ota'), str(source),
*[str(ROOT / 'src/helpers/ota' / name) for name in sources],
str(ROOT / 'src/Utils.cpp'), str(ROOT / 'src/Packet.cpp'), '-o', str(binary)],
capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
result = subprocess.run([str(binary)], capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
def test_async_ota_adapter_marks_only_response_types_and_retains_backpressure(self):
source = (ROOT / 'src/Mesh.cpp').read_text()
adapter = method(source, 'bool Mesh::otaSendAdapter(')
@@ -22,7 +239,16 @@ class ReplyTxIntegrationTest(unittest.TestCase):
#include <initializer_list>
#include <helpers/ota/OtaFormat.h>
namespace mesh {
struct Packet { bool radio_reply=false; };
struct Packet {
bool radio_reply=false, radio_local=true;
uint8_t radio_profile=0, radio_origin=0;
uint32_t radio_generation=0, radio_origin_generation=0;
};
namespace ota {
struct Route { uint32_t generation=0; uint8_t profile=0xFF; };
struct Context { struct Manager { Route replyRoute() const { return {}; } } manager; };
Context* ota_context_if_active() { return nullptr; }
}
struct Manager { int free=10; int getFreeCount() { return free; } };
static int queued;
static int queuedPacedOtaResponses(Manager*) { return queued; }