Files
MeshCore/test/test_ota_apply_delivery.py
T
mikecarper 74d1ad27bc Fix OTA retry accounting and radio delivery races
Keep catalog, manifest and leaf requests pending when the local transmit
queue rejects them instead of spending their network retry allowance.

Keep the packet selected before CAD so work becoming due during sensing
cannot bypass its own channel or pacing checks. Retire cancelled radio
retries before waiting for BUSY hardware or their retry deadline.

Wait for active transmission completion before applying firmware. Stop
admitting new transmissions after the apply drain deadline so busy queues
cannot postpone reboot indefinitely, and keep those deadlines armed across
millis rollover. Retain the existing persistence-flush backoff.

Add failing-before/fixed-after regression coverage for all four issues,
including the production OTA apply and TX guards in CI.

Validation: 1,496 native tests, 42 Python integration tests, and five
firmware builds passed across ESP32-S3, nRF52, and STM32 targets.
2026-09-15 06:45:28 -07:00

123 lines
4.8 KiB
Python

"""Run the production deferred OTA apply and TX guard with an active radio."""
from pathlib import Path
import shutil
import subprocess
import tempfile
import unittest
from test_ota_dev_reboot_guard import function_body
ROOT = Path(__file__).resolve().parents[1]
HARNESS = r'''
#include <cassert>
#include <cstdint>
#include <helpers/ota/OtaTiming.h>
#define PAYLOAD_TYPE_OTA 0x0c
#define TOTAL_FLOOD_RETRY_SLOTS 1
namespace mesh {
namespace ota {
struct OtaContext {
bool apply_pending=false, bootloader_apply_pending=false;
uint32_t apply_at=0, apply_hard=0;
};
OtaContext context;
OtaContext* active=&context;
int applications=0, bootloaders=0;
OtaContext& ota_ctx() { return context; }
OtaContext* ota_context_if_active() { return active; }
void ota_reboot_to_apply() { ++applications; }
void ota_reboot_to_bootloader_update() { ++bootloaders; }
}
struct Packet { uint8_t getPayloadType() const { return 0; } };
struct Clock { uint32_t now=0; uint32_t getMillis() const { return now; } };
struct Manager { int queued=0; int getOutboundTotal() const { return queued; } };
struct Mesh {
Clock clock; Clock* _ms=&clock;
Manager manager; Manager* _mgr=&manager;
Packet* outbound=nullptr;
bool flush_ok=true;
int _active_flood_retry_count=0;
struct { bool active=false, queued=false; Packet* packet=nullptr; uint8_t retry_attempts_sent=0; } _flood_retries[1];
uint8_t getEligibleFloodRetryMaxAttempts(const Packet*) const { return 0; }
bool isAnyTempRadioActive() const { return true; }
bool hasOutbound() const { return outbound != nullptr; }
bool millisHasNowPassed(uint32_t deadline) const { return (int32_t)(clock.now-deadline)>0; }
uint32_t futureMillis(uint32_t ms) const { return clock.now+ms; }
bool prepareForOtaReboot() { return flush_ok; }
bool allowPacketTransmit(const Packet* packet) const { @ALLOW@ }
void serviceApply() {
ota::OtaContext& oc=ota::ota_ctx();
if (oc.apply_pending) { @APPLY@ }
}
};
}
int main() {
using namespace mesh;
Mesh node;
Packet packet;
auto reset = [&]() {
ota::context={}; ota::active=&ota::context;
ota::applications=ota::bootloaders=0;
node.clock.now=0; node.outbound=nullptr;
node.manager.queued=0; node.flush_ok=true;
ota::context.apply_pending=true;
};
reset(); node.serviceApply();
node.clock.now=2000; node.outbound=&packet;
node.serviceApply();
assert(ota::applications==0); // dequeued is not the same as finished transmitting
node.outbound=nullptr; node.serviceApply();
assert(ota::applications==1);
reset(); node.serviceApply();
node.clock.now=15001; node.outbound=&packet; node.manager.queued=99;
assert(!node.allowPacketTransmit(&packet)); // busy traffic cannot keep extending reboot
node.serviceApply(); assert(ota::applications==0);
node.outbound=nullptr; node.serviceApply();
assert(ota::applications==1);
reset(); node.serviceApply();
node.clock.now=15001; node.flush_ok=false;
node.serviceApply(); assert(ota::applications==0);
assert(node.allowPacketTransmit(&packet)); // storage backoff releases the TX gate
node.flush_ok=true; node.clock.now+=1001; node.serviceApply();
assert(ota::applications==1);
reset(); node.clock.now=UINT32_MAX-1499; node.serviceApply();
assert(ota::context.apply_at!=0); // exact rollover must not re-arm forever
node.clock.now=1; node.outbound=&packet; node.serviceApply();
assert(ota::applications==0);
node.clock.now=2; node.outbound=nullptr;
ota::context.bootloader_apply_pending=true; node.serviceApply();
assert(ota::bootloaders==1 && ota::applications==0);
ota::context.apply_pending=false;
assert(node.allowPacketTransmit(&packet));
ota::active=nullptr;
assert(node.allowPacketTransmit(&packet));
assert(ota::active==nullptr); // idle inspection must not borrow an OTA workspace
}
'''
class OtaApplyDeliveryTest(unittest.TestCase):
def test_reply_completion_busy_queue_flush_failure_and_rollover(self):
source = (ROOT / "src/Mesh.cpp").read_text()
maintenance = function_body(source, "void __attribute__((noinline)) Mesh::serviceLoopMaintenance()")
apply = function_body(maintenance, "if (oc.apply_pending)")
allow = function_body(source, "bool Mesh::allowPacketTransmit(")
compiler = shutil.which("g++") or shutil.which("clang++")
self.assertIsNotNone(compiler)
with tempfile.TemporaryDirectory() as directory:
cpp = Path(directory) / "apply.cpp"
binary = Path(directory) / "apply"
cpp.write_text(HARNESS.replace("@APPLY@", apply).replace("@ALLOW@", allow))
subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", "-DENABLE_OTA=1",
"-I", str(ROOT / "src"), str(cpp), "-o", str(binary)], check=True)
subprocess.run([str(binary)], check=True)
if __name__ == "__main__":
unittest.main()