Bound failed RX retries without blocking power-saving sleep

This commit is contained in:
mikecarper committed 2026-10-16 01:27:24 -07:00
1 parent 862260de17
commit 339079c439
14 files changed
+463 -12

No files matched your search

+177
View File
@@ -0,0 +1,177 @@
struct RetryClock : mesh::MillisecondClock {
unsigned long getMillis() override { return now_ms; }
};
struct RetryDispatcher : mesh::Dispatcher {
int interval = 8000;
bool adaptive = true;
RetryDispatcher(RadioLibWrapper& r, RetryClock& c, StaticPoolPacketManager& m)
: Dispatcher(r, c, m) {}
mesh::DispatcherAction onRecvPacket(mesh::Packet*) override { return ACTION_RELEASE; }
int getAGCResetInterval() const override { return interval; }
bool useAdaptiveAGC() const override { return adaptive; }
uint32_t getCADFailRetryDelay() const override { return 5000; }
using Dispatcher::getNextQueueWakeDelay;
};
static uint32_t retryDelay(const RadioLibWrapper& w) {
uint32_t delay = UINT32_MAX;
assert(w.getReceiveRetryDelay(now_ms, delay));
return delay;
}
static void persistentFailure() {
uint8_t bytes[8];
for (unsigned mode : {0U, 1U, 2U}) {
RadioLibWrapper w; RetryClock clock; StaticPoolPacketManager manager{4};
RetryDispatcher d(w, clock, manager);
now_ms = 1000; d.begin(); selectMode(w, mode); w.arm_result = -5;
now_ms = 31000; // application setup may outlast the sampling timeout
unsigned sleep_windows = 0;
for (unsigned i = 0; i <= 60000; ++i, ++now_ms) {
d.loop();
assert(!w.isCalibratingNoiseFloor());
uint32_t delay = 0;
assert(d.getNextQueueWakeDelay(delay));
sleep_windows += mesh::power::powerSaveSleepSeconds(delay) > 0;
assert(w.arms <= 37); // initial ramp followed by at most one attempt/2s
}
assert(w.arms == 36 && sleep_windows > 20000);
assert(w._nf_refresh_requested && w._nf_calib_deadline == 0 && w.reads == 0);
assert(w.getRecoveryCount() >= 7);
std::printf("60s actual Dispatcher mode=%u: arms=%u recoveries=%u sleep_windows=%u\n",
mode, w.arms, unsigned(w.getRecoveryCount()), sleep_windows);
// A real success clears the ramp and enters normal settled calibration.
w.arm_result = 0; now_ms += retryDelay(w); w.recvRaw(bytes, sizeof(bytes));
assert(w.state == STATE_RX && !w._rx_retry_pending && w._rx_retry_delay_ms == 0);
if (mode == 1) { w.loop(); w.recvRaw(bytes, sizeof(bytes)); }
assert(w.isCalibratingNoiseFloor());
w.loop();
assert(w._nf_calib_deadline == now_ms + NF_CONTINUOUS_TIMEOUT_MS);
now_ms += NF_CALIB_SETTLE_MS; w.loop();
assert(w._floor_estimator.count() == 1);
}
}
static void rampAndRollover() {
uint8_t bytes[8];
for (uint32_t origin : {100U, UINT32_MAX - 19U}) {
RadioLibWrapper w; w.state = STATE_IDLE; w.arm_result = -5; now_ms = origin;
for (uint32_t expected : {20U, 40U, 80U, 160U, 320U, 640U, 1280U, 2000U, 2000U}) {
const unsigned arms = w.arms;
w.recvRaw(bytes, sizeof(bytes));
assert(w.arms == arms + 1 && retryDelay(w) == expected);
// Periodic/forced calibration, recovery and watchdog counter resets
// cannot turn the existing failed-arm deadline into an immediate retry.
w.triggerNoiseFloorCalibrate(7); w.recalibrateNoiseFloor(); w.resetAGC();
w.recoverRadio(false); w._startrx_fails = 0;
assert(retryDelay(w) == expected && w._rx_retry_delay_ms == expected);
now_ms += expected - 1;
w.recvRaw(bytes, sizeof(bytes)); assert(w.arms == arms + 1 && retryDelay(w) == 1);
++now_ms; assert(retryDelay(w) == 0);
}
w.begin(); assert(!w._rx_retry_pending && w._rx_retry_delay_ms == 0);
w.recvRaw(bytes, sizeof(bytes)); assert(retryDelay(w) == 20);
}
}
static void ownersAndImmediateRearms() {
uint8_t bytes[8];
for (bool busy : {false, true}) {
RadioLibWrapper w; w.state = STATE_IDLE; w.arm_result = -5;
w.recvRaw(bytes, sizeof(bytes)); now_ms += retryDelay(w);
w.busy = busy; w.packet = !busy;
const unsigned arms = w.arms;
w.recvRaw(bytes, sizeof(bytes));
assert(w.arms == arms && retryDelay(w) == 2000 && !w.hasPendingRadioInterrupt());
w.busy = w.packet = false;
// A completed packet bypasses the timer and is delivered with its owner.
w.events.record(); w.radio.packet_length = 1; w.arm_result = 0;
assert(w.recvRaw(bytes, sizeof(bytes)) == 1 && w.arms == arms + 1);
assert(!w._rx_retry_pending && w._rx_retry_delay_ms == 0 && !w.hasPendingRadioInterrupt());
}
{
RadioLibWrapper w; w.state = STATE_IDLE; w.arm_result = -5;
w.recvRaw(bytes, sizeof(bytes));
w.state = STATE_TX_WAIT; w.events.record();
const unsigned arms = w.arms;
assert(w.recvRaw(bytes, sizeof(bytes)) == 0 && w.arms == arms && w.hasPendingRadioInterrupt());
uint32_t ignored = 0; assert(!w.getReceiveRetryDelay(now_ms, ignored));
w.onSendFinished(); w.arm_result = 0;
w.recvRaw(bytes, sizeof(bytes));
assert(w.arms == arms + 1 && w.state == STATE_RX && !w.hasPendingRadioInterrupt());
}
{
RadioLibWrapper w; w.state = STATE_IDLE; w.arm_result = -5;
w.recvRaw(bytes, sizeof(bytes)); const unsigned arms = w.arms;
// Direct rearm, as used by profile/CAD restoration, has no retry wait.
w.arm_result = 0; w.startRecv();
assert(w.arms == arms + 1 && w.state == STATE_RX && !w._rx_retry_pending);
}
{
RadioLibWrapper w; w._rx_ps_armed = true; w.arm_result = -5;
w.startRecv(); assert(w.state == STATE_IDLE && !w._rx_ps_armed);
}
{
RadioLibWrapper w; w._nf_calib_active = true; w._rx_ps_armed = true;
assert(!w.isCalibratingNoiseFloor());
w.state = STATE_IDLE; w._rx_ps_armed = false; assert(!w.isCalibratingNoiseFloor());
w.state = STATE_RX; assert(w.isCalibratingNoiseFloor());
w._cw_active = true; uint32_t delay = 0; assert(!w.getReceiveRetryDelay(now_ms, delay));
}
}
static void pendingSetupAndTx() {
RadioLibWrapper w; w.begin(); now_ms += 100000;
assert(!w.isCalibratingNoiseFloor() && w._nf_calib_deadline == 0);
w.loop(); assert(w._nf_refresh_requested && w._nf_calib_deadline == 0);
w.state = STATE_TX_WAIT; now_ms += 100000; w.loop();
assert(!w.isCalibratingNoiseFloor() && w._nf_refresh_requested && w._nf_calib_deadline == 0);
w.onSendFinished(); w.startRecv(); w.loop();
assert(w._nf_calib_deadline == now_ms + NF_CONTINUOUS_TIMEOUT_MS);
}
static void retainedOutboundWake() {
RadioLibWrapper w; RetryClock clock; StaticPoolPacketManager manager{4};
RetryDispatcher d(w, clock, manager);
now_ms = 1000; d.begin(); w.arm_result = -5;
d.loop();
auto* packet = d.obtainNewPacket(); assert(packet);
packet->header = ROUTE_TYPE_DIRECT | (PAYLOAD_TYPE_RAW_CUSTOM << PH_TYPE_SHIFT);
packet->payload_len = 1; packet->payload[0] = 0x42;
assert(d.sendPacket(packet, 0)); d.loop(); assert(w.sends == 1 && d.hasOutbound());
uint32_t delay = 0;
assert(d.getNextQueueWakeDelay(delay) && delay == 20);
now_ms += 20; d.loop();
assert(w.arms == 2 && d.getNextQueueWakeDelay(delay) && delay == 40);
}
static void busyAfterFailure() {
RadioLibWrapper w; RetryClock clock; StaticPoolPacketManager manager{4};
RetryDispatcher d(w, clock, manager);
now_ms = 1000; d.begin(); w.arm_result = -5; d.loop(); w.busy = true;
unsigned sleep_windows = 0;
for (unsigned elapsed = 1; elapsed <= 30000; ++elapsed) {
now_ms = 1000 + elapsed; d.loop();
uint32_t delay = 0; assert(d.getNextQueueWakeDelay(delay));
if (elapsed > 8000) sleep_windows += mesh::power::powerSaveSleepSeconds(delay) > 0;
assert(w.arms == 1 && !w.isCalibratingNoiseFloor());
}
// A due AGC reset cannot admit SPI on BUSY. Its deferred deadline must join
// the failed-RX deadline instead of vetoing every timed sleep at 50 ms.
assert(sleep_windows > 8000);
w.busy = false; w.arm_result = 0; now_ms += retryDelay(w); d.loop();
assert(w.state == STATE_RX && !w._rx_retry_pending && w._rx_retry_delay_ms == 0);
w._nf_refresh_requested = false; w._nf_calib_active = false;
now_ms += 1; d.loop();
uint32_t ignored = 0; assert(!w.getReceiveRetryDelay(now_ms, ignored));
// The healthy non-RX-retry deferral contract remains 50 ms.
w.packet = true; now_ms += 10000; d.loop();
assert(d.getNextQueueWakeDelay(ignored) && ignored == 50);
std::printf("30s BUSY after RX error: arms=%u later_sleep_windows=%u\n", w.arms, sleep_windows);
}
int main() {
persistentFailure(); rampAndRollover(); ownersAndImmediateRearms();
pendingSetupAndTx(); retainedOutboundWake(); busyAfterFailure();
std::puts("PASS: receive retry lifecycle");
}
@@ -216,6 +216,36 @@ TEST(AGCResetScheduler, DeferredRetriesWaitFiftyMillisWithoutPromotingStage) {
EXPECT_TRUE(scheduler.due(39));
}
TEST(AGCResetScheduler, ReceiveRetryDeferralPreservesStageAndValidatesDelay) {
for (uint32_t origin : {100u, UINT32_MAX - 1999u}) {
AGCResetScheduler scheduler;
scheduler.configure(origin, 8000, true, 0);
scheduler.noteAttempt(origin + 8000u); // one admitted attempt, no promotion yet
scheduler.noteDeferred(origin, 2000);
EXPECT_EQ(scheduler.interval(), 8000u);
EXPECT_EQ(scheduler.nextAt(), origin + 2000u);
EXPECT_FALSE(scheduler.due(origin + 1999u));
EXPECT_TRUE(scheduler.due(origin + 2000u));
scheduler.noteAttempt(origin + 2000u);
EXPECT_EQ(scheduler.interval(), 16000u); // deferral retained the first attempt
}
for (uint32_t requested : {0u, uint32_t(INT32_MAX) + 1u, UINT32_MAX}) {
AGCResetScheduler scheduler;
scheduler.configure(0, 8000, true, 0);
scheduler.noteDeferred(UINT32_MAX - 10u, requested);
EXPECT_EQ(scheduler.nextAt(), 39u);
EXPECT_FALSE(scheduler.due(38));
EXPECT_TRUE(scheduler.due(39));
}
AGCResetScheduler scheduler;
scheduler.configure(0, 8000, true, 0);
scheduler.noteDeferred(100, INT32_MAX);
EXPECT_FALSE(scheduler.due(100 + uint32_t(INT32_MAX) - 1));
EXPECT_TRUE(scheduler.due(100 + uint32_t(INT32_MAX)));
scheduler.reset(); scheduler.noteDeferred(100, 2000);
uint32_t ignored = 0; EXPECT_FALSE(scheduler.nextDelay(100, ignored));
}
TEST(AGCResetScheduler, LateServiceDoesNotReplayMissedResetSlots) {
AGCResetScheduler scheduler;
scheduler.configure(10, 8000, true, 0);
+3
View File
@@ -88,6 +88,9 @@ struct RadioLibWrapper {
bool _rx_boosted_gain_valid = true, _cur_rx_boosted_gain = true;
bool _wd_last_busy = false;
uint8_t _startrx_fails = 0, _irq_probe_failures = 0;
uint32_t _rx_retry_at = 0;
uint16_t _rx_retry_delay_ms = 0;
bool _rx_retry_pending = false;
uint32_t _rx_ps_rx_us = 50000, _rx_ps_sleep_us = 50000;
uint32_t _wd_last_transition = 0, _wd_stuck_thresh = 0, _wd_observe_ms = 0;
uint32_t n_wd_soft = 0, n_wd_hard = 0;
+5
View File
@@ -108,6 +108,9 @@ struct RadioLibWrapper {
bool _rx_ps_continuous_fallback=false, _nf_calib_active=false;
float _last_snr=0, _last_rssi=0;
unsigned n_recv=0,n_recv_errors=0,n_sent=0;
uint32_t _rx_retry_at=0;
uint16_t _rx_retry_delay_ms=0;
bool _rx_retry_pending=false;
uint32_t last_recv_millis=0,last_radio_interrupt_millis=0,_nf_sample_from=0;
Board board;Board* _board=&board;
Chip chip;Chip* _radio=&chip;
@@ -121,6 +124,8 @@ public:
int recvRaw(uint8_t*,int);
void onSendFinished();
void idle(){state=STATE_IDLE;}
bool isChipBusy(){return false;}
bool isPacketPendingOrReceiving(){return hasPendingRadioInterrupt();}
bool recoverRadio(bool){return !hasPendingRadioInterrupt();}
void stopReceiveDutyCycle(){_rx_ps_armed=false;chip.standby();}
void startRecv(){if(chip.startReceive()==0)state=STATE_RX;}
+3
View File
@@ -84,6 +84,9 @@ struct RadioLibWrapper {
bool _rx_boosted_gain_valid = false, _cur_rx_boosted_gain = false;
bool _wd_last_busy = false;
uint8_t _startrx_fails = 0, _rx_mode_failures = 0;
uint32_t _rx_retry_at = 0;
uint16_t _rx_retry_delay_ms = 0;
bool _rx_retry_pending = false;
uint32_t _rx_mode_checked_at = 0, _rx_ps_rx_us = 50000, _rx_ps_sleep_us = 50000;
unsigned long _nf_last_calib = 0, _nf_calib_deadline = 0, _nf_sample_from = 0;
unsigned long _wd_last_transition = 0, _wd_stuck_thresh = 0, _wd_observe_ms = 0;
+145
View File
@@ -0,0 +1,145 @@
#!/usr/bin/env python3
"""Run failed RX starts through actual Dispatcher and radio lifecycle methods."""
from pathlib import Path
import os
import re
import subprocess
import tempfile
import unittest
from test_radio_receive_contract import HARNESS, ROOT, method
class RadioReceiveRetryTests(unittest.TestCase):
def build(self, folder, mutation=None):
source = (ROOT / 'src/helpers/radiolib/RadioLibWrappers.cpp').read_text()
header = (ROOT / 'src/helpers/radiolib/RadioLibWrappers.h').read_text()
dispatcher = (ROOT / 'src/Dispatcher.h').read_text()
rxps = (ROOT / 'src/helpers/radiolib/RXPowerSaving.h').read_text()
if mutation == 'unpaced':
source = source.replace(' if (_rx_retry_pending) {', ' if (false) {')
getter = method(header, 'bool isCalibratingNoiseFloor() const')
if mutation == 'pending_veto':
getter = getter.replace('isInRecvMode() && !_rx_ps_armed && ', '')
names = [('int16_t', 'performChannelScanWithTimeout'),
('bool', 'isPacketPendingOrReceiving'), ('bool', 'isChannelActive'),
('bool', 'isReceivingPassive'), ('bool', 'setRxPowerSaving'),
('bool', 'setRxBoostedGainMode'), ('void', 'startRecv'),
('void', 'checkReceiveMode'), ('void', 'loop'),
('void', 'triggerNoiseFloorCalibrate'), ('void', 'requestRestartRecv'),
('void', 'noiseFloorCalibCheck'), ('void', 'endNoiseFloorCalib'),
('void', 'requestNoiseFloorRefresh'), ('void', 'requestPeriodicNoiseFloorRefresh'),
('void', 'recalibrateNoiseFloor'), ('bool', 'resetAGCIfIdle'),
('bool', 'tryResetAGC'), ('void', 'resetAGC'), ('int', 'recvRaw'),
('void', 'onSendFinished'), ('void', 'begin'),
('bool', 'getReceiveRetryDelay'), ('void', 'rxPsWatchdogCheck')]
methods = method(rxps, 'inline bool rxPowerSavingUsesContinuousFallback(') + '\n'
methods += '\n'.join(method(source, f'{kind} RadioLibWrapper::{name}(')
for kind, name in names)
constants = '\n'.join(re.findall(r'^#define NF_\w+\s+[^\n]+', source, re.MULTILINE))
constants += '\n' + re.search(
r'^#define RX_POWERSAVING_CONTINUOUS_FALLBACK_US\s+[^\n]+', rxps, re.MULTILINE).group(0)
start = header.index('#if defined(USE_SX1262) || defined(USE_SX1268) || defined(USE_LLCC68)')
settle = header[start:header.index('#endif', start) + len('#endif')]
harness = HARNESS[:HARNESS.index('int main() {')]
harness = harness.replace('#include <RadioProfiles.h>',
'#include <Dispatcher.h>\n#include <helpers/StaticPoolPacketManager.h>\n'
'#include <helpers/PowerManagementUtils.h>\n#define BD_STARTUP_RX_PACKET 1')
harness = harness.replace('struct RadioLibWrapper {', 'struct RadioLibWrapper : mesh::Radio {')
harness = harness.replace('Radio radio{&events}; Radio* _radio',
'::Radio radio{&events}; ::Radio* _radio')
harness = harness.replace('unsigned long getEstAirtimeFor(int)', 'uint32_t getEstAirtimeFor(int)')
harness = harness.replace('void rxPsWatchdogCheck() {}', 'void rxPsWatchdogCheck();')
harness = harness.replace(' void serviceWatchdog()',
' unsigned after = 0;\n int getStartupReason() { return 0; }\n'
' void onAfterTransmit() { ++after; }\n void serviceWatchdog()')
harness = harness.replace(' int startChannelScan()', '''
unsigned packet_length = 0, direct_arms = 0;
int startReceive() { ++direct_arms; return start_result; }
int setPreambleLength(uint16_t) { return 0; }
int finishTransmit() { return 0; }
size_t getPacketLength() { return packet_length; }
float getSNR() { return 0; }
float getRSSI() { return 0; }
int readData(uint8_t* out, int len) { for (int i=0;i<len;++i) out[i]=0x42; return 0; }
int startChannelScan()''')
harness = harness.replace('return events.pending();', 'return _loop_event_pending || events.pending();')
harness = harness.replace('bool claimRadioInterrupt() { return events.claim(); }', '''
bool claimRadioInterrupt() {
const bool pending = _loop_event_pending; _loop_event_pending = false;
return events.claim() || pending;
}''')
harness = harness.replace(' @RECOVERY_GETTER@', '''
int recvRaw(uint8_t*, int) override;
void begin() override;
void onSendFinished() override;
bool getReceiveRetryDelay(uint32_t, uint32_t&) const override;
bool registerInterruptAction() { _loop_event_pending=false; events.claim(); return true; }
uint16_t currentPreambleLength() const { return 32; }
bool isPacketReady() { return true; }
bool isInRecvMode() const override { return state == STATE_RX; }
bool startSendRaw(const uint8_t*, int) override { ++sends; return false; }
bool isSendComplete() override { return false; }
float packetScore(float, int) override { return 0; }
bool _rx_hold_continuous = false, _loop_event_pending = false;
uint32_t last_radio_interrupt_millis = 0, last_recv_millis = 0;
float _last_snr = 0, _last_rssi = 0;
unsigned n_recv = 0, n_recv_errors = 0, sends = 0;
uint8_t _wd_stage = 0, _wd_strikes = 0, _irq_probe_failures = 0;
unsigned long _wd_observe_until = 0;
''' + getter.replace(' override', '') + '\n @RECOVERY_GETTER@')
harness = harness.replace(
'bool recoverRadio(bool use_hard) { use_hard ? ++hard : ++soft; return true; }',
method(source, 'bool RadioLibWrapper::recoverRadio(').replace('RadioLibWrapper::', '')
+ '\n bool supportsRadioDeepInit() const { return false; }\n'
+ ' bool restoreAfterDeepInit() { recalibrateNoiseFloor(); return false; }')
harness = (harness.replace('@METHODS@', methods).replace('@NF_CONSTANTS@', constants)
.replace('@SETTLE_CONSTANT@', settle)
.replace('@RECOVERY_GETTER@', method(header, 'uint32_t getRecoveryCount() const'))
.replace('@LEGACY_TRY_RESET@', method(dispatcher, 'virtual bool tryResetAGC('))
.replace('@LEGACY_RECOVERY_GETTER@', method(dispatcher, 'virtual uint32_t getRecoveryCount(')))
harness += (ROOT / 'test/fixtures/radio_receive_retry.cpp').read_text()
folder = Path(folder)
cpp, binary = folder / 'retry.cpp', folder / 'retry'
cpp.write_text(harness.replace('\u2014', '-'), encoding='ascii')
dispatcher_source = ROOT / 'src/Dispatcher.cpp'
if mutation == 'agc_fast':
original = dispatcher_source.read_text()
alignment = (' if (adaptive && _radio->getReceiveRetryDelay(deferred_at, receive_delay)\n'
' && receive_delay > retry_delay) retry_delay = receive_delay;\n')
self.assertEqual(original.count(alignment), 1)
dispatcher_source = folder / 'Dispatcher.cpp'
dispatcher_source.write_text(original.replace(alignment, ''), encoding='ascii')
built = subprocess.run([os.environ.get('CXX', 'g++'), '-std=c++17', '-O1',
'-DUSE_SX1262=1', '-fsanitize=address,undefined', '-fno-sanitize-recover=all',
'-fno-pie', '-no-pie', '-I', str(ROOT / 'test/mocks'), '-I', str(ROOT / 'src'),
str(cpp), str(dispatcher_source), str(ROOT / 'src/Packet.cpp'),
str(ROOT / 'src/helpers/StaticPoolPacketManager.cpp'), '-o', str(binary)],
capture_output=True, text=True, timeout=60)
self.assertEqual(built.returncode, 0, built.stderr)
return binary
def run_probe(self, binary):
env = os.environ.copy()
env['ASAN_OPTIONS'] = env.get('ASAN_OPTIONS', '') + ':detect_leaks=0'
return subprocess.run([str(binary)], capture_output=True, text=True, env=env, timeout=20)
def test_actual_dispatcher_and_receive_lifecycle(self):
with tempfile.TemporaryDirectory(prefix='meshcore-rx-retry-') as folder:
result = self.run_probe(self.build(folder))
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn('PASS: receive retry lifecycle', result.stdout)
print(result.stdout, end='')
def test_old_hot_poll_and_pending_calibration_veto_negative_controls(self):
for mutation in ('unpaced', 'pending_veto', 'agc_fast'):
with self.subTest(mutation=mutation), tempfile.TemporaryDirectory(
prefix='meshcore-rx-retry-negative-') as folder:
result = self.run_probe(self.build(folder, mutation))
self.assertNotEqual(result.returncode, 0)
self.assertIn('busyAfterFailure' if mutation == 'agc_fast'
else 'persistentFailure', result.stderr)
if __name__ == '__main__':
unittest.main()