diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 14f1fafcd..16a18031d 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -412,6 +412,7 @@ jobs: run: | python3 -B test/test_dispatcher_startup_watchdog.py -v python3 -B test/test_radio_receive_contract.py + python3 -B test/test_radio_receive_retry.py -v python3 -B test/test_adaptive_agc_roles.py -v python3 - <<'PY' from pathlib import Path diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 9a90d6a7c..c5fc127fa 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -2912,7 +2912,9 @@ noise floor, or a successful transmission as proof of full receiver sensitivity. TX cannot continually postpone adaptive maintenance. Pending RX/TX, a BUSY chip outside known RX duty cycling, carrier/Morse ownership, and an active noise-floor block can defer it; deferred attempts do not advance backoff and -are retried after 50 ms. The +normally retry after 50 ms. When a failed receive start already has a later +retry deadline, adaptive AGC waits at least until that deadline too, allowing +sleep during persistent receive-start failures. The 60-second bound is a scheduling interval, not permission to interrupt a packet. An armed RXPS radio uses the existing explicit wake/reset path, avoiding retry polling that could miss every short listen window. diff --git a/docs/test-results/radio_receive_calibration.md b/docs/test-results/radio_receive_calibration.md index 12adf80c2..928b565ca 100644 --- a/docs/test-results/radio_receive_calibration.md +++ b/docs/test-results/radio_receive_calibration.md @@ -11,6 +11,16 @@ Periodic requests coalesce while a block is active. RXPS resumes after a complet block, including a held block, or at timeout. Receive and transmit activity are never interrupted just to finish calibration. +A calibration request waiting for successful RX does not keep the MCU awake. +Repeated failed receive starts retry after 20, 40, 80, 160, 320, 640, and 1280 ms, +then at most once every 2000 ms. Only the repeated idle fallback is paced: +packet/TX completion and explicit retune/CAD restoration still rearm promptly. +Calibration and recovery requests retain the failed-start backoff; successful +RX or a new radio session clears it. A retry blocked by BUSY or an RX owner +waits 2000 ms without issuing another RX command. The Dispatcher includes this +retry deadline in its wake query, and deferred adaptive AGC joins a later RX +retry instead of forcing 50 ms wakeups throughout a persistent fault. + In single-profile RXPS, routine refreshes wait at least 60 seconds after the last completed or timed-out block before requesting another continuous-RX window. Both periodic maintenance and passive retry checks use this cooldown; @@ -108,11 +118,12 @@ radio windows of at least 12 hours use a 12-hour deadline; shorter windows disable that reboot watchdog until the window ends. The temporary timing overrides are held in RAM and do not replace saved normal-mode preferences. -The status read uses the two-byte GetStatus transaction. The pinned RadioLib -getStatus() helper requests zero copied data bytes, so its return value is not -usable for this mode check. The dedicated read restores the normal SPI command -layout afterward and does not wake a duty-cycling radio to inspect its mode. +The status read uses the two-byte GetStatus transaction. RadioLib 7.8.1 also +reads the response byte directly; the dedicated probe additionally preserves +the existing SPI layout, checks transport errors, and avoids waiting on BUSY. +It does not wake a duty-cycling radio to inspect its mode. Validation lives in `test_noise_floor_estimator`, `test_rx_power_saving`, `test_radio_liveness`, `test_radio_receive_contract.py`, and -`test_sx126x_receive_mode.py`. +`test_sx126x_receive_mode.py`, and `test_radio_receive_retry.py`. These host +checks do not establish battery-life or hardware fault-recovery results. diff --git a/src/Dispatcher.cpp b/src/Dispatcher.cpp index 8b26a6714..ebd2f830c 100644 --- a/src/Dispatcher.cpp +++ b/src/Dispatcher.cpp @@ -584,10 +584,18 @@ bool Dispatcher::getNextQueueWakeDelay(uint32_t& delay_millis) const { } } + uint32_t receive_retry_delay = 0; + const bool receive_retry = radio_available + && _radio->getReceiveRetryDelay(now, receive_retry_delay); if (outbound != NULL) { if (outbound_radio_retry_pending) { int32_t signed_retry_delay = (int32_t)(outbound_radio_retry_at - now); delay_millis = signed_retry_delay > 0 ? (uint32_t)signed_retry_delay : 0; + // This backoff branch services checkRecv(), but returns before AGC. + // Wake for a failed RX retry without making overdue AGC a hot poll. + if (receive_retry && receive_retry_delay < delay_millis) { + delay_millis = receive_retry_delay; + } } else { // TX completion/timeout still needs the normal fast lifecycle path. delay_millis = 0; @@ -595,6 +603,11 @@ bool Dispatcher::getNextQueueWakeDelay(uint32_t& delay_millis) const { return true; } + if (receive_retry) { + if (!found || receive_retry_delay < shortest_delay) shortest_delay = receive_retry_delay; + found = true; + } + if (outbound_restore_cr) { const int32_t restore_delay = (int32_t)(outbound_restore_retry_at - now); const uint32_t delay = restore_delay > 0 ? (uint32_t)restore_delay : 0; @@ -854,7 +867,11 @@ void Dispatcher::loop() { agc_reset.noteAttempt(_ms->getMillis()); } else { // Ownership/calibration deferral is not a reset or health evidence. - agc_reset.noteDeferred(_ms->getMillis()); + const uint32_t deferred_at = _ms->getMillis(); + uint32_t retry_delay = 50, receive_delay; + if (adaptive && _radio->getReceiveRetryDelay(deferred_at, receive_delay) + && receive_delay > retry_delay) retry_delay = receive_delay; + agc_reset.noteDeferred(deferred_at, retry_delay); } } diff --git a/src/Dispatcher.h b/src/Dispatcher.h index 09b67bccf..508206e2a 100644 --- a/src/Dispatcher.h +++ b/src/Dispatcher.h @@ -215,6 +215,11 @@ public: return getNoiseFloorCalibrationSecondsRemaining(); } virtual bool isCalibratingNoiseFloor() const { return false; } + // A failed receive start may need a later retry without keeping the MCU + // awake. Transports without a deferred receive retry retain their cadence. + virtual bool getReceiveRetryDelay(uint32_t now, uint32_t& delay_ms) const { + (void)now; (void)delay_ms; return false; + } virtual void triggerNoiseFloorCalibrate(int threshold) { } diff --git a/src/helpers/AGCResetScheduler.h b/src/helpers/AGCResetScheduler.h index b297849aa..a04d24071 100644 --- a/src/helpers/AGCResetScheduler.h +++ b/src/helpers/AGCResetScheduler.h @@ -54,8 +54,9 @@ public: // A busy operation changes only its retry deadline, never the reset cadence // or adaptive stage. This permits sleep instead of spinning on a due reset. - void noteDeferred(uint32_t now) { - if (_state) _next = now + 50; + void noteDeferred(uint32_t now, uint32_t retry_delay_ms = 50) { + if (retry_delay_ms == 0 || retry_delay_ms > INT32_MAX) retry_delay_ms = 50; + if (_state) _next = now + retry_delay_ms; } // Call only after an actual accepted reset sequence, not a busy/deferred one. diff --git a/src/helpers/radiolib/RadioLibWrappers.cpp b/src/helpers/radiolib/RadioLibWrappers.cpp index bcd4d16fa..684eb0014 100644 --- a/src/helpers/radiolib/RadioLibWrappers.cpp +++ b/src/helpers/radiolib/RadioLibWrappers.cpp @@ -207,6 +207,9 @@ void RadioLibWrapper::begin() { } #endif state = STATE_IDLE; + _rx_retry_at = 0; + _rx_retry_delay_ms = 0; + _rx_retry_pending = false; if (_board->getStartupReason() == BD_STARTUP_RX_PACKET) { // received a LoRa packet (while in deep sleep) _loop_event_pending = true; // Main-owned wake event; ISR remains sole publisher @@ -1877,6 +1880,8 @@ void RadioLibWrapper::startRecv() { // Keep the partial block, but wait for the frontend before its next sample. _nf_sample_from = millis() + NF_CALIB_SETTLE_MS; _startrx_fails = 0; + _rx_retry_pending = false; + _rx_retry_delay_ms = 0; if (_rx_ps_armed) { // (re)base the duty-cycle watchdog on the freshly armed cycle _wd_last_busy = isChipBusy(); @@ -1895,6 +1900,15 @@ void RadioLibWrapper::startRecv() { if (_wd_observe_ms > 1500) _wd_observe_ms = 1500; } } else { + state = STATE_IDLE; + _rx_ps_armed = false; + // Recover a transient failure promptly, then permit whole-second timed + // sleep under persistent failure. Direct retune/CAD/packet rearms remain + // immediate; only recvRaw's repeated idle fallback observes this deadline. + _rx_retry_delay_ms = _rx_retry_delay_ms == 0 ? 20 + : (_rx_retry_delay_ms < 1000 ? _rx_retry_delay_ms * 2 : 2000); + _rx_retry_at = millis() + _rx_retry_delay_ms; + _rx_retry_pending = true; if (_startrx_fails < 255) _startrx_fails++; MESH_DEBUG_PRINTLN("RadioLibWrapper: error: startReceiveMode(%d)", err); } @@ -1933,6 +1947,15 @@ bool RadioLibWrapper::isInRecvMode() const { return state == STATE_RX; } +bool RadioLibWrapper::getReceiveRetryDelay(uint32_t now, uint32_t& delay_ms) const { +#if MESH_ENABLE_TX_BEACONS + if (_beacon_owned) return false; +#endif + if (!_rx_retry_pending || state != STATE_IDLE || _cw_active) return false; + delay_ms = static_cast(now - _rx_retry_at) >= 0 ? 0 : _rx_retry_at - now; + return true; +} + // RX PowerSaving bool RadioLibWrapper::setRxPowerSaving(bool enabled, uint32_t rx_us, uint32_t sleep_us) { #if MESH_ENABLE_TX_BEACONS @@ -1965,7 +1988,7 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) { #if MESH_ENABLE_TX_BEACONS if (_beacon_owned) return 0; #endif - if (_cw_active) return 0; + if (_cw_active || state == STATE_TX_WAIT) return 0; int len = 0; #if defined(USE_LR2021) bool receiver_stopped = false; @@ -2005,6 +2028,8 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) { // Serial.print(" readData() -> "); Serial.println(len); n_recv++; last_recv_millis = millis(); + _rx_retry_pending = false; + _rx_retry_delay_ms = 0; } } } @@ -2022,6 +2047,9 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) { return 0; } #endif + // Serviced receive events retain the normal immediate rearm, even if an + // earlier idle start failed. A pending IRQ probe above still owns its FIFO. + _rx_retry_pending = false; state = STATE_IDLE; // need another startReceive() } @@ -2033,16 +2061,32 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) { int err = _radio->startReceive(); if (err == RADIOLIB_ERR_NONE) { state = STATE_RX; + _rx_retry_pending = false; + _rx_retry_delay_ms = 0; if (_nf_calib_active) { _nf_sample_from = millis() + NF_CALIB_SETTLE_MS; } } else { + _rx_retry_delay_ms = 20; // the delivered packet just proved RX was working + _rx_retry_at = millis() + _rx_retry_delay_ms; + _rx_retry_pending = true; MESH_DEBUG_PRINTLN("RadioLibWrapper: error: startReceive after packet (%d)", err); } return len; } if (state != STATE_RX) { + if (_rx_retry_pending) { + const uint32_t now = millis(); + if (static_cast(now - _rx_retry_at) < 0) return len; + // A command error can leave hardware receiving despite the idle software + // state. Preserve that owner and move the retry deadline forward instead + // of keeping a permanently due wake or issuing SPI while BUSY is high. + if (isChipBusy() || isPacketPendingOrReceiving() || hasPendingRadioInterrupt()) { + _rx_retry_at = now + 2000; + return len; + } + } startRecv(); } return len; @@ -2164,6 +2208,7 @@ void RadioLibWrapper::onSendFinished() { claimRadioInterrupt(); _irq_probe_failures = 0; _board->onAfterTransmit(); + _rx_retry_pending = false; // TX retirement gets its usual immediate RX rearm state = STATE_IDLE; } diff --git a/src/helpers/radiolib/RadioLibWrappers.h b/src/helpers/radiolib/RadioLibWrappers.h index 51f178640..5f3a47f8c 100644 --- a/src/helpers/radiolib/RadioLibWrappers.h +++ b/src/helpers/radiolib/RadioLibWrappers.h @@ -58,6 +58,11 @@ protected: NoiseFloorEstimator _secondary_floor_estimator; uint32_t _rx_mode_checked_at = 0; uint8_t _rx_mode_failures = 0; + // Independent of calibration and watchdog counters: repeated failed starts + // must retain their backoff through an AGC reset or radio recovery attempt. + uint32_t _rx_retry_at = 0; + uint16_t _rx_retry_delay_ms = 0; + bool _rx_retry_pending = false; unsigned long last_recv_millis; unsigned long last_radio_interrupt_millis; // updated on any ISR event, even CRC errors bool _rx_ps_enabled; @@ -530,10 +535,11 @@ public: // true while a noise-floor batch needs prompt loop service; the app's // hasPendingWork() keeps the MCU awake for the bounded spaced-sample window. bool isCalibratingNoiseFloor() const override { - return _nf_calib_active + return isInRecvMode() && !_rx_ps_armed && (_nf_calib_active || (_nf_refresh_requested - && (!_rx_ps_enabled || _rx_ps_continuous_fallback)); + && (!_rx_ps_enabled || _rx_ps_continuous_fallback))); } + bool getReceiveRetryDelay(uint32_t now, uint32_t& delay_ms) const override; void resetStats() { n_recv = n_sent = n_recv_errors = 0; } uint8_t getRadioState() const override; diff --git a/test/fixtures/radio_receive_retry.cpp b/test/fixtures/radio_receive_retry.cpp new file mode 100644 index 000000000..f08219554 --- /dev/null +++ b/test/fixtures/radio_receive_retry.cpp @@ -0,0 +1,177 @@ +struct RetryClock : mesh::MillisecondClock { + unsigned long getMillis() override { return now_ms; } +}; +struct RetryDispatcher : mesh::Dispatcher { + int interval = 8000; + bool adaptive = true; + RetryDispatcher(RadioLibWrapper& r, RetryClock& c, StaticPoolPacketManager& m) + : Dispatcher(r, c, m) {} + mesh::DispatcherAction onRecvPacket(mesh::Packet*) override { return ACTION_RELEASE; } + int getAGCResetInterval() const override { return interval; } + bool useAdaptiveAGC() const override { return adaptive; } + uint32_t getCADFailRetryDelay() const override { return 5000; } + using Dispatcher::getNextQueueWakeDelay; +}; + +static uint32_t retryDelay(const RadioLibWrapper& w) { + uint32_t delay = UINT32_MAX; + assert(w.getReceiveRetryDelay(now_ms, delay)); + return delay; +} + +static void persistentFailure() { + uint8_t bytes[8]; + for (unsigned mode : {0U, 1U, 2U}) { + RadioLibWrapper w; RetryClock clock; StaticPoolPacketManager manager{4}; + RetryDispatcher d(w, clock, manager); + now_ms = 1000; d.begin(); selectMode(w, mode); w.arm_result = -5; + now_ms = 31000; // application setup may outlast the sampling timeout + unsigned sleep_windows = 0; + for (unsigned i = 0; i <= 60000; ++i, ++now_ms) { + d.loop(); + assert(!w.isCalibratingNoiseFloor()); + uint32_t delay = 0; + assert(d.getNextQueueWakeDelay(delay)); + sleep_windows += mesh::power::powerSaveSleepSeconds(delay) > 0; + assert(w.arms <= 37); // initial ramp followed by at most one attempt/2s + } + assert(w.arms == 36 && sleep_windows > 20000); + assert(w._nf_refresh_requested && w._nf_calib_deadline == 0 && w.reads == 0); + assert(w.getRecoveryCount() >= 7); + std::printf("60s actual Dispatcher mode=%u: arms=%u recoveries=%u sleep_windows=%u\n", + mode, w.arms, unsigned(w.getRecoveryCount()), sleep_windows); + // A real success clears the ramp and enters normal settled calibration. + w.arm_result = 0; now_ms += retryDelay(w); w.recvRaw(bytes, sizeof(bytes)); + assert(w.state == STATE_RX && !w._rx_retry_pending && w._rx_retry_delay_ms == 0); + if (mode == 1) { w.loop(); w.recvRaw(bytes, sizeof(bytes)); } + assert(w.isCalibratingNoiseFloor()); + w.loop(); + assert(w._nf_calib_deadline == now_ms + NF_CONTINUOUS_TIMEOUT_MS); + now_ms += NF_CALIB_SETTLE_MS; w.loop(); + assert(w._floor_estimator.count() == 1); + } +} + +static void rampAndRollover() { + uint8_t bytes[8]; + for (uint32_t origin : {100U, UINT32_MAX - 19U}) { + RadioLibWrapper w; w.state = STATE_IDLE; w.arm_result = -5; now_ms = origin; + for (uint32_t expected : {20U, 40U, 80U, 160U, 320U, 640U, 1280U, 2000U, 2000U}) { + const unsigned arms = w.arms; + w.recvRaw(bytes, sizeof(bytes)); + assert(w.arms == arms + 1 && retryDelay(w) == expected); + // Periodic/forced calibration, recovery and watchdog counter resets + // cannot turn the existing failed-arm deadline into an immediate retry. + w.triggerNoiseFloorCalibrate(7); w.recalibrateNoiseFloor(); w.resetAGC(); + w.recoverRadio(false); w._startrx_fails = 0; + assert(retryDelay(w) == expected && w._rx_retry_delay_ms == expected); + now_ms += expected - 1; + w.recvRaw(bytes, sizeof(bytes)); assert(w.arms == arms + 1 && retryDelay(w) == 1); + ++now_ms; assert(retryDelay(w) == 0); + } + w.begin(); assert(!w._rx_retry_pending && w._rx_retry_delay_ms == 0); + w.recvRaw(bytes, sizeof(bytes)); assert(retryDelay(w) == 20); + } +} + +static void ownersAndImmediateRearms() { + uint8_t bytes[8]; + for (bool busy : {false, true}) { + RadioLibWrapper w; w.state = STATE_IDLE; w.arm_result = -5; + w.recvRaw(bytes, sizeof(bytes)); now_ms += retryDelay(w); + w.busy = busy; w.packet = !busy; + const unsigned arms = w.arms; + w.recvRaw(bytes, sizeof(bytes)); + assert(w.arms == arms && retryDelay(w) == 2000 && !w.hasPendingRadioInterrupt()); + w.busy = w.packet = false; + // A completed packet bypasses the timer and is delivered with its owner. + w.events.record(); w.radio.packet_length = 1; w.arm_result = 0; + assert(w.recvRaw(bytes, sizeof(bytes)) == 1 && w.arms == arms + 1); + assert(!w._rx_retry_pending && w._rx_retry_delay_ms == 0 && !w.hasPendingRadioInterrupt()); + } + { + RadioLibWrapper w; w.state = STATE_IDLE; w.arm_result = -5; + w.recvRaw(bytes, sizeof(bytes)); + w.state = STATE_TX_WAIT; w.events.record(); + const unsigned arms = w.arms; + assert(w.recvRaw(bytes, sizeof(bytes)) == 0 && w.arms == arms && w.hasPendingRadioInterrupt()); + uint32_t ignored = 0; assert(!w.getReceiveRetryDelay(now_ms, ignored)); + w.onSendFinished(); w.arm_result = 0; + w.recvRaw(bytes, sizeof(bytes)); + assert(w.arms == arms + 1 && w.state == STATE_RX && !w.hasPendingRadioInterrupt()); + } + { + RadioLibWrapper w; w.state = STATE_IDLE; w.arm_result = -5; + w.recvRaw(bytes, sizeof(bytes)); const unsigned arms = w.arms; + // Direct rearm, as used by profile/CAD restoration, has no retry wait. + w.arm_result = 0; w.startRecv(); + assert(w.arms == arms + 1 && w.state == STATE_RX && !w._rx_retry_pending); + } + { + RadioLibWrapper w; w._rx_ps_armed = true; w.arm_result = -5; + w.startRecv(); assert(w.state == STATE_IDLE && !w._rx_ps_armed); + } + { + RadioLibWrapper w; w._nf_calib_active = true; w._rx_ps_armed = true; + assert(!w.isCalibratingNoiseFloor()); + w.state = STATE_IDLE; w._rx_ps_armed = false; assert(!w.isCalibratingNoiseFloor()); + w.state = STATE_RX; assert(w.isCalibratingNoiseFloor()); + w._cw_active = true; uint32_t delay = 0; assert(!w.getReceiveRetryDelay(now_ms, delay)); + } +} + +static void pendingSetupAndTx() { + RadioLibWrapper w; w.begin(); now_ms += 100000; + assert(!w.isCalibratingNoiseFloor() && w._nf_calib_deadline == 0); + w.loop(); assert(w._nf_refresh_requested && w._nf_calib_deadline == 0); + w.state = STATE_TX_WAIT; now_ms += 100000; w.loop(); + assert(!w.isCalibratingNoiseFloor() && w._nf_refresh_requested && w._nf_calib_deadline == 0); + w.onSendFinished(); w.startRecv(); w.loop(); + assert(w._nf_calib_deadline == now_ms + NF_CONTINUOUS_TIMEOUT_MS); +} + +static void retainedOutboundWake() { + RadioLibWrapper w; RetryClock clock; StaticPoolPacketManager manager{4}; + RetryDispatcher d(w, clock, manager); + now_ms = 1000; d.begin(); w.arm_result = -5; + d.loop(); + auto* packet = d.obtainNewPacket(); assert(packet); + packet->header = ROUTE_TYPE_DIRECT | (PAYLOAD_TYPE_RAW_CUSTOM << PH_TYPE_SHIFT); + packet->payload_len = 1; packet->payload[0] = 0x42; + assert(d.sendPacket(packet, 0)); d.loop(); assert(w.sends == 1 && d.hasOutbound()); + uint32_t delay = 0; + assert(d.getNextQueueWakeDelay(delay) && delay == 20); + now_ms += 20; d.loop(); + assert(w.arms == 2 && d.getNextQueueWakeDelay(delay) && delay == 40); +} + +static void busyAfterFailure() { + RadioLibWrapper w; RetryClock clock; StaticPoolPacketManager manager{4}; + RetryDispatcher d(w, clock, manager); + now_ms = 1000; d.begin(); w.arm_result = -5; d.loop(); w.busy = true; + unsigned sleep_windows = 0; + for (unsigned elapsed = 1; elapsed <= 30000; ++elapsed) { + now_ms = 1000 + elapsed; d.loop(); + uint32_t delay = 0; assert(d.getNextQueueWakeDelay(delay)); + if (elapsed > 8000) sleep_windows += mesh::power::powerSaveSleepSeconds(delay) > 0; + assert(w.arms == 1 && !w.isCalibratingNoiseFloor()); + } + // A due AGC reset cannot admit SPI on BUSY. Its deferred deadline must join + // the failed-RX deadline instead of vetoing every timed sleep at 50 ms. + assert(sleep_windows > 8000); + w.busy = false; w.arm_result = 0; now_ms += retryDelay(w); d.loop(); + assert(w.state == STATE_RX && !w._rx_retry_pending && w._rx_retry_delay_ms == 0); + w._nf_refresh_requested = false; w._nf_calib_active = false; + now_ms += 1; d.loop(); + uint32_t ignored = 0; assert(!w.getReceiveRetryDelay(now_ms, ignored)); + // The healthy non-RX-retry deferral contract remains 50 ms. + w.packet = true; now_ms += 10000; d.loop(); + assert(d.getNextQueueWakeDelay(ignored) && ignored == 50); + std::printf("30s BUSY after RX error: arms=%u later_sleep_windows=%u\n", w.arms, sleep_windows); +} + +int main() { + persistentFailure(); rampAndRollover(); ownersAndImmediateRearms(); + pendingSetupAndTx(); retainedOutboundWake(); busyAfterFailure(); + std::puts("PASS: receive retry lifecycle"); +} diff --git a/test/test_agc_reset_scheduler/test_agc_reset_scheduler.cpp b/test/test_agc_reset_scheduler/test_agc_reset_scheduler.cpp index 18a104c8d..53b9d5d52 100644 --- a/test/test_agc_reset_scheduler/test_agc_reset_scheduler.cpp +++ b/test/test_agc_reset_scheduler/test_agc_reset_scheduler.cpp @@ -216,6 +216,36 @@ TEST(AGCResetScheduler, DeferredRetriesWaitFiftyMillisWithoutPromotingStage) { EXPECT_TRUE(scheduler.due(39)); } +TEST(AGCResetScheduler, ReceiveRetryDeferralPreservesStageAndValidatesDelay) { + for (uint32_t origin : {100u, UINT32_MAX - 1999u}) { + AGCResetScheduler scheduler; + scheduler.configure(origin, 8000, true, 0); + scheduler.noteAttempt(origin + 8000u); // one admitted attempt, no promotion yet + scheduler.noteDeferred(origin, 2000); + EXPECT_EQ(scheduler.interval(), 8000u); + EXPECT_EQ(scheduler.nextAt(), origin + 2000u); + EXPECT_FALSE(scheduler.due(origin + 1999u)); + EXPECT_TRUE(scheduler.due(origin + 2000u)); + scheduler.noteAttempt(origin + 2000u); + EXPECT_EQ(scheduler.interval(), 16000u); // deferral retained the first attempt + } + for (uint32_t requested : {0u, uint32_t(INT32_MAX) + 1u, UINT32_MAX}) { + AGCResetScheduler scheduler; + scheduler.configure(0, 8000, true, 0); + scheduler.noteDeferred(UINT32_MAX - 10u, requested); + EXPECT_EQ(scheduler.nextAt(), 39u); + EXPECT_FALSE(scheduler.due(38)); + EXPECT_TRUE(scheduler.due(39)); + } + AGCResetScheduler scheduler; + scheduler.configure(0, 8000, true, 0); + scheduler.noteDeferred(100, INT32_MAX); + EXPECT_FALSE(scheduler.due(100 + uint32_t(INT32_MAX) - 1)); + EXPECT_TRUE(scheduler.due(100 + uint32_t(INT32_MAX))); + scheduler.reset(); scheduler.noteDeferred(100, 2000); + uint32_t ignored = 0; EXPECT_FALSE(scheduler.nextDelay(100, ignored)); +} + TEST(AGCResetScheduler, LateServiceDoesNotReplayMissedResetSlots) { AGCResetScheduler scheduler; scheduler.configure(10, 8000, true, 0); diff --git a/test/test_beacon_radio.py b/test/test_beacon_radio.py index 0472583e8..86c2c3efd 100644 --- a/test/test_beacon_radio.py +++ b/test/test_beacon_radio.py @@ -88,6 +88,9 @@ struct RadioLibWrapper { bool _rx_boosted_gain_valid = true, _cur_rx_boosted_gain = true; bool _wd_last_busy = false; uint8_t _startrx_fails = 0, _irq_probe_failures = 0; + uint32_t _rx_retry_at = 0; + uint16_t _rx_retry_delay_ms = 0; + bool _rx_retry_pending = false; uint32_t _rx_ps_rx_us = 50000, _rx_ps_sleep_us = 50000; uint32_t _wd_last_transition = 0, _wd_stuck_thresh = 0, _wd_observe_ms = 0; uint32_t n_wd_soft = 0, n_wd_hard = 0; diff --git a/test/test_radio_interrupt_recovery.py b/test/test_radio_interrupt_recovery.py index b27ebf229..7ddb07fa9 100644 --- a/test/test_radio_interrupt_recovery.py +++ b/test/test_radio_interrupt_recovery.py @@ -108,6 +108,9 @@ struct RadioLibWrapper { bool _rx_ps_continuous_fallback=false, _nf_calib_active=false; float _last_snr=0, _last_rssi=0; unsigned n_recv=0,n_recv_errors=0,n_sent=0; + uint32_t _rx_retry_at=0; + uint16_t _rx_retry_delay_ms=0; + bool _rx_retry_pending=false; uint32_t last_recv_millis=0,last_radio_interrupt_millis=0,_nf_sample_from=0; Board board;Board* _board=&board; Chip chip;Chip* _radio=&chip; @@ -121,6 +124,8 @@ public: int recvRaw(uint8_t*,int); void onSendFinished(); void idle(){state=STATE_IDLE;} + bool isChipBusy(){return false;} + bool isPacketPendingOrReceiving(){return hasPendingRadioInterrupt();} bool recoverRadio(bool){return !hasPendingRadioInterrupt();} void stopReceiveDutyCycle(){_rx_ps_armed=false;chip.standby();} void startRecv(){if(chip.startReceive()==0)state=STATE_RX;} diff --git a/test/test_radio_receive_contract.py b/test/test_radio_receive_contract.py index bdc5e131a..a91cc293f 100644 --- a/test/test_radio_receive_contract.py +++ b/test/test_radio_receive_contract.py @@ -84,6 +84,9 @@ struct RadioLibWrapper { bool _rx_boosted_gain_valid = false, _cur_rx_boosted_gain = false; bool _wd_last_busy = false; uint8_t _startrx_fails = 0, _rx_mode_failures = 0; + uint32_t _rx_retry_at = 0; + uint16_t _rx_retry_delay_ms = 0; + bool _rx_retry_pending = false; uint32_t _rx_mode_checked_at = 0, _rx_ps_rx_us = 50000, _rx_ps_sleep_us = 50000; unsigned long _nf_last_calib = 0, _nf_calib_deadline = 0, _nf_sample_from = 0; unsigned long _wd_last_transition = 0, _wd_stuck_thresh = 0, _wd_observe_ms = 0; diff --git a/test/test_radio_receive_retry.py b/test/test_radio_receive_retry.py new file mode 100644 index 000000000..b9683d841 --- /dev/null +++ b/test/test_radio_receive_retry.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Run failed RX starts through actual Dispatcher and radio lifecycle methods.""" +from pathlib import Path +import os +import re +import subprocess +import tempfile +import unittest + +from test_radio_receive_contract import HARNESS, ROOT, method + + +class RadioReceiveRetryTests(unittest.TestCase): + def build(self, folder, mutation=None): + source = (ROOT / 'src/helpers/radiolib/RadioLibWrappers.cpp').read_text() + header = (ROOT / 'src/helpers/radiolib/RadioLibWrappers.h').read_text() + dispatcher = (ROOT / 'src/Dispatcher.h').read_text() + rxps = (ROOT / 'src/helpers/radiolib/RXPowerSaving.h').read_text() + if mutation == 'unpaced': + source = source.replace(' if (_rx_retry_pending) {', ' if (false) {') + getter = method(header, 'bool isCalibratingNoiseFloor() const') + if mutation == 'pending_veto': + getter = getter.replace('isInRecvMode() && !_rx_ps_armed && ', '') + names = [('int16_t', 'performChannelScanWithTimeout'), + ('bool', 'isPacketPendingOrReceiving'), ('bool', 'isChannelActive'), + ('bool', 'isReceivingPassive'), ('bool', 'setRxPowerSaving'), + ('bool', 'setRxBoostedGainMode'), ('void', 'startRecv'), + ('void', 'checkReceiveMode'), ('void', 'loop'), + ('void', 'triggerNoiseFloorCalibrate'), ('void', 'requestRestartRecv'), + ('void', 'noiseFloorCalibCheck'), ('void', 'endNoiseFloorCalib'), + ('void', 'requestNoiseFloorRefresh'), ('void', 'requestPeriodicNoiseFloorRefresh'), + ('void', 'recalibrateNoiseFloor'), ('bool', 'resetAGCIfIdle'), + ('bool', 'tryResetAGC'), ('void', 'resetAGC'), ('int', 'recvRaw'), + ('void', 'onSendFinished'), ('void', 'begin'), + ('bool', 'getReceiveRetryDelay'), ('void', 'rxPsWatchdogCheck')] + methods = method(rxps, 'inline bool rxPowerSavingUsesContinuousFallback(') + '\n' + methods += '\n'.join(method(source, f'{kind} RadioLibWrapper::{name}(') + for kind, name in names) + constants = '\n'.join(re.findall(r'^#define NF_\w+\s+[^\n]+', source, re.MULTILINE)) + constants += '\n' + re.search( + r'^#define RX_POWERSAVING_CONTINUOUS_FALLBACK_US\s+[^\n]+', rxps, re.MULTILINE).group(0) + start = header.index('#if defined(USE_SX1262) || defined(USE_SX1268) || defined(USE_LLCC68)') + settle = header[start:header.index('#endif', start) + len('#endif')] + harness = HARNESS[:HARNESS.index('int main() {')] + harness = harness.replace('#include ', + '#include \n#include \n' + '#include \n#define BD_STARTUP_RX_PACKET 1') + harness = harness.replace('struct RadioLibWrapper {', 'struct RadioLibWrapper : mesh::Radio {') + harness = harness.replace('Radio radio{&events}; Radio* _radio', + '::Radio radio{&events}; ::Radio* _radio') + harness = harness.replace('unsigned long getEstAirtimeFor(int)', 'uint32_t getEstAirtimeFor(int)') + harness = harness.replace('void rxPsWatchdogCheck() {}', 'void rxPsWatchdogCheck();') + harness = harness.replace(' void serviceWatchdog()', + ' unsigned after = 0;\n int getStartupReason() { return 0; }\n' + ' void onAfterTransmit() { ++after; }\n void serviceWatchdog()') + harness = harness.replace(' int startChannelScan()', ''' + unsigned packet_length = 0, direct_arms = 0; + int startReceive() { ++direct_arms; return start_result; } + int setPreambleLength(uint16_t) { return 0; } + int finishTransmit() { return 0; } + size_t getPacketLength() { return packet_length; } + float getSNR() { return 0; } + float getRSSI() { return 0; } + int readData(uint8_t* out, int len) { for (int i=0;igetReceiveRetryDelay(deferred_at, receive_delay)\n' + ' && receive_delay > retry_delay) retry_delay = receive_delay;\n') + self.assertEqual(original.count(alignment), 1) + dispatcher_source = folder / 'Dispatcher.cpp' + dispatcher_source.write_text(original.replace(alignment, ''), encoding='ascii') + built = subprocess.run([os.environ.get('CXX', 'g++'), '-std=c++17', '-O1', + '-DUSE_SX1262=1', '-fsanitize=address,undefined', '-fno-sanitize-recover=all', + '-fno-pie', '-no-pie', '-I', str(ROOT / 'test/mocks'), '-I', str(ROOT / 'src'), + str(cpp), str(dispatcher_source), str(ROOT / 'src/Packet.cpp'), + str(ROOT / 'src/helpers/StaticPoolPacketManager.cpp'), '-o', str(binary)], + capture_output=True, text=True, timeout=60) + self.assertEqual(built.returncode, 0, built.stderr) + return binary + + def run_probe(self, binary): + env = os.environ.copy() + env['ASAN_OPTIONS'] = env.get('ASAN_OPTIONS', '') + ':detect_leaks=0' + return subprocess.run([str(binary)], capture_output=True, text=True, env=env, timeout=20) + + def test_actual_dispatcher_and_receive_lifecycle(self): + with tempfile.TemporaryDirectory(prefix='meshcore-rx-retry-') as folder: + result = self.run_probe(self.build(folder)) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn('PASS: receive retry lifecycle', result.stdout) + print(result.stdout, end='') + + def test_old_hot_poll_and_pending_calibration_veto_negative_controls(self): + for mutation in ('unpaced', 'pending_veto', 'agc_fast'): + with self.subTest(mutation=mutation), tempfile.TemporaryDirectory( + prefix='meshcore-rx-retry-negative-') as folder: + result = self.run_probe(self.build(folder, mutation)) + self.assertNotEqual(result.returncode, 0) + self.assertIn('busyAfterFailure' if mutation == 'agc_fast' + else 'persistentFailure', result.stderr) + + +if __name__ == '__main__': + unittest.main()