Bound failed RX retries without blocking power-saving sleep

.github/workflows/run-unit-tests.yml

@@ -412,6 +412,7 @@ jobs:
         run: |
           python3 -B test/test_dispatcher_startup_watchdog.py -v
           python3 -B test/test_radio_receive_contract.py
+          python3 -B test/test_radio_receive_retry.py -v
           python3 -B test/test_adaptive_agc_roles.py -v
           python3 - <<'PY'
           from pathlib import Path

docs/cli_commands.md

@@ -2912,7 +2912,9 @@ noise floor, or a successful transmission as proof of full receiver sensitivity.
 TX cannot continually postpone adaptive maintenance. Pending RX/TX, a BUSY
 chip outside known RX duty cycling, carrier/Morse ownership, and an active
 noise-floor block can defer it; deferred attempts do not advance backoff and
-are retried after 50 ms. The
+normally retry after 50 ms. When a failed receive start already has a later
+retry deadline, adaptive AGC waits at least until that deadline too, allowing
+sleep during persistent receive-start failures. The
 60-second bound is a scheduling interval, not permission to interrupt a packet.
 An armed RXPS radio uses the existing explicit wake/reset path, avoiding retry
 polling that could miss every short listen window.

docs/test-results/radio_receive_calibration.md

@@ -11,6 +11,16 @@ Periodic requests coalesce while a block is active. RXPS resumes after a complet
 block, including a held block, or at timeout. Receive and transmit activity are
 never interrupted just to finish calibration.
 
+A calibration request waiting for successful RX does not keep the MCU awake.
+Repeated failed receive starts retry after 20, 40, 80, 160, 320, 640, and 1280 ms,
+then at most once every 2000 ms. Only the repeated idle fallback is paced:
+packet/TX completion and explicit retune/CAD restoration still rearm promptly.
+Calibration and recovery requests retain the failed-start backoff; successful
+RX or a new radio session clears it. A retry blocked by BUSY or an RX owner
+waits 2000 ms without issuing another RX command. The Dispatcher includes this
+retry deadline in its wake query, and deferred adaptive AGC joins a later RX
+retry instead of forcing 50 ms wakeups throughout a persistent fault.
+
 In single-profile RXPS, routine refreshes wait at least 60 seconds after the
 last completed or timed-out block before requesting another continuous-RX
 window. Both periodic maintenance and passive retry checks use this cooldown;
@@ -108,11 +118,12 @@ radio windows of at least 12 hours use a 12-hour deadline; shorter windows
 disable that reboot watchdog until the window ends. The temporary timing
 overrides are held in RAM and do not replace saved normal-mode preferences.
 
-The status read uses the two-byte GetStatus transaction. The pinned RadioLib
-getStatus() helper requests zero copied data bytes, so its return value is not
-usable for this mode check. The dedicated read restores the normal SPI command
-layout afterward and does not wake a duty-cycling radio to inspect its mode.
+The status read uses the two-byte GetStatus transaction. RadioLib 7.8.1 also
+reads the response byte directly; the dedicated probe additionally preserves
+the existing SPI layout, checks transport errors, and avoids waiting on BUSY.
+It does not wake a duty-cycling radio to inspect its mode.
 
 Validation lives in `test_noise_floor_estimator`, `test_rx_power_saving`,
 `test_radio_liveness`, `test_radio_receive_contract.py`, and
-`test_sx126x_receive_mode.py`.
+`test_sx126x_receive_mode.py`, and `test_radio_receive_retry.py`. These host
+checks do not establish battery-life or hardware fault-recovery results.

src/Dispatcher.cpp

@@ -584,10 +584,18 @@ bool Dispatcher::getNextQueueWakeDelay(uint32_t& delay_millis) const {
     }
   }
 
+  uint32_t receive_retry_delay = 0;
+  const bool receive_retry = radio_available
+      && _radio->getReceiveRetryDelay(now, receive_retry_delay);
   if (outbound != NULL) {
     if (outbound_radio_retry_pending) {
       int32_t signed_retry_delay = (int32_t)(outbound_radio_retry_at - now);
       delay_millis = signed_retry_delay > 0 ? (uint32_t)signed_retry_delay : 0;
+      // This backoff branch services checkRecv(), but returns before AGC.
+      // Wake for a failed RX retry without making overdue AGC a hot poll.
+      if (receive_retry && receive_retry_delay < delay_millis) {
+        delay_millis = receive_retry_delay;
+      }
     } else {
       // TX completion/timeout still needs the normal fast lifecycle path.
       delay_millis = 0;
@@ -595,6 +603,11 @@ bool Dispatcher::getNextQueueWakeDelay(uint32_t& delay_millis) const {
     return true;
   }
 
+  if (receive_retry) {
+    if (!found || receive_retry_delay < shortest_delay) shortest_delay = receive_retry_delay;
+    found = true;
+  }
+
   if (outbound_restore_cr) {
     const int32_t restore_delay = (int32_t)(outbound_restore_retry_at - now);
     const uint32_t delay = restore_delay > 0 ? (uint32_t)restore_delay : 0;
@@ -854,7 +867,11 @@ void Dispatcher::loop() {
       agc_reset.noteAttempt(_ms->getMillis());
     } else {
       // Ownership/calibration deferral is not a reset or health evidence.
-      agc_reset.noteDeferred(_ms->getMillis());
+      const uint32_t deferred_at = _ms->getMillis();
+      uint32_t retry_delay = 50, receive_delay;
+      if (adaptive && _radio->getReceiveRetryDelay(deferred_at, receive_delay)
+          && receive_delay > retry_delay) retry_delay = receive_delay;
+      agc_reset.noteDeferred(deferred_at, retry_delay);
     }
   }
 

src/Dispatcher.h

@@ -215,6 +215,11 @@ public:
     return getNoiseFloorCalibrationSecondsRemaining();
   }
   virtual bool isCalibratingNoiseFloor() const { return false; }
+  // A failed receive start may need a later retry without keeping the MCU
+  // awake. Transports without a deferred receive retry retain their cadence.
+  virtual bool getReceiveRetryDelay(uint32_t now, uint32_t& delay_ms) const {
+    (void)now; (void)delay_ms; return false;
+  }
 
   virtual void triggerNoiseFloorCalibrate(int threshold) { }
 

src/helpers/AGCResetScheduler.h

@@ -54,8 +54,9 @@ public:
 
   // A busy operation changes only its retry deadline, never the reset cadence
   // or adaptive stage. This permits sleep instead of spinning on a due reset.
-  void noteDeferred(uint32_t now) {
-    if (_state) _next = now + 50;
+  void noteDeferred(uint32_t now, uint32_t retry_delay_ms = 50) {
+    if (retry_delay_ms == 0 || retry_delay_ms > INT32_MAX) retry_delay_ms = 50;
+    if (_state) _next = now + retry_delay_ms;
   }
 
   // Call only after an actual accepted reset sequence, not a busy/deferred one.

src/helpers/radiolib/RadioLibWrappers.cpp

@@ -207,6 +207,9 @@ void RadioLibWrapper::begin() {
   }
 #endif
   state = STATE_IDLE;
+  _rx_retry_at = 0;
+  _rx_retry_delay_ms = 0;
+  _rx_retry_pending = false;
 
   if (_board->getStartupReason() == BD_STARTUP_RX_PACKET) {  // received a LoRa packet (while in deep sleep)
     _loop_event_pending = true; // Main-owned wake event; ISR remains sole publisher
@@ -1877,6 +1880,8 @@ void RadioLibWrapper::startRecv() {
     // Keep the partial block, but wait for the frontend before its next sample.
     _nf_sample_from = millis() + NF_CALIB_SETTLE_MS;
     _startrx_fails = 0;
+    _rx_retry_pending = false;
+    _rx_retry_delay_ms = 0;
     if (_rx_ps_armed) {
       // (re)base the duty-cycle watchdog on the freshly armed cycle
       _wd_last_busy = isChipBusy();
@@ -1895,6 +1900,15 @@ void RadioLibWrapper::startRecv() {
       if (_wd_observe_ms > 1500) _wd_observe_ms = 1500;
     }
   } else {
+    state = STATE_IDLE;
+    _rx_ps_armed = false;
+    // Recover a transient failure promptly, then permit whole-second timed
+    // sleep under persistent failure. Direct retune/CAD/packet rearms remain
+    // immediate; only recvRaw's repeated idle fallback observes this deadline.
+    _rx_retry_delay_ms = _rx_retry_delay_ms == 0 ? 20
+        : (_rx_retry_delay_ms < 1000 ? _rx_retry_delay_ms * 2 : 2000);
+    _rx_retry_at = millis() + _rx_retry_delay_ms;
+    _rx_retry_pending = true;
     if (_startrx_fails < 255) _startrx_fails++;
     MESH_DEBUG_PRINTLN("RadioLibWrapper: error: startReceiveMode(%d)", err);
   }
@@ -1933,6 +1947,15 @@ bool RadioLibWrapper::isInRecvMode() const {
   return state == STATE_RX;
 }
 
+bool RadioLibWrapper::getReceiveRetryDelay(uint32_t now, uint32_t& delay_ms) const {
+#if MESH_ENABLE_TX_BEACONS
+  if (_beacon_owned) return false;
+#endif
+  if (!_rx_retry_pending || state != STATE_IDLE || _cw_active) return false;
+  delay_ms = static_cast<int32_t>(now - _rx_retry_at) >= 0 ? 0 : _rx_retry_at - now;
+  return true;
+}
+
 // RX PowerSaving
 bool RadioLibWrapper::setRxPowerSaving(bool enabled, uint32_t rx_us, uint32_t sleep_us) {
 #if MESH_ENABLE_TX_BEACONS
@@ -1965,7 +1988,7 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) {
 #if MESH_ENABLE_TX_BEACONS
   if (_beacon_owned) return 0;
 #endif
-  if (_cw_active) return 0;
+  if (_cw_active || state == STATE_TX_WAIT) return 0;
   int len = 0;
   #if defined(USE_LR2021)
   bool receiver_stopped = false;
@@ -2005,6 +2028,8 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) {
         //  Serial.print("  readData() -> "); Serial.println(len);
           n_recv++;
           last_recv_millis = millis();
+          _rx_retry_pending = false;
+          _rx_retry_delay_ms = 0;
         }
       }
     }
@@ -2022,6 +2047,9 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) {
       return 0;
     }
     #endif
+    // Serviced receive events retain the normal immediate rearm, even if an
+    // earlier idle start failed. A pending IRQ probe above still owns its FIFO.
+    _rx_retry_pending = false;
     state = STATE_IDLE;   // need another startReceive()
   }
 
@@ -2033,16 +2061,32 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) {
     int err = _radio->startReceive();
     if (err == RADIOLIB_ERR_NONE) {
       state = STATE_RX;
+      _rx_retry_pending = false;
+      _rx_retry_delay_ms = 0;
       if (_nf_calib_active) {
         _nf_sample_from = millis() + NF_CALIB_SETTLE_MS;
       }
     } else {
+      _rx_retry_delay_ms = 20;  // the delivered packet just proved RX was working
+      _rx_retry_at = millis() + _rx_retry_delay_ms;
+      _rx_retry_pending = true;
       MESH_DEBUG_PRINTLN("RadioLibWrapper: error: startReceive after packet (%d)", err);
     }
     return len;
   }
 
   if (state != STATE_RX) {
+    if (_rx_retry_pending) {
+      const uint32_t now = millis();
+      if (static_cast<int32_t>(now - _rx_retry_at) < 0) return len;
+      // A command error can leave hardware receiving despite the idle software
+      // state. Preserve that owner and move the retry deadline forward instead
+      // of keeping a permanently due wake or issuing SPI while BUSY is high.
+      if (isChipBusy() || isPacketPendingOrReceiving() || hasPendingRadioInterrupt()) {
+        _rx_retry_at = now + 2000;
+        return len;
+      }
+    }
     startRecv();
   }
   return len;
@@ -2164,6 +2208,7 @@ void RadioLibWrapper::onSendFinished() {
   claimRadioInterrupt();
   _irq_probe_failures = 0;
   _board->onAfterTransmit();
+  _rx_retry_pending = false;  // TX retirement gets its usual immediate RX rearm
   state = STATE_IDLE;
 }
 

src/helpers/radiolib/RadioLibWrappers.h

@@ -58,6 +58,11 @@ protected:
   NoiseFloorEstimator _secondary_floor_estimator;
   uint32_t _rx_mode_checked_at = 0;
   uint8_t _rx_mode_failures = 0;
+  // Independent of calibration and watchdog counters: repeated failed starts
+  // must retain their backoff through an AGC reset or radio recovery attempt.
+  uint32_t _rx_retry_at = 0;
+  uint16_t _rx_retry_delay_ms = 0;
+  bool _rx_retry_pending = false;
   unsigned long last_recv_millis;
   unsigned long last_radio_interrupt_millis;  // updated on any ISR event, even CRC errors
   bool _rx_ps_enabled;
@@ -530,10 +535,11 @@ public:
   // true while a noise-floor batch needs prompt loop service; the app's
   // hasPendingWork() keeps the MCU awake for the bounded spaced-sample window.
   bool isCalibratingNoiseFloor() const override {
-    return _nf_calib_active
+    return isInRecvMode() && !_rx_ps_armed && (_nf_calib_active
         || (_nf_refresh_requested
-            && (!_rx_ps_enabled || _rx_ps_continuous_fallback));
+            && (!_rx_ps_enabled || _rx_ps_continuous_fallback)));
   }
+  bool getReceiveRetryDelay(uint32_t now, uint32_t& delay_ms) const override;
   void resetStats() { n_recv = n_sent = n_recv_errors = 0; }
 
   uint8_t getRadioState() const override;

test/fixtures/radio_receive_retry.cpp

@@ -0,0 +1,177 @@
+struct RetryClock : mesh::MillisecondClock {
+  unsigned long getMillis() override { return now_ms; }
+};
+struct RetryDispatcher : mesh::Dispatcher {
+  int interval = 8000;
+  bool adaptive = true;
+  RetryDispatcher(RadioLibWrapper& r, RetryClock& c, StaticPoolPacketManager& m)
+      : Dispatcher(r, c, m) {}
+  mesh::DispatcherAction onRecvPacket(mesh::Packet*) override { return ACTION_RELEASE; }
+  int getAGCResetInterval() const override { return interval; }
+  bool useAdaptiveAGC() const override { return adaptive; }
+  uint32_t getCADFailRetryDelay() const override { return 5000; }
+  using Dispatcher::getNextQueueWakeDelay;
+};
+
+static uint32_t retryDelay(const RadioLibWrapper& w) {
+  uint32_t delay = UINT32_MAX;
+  assert(w.getReceiveRetryDelay(now_ms, delay));
+  return delay;
+}
+
+static void persistentFailure() {
+  uint8_t bytes[8];
+  for (unsigned mode : {0U, 1U, 2U}) {
+    RadioLibWrapper w; RetryClock clock; StaticPoolPacketManager manager{4};
+    RetryDispatcher d(w, clock, manager);
+    now_ms = 1000; d.begin(); selectMode(w, mode); w.arm_result = -5;
+    now_ms = 31000; // application setup may outlast the sampling timeout
+    unsigned sleep_windows = 0;
+    for (unsigned i = 0; i <= 60000; ++i, ++now_ms) {
+      d.loop();
+      assert(!w.isCalibratingNoiseFloor());
+      uint32_t delay = 0;
+      assert(d.getNextQueueWakeDelay(delay));
+      sleep_windows += mesh::power::powerSaveSleepSeconds(delay) > 0;
+      assert(w.arms <= 37); // initial ramp followed by at most one attempt/2s
+    }
+    assert(w.arms == 36 && sleep_windows > 20000);
+    assert(w._nf_refresh_requested && w._nf_calib_deadline == 0 && w.reads == 0);
+    assert(w.getRecoveryCount() >= 7);
+    std::printf("60s actual Dispatcher mode=%u: arms=%u recoveries=%u sleep_windows=%u\n",
+                mode, w.arms, unsigned(w.getRecoveryCount()), sleep_windows);
+    // A real success clears the ramp and enters normal settled calibration.
+    w.arm_result = 0; now_ms += retryDelay(w); w.recvRaw(bytes, sizeof(bytes));
+    assert(w.state == STATE_RX && !w._rx_retry_pending && w._rx_retry_delay_ms == 0);
+    if (mode == 1) { w.loop(); w.recvRaw(bytes, sizeof(bytes)); }
+    assert(w.isCalibratingNoiseFloor());
+    w.loop();
+    assert(w._nf_calib_deadline == now_ms + NF_CONTINUOUS_TIMEOUT_MS);
+    now_ms += NF_CALIB_SETTLE_MS; w.loop();
+    assert(w._floor_estimator.count() == 1);
+  }
+}
+
+static void rampAndRollover() {
+  uint8_t bytes[8];
+  for (uint32_t origin : {100U, UINT32_MAX - 19U}) {
+    RadioLibWrapper w; w.state = STATE_IDLE; w.arm_result = -5; now_ms = origin;
+    for (uint32_t expected : {20U, 40U, 80U, 160U, 320U, 640U, 1280U, 2000U, 2000U}) {
+      const unsigned arms = w.arms;
+      w.recvRaw(bytes, sizeof(bytes));
+      assert(w.arms == arms + 1 && retryDelay(w) == expected);
+      // Periodic/forced calibration, recovery and watchdog counter resets
+      // cannot turn the existing failed-arm deadline into an immediate retry.
+      w.triggerNoiseFloorCalibrate(7); w.recalibrateNoiseFloor(); w.resetAGC();
+      w.recoverRadio(false); w._startrx_fails = 0;
+      assert(retryDelay(w) == expected && w._rx_retry_delay_ms == expected);
+      now_ms += expected - 1;
+      w.recvRaw(bytes, sizeof(bytes)); assert(w.arms == arms + 1 && retryDelay(w) == 1);
+      ++now_ms; assert(retryDelay(w) == 0);
+    }
+    w.begin(); assert(!w._rx_retry_pending && w._rx_retry_delay_ms == 0);
+    w.recvRaw(bytes, sizeof(bytes)); assert(retryDelay(w) == 20);
+  }
+}
+
+static void ownersAndImmediateRearms() {
+  uint8_t bytes[8];
+  for (bool busy : {false, true}) {
+    RadioLibWrapper w; w.state = STATE_IDLE; w.arm_result = -5;
+    w.recvRaw(bytes, sizeof(bytes)); now_ms += retryDelay(w);
+    w.busy = busy; w.packet = !busy;
+    const unsigned arms = w.arms;
+    w.recvRaw(bytes, sizeof(bytes));
+    assert(w.arms == arms && retryDelay(w) == 2000 && !w.hasPendingRadioInterrupt());
+    w.busy = w.packet = false;
+    // A completed packet bypasses the timer and is delivered with its owner.
+    w.events.record(); w.radio.packet_length = 1; w.arm_result = 0;
+    assert(w.recvRaw(bytes, sizeof(bytes)) == 1 && w.arms == arms + 1);
+    assert(!w._rx_retry_pending && w._rx_retry_delay_ms == 0 && !w.hasPendingRadioInterrupt());
+  }
+  {
+    RadioLibWrapper w; w.state = STATE_IDLE; w.arm_result = -5;
+    w.recvRaw(bytes, sizeof(bytes));
+    w.state = STATE_TX_WAIT; w.events.record();
+    const unsigned arms = w.arms;
+    assert(w.recvRaw(bytes, sizeof(bytes)) == 0 && w.arms == arms && w.hasPendingRadioInterrupt());
+    uint32_t ignored = 0; assert(!w.getReceiveRetryDelay(now_ms, ignored));
+    w.onSendFinished(); w.arm_result = 0;
+    w.recvRaw(bytes, sizeof(bytes));
+    assert(w.arms == arms + 1 && w.state == STATE_RX && !w.hasPendingRadioInterrupt());
+  }
+  {
+    RadioLibWrapper w; w.state = STATE_IDLE; w.arm_result = -5;
+    w.recvRaw(bytes, sizeof(bytes)); const unsigned arms = w.arms;
+    // Direct rearm, as used by profile/CAD restoration, has no retry wait.
+    w.arm_result = 0; w.startRecv();
+    assert(w.arms == arms + 1 && w.state == STATE_RX && !w._rx_retry_pending);
+  }
+  {
+    RadioLibWrapper w; w._rx_ps_armed = true; w.arm_result = -5;
+    w.startRecv(); assert(w.state == STATE_IDLE && !w._rx_ps_armed);
+  }
+  {
+    RadioLibWrapper w; w._nf_calib_active = true; w._rx_ps_armed = true;
+    assert(!w.isCalibratingNoiseFloor());
+    w.state = STATE_IDLE; w._rx_ps_armed = false; assert(!w.isCalibratingNoiseFloor());
+    w.state = STATE_RX; assert(w.isCalibratingNoiseFloor());
+    w._cw_active = true; uint32_t delay = 0; assert(!w.getReceiveRetryDelay(now_ms, delay));
+  }
+}
+
+static void pendingSetupAndTx() {
+  RadioLibWrapper w; w.begin(); now_ms += 100000;
+  assert(!w.isCalibratingNoiseFloor() && w._nf_calib_deadline == 0);
+  w.loop(); assert(w._nf_refresh_requested && w._nf_calib_deadline == 0);
+  w.state = STATE_TX_WAIT; now_ms += 100000; w.loop();
+  assert(!w.isCalibratingNoiseFloor() && w._nf_refresh_requested && w._nf_calib_deadline == 0);
+  w.onSendFinished(); w.startRecv(); w.loop();
+  assert(w._nf_calib_deadline == now_ms + NF_CONTINUOUS_TIMEOUT_MS);
+}
+
+static void retainedOutboundWake() {
+  RadioLibWrapper w; RetryClock clock; StaticPoolPacketManager manager{4};
+  RetryDispatcher d(w, clock, manager);
+  now_ms = 1000; d.begin(); w.arm_result = -5;
+  d.loop();
+  auto* packet = d.obtainNewPacket(); assert(packet);
+  packet->header = ROUTE_TYPE_DIRECT | (PAYLOAD_TYPE_RAW_CUSTOM << PH_TYPE_SHIFT);
+  packet->payload_len = 1; packet->payload[0] = 0x42;
+  assert(d.sendPacket(packet, 0)); d.loop(); assert(w.sends == 1 && d.hasOutbound());
+  uint32_t delay = 0;
+  assert(d.getNextQueueWakeDelay(delay) && delay == 20);
+  now_ms += 20; d.loop();
+  assert(w.arms == 2 && d.getNextQueueWakeDelay(delay) && delay == 40);
+}
+
+static void busyAfterFailure() {
+  RadioLibWrapper w; RetryClock clock; StaticPoolPacketManager manager{4};
+  RetryDispatcher d(w, clock, manager);
+  now_ms = 1000; d.begin(); w.arm_result = -5; d.loop(); w.busy = true;
+  unsigned sleep_windows = 0;
+  for (unsigned elapsed = 1; elapsed <= 30000; ++elapsed) {
+    now_ms = 1000 + elapsed; d.loop();
+    uint32_t delay = 0; assert(d.getNextQueueWakeDelay(delay));
+    if (elapsed > 8000) sleep_windows += mesh::power::powerSaveSleepSeconds(delay) > 0;
+    assert(w.arms == 1 && !w.isCalibratingNoiseFloor());
+  }
+  // A due AGC reset cannot admit SPI on BUSY. Its deferred deadline must join
+  // the failed-RX deadline instead of vetoing every timed sleep at 50 ms.
+  assert(sleep_windows > 8000);
+  w.busy = false; w.arm_result = 0; now_ms += retryDelay(w); d.loop();
+  assert(w.state == STATE_RX && !w._rx_retry_pending && w._rx_retry_delay_ms == 0);
+  w._nf_refresh_requested = false; w._nf_calib_active = false;
+  now_ms += 1; d.loop();
+  uint32_t ignored = 0; assert(!w.getReceiveRetryDelay(now_ms, ignored));
+  // The healthy non-RX-retry deferral contract remains 50 ms.
+  w.packet = true; now_ms += 10000; d.loop();
+  assert(d.getNextQueueWakeDelay(ignored) && ignored == 50);
+  std::printf("30s BUSY after RX error: arms=%u later_sleep_windows=%u\n", w.arms, sleep_windows);
+}
+
+int main() {
+  persistentFailure(); rampAndRollover(); ownersAndImmediateRearms();
+  pendingSetupAndTx(); retainedOutboundWake(); busyAfterFailure();
+  std::puts("PASS: receive retry lifecycle");
+}

test/test_agc_reset_scheduler/test_agc_reset_scheduler.cpp

@@ -216,6 +216,36 @@ TEST(AGCResetScheduler, DeferredRetriesWaitFiftyMillisWithoutPromotingStage) {
   EXPECT_TRUE(scheduler.due(39));
 }
 
+TEST(AGCResetScheduler, ReceiveRetryDeferralPreservesStageAndValidatesDelay) {
+  for (uint32_t origin : {100u, UINT32_MAX - 1999u}) {
+    AGCResetScheduler scheduler;
+    scheduler.configure(origin, 8000, true, 0);
+    scheduler.noteAttempt(origin + 8000u); // one admitted attempt, no promotion yet
+    scheduler.noteDeferred(origin, 2000);
+    EXPECT_EQ(scheduler.interval(), 8000u);
+    EXPECT_EQ(scheduler.nextAt(), origin + 2000u);
+    EXPECT_FALSE(scheduler.due(origin + 1999u));
+    EXPECT_TRUE(scheduler.due(origin + 2000u));
+    scheduler.noteAttempt(origin + 2000u);
+    EXPECT_EQ(scheduler.interval(), 16000u); // deferral retained the first attempt
+  }
+  for (uint32_t requested : {0u, uint32_t(INT32_MAX) + 1u, UINT32_MAX}) {
+    AGCResetScheduler scheduler;
+    scheduler.configure(0, 8000, true, 0);
+    scheduler.noteDeferred(UINT32_MAX - 10u, requested);
+    EXPECT_EQ(scheduler.nextAt(), 39u);
+    EXPECT_FALSE(scheduler.due(38));
+    EXPECT_TRUE(scheduler.due(39));
+  }
+  AGCResetScheduler scheduler;
+  scheduler.configure(0, 8000, true, 0);
+  scheduler.noteDeferred(100, INT32_MAX);
+  EXPECT_FALSE(scheduler.due(100 + uint32_t(INT32_MAX) - 1));
+  EXPECT_TRUE(scheduler.due(100 + uint32_t(INT32_MAX)));
+  scheduler.reset(); scheduler.noteDeferred(100, 2000);
+  uint32_t ignored = 0; EXPECT_FALSE(scheduler.nextDelay(100, ignored));
+}
+
 TEST(AGCResetScheduler, LateServiceDoesNotReplayMissedResetSlots) {
   AGCResetScheduler scheduler;
   scheduler.configure(10, 8000, true, 0);

test/test_beacon_radio.py

@@ -88,6 +88,9 @@ struct RadioLibWrapper {
     bool _rx_boosted_gain_valid = true, _cur_rx_boosted_gain = true;
     bool _wd_last_busy = false;
     uint8_t _startrx_fails = 0, _irq_probe_failures = 0;
+    uint32_t _rx_retry_at = 0;
+    uint16_t _rx_retry_delay_ms = 0;
+    bool _rx_retry_pending = false;
     uint32_t _rx_ps_rx_us = 50000, _rx_ps_sleep_us = 50000;
     uint32_t _wd_last_transition = 0, _wd_stuck_thresh = 0, _wd_observe_ms = 0;
     uint32_t n_wd_soft = 0, n_wd_hard = 0;

test/test_radio_interrupt_recovery.py

@@ -108,6 +108,9 @@ struct RadioLibWrapper {
  bool _rx_ps_continuous_fallback=false, _nf_calib_active=false;
  float _last_snr=0, _last_rssi=0;
  unsigned n_recv=0,n_recv_errors=0,n_sent=0;
+ uint32_t _rx_retry_at=0;
+ uint16_t _rx_retry_delay_ms=0;
+ bool _rx_retry_pending=false;
  uint32_t last_recv_millis=0,last_radio_interrupt_millis=0,_nf_sample_from=0;
  Board board;Board* _board=&board;
  Chip chip;Chip* _radio=&chip;
@@ -121,6 +124,8 @@ public:
  int recvRaw(uint8_t*,int);
  void onSendFinished();
  void idle(){state=STATE_IDLE;}
+ bool isChipBusy(){return false;}
+ bool isPacketPendingOrReceiving(){return hasPendingRadioInterrupt();}
  bool recoverRadio(bool){return !hasPendingRadioInterrupt();}
  void stopReceiveDutyCycle(){_rx_ps_armed=false;chip.standby();}
  void startRecv(){if(chip.startReceive()==0)state=STATE_RX;}

test/test_radio_receive_contract.py

@@ -84,6 +84,9 @@ struct RadioLibWrapper {
   bool _rx_boosted_gain_valid = false, _cur_rx_boosted_gain = false;
   bool _wd_last_busy = false;
   uint8_t _startrx_fails = 0, _rx_mode_failures = 0;
+  uint32_t _rx_retry_at = 0;
+  uint16_t _rx_retry_delay_ms = 0;
+  bool _rx_retry_pending = false;
   uint32_t _rx_mode_checked_at = 0, _rx_ps_rx_us = 50000, _rx_ps_sleep_us = 50000;
   unsigned long _nf_last_calib = 0, _nf_calib_deadline = 0, _nf_sample_from = 0;
   unsigned long _wd_last_transition = 0, _wd_stuck_thresh = 0, _wd_observe_ms = 0;

test/test_radio_receive_retry.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""Run failed RX starts through actual Dispatcher and radio lifecycle methods."""
+from pathlib import Path
+import os
+import re
+import subprocess
+import tempfile
+import unittest
+
+from test_radio_receive_contract import HARNESS, ROOT, method
+
+
+class RadioReceiveRetryTests(unittest.TestCase):
+    def build(self, folder, mutation=None):
+        source = (ROOT / 'src/helpers/radiolib/RadioLibWrappers.cpp').read_text()
+        header = (ROOT / 'src/helpers/radiolib/RadioLibWrappers.h').read_text()
+        dispatcher = (ROOT / 'src/Dispatcher.h').read_text()
+        rxps = (ROOT / 'src/helpers/radiolib/RXPowerSaving.h').read_text()
+        if mutation == 'unpaced':
+            source = source.replace('    if (_rx_retry_pending) {', '    if (false) {')
+        getter = method(header, 'bool isCalibratingNoiseFloor() const')
+        if mutation == 'pending_veto':
+            getter = getter.replace('isInRecvMode() && !_rx_ps_armed && ', '')
+        names = [('int16_t', 'performChannelScanWithTimeout'),
+                 ('bool', 'isPacketPendingOrReceiving'), ('bool', 'isChannelActive'),
+                 ('bool', 'isReceivingPassive'), ('bool', 'setRxPowerSaving'),
+                 ('bool', 'setRxBoostedGainMode'), ('void', 'startRecv'),
+                 ('void', 'checkReceiveMode'), ('void', 'loop'),
+                 ('void', 'triggerNoiseFloorCalibrate'), ('void', 'requestRestartRecv'),
+                 ('void', 'noiseFloorCalibCheck'), ('void', 'endNoiseFloorCalib'),
+                 ('void', 'requestNoiseFloorRefresh'), ('void', 'requestPeriodicNoiseFloorRefresh'),
+                 ('void', 'recalibrateNoiseFloor'), ('bool', 'resetAGCIfIdle'),
+                 ('bool', 'tryResetAGC'), ('void', 'resetAGC'), ('int', 'recvRaw'),
+                 ('void', 'onSendFinished'), ('void', 'begin'),
+                 ('bool', 'getReceiveRetryDelay'), ('void', 'rxPsWatchdogCheck')]
+        methods = method(rxps, 'inline bool rxPowerSavingUsesContinuousFallback(') + '\n'
+        methods += '\n'.join(method(source, f'{kind} RadioLibWrapper::{name}(')
+                             for kind, name in names)
+        constants = '\n'.join(re.findall(r'^#define NF_\w+\s+[^\n]+', source, re.MULTILINE))
+        constants += '\n' + re.search(
+            r'^#define RX_POWERSAVING_CONTINUOUS_FALLBACK_US\s+[^\n]+', rxps, re.MULTILINE).group(0)
+        start = header.index('#if defined(USE_SX1262) || defined(USE_SX1268) || defined(USE_LLCC68)')
+        settle = header[start:header.index('#endif', start) + len('#endif')]
+        harness = HARNESS[:HARNESS.index('int main() {')]
+        harness = harness.replace('#include <RadioProfiles.h>',
+            '#include <Dispatcher.h>\n#include <helpers/StaticPoolPacketManager.h>\n'
+            '#include <helpers/PowerManagementUtils.h>\n#define BD_STARTUP_RX_PACKET 1')
+        harness = harness.replace('struct RadioLibWrapper {', 'struct RadioLibWrapper : mesh::Radio {')
+        harness = harness.replace('Radio radio{&events}; Radio* _radio',
+                                  '::Radio radio{&events}; ::Radio* _radio')
+        harness = harness.replace('unsigned long getEstAirtimeFor(int)', 'uint32_t getEstAirtimeFor(int)')
+        harness = harness.replace('void rxPsWatchdogCheck() {}', 'void rxPsWatchdogCheck();')
+        harness = harness.replace('  void serviceWatchdog()',
+            '  unsigned after = 0;\n  int getStartupReason() { return 0; }\n'
+            '  void onAfterTransmit() { ++after; }\n  void serviceWatchdog()')
+        harness = harness.replace('  int startChannelScan()', '''
+  unsigned packet_length = 0, direct_arms = 0;
+  int startReceive() { ++direct_arms; return start_result; }
+  int setPreambleLength(uint16_t) { return 0; }
+  int finishTransmit() { return 0; }
+  size_t getPacketLength() { return packet_length; }
+  float getSNR() { return 0; }
+  float getRSSI() { return 0; }
+  int readData(uint8_t* out, int len) { for (int i=0;i<len;++i) out[i]=0x42; return 0; }
+  int startChannelScan()''')
+        harness = harness.replace('return events.pending();', 'return _loop_event_pending || events.pending();')
+        harness = harness.replace('bool claimRadioInterrupt() { return events.claim(); }', '''
+  bool claimRadioInterrupt() {
+    const bool pending = _loop_event_pending; _loop_event_pending = false;
+    return events.claim() || pending;
+  }''')
+        harness = harness.replace('  @RECOVERY_GETTER@', '''
+  int recvRaw(uint8_t*, int) override;
+  void begin() override;
+  void onSendFinished() override;
+  bool getReceiveRetryDelay(uint32_t, uint32_t&) const override;
+  bool registerInterruptAction() { _loop_event_pending=false; events.claim(); return true; }
+  uint16_t currentPreambleLength() const { return 32; }
+  bool isPacketReady() { return true; }
+  bool isInRecvMode() const override { return state == STATE_RX; }
+  bool startSendRaw(const uint8_t*, int) override { ++sends; return false; }
+  bool isSendComplete() override { return false; }
+  float packetScore(float, int) override { return 0; }
+  bool _rx_hold_continuous = false, _loop_event_pending = false;
+  uint32_t last_radio_interrupt_millis = 0, last_recv_millis = 0;
+  float _last_snr = 0, _last_rssi = 0;
+  unsigned n_recv = 0, n_recv_errors = 0, sends = 0;
+  uint8_t _wd_stage = 0, _wd_strikes = 0, _irq_probe_failures = 0;
+  unsigned long _wd_observe_until = 0;
+  ''' + getter.replace(' override', '') + '\n  @RECOVERY_GETTER@')
+        harness = harness.replace(
+            'bool recoverRadio(bool use_hard) { use_hard ? ++hard : ++soft; return true; }',
+            method(source, 'bool RadioLibWrapper::recoverRadio(').replace('RadioLibWrapper::', '')
+            + '\n bool supportsRadioDeepInit() const { return false; }\n'
+            + ' bool restoreAfterDeepInit() { recalibrateNoiseFloor(); return false; }')
+        harness = (harness.replace('@METHODS@', methods).replace('@NF_CONSTANTS@', constants)
+            .replace('@SETTLE_CONSTANT@', settle)
+            .replace('@RECOVERY_GETTER@', method(header, 'uint32_t getRecoveryCount() const'))
+            .replace('@LEGACY_TRY_RESET@', method(dispatcher, 'virtual bool tryResetAGC('))
+            .replace('@LEGACY_RECOVERY_GETTER@', method(dispatcher, 'virtual uint32_t getRecoveryCount(')))
+        harness += (ROOT / 'test/fixtures/radio_receive_retry.cpp').read_text()
+        folder = Path(folder)
+        cpp, binary = folder / 'retry.cpp', folder / 'retry'
+        cpp.write_text(harness.replace('\u2014', '-'), encoding='ascii')
+        dispatcher_source = ROOT / 'src/Dispatcher.cpp'
+        if mutation == 'agc_fast':
+            original = dispatcher_source.read_text()
+            alignment = ('      if (adaptive && _radio->getReceiveRetryDelay(deferred_at, receive_delay)\n'
+                         '          && receive_delay > retry_delay) retry_delay = receive_delay;\n')
+            self.assertEqual(original.count(alignment), 1)
+            dispatcher_source = folder / 'Dispatcher.cpp'
+            dispatcher_source.write_text(original.replace(alignment, ''), encoding='ascii')
+        built = subprocess.run([os.environ.get('CXX', 'g++'), '-std=c++17', '-O1',
+            '-DUSE_SX1262=1', '-fsanitize=address,undefined', '-fno-sanitize-recover=all',
+            '-fno-pie', '-no-pie', '-I', str(ROOT / 'test/mocks'), '-I', str(ROOT / 'src'),
+            str(cpp), str(dispatcher_source), str(ROOT / 'src/Packet.cpp'),
+            str(ROOT / 'src/helpers/StaticPoolPacketManager.cpp'), '-o', str(binary)],
+            capture_output=True, text=True, timeout=60)
+        self.assertEqual(built.returncode, 0, built.stderr)
+        return binary
+
+    def run_probe(self, binary):
+        env = os.environ.copy()
+        env['ASAN_OPTIONS'] = env.get('ASAN_OPTIONS', '') + ':detect_leaks=0'
+        return subprocess.run([str(binary)], capture_output=True, text=True, env=env, timeout=20)
+
+    def test_actual_dispatcher_and_receive_lifecycle(self):
+        with tempfile.TemporaryDirectory(prefix='meshcore-rx-retry-') as folder:
+            result = self.run_probe(self.build(folder))
+            self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+            self.assertIn('PASS: receive retry lifecycle', result.stdout)
+            print(result.stdout, end='')
+
+    def test_old_hot_poll_and_pending_calibration_veto_negative_controls(self):
+        for mutation in ('unpaced', 'pending_veto', 'agc_fast'):
+            with self.subTest(mutation=mutation), tempfile.TemporaryDirectory(
+                    prefix='meshcore-rx-retry-negative-') as folder:
+                result = self.run_probe(self.build(folder, mutation))
+                self.assertNotEqual(result.returncode, 0)
+                self.assertIn('busyAfterFailure' if mutation == 'agc_fast'
+                              else 'persistentFailure', result.stderr)
+
+
+if __name__ == '__main__':
+    unittest.main()