rich/MeshCore
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden nRF52 storage and hardware recovery

Browse Source
This commit is contained in:
mikecarper committed 2026-07-21 00:01:01 -07:00
1 parent d5a00da2a0
commit bda0722ad8
40 files changed
+2069 -275
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/Dispatcher.cpp
+61 -28
View File
@@ -16,6 +16,13 @@ namespace mesh {
#define NOISE_FLOOR_CALIB_INTERVAL 2000 // refresh every 2 seconds
#endif
#ifndef RADIO_LIVENESS_SOFT_MS
#define RADIO_LIVENESS_SOFT_MS (30UL * 60UL * 1000UL)
#endif
#ifndef RADIO_LIVENESS_HARD_MS
#define RADIO_LIVENESS_HARD_MS (12UL * 60UL * 60UL * 1000UL)
#endif
void Dispatcher::begin() {
n_sent_flood = n_sent_direct = 0;
n_recv_flood = n_recv_direct = 0;
@@ -29,12 +36,11 @@ void Dispatcher::begin() {
_radio->begin();
prev_isrecv_mode = _radio->isInRecvMode();
#ifdef WITH_MQTT_BRIDGE
// Use begin() as the watchdog baseline even if the radio never reports an
// RX/IRQ timestamp. This lets a radio that is dead from startup recover.
last_radio_active_ms = _ms->getMillis();
last_watchdog_recovery = last_radio_active_ms;
#endif
last_observed_radio_irq = _radio->getLastRadioInterruptMillis();
radio_liveness.begin(last_radio_active_ms);
nonrx_soft_recovery_attempted = false;
}
float Dispatcher::getAirtimeBudgetFactor() const {
@@ -130,43 +136,67 @@ void Dispatcher::loop() {
}
_radio->loop();
const unsigned long now = _ms->getMillis();
const unsigned long latest_irq = _radio->getLastRadioInterruptMillis();
if (latest_irq != 0 && latest_irq != last_observed_radio_irq) {
last_observed_radio_irq = latest_irq;
last_radio_active_ms = now;
radio_liveness.noteActivity(now);
}
// check for radio 'stuck' in mode other than Rx
bool is_recv = _radio->isInRecvMode();
if (is_recv != prev_isrecv_mode) {
prev_isrecv_mode = is_recv;
if (!is_recv) {
radio_nonrx_start = _ms->getMillis();
radio_nonrx_start = now;
} else {
nonrx_soft_recovery_attempted = false;
}
}
if (!is_recv && _ms->getMillis() - radio_nonrx_start > 8000) { // radio has not been in Rx mode for 8 seconds!
bool recovered_this_loop = false;
if (!is_recv && outbound == NULL && now - radio_nonrx_start > 8000) {
_err_flags |= ERR_EVENT_STARTRX_TIMEOUT;
const bool hard = nonrx_soft_recovery_attempted;
MESH_DEBUG_PRINTLN("Radio watchdog: radio outside RX for %lu ms; %s recovery",
now - radio_nonrx_start, hard ? "hard" : "soft");
if (_radio->recoverRadio(hard)) {
last_watchdog_recovery = now;
recovered_this_loop = true;
}
nonrx_soft_recovery_attempted = true;
radio_nonrx_start = now; // bounded retry cadence if recovery is unsupported
}
// Radio watchdog: detect radio stuck in RX mode but not receiving any packets.
// Observer-only feature (gated behind WITH_MQTT_BRIDGE); configured via the
// MQTTPrefs radio_watchdog_minutes setting.
// Continuous-RX can fail while the MCU and main loop remain healthy, so a
// CPU watchdog cannot detect it. Use packet/IRQ/TX activity as proof of
// radio life and stage recovery from a harmless re-arm to a radio-only
// hardware reset when the driver supports one. Observer configuration may
// request a faster soft check.
uint32_t soft_liveness_ms = RADIO_LIVENESS_SOFT_MS;
#ifdef WITH_MQTT_BRIDGE
{
const uint32_t watchdog_ms = getRadioWatchdogMillis();
if (watchdog_ms > 0) {
const unsigned long now = _ms->getMillis();
unsigned long silent_ms = now - last_radio_active_ms;
const unsigned long last_recv = _radio->getLastRecvMillis();
const unsigned long last_irq = _radio->getLastRadioInterruptMillis();
if (last_recv != 0 && now - last_recv < silent_ms) silent_ms = now - last_recv;
if (last_irq != 0 && now - last_irq < silent_ms) silent_ms = now - last_irq;
const unsigned long since_recovery = now - last_watchdog_recovery;
if (is_recv && silent_ms >= watchdog_ms && since_recovery >= watchdog_ms) {
_err_flags |= ERR_EVENT_RADIO_WATCHDOG;
MESH_DEBUG_PRINTLN("Radio watchdog: silent %lu ms, state=%d, recovering", silent_ms, _radio->getRadioState());
_radio->idle();
_radio->startRecv();
last_watchdog_recovery = now;
last_radio_active_ms = now;
}
const uint32_t configured_watchdog_ms = getRadioWatchdogMillis();
if (configured_watchdog_ms > 0) soft_liveness_ms = configured_watchdog_ms;
#endif
uint32_t hard_liveness_ms = RADIO_LIVENESS_HARD_MS;
if (soft_liveness_ms > hard_liveness_ms / 2) {
hard_liveness_ms = soft_liveness_ms <= 0x7FFFFFFFUL
? soft_liveness_ms * 2 : 0xFFFFFFFFUL;
}
if (is_recv && outbound == NULL && !recovered_this_loop) {
const RadioRecoveryAction action = radio_liveness.poll(
now, soft_liveness_ms, hard_liveness_ms);
if (action != RadioRecoveryAction::NONE) {
const bool hard = action == RadioRecoveryAction::HARD;
_err_flags |= ERR_EVENT_RADIO_WATCHDOG;
MESH_DEBUG_PRINTLN("Radio watchdog: no hardware activity for %lu ms, state=%d, %s recovery",
now - last_radio_active_ms, _radio->getRadioState(),
hard ? "hard" : "soft");
const bool recovered = _radio->recoverRadio(hard);
if (recovered) last_watchdog_recovery = now;
if (hard) radio_liveness.noteHardRecoveryResult(now, recovered);
}
}
#endif // WITH_MQTT_BRIDGE (radio watchdog)
if (outbound) { // waiting for outbound send to be completed
if (_radio->isSendComplete()) {
@@ -192,6 +222,7 @@ void Dispatcher::loop() {
_radio->onSendFinished();
last_radio_active_ms = _ms->getMillis(); // TX success → radio is alive
radio_liveness.noteActivity(last_radio_active_ms);
restoreOutboundTxOverrides();
logTx(outbound, 2 + outbound->getPathByteLen() + outbound->payload_len);
onSendComplete(outbound);
@@ -317,6 +348,8 @@ void Dispatcher::checkRecv() {
uint8_t raw[MAX_TRANS_UNIT+1];
int len = _radio->recvRaw(raw, MAX_TRANS_UNIT);
if (len > 0) {
last_radio_active_ms = _ms->getMillis();
radio_liveness.noteActivity(last_radio_active_ms);
logRxRaw(_radio->getLastSNR(), _radio->getLastRSSI(), raw, len);
pkt = _mgr->allocNew();