From 7e4f75c9a5f31f74580de833f250930e98447a68 Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 10 Apr 2026 08:48:22 -0700 Subject: [PATCH] Add radio diagnostics support in MyMesh and CommonCLI Implement a new method `formatRadioDiagReply` in MyMesh to format and return radio diagnostic information. Update CommonCLI to handle the new command for retrieving radio diagnostics. Enhance the Dispatcher class with a radio watchdog feature to monitor and recover from silent radio states. Introduce new error flags and methods in the Dispatcher and RadioLibWrapper classes to support these diagnostics, improving overall radio management and troubleshooting capabilities. --- examples/simple_repeater/MyMesh.cpp | 4 ++++ examples/simple_repeater/MyMesh.h | 1 + src/Dispatcher.cpp | 13 +++++++++++++ src/Dispatcher.h | 14 ++++++++++++++ src/helpers/CommonCLI.cpp | 13 ++++++++++--- src/helpers/CommonCLI.h | 1 + src/helpers/StatsFormatHelper.h | 18 ++++++++++++++++++ src/helpers/bridges/MQTTBridge.cpp | 12 ++++++++++++ src/helpers/radiolib/RadioLibWrappers.cpp | 5 +++++ src/helpers/radiolib/RadioLibWrappers.h | 11 ++++++++--- 10 files changed, 86 insertions(+), 6 deletions(-) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 8c06bce9..fe14f7cb 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1196,6 +1196,10 @@ void MyMesh::formatRadioStatsReply(char *reply) { StatsFormatHelper::formatRadioStats(reply, _radio, radio_driver, getTotalAirTime(), getReceiveAirTime()); } +void MyMesh::formatRadioDiagReply(char *reply) { + StatsFormatHelper::formatRadioDiag(reply, _radio, radio_driver, *_ms, _err_flags, hasOutbound()); +} + void MyMesh::formatPacketStatsReply(char *reply) { StatsFormatHelper::formatPacketStats(reply, radio_driver, getNumSentFlood(), getNumSentDirect(), getNumRecvFlood(), getNumRecvDirect()); diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index d957f6d9..c6c8db5b 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -220,6 +220,7 @@ public: void removeNeighbor(const uint8_t* pubkey, int key_len) override; void formatStatsReply(char *reply) override; void formatRadioStatsReply(char *reply) override; + void formatRadioDiagReply(char *reply) override; void formatPacketStatsReply(char *reply) override; mesh::LocalIdentity& getSelfId() override { return self_id; } diff --git a/src/Dispatcher.cpp b/src/Dispatcher.cpp index 35eca0a9..a837d47a 100644 --- a/src/Dispatcher.cpp +++ b/src/Dispatcher.cpp @@ -58,6 +58,19 @@ void Dispatcher::loop() { _err_flags |= ERR_EVENT_STARTRX_TIMEOUT; } + // Radio watchdog: detect radio stuck in RX mode but not receiving any packets + if (is_recv && _radio->getLastRecvMillis() > 0) { + unsigned long silent_ms = _ms->getMillis() - _radio->getLastRecvMillis(); + unsigned long since_recovery = _ms->getMillis() - last_watchdog_recovery; + if (silent_ms > RADIO_WATCHDOG_MS && since_recovery > RADIO_WATCHDOG_MS) { + _err_flags |= ERR_EVENT_RADIO_WATCHDOG; + MESH_DEBUG_PRINTLN("Radio watchdog: silent %lu ms, state=%d, recovering", silent_ms, _radio->getRadioState()); + _radio->idle(); + _radio->startRecv(); + last_watchdog_recovery = _ms->getMillis(); + } + } + if (outbound) { // waiting for outbound send to be completed if (_radio->isSendComplete()) { long t = _ms->getMillis() - outbound_start; diff --git a/src/Dispatcher.h b/src/Dispatcher.h index 4cde40bc..dbbc4b6f 100644 --- a/src/Dispatcher.h +++ b/src/Dispatcher.h @@ -61,12 +61,18 @@ public: */ virtual void loop() { } + virtual void idle() { } + virtual void startRecv() { } + virtual int getNoiseFloor() const { return 0; } virtual void triggerNoiseFloorCalibrate(int threshold) { } virtual void resetAGC() { } + virtual uint8_t getRadioState() const { return 0; } + virtual unsigned long getLastRecvMillis() const { return 0; } + virtual bool isInRecvMode() const = 0; /** @@ -112,6 +118,11 @@ typedef uint32_t DispatcherAction; #define ERR_EVENT_FULL (1 << 0) #define ERR_EVENT_CAD_TIMEOUT (1 << 1) #define ERR_EVENT_STARTRX_TIMEOUT (1 << 2) +#define ERR_EVENT_RADIO_WATCHDOG (1 << 3) + +#ifndef RADIO_WATCHDOG_MS + #define RADIO_WATCHDOG_MS 300000 // 5 minutes +#endif /** * \brief The low-level task that manages detecting incoming Packets, and the queueing @@ -120,6 +131,7 @@ typedef uint32_t DispatcherAction; class Dispatcher { Packet* outbound; // current outbound packet unsigned long outbound_expiry, outbound_start, total_air_time, rx_air_time; + unsigned long last_watchdog_recovery; unsigned long next_tx_time; unsigned long cad_busy_start; unsigned long radio_nonrx_start; @@ -147,6 +159,7 @@ protected: _err_flags = 0; radio_nonrx_start = 0; prev_isrecv_mode = true; + last_watchdog_recovery = 0; } virtual DispatcherAction onRecvPacket(Packet* pkt) = 0; @@ -180,6 +193,7 @@ public: uint32_t getNumRecvFlood() const { return n_recv_flood; } uint32_t getNumRecvDirect() const { return n_recv_direct; } uint16_t getErrFlags() const { return _err_flags; } // Get error flags + bool hasOutbound() const { return outbound != NULL; } void resetStats() { n_sent_flood = n_sent_direct = n_recv_flood = n_recv_direct = 0; _err_flags = 0; diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 2996a0ef..c19b0f21 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #endif #ifdef WITH_MQTT_BRIDGE #include "bridges/MQTTBridge.h" @@ -643,9 +644,13 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, const char* command, ch strcpy(reply, "ERR: clock cannot go backwards"); } } else if (memcmp(command, "memory", 6) == 0) { - sprintf(reply, "Free: %d, Min: %d, Max: %d, Queue: %d", - ESP.getFreeHeap(), ESP.getMinFreeHeap(), ESP.getMaxAllocHeap(), - _callbacks->getQueueSize()); + sprintf(reply, "Free: %d, Min: %d, Max: %d, Queue: %d, IntFree: %d, IntMax: %d, PSRAM: %d/%d", + ESP.getFreeHeap(), ESP.getMinFreeHeap(), ESP.getMaxAllocHeap(), + _callbacks->getQueueSize(), + (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL), + (int)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL), + (int)heap_caps_get_free_size(MALLOC_CAP_SPIRAM), + (int)heap_caps_get_total_size(MALLOC_CAP_SPIRAM)); } else if (memcmp(command, "tls.bundletest ", 15) == 0) { #ifdef ESP_PLATFORM if (WiFi.status() != WL_CONNECTED) { @@ -1708,6 +1713,8 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, const char* command, ch strcpy(reply, " EOF"); } else if (sender_timestamp == 0 && memcmp(command, "stats-packets", 13) == 0 && (command[13] == 0 || command[13] == ' ')) { _callbacks->formatPacketStatsReply(reply); + } else if (sender_timestamp == 0 && memcmp(command, "stats-radio-diag", 16) == 0 && (command[16] == 0 || command[16] == ' ')) { + _callbacks->formatRadioDiagReply(reply); } else if (sender_timestamp == 0 && memcmp(command, "stats-radio", 11) == 0 && (command[11] == 0 || command[11] == ' ')) { _callbacks->formatRadioStatsReply(reply); } else if (sender_timestamp == 0 && memcmp(command, "stats-core", 10) == 0 && (command[10] == 0 || command[10] == ' ')) { diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index 966212db..1f80dc72 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -235,6 +235,7 @@ public: }; virtual void formatStatsReply(char *reply) = 0; virtual void formatRadioStatsReply(char *reply) = 0; + virtual void formatRadioDiagReply(char *reply) { strcpy(reply, "Not supported"); } virtual void formatPacketStatsReply(char *reply) = 0; virtual mesh::LocalIdentity& getSelfId() = 0; virtual void saveIdentity(const mesh::LocalIdentity& new_id) = 0; diff --git a/src/helpers/StatsFormatHelper.h b/src/helpers/StatsFormatHelper.h index 5aa01da9..7cce0552 100644 --- a/src/helpers/StatsFormatHelper.h +++ b/src/helpers/StatsFormatHelper.h @@ -34,6 +34,24 @@ public: ); } + template + static void formatRadioDiag(char* reply, + mesh::Radio* radio, + RadioDriverType& driver, + mesh::MillisecondClock& ms, + uint16_t err_flags, + bool has_outbound) { + uint8_t st = radio->getRadioState(); + const char* state_name = (st & 16) ? "INT_READY" : (st == 0 ? "IDLE" : (st == 1 ? "RX" : (st == 3 ? "TX_WAIT" : "?"))); + unsigned long last_rx = radio->getLastRecvMillis(); + unsigned long ago_secs = (last_rx > 0) ? (ms.getMillis() - last_rx) / 1000 : 0; + sprintf(reply, + "state=%d(%s), recv=%u, sent=%u, errors=%u, err_flags=%u, outbound=%s, last_rx=%lus ago", + st, state_name, + driver.getPacketsRecv(), driver.getPacketsSent(), driver.getPacketsRecvErrors(), + err_flags, has_outbound ? "yes" : "no", ago_secs); + } + template static void formatPacketStats(char* reply, RadioDriverType& driver, diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 5418bd09..9e0efde5 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1289,6 +1289,9 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns _last_slot_reconnect_ms = now_millis; MQTT_DEBUG_PRINTLN("MQTT%d int_heap=%d at token renewal reconnect", index + 1, (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); + MQTT_DEBUG_PRINTLN(" radio_state=%d, last_rx=%lums ago", + _radio ? _radio->getRadioState() : -1, + (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0); } else { // Token renewed but old one still valid — just update credentials for next reconnect slot.client->setCredentials(_jwt_username, slot.auth_token); @@ -1314,6 +1317,9 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns _last_slot_reconnect_ms = now_millis; MQTT_DEBUG_PRINTLN("MQTT%d circuit breaker probe (attempting single reconnect after %lu ms, int_heap=%d)", index + 1, probe_elapsed, (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); + MQTT_DEBUG_PRINTLN(" radio_state=%d, last_rx=%lums ago", + _radio ? _radio->getRadioState() : -1, + (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0); if (slot_uses_jwt) { unsigned long current_time = time(nullptr); bool token_still_valid = slot.token_expires_at > 0 && @@ -1378,6 +1384,9 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns } MQTT_DEBUG_PRINTLN("MQTT%d reconnecting (backoff level %d, failures at max: %d, int_heap=%d)", index + 1, slot.reconnect_backoff, slot.max_backoff_failures, (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); + MQTT_DEBUG_PRINTLN(" radio_state=%d, last_rx=%lums ago", + _radio ? _radio->getRadioState() : -1, + (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0); reconnect_attempted = true; _last_slot_reconnect_ms = now_millis; if (slot_uses_jwt) { @@ -1805,6 +1814,9 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) { if (!_wifi_status_initialized) { _last_wifi_status = current_wifi_status; _wifi_status_initialized = true; + if (current_wifi_status != WL_CONNECTED) { + _wifi_disconnected_time = now; + } } if (now - _last_wifi_check <= 10000) { return false; diff --git a/src/helpers/radiolib/RadioLibWrappers.cpp b/src/helpers/radiolib/RadioLibWrappers.cpp index 2216ca8f..30995fd0 100644 --- a/src/helpers/radiolib/RadioLibWrappers.cpp +++ b/src/helpers/radiolib/RadioLibWrappers.cpp @@ -120,6 +120,7 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) { } else { // Serial.print(" readData() -> "); Serial.println(len); n_recv++; + last_recv_millis = millis(); } } state = STATE_IDLE; // need another startReceive() @@ -174,6 +175,10 @@ bool RadioLibWrapper::isChannelActive() { : getCurrentRSSI() > _noise_floor + _threshold; } +uint8_t RadioLibWrapper::getRadioState() const { + return state; +} + float RadioLibWrapper::getLastRSSI() const { return _radio->getRSSI(); } diff --git a/src/helpers/radiolib/RadioLibWrappers.h b/src/helpers/radiolib/RadioLibWrappers.h index 29e77cec..405bb0de 100644 --- a/src/helpers/radiolib/RadioLibWrappers.h +++ b/src/helpers/radiolib/RadioLibWrappers.h @@ -11,15 +11,17 @@ protected: int16_t _noise_floor, _threshold; uint16_t _num_floor_samples; int32_t _floor_sample_sum; + unsigned long last_recv_millis; - void idle(); - void startRecv(); float packetScoreInt(float snr, int sf, int packet_len); virtual bool isReceivingPacket() =0; virtual void doResetAGC(); public: - RadioLibWrapper(PhysicalLayer& radio, mesh::MainBoard& board) : _radio(&radio), _board(&board) { n_recv = n_sent = 0; } + RadioLibWrapper(PhysicalLayer& radio, mesh::MainBoard& board) : _radio(&radio), _board(&board) { n_recv = n_sent = 0; last_recv_millis = 0; } + + void idle() override; + void startRecv() override; void begin() override; virtual void powerOff() { _radio->sleep(); } @@ -50,6 +52,9 @@ public: uint32_t getPacketsSent() const { return n_sent; } void resetStats() { n_recv = n_sent = n_recv_errors = 0; } + uint8_t getRadioState() const override; + unsigned long getLastRecvMillis() const override { return last_recv_millis; } + virtual float getLastRSSI() const override; virtual float getLastSNR() const override;