Add radio diagnostics support in MyMesh and CommonCLI

Implement a new method `formatRadioDiagReply` in MyMesh to format and return radio diagnostic information. Update CommonCLI to handle the new command for retrieving radio diagnostics. Enhance the Dispatcher class with a radio watchdog feature to monitor and recover from silent radio states. Introduce new error flags and methods in the Dispatcher and RadioLibWrapper classes to support these diagnostics, improving overall radio management and troubleshooting capabilities.

examples/simple_repeater/MyMesh.cpp

@@ -1196,6 +1196,10 @@ void MyMesh::formatRadioStatsReply(char *reply) {
   StatsFormatHelper::formatRadioStats(reply, _radio, radio_driver, getTotalAirTime(), getReceiveAirTime());
 }
 
+void MyMesh::formatRadioDiagReply(char *reply) {
+  StatsFormatHelper::formatRadioDiag(reply, _radio, radio_driver, *_ms, _err_flags, hasOutbound());
+}
+
 void MyMesh::formatPacketStatsReply(char *reply) {
   StatsFormatHelper::formatPacketStats(reply, radio_driver, getNumSentFlood(), getNumSentDirect(), 
                                        getNumRecvFlood(), getNumRecvDirect());

examples/simple_repeater/MyMesh.h

@@ -220,6 +220,7 @@ public:
   void removeNeighbor(const uint8_t* pubkey, int key_len) override;
   void formatStatsReply(char *reply) override;
   void formatRadioStatsReply(char *reply) override;
+  void formatRadioDiagReply(char *reply) override;
   void formatPacketStatsReply(char *reply) override;
 
   mesh::LocalIdentity& getSelfId() override { return self_id; }

src/Dispatcher.cpp

@@ -58,6 +58,19 @@ void Dispatcher::loop() {
     _err_flags |= ERR_EVENT_STARTRX_TIMEOUT;
   }
 
+  // Radio watchdog: detect radio stuck in RX mode but not receiving any packets
+  if (is_recv && _radio->getLastRecvMillis() > 0) {
+    unsigned long silent_ms = _ms->getMillis() - _radio->getLastRecvMillis();
+    unsigned long since_recovery = _ms->getMillis() - last_watchdog_recovery;
+    if (silent_ms > RADIO_WATCHDOG_MS && since_recovery > RADIO_WATCHDOG_MS) {
+      _err_flags |= ERR_EVENT_RADIO_WATCHDOG;
+      MESH_DEBUG_PRINTLN("Radio watchdog: silent %lu ms, state=%d, recovering", silent_ms, _radio->getRadioState());
+      _radio->idle();
+      _radio->startRecv();
+      last_watchdog_recovery = _ms->getMillis();
+    }
+  }
+
   if (outbound) {  // waiting for outbound send to be completed
     if (_radio->isSendComplete()) {
       long t = _ms->getMillis() - outbound_start;

src/Dispatcher.h

@@ -61,12 +61,18 @@ public:
    */
   virtual void loop() { }
 
+  virtual void idle() { }
+  virtual void startRecv() { }
+
   virtual int getNoiseFloor() const { return 0; }
 
   virtual void triggerNoiseFloorCalibrate(int threshold) { }
 
   virtual void resetAGC() { }
 
+  virtual uint8_t getRadioState() const { return 0; }
+  virtual unsigned long getLastRecvMillis() const { return 0; }
+
   virtual bool isInRecvMode() const = 0;
 
   /**
@@ -112,6 +118,11 @@ typedef uint32_t  DispatcherAction;
 #define ERR_EVENT_FULL              (1 << 0)
 #define ERR_EVENT_CAD_TIMEOUT       (1 << 1)
 #define ERR_EVENT_STARTRX_TIMEOUT   (1 << 2)
+#define ERR_EVENT_RADIO_WATCHDOG    (1 << 3)
+
+#ifndef RADIO_WATCHDOG_MS
+  #define RADIO_WATCHDOG_MS  300000   // 5 minutes
+#endif
 
 /**
  * \brief  The low-level task that manages detecting incoming Packets, and the queueing
@@ -120,6 +131,7 @@ typedef uint32_t  DispatcherAction;
 class Dispatcher {
   Packet* outbound;  // current outbound packet
   unsigned long outbound_expiry, outbound_start, total_air_time, rx_air_time;
+  unsigned long last_watchdog_recovery;
   unsigned long next_tx_time;
   unsigned long cad_busy_start;
   unsigned long radio_nonrx_start;
@@ -147,6 +159,7 @@ protected:
     _err_flags = 0;
     radio_nonrx_start = 0;
     prev_isrecv_mode = true;
+    last_watchdog_recovery = 0;
   }
 
   virtual DispatcherAction onRecvPacket(Packet* pkt) = 0;
@@ -180,6 +193,7 @@ public:
   uint32_t getNumRecvFlood() const { return n_recv_flood; }
   uint32_t getNumRecvDirect() const { return n_recv_direct; }
   uint16_t getErrFlags() const { return _err_flags; }  // Get error flags
+  bool hasOutbound() const { return outbound != NULL; }
   void resetStats() {
     n_sent_flood = n_sent_direct = n_recv_flood = n_recv_direct = 0;
     _err_flags = 0;

src/helpers/CommonCLI.cpp

@@ -11,6 +11,7 @@
 #include <WiFi.h>
 #include <WiFiClientSecure.h>
 #include <esp_wifi.h>
+#include <esp_heap_caps.h>
 #endif
 #ifdef WITH_MQTT_BRIDGE
 #include "bridges/MQTTBridge.h"
@@ -643,9 +644,13 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, const char* command, ch
         strcpy(reply, "ERR: clock cannot go backwards");
       }
     } else if (memcmp(command, "memory", 6) == 0) {
-      sprintf(reply, "Free: %d, Min: %d, Max: %d, Queue: %d", 
+      sprintf(reply, "Free: %d, Min: %d, Max: %d, Queue: %d, IntFree: %d, IntMax: %d, PSRAM: %d/%d",
-              ESP.getFreeHeap(), ESP.getMinFreeHeap(), ESP.getMaxAllocHeap(), 
+              ESP.getFreeHeap(), ESP.getMinFreeHeap(), ESP.getMaxAllocHeap(),
-              _callbacks->getQueueSize());
+              _callbacks->getQueueSize(),
+              (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL),
+              (int)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL),
+              (int)heap_caps_get_free_size(MALLOC_CAP_SPIRAM),
+              (int)heap_caps_get_total_size(MALLOC_CAP_SPIRAM));
     } else if (memcmp(command, "tls.bundletest ", 15) == 0) {
 #ifdef ESP_PLATFORM
       if (WiFi.status() != WL_CONNECTED) {
@@ -1708,6 +1713,8 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, const char* command, ch
       strcpy(reply, "   EOF");
     } else if (sender_timestamp == 0 && memcmp(command, "stats-packets", 13) == 0 && (command[13] == 0 || command[13] == ' ')) {
       _callbacks->formatPacketStatsReply(reply);
+    } else if (sender_timestamp == 0 && memcmp(command, "stats-radio-diag", 16) == 0 && (command[16] == 0 || command[16] == ' ')) {
+      _callbacks->formatRadioDiagReply(reply);
     } else if (sender_timestamp == 0 && memcmp(command, "stats-radio", 11) == 0 && (command[11] == 0 || command[11] == ' ')) {
       _callbacks->formatRadioStatsReply(reply);
     } else if (sender_timestamp == 0 && memcmp(command, "stats-core", 10) == 0 && (command[10] == 0 || command[10] == ' ')) {

src/helpers/CommonCLI.h

@@ -235,6 +235,7 @@ public:
   };
   virtual void formatStatsReply(char *reply) = 0;
   virtual void formatRadioStatsReply(char *reply) = 0;
+  virtual void formatRadioDiagReply(char *reply) { strcpy(reply, "Not supported"); }
   virtual void formatPacketStatsReply(char *reply) = 0;
   virtual mesh::LocalIdentity& getSelfId() = 0;
   virtual void saveIdentity(const mesh::LocalIdentity& new_id) = 0;

src/helpers/StatsFormatHelper.h

@@ -34,6 +34,24 @@ public:
     );
   }
 
+  template<typename RadioDriverType>
+  static void formatRadioDiag(char* reply,
+                              mesh::Radio* radio,
+                              RadioDriverType& driver,
+                              mesh::MillisecondClock& ms,
+                              uint16_t err_flags,
+                              bool has_outbound) {
+    uint8_t st = radio->getRadioState();
+    const char* state_name = (st & 16) ? "INT_READY" : (st == 0 ? "IDLE" : (st == 1 ? "RX" : (st == 3 ? "TX_WAIT" : "?")));
+    unsigned long last_rx = radio->getLastRecvMillis();
+    unsigned long ago_secs = (last_rx > 0) ? (ms.getMillis() - last_rx) / 1000 : 0;
+    sprintf(reply,
+      "state=%d(%s), recv=%u, sent=%u, errors=%u, err_flags=%u, outbound=%s, last_rx=%lus ago",
+      st, state_name,
+      driver.getPacketsRecv(), driver.getPacketsSent(), driver.getPacketsRecvErrors(),
+      err_flags, has_outbound ? "yes" : "no", ago_secs);
+  }
+
   template<typename RadioDriverType>
   static void formatPacketStats(char* reply,
                                RadioDriverType& driver,

src/helpers/bridges/MQTTBridge.cpp

@@ -1289,6 +1289,9 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns
           _last_slot_reconnect_ms = now_millis;
           MQTT_DEBUG_PRINTLN("MQTT%d int_heap=%d at token renewal reconnect", index + 1,
               (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL));
+          MQTT_DEBUG_PRINTLN("  radio_state=%d, last_rx=%lums ago",
+              _radio ? _radio->getRadioState() : -1,
+              (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0);
         } else {
           // Token renewed but old one still valid — just update credentials for next reconnect
           slot.client->setCredentials(_jwt_username, slot.auth_token);
@@ -1314,6 +1317,9 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns
       _last_slot_reconnect_ms = now_millis;
       MQTT_DEBUG_PRINTLN("MQTT%d circuit breaker probe (attempting single reconnect after %lu ms, int_heap=%d)", index + 1, probe_elapsed,
           (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL));
+      MQTT_DEBUG_PRINTLN("  radio_state=%d, last_rx=%lums ago",
+          _radio ? _radio->getRadioState() : -1,
+          (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0);
       if (slot_uses_jwt) {
         unsigned long current_time = time(nullptr);
         bool token_still_valid = slot.token_expires_at > 0 &&
@@ -1378,6 +1384,9 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns
       }
       MQTT_DEBUG_PRINTLN("MQTT%d reconnecting (backoff level %d, failures at max: %d, int_heap=%d)", index + 1, slot.reconnect_backoff, slot.max_backoff_failures,
           (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL));
+      MQTT_DEBUG_PRINTLN("  radio_state=%d, last_rx=%lums ago",
+          _radio ? _radio->getRadioState() : -1,
+          (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0);
       reconnect_attempted = true;
       _last_slot_reconnect_ms = now_millis;
       if (slot_uses_jwt) {
@@ -1805,6 +1814,9 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) {
   if (!_wifi_status_initialized) {
     _last_wifi_status = current_wifi_status;
     _wifi_status_initialized = true;
+    if (current_wifi_status != WL_CONNECTED) {
+      _wifi_disconnected_time = now;
+    }
   }
   if (now - _last_wifi_check <= 10000) {
     return false;

src/helpers/radiolib/RadioLibWrappers.cpp

@@ -120,6 +120,7 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) {
       } else {
       //  Serial.print("  readData() -> "); Serial.println(len);
         n_recv++;
+        last_recv_millis = millis();
       }
     }
     state = STATE_IDLE;   // need another startReceive()
@@ -174,6 +175,10 @@ bool RadioLibWrapper::isChannelActive() {
           : getCurrentRSSI() > _noise_floor + _threshold;
 }
 
+uint8_t RadioLibWrapper::getRadioState() const {
+  return state;
+}
+
 float RadioLibWrapper::getLastRSSI() const {
   return _radio->getRSSI();
 }

src/helpers/radiolib/RadioLibWrappers.h

@@ -11,15 +11,17 @@ protected:
   int16_t _noise_floor, _threshold;
   uint16_t _num_floor_samples;
   int32_t _floor_sample_sum;
+  unsigned long last_recv_millis;
 
-  void idle();
-  void startRecv();
   float packetScoreInt(float snr, int sf, int packet_len);
   virtual bool isReceivingPacket() =0;
   virtual void doResetAGC();
 
 public:
-  RadioLibWrapper(PhysicalLayer& radio, mesh::MainBoard& board) : _radio(&radio), _board(&board) { n_recv = n_sent = 0; }
+  RadioLibWrapper(PhysicalLayer& radio, mesh::MainBoard& board) : _radio(&radio), _board(&board) { n_recv = n_sent = 0; last_recv_millis = 0; }
+
+  void idle() override;
+  void startRecv() override;
 
   void begin() override;
   virtual void powerOff() { _radio->sleep(); }
@@ -50,6 +52,9 @@ public:
   uint32_t getPacketsSent() const { return n_sent; }
   void resetStats() { n_recv = n_sent = n_recv_errors = 0; }
 
+  uint8_t getRadioState() const override;
+  unsigned long getLastRecvMillis() const override { return last_recv_millis; }
+
   virtual float getLastRSSI() const override;
   virtual float getLastSNR() const override;