diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 8c06bce9..fe14f7cb 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1196,6 +1196,10 @@ void MyMesh::formatRadioStatsReply(char *reply) { StatsFormatHelper::formatRadioStats(reply, _radio, radio_driver, getTotalAirTime(), getReceiveAirTime()); } +void MyMesh::formatRadioDiagReply(char *reply) { + StatsFormatHelper::formatRadioDiag(reply, _radio, radio_driver, *_ms, _err_flags, hasOutbound()); +} + void MyMesh::formatPacketStatsReply(char *reply) { StatsFormatHelper::formatPacketStats(reply, radio_driver, getNumSentFlood(), getNumSentDirect(), getNumRecvFlood(), getNumRecvDirect()); diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index d957f6d9..c6c8db5b 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -220,6 +220,7 @@ public: void removeNeighbor(const uint8_t* pubkey, int key_len) override; void formatStatsReply(char *reply) override; void formatRadioStatsReply(char *reply) override; + void formatRadioDiagReply(char *reply) override; void formatPacketStatsReply(char *reply) override; mesh::LocalIdentity& getSelfId() override { return self_id; } diff --git a/src/Dispatcher.cpp b/src/Dispatcher.cpp index 35eca0a9..a837d47a 100644 --- a/src/Dispatcher.cpp +++ b/src/Dispatcher.cpp @@ -58,6 +58,19 @@ void Dispatcher::loop() { _err_flags |= ERR_EVENT_STARTRX_TIMEOUT; } + // Radio watchdog: detect radio stuck in RX mode but not receiving any packets + if (is_recv && _radio->getLastRecvMillis() > 0) { + unsigned long silent_ms = _ms->getMillis() - _radio->getLastRecvMillis(); + unsigned long since_recovery = _ms->getMillis() - last_watchdog_recovery; + if (silent_ms > RADIO_WATCHDOG_MS && since_recovery > RADIO_WATCHDOG_MS) { + _err_flags |= ERR_EVENT_RADIO_WATCHDOG; + MESH_DEBUG_PRINTLN("Radio watchdog: silent %lu ms, state=%d, recovering", silent_ms, _radio->getRadioState()); + _radio->idle(); + _radio->startRecv(); + last_watchdog_recovery = _ms->getMillis(); + } + } + if (outbound) { // waiting for outbound send to be completed if (_radio->isSendComplete()) { long t = _ms->getMillis() - outbound_start; diff --git a/src/Dispatcher.h b/src/Dispatcher.h index 4cde40bc..dbbc4b6f 100644 --- a/src/Dispatcher.h +++ b/src/Dispatcher.h @@ -61,12 +61,18 @@ public: */ virtual void loop() { } + virtual void idle() { } + virtual void startRecv() { } + virtual int getNoiseFloor() const { return 0; } virtual void triggerNoiseFloorCalibrate(int threshold) { } virtual void resetAGC() { } + virtual uint8_t getRadioState() const { return 0; } + virtual unsigned long getLastRecvMillis() const { return 0; } + virtual bool isInRecvMode() const = 0; /** @@ -112,6 +118,11 @@ typedef uint32_t DispatcherAction; #define ERR_EVENT_FULL (1 << 0) #define ERR_EVENT_CAD_TIMEOUT (1 << 1) #define ERR_EVENT_STARTRX_TIMEOUT (1 << 2) +#define ERR_EVENT_RADIO_WATCHDOG (1 << 3) + +#ifndef RADIO_WATCHDOG_MS + #define RADIO_WATCHDOG_MS 300000 // 5 minutes +#endif /** * \brief The low-level task that manages detecting incoming Packets, and the queueing @@ -120,6 +131,7 @@ typedef uint32_t DispatcherAction; class Dispatcher { Packet* outbound; // current outbound packet unsigned long outbound_expiry, outbound_start, total_air_time, rx_air_time; + unsigned long last_watchdog_recovery; unsigned long next_tx_time; unsigned long cad_busy_start; unsigned long radio_nonrx_start; @@ -147,6 +159,7 @@ protected: _err_flags = 0; radio_nonrx_start = 0; prev_isrecv_mode = true; + last_watchdog_recovery = 0; } virtual DispatcherAction onRecvPacket(Packet* pkt) = 0; @@ -180,6 +193,7 @@ public: uint32_t getNumRecvFlood() const { return n_recv_flood; } uint32_t getNumRecvDirect() const { return n_recv_direct; } uint16_t getErrFlags() const { return _err_flags; } // Get error flags + bool hasOutbound() const { return outbound != NULL; } void resetStats() { n_sent_flood = n_sent_direct = n_recv_flood = n_recv_direct = 0; _err_flags = 0; diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 2996a0ef..c19b0f21 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #endif #ifdef WITH_MQTT_BRIDGE #include "bridges/MQTTBridge.h" @@ -643,9 +644,13 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, const char* command, ch strcpy(reply, "ERR: clock cannot go backwards"); } } else if (memcmp(command, "memory", 6) == 0) { - sprintf(reply, "Free: %d, Min: %d, Max: %d, Queue: %d", - ESP.getFreeHeap(), ESP.getMinFreeHeap(), ESP.getMaxAllocHeap(), - _callbacks->getQueueSize()); + sprintf(reply, "Free: %d, Min: %d, Max: %d, Queue: %d, IntFree: %d, IntMax: %d, PSRAM: %d/%d", + ESP.getFreeHeap(), ESP.getMinFreeHeap(), ESP.getMaxAllocHeap(), + _callbacks->getQueueSize(), + (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL), + (int)heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL), + (int)heap_caps_get_free_size(MALLOC_CAP_SPIRAM), + (int)heap_caps_get_total_size(MALLOC_CAP_SPIRAM)); } else if (memcmp(command, "tls.bundletest ", 15) == 0) { #ifdef ESP_PLATFORM if (WiFi.status() != WL_CONNECTED) { @@ -1708,6 +1713,8 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, const char* command, ch strcpy(reply, " EOF"); } else if (sender_timestamp == 0 && memcmp(command, "stats-packets", 13) == 0 && (command[13] == 0 || command[13] == ' ')) { _callbacks->formatPacketStatsReply(reply); + } else if (sender_timestamp == 0 && memcmp(command, "stats-radio-diag", 16) == 0 && (command[16] == 0 || command[16] == ' ')) { + _callbacks->formatRadioDiagReply(reply); } else if (sender_timestamp == 0 && memcmp(command, "stats-radio", 11) == 0 && (command[11] == 0 || command[11] == ' ')) { _callbacks->formatRadioStatsReply(reply); } else if (sender_timestamp == 0 && memcmp(command, "stats-core", 10) == 0 && (command[10] == 0 || command[10] == ' ')) { diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index 966212db..1f80dc72 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -235,6 +235,7 @@ public: }; virtual void formatStatsReply(char *reply) = 0; virtual void formatRadioStatsReply(char *reply) = 0; + virtual void formatRadioDiagReply(char *reply) { strcpy(reply, "Not supported"); } virtual void formatPacketStatsReply(char *reply) = 0; virtual mesh::LocalIdentity& getSelfId() = 0; virtual void saveIdentity(const mesh::LocalIdentity& new_id) = 0; diff --git a/src/helpers/StatsFormatHelper.h b/src/helpers/StatsFormatHelper.h index 5aa01da9..7cce0552 100644 --- a/src/helpers/StatsFormatHelper.h +++ b/src/helpers/StatsFormatHelper.h @@ -34,6 +34,24 @@ public: ); } + template + static void formatRadioDiag(char* reply, + mesh::Radio* radio, + RadioDriverType& driver, + mesh::MillisecondClock& ms, + uint16_t err_flags, + bool has_outbound) { + uint8_t st = radio->getRadioState(); + const char* state_name = (st & 16) ? "INT_READY" : (st == 0 ? "IDLE" : (st == 1 ? "RX" : (st == 3 ? "TX_WAIT" : "?"))); + unsigned long last_rx = radio->getLastRecvMillis(); + unsigned long ago_secs = (last_rx > 0) ? (ms.getMillis() - last_rx) / 1000 : 0; + sprintf(reply, + "state=%d(%s), recv=%u, sent=%u, errors=%u, err_flags=%u, outbound=%s, last_rx=%lus ago", + st, state_name, + driver.getPacketsRecv(), driver.getPacketsSent(), driver.getPacketsRecvErrors(), + err_flags, has_outbound ? "yes" : "no", ago_secs); + } + template static void formatPacketStats(char* reply, RadioDriverType& driver, diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 5418bd09..9e0efde5 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1289,6 +1289,9 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns _last_slot_reconnect_ms = now_millis; MQTT_DEBUG_PRINTLN("MQTT%d int_heap=%d at token renewal reconnect", index + 1, (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); + MQTT_DEBUG_PRINTLN(" radio_state=%d, last_rx=%lums ago", + _radio ? _radio->getRadioState() : -1, + (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0); } else { // Token renewed but old one still valid — just update credentials for next reconnect slot.client->setCredentials(_jwt_username, slot.auth_token); @@ -1314,6 +1317,9 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns _last_slot_reconnect_ms = now_millis; MQTT_DEBUG_PRINTLN("MQTT%d circuit breaker probe (attempting single reconnect after %lu ms, int_heap=%d)", index + 1, probe_elapsed, (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); + MQTT_DEBUG_PRINTLN(" radio_state=%d, last_rx=%lums ago", + _radio ? _radio->getRadioState() : -1, + (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0); if (slot_uses_jwt) { unsigned long current_time = time(nullptr); bool token_still_valid = slot.token_expires_at > 0 && @@ -1378,6 +1384,9 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns } MQTT_DEBUG_PRINTLN("MQTT%d reconnecting (backoff level %d, failures at max: %d, int_heap=%d)", index + 1, slot.reconnect_backoff, slot.max_backoff_failures, (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); + MQTT_DEBUG_PRINTLN(" radio_state=%d, last_rx=%lums ago", + _radio ? _radio->getRadioState() : -1, + (_radio && _radio->getLastRecvMillis() > 0) ? (_ms->getMillis() - _radio->getLastRecvMillis()) : 0); reconnect_attempted = true; _last_slot_reconnect_ms = now_millis; if (slot_uses_jwt) { @@ -1805,6 +1814,9 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) { if (!_wifi_status_initialized) { _last_wifi_status = current_wifi_status; _wifi_status_initialized = true; + if (current_wifi_status != WL_CONNECTED) { + _wifi_disconnected_time = now; + } } if (now - _last_wifi_check <= 10000) { return false; diff --git a/src/helpers/radiolib/RadioLibWrappers.cpp b/src/helpers/radiolib/RadioLibWrappers.cpp index 2216ca8f..30995fd0 100644 --- a/src/helpers/radiolib/RadioLibWrappers.cpp +++ b/src/helpers/radiolib/RadioLibWrappers.cpp @@ -120,6 +120,7 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) { } else { // Serial.print(" readData() -> "); Serial.println(len); n_recv++; + last_recv_millis = millis(); } } state = STATE_IDLE; // need another startReceive() @@ -174,6 +175,10 @@ bool RadioLibWrapper::isChannelActive() { : getCurrentRSSI() > _noise_floor + _threshold; } +uint8_t RadioLibWrapper::getRadioState() const { + return state; +} + float RadioLibWrapper::getLastRSSI() const { return _radio->getRSSI(); } diff --git a/src/helpers/radiolib/RadioLibWrappers.h b/src/helpers/radiolib/RadioLibWrappers.h index 29e77cec..405bb0de 100644 --- a/src/helpers/radiolib/RadioLibWrappers.h +++ b/src/helpers/radiolib/RadioLibWrappers.h @@ -11,15 +11,17 @@ protected: int16_t _noise_floor, _threshold; uint16_t _num_floor_samples; int32_t _floor_sample_sum; + unsigned long last_recv_millis; - void idle(); - void startRecv(); float packetScoreInt(float snr, int sf, int packet_len); virtual bool isReceivingPacket() =0; virtual void doResetAGC(); public: - RadioLibWrapper(PhysicalLayer& radio, mesh::MainBoard& board) : _radio(&radio), _board(&board) { n_recv = n_sent = 0; } + RadioLibWrapper(PhysicalLayer& radio, mesh::MainBoard& board) : _radio(&radio), _board(&board) { n_recv = n_sent = 0; last_recv_millis = 0; } + + void idle() override; + void startRecv() override; void begin() override; virtual void powerOff() { _radio->sleep(); } @@ -50,6 +52,9 @@ public: uint32_t getPacketsSent() const { return n_sent; } void resetStats() { n_recv = n_sent = n_recv_errors = 0; } + uint8_t getRadioState() const override; + unsigned long getLastRecvMillis() const override { return last_recv_millis; } + virtual float getLastRSSI() const override; virtual float getLastSNR() const override;