From c055228c91175723d55915798e1c751b86a597a7 Mon Sep 17 00:00:00 2001 From: agessaman Date: Thu, 22 Jan 2026 07:51:38 -0800 Subject: [PATCH] Refactor MQTTBridge connection handling for improved stability and responsiveness - Simplified connection attempt logic to allow immediate reconnects after failures. - Added checks to ensure the MQTT client is disconnected before new connection attempts. - Enhanced error handling during publish operations to reset connection state on failures. - Removed aggressive health checks that caused connection instability, relying on the MQTT client library for connection management. - Updated status publishing to verify connection state before attempting to publish, ensuring accurate broker status. --- examples/simple_repeater/MyMesh.cpp | 4 +- src/helpers/bridges/MQTTBridge.cpp | 95 +++++++++++++++++++---- src/helpers/radiolib/RadioLibWrappers.cpp | 22 +----- 3 files changed, 81 insertions(+), 40 deletions(-) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 8a3c229e0..18555f927 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -758,9 +758,7 @@ bool MyMesh::onPeerPathRecv(mesh::Packet *packet, int sender_idx, const uint8_t void MyMesh::onControlDataRecv(mesh::Packet* packet) { uint8_t type = packet->payload[0] & 0xF0; // just test upper 4 bits - if (type == CTL_TYPE_NODE_DISCOVER_REQ && packet->payload_len >= 6 - && !_prefs.disable_fwd && discover_limiter.allow(rtc_clock.getCurrentTime()) - ) { + if (type == CTL_TYPE_NODE_DISCOVER_REQ && packet->payload_len >= 6 && discover_limiter.allow(rtc_clock.getCurrentTime())) { int i = 1; uint8_t filter = packet->payload[i++]; uint32_t tag; diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 015f4ec1d..3ed729c57 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -963,9 +963,11 @@ void MQTTBridge::connectToBrokers() { if (!_brokers[i].enabled) continue; // Check if we need to attempt connection - if (!_brokers[i].connected && - millis() - _brokers[i].last_attempt > _brokers[i].reconnect_interval) { - + // Allow immediate reconnect if last_attempt is 0 (was reset due to failure) + bool can_attempt = (_brokers[i].last_attempt == 0) || + (millis() - _brokers[i].last_attempt > _brokers[i].reconnect_interval); + + if (!_brokers[i].connected && can_attempt) { MQTT_DEBUG_PRINTLN("Connecting to broker %d: %s:%d", i, _brokers[i].host, _brokers[i].port); // Generate unique client ID @@ -982,6 +984,12 @@ void MQTTBridge::connectToBrokers() { _mqtt_client->setCredentials(_brokers[i].username, _brokers[i].password); } + // Ensure we're disconnected before attempting new connection + if (_mqtt_client->connected()) { + _mqtt_client->disconnect(); + vTaskDelay(pdMS_TO_TICKS(100)); // Brief delay to allow disconnect to complete + } + // Connect to the broker (PsychicMqttClient uses async connection) _mqtt_client->connect(); @@ -992,18 +1000,26 @@ void MQTTBridge::connectToBrokers() { // Maintain connection and check for stale connections if (_brokers[i].connected) { - // Check actual connection state - if it's stale, mark as disconnected - // PsychicMqttClient handles automatic reconnection internally, so we don't need - // to force periodic reconnections. Only mark as disconnected if actually disconnected. + // Check actual connection state - if it's stale, mark as disconnected and trigger reconnect + // PsychicMqttClient handles automatic reconnection internally, but we need to detect stale state if (!_mqtt_client->connected()) { + MQTT_DEBUG_PRINTLN("Broker %d connection lost, marking for reconnect", i); _brokers[i].connected = false; _active_brokers--; + _brokers[i].last_attempt = 0; // Reset attempt time to allow immediate reconnect // Update cached broker status _cached_has_brokers = isAnyBrokerConnected(); } // Removed aggressive 4-hour health check that was causing connection instability. // The MQTT client library handles connection health internally, and forcing // disconnections on healthy connections was causing hours of downtime. + } else { + // Not connected - ensure we attempt reconnection if enough time has passed + // Reset last_attempt if it's been too long (prevents getting stuck) + if (_brokers[i].last_attempt > 0 && (millis() - _brokers[i].last_attempt) > 300000) { + // Been trying for more than 5 minutes - reset to allow fresh attempt + _brokers[i].last_attempt = 0; + } } } @@ -1177,6 +1193,7 @@ bool MQTTBridge::publishStatus() { #endif // Use cached destination status to avoid redundant checks + // Note: Connection state is verified in connectToBrokers() which runs before publishStatus() bool has_custom_brokers = _cached_has_brokers && _config_valid; bool has_destinations = has_custom_brokers || _cached_has_analyzer_servers; @@ -1184,6 +1201,26 @@ bool MQTTBridge::publishStatus() { return false; // No destinations available } + // Final verification: ensure client is actually connected before attempting publish + // This catches cases where connection state got out of sync + if (has_custom_brokers && _mqtt_client && !_mqtt_client->connected()) { + // Client says disconnected but our state says connected - fix the state + MQTT_DEBUG_PRINTLN("Status publish: Client disconnected but state says connected - fixing state"); + for (int i = 0; i < MAX_MQTT_BROKERS_COUNT; i++) { + if (_brokers[i].enabled && _brokers[i].connected) { + _brokers[i].connected = false; + _active_brokers--; + _brokers[i].last_attempt = 0; // Allow immediate reconnect + } + } + _cached_has_brokers = false; + has_custom_brokers = false; + has_destinations = has_custom_brokers || _cached_has_analyzer_servers; + if (!has_destinations) { + return false; // No destinations after fixing state + } + } + // Status messages with stats can be larger (~400-500 bytes), so increase buffer size char json_buffer[768]; // Increased from 512 to accommodate stats object char origin_id[65]; @@ -1263,13 +1300,15 @@ bool MQTTBridge::publishStatus() { size_t json_len = strlen(json_buffer); // Cache length to avoid multiple strlen() calls // Publish to all connected custom brokers + // Double-check client is actually connected before attempting publish if (_config_valid && _mqtt_client && _mqtt_client->connected()) { // Track last broker URI to avoid calling setServer() unnecessarily (memory optimization) // setServer() may allocate memory, so we only call it when the broker changes static char last_broker_uri_status[128] = ""; for (int i = 0; i < MAX_MQTT_BROKERS_COUNT; i++) { - if (_brokers[i].enabled && _brokers[i].connected) { + // Verify broker is actually connected (state might be stale) + if (_brokers[i].enabled && _brokers[i].connected && _mqtt_client->connected()) { // Build broker URI char broker_uri[128]; @@ -1287,15 +1326,22 @@ bool MQTTBridge::publishStatus() { if (publish_result > 0) { published = true; } else { - // Publish failed - connection may be stale, mark as disconnected + // Publish failed - connection may be stale, force disconnect and mark for reconnect static unsigned long last_status_publish_fail_log = 0; unsigned long now = millis(); if (now - last_status_publish_fail_log > 60000) { // Log every minute max - MQTT_DEBUG_PRINTLN("Status publish failed (result=%d), marking broker %d as disconnected", publish_result, i); + MQTT_DEBUG_PRINTLN("Status publish failed (result=%d), forcing broker %d reconnect", publish_result, i); last_status_publish_fail_log = now; } + // Force disconnect to trigger reconnection + if (_mqtt_client->connected()) { + _mqtt_client->disconnect(); + } _brokers[i].connected = false; _active_brokers--; + _brokers[i].last_attempt = 0; // Reset attempt time to allow immediate reconnect + // Update cached broker status + _cached_has_brokers = isAnyBrokerConnected(); } } } @@ -1403,13 +1449,15 @@ void MQTTBridge::publishPacket(mesh::Packet* packet, bool is_tx, size_t json_len = strlen(json_buffer); // Cache length to avoid multiple strlen() calls // Publish to custom brokers (only if config is valid) + // Double-check client is actually connected before attempting publish if (_config_valid && _mqtt_client && _mqtt_client->connected()) { // Track last broker URI to avoid calling setServer() unnecessarily (memory optimization) // setServer() may allocate memory, so we only call it when the broker changes static char last_broker_uri[128] = ""; for (int i = 0; i < MAX_MQTT_BROKERS_COUNT; i++) { - if (_brokers[i].enabled && _brokers[i].connected) { + // Verify broker is actually connected (state might be stale) + if (_brokers[i].enabled && _brokers[i].connected && _mqtt_client->connected()) { // Build broker URI char broker_uri[128]; snprintf(broker_uri, sizeof(broker_uri), "mqtt://%s:%d", _brokers[i].host, _brokers[i].port); @@ -1425,16 +1473,22 @@ void MQTTBridge::publishPacket(mesh::Packet* packet, bool is_tx, // This prevents blocking the main loop when MQTT broker is slow or unresponsive int publish_result = _mqtt_client->publish(topic, 1, false, json_buffer, json_len); // qos=1, retained=false if (publish_result <= 0) { - // Publish failed - connection may be stale, mark as disconnected - // This prevents accumulation of failed publishes that block the loop + // Publish failed - connection may be stale, force disconnect and mark for reconnect static unsigned long last_publish_fail_log = 0; unsigned long now = millis(); if (now - last_publish_fail_log > 60000) { // Log every minute max - MQTT_DEBUG_PRINTLN("Publish failed (result=%d), marking broker %d as disconnected", publish_result, i); + MQTT_DEBUG_PRINTLN("Publish failed (result=%d), forcing broker %d reconnect", publish_result, i); last_publish_fail_log = now; } + // Force disconnect to trigger reconnection + if (_mqtt_client->connected()) { + _mqtt_client->disconnect(); + } _brokers[i].connected = false; _active_brokers--; + _brokers[i].last_attempt = 0; // Reset attempt time to allow immediate reconnect + // Update cached broker status + _cached_has_brokers = isAnyBrokerConnected(); } } } @@ -1502,13 +1556,15 @@ void MQTTBridge::publishRaw(mesh::Packet* packet) { size_t json_len = strlen(json_buffer); // Cache length to avoid multiple strlen() calls // Publish to custom brokers (only if config is valid) + // Double-check client is actually connected before attempting publish if (_config_valid && _mqtt_client && _mqtt_client->connected()) { // Track last broker URI to avoid calling setServer() unnecessarily (memory optimization) // setServer() may allocate memory, so we only call it when the broker changes static char last_broker_uri_raw[128] = ""; for (int i = 0; i < MAX_MQTT_BROKERS_COUNT; i++) { - if (_brokers[i].enabled && _brokers[i].connected) { + // Verify broker is actually connected (state might be stale) + if (_brokers[i].enabled && _brokers[i].connected && _mqtt_client->connected()) { // Build broker URI char broker_uri[128]; snprintf(broker_uri, sizeof(broker_uri), "mqtt://%s:%d", _brokers[i].host, _brokers[i].port); @@ -1523,15 +1579,22 @@ void MQTTBridge::publishRaw(mesh::Packet* packet) { // Publish with timeout check - don't block if connection is slow int publish_result = _mqtt_client->publish(topic, 1, false, json_buffer, json_len); // qos=1, retained=false if (publish_result <= 0) { - // Publish failed - connection may be stale, mark as disconnected + // Publish failed - connection may be stale, force disconnect and mark for reconnect static unsigned long last_raw_publish_fail_log = 0; unsigned long now = millis(); if (now - last_raw_publish_fail_log > 60000) { // Log every minute max - MQTT_DEBUG_PRINTLN("Raw publish failed (result=%d), marking broker %d as disconnected", publish_result, i); + MQTT_DEBUG_PRINTLN("Raw publish failed (result=%d), forcing broker %d reconnect", publish_result, i); last_raw_publish_fail_log = now; } + // Force disconnect to trigger reconnection + if (_mqtt_client->connected()) { + _mqtt_client->disconnect(); + } _brokers[i].connected = false; _active_brokers--; + _brokers[i].last_attempt = 0; // Reset attempt time to allow immediate reconnect + // Update cached broker status + _cached_has_brokers = isAnyBrokerConnected(); } } } diff --git a/src/helpers/radiolib/RadioLibWrappers.cpp b/src/helpers/radiolib/RadioLibWrappers.cpp index 8b74fad2e..e34078211 100644 --- a/src/helpers/radiolib/RadioLibWrappers.cpp +++ b/src/helpers/radiolib/RadioLibWrappers.cpp @@ -103,27 +103,7 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) { if (len > sz) { len = sz; } int err = _radio->readData(bytes, len); if (err != RADIOLIB_ERR_NONE) { - // Throttle error logging to avoid spam (CRC errors are common in noisy environments) - static unsigned long last_crc_error_log = 0; - static unsigned long crc_error_count = 0; - unsigned long now = millis(); - crc_error_count++; - - // Log every 10 seconds max, or on first error - if (last_crc_error_log == 0 || (now - last_crc_error_log > 10000)) { - MESH_DEBUG_PRINTLN("RadioLibWrapper: error: readData(%d) - %lu errors in last 10s", err, crc_error_count); - last_crc_error_log = now; - crc_error_count = 0; - } - - // For CRC mismatch errors (-7), reset radio state to prevent stuck conditions - // Similar to CustomLR1110's handling of header errors - if (err == -7) { // RADIOLIB_ERR_CRC_MISMATCH - // Call standby() to reset radio to known-good state - // This helps recover from false packet detections or corrupted state - _radio->standby(); - } - + MESH_DEBUG_PRINTLN("RadioLibWrapper: error: readData(%d)", err); len = 0; } else { // Serial.print(" readData() -> "); Serial.println(len);