Refactor MQTTBridge connection handling for improved stability and responsiveness

- Simplified connection attempt logic to allow immediate reconnects after failures.
- Added checks to ensure the MQTT client is disconnected before new connection attempts.
- Enhanced error handling during publish operations to reset connection state on failures.
- Removed aggressive health checks that caused connection instability, relying on the MQTT client library for connection management.
- Updated status publishing to verify connection state before attempting to publish, ensuring accurate broker status.
This commit is contained in:
agessaman committed 2026-01-22 07:51:38 -08:00
1 parent 39ba020fab
commit c055228c91
3 files changed
+81 -40

No files matched your search

+1 -3
View File
@@ -758,9 +758,7 @@ bool MyMesh::onPeerPathRecv(mesh::Packet *packet, int sender_idx, const uint8_t
void MyMesh::onControlDataRecv(mesh::Packet* packet) {
uint8_t type = packet->payload[0] & 0xF0; // just test upper 4 bits
if (type == CTL_TYPE_NODE_DISCOVER_REQ && packet->payload_len >= 6
&& !_prefs.disable_fwd && discover_limiter.allow(rtc_clock.getCurrentTime())
) {
if (type == CTL_TYPE_NODE_DISCOVER_REQ && packet->payload_len >= 6 && discover_limiter.allow(rtc_clock.getCurrentTime())) {
int i = 1;
uint8_t filter = packet->payload[i++];
uint32_t tag;
+79 -16
View File
@@ -963,9 +963,11 @@ void MQTTBridge::connectToBrokers() {
if (!_brokers[i].enabled) continue;
// Check if we need to attempt connection
if (!_brokers[i].connected &&
millis() - _brokers[i].last_attempt > _brokers[i].reconnect_interval) {
// Allow immediate reconnect if last_attempt is 0 (was reset due to failure)
bool can_attempt = (_brokers[i].last_attempt == 0) ||
(millis() - _brokers[i].last_attempt > _brokers[i].reconnect_interval);
if (!_brokers[i].connected && can_attempt) {
MQTT_DEBUG_PRINTLN("Connecting to broker %d: %s:%d", i, _brokers[i].host, _brokers[i].port);
// Generate unique client ID
@@ -982,6 +984,12 @@ void MQTTBridge::connectToBrokers() {
_mqtt_client->setCredentials(_brokers[i].username, _brokers[i].password);
}
// Ensure we're disconnected before attempting new connection
if (_mqtt_client->connected()) {
_mqtt_client->disconnect();
vTaskDelay(pdMS_TO_TICKS(100)); // Brief delay to allow disconnect to complete
}
// Connect to the broker (PsychicMqttClient uses async connection)
_mqtt_client->connect();
@@ -992,18 +1000,26 @@ void MQTTBridge::connectToBrokers() {
// Maintain connection and check for stale connections
if (_brokers[i].connected) {
// Check actual connection state - if it's stale, mark as disconnected
// PsychicMqttClient handles automatic reconnection internally, so we don't need
// to force periodic reconnections. Only mark as disconnected if actually disconnected.
// Check actual connection state - if it's stale, mark as disconnected and trigger reconnect
// PsychicMqttClient handles automatic reconnection internally, but we need to detect stale state
if (!_mqtt_client->connected()) {
MQTT_DEBUG_PRINTLN("Broker %d connection lost, marking for reconnect", i);
_brokers[i].connected = false;
_active_brokers--;
_brokers[i].last_attempt = 0; // Reset attempt time to allow immediate reconnect
// Update cached broker status
_cached_has_brokers = isAnyBrokerConnected();
}
// Removed aggressive 4-hour health check that was causing connection instability.
// The MQTT client library handles connection health internally, and forcing
// disconnections on healthy connections was causing hours of downtime.
} else {
// Not connected - ensure we attempt reconnection if enough time has passed
// Reset last_attempt if it's been too long (prevents getting stuck)
if (_brokers[i].last_attempt > 0 && (millis() - _brokers[i].last_attempt) > 300000) {
// Been trying for more than 5 minutes - reset to allow fresh attempt
_brokers[i].last_attempt = 0;
}
}
}
@@ -1177,6 +1193,7 @@ bool MQTTBridge::publishStatus() {
#endif
// Use cached destination status to avoid redundant checks
// Note: Connection state is verified in connectToBrokers() which runs before publishStatus()
bool has_custom_brokers = _cached_has_brokers && _config_valid;
bool has_destinations = has_custom_brokers || _cached_has_analyzer_servers;
@@ -1184,6 +1201,26 @@ bool MQTTBridge::publishStatus() {
return false; // No destinations available
}
// Final verification: ensure client is actually connected before attempting publish
// This catches cases where connection state got out of sync
if (has_custom_brokers && _mqtt_client && !_mqtt_client->connected()) {
// Client says disconnected but our state says connected - fix the state
MQTT_DEBUG_PRINTLN("Status publish: Client disconnected but state says connected - fixing state");
for (int i = 0; i < MAX_MQTT_BROKERS_COUNT; i++) {
if (_brokers[i].enabled && _brokers[i].connected) {
_brokers[i].connected = false;
_active_brokers--;
_brokers[i].last_attempt = 0; // Allow immediate reconnect
}
}
_cached_has_brokers = false;
has_custom_brokers = false;
has_destinations = has_custom_brokers || _cached_has_analyzer_servers;
if (!has_destinations) {
return false; // No destinations after fixing state
}
}
// Status messages with stats can be larger (~400-500 bytes), so increase buffer size
char json_buffer[768]; // Increased from 512 to accommodate stats object
char origin_id[65];
@@ -1263,13 +1300,15 @@ bool MQTTBridge::publishStatus() {
size_t json_len = strlen(json_buffer); // Cache length to avoid multiple strlen() calls
// Publish to all connected custom brokers
// Double-check client is actually connected before attempting publish
if (_config_valid && _mqtt_client && _mqtt_client->connected()) {
// Track last broker URI to avoid calling setServer() unnecessarily (memory optimization)
// setServer() may allocate memory, so we only call it when the broker changes
static char last_broker_uri_status[128] = "";
for (int i = 0; i < MAX_MQTT_BROKERS_COUNT; i++) {
if (_brokers[i].enabled && _brokers[i].connected) {
// Verify broker is actually connected (state might be stale)
if (_brokers[i].enabled && _brokers[i].connected && _mqtt_client->connected()) {
// Build broker URI
char broker_uri[128];
@@ -1287,15 +1326,22 @@ bool MQTTBridge::publishStatus() {
if (publish_result > 0) {
published = true;
} else {
// Publish failed - connection may be stale, mark as disconnected
// Publish failed - connection may be stale, force disconnect and mark for reconnect
static unsigned long last_status_publish_fail_log = 0;
unsigned long now = millis();
if (now - last_status_publish_fail_log > 60000) { // Log every minute max
MQTT_DEBUG_PRINTLN("Status publish failed (result=%d), marking broker %d as disconnected", publish_result, i);
MQTT_DEBUG_PRINTLN("Status publish failed (result=%d), forcing broker %d reconnect", publish_result, i);
last_status_publish_fail_log = now;
}
// Force disconnect to trigger reconnection
if (_mqtt_client->connected()) {
_mqtt_client->disconnect();
}
_brokers[i].connected = false;
_active_brokers--;
_brokers[i].last_attempt = 0; // Reset attempt time to allow immediate reconnect
// Update cached broker status
_cached_has_brokers = isAnyBrokerConnected();
}
}
}
@@ -1403,13 +1449,15 @@ void MQTTBridge::publishPacket(mesh::Packet* packet, bool is_tx,
size_t json_len = strlen(json_buffer); // Cache length to avoid multiple strlen() calls
// Publish to custom brokers (only if config is valid)
// Double-check client is actually connected before attempting publish
if (_config_valid && _mqtt_client && _mqtt_client->connected()) {
// Track last broker URI to avoid calling setServer() unnecessarily (memory optimization)
// setServer() may allocate memory, so we only call it when the broker changes
static char last_broker_uri[128] = "";
for (int i = 0; i < MAX_MQTT_BROKERS_COUNT; i++) {
if (_brokers[i].enabled && _brokers[i].connected) {
// Verify broker is actually connected (state might be stale)
if (_brokers[i].enabled && _brokers[i].connected && _mqtt_client->connected()) {
// Build broker URI
char broker_uri[128];
snprintf(broker_uri, sizeof(broker_uri), "mqtt://%s:%d", _brokers[i].host, _brokers[i].port);
@@ -1425,16 +1473,22 @@ void MQTTBridge::publishPacket(mesh::Packet* packet, bool is_tx,
// This prevents blocking the main loop when MQTT broker is slow or unresponsive
int publish_result = _mqtt_client->publish(topic, 1, false, json_buffer, json_len); // qos=1, retained=false
if (publish_result <= 0) {
// Publish failed - connection may be stale, mark as disconnected
// This prevents accumulation of failed publishes that block the loop
// Publish failed - connection may be stale, force disconnect and mark for reconnect
static unsigned long last_publish_fail_log = 0;
unsigned long now = millis();
if (now - last_publish_fail_log > 60000) { // Log every minute max
MQTT_DEBUG_PRINTLN("Publish failed (result=%d), marking broker %d as disconnected", publish_result, i);
MQTT_DEBUG_PRINTLN("Publish failed (result=%d), forcing broker %d reconnect", publish_result, i);
last_publish_fail_log = now;
}
// Force disconnect to trigger reconnection
if (_mqtt_client->connected()) {
_mqtt_client->disconnect();
}
_brokers[i].connected = false;
_active_brokers--;
_brokers[i].last_attempt = 0; // Reset attempt time to allow immediate reconnect
// Update cached broker status
_cached_has_brokers = isAnyBrokerConnected();
}
}
}
@@ -1502,13 +1556,15 @@ void MQTTBridge::publishRaw(mesh::Packet* packet) {
size_t json_len = strlen(json_buffer); // Cache length to avoid multiple strlen() calls
// Publish to custom brokers (only if config is valid)
// Double-check client is actually connected before attempting publish
if (_config_valid && _mqtt_client && _mqtt_client->connected()) {
// Track last broker URI to avoid calling setServer() unnecessarily (memory optimization)
// setServer() may allocate memory, so we only call it when the broker changes
static char last_broker_uri_raw[128] = "";
for (int i = 0; i < MAX_MQTT_BROKERS_COUNT; i++) {
if (_brokers[i].enabled && _brokers[i].connected) {
// Verify broker is actually connected (state might be stale)
if (_brokers[i].enabled && _brokers[i].connected && _mqtt_client->connected()) {
// Build broker URI
char broker_uri[128];
snprintf(broker_uri, sizeof(broker_uri), "mqtt://%s:%d", _brokers[i].host, _brokers[i].port);
@@ -1523,15 +1579,22 @@ void MQTTBridge::publishRaw(mesh::Packet* packet) {
// Publish with timeout check - don't block if connection is slow
int publish_result = _mqtt_client->publish(topic, 1, false, json_buffer, json_len); // qos=1, retained=false
if (publish_result <= 0) {
// Publish failed - connection may be stale, mark as disconnected
// Publish failed - connection may be stale, force disconnect and mark for reconnect
static unsigned long last_raw_publish_fail_log = 0;
unsigned long now = millis();
if (now - last_raw_publish_fail_log > 60000) { // Log every minute max
MQTT_DEBUG_PRINTLN("Raw publish failed (result=%d), marking broker %d as disconnected", publish_result, i);
MQTT_DEBUG_PRINTLN("Raw publish failed (result=%d), forcing broker %d reconnect", publish_result, i);
last_raw_publish_fail_log = now;
}
// Force disconnect to trigger reconnection
if (_mqtt_client->connected()) {
_mqtt_client->disconnect();
}
_brokers[i].connected = false;
_active_brokers--;
_brokers[i].last_attempt = 0; // Reset attempt time to allow immediate reconnect
// Update cached broker status
_cached_has_brokers = isAnyBrokerConnected();
}
}
}
+1 -21
View File
@@ -103,27 +103,7 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) {
if (len > sz) { len = sz; }
int err = _radio->readData(bytes, len);
if (err != RADIOLIB_ERR_NONE) {
// Throttle error logging to avoid spam (CRC errors are common in noisy environments)
static unsigned long last_crc_error_log = 0;
static unsigned long crc_error_count = 0;
unsigned long now = millis();
crc_error_count++;
// Log every 10 seconds max, or on first error
if (last_crc_error_log == 0 || (now - last_crc_error_log > 10000)) {
MESH_DEBUG_PRINTLN("RadioLibWrapper: error: readData(%d) - %lu errors in last 10s", err, crc_error_count);
last_crc_error_log = now;
crc_error_count = 0;
}
// For CRC mismatch errors (-7), reset radio state to prevent stuck conditions
// Similar to CustomLR1110's handling of header errors
if (err == -7) { // RADIOLIB_ERR_CRC_MISMATCH
// Call standby() to reset radio to known-good state
// This helps recover from false packet detections or corrupted state
_radio->standby();
}
MESH_DEBUG_PRINTLN("RadioLibWrapper: error: readData(%d)", err);
len = 0;
} else {
// Serial.print(" readData() -> "); Serial.println(len);