Refactor MQTTBridge connection handling for improved stability and responsiveness
- Simplified connection attempt logic to allow immediate reconnects after failures. - Added checks to ensure the MQTT client is disconnected before new connection attempts. - Enhanced error handling during publish operations to reset connection state on failures. - Removed aggressive health checks that caused connection instability, relying on the MQTT client library for connection management. - Updated status publishing to verify connection state before attempting to publish, ensuring accurate broker status.
This commit is contained in:
1 parent
39ba020fab
commit
c055228c91
3 files changed
+81
-40
No files matched your search
@@ -758,9 +758,7 @@ bool MyMesh::onPeerPathRecv(mesh::Packet *packet, int sender_idx, const uint8_t
|
||||
|
||||
void MyMesh::onControlDataRecv(mesh::Packet* packet) {
|
||||
uint8_t type = packet->payload[0] & 0xF0; // just test upper 4 bits
|
||||
if (type == CTL_TYPE_NODE_DISCOVER_REQ && packet->payload_len >= 6
|
||||
&& !_prefs.disable_fwd && discover_limiter.allow(rtc_clock.getCurrentTime())
|
||||
) {
|
||||
if (type == CTL_TYPE_NODE_DISCOVER_REQ && packet->payload_len >= 6 && discover_limiter.allow(rtc_clock.getCurrentTime())) {
|
||||
int i = 1;
|
||||
uint8_t filter = packet->payload[i++];
|
||||
uint32_t tag;
|
||||
|
||||
@@ -963,9 +963,11 @@ void MQTTBridge::connectToBrokers() {
|
||||
if (!_brokers[i].enabled) continue;
|
||||
|
||||
// Check if we need to attempt connection
|
||||
if (!_brokers[i].connected &&
|
||||
millis() - _brokers[i].last_attempt > _brokers[i].reconnect_interval) {
|
||||
|
||||
// Allow immediate reconnect if last_attempt is 0 (was reset due to failure)
|
||||
bool can_attempt = (_brokers[i].last_attempt == 0) ||
|
||||
(millis() - _brokers[i].last_attempt > _brokers[i].reconnect_interval);
|
||||
|
||||
if (!_brokers[i].connected && can_attempt) {
|
||||
MQTT_DEBUG_PRINTLN("Connecting to broker %d: %s:%d", i, _brokers[i].host, _brokers[i].port);
|
||||
|
||||
// Generate unique client ID
|
||||
@@ -982,6 +984,12 @@ void MQTTBridge::connectToBrokers() {
|
||||
_mqtt_client->setCredentials(_brokers[i].username, _brokers[i].password);
|
||||
}
|
||||
|
||||
// Ensure we're disconnected before attempting new connection
|
||||
if (_mqtt_client->connected()) {
|
||||
_mqtt_client->disconnect();
|
||||
vTaskDelay(pdMS_TO_TICKS(100)); // Brief delay to allow disconnect to complete
|
||||
}
|
||||
|
||||
// Connect to the broker (PsychicMqttClient uses async connection)
|
||||
_mqtt_client->connect();
|
||||
|
||||
@@ -992,18 +1000,26 @@ void MQTTBridge::connectToBrokers() {
|
||||
|
||||
// Maintain connection and check for stale connections
|
||||
if (_brokers[i].connected) {
|
||||
// Check actual connection state - if it's stale, mark as disconnected
|
||||
// PsychicMqttClient handles automatic reconnection internally, so we don't need
|
||||
// to force periodic reconnections. Only mark as disconnected if actually disconnected.
|
||||
// Check actual connection state - if it's stale, mark as disconnected and trigger reconnect
|
||||
// PsychicMqttClient handles automatic reconnection internally, but we need to detect stale state
|
||||
if (!_mqtt_client->connected()) {
|
||||
MQTT_DEBUG_PRINTLN("Broker %d connection lost, marking for reconnect", i);
|
||||
_brokers[i].connected = false;
|
||||
_active_brokers--;
|
||||
_brokers[i].last_attempt = 0; // Reset attempt time to allow immediate reconnect
|
||||
// Update cached broker status
|
||||
_cached_has_brokers = isAnyBrokerConnected();
|
||||
}
|
||||
// Removed aggressive 4-hour health check that was causing connection instability.
|
||||
// The MQTT client library handles connection health internally, and forcing
|
||||
// disconnections on healthy connections was causing hours of downtime.
|
||||
} else {
|
||||
// Not connected - ensure we attempt reconnection if enough time has passed
|
||||
// Reset last_attempt if it's been too long (prevents getting stuck)
|
||||
if (_brokers[i].last_attempt > 0 && (millis() - _brokers[i].last_attempt) > 300000) {
|
||||
// Been trying for more than 5 minutes - reset to allow fresh attempt
|
||||
_brokers[i].last_attempt = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1177,6 +1193,7 @@ bool MQTTBridge::publishStatus() {
|
||||
#endif
|
||||
|
||||
// Use cached destination status to avoid redundant checks
|
||||
// Note: Connection state is verified in connectToBrokers() which runs before publishStatus()
|
||||
bool has_custom_brokers = _cached_has_brokers && _config_valid;
|
||||
bool has_destinations = has_custom_brokers || _cached_has_analyzer_servers;
|
||||
|
||||
@@ -1184,6 +1201,26 @@ bool MQTTBridge::publishStatus() {
|
||||
return false; // No destinations available
|
||||
}
|
||||
|
||||
// Final verification: ensure client is actually connected before attempting publish
|
||||
// This catches cases where connection state got out of sync
|
||||
if (has_custom_brokers && _mqtt_client && !_mqtt_client->connected()) {
|
||||
// Client says disconnected but our state says connected - fix the state
|
||||
MQTT_DEBUG_PRINTLN("Status publish: Client disconnected but state says connected - fixing state");
|
||||
for (int i = 0; i < MAX_MQTT_BROKERS_COUNT; i++) {
|
||||
if (_brokers[i].enabled && _brokers[i].connected) {
|
||||
_brokers[i].connected = false;
|
||||
_active_brokers--;
|
||||
_brokers[i].last_attempt = 0; // Allow immediate reconnect
|
||||
}
|
||||
}
|
||||
_cached_has_brokers = false;
|
||||
has_custom_brokers = false;
|
||||
has_destinations = has_custom_brokers || _cached_has_analyzer_servers;
|
||||
if (!has_destinations) {
|
||||
return false; // No destinations after fixing state
|
||||
}
|
||||
}
|
||||
|
||||
// Status messages with stats can be larger (~400-500 bytes), so increase buffer size
|
||||
char json_buffer[768]; // Increased from 512 to accommodate stats object
|
||||
char origin_id[65];
|
||||
@@ -1263,13 +1300,15 @@ bool MQTTBridge::publishStatus() {
|
||||
size_t json_len = strlen(json_buffer); // Cache length to avoid multiple strlen() calls
|
||||
|
||||
// Publish to all connected custom brokers
|
||||
// Double-check client is actually connected before attempting publish
|
||||
if (_config_valid && _mqtt_client && _mqtt_client->connected()) {
|
||||
// Track last broker URI to avoid calling setServer() unnecessarily (memory optimization)
|
||||
// setServer() may allocate memory, so we only call it when the broker changes
|
||||
static char last_broker_uri_status[128] = "";
|
||||
|
||||
for (int i = 0; i < MAX_MQTT_BROKERS_COUNT; i++) {
|
||||
if (_brokers[i].enabled && _brokers[i].connected) {
|
||||
// Verify broker is actually connected (state might be stale)
|
||||
if (_brokers[i].enabled && _brokers[i].connected && _mqtt_client->connected()) {
|
||||
|
||||
// Build broker URI
|
||||
char broker_uri[128];
|
||||
@@ -1287,15 +1326,22 @@ bool MQTTBridge::publishStatus() {
|
||||
if (publish_result > 0) {
|
||||
published = true;
|
||||
} else {
|
||||
// Publish failed - connection may be stale, mark as disconnected
|
||||
// Publish failed - connection may be stale, force disconnect and mark for reconnect
|
||||
static unsigned long last_status_publish_fail_log = 0;
|
||||
unsigned long now = millis();
|
||||
if (now - last_status_publish_fail_log > 60000) { // Log every minute max
|
||||
MQTT_DEBUG_PRINTLN("Status publish failed (result=%d), marking broker %d as disconnected", publish_result, i);
|
||||
MQTT_DEBUG_PRINTLN("Status publish failed (result=%d), forcing broker %d reconnect", publish_result, i);
|
||||
last_status_publish_fail_log = now;
|
||||
}
|
||||
// Force disconnect to trigger reconnection
|
||||
if (_mqtt_client->connected()) {
|
||||
_mqtt_client->disconnect();
|
||||
}
|
||||
_brokers[i].connected = false;
|
||||
_active_brokers--;
|
||||
_brokers[i].last_attempt = 0; // Reset attempt time to allow immediate reconnect
|
||||
// Update cached broker status
|
||||
_cached_has_brokers = isAnyBrokerConnected();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1403,13 +1449,15 @@ void MQTTBridge::publishPacket(mesh::Packet* packet, bool is_tx,
|
||||
size_t json_len = strlen(json_buffer); // Cache length to avoid multiple strlen() calls
|
||||
|
||||
// Publish to custom brokers (only if config is valid)
|
||||
// Double-check client is actually connected before attempting publish
|
||||
if (_config_valid && _mqtt_client && _mqtt_client->connected()) {
|
||||
// Track last broker URI to avoid calling setServer() unnecessarily (memory optimization)
|
||||
// setServer() may allocate memory, so we only call it when the broker changes
|
||||
static char last_broker_uri[128] = "";
|
||||
|
||||
for (int i = 0; i < MAX_MQTT_BROKERS_COUNT; i++) {
|
||||
if (_brokers[i].enabled && _brokers[i].connected) {
|
||||
// Verify broker is actually connected (state might be stale)
|
||||
if (_brokers[i].enabled && _brokers[i].connected && _mqtt_client->connected()) {
|
||||
// Build broker URI
|
||||
char broker_uri[128];
|
||||
snprintf(broker_uri, sizeof(broker_uri), "mqtt://%s:%d", _brokers[i].host, _brokers[i].port);
|
||||
@@ -1425,16 +1473,22 @@ void MQTTBridge::publishPacket(mesh::Packet* packet, bool is_tx,
|
||||
// This prevents blocking the main loop when MQTT broker is slow or unresponsive
|
||||
int publish_result = _mqtt_client->publish(topic, 1, false, json_buffer, json_len); // qos=1, retained=false
|
||||
if (publish_result <= 0) {
|
||||
// Publish failed - connection may be stale, mark as disconnected
|
||||
// This prevents accumulation of failed publishes that block the loop
|
||||
// Publish failed - connection may be stale, force disconnect and mark for reconnect
|
||||
static unsigned long last_publish_fail_log = 0;
|
||||
unsigned long now = millis();
|
||||
if (now - last_publish_fail_log > 60000) { // Log every minute max
|
||||
MQTT_DEBUG_PRINTLN("Publish failed (result=%d), marking broker %d as disconnected", publish_result, i);
|
||||
MQTT_DEBUG_PRINTLN("Publish failed (result=%d), forcing broker %d reconnect", publish_result, i);
|
||||
last_publish_fail_log = now;
|
||||
}
|
||||
// Force disconnect to trigger reconnection
|
||||
if (_mqtt_client->connected()) {
|
||||
_mqtt_client->disconnect();
|
||||
}
|
||||
_brokers[i].connected = false;
|
||||
_active_brokers--;
|
||||
_brokers[i].last_attempt = 0; // Reset attempt time to allow immediate reconnect
|
||||
// Update cached broker status
|
||||
_cached_has_brokers = isAnyBrokerConnected();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1502,13 +1556,15 @@ void MQTTBridge::publishRaw(mesh::Packet* packet) {
|
||||
size_t json_len = strlen(json_buffer); // Cache length to avoid multiple strlen() calls
|
||||
|
||||
// Publish to custom brokers (only if config is valid)
|
||||
// Double-check client is actually connected before attempting publish
|
||||
if (_config_valid && _mqtt_client && _mqtt_client->connected()) {
|
||||
// Track last broker URI to avoid calling setServer() unnecessarily (memory optimization)
|
||||
// setServer() may allocate memory, so we only call it when the broker changes
|
||||
static char last_broker_uri_raw[128] = "";
|
||||
|
||||
for (int i = 0; i < MAX_MQTT_BROKERS_COUNT; i++) {
|
||||
if (_brokers[i].enabled && _brokers[i].connected) {
|
||||
// Verify broker is actually connected (state might be stale)
|
||||
if (_brokers[i].enabled && _brokers[i].connected && _mqtt_client->connected()) {
|
||||
// Build broker URI
|
||||
char broker_uri[128];
|
||||
snprintf(broker_uri, sizeof(broker_uri), "mqtt://%s:%d", _brokers[i].host, _brokers[i].port);
|
||||
@@ -1523,15 +1579,22 @@ void MQTTBridge::publishRaw(mesh::Packet* packet) {
|
||||
// Publish with timeout check - don't block if connection is slow
|
||||
int publish_result = _mqtt_client->publish(topic, 1, false, json_buffer, json_len); // qos=1, retained=false
|
||||
if (publish_result <= 0) {
|
||||
// Publish failed - connection may be stale, mark as disconnected
|
||||
// Publish failed - connection may be stale, force disconnect and mark for reconnect
|
||||
static unsigned long last_raw_publish_fail_log = 0;
|
||||
unsigned long now = millis();
|
||||
if (now - last_raw_publish_fail_log > 60000) { // Log every minute max
|
||||
MQTT_DEBUG_PRINTLN("Raw publish failed (result=%d), marking broker %d as disconnected", publish_result, i);
|
||||
MQTT_DEBUG_PRINTLN("Raw publish failed (result=%d), forcing broker %d reconnect", publish_result, i);
|
||||
last_raw_publish_fail_log = now;
|
||||
}
|
||||
// Force disconnect to trigger reconnection
|
||||
if (_mqtt_client->connected()) {
|
||||
_mqtt_client->disconnect();
|
||||
}
|
||||
_brokers[i].connected = false;
|
||||
_active_brokers--;
|
||||
_brokers[i].last_attempt = 0; // Reset attempt time to allow immediate reconnect
|
||||
// Update cached broker status
|
||||
_cached_has_brokers = isAnyBrokerConnected();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,27 +103,7 @@ int RadioLibWrapper::recvRaw(uint8_t* bytes, int sz) {
|
||||
if (len > sz) { len = sz; }
|
||||
int err = _radio->readData(bytes, len);
|
||||
if (err != RADIOLIB_ERR_NONE) {
|
||||
// Throttle error logging to avoid spam (CRC errors are common in noisy environments)
|
||||
static unsigned long last_crc_error_log = 0;
|
||||
static unsigned long crc_error_count = 0;
|
||||
unsigned long now = millis();
|
||||
crc_error_count++;
|
||||
|
||||
// Log every 10 seconds max, or on first error
|
||||
if (last_crc_error_log == 0 || (now - last_crc_error_log > 10000)) {
|
||||
MESH_DEBUG_PRINTLN("RadioLibWrapper: error: readData(%d) - %lu errors in last 10s", err, crc_error_count);
|
||||
last_crc_error_log = now;
|
||||
crc_error_count = 0;
|
||||
}
|
||||
|
||||
// For CRC mismatch errors (-7), reset radio state to prevent stuck conditions
|
||||
// Similar to CustomLR1110's handling of header errors
|
||||
if (err == -7) { // RADIOLIB_ERR_CRC_MISMATCH
|
||||
// Call standby() to reset radio to known-good state
|
||||
// This helps recover from false packet detections or corrupted state
|
||||
_radio->standby();
|
||||
}
|
||||
|
||||
MESH_DEBUG_PRINTLN("RadioLibWrapper: error: readData(%d)", err);
|
||||
len = 0;
|
||||
} else {
|
||||
// Serial.print(" readData() -> "); Serial.println(len);
|
||||
|
||||
Reference in new issue
Block a user