fix(mqtt): gate reconnect-backoff reset on 2 min of connection stability

A CONNACK alone reset the backoff ladder, so a flapping broker (accepts
then drops within seconds) retried at the 10 s rung forever - each
attempt a full ~40 KB TLS session alloc/free on internal heap, a known
fragmentation driver. The ladder now clears only after the connection
survives 2 minutes (at least one 75 s keepalive round-trip); flapping
endpoints degrade to the 300 s rung and then the existing 30-minute
circuit-breaker probes, and recover automatically once stable.
This commit is contained in:
agessaman committed 2026-07-10 19:03:38 -07:00
1 parent cd6ad2333a
commit c119ad2ab9
2 files changed
+29 -3

No files matched your search

+27 -3
View File
@@ -1119,8 +1119,18 @@ void MQTTBridge::initSlotClients() {
slot.client->onConnect([this, index](bool sessionPresent) {
MQTT_DEBUG_PRINTLN("MQTT%d connected", index + 1);
_slots[index].connected = true;
_slots[index].reconnect_backoff = 0;
_slots[index].max_backoff_failures = 0;
// NOTE: reconnect_backoff / max_backoff_failures are NOT reset here.
// A CONNACK alone doesn't prove the link is healthy — a broker that
// accepts and then drops within seconds would reset the ladder every
// cycle and retry at the 10 s rung forever, and each retry is a full
// TLS session alloc/free (~40 KB of internal-heap churn, a known
// fragmentation driver). The ladder is instead cleared by
// maintainSlotConnection() once the connection has stayed up for
// BACKOFF_STABLE_RESET_MS, so flapping endpoints keep their earned
// backoff level. The breaker itself does clear now: while connected
// the diag/status must not claim the slot gave up, and the next
// disconnect should be governed by the (still-elevated) ladder.
_slots[index].connected_at_ms = millis();
_slots[index].circuit_breaker_tripped = false;
_slots[index].last_tls_err = 0;
_slots[index].last_tls_stack_err = 0;
@@ -1140,6 +1150,7 @@ void MQTTBridge::initSlotClients() {
_slots[index].current_outage_started_ms = millis();
}
_slots[index].connected = false;
_slots[index].connected_at_ms = 0; // stability clock only runs while connected
updateCachedConnectionStatus();
});
slot.client->onError([this, index](esp_mqtt_error_codes error) {
@@ -1453,7 +1464,20 @@ void MQTTBridge::maintainSlotConnections() {
void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, unsigned long current_time, bool time_synced, bool& reconnect_attempted, bool& teardown_attempted) {
MQTTSlot& slot = _slots[index];
if (slot.connected) {
// Forgive past failures only after the connection has proven stable.
// 2 minutes covers at least one keepalive round-trip (keepalive is 75 s),
// so a link that can't survive a single keepalive period never resets the
// ladder. Flapping endpoints therefore stay at their earned backoff rung
// (worst case the 300 s rung / 30-minute breaker probes) instead of
// hammering full TLS handshakes at the 10 s rung — see the onConnect
// handler in initSlotClients() for why this doesn't happen on CONNACK.
static const unsigned long BACKOFF_STABLE_RESET_MS = 120000UL;
if (slot.connected &&
(slot.reconnect_backoff != 0 || slot.max_backoff_failures != 0) &&
slot.connected_at_ms != 0 &&
(now_millis - slot.connected_at_ms) >= BACKOFF_STABLE_RESET_MS) {
MQTT_DEBUG_PRINTLN("MQTT%d stable for %lus - clearing reconnect backoff (was level %d)",
index + 1, (now_millis - slot.connected_at_ms) / 1000UL, slot.reconnect_backoff);
slot.reconnect_backoff = 0;
slot.max_backoff_failures = 0;
}
+2
View File
@@ -89,6 +89,8 @@ private:
uint8_t reconnect_backoff; // 0..4 index into backoff table
uint8_t max_backoff_failures; // consecutive failures at max backoff level
bool circuit_breaker_tripped; // true = stop reconnecting until reconfigured
unsigned long connected_at_ms; // millis() of last successful connect (0 = not connected);
// gates the stability-based backoff reset in maintenance
unsigned long last_reconnect_attempt;
unsigned long last_log_time; // Throttle disconnect log messages
unsigned long last_deferred_log_ms; // Throttle "connect deferred" log spam (Phase 1)