Recover stalled MQTT transports and broker acknowledgments

This commit is contained in:
mikecarper committed 2026-10-14 10:34:40 -07:00
1 parent 8a49a98709
commit 3347a62780
17 files changed
+1201 -23

No files matched your search

+79
View File
@@ -0,0 +1,79 @@
#pragma once
#include <stdint.h>
// Owner-task transport recovery, separate from the reconnect/backoff policy.
// A CONNECTED callback or accepted/enqueued publish is not proof of delivery.
namespace MQTTTransportHealth {
// Twelve times the normal 10-second socket budget gives DNS/TLS/WebSocket/
// CONNACK setup ample room, while bounding a lost final callback. PUBACK uses
// the same generous budget: longer than the 15-second retry and SDK outbox
// expiry, so an ordinary retransmit cannot trigger a premature reconnect.
static const uint32_t kAttemptTimeoutMs = 120000UL;
static const uint32_t kAckTimeoutMs = 120000UL;
enum class RecoveryReason : uint8_t {
None, TransportError, AdapterDisconnected, HandshakeTimeout,
PublishFailure, AckTimeout
};
inline const char* reasonLabel(RecoveryReason reason) {
switch (reason) {
case RecoveryReason::TransportError: return "error";
case RecoveryReason::AdapterDisconnected: return "state";
case RecoveryReason::HandshakeTimeout: return "handshake";
case RecoveryReason::PublishFailure: return "write";
case RecoveryReason::AckTimeout: return "puback";
default: return "none";
}
}
struct State {
bool attempt_active = false;
uint32_t attempt_started_ms = 0;
bool ack_pending = false;
uint32_t ack_started_ms = 0;
uint32_t ack_snapshot = 0;
RecoveryReason recovery_reason = RecoveryReason::None;
RecoveryReason last_recovery_reason = RecoveryReason::None;
uint32_t recoveries = 0;
void beginAttempt(uint32_t now) {
attempt_active = true;
attempt_started_ms = now;
}
void completeAttempt() { attempt_active = false; }
void observeAck(uint32_t count) {
if (ack_pending && count != ack_snapshot) ack_pending = false;
}
void acceptedQos1(uint32_t now, uint32_t ack_before) {
observeAck(ack_before);
if (!ack_pending) {
ack_pending = true;
ack_started_ms = now;
ack_snapshot = ack_before;
}
}
bool attemptExpired(uint32_t now) const {
return attempt_active && now - attempt_started_ms >= kAttemptTimeoutMs;
}
bool ackExpired(uint32_t now) const {
return ack_pending && now - ack_started_ms >= kAckTimeoutMs;
}
void requestRecovery(RecoveryReason reason) {
if (recovery_reason == RecoveryReason::None) recovery_reason = reason;
}
void resetSession() {
attempt_active = false;
ack_pending = false;
}
void recovered() {
if (recoveries != UINT32_MAX) ++recoveries;
last_recovery_reason = recovery_reason;
recovery_reason = RecoveryReason::None;
resetSession();
}
};
} // namespace MQTTTransportHealth
+242 -13
View File
@@ -426,6 +426,10 @@ void MQTTBridge::formatMqttStatusReply(char* buf, size_t bufsize, const MQTTPref
} else if (!b->isSlotReady(i)) {
name = slot.preset ? slot.preset->name : "custom";
state = "wait";
} else if (b->_slot_health_recovering[i].load(std::memory_order_acquire) ||
b->_slot_transport_fault_pending[i].load(std::memory_order_acquire)) {
name = slot.preset ? slot.preset->name : "custom";
state = "recover";
} else if (slot.connected) {
name = slot.preset ? slot.preset->name : "custom";
state = "ok";
@@ -551,6 +555,9 @@ bool MQTTBridge::getSlotStatusSnapshot(int slot_index, SlotStatusSnapshot* out)
out->name = slot.preset ? slot.preset->name : "custom";
if (!b->isSlotReady(slot_index)) {
out->state = "wait";
} else if (b->_slot_health_recovering[slot_index].load(std::memory_order_acquire) ||
b->_slot_transport_fault_pending[slot_index].load(std::memory_order_acquire)) {
out->state = "recover";
} else if (slot.connected) {
out->state = "ok";
} else if (slot.circuit_breaker_tripped) {
@@ -657,6 +664,9 @@ void MQTTBridge::formatSlotDiagReply(char* buf, size_t bufsize, int slot_index)
} else if (!slot.client) {
// Ready to connect but the client object could not be allocated.
state = "no client";
} else if (b->_slot_health_recovering[slot_index].load(std::memory_order_acquire) ||
b->_slot_transport_fault_pending[slot_index].load(std::memory_order_acquire)) {
state = "recover";
} else if (slot.connected) {
state = "ok";
} else if (slot.circuit_breaker_tripped) {
@@ -689,7 +699,7 @@ void MQTTBridge::formatSlotDiagReply(char* buf, size_t bufsize, int slot_index)
}
// Connected with no errors: nothing more to say about the connection.
if (slot.connected && slot.last_error_time == 0) {
if (slot.connected && strcmp(state, "ok") == 0 && slot.last_error_time == 0) {
replyAppendf(buf, bufsize, &pos, ", no errors");
} else if (slot.last_error_time > 0) {
// TLS error with human-friendly description
@@ -743,6 +753,28 @@ void MQTTBridge::formatSlotDiagReply(char* buf, size_t bufsize, int slot_index)
}
}
// Recovery/ACK age are owner-task snapshots. Append complete fields only;
// error details and the existing broker ACK count retain reply priority.
const uint32_t recoveries = b->_slot_health_recoveries[slot_index].load(std::memory_order_relaxed);
const auto recovery_reason = static_cast<MQTTTransportHealth::RecoveryReason>(
b->_slot_health_reason[slot_index].load(std::memory_order_relaxed));
if (recoveries || recovery_reason != MQTTTransportHealth::RecoveryReason::None) {
char recovery_text[48];
const int len = snprintf(recovery_text, sizeof(recovery_text), ", rec:%lu/%s",
static_cast<unsigned long>(recoveries), MQTTTransportHealth::reasonLabel(recovery_reason));
if (len > 0 && len < (int)sizeof(recovery_text) && len <= (int)bufsize - 1 - pos) {
replyAppendf(buf, bufsize, &pos, "%s", recovery_text);
}
}
const uint32_t ack_wait = b->_slot_health_ack_wait_secs[slot_index].load(std::memory_order_relaxed);
if (ack_wait != UINT32_MAX) {
char wait_text[32];
const int len = snprintf(wait_text, sizeof(wait_text), ", ack_wait:%lus", static_cast<unsigned long>(ack_wait));
if (len > 0 && len < (int)sizeof(wait_text) && len <= (int)bufsize - 1 - pos) {
replyAppendf(buf, bufsize, &pos, "%s", wait_text);
}
}
// Appended last so it never displaces connection diagnostics. replyAppendf
// clamps rather than overflows, but a clipped type list is worse than no
// list: "...,13,14," parses as a real, different allowlist, and this is the
@@ -875,11 +907,11 @@ MQTTBridge::MQTTBridge(const MQTTNodeInfo& node_info, MQTTPrefs *obs,
// Initialize all slots to empty/disabled state
for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) {
memset(&_slots[i], 0, sizeof(MQTTSlot));
_slots[i] = MQTTSlot{};
_slots[i].enabled = false;
_slots[i].client = nullptr;
_slots[i].preset = nullptr;
// auth_token == nullptr after memset above - allocated on first token creation
// auth_token == nullptr after value initialization - allocated on first token creation
_slots[i].connected = false;
_slots[i].initial_connect_done = false;
_slots[i].token_expires_at = 0;
@@ -893,6 +925,13 @@ MQTTBridge::MQTTBridge(const MQTTNodeInfo& node_info, MQTTPrefs *obs,
_slot_reconfigure_pending[i] = false;
_slot_force_jwt_mint[i] = false;
_slot_attempt_pending[i] = false;
_slot_transport_fault_pending[i] = false;
_slot_health_reset_pending[i] = false;
_slots[i].transport_health = MQTTTransportHealth::State();
_slot_health_recoveries[i] = 0;
_slot_health_reason[i] = static_cast<uint8_t>(MQTTTransportHealth::RecoveryReason::None);
_slot_health_recovering[i] = false;
_slot_health_ack_wait_secs[i] = UINT32_MAX;
_status_publish_pending[i] = false;
}
@@ -1024,6 +1063,11 @@ void MQTTBridge::begin() {
_slots[i].broker_uri[0] = '\0';
_slots[i].port = 1883;
_slot_reconfigure_pending[i] = false;
_slot_transport_fault_pending[i] = false;
_slot_health_reset_pending[i] = false;
_slots[i].transport_health.resetSession();
_slots[i].transport_health.recovery_reason = MQTTTransportHealth::RecoveryReason::None;
updateSlotTransportDiag(i, static_cast<uint32_t>(millis()));
}
_cached_has_connected_slots = false;
_slots_setup_done = false;
@@ -1617,13 +1661,33 @@ bool MQTTBridge::serviceNetworkPower(uint32_t now) {
cancelNtpRefresh();
_ntp_sync_pending = false;
for (int i = 0; i < RUNTIME_MQTT_SLOTS; ++i) {
// A foreground/probe request can resume the network policy after a
// failed suspension. Keep a transport reservation until stop is proven.
if (_slots[i].client && _slots[i].client->isStarted()) {
_slot_transport_fault_pending[i].store(true, std::memory_order_release);
}
_slot_attempt_pending[i] = false;
if (_slots[i].client && _slots[i].client->isStarted()) {
_slots[i].client->disconnect();
// The wrapper only clears isStarted after SDK stop succeeds. Never
// hand station ownership off while any transport still runs.
if (_slots[i].client->isStarted()) return false;
if (_slots[i].client->isStarted()) {
_slots[i].transport_health.requestRecovery(MQTTTransportHealth::RecoveryReason::AdapterDisconnected);
_slots[i].connected = false;
_slots[i].connected_at_ms = 0;
if (_slots[i].current_outage_started_ms == 0) {
_slots[i].current_outage_started_ms = now ? now : 1;
}
updateSlotTransportDiag(i, static_cast<uint32_t>(millis()));
updateCachedConnectionStatus();
return false;
}
}
_slot_transport_fault_pending[i].store(false, std::memory_order_release);
_slot_health_reset_pending[i].store(false, std::memory_order_release);
_slots[i].transport_health.resetSession();
_slots[i].transport_health.recovery_reason = MQTTTransportHealth::RecoveryReason::None;
updateSlotTransportDiag(i, static_cast<uint32_t>(millis()));
_slots[i].connected = false;
_slots[i].connected_at_ms = 0;
_status_publish_pending[i] = false;
@@ -2040,8 +2104,8 @@ void MQTTBridge::mqttTaskLoop() {
}
}
// Update cached connection status periodically (every 5 seconds)
// This ensures cache stays accurate even if callbacks miss updates
// Refresh the aggregate cache periodically; owner maintenance above
// reconciles adapter state and bounds missed terminal callbacks.
static unsigned long last_slot_status_update = 0;
if (now - last_slot_status_update > 5000) {
updateCachedConnectionStatus();
@@ -2092,6 +2156,7 @@ bool MQTTBridge::ensureSlotClient(int index) {
MQTT_DEBUG_PRINTLN("MQTT%d connected", index + 1);
_slot_attempt_pending[index] = false;
_slots[index].connected = true;
_slot_health_reset_pending[index].store(true, std::memory_order_release);
_slot_force_jwt_mint[index] = false;
// NOTE: reconnect_backoff / max_backoff_failures are NOT reset here.
// A CONNACK alone doesn't prove the link is healthy -- a broker that
@@ -2142,6 +2207,10 @@ bool MQTTBridge::ensureSlotClient(int index) {
updateCachedConnectionStatus();
});
slot.client->onError([this, index](esp_mqtt_error_codes error) {
// The adapter is already disconnected at MQTT_EVENT_ERROR. Do not stop
// its SDK task from inside that task's callback, or release the global
// handshake reservation before the owner proves the SDK has stopped.
_slot_transport_fault_pending[index].store(true, std::memory_order_release);
_slots[index].last_tls_err = error.esp_tls_last_esp_err;
_slots[index].last_tls_stack_err = error.esp_tls_stack_err;
_slots[index].last_sock_errno = error.esp_transport_sock_errno;
@@ -2255,7 +2324,9 @@ bool MQTTBridge::hasPendingSlotConnection() const {
// Callbacks run on esp-mqtt tasks. Time spacing alone cannot establish that
// an earlier handshake has finished, particularly on a slow WiFi link.
for (int i = 0; i < RUNTIME_MQTT_SLOTS; ++i) {
if (_slot_attempt_pending[i].load(std::memory_order_acquire)) return true;
if (_slot_attempt_pending[i].load(std::memory_order_acquire) ||
_slot_transport_fault_pending[i].load(std::memory_order_acquire) ||
_slots[i].transport_health.recovery_reason != MQTTTransportHealth::RecoveryReason::None) return true;
}
return false;
}
@@ -2326,6 +2397,9 @@ bool MQTTBridge::setupSlot(int index) {
slot.client->disconnect();
}
if (slot.client->isStarted()) {
_slot_transport_fault_pending[index].store(true, std::memory_order_release);
slot.transport_health.requestRecovery(MQTTTransportHealth::RecoveryReason::AdapterDisconnected);
updateSlotTransportDiag(index, static_cast<uint32_t>(millis()));
slot.last_reconnect_attempt = millis();
return false;
}
@@ -2558,7 +2632,14 @@ void MQTTBridge::teardownSlot(int index, bool force) {
delay(50);
#endif
}
if (slot.client && !force && slot.client->isStarted()) return;
if (slot.client && !force && slot.client->isStarted()) {
// Even a deliberate reconfigure must retain admission ownership if SDK
// stop failed. The worker will retry cleanup; no second handshake may run.
_slot_transport_fault_pending[index].store(true, std::memory_order_release);
slot.transport_health.requestRecovery(MQTTTransportHealth::RecoveryReason::AdapterDisconnected);
updateSlotTransportDiag(index, static_cast<uint32_t>(millis()));
return;
}
// Invalidate the token but keep the buffer: the client survives teardown and still
// holds this pointer in its config (see MQTTSlot::auth_token).
@@ -2575,6 +2656,11 @@ void MQTTBridge::teardownSlot(int index, bool force) {
slot.last_log_time = 0;
slot.last_deferred_log_ms = 0;
_slot_force_jwt_mint[index] = false;
_slot_transport_fault_pending[index] = false;
_slot_health_reset_pending[index] = false;
slot.transport_health.resetSession();
slot.transport_health.recovery_reason = MQTTTransportHealth::RecoveryReason::None;
updateSlotTransportDiag(index, static_cast<uint32_t>(millis()));
}
esp_err_t MQTTBridge::reconnectSlotClient(int index) {
@@ -2586,6 +2672,7 @@ esp_err_t MQTTBridge::reconnectSlotClient(int index) {
const bool was_pending = _slot_attempt_pending[index];
const bool starting = !slot.client->isStarted();
slot.transport_health.beginAttempt(static_cast<uint32_t>(millis()));
_slot_attempt_pending[index] = true; // callbacks may run before the call returns
esp_err_t result;
if (starting) {
@@ -2597,6 +2684,7 @@ esp_err_t MQTTBridge::reconnectSlotClient(int index) {
if (result != ESP_OK) {
if (MQTTConnectionHealth::clearPendingAfterRejectedRequest(starting, was_pending)) {
_slot_attempt_pending[index] = false;
slot.transport_health.completeAttempt();
}
if (starting) {
slot.start_failures = MQTTConnectionHealth::incrementFailures(slot.start_failures);
@@ -2605,8 +2693,112 @@ esp_err_t MQTTBridge::reconnectSlotClient(int index) {
return result;
}
void MQTTBridge::updateSlotTransportDiag(int index, uint32_t now) {
const MQTTTransportHealth::State& health = _slots[index].transport_health;
_slot_health_recoveries[index].store(health.recoveries, std::memory_order_relaxed);
const auto reason = health.recovery_reason != MQTTTransportHealth::RecoveryReason::None
? health.recovery_reason : health.last_recovery_reason;
_slot_health_reason[index].store(static_cast<uint8_t>(reason), std::memory_order_relaxed);
_slot_health_recovering[index].store(
health.recovery_reason != MQTTTransportHealth::RecoveryReason::None,
std::memory_order_release);
_slot_health_ack_wait_secs[index].store(
health.ack_pending ? (now - health.ack_started_ms) / 1000UL : UINT32_MAX,
std::memory_order_relaxed);
}
bool MQTTBridge::recoverSlotTransport(int index, uint32_t now) {
MQTTSlot& slot = _slots[index];
auto& health = slot.transport_health;
if (!slot.client || health.recovery_reason == MQTTTransportHealth::RecoveryReason::None) return false;
MQTT_DEBUG_PRINTLN("MQTT%d transport recovery: %s", index + 1,
MQTTTransportHealth::reasonLabel(health.recovery_reason));
const bool failed_attempt = MQTTConnectionHealth::disconnectFailedAttempt(
_slot_attempt_pending[index].load(std::memory_order_acquire), slot.connected);
const uint32_t failures_before_stop = slot.connect_failures;
// Retain the client, SDK buffers and credentials. disconnect() closes the
// transport and stops its task, and leaves isStarted() true if SDK stop fails.
// Never vTaskDelete a live SDK task or admit a second TLS handshake on failure.
slot.client->disconnect();
if (slot.client->isStarted()) {
updateSlotTransportDiag(index, static_cast<uint32_t>(millis()));
return false;
}
// SDK stop can consume its socket budget. Space the next handshake from
// completed cleanup, not from the earlier watchdog observation.
now = static_cast<uint32_t>(millis());
// A real DISCONNECTED callback may already have accounted this failure.
// ERROR-only/lost-callback attempts need the same accounting, once.
if (failed_attempt && slot.connect_failures == failures_before_stop) {
slot.connect_failures = MQTTConnectionHealth::incrementFailures(slot.connect_failures);
}
_slot_attempt_pending[index].store(false, std::memory_order_release);
_slot_transport_fault_pending[index].store(false, std::memory_order_release);
_slot_health_reset_pending[index].store(false, std::memory_order_release);
_status_publish_pending[index] = false;
slot.connected = false;
slot.connected_at_ms = 0;
if (slot.current_outage_started_ms == 0) slot.current_outage_started_ms = now ? now : 1;
slot.last_reconnect_attempt = now;
_last_slot_reconnect_ms = now;
health.recovered();
updateSlotTransportDiag(index, now);
updateCachedConnectionStatus();
return true;
}
void MQTTBridge::serviceSlotTransportHealth(uint32_t now) {
if (_stop_requested.load(std::memory_order_acquire)) return;
for (int i = 0; i < RUNTIME_MQTT_SLOTS; ++i) {
MQTTSlot& slot = _slots[i];
auto& health = slot.transport_health;
if (_slot_health_reset_pending[i].exchange(false, std::memory_order_acq_rel)) {
// A late CONNACK does not release an already-required SDK stop. Its
// session clocks may reset, but only proven cleanup clears recovery.
health.resetSession();
}
const bool pending = _slot_attempt_pending[i].load(std::memory_order_acquire);
if (!pending) health.completeAttempt();
if (slot.client) {
health.observeAck(static_cast<uint32_t>(slot.client->getPublishAck()));
if (_slot_transport_fault_pending[i].load(std::memory_order_acquire)) {
health.requestRecovery(MQTTTransportHealth::RecoveryReason::TransportError);
} else if (slot.connected && !slot.client->connected()) {
health.requestRecovery(MQTTTransportHealth::RecoveryReason::AdapterDisconnected);
} else if (pending && health.attemptExpired(now)) {
health.requestRecovery(MQTTTransportHealth::RecoveryReason::HandshakeTimeout);
} else if (slot.connected && health.ackExpired(now)) {
health.requestRecovery(MQTTTransportHealth::RecoveryReason::AckTimeout);
}
if (!slot.connected && !pending) health.resetSession();
}
if (health.recovery_reason != MQTTTransportHealth::RecoveryReason::None) {
// Stop advertising a healthy link immediately, including while SDK
// cleanup is still failing. Admission is held by the recovery state,
// independently of this observable connection flag.
slot.connected = false;
slot.connected_at_ms = 0;
if (slot.current_outage_started_ms == 0) slot.current_outage_started_ms = now ? now : 1;
}
updateSlotTransportDiag(i, now);
}
updateCachedConnectionStatus();
// Close no more than one SDK transport per pass. A failed stop keeps its
// recovery reservation and blocks all new handshakes on subsequent passes.
for (int i = 0; i < RUNTIME_MQTT_SLOTS; ++i) {
if (_slots[i].client && _slots[i].transport_health.recovery_reason != MQTTTransportHealth::RecoveryReason::None) {
recoverSlotTransport(i, now);
break;
}
}
}
void MQTTBridge::maintainSlotConnections() {
if (_stop_requested.load(std::memory_order_acquire)) return;
serviceSlotTransportHealth(static_cast<uint32_t>(millis()));
if (!_identity) return;
// Check WiFi status first
@@ -3003,6 +3195,14 @@ bool MQTTBridge::publishToSlot(int index, const char* topic, const char* payload
if (_stop_requested.load(std::memory_order_acquire)) return false;
if (index < 0 || index >= RUNTIME_MQTT_SLOTS) return false;
MQTTSlot& slot = _slots[index];
// On-connect status is processed before maintenance. Consume this new-
// session latch BEFORE enqueueing it, or maintenance would erase its ACK
// watch immediately afterwards. Only this owner task mutates health state.
if (_slot_health_reset_pending[index].exchange(false, std::memory_order_acq_rel)) {
slot.transport_health.resetSession();
}
if (slot.transport_health.recovery_reason != MQTTTransportHealth::RecoveryReason::None ||
_slot_transport_fault_pending[index].load(std::memory_order_acquire)) return false;
if (!slot.client || !slot.connected) {
unsigned long now = millis();
if (now - slot.last_log_time > SLOT_LOG_INTERVAL) {
@@ -3020,16 +3220,23 @@ bool MQTTBridge::publishToSlot(int index, const char* topic, const char* payload
// write bypasses that drain ceiling entirely and does not store in the outbox. It can
// block the (Core-0, prio-1) MQTT task on a stalled socket, but only up to
// network_timeout_ms (configured in optimizeMqttClientConfig); mesh RX (Core 1) and the
// WiFi/TCP stack (higher-prio system tasks) are unaffected, and a failed write flips
// the slot to disconnected so subsequent packets skip it.
// WiFi/TCP stack (higher-prio system tasks) are unaffected. A failed write reserves
// owner-task transport recovery, so subsequent packets skip the failed slot.
// - QoS 1 (low-rate retained status): async, so it keeps the durable outbox + retransmit.
//
// Return convention: QoS 0 sync publish returns msg_id == 0 on success (no PUBACK
// tracking). Negative values (-1 write/failure) are the only actual failures; the queue
// retry/drop path below handles them.
bool async = (qos > 0);
// Take the actual broker ACK counter before the SDK call: a fast PUBACK can
// arrive before enqueue returns. A post-call snapshot would lose that proof.
const uint32_t ack_before = static_cast<uint32_t>(slot.client->getPublishAck());
int result = slot.client->publish(topic, qos, retained, payload, (int)payload_len, async);
if (result < 0) {
if (!async) {
slot.transport_health.requestRecovery(MQTTTransportHealth::RecoveryReason::PublishFailure);
updateSlotTransportDiag(index, static_cast<uint32_t>(millis()));
}
// QoS0 packet/raw publishes are best-effort and may be retried from the
// bridge queue; avoid logging transient first-attempt failures here.
if (qos > 0) {
@@ -3042,6 +3249,12 @@ bool MQTTBridge::publishToSlot(int index, const char* topic, const char* payload
}
return false;
}
if (qos > 0) {
slot.transport_health.acceptedQos1(static_cast<uint32_t>(millis()), ack_before);
// Also reflect an ACK delivered during enqueue, not just on the next tick.
slot.transport_health.observeAck(static_cast<uint32_t>(slot.client->getPublishAck()));
updateSlotTransportDiag(index, static_cast<uint32_t>(millis()));
}
return true;
}
@@ -3189,8 +3402,7 @@ void MQTTBridge::publishStatusToSlot(int index) {
// on-connect status must not force retain=true. Custom slots default to
// non-retained here too, keeping both status paths consistent.
bool use_retain = slot.preset ? slot.preset->allow_retain : false;
int result = slot.client->publish(status_topic, 1, use_retain, json_buffer, len);
if (result <= 0) {
if (!publishToSlot(index, status_topic, json_buffer, static_cast<size_t>(len), use_retain, 1)) {
MQTT_DEBUG_PRINTLN("MQTT%d status publish failed", index + 1);
}
}
@@ -3453,11 +3665,28 @@ bool MQTTBridge::handleWiFiConnection(unsigned long now) {
s_wifi_connected_at = 0;
// Disconnect all slot clients when WiFi drops
for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) {
if (_slots[i].client && _slots[i].connected) {
// Include pending handshakes, not just sessions that reached CONNACK.
if (_slots[i].client && _slots[i].client->isStarted()) {
_slot_transport_fault_pending[i].store(true, std::memory_order_release);
_slot_attempt_pending[i] = false;
_slots[i].client->disconnect();
_slots[i].connected = false;
_slots[i].connected_at_ms = 0;
if (_slots[i].client->isStarted()) {
_slots[i].transport_health.requestRecovery(MQTTTransportHealth::RecoveryReason::AdapterDisconnected);
if (_slots[i].current_outage_started_ms == 0) {
_slots[i].current_outage_started_ms = now ? now : 1;
}
} else {
_slot_transport_fault_pending[i].store(false, std::memory_order_release);
_slot_health_reset_pending[i].store(false, std::memory_order_release);
_slots[i].transport_health.resetSession();
_slots[i].transport_health.recovery_reason = MQTTTransportHealth::RecoveryReason::None;
}
updateSlotTransportDiag(i, static_cast<uint32_t>(millis()));
}
}
updateCachedConnectionStatus();
} else if (snapshot.down) {
// Backoff ladder + wrap-safe timing live in MQTTConnectionPolicy (Phase 6),
// exercised by host tests. Behavior is unchanged: both the link-down
+15
View File
@@ -12,6 +12,7 @@
#include "helpers/JWTHelper.h"
#include "helpers/MQTTConnectionPolicy.h"
#include "helpers/MQTTConnectionHealth.h"
#include "helpers/MQTTTransportHealth.h"
#include "helpers/MQTTPayloadBuilder.h"
#include "helpers/MQTTPacketFilter.h"
#include "helpers/MQTTPresets.h"
@@ -158,6 +159,10 @@ private:
// DISCONNECTED (including a failed initial attempt). first_disconnect_time is intentionally separate
// so the existing 'mqttN.diag' "first_disc" semantics don't change.
unsigned long current_outage_started_ms;
// Only the bridge worker mutates this state; SDK callbacks raise atomic
// latches instead. Retained across reconnects to expose recovery history.
MQTTTransportHealth::State transport_health;
};
MQTTSlot _slots[RUNTIME_MQTT_SLOTS];
@@ -260,6 +265,13 @@ private:
// Set before starting/reconnecting, cleared by connect/failed-attempt/stop.
// Must be set before the SDK call because its event task can callback at once.
std::atomic<bool> _slot_attempt_pending[RUNTIME_MQTT_SLOTS];
std::atomic<bool> _slot_transport_fault_pending[RUNTIME_MQTT_SLOTS];
std::atomic<bool> _slot_health_reset_pending[RUNTIME_MQTT_SLOTS];
// Read-only CLI snapshots: never read the owner-only State across tasks.
std::atomic<uint32_t> _slot_health_recoveries[RUNTIME_MQTT_SLOTS];
std::atomic<uint8_t> _slot_health_reason[RUNTIME_MQTT_SLOTS];
std::atomic<bool> _slot_health_recovering[RUNTIME_MQTT_SLOTS];
std::atomic<uint32_t> _slot_health_ack_wait_secs[RUNTIME_MQTT_SLOTS];
// Pending on-connect status publish: set from the onConnect callback (which
// runs on the esp-mqtt event task, NOT this bridge task), consumed by the MQTT
@@ -509,6 +521,9 @@ private:
bool canStartSlotConnection(int index) const;
void teardownSlot(int index, bool force = false);
esp_err_t reconnectSlotClient(int index);
bool recoverSlotTransport(int index, uint32_t now);
void serviceSlotTransportHealth(uint32_t now);
void updateSlotTransportDiag(int index, uint32_t now);
void maintainSlotConnections(); // Maintain all slot connections (token renewal, reconnect)
void maintainSlotConnection(int index, unsigned long now_millis, unsigned long current_time, bool time_synced, bool& reconnect_attempted, bool& teardown_attempted);
bool createSlotAuthToken(int index); // Create/renew JWT token for a slot