From 74e45a795c13e28ae122522e2eb9cbb9421343f9 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 6 Apr 2026 18:00:50 -0700 Subject: [PATCH] Enhance MQTTMessageBuilder to include internal heap memory statistics in status messages. Update function signature and JSON serialization to accommodate the new internal_heap parameter, improving diagnostics for memory management in MQTTBridge. --- src/helpers/MQTTMessageBuilder.cpp | 11 +++++--- src/helpers/MQTTMessageBuilder.h | 4 ++- src/helpers/bridges/MQTTBridge.cpp | 42 +++++++++++++++++++++++++----- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/helpers/MQTTMessageBuilder.cpp b/src/helpers/MQTTMessageBuilder.cpp index e2a8d07db..8597c230a 100644 --- a/src/helpers/MQTTMessageBuilder.cpp +++ b/src/helpers/MQTTMessageBuilder.cpp @@ -22,7 +22,8 @@ int MQTTMessageBuilder::buildStatusMessage( int noise_floor, int tx_air_secs, int rx_air_secs, - int recv_errors + int recv_errors, + int internal_heap ) { // Use StaticJsonDocument to avoid heap fragmentation (fixed-size stack allocation) StaticJsonDocument<768> doc; // Increased size to accommodate stats @@ -38,8 +39,9 @@ int MQTTMessageBuilder::buildStatusMessage( root["client_version"] = client_version; // Add stats object if any stats are provided - if (battery_mv >= 0 || uptime_secs >= 0 || errors >= 0 || queue_len >= 0 || - noise_floor > -999 || tx_air_secs >= 0 || rx_air_secs >= 0 || recv_errors >= 0) { + if (battery_mv >= 0 || uptime_secs >= 0 || errors >= 0 || queue_len >= 0 || + noise_floor > -999 || tx_air_secs >= 0 || rx_air_secs >= 0 || recv_errors >= 0 || + internal_heap >= 0) { JsonObject stats = root.createNestedObject("stats"); if (battery_mv >= 0) { @@ -66,6 +68,9 @@ int MQTTMessageBuilder::buildStatusMessage( if (recv_errors >= 0) { stats["recv_errors"] = recv_errors; } + if (internal_heap >= 0) { + stats["internal_heap"] = internal_heap; + } } size_t len = serializeJson(root, buffer, buffer_size); diff --git a/src/helpers/MQTTMessageBuilder.h b/src/helpers/MQTTMessageBuilder.h index 4f525daec..96efabcd0 100644 --- a/src/helpers/MQTTMessageBuilder.h +++ b/src/helpers/MQTTMessageBuilder.h @@ -38,6 +38,7 @@ public: * @param tx_air_secs TX air time in seconds (optional, -1 to omit) * @param rx_air_secs RX air time in seconds (optional, -1 to omit) * @param recv_errors Radio receive/CRC errors (optional, -1 to omit) + * @param internal_heap Internal heap free bytes (optional, -1 to omit) * @return Length of JSON string, or 0 on error */ static int buildStatusMessage( @@ -58,7 +59,8 @@ public: int noise_floor = -999, int tx_air_secs = -1, int rx_air_secs = -1, - int recv_errors = -1 + int recv_errors = -1, + int internal_heap = -1 ); /** diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 612f6f490..5418bd09d 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -198,6 +198,7 @@ const char* MQTTBridge::tlsErrorStr(int32_t err) { case 0x8002: return "socket error"; case 0x8004: return "connect refused"; case 0x8006: return "TLS timeout"; + case 0x8008: return "connection timeout"; case 0x800B: return "cert verify failed"; case 0x8010: return "mbedTLS error"; default: return nullptr; @@ -1286,6 +1287,8 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns slot.client->connect(); // restart stopped client; reconnect() fails silently on a stopped client reconnect_attempted = true; _last_slot_reconnect_ms = now_millis; + MQTT_DEBUG_PRINTLN("MQTT%d int_heap=%d at token renewal reconnect", index + 1, + (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); } else { // Token renewed but old one still valid — just update credentials for next reconnect slot.client->setCredentials(_jwt_username, slot.auth_token); @@ -1309,7 +1312,8 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns slot.last_reconnect_attempt = now_millis; reconnect_attempted = true; _last_slot_reconnect_ms = now_millis; - MQTT_DEBUG_PRINTLN("MQTT%d circuit breaker probe (attempting single reconnect after %lu ms)", index + 1, probe_elapsed); + MQTT_DEBUG_PRINTLN("MQTT%d circuit breaker probe (attempting single reconnect after %lu ms, int_heap=%d)", index + 1, probe_elapsed, + (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); if (slot_uses_jwt) { unsigned long current_time = time(nullptr); bool token_still_valid = slot.token_expires_at > 0 && @@ -1372,7 +1376,8 @@ void MQTTBridge::maintainSlotConnection(int index, unsigned long now_millis, uns return; } } - MQTT_DEBUG_PRINTLN("MQTT%d reconnecting (backoff level %d, failures at max: %d)", index + 1, slot.reconnect_backoff, slot.max_backoff_failures); + MQTT_DEBUG_PRINTLN("MQTT%d reconnecting (backoff level %d, failures at max: %d, int_heap=%d)", index + 1, slot.reconnect_backoff, slot.max_backoff_failures, + (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL)); reconnect_attempted = true; _last_slot_reconnect_ms = now_millis; if (slot_uses_jwt) { @@ -1662,11 +1667,14 @@ void MQTTBridge::publishStatusToSlot(int index) { recv_errors = (int)_radio->getPacketsRecvErrors(); } + // Internal heap free (for diagnosing repeater hangs from internal heap exhaustion) + int internal_heap_free = (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL); + int len = MQTTMessageBuilder::buildStatusMessage( _origin, origin_id, _board_model, _firmware_version, radio_info, client_version, "online", timestamp, json_buffer, STATUS_JSON_BUFFER_SIZE, battery_mv, uptime_secs, errors, _queue_count, noise_floor, - tx_air_secs, rx_air_secs, recv_errors + tx_air_secs, rx_air_secs, recv_errors, internal_heap_free ); if (len > 0) { @@ -2277,11 +2285,14 @@ bool MQTTBridge::publishStatus() { recv_errors = (int)_radio->getPacketsRecvErrors(); } + // Internal heap free (for diagnosing repeater hangs from internal heap exhaustion) + int internal_heap_free = (int)heap_caps_get_free_size(MALLOC_CAP_INTERNAL); + int len = MQTTMessageBuilder::buildStatusMessage( _origin, origin_id, _board_model, _firmware_version, radio_info, client_version, "online", timestamp, json_buffer, STATUS_JSON_BUFFER_SIZE, battery_mv, uptime_secs, errors, _queue_count, noise_floor, - tx_air_secs, rx_air_secs, recv_errors + tx_air_secs, rx_air_secs, recv_errors, internal_heap_free ); if (len > 0) { @@ -2615,13 +2626,14 @@ void MQTTBridge::runCriticalMemoryCheckAndRecovery() { size_t free_h = ESP.getFreeHeap(); size_t max_alloc = ESP.getMaxAllocHeap(); + size_t internal_free = heap_caps_get_free_size(MALLOC_CAP_INTERNAL); + size_t internal_max_block = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL); #ifdef MQTT_MEMORY_DEBUG - unsigned long internal_f = heap_caps_get_free_size(MALLOC_CAP_INTERNAL); unsigned long spiram_f = 0; #ifdef BOARD_HAS_PSRAM spiram_f = heap_caps_get_free_size(MALLOC_CAP_SPIRAM); #endif - agentLogHeap("MQTTBridge.cpp:runCriticalMemoryCheckAndRecovery", "critical_memory_check", "H1_H4", free_h, max_alloc, internal_f, spiram_f); + agentLogHeap("MQTTBridge.cpp:runCriticalMemoryCheckAndRecovery", "critical_memory_check", "H1_H4", free_h, max_alloc, internal_free, spiram_f); #endif // Pressure timer: track how long max_alloc has been below moderate threshold @@ -2637,11 +2649,29 @@ void MQTTBridge::runCriticalMemoryCheckAndRecovery() { static unsigned long last_critical_log = 0; if (now - last_critical_log >= CRITICAL_LOG_INTERVAL_MS) { last_critical_log = now; + + // Always log heap state including internal heap (critical for diagnosing + // repeater hangs caused by internal heap exhaustion masked by PSRAM) + MQTT_DEBUG_PRINTLN("Heap: free=%d max=%d int_free=%d int_max=%d", + (int)free_h, (int)max_alloc, (int)internal_free, (int)internal_max_block); + if (max_alloc < PRESSURE_THRESHOLD_CRITICAL) { MQTT_DEBUG_PRINTLN("CRITICAL: Low memory! Free: %d, Max: %d", (int)free_h, (int)max_alloc); } else if (max_alloc < PRESSURE_THRESHOLD_MODERATE) { MQTT_DEBUG_PRINTLN("WARNING: Memory pressure. Free: %d, Max: %d", (int)free_h, (int)max_alloc); } + + // Internal heap pressure check (PSRAM boards only — total heap can look fine + // while internal heap is exhausted, starving WiFi driver and Core 1) + #if defined(BOARD_HAS_PSRAM) + const size_t INTERNAL_HEAP_CRITICAL = 40000; + const size_t INTERNAL_BLOCK_CRITICAL = 20000; + if (internal_free < INTERNAL_HEAP_CRITICAL || internal_max_block < INTERNAL_BLOCK_CRITICAL) { + MQTT_DEBUG_PRINTLN("CRITICAL: Internal heap low! int_free=%d int_max_block=%d", + (int)internal_free, (int)internal_max_block); + } + #endif + // Log slot client count int n_active = 0; for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) {