From 3c5b25c2cfe3d8cbc0885420a34af1244a61fa73 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 26 Jul 2026 19:10:51 -0700 Subject: [PATCH] feat(mqtt): enhance neighbor discovery JSON handling Introduce new methods to manage neighbor discovery JSON budget and entries in MyMesh. This includes tracking the number of queried and published neighbors, measuring JSON sizes, and handling truncation when the buffer limit is reached. These improvements optimize the neighbor discovery process and ensure efficient JSON message construction for MQTT communications. --- examples/simple_repeater/MyMesh.cpp | 97 +++++++++++--- examples/simple_repeater/MyMesh.h | 7 + examples/simple_room_server/MyMesh.cpp | 97 +++++++++++--- examples/simple_room_server/MyMesh.h | 7 + src/helpers/MQTTMessageBuilder.cpp | 24 +++- src/helpers/MQTTMessageBuilder.h | 13 +- src/helpers/MQTTPayloadBuilder.cpp | 95 +++++++++++--- src/helpers/MQTTPayloadBuilder.h | 18 ++- src/helpers/bridges/MQTTBridge.cpp | 4 +- .../test_mqtt_payload_builder.cpp | 123 +++++++++++++++++- 10 files changed, 423 insertions(+), 62 deletions(-) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index a8f3cb927..6fabf045e 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -564,6 +564,7 @@ void MyMesh::logTx(mesh::Packet *pkt, int len) { NeighborDiscoverEntry& entry = neighbor_discover[neighbor_discover_next]; if (entry.status == ND_QUEUED) { entry.status = ND_PENDING; + neighbor_discover_queried_count++; neighbor_discover_request = NULL; neighbor_discover_until = futureMillis(neighborDiscoverQueryTimeoutMs()); } @@ -1066,6 +1067,10 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc #if defined(WITH_MQTT_NEIGHBORS) neighbor_discover_count = 0; neighbor_discover_next = 0; + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_json_size = 0; + neighbor_discover_truncated = false; neighbor_discover_active = false; neighbor_table_refresh_active = false; neighbor_table_refresh_periodic = false; @@ -1073,6 +1078,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc neighbor_discover_request = NULL; next_neighbors_publish = 0; self_scopes_buf[0] = 0; + neighbor_discover_origin[0] = 0; #endif memset(default_scope.key, 0, sizeof(default_scope.key)); @@ -1876,6 +1882,57 @@ uint32_t MyMesh::neighborDiscoverQueryTimeoutMs() const { + response_airtime * 2UL; } +void MyMesh::resetNeighborDiscoverJsonBudget() { + getLocalScopes(self_scopes_buf, sizeof(self_scopes_buf)); + MQTTBridge::getEffectiveMqttOrigin( + &_prefs, _cli.getObserverPrefs(), + neighbor_discover_origin, sizeof(neighbor_discover_origin)); + + char self_pubkey_hex[65]; + mesh::Utils::toHex(self_pubkey_hex, self_id.pub_key, PUB_KEY_SIZE); + char timestamp[40]; + MQTTMessageBuilder::formatIsoTimestampForMqtt( + getRTCClock()->getCurrentTime(), 0, nullptr, timestamp, sizeof(timestamp)); + + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_truncated = false; + neighbor_discover_json_size = MQTTMessageBuilder::measureNeighborsMessageBase( + neighbor_discover_origin, self_pubkey_hex, timestamp, self_scopes_buf, + neighbor_discover_count); +} + +// Account for one terminal result. The base measurement reserves maximum-width +// progress metadata; UINT32_MAX likewise reserves the widest heard-age value. +// If this result cannot fit, stop before transmitting another scope request. +bool MyMesh::completeNeighborDiscoverEntry() { + NeighborDiscoverEntry& entry = neighbor_discover[neighbor_discover_next]; + char pubkey_hex[65]; + mesh::Utils::toHex(pubkey_hex, entry.id.pub_key, PUB_KEY_SIZE); + + MQTTMessageBuilder::NeighborsMessageEntry measured = { + pubkey_hex, + entry.snr / 4.0f, + UINT32_MAX, + entry.scopes, + entry.status == ND_RESPONDED ? "responded" + : (entry.status == ND_SEND_FAILED ? "send_failed" : "timeout") + }; + size_t added = MQTTMessageBuilder::measureNeighborsMessageEntry(measured); + if (neighbor_discover_publish_count > 0) added++; // array comma + + if (neighbor_discover_json_size + added >= MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE) { + neighbor_discover_truncated = true; + finishNeighborDiscover(); + return false; + } + + neighbor_discover_json_size += added; + neighbor_discover_publish_count++; + neighbor_discover_next++; + return true; +} + // Match a RESPONSE against the pending overlay entry by tag; copy its scope // string (payload after the 8-byte {tag}{clock} header) into the entry. bool MyMesh::handleNeighborDiscoverResponse(int overlay_idx, const uint8_t* data, size_t len) { @@ -1945,14 +2002,9 @@ struct NeighborsDocAllocator : ArduinoJson::Allocator { // Build the neighbors-table JSON and hand it to the bridge, then reschedule. void MyMesh::finishNeighborDiscover() { - getLocalScopes(self_scopes_buf, sizeof(self_scopes_buf)); - char self_pubkey_hex[65]; mesh::Utils::toHex(self_pubkey_hex, self_id.pub_key, PUB_KEY_SIZE); - char origin[32]; - MQTTBridge::getEffectiveMqttOrigin(&_prefs, _cli.getObserverPrefs(), origin, sizeof(origin)); - char timestamp[40]; MQTTMessageBuilder::formatIsoTimestampForMqtt(getRTCClock()->getCurrentTime(), 0, nullptr, timestamp, sizeof(timestamp)); @@ -1960,7 +2012,7 @@ void MyMesh::finishNeighborDiscover() { MQTTMessageBuilder::NeighborsMessageEntry entries[MAX_NEIGHBOURS]; uint32_t now_secs = getRTCClock()->getCurrentTime(); - for (int i = 0; i < neighbor_discover_count; i++) { + for (int i = 0; i < neighbor_discover_publish_count; i++) { auto& entry = neighbor_discover[i]; mesh::Utils::toHex(pubkey_hex[i], entry.id.pub_key, PUB_KEY_SIZE); entries[i].pubkey_hex = pubkey_hex[i]; @@ -1976,7 +2028,7 @@ void MyMesh::finishNeighborDiscover() { } // insertion sort: most useful first (JSON builder drops the tail on overflow) - for (int i = 1; i < neighbor_discover_count; i++) { + for (int i = 1; i < neighbor_discover_publish_count; i++) { MQTTMessageBuilder::NeighborsMessageEntry entry = entries[i]; int j = i; while (j > 0 && neighborPublishEntryComesBefore(entry, entries[j - 1])) { @@ -1995,6 +2047,10 @@ void MyMesh::finishNeighborDiscover() { neighbor_discover_active = false; neighbor_discover_count = 0; neighbor_discover_next = 0; + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_json_size = 0; + neighbor_discover_truncated = false; neighbor_discover_until = 0; neighbor_discover_request = NULL; if (_cli.getObserverPrefs()->mqtt_neighbors_enabled) { @@ -2010,9 +2066,11 @@ void MyMesh::finishNeighborDiscover() { JsonDocument doc; #endif int json_len = MQTTMessageBuilder::buildNeighborsMessage( - doc, origin, self_pubkey_hex, timestamp, self_scopes_buf, - entries, neighbor_discover_count, - json_buf, MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE); + doc, neighbor_discover_origin, self_pubkey_hex, timestamp, self_scopes_buf, + entries, neighbor_discover_publish_count, + json_buf, MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE, + neighbor_discover_count, neighbor_discover_queried_count, + neighbor_discover_truncated); if (json_len > 0 && bridge) { bridge->requestPublishNeighbors(json_buf, (size_t)json_len); @@ -2027,6 +2085,10 @@ void MyMesh::finishNeighborDiscover() { neighbor_discover_active = false; neighbor_discover_count = 0; neighbor_discover_next = 0; + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_json_size = 0; + neighbor_discover_truncated = false; neighbor_discover_until = 0; neighbor_discover_request = NULL; if (_cli.getObserverPrefs()->mqtt_neighbors_enabled) { @@ -2049,7 +2111,7 @@ void MyMesh::loopNeighborDiscover() { if (!millisHasNowPassed(neighbor_discover_until)) return; if (cancelNeighborDiscoverRequest()) { entry.status = ND_SEND_FAILED; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); return; } if (isCurrentOutbound(neighbor_discover_request)) { @@ -2058,17 +2120,18 @@ void MyMesh::loopNeighborDiscover() { } neighbor_discover_request = NULL; // packet manager already shed it entry.status = ND_SEND_FAILED; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); return; } if (entry.status == ND_PENDING) { if (!millisHasNowPassed(neighbor_discover_until)) return; entry.status = ND_TIMEOUT; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); return; } - if (entry.status == ND_RESPONDED || entry.status == ND_SEND_FAILED) { - neighbor_discover_next++; + if (entry.status == ND_RESPONDED || entry.status == ND_SEND_FAILED + || entry.status == ND_TIMEOUT) { + completeNeighborDiscoverEntry(); return; } if (entry.status != ND_UNSENT) { @@ -2085,7 +2148,7 @@ void MyMesh::loopNeighborDiscover() { neighbor_discover_until = futureMillis(NEIGHBOR_DISCOVER_QUEUE_TIMEOUT_MS); } else { entry.status = ND_SEND_FAILED; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); } } @@ -2109,7 +2172,6 @@ bool MyMesh::startNeighborDiscover(char* reply) { return false; // reply already set } - getLocalScopes(self_scopes_buf, sizeof(self_scopes_buf)); neighbor_discover_count = 0; for (int i = 0; i < MAX_NEIGHBOURS; i++) { if (neighbours[i].heard_timestamp > 0) { @@ -2142,6 +2204,7 @@ bool MyMesh::startNeighborDiscover(char* reply) { } neighbor_discover_next = 0; + resetNeighborDiscoverJsonBudget(); neighbor_discover_active = true; neighbor_discover_until = 0; neighbor_discover_request = NULL; diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index bfca869ec..76fae199c 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -170,6 +170,10 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks NeighborDiscoverEntry neighbor_discover[MAX_NEIGHBOURS]; uint8_t neighbor_discover_count; uint8_t neighbor_discover_next; // newest-first entry currently being queried + uint8_t neighbor_discover_publish_count; // completed prefix that fits the JSON buffer + uint8_t neighbor_discover_queried_count; // requests confirmed transmitted + size_t neighbor_discover_json_size; + bool neighbor_discover_truncated; bool neighbor_discover_active; // scope-query phase in flight bool neighbor_table_refresh_active; // zero-hop table refresh (stage 1) in flight bool neighbor_table_refresh_periodic; // that refresh was kicked by the periodic timer @@ -177,10 +181,13 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks mesh::Packet* neighbor_discover_request; // request awaiting TX completion unsigned long next_neighbors_publish; // periodic publish deadline (0 = fire ASAP) char self_scopes_buf[96]; + char neighbor_discover_origin[32]; mesh::Packet* sendAnonRegionsReq(const mesh::Identity& target, uint32_t& tag); bool cancelNeighborDiscoverRequest(); uint32_t neighborDiscoverQueryTimeoutMs() const; + bool completeNeighborDiscoverEntry(); + void resetNeighborDiscoverJsonBudget(); bool neighborDiscoverReady(char* reply); bool startNeighborDiscover(char* reply); void loopNeighborDiscover(); diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index f161f68c6..d7409f1bd 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -256,6 +256,7 @@ void MyMesh::logTx(mesh::Packet *pkt, int len) { NeighborDiscoverEntry& entry = neighbor_discover[neighbor_discover_next]; if (entry.status == ND_QUEUED) { entry.status = ND_PENDING; + neighbor_discover_queried_count++; neighbor_discover_request = NULL; neighbor_discover_until = futureMillis(neighborDiscoverQueryTimeoutMs()); } @@ -885,6 +886,10 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc pending_discover_until = 0; neighbor_discover_count = 0; neighbor_discover_next = 0; + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_json_size = 0; + neighbor_discover_truncated = false; neighbor_discover_active = false; neighbor_table_refresh_active = false; neighbor_table_refresh_periodic = false; @@ -892,6 +897,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc neighbor_discover_request = NULL; next_neighbors_publish = 0; self_scopes_buf[0] = 0; + neighbor_discover_origin[0] = 0; memset(neighbours, 0, sizeof(neighbours)); #endif @@ -1723,6 +1729,57 @@ uint32_t MyMesh::neighborDiscoverQueryTimeoutMs() const { + response_airtime * 2UL; } +void MyMesh::resetNeighborDiscoverJsonBudget() { + getLocalScopes(self_scopes_buf, sizeof(self_scopes_buf)); + MQTTBridge::getEffectiveMqttOrigin( + &_prefs, _cli.getObserverPrefs(), + neighbor_discover_origin, sizeof(neighbor_discover_origin)); + + char self_pubkey_hex[65]; + mesh::Utils::toHex(self_pubkey_hex, self_id.pub_key, PUB_KEY_SIZE); + char timestamp[40]; + MQTTMessageBuilder::formatIsoTimestampForMqtt( + getRTCClock()->getCurrentTime(), 0, nullptr, timestamp, sizeof(timestamp)); + + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_truncated = false; + neighbor_discover_json_size = MQTTMessageBuilder::measureNeighborsMessageBase( + neighbor_discover_origin, self_pubkey_hex, timestamp, self_scopes_buf, + neighbor_discover_count); +} + +// Account for one terminal result. The base measurement reserves maximum-width +// progress metadata; UINT32_MAX likewise reserves the widest heard-age value. +// If this result cannot fit, stop before transmitting another scope request. +bool MyMesh::completeNeighborDiscoverEntry() { + NeighborDiscoverEntry& entry = neighbor_discover[neighbor_discover_next]; + char pubkey_hex[65]; + mesh::Utils::toHex(pubkey_hex, entry.id.pub_key, PUB_KEY_SIZE); + + MQTTMessageBuilder::NeighborsMessageEntry measured = { + pubkey_hex, + entry.snr / 4.0f, + UINT32_MAX, + entry.scopes, + entry.status == ND_RESPONDED ? "responded" + : (entry.status == ND_SEND_FAILED ? "send_failed" : "timeout") + }; + size_t added = MQTTMessageBuilder::measureNeighborsMessageEntry(measured); + if (neighbor_discover_publish_count > 0) added++; // array comma + + if (neighbor_discover_json_size + added >= MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE) { + neighbor_discover_truncated = true; + finishNeighborDiscover(); + return false; + } + + neighbor_discover_json_size += added; + neighbor_discover_publish_count++; + neighbor_discover_next++; + return true; +} + // Match a RESPONSE against the pending overlay entry by tag; copy its scope // string (payload after the 8-byte {tag}{clock} header) into the entry. bool MyMesh::handleNeighborDiscoverResponse(int overlay_idx, const uint8_t* data, size_t len) { @@ -1792,14 +1849,9 @@ struct NeighborsDocAllocator : ArduinoJson::Allocator { // Build the neighbors-table JSON and hand it to the bridge, then reschedule. void MyMesh::finishNeighborDiscover() { - getLocalScopes(self_scopes_buf, sizeof(self_scopes_buf)); - char self_pubkey_hex[65]; mesh::Utils::toHex(self_pubkey_hex, self_id.pub_key, PUB_KEY_SIZE); - char origin[32]; - MQTTBridge::getEffectiveMqttOrigin(&_prefs, _cli.getObserverPrefs(), origin, sizeof(origin)); - char timestamp[40]; MQTTMessageBuilder::formatIsoTimestampForMqtt(getRTCClock()->getCurrentTime(), 0, nullptr, timestamp, sizeof(timestamp)); @@ -1807,7 +1859,7 @@ void MyMesh::finishNeighborDiscover() { MQTTMessageBuilder::NeighborsMessageEntry entries[MAX_NEIGHBOURS]; uint32_t now_secs = getRTCClock()->getCurrentTime(); - for (int i = 0; i < neighbor_discover_count; i++) { + for (int i = 0; i < neighbor_discover_publish_count; i++) { auto& entry = neighbor_discover[i]; mesh::Utils::toHex(pubkey_hex[i], entry.id.pub_key, PUB_KEY_SIZE); entries[i].pubkey_hex = pubkey_hex[i]; @@ -1823,7 +1875,7 @@ void MyMesh::finishNeighborDiscover() { } // insertion sort: most useful first (JSON builder drops the tail on overflow) - for (int i = 1; i < neighbor_discover_count; i++) { + for (int i = 1; i < neighbor_discover_publish_count; i++) { MQTTMessageBuilder::NeighborsMessageEntry entry = entries[i]; int j = i; while (j > 0 && neighborPublishEntryComesBefore(entry, entries[j - 1])) { @@ -1842,6 +1894,10 @@ void MyMesh::finishNeighborDiscover() { neighbor_discover_active = false; neighbor_discover_count = 0; neighbor_discover_next = 0; + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_json_size = 0; + neighbor_discover_truncated = false; neighbor_discover_until = 0; neighbor_discover_request = NULL; if (_cli.getObserverPrefs()->mqtt_neighbors_enabled) { @@ -1857,9 +1913,11 @@ void MyMesh::finishNeighborDiscover() { JsonDocument doc; #endif int json_len = MQTTMessageBuilder::buildNeighborsMessage( - doc, origin, self_pubkey_hex, timestamp, self_scopes_buf, - entries, neighbor_discover_count, - json_buf, MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE); + doc, neighbor_discover_origin, self_pubkey_hex, timestamp, self_scopes_buf, + entries, neighbor_discover_publish_count, + json_buf, MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE, + neighbor_discover_count, neighbor_discover_queried_count, + neighbor_discover_truncated); if (json_len > 0 && bridge) { bridge->requestPublishNeighbors(json_buf, (size_t)json_len); @@ -1874,6 +1932,10 @@ void MyMesh::finishNeighborDiscover() { neighbor_discover_active = false; neighbor_discover_count = 0; neighbor_discover_next = 0; + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_json_size = 0; + neighbor_discover_truncated = false; neighbor_discover_until = 0; neighbor_discover_request = NULL; if (_cli.getObserverPrefs()->mqtt_neighbors_enabled) { @@ -1896,7 +1958,7 @@ void MyMesh::loopNeighborDiscover() { if (!millisHasNowPassed(neighbor_discover_until)) return; if (cancelNeighborDiscoverRequest()) { entry.status = ND_SEND_FAILED; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); return; } if (isCurrentOutbound(neighbor_discover_request)) { @@ -1905,17 +1967,18 @@ void MyMesh::loopNeighborDiscover() { } neighbor_discover_request = NULL; // packet manager already shed it entry.status = ND_SEND_FAILED; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); return; } if (entry.status == ND_PENDING) { if (!millisHasNowPassed(neighbor_discover_until)) return; entry.status = ND_TIMEOUT; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); return; } - if (entry.status == ND_RESPONDED || entry.status == ND_SEND_FAILED) { - neighbor_discover_next++; + if (entry.status == ND_RESPONDED || entry.status == ND_SEND_FAILED + || entry.status == ND_TIMEOUT) { + completeNeighborDiscoverEntry(); return; } if (entry.status != ND_UNSENT) { @@ -1932,7 +1995,7 @@ void MyMesh::loopNeighborDiscover() { neighbor_discover_until = futureMillis(NEIGHBOR_DISCOVER_QUEUE_TIMEOUT_MS); } else { entry.status = ND_SEND_FAILED; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); } } @@ -1956,7 +2019,6 @@ bool MyMesh::startNeighborDiscover(char* reply) { return false; // reply already set } - getLocalScopes(self_scopes_buf, sizeof(self_scopes_buf)); neighbor_discover_count = 0; for (int i = 0; i < MAX_NEIGHBOURS; i++) { if (neighbours[i].heard_timestamp > 0) { @@ -1989,6 +2051,7 @@ bool MyMesh::startNeighborDiscover(char* reply) { } neighbor_discover_next = 0; + resetNeighborDiscoverJsonBudget(); neighbor_discover_active = true; neighbor_discover_until = 0; neighbor_discover_request = NULL; diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index 2b064d5f3..ff60c6ce0 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -163,6 +163,10 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks NeighborDiscoverEntry neighbor_discover[MAX_NEIGHBOURS]; uint8_t neighbor_discover_count; uint8_t neighbor_discover_next; + uint8_t neighbor_discover_publish_count; + uint8_t neighbor_discover_queried_count; // requests confirmed transmitted + size_t neighbor_discover_json_size; + bool neighbor_discover_truncated; bool neighbor_discover_active; bool neighbor_table_refresh_active; bool neighbor_table_refresh_periodic; @@ -170,12 +174,15 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks mesh::Packet* neighbor_discover_request; unsigned long next_neighbors_publish; char self_scopes_buf[96]; + char neighbor_discover_origin[32]; void putNeighbour(const mesh::Identity& id, uint32_t timestamp, float snr); void sendNodeDiscoverReq(); mesh::Packet* sendAnonRegionsReq(const mesh::Identity& target, uint32_t& tag); bool cancelNeighborDiscoverRequest(); uint32_t neighborDiscoverQueryTimeoutMs() const; + bool completeNeighborDiscoverEntry(); + void resetNeighborDiscoverJsonBudget(); bool neighborDiscoverReady(char* reply); bool startNeighborDiscover(char* reply); void loopNeighborDiscover(); diff --git a/src/helpers/MQTTMessageBuilder.cpp b/src/helpers/MQTTMessageBuilder.cpp index b90295442..b5dd826df 100644 --- a/src/helpers/MQTTMessageBuilder.cpp +++ b/src/helpers/MQTTMessageBuilder.cpp @@ -111,11 +111,31 @@ int MQTTMessageBuilder::buildNeighborsMessage( const NeighborsMessageEntry* neighbors, int neighbor_count, char* buffer, - size_t buffer_size + size_t buffer_size, + int total_neighbors, + int queried_neighbors, + bool truncated ) { return MQTTPayloadBuilder::buildNeighborsMessage( doc, origin, origin_id, timestamp, self_scopes, neighbors, neighbor_count, - buffer, buffer_size); + buffer, buffer_size, total_neighbors, queried_neighbors, truncated); +} + +size_t MQTTMessageBuilder::measureNeighborsMessageBase( + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + int total_neighbors +) { + return MQTTPayloadBuilder::measureNeighborsMessageBase( + origin, origin_id, timestamp, self_scopes, total_neighbors); +} + +size_t MQTTMessageBuilder::measureNeighborsMessageEntry( + const NeighborsMessageEntry& neighbor +) { + return MQTTPayloadBuilder::measureNeighborsMessageEntry(neighbor); } int MQTTMessageBuilder::buildPacketJSON( diff --git a/src/helpers/MQTTMessageBuilder.h b/src/helpers/MQTTMessageBuilder.h index fc0c47055..2d45af86e 100644 --- a/src/helpers/MQTTMessageBuilder.h +++ b/src/helpers/MQTTMessageBuilder.h @@ -167,8 +167,19 @@ public: const NeighborsMessageEntry* neighbors, int neighbor_count, char* buffer, - size_t buffer_size + size_t buffer_size, + int total_neighbors = -1, + int queried_neighbors = -1, + bool truncated = false ); + static size_t measureNeighborsMessageBase( + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + int total_neighbors + ); + static size_t measureNeighborsMessageEntry(const NeighborsMessageEntry& neighbor); /** * Convert packet to JSON message diff --git a/src/helpers/MQTTPayloadBuilder.cpp b/src/helpers/MQTTPayloadBuilder.cpp index 5a40e502b..77d9054c4 100644 --- a/src/helpers/MQTTPayloadBuilder.cpp +++ b/src/helpers/MQTTPayloadBuilder.cpp @@ -185,6 +185,68 @@ int MQTTPayloadBuilder::buildRawMessage( return serializeComplete(root, buffer, buffer_size); } +static JsonArray buildNeighborsMessageBase( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + int total_neighbors, + int queried_neighbors, + bool truncated +) { + doc.clear(); + JsonObject root = doc.to(); + root["timestamp"] = timestamp; + root["origin"] = origin; + root["origin_id"] = origin_id; + if (total_neighbors >= 0) { + root["total_neighbors"] = total_neighbors; + root["queried_neighbors"] = queried_neighbors >= 0 ? queried_neighbors : total_neighbors; + root["truncated"] = truncated; + } + + JsonObject self = root["self"].to(); + self["scopes"] = self_scopes ? self_scopes : ""; + return root["neighbors"].to(); +} + +static void addNeighborsMessageEntry( + JsonArray& arr, + const MQTTPayloadBuilder::NeighborsMessageEntry& neighbor +) { + JsonObject nb = arr.add(); + nb["pubkey"] = neighbor.pubkey_hex; + nb["snr"] = neighbor.snr; + nb["heard_secs_ago"] = neighbor.heard_secs_ago; + nb["scopes"] = neighbor.scopes ? neighbor.scopes : ""; + nb["status"] = neighbor.status; +} + +size_t MQTTPayloadBuilder::measureNeighborsMessageBase( + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + int total_neighbors +) { + JsonDocument doc; + buildNeighborsMessageBase( + doc, origin, origin_id, timestamp, self_scopes, + total_neighbors, total_neighbors, false); + return measureJson(doc); +} + +size_t MQTTPayloadBuilder::measureNeighborsMessageEntry( + const NeighborsMessageEntry& neighbor +) { + JsonDocument doc; + JsonArray arr = doc.to(); + addNeighborsMessageEntry(arr, neighbor); + JsonObject entry = arr[0]; + return measureJson(entry); +} + int MQTTPayloadBuilder::buildNeighborsMessage( JsonDocument& doc, const char* origin, @@ -194,38 +256,31 @@ int MQTTPayloadBuilder::buildNeighborsMessage( const NeighborsMessageEntry* neighbors, int neighbor_count, char* buffer, - size_t buffer_size + size_t buffer_size, + int total_neighbors, + int queried_neighbors, + bool truncated ) { if (!buffer || buffer_size == 0) return 0; - doc.clear(); - JsonObject root = doc.to(); - root["timestamp"] = timestamp; - root["origin"] = origin; - root["origin_id"] = origin_id; - - JsonObject self = root["self"].to(); - self["scopes"] = self_scopes ? self_scopes : ""; - - JsonArray arr = root["neighbors"].to(); - if (measureJson(root) >= buffer_size) return 0; + JsonArray arr = buildNeighborsMessageBase( + doc, origin, origin_id, timestamp, self_scopes, + total_neighbors, queried_neighbors, truncated); + if (doc.overflowed() || arr.isNull() || measureJson(doc) >= buffer_size) return 0; for (int i = 0; i < neighbor_count; i++) { - JsonObject nb = arr.add(); - nb["pubkey"] = neighbors[i].pubkey_hex; - nb["snr"] = neighbors[i].snr; - nb["heard_secs_ago"] = neighbors[i].heard_secs_ago; - nb["scopes"] = neighbors[i].scopes ? neighbors[i].scopes : ""; - nb["status"] = neighbors[i].status; + addNeighborsMessageEntry(arr, neighbors[i]); + if (doc.overflowed()) return 0; // Entries arrive ordered most- to least-useful. Stop as soon as the next // one would fill the fixed publish buffer, dropping the remaining tail so // document growth stays bounded. - if (measureJson(root) >= buffer_size) { + if (measureJson(doc) >= buffer_size) { arr.remove(arr.size() - 1); + if (total_neighbors >= 0) doc["truncated"] = true; break; } } - return serializeComplete(root, buffer, buffer_size); + return serializeComplete(doc.as(), buffer, buffer_size); } diff --git a/src/helpers/MQTTPayloadBuilder.h b/src/helpers/MQTTPayloadBuilder.h index a6fa333d1..f9e5c9598 100644 --- a/src/helpers/MQTTPayloadBuilder.h +++ b/src/helpers/MQTTPayloadBuilder.h @@ -89,7 +89,21 @@ public: const NeighborsMessageEntry* neighbors, int neighbor_count, char* buffer, - size_t buffer_size + size_t buffer_size, + int total_neighbors = -1, + int queried_neighbors = -1, + bool truncated = false ); -}; + // Exact serialized-size components used by paced neighbor discovery. The + // base reserves the largest progress metadata values for this snapshot; + // callers add each entry size plus one byte for commas after the first. + static size_t measureNeighborsMessageBase( + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + int total_neighbors + ); + static size_t measureNeighborsMessageEntry(const NeighborsMessageEntry& neighbor); +}; diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 4102cbe5c..63a04306e 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -3314,8 +3314,10 @@ bool MQTTBridge::publishNeighbors() { // MeshRank slots reject non-packets by contract, so buildTopicForSlot // returns false for them here and the slot is skipped. if (buildTopicForSlot(i, MSG_NEIGHBORS, topic, sizeof(topic))) { + // Neighbor snapshots are periodically refreshed. Publish synchronously + // at QoS 0 to avoid the QoS 1 outbox, retaining where the broker allows. bool use_retain = _slots[i].preset ? _slots[i].preset->allow_retain : false; - if (publishToSlot(i, topic, _neighbors_json_buffer, use_retain, 1)) { + if (publishToSlot(i, topic, _neighbors_json_buffer, use_retain, 0)) { published = true; } } diff --git a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp index d7f811308..4a1e40deb 100644 --- a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp +++ b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp @@ -12,6 +12,12 @@ namespace { constexpr const char* kTimestamp = "2026-07-18T12:34:56.123456+00:00"; +struct RejectAllJsonAllocations : ArduinoJson::Allocator { + void* allocate(size_t) override { return nullptr; } + void deallocate(void*) override {} + void* reallocate(void*, size_t) override { return nullptr; } +}; + static int buildMinimalStatus(JsonDocument& scratch, char* buffer, size_t buffer_size) { return MQTTPayloadBuilder::buildStatusMessage( scratch, "DEN Repeater", "0123456789ABCDEF", "Heltec V3", "v1.16.0", @@ -237,7 +243,7 @@ TEST(MQTTPayloadBuilder, NeighborsMessageRoundTripsSelfAndEntries) { char buffer[1024]; int len = MQTTPayloadBuilder::buildNeighborsMessage( scratch, "DEN Repeater", "0123456789ABCDEF", kTimestamp, "DEN,APRS", - neighbors, 2, buffer, sizeof(buffer)); + neighbors, 2, buffer, sizeof(buffer), 5, 2, true); ASSERT_GT(len, 0); EXPECT_EQ(static_cast(len), strlen(buffer)); @@ -247,6 +253,9 @@ TEST(MQTTPayloadBuilder, NeighborsMessageRoundTripsSelfAndEntries) { EXPECT_STREQ("DEN Repeater", parsed["origin"].as()); EXPECT_STREQ("0123456789ABCDEF", parsed["origin_id"].as()); EXPECT_STREQ("DEN,APRS", parsed["self"]["scopes"].as()); + EXPECT_EQ(5, parsed["total_neighbors"].as()); + EXPECT_EQ(2, parsed["queried_neighbors"].as()); + EXPECT_TRUE(parsed["truncated"].as()); JsonArray arr = parsed["neighbors"].as(); ASSERT_EQ(2U, arr.size()); @@ -260,6 +269,116 @@ TEST(MQTTPayloadBuilder, NeighborsMessageRoundTripsSelfAndEntries) { EXPECT_STREQ("stale", arr[1]["status"].as()); } +TEST(MQTTPayloadBuilder, NeighborsMessageMeasurementsMatchCompletePayload) { + MQTTPayloadBuilder::NeighborsMessageEntry neighbors[] = { + {"0011223344556677", 9.75f, UINT32_MAX, "DEN,APRS", "responded"}, + {"8899AABBCCDDEEFF", -3.5f, UINT32_MAX, "", "timeout"}, + }; + + size_t measured = + MQTTPayloadBuilder::measureNeighborsMessageBase( + "DEN Repeater", "0123456789ABCDEF", kTimestamp, "DEN,APRS", 2) + + MQTTPayloadBuilder::measureNeighborsMessageEntry(neighbors[0]) + + 1 // array comma + + MQTTPayloadBuilder::measureNeighborsMessageEntry(neighbors[1]); + + JsonDocument scratch; + char buffer[1024]; + int len = MQTTPayloadBuilder::buildNeighborsMessage( + scratch, "DEN Repeater", "0123456789ABCDEF", kTimestamp, "DEN,APRS", + neighbors, 2, buffer, sizeof(buffer), 2, 2, false); + + ASSERT_GT(len, 0); + EXPECT_EQ(measured, static_cast(len)); +} + +TEST(MQTTPayloadBuilder, NeighborsMessageMeasuredPrefixStopsBeforeOverflow) { + MQTTPayloadBuilder::NeighborsMessageEntry neighbors[20]; + static char keys[20][17]; + static char long_scopes[96]; + memset(long_scopes, 'S', sizeof(long_scopes) - 1); + long_scopes[sizeof(long_scopes) - 1] = '\0'; + for (int i = 0; i < 20; i++) { + snprintf(keys[i], sizeof(keys[i]), "%016X", i); + neighbors[i] = { + keys[i], static_cast(i), UINT32_MAX, + long_scopes, "responded"}; + } + + constexpr size_t kBufferSize = 512; + size_t used = MQTTPayloadBuilder::measureNeighborsMessageBase( + "node", "id", kTimestamp, "DEN", 20); + int published = 0; + int queried = 0; + while (queried < 20) { + size_t added = + MQTTPayloadBuilder::measureNeighborsMessageEntry(neighbors[queried]) + + (published > 0 ? 1U : 0U); + queried++; + if (used + added >= kBufferSize) break; + used += added; + published++; + } + + ASSERT_GT(published, 0); + ASSERT_LT(published, 20); + ASSERT_EQ(published + 1, queried); // only the non-fitting candidate was extra + + JsonDocument scratch; + char buffer[kBufferSize]; + int len = MQTTPayloadBuilder::buildNeighborsMessage( + scratch, "node", "id", kTimestamp, "DEN", + neighbors, published, buffer, sizeof(buffer), + 20, queried, true); + + ASSERT_GT(len, 0); + EXPECT_LT(static_cast(len), sizeof(buffer)); + JsonDocument parsed; + ASSERT_FALSE(deserializeJson(parsed, buffer)); + EXPECT_EQ(static_cast(published), + parsed["neighbors"].as().size()); + EXPECT_EQ(20, parsed["total_neighbors"].as()); + EXPECT_EQ(queried, parsed["queried_neighbors"].as()); + EXPECT_TRUE(parsed["truncated"].as()); +} + +TEST(MQTTPayloadBuilder, NeighborsMessageFallbackMarksTruncated) { + MQTTPayloadBuilder::NeighborsMessageEntry neighbors[] = { + {"0011223344556677", 9.75f, 1, "DEN", "responded"}, + {"8899AABBCCDDEEFF", -3.5f, 2, "APRS", "responded"}, + }; + size_t first_only_size = + MQTTPayloadBuilder::measureNeighborsMessageBase( + "node", "id", kTimestamp, "DEN", 2) + + MQTTPayloadBuilder::measureNeighborsMessageEntry(neighbors[0]); + std::vector buffer(first_only_size + 1); + + JsonDocument scratch; + int len = MQTTPayloadBuilder::buildNeighborsMessage( + scratch, "node", "id", kTimestamp, "DEN", + neighbors, 2, buffer.data(), buffer.size(), + 2, 2, false); + + ASSERT_GT(len, 0); + JsonDocument parsed; + ASSERT_FALSE(deserializeJson(parsed, buffer.data())); + EXPECT_EQ(1U, parsed["neighbors"].as().size()); + EXPECT_TRUE(parsed["truncated"].as()); +} + +TEST(MQTTPayloadBuilder, NeighborsMessageFailsCleanlyOnAllocationFailure) { + MQTTPayloadBuilder::NeighborsMessageEntry neighbor = { + "0011223344556677", 9.75f, 1, "DEN", "responded"}; + RejectAllJsonAllocations allocator; + JsonDocument scratch(&allocator); + char buffer[512]; + + EXPECT_EQ(0, MQTTPayloadBuilder::buildNeighborsMessage( + scratch, "node", "id", kTimestamp, "DEN", + &neighbor, 1, buffer, sizeof(buffer), + 1, 1, false)); +} + TEST(MQTTPayloadBuilder, NeighborsMessageHandlesEmptyTableAndNullScopes) { JsonDocument scratch; char buffer[256]; @@ -304,6 +423,7 @@ TEST(MQTTPayloadBuilder, NeighborsMessageDropsTailWhenBufferFills) { ASSERT_FALSE(arr.isNull()); EXPECT_GT(arr.size(), 0U); EXPECT_LT(arr.size(), 20U); + EXPECT_FALSE(parsed["truncated"].is()); // Kept entries are the head of the input, in order. EXPECT_STREQ(keys[0], arr[0]["pubkey"].as()); } @@ -314,4 +434,3 @@ int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } -