diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index a8f3cb927..6fabf045e 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -564,6 +564,7 @@ void MyMesh::logTx(mesh::Packet *pkt, int len) { NeighborDiscoverEntry& entry = neighbor_discover[neighbor_discover_next]; if (entry.status == ND_QUEUED) { entry.status = ND_PENDING; + neighbor_discover_queried_count++; neighbor_discover_request = NULL; neighbor_discover_until = futureMillis(neighborDiscoverQueryTimeoutMs()); } @@ -1066,6 +1067,10 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc #if defined(WITH_MQTT_NEIGHBORS) neighbor_discover_count = 0; neighbor_discover_next = 0; + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_json_size = 0; + neighbor_discover_truncated = false; neighbor_discover_active = false; neighbor_table_refresh_active = false; neighbor_table_refresh_periodic = false; @@ -1073,6 +1078,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc neighbor_discover_request = NULL; next_neighbors_publish = 0; self_scopes_buf[0] = 0; + neighbor_discover_origin[0] = 0; #endif memset(default_scope.key, 0, sizeof(default_scope.key)); @@ -1876,6 +1882,57 @@ uint32_t MyMesh::neighborDiscoverQueryTimeoutMs() const { + response_airtime * 2UL; } +void MyMesh::resetNeighborDiscoverJsonBudget() { + getLocalScopes(self_scopes_buf, sizeof(self_scopes_buf)); + MQTTBridge::getEffectiveMqttOrigin( + &_prefs, _cli.getObserverPrefs(), + neighbor_discover_origin, sizeof(neighbor_discover_origin)); + + char self_pubkey_hex[65]; + mesh::Utils::toHex(self_pubkey_hex, self_id.pub_key, PUB_KEY_SIZE); + char timestamp[40]; + MQTTMessageBuilder::formatIsoTimestampForMqtt( + getRTCClock()->getCurrentTime(), 0, nullptr, timestamp, sizeof(timestamp)); + + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_truncated = false; + neighbor_discover_json_size = MQTTMessageBuilder::measureNeighborsMessageBase( + neighbor_discover_origin, self_pubkey_hex, timestamp, self_scopes_buf, + neighbor_discover_count); +} + +// Account for one terminal result. The base measurement reserves maximum-width +// progress metadata; UINT32_MAX likewise reserves the widest heard-age value. +// If this result cannot fit, stop before transmitting another scope request. +bool MyMesh::completeNeighborDiscoverEntry() { + NeighborDiscoverEntry& entry = neighbor_discover[neighbor_discover_next]; + char pubkey_hex[65]; + mesh::Utils::toHex(pubkey_hex, entry.id.pub_key, PUB_KEY_SIZE); + + MQTTMessageBuilder::NeighborsMessageEntry measured = { + pubkey_hex, + entry.snr / 4.0f, + UINT32_MAX, + entry.scopes, + entry.status == ND_RESPONDED ? "responded" + : (entry.status == ND_SEND_FAILED ? "send_failed" : "timeout") + }; + size_t added = MQTTMessageBuilder::measureNeighborsMessageEntry(measured); + if (neighbor_discover_publish_count > 0) added++; // array comma + + if (neighbor_discover_json_size + added >= MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE) { + neighbor_discover_truncated = true; + finishNeighborDiscover(); + return false; + } + + neighbor_discover_json_size += added; + neighbor_discover_publish_count++; + neighbor_discover_next++; + return true; +} + // Match a RESPONSE against the pending overlay entry by tag; copy its scope // string (payload after the 8-byte {tag}{clock} header) into the entry. bool MyMesh::handleNeighborDiscoverResponse(int overlay_idx, const uint8_t* data, size_t len) { @@ -1945,14 +2002,9 @@ struct NeighborsDocAllocator : ArduinoJson::Allocator { // Build the neighbors-table JSON and hand it to the bridge, then reschedule. void MyMesh::finishNeighborDiscover() { - getLocalScopes(self_scopes_buf, sizeof(self_scopes_buf)); - char self_pubkey_hex[65]; mesh::Utils::toHex(self_pubkey_hex, self_id.pub_key, PUB_KEY_SIZE); - char origin[32]; - MQTTBridge::getEffectiveMqttOrigin(&_prefs, _cli.getObserverPrefs(), origin, sizeof(origin)); - char timestamp[40]; MQTTMessageBuilder::formatIsoTimestampForMqtt(getRTCClock()->getCurrentTime(), 0, nullptr, timestamp, sizeof(timestamp)); @@ -1960,7 +2012,7 @@ void MyMesh::finishNeighborDiscover() { MQTTMessageBuilder::NeighborsMessageEntry entries[MAX_NEIGHBOURS]; uint32_t now_secs = getRTCClock()->getCurrentTime(); - for (int i = 0; i < neighbor_discover_count; i++) { + for (int i = 0; i < neighbor_discover_publish_count; i++) { auto& entry = neighbor_discover[i]; mesh::Utils::toHex(pubkey_hex[i], entry.id.pub_key, PUB_KEY_SIZE); entries[i].pubkey_hex = pubkey_hex[i]; @@ -1976,7 +2028,7 @@ void MyMesh::finishNeighborDiscover() { } // insertion sort: most useful first (JSON builder drops the tail on overflow) - for (int i = 1; i < neighbor_discover_count; i++) { + for (int i = 1; i < neighbor_discover_publish_count; i++) { MQTTMessageBuilder::NeighborsMessageEntry entry = entries[i]; int j = i; while (j > 0 && neighborPublishEntryComesBefore(entry, entries[j - 1])) { @@ -1995,6 +2047,10 @@ void MyMesh::finishNeighborDiscover() { neighbor_discover_active = false; neighbor_discover_count = 0; neighbor_discover_next = 0; + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_json_size = 0; + neighbor_discover_truncated = false; neighbor_discover_until = 0; neighbor_discover_request = NULL; if (_cli.getObserverPrefs()->mqtt_neighbors_enabled) { @@ -2010,9 +2066,11 @@ void MyMesh::finishNeighborDiscover() { JsonDocument doc; #endif int json_len = MQTTMessageBuilder::buildNeighborsMessage( - doc, origin, self_pubkey_hex, timestamp, self_scopes_buf, - entries, neighbor_discover_count, - json_buf, MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE); + doc, neighbor_discover_origin, self_pubkey_hex, timestamp, self_scopes_buf, + entries, neighbor_discover_publish_count, + json_buf, MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE, + neighbor_discover_count, neighbor_discover_queried_count, + neighbor_discover_truncated); if (json_len > 0 && bridge) { bridge->requestPublishNeighbors(json_buf, (size_t)json_len); @@ -2027,6 +2085,10 @@ void MyMesh::finishNeighborDiscover() { neighbor_discover_active = false; neighbor_discover_count = 0; neighbor_discover_next = 0; + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_json_size = 0; + neighbor_discover_truncated = false; neighbor_discover_until = 0; neighbor_discover_request = NULL; if (_cli.getObserverPrefs()->mqtt_neighbors_enabled) { @@ -2049,7 +2111,7 @@ void MyMesh::loopNeighborDiscover() { if (!millisHasNowPassed(neighbor_discover_until)) return; if (cancelNeighborDiscoverRequest()) { entry.status = ND_SEND_FAILED; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); return; } if (isCurrentOutbound(neighbor_discover_request)) { @@ -2058,17 +2120,18 @@ void MyMesh::loopNeighborDiscover() { } neighbor_discover_request = NULL; // packet manager already shed it entry.status = ND_SEND_FAILED; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); return; } if (entry.status == ND_PENDING) { if (!millisHasNowPassed(neighbor_discover_until)) return; entry.status = ND_TIMEOUT; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); return; } - if (entry.status == ND_RESPONDED || entry.status == ND_SEND_FAILED) { - neighbor_discover_next++; + if (entry.status == ND_RESPONDED || entry.status == ND_SEND_FAILED + || entry.status == ND_TIMEOUT) { + completeNeighborDiscoverEntry(); return; } if (entry.status != ND_UNSENT) { @@ -2085,7 +2148,7 @@ void MyMesh::loopNeighborDiscover() { neighbor_discover_until = futureMillis(NEIGHBOR_DISCOVER_QUEUE_TIMEOUT_MS); } else { entry.status = ND_SEND_FAILED; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); } } @@ -2109,7 +2172,6 @@ bool MyMesh::startNeighborDiscover(char* reply) { return false; // reply already set } - getLocalScopes(self_scopes_buf, sizeof(self_scopes_buf)); neighbor_discover_count = 0; for (int i = 0; i < MAX_NEIGHBOURS; i++) { if (neighbours[i].heard_timestamp > 0) { @@ -2142,6 +2204,7 @@ bool MyMesh::startNeighborDiscover(char* reply) { } neighbor_discover_next = 0; + resetNeighborDiscoverJsonBudget(); neighbor_discover_active = true; neighbor_discover_until = 0; neighbor_discover_request = NULL; diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index bfca869ec..76fae199c 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -170,6 +170,10 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks NeighborDiscoverEntry neighbor_discover[MAX_NEIGHBOURS]; uint8_t neighbor_discover_count; uint8_t neighbor_discover_next; // newest-first entry currently being queried + uint8_t neighbor_discover_publish_count; // completed prefix that fits the JSON buffer + uint8_t neighbor_discover_queried_count; // requests confirmed transmitted + size_t neighbor_discover_json_size; + bool neighbor_discover_truncated; bool neighbor_discover_active; // scope-query phase in flight bool neighbor_table_refresh_active; // zero-hop table refresh (stage 1) in flight bool neighbor_table_refresh_periodic; // that refresh was kicked by the periodic timer @@ -177,10 +181,13 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks mesh::Packet* neighbor_discover_request; // request awaiting TX completion unsigned long next_neighbors_publish; // periodic publish deadline (0 = fire ASAP) char self_scopes_buf[96]; + char neighbor_discover_origin[32]; mesh::Packet* sendAnonRegionsReq(const mesh::Identity& target, uint32_t& tag); bool cancelNeighborDiscoverRequest(); uint32_t neighborDiscoverQueryTimeoutMs() const; + bool completeNeighborDiscoverEntry(); + void resetNeighborDiscoverJsonBudget(); bool neighborDiscoverReady(char* reply); bool startNeighborDiscover(char* reply); void loopNeighborDiscover(); diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index f161f68c6..d7409f1bd 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -256,6 +256,7 @@ void MyMesh::logTx(mesh::Packet *pkt, int len) { NeighborDiscoverEntry& entry = neighbor_discover[neighbor_discover_next]; if (entry.status == ND_QUEUED) { entry.status = ND_PENDING; + neighbor_discover_queried_count++; neighbor_discover_request = NULL; neighbor_discover_until = futureMillis(neighborDiscoverQueryTimeoutMs()); } @@ -885,6 +886,10 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc pending_discover_until = 0; neighbor_discover_count = 0; neighbor_discover_next = 0; + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_json_size = 0; + neighbor_discover_truncated = false; neighbor_discover_active = false; neighbor_table_refresh_active = false; neighbor_table_refresh_periodic = false; @@ -892,6 +897,7 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc neighbor_discover_request = NULL; next_neighbors_publish = 0; self_scopes_buf[0] = 0; + neighbor_discover_origin[0] = 0; memset(neighbours, 0, sizeof(neighbours)); #endif @@ -1723,6 +1729,57 @@ uint32_t MyMesh::neighborDiscoverQueryTimeoutMs() const { + response_airtime * 2UL; } +void MyMesh::resetNeighborDiscoverJsonBudget() { + getLocalScopes(self_scopes_buf, sizeof(self_scopes_buf)); + MQTTBridge::getEffectiveMqttOrigin( + &_prefs, _cli.getObserverPrefs(), + neighbor_discover_origin, sizeof(neighbor_discover_origin)); + + char self_pubkey_hex[65]; + mesh::Utils::toHex(self_pubkey_hex, self_id.pub_key, PUB_KEY_SIZE); + char timestamp[40]; + MQTTMessageBuilder::formatIsoTimestampForMqtt( + getRTCClock()->getCurrentTime(), 0, nullptr, timestamp, sizeof(timestamp)); + + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_truncated = false; + neighbor_discover_json_size = MQTTMessageBuilder::measureNeighborsMessageBase( + neighbor_discover_origin, self_pubkey_hex, timestamp, self_scopes_buf, + neighbor_discover_count); +} + +// Account for one terminal result. The base measurement reserves maximum-width +// progress metadata; UINT32_MAX likewise reserves the widest heard-age value. +// If this result cannot fit, stop before transmitting another scope request. +bool MyMesh::completeNeighborDiscoverEntry() { + NeighborDiscoverEntry& entry = neighbor_discover[neighbor_discover_next]; + char pubkey_hex[65]; + mesh::Utils::toHex(pubkey_hex, entry.id.pub_key, PUB_KEY_SIZE); + + MQTTMessageBuilder::NeighborsMessageEntry measured = { + pubkey_hex, + entry.snr / 4.0f, + UINT32_MAX, + entry.scopes, + entry.status == ND_RESPONDED ? "responded" + : (entry.status == ND_SEND_FAILED ? "send_failed" : "timeout") + }; + size_t added = MQTTMessageBuilder::measureNeighborsMessageEntry(measured); + if (neighbor_discover_publish_count > 0) added++; // array comma + + if (neighbor_discover_json_size + added >= MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE) { + neighbor_discover_truncated = true; + finishNeighborDiscover(); + return false; + } + + neighbor_discover_json_size += added; + neighbor_discover_publish_count++; + neighbor_discover_next++; + return true; +} + // Match a RESPONSE against the pending overlay entry by tag; copy its scope // string (payload after the 8-byte {tag}{clock} header) into the entry. bool MyMesh::handleNeighborDiscoverResponse(int overlay_idx, const uint8_t* data, size_t len) { @@ -1792,14 +1849,9 @@ struct NeighborsDocAllocator : ArduinoJson::Allocator { // Build the neighbors-table JSON and hand it to the bridge, then reschedule. void MyMesh::finishNeighborDiscover() { - getLocalScopes(self_scopes_buf, sizeof(self_scopes_buf)); - char self_pubkey_hex[65]; mesh::Utils::toHex(self_pubkey_hex, self_id.pub_key, PUB_KEY_SIZE); - char origin[32]; - MQTTBridge::getEffectiveMqttOrigin(&_prefs, _cli.getObserverPrefs(), origin, sizeof(origin)); - char timestamp[40]; MQTTMessageBuilder::formatIsoTimestampForMqtt(getRTCClock()->getCurrentTime(), 0, nullptr, timestamp, sizeof(timestamp)); @@ -1807,7 +1859,7 @@ void MyMesh::finishNeighborDiscover() { MQTTMessageBuilder::NeighborsMessageEntry entries[MAX_NEIGHBOURS]; uint32_t now_secs = getRTCClock()->getCurrentTime(); - for (int i = 0; i < neighbor_discover_count; i++) { + for (int i = 0; i < neighbor_discover_publish_count; i++) { auto& entry = neighbor_discover[i]; mesh::Utils::toHex(pubkey_hex[i], entry.id.pub_key, PUB_KEY_SIZE); entries[i].pubkey_hex = pubkey_hex[i]; @@ -1823,7 +1875,7 @@ void MyMesh::finishNeighborDiscover() { } // insertion sort: most useful first (JSON builder drops the tail on overflow) - for (int i = 1; i < neighbor_discover_count; i++) { + for (int i = 1; i < neighbor_discover_publish_count; i++) { MQTTMessageBuilder::NeighborsMessageEntry entry = entries[i]; int j = i; while (j > 0 && neighborPublishEntryComesBefore(entry, entries[j - 1])) { @@ -1842,6 +1894,10 @@ void MyMesh::finishNeighborDiscover() { neighbor_discover_active = false; neighbor_discover_count = 0; neighbor_discover_next = 0; + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_json_size = 0; + neighbor_discover_truncated = false; neighbor_discover_until = 0; neighbor_discover_request = NULL; if (_cli.getObserverPrefs()->mqtt_neighbors_enabled) { @@ -1857,9 +1913,11 @@ void MyMesh::finishNeighborDiscover() { JsonDocument doc; #endif int json_len = MQTTMessageBuilder::buildNeighborsMessage( - doc, origin, self_pubkey_hex, timestamp, self_scopes_buf, - entries, neighbor_discover_count, - json_buf, MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE); + doc, neighbor_discover_origin, self_pubkey_hex, timestamp, self_scopes_buf, + entries, neighbor_discover_publish_count, + json_buf, MQTTBridge::NEIGHBORS_JSON_BUFFER_SIZE, + neighbor_discover_count, neighbor_discover_queried_count, + neighbor_discover_truncated); if (json_len > 0 && bridge) { bridge->requestPublishNeighbors(json_buf, (size_t)json_len); @@ -1874,6 +1932,10 @@ void MyMesh::finishNeighborDiscover() { neighbor_discover_active = false; neighbor_discover_count = 0; neighbor_discover_next = 0; + neighbor_discover_publish_count = 0; + neighbor_discover_queried_count = 0; + neighbor_discover_json_size = 0; + neighbor_discover_truncated = false; neighbor_discover_until = 0; neighbor_discover_request = NULL; if (_cli.getObserverPrefs()->mqtt_neighbors_enabled) { @@ -1896,7 +1958,7 @@ void MyMesh::loopNeighborDiscover() { if (!millisHasNowPassed(neighbor_discover_until)) return; if (cancelNeighborDiscoverRequest()) { entry.status = ND_SEND_FAILED; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); return; } if (isCurrentOutbound(neighbor_discover_request)) { @@ -1905,17 +1967,18 @@ void MyMesh::loopNeighborDiscover() { } neighbor_discover_request = NULL; // packet manager already shed it entry.status = ND_SEND_FAILED; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); return; } if (entry.status == ND_PENDING) { if (!millisHasNowPassed(neighbor_discover_until)) return; entry.status = ND_TIMEOUT; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); return; } - if (entry.status == ND_RESPONDED || entry.status == ND_SEND_FAILED) { - neighbor_discover_next++; + if (entry.status == ND_RESPONDED || entry.status == ND_SEND_FAILED + || entry.status == ND_TIMEOUT) { + completeNeighborDiscoverEntry(); return; } if (entry.status != ND_UNSENT) { @@ -1932,7 +1995,7 @@ void MyMesh::loopNeighborDiscover() { neighbor_discover_until = futureMillis(NEIGHBOR_DISCOVER_QUEUE_TIMEOUT_MS); } else { entry.status = ND_SEND_FAILED; - neighbor_discover_next++; + completeNeighborDiscoverEntry(); } } @@ -1956,7 +2019,6 @@ bool MyMesh::startNeighborDiscover(char* reply) { return false; // reply already set } - getLocalScopes(self_scopes_buf, sizeof(self_scopes_buf)); neighbor_discover_count = 0; for (int i = 0; i < MAX_NEIGHBOURS; i++) { if (neighbours[i].heard_timestamp > 0) { @@ -1989,6 +2051,7 @@ bool MyMesh::startNeighborDiscover(char* reply) { } neighbor_discover_next = 0; + resetNeighborDiscoverJsonBudget(); neighbor_discover_active = true; neighbor_discover_until = 0; neighbor_discover_request = NULL; diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index 2b064d5f3..ff60c6ce0 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -163,6 +163,10 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks NeighborDiscoverEntry neighbor_discover[MAX_NEIGHBOURS]; uint8_t neighbor_discover_count; uint8_t neighbor_discover_next; + uint8_t neighbor_discover_publish_count; + uint8_t neighbor_discover_queried_count; // requests confirmed transmitted + size_t neighbor_discover_json_size; + bool neighbor_discover_truncated; bool neighbor_discover_active; bool neighbor_table_refresh_active; bool neighbor_table_refresh_periodic; @@ -170,12 +174,15 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks mesh::Packet* neighbor_discover_request; unsigned long next_neighbors_publish; char self_scopes_buf[96]; + char neighbor_discover_origin[32]; void putNeighbour(const mesh::Identity& id, uint32_t timestamp, float snr); void sendNodeDiscoverReq(); mesh::Packet* sendAnonRegionsReq(const mesh::Identity& target, uint32_t& tag); bool cancelNeighborDiscoverRequest(); uint32_t neighborDiscoverQueryTimeoutMs() const; + bool completeNeighborDiscoverEntry(); + void resetNeighborDiscoverJsonBudget(); bool neighborDiscoverReady(char* reply); bool startNeighborDiscover(char* reply); void loopNeighborDiscover(); diff --git a/src/helpers/MQTTMessageBuilder.cpp b/src/helpers/MQTTMessageBuilder.cpp index b90295442..b5dd826df 100644 --- a/src/helpers/MQTTMessageBuilder.cpp +++ b/src/helpers/MQTTMessageBuilder.cpp @@ -111,11 +111,31 @@ int MQTTMessageBuilder::buildNeighborsMessage( const NeighborsMessageEntry* neighbors, int neighbor_count, char* buffer, - size_t buffer_size + size_t buffer_size, + int total_neighbors, + int queried_neighbors, + bool truncated ) { return MQTTPayloadBuilder::buildNeighborsMessage( doc, origin, origin_id, timestamp, self_scopes, neighbors, neighbor_count, - buffer, buffer_size); + buffer, buffer_size, total_neighbors, queried_neighbors, truncated); +} + +size_t MQTTMessageBuilder::measureNeighborsMessageBase( + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + int total_neighbors +) { + return MQTTPayloadBuilder::measureNeighborsMessageBase( + origin, origin_id, timestamp, self_scopes, total_neighbors); +} + +size_t MQTTMessageBuilder::measureNeighborsMessageEntry( + const NeighborsMessageEntry& neighbor +) { + return MQTTPayloadBuilder::measureNeighborsMessageEntry(neighbor); } int MQTTMessageBuilder::buildPacketJSON( diff --git a/src/helpers/MQTTMessageBuilder.h b/src/helpers/MQTTMessageBuilder.h index fc0c47055..2d45af86e 100644 --- a/src/helpers/MQTTMessageBuilder.h +++ b/src/helpers/MQTTMessageBuilder.h @@ -167,8 +167,19 @@ public: const NeighborsMessageEntry* neighbors, int neighbor_count, char* buffer, - size_t buffer_size + size_t buffer_size, + int total_neighbors = -1, + int queried_neighbors = -1, + bool truncated = false ); + static size_t measureNeighborsMessageBase( + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + int total_neighbors + ); + static size_t measureNeighborsMessageEntry(const NeighborsMessageEntry& neighbor); /** * Convert packet to JSON message diff --git a/src/helpers/MQTTPayloadBuilder.cpp b/src/helpers/MQTTPayloadBuilder.cpp index 5a40e502b..77d9054c4 100644 --- a/src/helpers/MQTTPayloadBuilder.cpp +++ b/src/helpers/MQTTPayloadBuilder.cpp @@ -185,6 +185,68 @@ int MQTTPayloadBuilder::buildRawMessage( return serializeComplete(root, buffer, buffer_size); } +static JsonArray buildNeighborsMessageBase( + JsonDocument& doc, + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + int total_neighbors, + int queried_neighbors, + bool truncated +) { + doc.clear(); + JsonObject root = doc.to(); + root["timestamp"] = timestamp; + root["origin"] = origin; + root["origin_id"] = origin_id; + if (total_neighbors >= 0) { + root["total_neighbors"] = total_neighbors; + root["queried_neighbors"] = queried_neighbors >= 0 ? queried_neighbors : total_neighbors; + root["truncated"] = truncated; + } + + JsonObject self = root["self"].to(); + self["scopes"] = self_scopes ? self_scopes : ""; + return root["neighbors"].to(); +} + +static void addNeighborsMessageEntry( + JsonArray& arr, + const MQTTPayloadBuilder::NeighborsMessageEntry& neighbor +) { + JsonObject nb = arr.add(); + nb["pubkey"] = neighbor.pubkey_hex; + nb["snr"] = neighbor.snr; + nb["heard_secs_ago"] = neighbor.heard_secs_ago; + nb["scopes"] = neighbor.scopes ? neighbor.scopes : ""; + nb["status"] = neighbor.status; +} + +size_t MQTTPayloadBuilder::measureNeighborsMessageBase( + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + int total_neighbors +) { + JsonDocument doc; + buildNeighborsMessageBase( + doc, origin, origin_id, timestamp, self_scopes, + total_neighbors, total_neighbors, false); + return measureJson(doc); +} + +size_t MQTTPayloadBuilder::measureNeighborsMessageEntry( + const NeighborsMessageEntry& neighbor +) { + JsonDocument doc; + JsonArray arr = doc.to(); + addNeighborsMessageEntry(arr, neighbor); + JsonObject entry = arr[0]; + return measureJson(entry); +} + int MQTTPayloadBuilder::buildNeighborsMessage( JsonDocument& doc, const char* origin, @@ -194,38 +256,31 @@ int MQTTPayloadBuilder::buildNeighborsMessage( const NeighborsMessageEntry* neighbors, int neighbor_count, char* buffer, - size_t buffer_size + size_t buffer_size, + int total_neighbors, + int queried_neighbors, + bool truncated ) { if (!buffer || buffer_size == 0) return 0; - doc.clear(); - JsonObject root = doc.to(); - root["timestamp"] = timestamp; - root["origin"] = origin; - root["origin_id"] = origin_id; - - JsonObject self = root["self"].to(); - self["scopes"] = self_scopes ? self_scopes : ""; - - JsonArray arr = root["neighbors"].to(); - if (measureJson(root) >= buffer_size) return 0; + JsonArray arr = buildNeighborsMessageBase( + doc, origin, origin_id, timestamp, self_scopes, + total_neighbors, queried_neighbors, truncated); + if (doc.overflowed() || arr.isNull() || measureJson(doc) >= buffer_size) return 0; for (int i = 0; i < neighbor_count; i++) { - JsonObject nb = arr.add(); - nb["pubkey"] = neighbors[i].pubkey_hex; - nb["snr"] = neighbors[i].snr; - nb["heard_secs_ago"] = neighbors[i].heard_secs_ago; - nb["scopes"] = neighbors[i].scopes ? neighbors[i].scopes : ""; - nb["status"] = neighbors[i].status; + addNeighborsMessageEntry(arr, neighbors[i]); + if (doc.overflowed()) return 0; // Entries arrive ordered most- to least-useful. Stop as soon as the next // one would fill the fixed publish buffer, dropping the remaining tail so // document growth stays bounded. - if (measureJson(root) >= buffer_size) { + if (measureJson(doc) >= buffer_size) { arr.remove(arr.size() - 1); + if (total_neighbors >= 0) doc["truncated"] = true; break; } } - return serializeComplete(root, buffer, buffer_size); + return serializeComplete(doc.as(), buffer, buffer_size); } diff --git a/src/helpers/MQTTPayloadBuilder.h b/src/helpers/MQTTPayloadBuilder.h index a6fa333d1..f9e5c9598 100644 --- a/src/helpers/MQTTPayloadBuilder.h +++ b/src/helpers/MQTTPayloadBuilder.h @@ -89,7 +89,21 @@ public: const NeighborsMessageEntry* neighbors, int neighbor_count, char* buffer, - size_t buffer_size + size_t buffer_size, + int total_neighbors = -1, + int queried_neighbors = -1, + bool truncated = false ); -}; + // Exact serialized-size components used by paced neighbor discovery. The + // base reserves the largest progress metadata values for this snapshot; + // callers add each entry size plus one byte for commas after the first. + static size_t measureNeighborsMessageBase( + const char* origin, + const char* origin_id, + const char* timestamp, + const char* self_scopes, + int total_neighbors + ); + static size_t measureNeighborsMessageEntry(const NeighborsMessageEntry& neighbor); +}; diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 4102cbe5c..63a04306e 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -3314,8 +3314,10 @@ bool MQTTBridge::publishNeighbors() { // MeshRank slots reject non-packets by contract, so buildTopicForSlot // returns false for them here and the slot is skipped. if (buildTopicForSlot(i, MSG_NEIGHBORS, topic, sizeof(topic))) { + // Neighbor snapshots are periodically refreshed. Publish synchronously + // at QoS 0 to avoid the QoS 1 outbox, retaining where the broker allows. bool use_retain = _slots[i].preset ? _slots[i].preset->allow_retain : false; - if (publishToSlot(i, topic, _neighbors_json_buffer, use_retain, 1)) { + if (publishToSlot(i, topic, _neighbors_json_buffer, use_retain, 0)) { published = true; } } diff --git a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp index d7f811308..4a1e40deb 100644 --- a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp +++ b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp @@ -12,6 +12,12 @@ namespace { constexpr const char* kTimestamp = "2026-07-18T12:34:56.123456+00:00"; +struct RejectAllJsonAllocations : ArduinoJson::Allocator { + void* allocate(size_t) override { return nullptr; } + void deallocate(void*) override {} + void* reallocate(void*, size_t) override { return nullptr; } +}; + static int buildMinimalStatus(JsonDocument& scratch, char* buffer, size_t buffer_size) { return MQTTPayloadBuilder::buildStatusMessage( scratch, "DEN Repeater", "0123456789ABCDEF", "Heltec V3", "v1.16.0", @@ -237,7 +243,7 @@ TEST(MQTTPayloadBuilder, NeighborsMessageRoundTripsSelfAndEntries) { char buffer[1024]; int len = MQTTPayloadBuilder::buildNeighborsMessage( scratch, "DEN Repeater", "0123456789ABCDEF", kTimestamp, "DEN,APRS", - neighbors, 2, buffer, sizeof(buffer)); + neighbors, 2, buffer, sizeof(buffer), 5, 2, true); ASSERT_GT(len, 0); EXPECT_EQ(static_cast(len), strlen(buffer)); @@ -247,6 +253,9 @@ TEST(MQTTPayloadBuilder, NeighborsMessageRoundTripsSelfAndEntries) { EXPECT_STREQ("DEN Repeater", parsed["origin"].as()); EXPECT_STREQ("0123456789ABCDEF", parsed["origin_id"].as()); EXPECT_STREQ("DEN,APRS", parsed["self"]["scopes"].as()); + EXPECT_EQ(5, parsed["total_neighbors"].as()); + EXPECT_EQ(2, parsed["queried_neighbors"].as()); + EXPECT_TRUE(parsed["truncated"].as()); JsonArray arr = parsed["neighbors"].as(); ASSERT_EQ(2U, arr.size()); @@ -260,6 +269,116 @@ TEST(MQTTPayloadBuilder, NeighborsMessageRoundTripsSelfAndEntries) { EXPECT_STREQ("stale", arr[1]["status"].as()); } +TEST(MQTTPayloadBuilder, NeighborsMessageMeasurementsMatchCompletePayload) { + MQTTPayloadBuilder::NeighborsMessageEntry neighbors[] = { + {"0011223344556677", 9.75f, UINT32_MAX, "DEN,APRS", "responded"}, + {"8899AABBCCDDEEFF", -3.5f, UINT32_MAX, "", "timeout"}, + }; + + size_t measured = + MQTTPayloadBuilder::measureNeighborsMessageBase( + "DEN Repeater", "0123456789ABCDEF", kTimestamp, "DEN,APRS", 2) + + MQTTPayloadBuilder::measureNeighborsMessageEntry(neighbors[0]) + + 1 // array comma + + MQTTPayloadBuilder::measureNeighborsMessageEntry(neighbors[1]); + + JsonDocument scratch; + char buffer[1024]; + int len = MQTTPayloadBuilder::buildNeighborsMessage( + scratch, "DEN Repeater", "0123456789ABCDEF", kTimestamp, "DEN,APRS", + neighbors, 2, buffer, sizeof(buffer), 2, 2, false); + + ASSERT_GT(len, 0); + EXPECT_EQ(measured, static_cast(len)); +} + +TEST(MQTTPayloadBuilder, NeighborsMessageMeasuredPrefixStopsBeforeOverflow) { + MQTTPayloadBuilder::NeighborsMessageEntry neighbors[20]; + static char keys[20][17]; + static char long_scopes[96]; + memset(long_scopes, 'S', sizeof(long_scopes) - 1); + long_scopes[sizeof(long_scopes) - 1] = '\0'; + for (int i = 0; i < 20; i++) { + snprintf(keys[i], sizeof(keys[i]), "%016X", i); + neighbors[i] = { + keys[i], static_cast(i), UINT32_MAX, + long_scopes, "responded"}; + } + + constexpr size_t kBufferSize = 512; + size_t used = MQTTPayloadBuilder::measureNeighborsMessageBase( + "node", "id", kTimestamp, "DEN", 20); + int published = 0; + int queried = 0; + while (queried < 20) { + size_t added = + MQTTPayloadBuilder::measureNeighborsMessageEntry(neighbors[queried]) + + (published > 0 ? 1U : 0U); + queried++; + if (used + added >= kBufferSize) break; + used += added; + published++; + } + + ASSERT_GT(published, 0); + ASSERT_LT(published, 20); + ASSERT_EQ(published + 1, queried); // only the non-fitting candidate was extra + + JsonDocument scratch; + char buffer[kBufferSize]; + int len = MQTTPayloadBuilder::buildNeighborsMessage( + scratch, "node", "id", kTimestamp, "DEN", + neighbors, published, buffer, sizeof(buffer), + 20, queried, true); + + ASSERT_GT(len, 0); + EXPECT_LT(static_cast(len), sizeof(buffer)); + JsonDocument parsed; + ASSERT_FALSE(deserializeJson(parsed, buffer)); + EXPECT_EQ(static_cast(published), + parsed["neighbors"].as().size()); + EXPECT_EQ(20, parsed["total_neighbors"].as()); + EXPECT_EQ(queried, parsed["queried_neighbors"].as()); + EXPECT_TRUE(parsed["truncated"].as()); +} + +TEST(MQTTPayloadBuilder, NeighborsMessageFallbackMarksTruncated) { + MQTTPayloadBuilder::NeighborsMessageEntry neighbors[] = { + {"0011223344556677", 9.75f, 1, "DEN", "responded"}, + {"8899AABBCCDDEEFF", -3.5f, 2, "APRS", "responded"}, + }; + size_t first_only_size = + MQTTPayloadBuilder::measureNeighborsMessageBase( + "node", "id", kTimestamp, "DEN", 2) + + MQTTPayloadBuilder::measureNeighborsMessageEntry(neighbors[0]); + std::vector buffer(first_only_size + 1); + + JsonDocument scratch; + int len = MQTTPayloadBuilder::buildNeighborsMessage( + scratch, "node", "id", kTimestamp, "DEN", + neighbors, 2, buffer.data(), buffer.size(), + 2, 2, false); + + ASSERT_GT(len, 0); + JsonDocument parsed; + ASSERT_FALSE(deserializeJson(parsed, buffer.data())); + EXPECT_EQ(1U, parsed["neighbors"].as().size()); + EXPECT_TRUE(parsed["truncated"].as()); +} + +TEST(MQTTPayloadBuilder, NeighborsMessageFailsCleanlyOnAllocationFailure) { + MQTTPayloadBuilder::NeighborsMessageEntry neighbor = { + "0011223344556677", 9.75f, 1, "DEN", "responded"}; + RejectAllJsonAllocations allocator; + JsonDocument scratch(&allocator); + char buffer[512]; + + EXPECT_EQ(0, MQTTPayloadBuilder::buildNeighborsMessage( + scratch, "node", "id", kTimestamp, "DEN", + &neighbor, 1, buffer, sizeof(buffer), + 1, 1, false)); +} + TEST(MQTTPayloadBuilder, NeighborsMessageHandlesEmptyTableAndNullScopes) { JsonDocument scratch; char buffer[256]; @@ -304,6 +423,7 @@ TEST(MQTTPayloadBuilder, NeighborsMessageDropsTailWhenBufferFills) { ASSERT_FALSE(arr.isNull()); EXPECT_GT(arr.size(), 0U); EXPECT_LT(arr.size(), 20U); + EXPECT_FALSE(parsed["truncated"].is()); // Kept entries are the head of the input, in order. EXPECT_STREQ(keys[0], arr[0]["pubkey"].as()); } @@ -314,4 +434,3 @@ int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } -