From 7a463937cabd4d4976f02059137f0bcc369f8fc0 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Thu, 8 Oct 2026 14:41:16 -0700 Subject: [PATCH] Reduce shared RAM usage without lowering firmware capacities Specialize repeater ACL session state, compact recent and flood records, stream Merkle proofs, and release cold OTA buffers safely. Preserve saved formats, stack sizes, table capacities, and OTA source limits. Account for lazy buffers when active and add compatibility, ARM alignment, allocation failure, and queued-response tests. Validated six representative board/role builds, 1736 native test cases, sanitizer fixtures, and strict documentation. Document the preexisting Wio-E5 Mini flash overflow separately. --- .github/workflows/run-unit-tests.yml | 6 + docs/research/firmware_memory_budget.md | 96 ++++- examples/simple_repeater/MyMesh.cpp | 22 +- examples/simple_repeater/MyMesh.h | 43 ++- examples/simple_repeater/main.cpp | 2 + .../simple_room_server/FloodRuleEngine.cpp | 3 +- examples/simple_room_server/FloodRuleEngine.h | 29 +- platformio.ini | 6 +- scripts/check_firmware_ram.py | 61 ++- scripts/client_acl_role.py | 55 +++ src/helpers/ClientACL.h | 16 +- src/helpers/FloodFilterPolicy.h | 35 ++ src/helpers/SimpleMeshTables.h | 108 +++++- src/helpers/ota/MerkleTree.cpp | 25 +- src/helpers/ota/MerkleTree.h | 4 +- src/helpers/ota/OtaContext.h | 4 +- src/helpers/ota/OtaManager.cpp | 97 +++-- src/helpers/ota/OtaManager.h | 28 +- src/helpers/ota/OtaSelf.cpp | 9 +- .../test_client_acl_spiffs.cpp | 1 + test/fixtures/ota_cold_buffers/test.cpp | 359 ++++++++++++++++++ test/test_client_acl_role.py | 163 ++++++++ test/test_esp32_tinyusb_cooperative_output.py | 6 + test/test_firmware_ram.py | 60 ++- test/test_flood_policy_capacity.py | 66 ++++ test/test_flood_rule_layout.py | 238 ++++++++++++ .../test_simple_mesh_tables.cpp | 86 ++++- test/test_ota_cold_buffers.py | 43 +++ test/test_recent_repeater_storage.py | 231 +++++++++++ .../test_transport_filter.cpp | 31 ++ 30 files changed, 1783 insertions(+), 150 deletions(-) create mode 100644 scripts/client_acl_role.py create mode 100644 test/fixtures/ota_cold_buffers/test.cpp create mode 100644 test/test_client_acl_role.py create mode 100644 test/test_flood_rule_layout.py create mode 100644 test/test_ota_cold_buffers.py create mode 100644 test/test_recent_repeater_storage.py diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 899fac538..f4e66f00f 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -174,6 +174,7 @@ jobs: python3 -B test/test_common_prefs_commit.py -v python3 -B test/test_common_prefs_legacy_rxgain.py -v python3 -B test/test_flood_policy_capacity.py -v + python3 -B test/test_flood_rule_layout.py -v python3 -B test/test_mqtt_prefs_commit.py -v python3 -B test/test_identity_and_settings_recovery.py -v python3 -B test/test_contact_cache.py -v @@ -243,6 +244,7 @@ jobs: python3 -B test/test_ota_device_deflate_encoder.py -v python3 -B test/test_ota_device_deflate_role.py -v python3 -B test/test_client_acl_spiffs.py + python3 -B test/test_client_acl_role.py -v python3 -B test/test_client_acl_metadata.py -v python3 -B test/test_client_acl_cli.py -v python3 -B test/test_client_acl_response.py -v @@ -428,9 +430,13 @@ jobs: python3 -B test/test_replay_reset_integration.py -v - name: Verify firmware memory budgets and nRF52 Bluetooth startup + env: + MESHCORE_REQUIRE_ARM_COMPILER: '1' run: | bash test/test_build_size_detection.sh python3 -B test/test_firmware_ram.py + python3 -B test/test_recent_repeater_storage.py -v + python3 -B test/test_ota_cold_buffers.py -v python3 -B test/test_neighbor_scope_storage.py python3 -B test/test_ed25519_compact.py python3 -B test/test_cascade_release_package.py diff --git a/docs/research/firmware_memory_budget.md b/docs/research/firmware_memory_budget.md index 7c4195bbb..fee39bfbc 100644 --- a/docs/research/firmware_memory_budget.md +++ b/docs/research/firmware_memory_budget.md @@ -60,10 +60,19 @@ while Ethernet owns SPI. Disabling Ethernet parks the worker after socket/SPI cleanup; the main loop deletes that parked task synchronously and frees its TX queue before releasing ownership. The worker stack therefore cannot remain queued for idle-task reclamation while OTA allocates again. The budget counts the larger of the -4,928-byte Ethernet worker/queue or 5,168-byte external OTA workspaces, with the UART +4,928-byte Ethernet worker/queue or 11,344-byte external OTA workspaces, with the UART allowance alongside either mode. It keeps the 8 KiB loop stack, 63 flood rules, and reserved 64 KiB OTA handoff arena. +The shared RAM changes now reserve the lazy folder-source buffers as well: +up to 4,096 bytes of source leaves and one 2,048-byte encoded output block, +plus allocation overhead, for the RAK4631. These can coexist with own-image +metadata and the encoder. The combined budget therefore reserves 11,344 bytes +for external OTA, or 10,304 with the device encoder disabled, against the +4,928-byte Ethernet worker/queue. Ethernet and external OTA remain exclusive. +Freeing static buffers increases available heap; their active allocation cost +still appears in the memory check. + This combined config uses a source-bounded 2 KiB SSD1306 allowance. Its named 128x64 geometry needs one 1,024-byte framebuffer, reused on later starts; the driver itself is embedded in a linked global, measured at 156 bytes in @@ -75,6 +84,91 @@ bound. This corrects a padded allocation estimate; it does not free actual RAM. Other images retain their existing display budgets. These are local test configs, not a qualification of a published or physically tested image. +## Shared RAM reductions + +These changes apply to every board that compiles the corresponding feature. +They retain stack sizes, radio packet pools, USB buffering, table capacities, +stored settings, OTA block sizes and source-image limits. + +| Change | Scope | RAM reduction | +| --- | --- | --- | +| Repeater ACL session state | Pure repeater builds on all platforms | 36 bytes per client on 32-bit targets; 1,152 bytes for 32 clients | +| Compact recent-repeater records | Repeaters with recent history enabled | 12 to 9 bytes per record; 1,536 bytes for 512 records | +| Compact flood-rule flags and aligned counters | Generalized repeater and room rule engines | 200 to 192 bytes per rule; 504 bytes for 63 repeater rules or 248 bytes for 31 room rules | +| Streaming Merkle proof generation | All OTA roles and platforms | Removes full-leaf-table proof scratch; compressed output needs only one logical block | +| Source buffers allocated on demand | All OTA roles and platforms | Leaves allocate only the selected image's digest count; unused leaves/output return to the heap | + +The ACL source-role pre-script contributes one common compiler definition for +the entire build. Room/sensor and unknown roles keep the complete session +layout. The persisted four-byte room synchronization field stays in repeater +records, so existing ACL files and their checksums remain compatible. + +Recent-history timestamps retain all 32 bits using explicit byte operations, +avoiding unaligned multi-byte accesses on ARM. The table belongs to the mesh +loop; network command handlers marshal their work to that loop. These byte +operations do not provide synchronization for hypothetical concurrent users. +Flood flags remain internal; FPF files serialize each field explicitly and +never expose compiler bitfield layout. Multibyte counters remain aligned. +Rule deduplication compares configured fields explicitly, so neither live +rate-window counters nor struct padding can make different rules share a slot. + +OTA leaf capacity remains controlled by `OTA_PROOFGEN_SCRATCH`, independently +of output-buffer size. The maximum source buffers are 4 KiB plus one block on +ordinary nRF52, 8 KiB plus one block with SD storage, and 16 KiB plus one block +on ESP32. Generic folder sources need no output allocation for raw DATA or +proofs; own-image raw serving retains its caller-owned compatibility buffer. +Its nRF52 allowance remains in the budget even when the device encoder is off. +Queued DATA, PROOF and MANIFEST responses pin their buffers through radio +backpressure. +Explicit cancellation or source detach releases the reloadable cache; an +unpinned cache also expires after 30 seconds. Attached source descriptors +remain advertised and reload on the next request. Allocation failure refuses +source loading safely, or falls back to raw DATA if only compression output +is unavailable. Receive reassembly and serving/inflate buffers remain separate. + +On RAK4631, the OTA context's permanent allocation falls from 28,816 to 20,640 +bytes, an 8,176-byte reduction before table savings. This is idle heap capacity, +not an additional 8 KiB saving on top of the active-operation savings. The RAM +policy reserves lazy buffers when active and separately checks the largest +contiguous allocation. ESP32 history uses its actual default capacities: +256 records on classic ESP32 and 2,048 on S2/S3/C-series, unless overridden. + +The matched RAK4631 Full combined trial on 8 October 2026 retained all sensor +drivers, 32 ACL clients, 63 flood rules, 512 recent repeater records, the 8 KiB loop +stack and the reserved 64 KiB OTA arena: + +| Linked measurement | Before shared RAM changes | After all five changes | +| --- | ---: | ---: | +| Available heap capacity before startup | 73,580 bytes | 83,300 bytes | +| Reserved startup/active allocations | 72,584 bytes | 77,104 bytes | +| Capacity beyond those reservations | 996 bytes | 6,196 bytes | +| Application flash | 585,700 bytes | 586,260 bytes | + +The permanent allocations shrink by 9,720 bytes. The smaller ACL and flood +tables reduce startup allowances by another 1,656 bytes, while the new +6,176-byte folder-source allowance counts buffers when active. The resulting +capacity beyond reservations improves by 5,200 bytes; flash grows by 560 bytes. +The reports are bound to ELF hashes +`a16c02e43c12d7dadd94b92b9b80cdd22eced7f67e085b1fffb62ea39f09fcce` +and `f78110d1609334621098cb6db2039227c2f8fc9ac7f9e6ef7dc3cc63eb8d502f`, +respectively. These are linked capacity checks, not measured free heap after +boot or physical Ethernet/OTA soak results. + +The five changes pass representative repeater builds for RAK4631 combined +Full, Heltec V4 (ESP32-S3), Heltec V2 (classic ESP32), and Seeed XIAO RP2040. +RAK4631 room-server and Heltec V4 sensor builds also pass while retaining +their complete ACL session layouts. Wio E5 Mini repeater compilation passes, +but linking exceeds flash capacity: the baseline already overflowed by +27,200 bytes, and these changes add 224 bytes for a 27,424-byte overflow. +That STM32 image is unqualified and is not release-ready. + +Regression coverage includes unchanged ACL/FPF file bytes, full capacities, +timestamp rollover, flag combinations, ARM access widths/alignment, streamed +proofs against the existing reference algorithm, allocation failures, source +reload, queued-response lifetimes, and concurrent compressed serving/receiving. +Synthetic and representative build results are distinct from physical soak +qualification of every board. + Generate matched Full/Reduced baseline and combined configs outside the repository without starting PlatformIO: diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 7f7c41407..e7e695a23 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -2474,7 +2474,9 @@ void MyMesh::formatRecentRepeatersReply(char *reply, int page, if (info == NULL) break; char prefix[MAX_ROUTE_HASH_BYTES * 2 + 1]; char snr[12]; - mesh::Utils::toHex(prefix, info->prefix, info->prefix_len); + alignas(uint32_t) uint8_t recorded_prefix[MAX_ROUTE_HASH_BYTES]; + SimpleMeshTables::copyRecentRepeaterPrefix(recorded_prefix, *info); + mesh::Utils::toHex(prefix, recorded_prefix, info->prefix_len); prefix[info->prefix_len * 2] = 0; formatLocalSnrX4(snr, sizeof(snr), info->snr_x4); if (is_search) { @@ -2506,7 +2508,9 @@ void MyMesh::printRecentRepeatersSerial() { const auto* info = tables ? tables->getRecentRepeaterBySortedIdx(row++) : nullptr; if (!info) return 0; char prefix[MAX_ROUTE_HASH_BYTES * 2 + 1], snr[12]; - mesh::Utils::toHex(prefix, info->prefix, info->prefix_len); + alignas(uint32_t) uint8_t recorded_prefix[MAX_ROUTE_HASH_BYTES]; + SimpleMeshTables::copyRecentRepeaterPrefix(recorded_prefix, *info); + mesh::Utils::toHex(prefix, recorded_prefix, info->prefix_len); formatLocalSnrX4(snr, sizeof(snr), info->snr_x4); return snprintf(out, capacity, "%s,%s%s\r\n", prefix, snr[0] == '-' ? "" : " ", snr); }, this, "Recent repeaters:\r\n"); @@ -2542,7 +2546,9 @@ void MyMesh::printRecentRepeatersSerial() { if (info == NULL) break; char prefix[MAX_ROUTE_HASH_BYTES * 2 + 1]; char snr[12]; - mesh::Utils::toHex(prefix, info->prefix, info->prefix_len); + alignas(uint32_t) uint8_t recorded_prefix[MAX_ROUTE_HASH_BYTES]; + SimpleMeshTables::copyRecentRepeaterPrefix(recorded_prefix, *info); + mesh::Utils::toHex(prefix, recorded_prefix, info->prefix_len); prefix[info->prefix_len * 2] = 0; formatLocalSnrX4(snr, sizeof(snr), info->snr_x4); mesh::usbConsolePort().printf("%s,%s%s\n", prefix, snr[0] == '-' ? "" : " ", snr); @@ -5329,13 +5335,15 @@ void MyMesh::servicePendingSerialOutput() { } char prefix[MAX_ROUTE_HASH_BYTES * 2 + 1]; char snr[12]; - mesh::Utils::toHex(prefix, info->prefix, info->prefix_len); + alignas(uint32_t) uint8_t recorded_prefix[MAX_ROUTE_HASH_BYTES]; + SimpleMeshTables::copyRecentRepeaterPrefix(recorded_prefix, *info); + mesh::Utils::toHex(prefix, recorded_prefix, info->prefix_len); formatLocalSnrX4(snr, sizeof(snr), info->snr_x4); length = snprintf(record, sizeof(record), "%s,%s%s\n", prefix, snr[0] == '-' ? "" : " ", snr); // Snapshot the row before a short write. Its next pass must retry only // the retained suffix, even if the live recent-repeater table changes. - serial_recent_cursor = *info; + SimpleMeshTables::copyRecentRepeaterInfo(serial_recent_cursor, *info); serial_recent_cursor_index = next_index; serial_recent_has_cursor = true; } else { @@ -8291,9 +8299,7 @@ void MyMesh::setFloodPacketFilter(const char* args, char* reply, if (i == flood_channel_data_rule_slot) continue; const auto& entry = flood_packet_filters[i]; if (entry.active - && memcmp(&entry, &candidate, - offsetof(FloodPacketFilterEntry, - rate_window_started)) == 0) { + && FloodFilterPolicy::sameRuleConfiguration(entry, candidate)) { slot = i; break; } diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 456188ef1..8e4a679de 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -408,18 +408,27 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks uint32_t last_heard_millis; }; struct FloodPacketFilterEntry { +#if MESH_ENABLE_FLOOD_RULE_ENGINE + // Keep multibyte counters naturally aligned. These seven independent + // flags share a byte; their bit order is never persisted (FPF7 serializes + // each field explicitly). active/retry remain addressable for the stored + // rule decoders. Rule mutations run on the mesh loop. + uint32_t rate_window_started; + uint16_t rate_per_minute; + uint16_t rate_window_count; + bool suspend_on_temp_radio : 1; + bool match_blacklisted_path : 1; + bool scope_uses_slow_timing : 1; + bool drop_on_match : 1; + bool rate_limit_enabled : 1; + bool stop_on_match : 1; + bool rate_window_active : 1; bool active; + bool retry_on_match; uint8_t payload_type; uint8_t min_hops; uint8_t max_hops; - bool suspend_on_temp_radio; char scope_name[FLOOD_PACKET_FILTER_SCOPE_NAME_LEN]; - bool match_blacklisted_path; -#if !MESH_ENABLE_FLOOD_RULE_ENGINE - bool scope_requires_region_match; -#endif - bool scope_uses_slow_timing; -#if MESH_ENABLE_FLOOD_RULE_ENGINE uint8_t incoming_scope_kind; char incoming_scope_name[FLOOD_PACKET_FILTER_SCOPE_NAME_LEN]; uint8_t channel_key_len; @@ -430,16 +439,18 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks uint8_t path_hops; uint8_t path[FLOOD_PACKET_FILTER_PATH_PREFIX_BYTES_MAX]; char target_region_name[FLOOD_PACKET_FILTER_SCOPE_NAME_LEN]; - bool drop_on_match; - bool rate_limit_enabled; - uint16_t rate_per_minute; uint8_t priority; - bool stop_on_match; - bool retry_on_match; uint8_t transport_modes; - uint32_t rate_window_started; - uint16_t rate_window_count; - bool rate_window_active; +#else + bool active; + uint8_t payload_type; + uint8_t min_hops; + uint8_t max_hops; + bool suspend_on_temp_radio; + char scope_name[FLOOD_PACKET_FILTER_SCOPE_NAME_LEN]; + bool match_blacklisted_path; + bool scope_requires_region_match; + bool scope_uses_slow_timing; #endif }; struct FloodChannelScopeEntry { @@ -483,7 +494,7 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks }; mutable FloodRetryBridgeState flood_retry_bridge_states[TOTAL_FLOOD_RETRY_SLOTS]; FloodRetryBridgeReachability flood_retry_bridge_reachability[FLOOD_RETRY_BRIDGE_BUCKETS + 1]; - static_assert(sizeof(FloodPacketFilterEntry) <= (MESH_ENABLE_FLOOD_RULE_ENGINE ? 200 : 40), + static_assert(sizeof(FloodPacketFilterEntry) <= (MESH_ENABLE_FLOOD_RULE_ENGINE ? 192 : 40), "Update the flood-table runtime RAM budget in check_firmware_ram.py"); // The rule table is the single largest member of this object. It is heap // allocated in the constructor (before setup(), while the heap is still diff --git a/examples/simple_repeater/main.cpp b/examples/simple_repeater/main.cpp index 4cf0fbce5..77459b1ef 100644 --- a/examples/simple_repeater/main.cpp +++ b/examples/simple_repeater/main.cpp @@ -52,6 +52,8 @@ extern "C" bool btInUse() { return false; } StdRNG fast_rng; #if MAX_RECENT_REPEATERS > 0 +// Compact rows retain the complete prefix/SNR/32-bit age at every capacity. +// All accesses are owned by this loop, including deferred network commands. #if defined(ESP32) // Classic ESP32 has a much smaller link-time DRAM window than its runtime heap // map. Allocate this large, fixed-capacity history at startup so enabling WiFi diff --git a/examples/simple_room_server/FloodRuleEngine.cpp b/examples/simple_room_server/FloodRuleEngine.cpp index 279c5c582..d7cae754d 100644 --- a/examples/simple_room_server/FloodRuleEngine.cpp +++ b/examples/simple_room_server/FloodRuleEngine.cpp @@ -1732,8 +1732,7 @@ void FloodRuleEngine::set(const char* args, char* reply, if (slot < 0) { for (int i = 0; i < RULE_SLOTS; i++) { if (_entries[i].active - && memcmp(&_entries[i], &candidate, - offsetof(Entry, rate_window_started)) == 0) { + && FloodFilterPolicy::sameRuleConfiguration(_entries[i], candidate)) { slot = i; break; } diff --git a/examples/simple_room_server/FloodRuleEngine.h b/examples/simple_room_server/FloodRuleEngine.h index e1456dd31..903702a2a 100644 --- a/examples/simple_room_server/FloodRuleEngine.h +++ b/examples/simple_room_server/FloodRuleEngine.h @@ -42,14 +42,25 @@ public: private: struct Entry { + // Native alignment for counters; explicit FPF7 serialization does not + // expose the in-memory flag bit order. Keep active/retry addressable for + // the stored rule decoders, and mutate rules only on the mesh loop. + uint32_t rate_window_started; + uint16_t rate_per_minute; + uint16_t rate_window_count; + bool suspend_on_temp_radio : 1; + bool match_blacklisted_path : 1; + bool scope_uses_slow_timing : 1; + bool drop_on_match : 1; + bool rate_limit_enabled : 1; + bool stop_on_match : 1; + bool rate_window_active : 1; bool active; + bool retry_on_match; uint8_t payload_type; uint8_t min_hops; uint8_t max_hops; - bool suspend_on_temp_radio; char scope_name[NAME_LEN]; - bool match_blacklisted_path; - bool scope_uses_slow_timing; uint8_t incoming_scope_kind; char incoming_scope_name[NAME_LEN]; uint8_t channel_key_len; @@ -60,24 +71,16 @@ private: uint8_t path_hops; uint8_t path[PATH_PREFIX_BYTES_MAX]; char target_region_name[NAME_LEN]; - bool drop_on_match; - bool rate_limit_enabled; - uint16_t rate_per_minute; uint8_t priority; - bool stop_on_match; - bool retry_on_match; uint8_t transport_modes; - uint32_t rate_window_started; - uint16_t rate_window_count; - bool rate_window_active; }; - // This table is 6,200 bytes. Keep it in the constructor-time heap rather + // This table is at most 5,952 bytes. Keep it in the constructor-time heap rather // than inside the global MyMesh object: classic ESP32 has a small static // DRAM window but substantially more internal heap at this point in boot. // Allocation is deliberately all-or-nothing. A node must never fall back // to silently running without its configured flood rules. - static_assert(sizeof(Entry) == 200, + static_assert(sizeof(Entry) <= 192, "Update the flood-rule runtime RAM budget in check_firmware_ram.py"); FILESYSTEM* _fs; RegionMap* _regions; diff --git a/platformio.ini b/platformio.ini index 2edfe5905..5940276e6 100644 --- a/platformio.ini +++ b/platformio.ini @@ -83,6 +83,7 @@ monitor_filters = esp32_exception_decoder ; pio_endf appends the EndF self-identity trailer. (nRF52 single-slot OTA is enabled per-board, not here - ; it needs a custom bootloader; RP2040/STM32 have no A/B path yet.) extra_scripts = + pre:scripts/client_acl_role.py pre:scripts/generate_webconfig_html.py pre:scripts/meshcore_image_identity.py pre:scripts/ota_device_deflate_role.py @@ -159,6 +160,7 @@ platform_packages = framework-arduinoadafruitnrf52 @ https://github.com/meshcore-dev/Adafruit_nRF52_Arduino#d541301665b40959682252911e57b11df3ee651a platformio/toolchain-gccarmnoneeabi@^1.140201.0 extra_scripts = create-uf2.py + pre:scripts/client_acl_role.py pre:scripts/ota_device_deflate_role.py pre:scripts/nrf52_usb_power_fix.py pre:scripts/nrf52_internal_flash_fix.py @@ -307,7 +309,8 @@ lib_deps = ${nrf52_lora_ota.lib_deps} [rp2040_base] extends = arduino_base -extra_scripts = post:scripts/check_firmware_ram.py +extra_scripts = pre:scripts/client_acl_role.py + post:scripts/check_firmware_ram.py upload_protocol = picotool board_build.core = earlephilhower platform = https://github.com/maxgerhardt/platform-raspberrypi.git ; framework-arduinopico @ 1.50600.0+sha.6a1d13e9 @@ -326,6 +329,7 @@ extends = arduino_base ; 20 / Arduino 3 changes those signatures; keep release builds reproducible. platform = ststm32@19.5.0 extra_scripts = post:arch/stm32/build_hex.py + pre:scripts/client_acl_role.py post:scripts/check_firmware_ram.py build_flags = ${arduino_base.build_flags} -D STM32_PLATFORM diff --git a/scripts/check_firmware_ram.py b/scripts/check_firmware_ram.py index ea51f1405..95c7acb49 100644 --- a/scripts/check_firmware_ram.py +++ b/scripts/check_firmware_ram.py @@ -48,7 +48,9 @@ def integer(defines, name, default): def requirements(platform, defines, target): if platform not in SUPPORTED: raise ValueError(f"no memory policy for {platform}") - companion = bool(re.search(r"companion|comp_radio|comp_.*radio", target, re.I)) + qualified_repeater = integer(defines, "MESH_CLIENT_REPEATER_ONLY", 0) != 0 + repeater = qualified_repeater or "repeater" in target.lower() + companion = not qualified_repeater and bool(re.search(r"companion|comp_radio|comp_.*radio", target, re.I)) full = "COMPANION_RADIO_FULL" in defines or "companion_radio_full" in target.lower() display = str(defines.get("DISPLAY_CLASS", "")).strip('"') if display == "SCIndicatorDisplay": @@ -97,6 +99,8 @@ def requirements(platform, defines, target): # 4 KiB for the SDK UART task, buffers and allocator overhead. parts["uart_bridge_and_driver"] = 8192 if "ENABLE_OTA" in defines: + # Existing own-image/manual-source allowance is independent of + # the generic folder leaf/output buffers counted below. parts["ota_source_scratch"] = 8192 if ("WEBCONFIG_DISABLED" not in defines and ((companion and "WIFI_SSID" in defines) @@ -106,8 +110,15 @@ def requirements(platform, defines, target): # Larger replies grow on demand only while preserving 32 KiB # of free internal heap, and shrink after the browser reads. parts["browser_terminal_scrollback"] = 4096 - if not companion: - parts["neighbor_history"] = integer(defines, "MAX_RECENT_REPEATERS", 50) * 12 + if repeater: + # Match MyMesh's platform defaults. The post-hook supplies the + # MCU-specific classic ESP32 default before sdkconfig is included. + default_recent = integer(defines, "MESH_RAM_DEFAULT_RECENT_REPEATERS", + 256 if "CONFIG_IDF_TARGET_ESP32" in defines else 2048) + count = integer(defines, "MAX_RECENT_REPEATERS", default_recent) + if not integer(defines, "MESH_ENABLE_RECENT_REPEATERS", 1): + count = integer(defines, "MAX_RECENT_REPEATERS", 0) + parts["neighbor_history"] = count * 9 + (16 if count else 0) else: parts["core_filesystems_sensors"] = 8192 # Packet bytes plus all three queue tables and allocation overhead. @@ -115,19 +126,30 @@ def requirements(platform, defines, target): # These tables moved out of .bss, so linker heap bounds now include their # space. Count it as startup allocation instead; C++ assertions bind the # per-entry bounds to the production structures. - if re.search(r"repeater|room_server|sensor", target, re.I) or "COMPANION_MESH_CLOCK_SYNC" in defines: - parts["client_table"] = integer(defines, "MAX_CLIENTS", 32) * 320 + 16 - if "repeater" in target.lower(): + if qualified_repeater or re.search(r"repeater|room_server|sensor", target, re.I) or "COMPANION_MESH_CLOCK_SYNC" in defines: + client_bytes = 284 if integer(defines, "MESH_CLIENT_REPEATER_ONLY", 0) else 320 + parts["client_table"] = integer(defines, "MAX_CLIENTS", 32) * client_bytes + 16 + if repeater: engine = integer(defines, "MESH_ENABLE_FLOOD_RULE_ENGINE", int(platform != "STM32_PLATFORM")) slots = integer(defines, "FLOOD_PACKET_FILTER_SLOTS", 63 if engine else 16) - parts["flood_filter_table"] = slots * (200 if engine else 40) + 16 + parts["flood_filter_table"] = slots * (192 if engine else 40) + 16 if "room_server" in target.lower() and integer( defines, "MESH_ENABLE_ROOM_FLOOD_RULE_ENGINE", 0): # FloodRuleEngine keeps all 31 persisted rules in one startup heap - # allocation. Its C++ assertion pins Entry to the budgeted 200 bytes. - parts["room_flood_rule_table"] = 31 * 200 + 16 + # allocation. Its C++ assertion pins Entry to the budgeted 192 bytes. + parts["room_flood_rule_table"] = 31 * 192 + 16 if "ENABLE_OTA" in defines and "OTA_HEAP_CONTEXT" in defines: parts["ota_context"] = 16384 + 16 + if "ENABLE_OTA" in defines: + # These are lazy on every platform. Freeing static arrays increases + # linked heap capacity but cannot erase their active-operation cost. + # Leaves are sized to the selected source, up to the unchanged limit; + # output holds only one block. They can overlap self-serving/fetching. + leaf_limit = integer(defines, "OTA_PROOFGEN_SCRATCH", + 16384 if platform == "ESP32_PLATFORM" else + 8192 if "OTA_SD_STORE" in defines else 4096) + block_size = integer(defines, "OTA_MAX_BLOCK", 2048) + parts["ota_generic_source_buffers"] = leaf_limit + block_size + 32 # Match OtaDeflateConfig.h's compile eligibility and build-role override. # Adaptive RAK may qualify only at runtime; reserve its worst eligible case. # The shared role hook qualifies sources, not environment names. An @@ -144,12 +166,13 @@ def requirements(platform, defines, target): # The encoder uses input offsets, not pointers: sizeof(uint16_t) per # bucket on every platform. This allocation is freed before radio TX. parts["ota_encoder_workspace"] = (1 << hash_bits) * 2 + 16 - if platform == "NRF52_PLATFORM": - # The running nRF52840 app is <1 MiB: <=512 2-KiB leaves. Self-serve - # holds its leaf cache plus max(leaves,2048) proof/output scratch. - # Unlike ESP32's existing source scratch reserve these allocations - # were formerly covered only by the general transient margin. - parts["ota_self_source_scratch"] = 2 * 2048 + 2 * 16 + if ("ENABLE_OTA" in defines and platform == "NRF52_PLATFORM" + and "OTA_SEEDER_ONLY" not in defines and any(name in defines for name in ( + "OTA_QSPI_STORE", "OTA_SD_STORE", "OTA_RAK_AUTO_STORE", "OTA_TOWER_AUTO_STORE"))): + # Own-image leaves/output also exist without a device encoder. The + # running nRF52840 app is <1 MiB: <=512 2-KiB leaves. Compression needs + # one block; raw diagnostic serving retains a leaf-sized output. + parts["ota_self_source_scratch"] = 2 * 2048 + 2 * 16 if combined_rak_ethernet: words = integer(defines, "RAK4631_ETHERNET_TASK_STACK_WORDS", 1024) if words < 1024: @@ -170,7 +193,7 @@ def requirements(platform, defines, target): # allocated only while Ethernet is enabled; reserve its capacity plus # 64 bytes for allocator overhead alongside the worker stack and TCB. external_ota = sum(parts.pop(name, 0) for name in ( - "ota_encoder_workspace", "ota_self_source_scratch")) + "ota_encoder_workspace", "ota_self_source_scratch", "ota_generic_source_buffers")) parts["ethernet_or_external_ota_workspaces"] = max(words * 4 + 256 + tx_capacity + 64, external_ota) if display and display != "NullDisplayDriver": @@ -203,9 +226,9 @@ def requirements(platform, defines, target): largest = max(display_heap, parts["radio_packet_pool"], 8192 if platform == "ESP32_PLATFORM" else 0) if "expanded_message_previews" in parts: largest = max(largest, 8192 + parts["expanded_message_previews"]) - for name in ("client_table", "flood_filter_table", "room_flood_rule_table", + for name in ("client_table", "flood_filter_table", "room_flood_rule_table", "neighbor_history", "ota_context", "screen_objects_and_history", "ota_encoder_workspace", - "ethernet_or_external_ota_workspaces"): + "ota_generic_source_buffers", "ethernet_or_external_ota_workspaces"): largest = max(largest, parts.get(name, 0)) return {"required_heap_bytes": required, "required_contiguous_bytes": largest, "components": parts, "display": display, "full_companion": full} @@ -304,6 +327,8 @@ def heap_regions(elf, platform, mcu): def check_firmware(elf_path, platform, mcu, defines, target, output=None): elf = FirmwareElf(elf_path) + if platform == "ESP32_PLATFORM": + defines = {**defines, "MESH_RAM_DEFAULT_RECENT_REPEATERS": 256 if str(mcu).lower() == "esp32" else 2048} policy = requirements(platform, defines, target) regions = heap_regions(elf, platform, mcu) available = sum(end - start for start, end, _ in regions) diff --git a/scripts/client_acl_role.py b/scripts/client_acl_role.py new file mode 100644 index 000000000..e7dc98786 --- /dev/null +++ b/scripts/client_acl_role.py @@ -0,0 +1,55 @@ +"""Use the smaller ClientInfo layout only for an effective repeater source role. + +One shared compiler definition covers application and helper translation units. +Unknown/custom roles keep the complete layout; room and sensor session fields +are never removed from builds that use them. +""" + +import importlib.util +from pathlib import Path + + +SCRIPT_DIR = Path(__file__).resolve().parent if "__file__" in globals() else Path("scripts").resolve() +SPEC = importlib.util.spec_from_file_location("client_acl_source_policy", SCRIPT_DIR / "nrf52_flash_trim.py") +SOURCE_POLICY = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(SOURCE_POLICY) + +DEFINITION = "MESH_CLIENT_REPEATER_ONLY" +REPEATER_SOURCES = ("../examples/simple_repeater/main.cpp", "../examples/simple_repeater/MyMesh.cpp") +OTHER_ROLES = ("simple_room_server", "simple_sensor", "companion_radio", "simple_secure_chat", + "kiss_modem", "partition_expander", "esp32_partition_migrator", "esp32_partition_legacy_seed") + + +def role_policy(src_filter, flags=(), cppdefines=()): + values = SOURCE_POLICY.flag_defines(flags, cppdefines) + selected = SOURCE_POLICY.source_selected + qualified = (all(selected(src_filter, source) for source in REPEATER_SOURCES) + and not any(selected(src_filter, "../examples/" + role + "/main.cpp") + or selected(src_filter, "../examples/" + role + "/MyMesh.cpp") + or selected(src_filter, "../examples/" + role + "/SensorMesh.cpp") + for role in OTHER_ROLES) + and not any(name.startswith("COMPANION_") for name in values)) + if DEFINITION in values: + value = values[DEFINITION] + text = "0" if value is None else str(value).strip("\"'() ") + if text not in ("0", "1"): + raise ValueError(DEFINITION + " override must be 0 or 1") + if text == "1" and not qualified: + raise ValueError(DEFINITION + "=1 requires only the repeater source role") + return {} + return {DEFINITION: 1} if qualified else {} + + +def install(build_env): + src_filter = build_env.get("SRC_FILTER") + if src_filter is None: + src_filter = build_env.GetProjectOption("build_src_filter", "") + definitions = role_policy(src_filter, build_env.get("BUILD_FLAGS", ()), + build_env.get("CPPDEFINES", ())) + if definitions: + build_env.AppendUnique(CPPDEFINES=list(definitions.items())) + + +if "Import" in globals(): + Import("env") + install(env) diff --git a/src/helpers/ClientACL.h b/src/helpers/ClientACL.h index 7dae15c75..d32c3ea0d 100644 --- a/src/helpers/ClientACL.h +++ b/src/helpers/ClientACL.h @@ -16,6 +16,16 @@ #define OUT_PATH_FORCE_FLOOD 0xFE #define OUT_PATH_UNKNOWN 0xFF +// The source-role pre-script sets this for the complete build, including +// ClientACL.cpp. Never infer it from a local MyMesh header: that would give +// different translation units incompatible ClientInfo layouts. +#ifndef MESH_CLIENT_REPEATER_ONLY +#define MESH_CLIENT_REPEATER_ONLY 0 +#endif +#if MESH_CLIENT_REPEATER_ONLY != 0 && MESH_CLIENT_REPEATER_ONLY != 1 +#error "MESH_CLIENT_REPEATER_ONLY must be 0 or 1" +#endif + struct ClientInfo { mesh::Identity id; uint8_t permissions; @@ -34,12 +44,15 @@ struct ClientInfo { union { struct { uint32_t sync_since; // sync messages SINCE this timestamp (by OUR clock) +#if !MESH_CLIENT_REPEATER_ONLY uint32_t last_post_timestamp; // sender timestamp for room posts only (transient) uint32_t pending_ack; uint32_t push_post_timestamp; unsigned long ack_timeout; uint8_t push_failures; +#endif } room; +#if !MESH_CLIENT_REPEATER_ONLY struct { uint32_t expiry_timestamp; // epoch seconds uint32_t push_tag; @@ -48,6 +61,7 @@ struct ClientInfo { uint8_t min_deltas[14]; // LPP encoded uint8_t prev_telem[14]; // LPP encoded } sensor; +#endif } extra; bool isAdmin() const { return (permissions & PERM_ACL_ROLE_MASK) == PERM_ACL_ADMIN; } @@ -60,7 +74,7 @@ struct ClientInfo { #define MAX_CLIENTS 32 #endif -static_assert(sizeof(void*) != 4 || sizeof(ClientInfo) <= 320, +static_assert(sizeof(void*) != 4 || sizeof(ClientInfo) <= (MESH_CLIENT_REPEATER_ONLY ? 284 : 320), "Update the client-table runtime RAM budget in check_firmware_ram.py"); struct ClientLoginReplayClampResult { diff --git a/src/helpers/FloodFilterPolicy.h b/src/helpers/FloodFilterPolicy.h index 780a27dd2..4135d5b2e 100644 --- a/src/helpers/FloodFilterPolicy.h +++ b/src/helpers/FloodFilterPolicy.h @@ -49,6 +49,41 @@ enum ChannelScopeGate { CHANNEL_SCOPE_REQUIRED_REJECTED, }; +// Rules can be reordered/compacted in RAM without changing their identity. +// Compare configured fields explicitly: padding and live rate-window state +// must never decide whether an unnumbered setter reuses an existing slot. +template +bool sameRuleConfiguration(const Entry& left, const Entry& right) { + return left.active == right.active + && left.payload_type == right.payload_type + && left.min_hops == right.min_hops + && left.max_hops == right.max_hops + && left.suspend_on_temp_radio == right.suspend_on_temp_radio + && memcmp(left.scope_name, right.scope_name, sizeof(left.scope_name)) == 0 + && left.match_blacklisted_path == right.match_blacklisted_path + && left.scope_uses_slow_timing == right.scope_uses_slow_timing + && left.incoming_scope_kind == right.incoming_scope_kind + && memcmp(left.incoming_scope_name, right.incoming_scope_name, + sizeof(left.incoming_scope_name)) == 0 + && left.channel_key_len == right.channel_key_len + && left.channel_hash == right.channel_hash + && memcmp(left.channel_secret, right.channel_secret, + sizeof(left.channel_secret)) == 0 + && memcmp(left.channel_name, right.channel_name, sizeof(left.channel_name)) == 0 + && left.path_hash_size == right.path_hash_size + && left.path_hops == right.path_hops + && memcmp(left.path, right.path, sizeof(left.path)) == 0 + && memcmp(left.target_region_name, right.target_region_name, + sizeof(left.target_region_name)) == 0 + && left.drop_on_match == right.drop_on_match + && left.rate_limit_enabled == right.rate_limit_enabled + && left.rate_per_minute == right.rate_per_minute + && left.priority == right.priority + && left.stop_on_match == right.stop_on_match + && left.retry_on_match == right.retry_on_match + && left.transport_modes == right.transport_modes; +} + inline bool channelKeyLengthSupported(uint8_t key_len) { return key_len == 0 || key_len == CHANNEL_HASH_ONLY_LEN || key_len == CHANNEL_KEY_128_LEN diff --git a/src/helpers/SimpleMeshTables.h b/src/helpers/SimpleMeshTables.h index fc3c5435a..772c3ea9c 100644 --- a/src/helpers/SimpleMeshTables.h +++ b/src/helpers/SimpleMeshTables.h @@ -1,6 +1,7 @@ #pragma once #include +#include #if ARDUINO #include #endif @@ -29,6 +30,17 @@ #define ACK_VALID_BYTES ((MAX_PACKET_ACKS + 7) / 8) #define MAX_ROUTE_HASH_BYTES 3 +inline int routeHashPrefixBytesCompare(const uint8_t* a, const uint8_t* b, + uint8_t length) { + const volatile uint8_t* left = a; + const volatile uint8_t* right = b; + for (uint8_t i = 0; i < length; ++i) { + const uint8_t av = left[i], bv = right[i]; + if (av != bv) return (int)av - (int)bv; + } + return 0; +} + inline bool routeHashPrefixesOverlap(const uint8_t* a, uint8_t a_len, const uint8_t* b, uint8_t b_len) { if (a == NULL || b == NULL || a_len == 0 || b_len == 0 @@ -36,18 +48,72 @@ inline bool routeHashPrefixesOverlap(const uint8_t* a, uint8_t a_len, return false; } uint8_t compare_len = a_len < b_len ? a_len : b_len; - return memcmp(a, b, compare_len) == 0; + return routeHashPrefixBytesCompare(a, b, compare_len) == 0; } class SimpleMeshTables : public mesh::MeshTables { public: + // Recent rows belong to the mesh loop, including CLI requests marshaled from + // network tasks. They must not be read or changed concurrently by an ISR or + // worker. Byte access avoids unaligned uint32_t loads in the compact rows; + // volatile here constrains access width, not synchronization or atomicity. + struct RecentRepeaterMillis { + uint8_t bytes[4]; + + operator uint32_t() const { + const volatile uint8_t* value = bytes; + return (uint32_t)value[0] + | ((uint32_t)value[1] << 8) + | ((uint32_t)value[2] << 16) + | ((uint32_t)value[3] << 24); + } + + RecentRepeaterMillis& operator=(uint32_t millis) { + volatile uint8_t* value = bytes; + value[0] = (uint8_t)millis; + value[1] = (uint8_t)(millis >> 8); + value[2] = (uint8_t)(millis >> 16); + value[3] = (uint8_t)(millis >> 24); + return *this; + } + }; + struct RecentRepeaterInfo { // Identity and link quality for a next-hop path prefix. uint8_t prefix[MAX_ROUTE_HASH_BYTES]; - uint8_t prefix_len; - int8_t snr_x4; - uint32_t last_heard_millis; + volatile uint8_t prefix_len; + volatile int8_t snr_x4; + RecentRepeaterMillis last_heard_millis; }; + static_assert(sizeof(RecentRepeaterMillis) == 4 && alignof(RecentRepeaterMillis) == 1, + "Recent timestamps must retain all 32 bits without alignment padding"); + static_assert(sizeof(RecentRepeaterInfo) == 9 && alignof(RecentRepeaterInfo) == 1, + "Recent rows must not add padding to the externally allocated table"); + static_assert(std::is_trivially_copyable::value, + "Recent rows must support memset, compaction, and cursor copies"); + + // Use these helpers instead of implicit row copies or memcpy/memset. M4 + // compilers may otherwise widen byte-aligned aggregate operations, which + // faults when unaligned accesses trap. Keep every compact-row access a byte, + // even when LTO can see the length or a concrete source/destination address. + static void copyRecentRepeaterInfo(RecentRepeaterInfo& destination, + const RecentRepeaterInfo& source) { + volatile uint8_t* out = reinterpret_cast(&destination); + const volatile uint8_t* in = reinterpret_cast(&source); + for (size_t i = 0; i < sizeof(RecentRepeaterInfo); ++i) out[i] = in[i]; + } + + static void clearRecentRepeaterInfo(RecentRepeaterInfo& row) { + volatile uint8_t* out = reinterpret_cast(&row); + for (size_t i = 0; i < sizeof(RecentRepeaterInfo); ++i) out[i] = 0; + } + + static void copyRecentRepeaterPrefix(uint8_t* destination, + const RecentRepeaterInfo& source) { + volatile uint8_t* out = destination; + const volatile uint8_t* in = source.prefix; + for (size_t i = 0; i < MAX_ROUTE_HASH_BYTES; ++i) out[i] = in[i]; + } private: uint8_t _hashes[MAX_PACKET_HASHES*MAX_HASH_SIZE]; @@ -182,7 +248,7 @@ private: if (a.snr_x4 != b.snr_x4) { return a.snr_x4 > b.snr_x4; // Highest SNR first within each prefix length. } - int cmp = memcmp(a.prefix, b.prefix, a.prefix_len); + int cmp = routeHashPrefixBytesCompare(a.prefix, b.prefix, a.prefix_len); if (cmp != 0) { return cmp < 0; } @@ -261,9 +327,7 @@ public: memset(_ack_valid, 0, sizeof(_ack_valid)); _next_ack_idx = 0; _direct_dups = _flood_dups = 0; - if (_max_recent_repeaters > 0) { - memset(_recent_repeaters, 0, _max_recent_repeaters * sizeof(RecentRepeaterInfo)); - } + clearRecentRepeaters(); } #ifdef ESP32 @@ -398,7 +462,8 @@ public: // path prefix does not collapse 2/3-byte repeaters sharing its first byte. for (int i = 0; i < _recent_repeater_count; i++) { RecentRepeaterInfo& existing = _recent_repeaters[i]; - if (existing.prefix_len != prefix_len || memcmp(existing.prefix, prefix, prefix_len) != 0) { + if (existing.prefix_len != prefix_len + || routeHashPrefixBytesCompare(existing.prefix, prefix, prefix_len) != 0) { #if ARDUINO uint32_t age = (uint32_t)(now - existing.last_heard_millis); if (!have_oldest || age > oldest_age) { @@ -428,8 +493,11 @@ public: } RecentRepeaterInfo& slot = _recent_repeaters[slot_idx]; - memset(slot.prefix, 0, sizeof(slot.prefix)); - memcpy(slot.prefix, prefix, prefix_len); + volatile uint8_t* out_prefix = slot.prefix; + const volatile uint8_t* in_prefix = prefix; + for (uint8_t i = 0; i < MAX_ROUTE_HASH_BYTES; ++i) { + out_prefix[i] = i < prefix_len ? in_prefix[i] : 0; + } slot.prefix_len = prefix_len; slot.snr_x4 = snr_x4; #if ARDUINO @@ -467,9 +535,10 @@ public: } // Advance a cooperative live view in one bounded table scan. Unlike a // rank lookup, this never replays every earlier rank as the list grows. - // The caller copies the last accepted row: intervening RX may change the - // live ordering, but cannot invalidate that saved sort key or force a - // snapshot allocation. The existing paginated/rank APIs are unchanged. + // The caller uses copyRecentRepeaterInfo for the last accepted row: + // intervening RX may change the live ordering, but cannot invalidate that + // saved sort key or force a snapshot allocation. The existing paginated/rank + // APIs are unchanged. const RecentRepeaterInfo* getNextRecentRepeaterBySortKey( const RecentRepeaterInfo* after, int after_index, int& result_index) const { const RecentRepeaterInfo* best = NULL; @@ -523,7 +592,8 @@ public: const RecentRepeaterInfo* best = NULL; for (int i = 0; i < _recent_repeater_count; i++) { const RecentRepeaterInfo* info = &_recent_repeaters[i]; - if (info->prefix_len == hash_len && memcmp(info->prefix, hash, hash_len) == 0) { + if (info->prefix_len == hash_len + && routeHashPrefixBytesCompare(info->prefix, hash, hash_len) == 0) { return info; } if (routeHashPrefixesOverlap(info->prefix, info->prefix_len, hash, hash_len)) { @@ -536,9 +606,7 @@ public: return best; } void clearRecentRepeaters() { - if (_max_recent_repeaters > 0) { - memset(_recent_repeaters, 0, _max_recent_repeaters * sizeof(RecentRepeaterInfo)); - } + for (int i = 0; i < _max_recent_repeaters; ++i) clearRecentRepeaterInfo(_recent_repeaters[i]); _recent_repeater_count = 0; } int expireRecentRepeaters(uint32_t now_millis, uint32_t max_age_millis) { @@ -552,9 +620,9 @@ public: if ((uint32_t)(now_millis - info.last_heard_millis) > max_age_millis) { _recent_repeater_count--; if (i != _recent_repeater_count) { - info = _recent_repeaters[_recent_repeater_count]; + copyRecentRepeaterInfo(info, _recent_repeaters[_recent_repeater_count]); } - memset(&_recent_repeaters[_recent_repeater_count], 0, sizeof(RecentRepeaterInfo)); + clearRecentRepeaterInfo(_recent_repeaters[_recent_repeater_count]); expired++; } else { i++; diff --git a/src/helpers/ota/MerkleTree.cpp b/src/helpers/ota/MerkleTree.cpp index 58c126faf..f7b6cb420 100644 --- a/src/helpers/ota/MerkleTree.cpp +++ b/src/helpers/ota/MerkleTree.cpp @@ -105,26 +105,29 @@ bool merkle_verify_from_leaf(const uint8_t leaf[4], uint32_t index, uint8_t merkle_gen_proof(const uint8_t* leaves, uint32_t count, uint32_t index, uint8_t* scratch, uint8_t* out_siblings) { - if (count == 0 || index >= count) return 0; - memcpy(scratch, leaves, (size_t)count * 4); + (void)scratch; + if (!leaves || !out_siblings || count == 0 || index >= count) return 0; uint32_t n = count, idx = index; + uint64_t width = 1; uint8_t p = 0; + MerkleAccumulator accumulator; while (n > 1) { bool is_last_odd = (n & 1u) && (idx == n - 1); if (!is_last_odd) { uint32_t s = (idx & 1u) ? idx - 1 : idx + 1; - memcpy(out_siblings + (size_t)p * 4, scratch + (size_t)s * 4, 4); + const uint64_t first = (uint64_t)s * width; + uint64_t end = first + width; + if (end > count) end = count; + accumulator.reset(); + for (uint64_t leaf = first; leaf < end; ++leaf) { + if (!accumulator.add(leaves + (size_t)leaf * 4)) return 0; + } + if (!accumulator.finish(out_siblings + (size_t)p * 4)) return 0; p++; } - // reduce one level in place (parent m written from children 2m,2m+1; m <= i so it's safe) - uint32_t m = 0; - for (uint32_t i = 0; i < n; i += 2) { - if (i + 1 < n) merkle_combine(scratch + (size_t)m * 4, scratch + (size_t)i * 4, scratch + (size_t)(i + 1) * 4); - else memmove(scratch + (size_t)m * 4, scratch + (size_t)i * 4, 4); - m++; - } idx >>= 1; - n = (n + 1) >> 1; + n = (n >> 1) + (n & 1u); + width <<= 1; } return p; } diff --git a/src/helpers/ota/MerkleTree.h b/src/helpers/ota/MerkleTree.h index 122790358..c24c1ad17 100644 --- a/src/helpers/ota/MerkleTree.h +++ b/src/helpers/ota/MerkleTree.h @@ -55,7 +55,9 @@ bool merkle_verify_from_leaf(const uint8_t leaf[4], uint32_t index, const uint8_t root[4], uint32_t count); // Generate the proof (ordered sibling digests) for block `index`, for a server holding leaves[]. -// `scratch` must be >= count*4 bytes (working buffer); `out_siblings` >= 32*4 bytes. +// Sibling subtrees are reduced with O(log count) storage. `scratch` is retained +// for source compatibility but is unused and may be nullptr; `out_siblings` +// must hold 32*4 bytes. Input leaves are never modified. // Returns the number of 4-byte siblings written. Output matches the wire form merkle_verify expects. uint8_t merkle_gen_proof(const uint8_t* leaves, uint32_t count, uint32_t index, uint8_t* scratch, uint8_t* out_siblings); diff --git a/src/helpers/ota/OtaContext.h b/src/helpers/ota/OtaContext.h index 6864ea415..ae495180c 100644 --- a/src/helpers/ota/OtaContext.h +++ b/src/helpers/ota/OtaContext.h @@ -160,8 +160,8 @@ struct OtaContext { bool self_serve_supported = false; // flash-backed self-serve: cached merkle leaves (heap, freed on re-serve) + assembled manifest of our // own running firmware. The payload is read from flash per block; only the metadata is held in RAM. - // serve_self_proof is the proof-gen working buffer (>= block_count*4) - sized to OUR image's block - // count (the manager's fixed 4 KiB scratch covers <=1024 blocks, about 2 MiB at the new default). + // serve_self_proof is the encoded-DATA output buffer on encoder-enabled + // builds. Streaming Merkle proofs need no full copy of the leaf table. uint8_t* serve_self_leaves = nullptr; uint8_t* serve_self_proof = nullptr; diff --git a/src/helpers/ota/OtaManager.cpp b/src/helpers/ota/OtaManager.cpp index 80f05e643..d8164cf04 100644 --- a/src/helpers/ota/OtaManager.cpp +++ b/src/helpers/ota/OtaManager.cpp @@ -13,12 +13,8 @@ namespace ota { OtaManager::~OtaManager() { free(_catalog_heap); free(_leaves_buf); -#if defined(ESP32_PLATFORM) - // A shared Companion workspace is destroyed between mOTA sessions. Return - // the lazy source/proof buffers before its queue slots are reused. free(_src_leaves); free(_scratch); -#endif } bool OtaManager::set_speed(float speed) { @@ -61,24 +57,56 @@ bool OtaManager::expandCatalog() { return true; } -uint8_t* OtaManager::ensureSourceLeaves() { -#if defined(ESP32_PLATFORM) - if (!_src_leaves) { - _src_leaves = static_cast(malloc(OTA_PROOFGEN_SCRATCH)); +uint8_t* OtaManager::ensureSourceLeaves(uint32_t bytes) { + if (!bytes || bytes > OTA_PROOFGEN_SCRATCH) return nullptr; + if (_src_leaves_capacity < bytes) { + // loadSource revoked the cached view first. Free before growing, so even + // an allocation failure cannot overlap two images' maximum leaf caches. + free(_src_leaves); + _src_leaves = nullptr; + _src_leaves_capacity = 0; + _src_leaves = static_cast(malloc(bytes)); + if (_src_leaves) _src_leaves_capacity = bytes; } -#endif return _src_leaves; } uint8_t* OtaManager::ensureScratch() { -#if defined(ESP32_PLATFORM) if (!_scratch) { - _scratch = static_cast(malloc(OTA_PROOFGEN_SCRATCH)); + _scratch = static_cast(malloc(OTA_MAX_BLOCK)); } -#endif return _scratch; } +void OtaManager::releaseColdBuffers() { + // Both queues pin their selected source and any encoded DATA behind radio + // backpressure. No cached view or wire pointer may outlive its allocation. + if (_n_serve_jobs || _n_manifest_jobs) return; + _srcv.valid = false; + _srcv.m.leaves = nullptr; + _srcv.scratch = nullptr; + _srcv.scratch_sz = 0; + if (_view0.scratch == _scratch) { + _view0.scratch = nullptr; + _view0.scratch_sz = 0; + } + _serve_block_loaded = false; + _serve_wire_block = nullptr; + _serve_wire_len = 0; + free(_src_leaves); + _src_leaves = nullptr; + _src_leaves_capacity = 0; + free(_scratch); + _scratch = nullptr; +} + +void OtaManager::releaseIdleColdBuffers() { + if ((_src_leaves || _scratch) && + (uint32_t)(_now_ms - _source_buffer_used_at) >= OTA_SOURCE_BUFFER_IDLE_MS) { + releaseColdBuffers(); + } +} + void OtaManager::begin(uint32_t my_target_id, OtaSend send, void* ctx) { _target = my_target_id; _send = send; _ctx = ctx; _migration_target = 0; @@ -94,6 +122,7 @@ void OtaManager::begin(uint32_t my_target_id, OtaSend send, void* ctx) { memset(_src_advertised, 0, sizeof(_src_advertised)); _n_src = 0; _n_cat = 0; clearPendingEgress(); + releaseColdBuffers(); _adaptive_packet_speed = 1.0f; _pace_clean_blocks = 0; _pace_has_mid = false; @@ -112,17 +141,16 @@ bool OtaManager::serve(const uint8_t* mota, uint32_t len) { if (!mota || !mota_parse(mota, len, parsed)) return false; if (parsed.block_size() == 0 || parsed.block_size() > OTA_MAX_BLOCK || (uint64_t)parsed.block_count * 4 > OTA_PROOFGEN_SCRATCH) return false; - uint8_t* scratch = ensureScratch(); - if (!scratch) return false; // Parsing can fail after writing fields and pointers. Publish the replacement // only after all checks pass, preserving the old image and its queued replies. _view0.m = parsed; _view0.mfl = (uint16_t)(_view0.m.leaves - _view0.m.manifest_start); // contiguous container _view0.read = nullptr; _view0.read_ctx = nullptr; // payload is contiguous _view0.m.payload _view0.read_deflated = nullptr; _view0.deflate_ctx = nullptr; - _view0.scratch = scratch; _view0.scratch_sz = OTA_PROOFGEN_SCRATCH; // <=1024 leaves/blocks + _view0.scratch = nullptr; _view0.scratch_sz = 0; // allocate only if compressed DATA is requested _view0.valid = true; clearPendingEgress(); + releaseColdBuffers(); if (_n_src_obj) refresh_sources(); else registerSelfEntry(); return true; } @@ -131,7 +159,7 @@ bool OtaManager::serve_self(const uint8_t* manifest, uint16_t mfl, const uint8_t uint32_t block_count, uint8_t* proof_scratch, uint32_t proof_scratch_sz, ServeReadFn read, void* ctx) { if (!manifest || mfl != MOTA_MFL || !leaves || !proof_scratch || !read) return false; - if (proof_scratch_sz < (uint64_t)block_count * 4) return false; // proof-gen needs count*4 working bytes + if (proof_scratch_sz == 0) return false; MotaManifest parsed; if (!mota_parse_manifest(manifest, mfl, parsed)) return false; // fixed fields: root, image_hash, sizes if (parsed.block_size() == 0 || parsed.block_size() > OTA_MAX_BLOCK || @@ -143,9 +171,10 @@ bool OtaManager::serve_self(const uint8_t* manifest, uint16_t mfl, const uint8_t _view0.m.block_count = block_count; _view0.mfl = mfl; _view0.read = read; _view0.read_ctx = ctx; _view0.read_deflated = nullptr; _view0.deflate_ctx = nullptr; - _view0.scratch = proof_scratch; _view0.scratch_sz = proof_scratch_sz; // sized for our (large) image + _view0.scratch = proof_scratch; _view0.scratch_sz = proof_scratch_sz; // optional compression output _view0.valid = true; clearPendingEgress(); + releaseColdBuffers(); if (_n_src_obj) refresh_sources(); else registerSelfEntry(); return true; } @@ -203,6 +232,7 @@ bool OtaManager::remove_source(MotaSource* src) { void OtaManager::refresh_sources() { clearPendingEgress(); + releaseColdBuffers(); _n_serve = 0; if (_view0.valid) registerSelfEntry(); for (uint8_t s = 0; s < _n_src_obj; s++) { @@ -258,6 +288,7 @@ bool OtaManager::sourceStats(const MotaSource* src, uint16_t& offered, uint16_t& void OtaManager::clear_sources() { clearPendingEgress(); + releaseColdBuffers(); _n_src_obj = 0; _srcv.valid = false; memset(_src_list, 0, sizeof(_src_list)); memset(_src_offered, 0, sizeof(_src_offered)); @@ -269,6 +300,7 @@ void OtaManager::clear_sources() { void OtaManager::clear_primary() { clearPendingEgress(); _view0.valid = false; + releaseColdBuffers(); if (_n_src_obj) refresh_sources(); else _n_serve = 0; } @@ -281,7 +313,10 @@ int OtaManager::serveEntryIndex(const uint8_t* mid) const { OtaManager::ServeView* OtaManager::resolve(const uint8_t* mid) { if (_view0.valid && memcmp(mid, _view0.m.merkle_root, 4) == 0) return &_view0; - if (_srcv.valid && memcmp(mid, _srcv_mid, 4) == 0) return &_srcv; + if (_srcv.valid && memcmp(mid, _srcv_mid, 4) == 0) { + _source_buffer_used_at = _now_ms; + return &_srcv; + } int i = serveEntryIndex(mid); if (i < 0) return nullptr; if (_serve[i].is_self) return _view0.valid ? &_view0 : nullptr; @@ -299,12 +334,12 @@ bool OtaManager::loadSource(const ServeEntry& e) { uint16_t mfl = (uint16_t)(d.leaves_off - 8); if (mfl == 0 || mfl > sizeof(_src_manifest)) return false; if (d.block_count == 0 || (uint64_t)d.block_count * 4 > OTA_PROOFGEN_SCRATCH) return false; - uint8_t* src_leaves = ensureSourceLeaves(); - uint8_t* scratch = ensureScratch(); - if (!src_leaves || !scratch) return false; // Both images share these buffers. Even a failed read can overwrite part of // the old manifest/leaves, so its cached MID must stop resolving immediately. _srcv.valid = false; + uint8_t* src_leaves = ensureSourceLeaves(d.block_count * 4u); + if (!src_leaves) return false; + _source_buffer_used_at = _now_ms; bool ok = e.src->read(e.src_idx, 8, _src_manifest, mfl); if (!ok || !mota_parse_manifest(_src_manifest, mfl, _srcv.m)) return false; if (_srcv.m.block_size() == 0 || _srcv.m.block_size() > OTA_MAX_BLOCK) return false; @@ -322,7 +357,7 @@ bool OtaManager::loadSource(const ServeEntry& e) { _srcv.read_deflated = (d.source_caps & MOTA_SOURCE_CAP_DEFLATE_BLOCK) ? srcDeflateReadTramp : nullptr; _srcv.deflate_ctx = _srcv.read_deflated ? &_srcv_rdctx : nullptr; - _srcv.scratch = scratch; _srcv.scratch_sz = OTA_PROOFGEN_SCRATCH; + _srcv.scratch = nullptr; _srcv.scratch_sz = 0; memcpy(_srcv_mid, d.mid, 4); _srcv.valid = true; return true; @@ -562,6 +597,10 @@ void OtaManager::clearPendingEgress() { _serve_wire_len = 0; _serve_wire_deflated = false; memset(_serve_wire_id, 0, sizeof(_serve_wire_id)); + // Explicit cancellation also runs when the temporary OTA radio ends, when + // loop() is no longer called. Keep source descriptors advertised, but drop + // their reloadable working cache rather than pinning heap until next use. + releaseColdBuffers(); } void OtaManager::popManifestJob() { @@ -663,8 +702,13 @@ bool OtaManager::loadActiveServeBlock() { _serve_wire_deflated = false; // A host-backed MotaSource can provide a pre-compressed block without putting an encoder in firmware. - // Other sources may opt into a caller-supplied encoder. Both write into the proof scratch, which is safe - // until every DATA fragment has left; proof generation reuses it only after that point. + // Other sources may opt into a caller-supplied encoder. Encoded DATA remains + // pinned until every fragment has left. Streaming proofs need no output buffer. + if (job.wire_v2 && job.allow_deflate && (v->read_deflated || _deflate_encode) && !v->scratch) { + v->scratch = ensureScratch(); + v->scratch_sz = v->scratch ? OTA_MAX_BLOCK : 0; + } + _source_buffer_used_at = _now_ms; if (job.wire_v2 && job.allow_deflate && v->scratch && v->scratch_sz >= blen) { uint16_t encoded_len = 0; bool encoded = false; @@ -765,7 +809,6 @@ bool OtaManager::handleReqProof(const uint8_t* m, uint16_t n) { if (!decode_req_proof(m, n, rp)) return false; ServeView* v = resolve(rp.manifest_id); if (!v || rp.block_idx >= v->m.block_count) return false; - if ((uint64_t)v->m.block_count * 4 > v->scratch_sz) return false; return queueServeJob(v->m.merkle_root, rp.block_idx, 0); // proof-only, or merge with queued DATA } @@ -796,8 +839,7 @@ void OtaManager::serviceEgress(OtaReplyRouteValid route_valid, void* route_ctx) return; } ServeView* v = resolve(job.mid); - if (!v || job.block >= v->m.block_count - || (uint64_t)v->m.block_count * 4 > v->scratch_sz) { + if (!v || job.block >= v->m.block_count) { popServeJob(); return; } @@ -2179,6 +2221,7 @@ uint32_t OtaManager::pickMissingBlock() { } void OtaManager::loop() { + releaseIdleColdBuffers(); // Each source owns its own pending/retry state, so a burst of advertisements cannot overwrite another // seeder's query. Initial requests ask for all; recovery asks only for missing HAVE fragments. for (uint8_t i = 0; i < _n_src; i++) { diff --git a/src/helpers/ota/OtaManager.h b/src/helpers/ota/OtaManager.h index 3a19c92cb..90d0ff940 100644 --- a/src/helpers/ota/OtaManager.h +++ b/src/helpers/ota/OtaManager.h @@ -63,14 +63,20 @@ typedef bool (*ServeDeflateReadFn)(void* ctx, uint16_t block, uint8_t* dst, #ifndef OTA_PROOFGEN_SCRATCH #if defined(ESP32_PLATFORM) - #define OTA_PROOFGEN_SCRATCH 16384 // heap-backed; can seed <=4096 blocks (8 MiB at 2 KiB/block) + #define OTA_PROOFGEN_SCRATCH 16384 // maximum source leaves; <=4096 blocks (8 MiB at 2 KiB/block) #elif defined(OTA_SD_STORE) #define OTA_PROOFGEN_SCRATCH 8192 // SD archive can seed <=2048 blocks (about 4 MiB at 2 KiB/block) #else - #define OTA_PROOFGEN_SCRATCH 4096 // server proof-gen working buffer (supports up to 1024 blocks) + #define OTA_PROOFGEN_SCRATCH 4096 // maximum source leaves (supports up to 1024 blocks) #endif #endif +#ifndef OTA_SOURCE_BUFFER_IDLE_MS +#define OTA_SOURCE_BUFFER_IDLE_MS 30000u // retain an unpinned source cache briefly between requests +#endif +static_assert(OTA_SOURCE_BUFFER_IDLE_MS > 0u && OTA_SOURCE_BUFFER_IDLE_MS < 0x80000000u, + "OTA source-buffer idle interval must support wrap-safe elapsed time"); + #ifndef OTA_MAX_BLOCK #define OTA_MAX_BLOCK 2048 // largest logical block (Merkle leaf / independent DEFLATE unit) #endif @@ -291,7 +297,7 @@ public: void* read_ctx = nullptr; ServeDeflateReadFn read_deflated = nullptr; // optional host/source-generated raw-DEFLATE block void* deflate_ctx = nullptr; - uint8_t* scratch = nullptr; // proof-gen working buffer (>= block_count*4) + uint8_t* scratch = nullptr; // optional encoded DATA output, not Merkle proof storage uint32_t scratch_sz = 0; }; // A lightweight catalog entry: what we advertise per mota + how to load its ServeView on demand. @@ -686,15 +692,13 @@ private: uint16_t _src_advertised[OTA_MAX_SOURCE_OBJ] = {0}; uint8_t _n_src_obj = 0; uint8_t _src_manifest[OTA_SRC_MANIFEST_MAX]; // manifest-minus-leaves of the loaded source mota -#if defined(ESP32_PLATFORM) - // Folder-source leaves and proof scratch are cold-path working storage. Keep - // them off classic ESP32 .bss and allocate them on first use. + // Cold source leaves are sized to the selected image, within the unchanged + // capacity limit. Encoded DATA needs at most one logical block. These buffers + // remain pinned until queued egress drains, then return to the heap at idle. uint8_t* _src_leaves = nullptr; + uint32_t _src_leaves_capacity = 0; uint8_t* _scratch = nullptr; -#else - uint8_t _src_leaves[OTA_PROOFGEN_SCRATCH]; // leaves[] of the loaded source mota (<=1024 blocks) - uint8_t _scratch[OTA_PROOFGEN_SCRATCH]; // proof-gen / fetch root-check working buffer -#endif + uint32_t _source_buffer_used_at = 0; // Server-side response descriptors are tiny. The active logical block has its own buffer so proof generation // or a simultaneous fetch cannot overwrite DATA retained behind radio-queue backpressure. @@ -827,8 +831,10 @@ private: bool _diffing = false; // leaves are in; diffing the seed a batch per tick uint32_t _diff_idx = 0; // next block index to diff against the seed - uint8_t* ensureSourceLeaves(); + uint8_t* ensureSourceLeaves(uint32_t bytes); uint8_t* ensureScratch(); + void releaseColdBuffers(); + void releaseIdleColdBuffers(); // discovery: heard sources (beacon senders) + the catalog assembled from their OTA_HAVE replies struct Source { diff --git a/src/helpers/ota/OtaSelf.cpp b/src/helpers/ota/OtaSelf.cpp index aa89c64e8..de7817bd8 100644 --- a/src/helpers/ota/OtaSelf.cpp +++ b/src/helpers/ota/OtaSelf.cpp @@ -228,11 +228,10 @@ bool ota_serve_self(OtaContext& c, uint32_t fw_version) { c.serve_self_leaves = (uint8_t*)malloc((size_t)bc * 4); size_t proof_size = (size_t)bc * 4; #if MESHCORE_OTA_DEVICE_DEFLATE - // Proof generation and transport encoding use this buffer at different - // times. Small nRF52 images can have <2 KiB of leaves: keep enough room for - // a whole encoded block without allocating a second input/output buffer. - // Receiver-only/Companion diagnostic raw exports retain the old allocation. - if (proof_size < BS) proof_size = BS; + // Streaming proofs no longer need a copy of the leaf table. This buffer + // holds only one encoded DATA block, regardless of the firmware's size. + // Receiver-only/Companion diagnostic raw exports retain their allocation. + proof_size = BS; #endif c.serve_self_proof = (uint8_t*)malloc(proof_size); if (!c.serve_self_leaves || !c.serve_self_proof) { diff --git a/test/fixtures/client_acl_spiffs/test_client_acl_spiffs.cpp b/test/fixtures/client_acl_spiffs/test_client_acl_spiffs.cpp index 5353c37d0..211308468 100644 --- a/test/fixtures/client_acl_spiffs/test_client_acl_spiffs.cpp +++ b/test/fixtures/client_acl_spiffs/test_client_acl_spiffs.cpp @@ -585,4 +585,5 @@ int main() { std::printf("PASS: %s\n", test.name); } std::puts("27 ClientACL SPIFFS checks passed"); + return 0; } diff --git a/test/fixtures/ota_cold_buffers/test.cpp b/test/fixtures/ota_cold_buffers/test.cpp new file mode 100644 index 000000000..328a08d27 --- /dev/null +++ b/test/fixtures/ota_cold_buffers/test.cpp @@ -0,0 +1,359 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +using namespace mesh::ota; + +// Only production malloc/free calls are wrapped. C++ fixture containers use +// operator new, so the ledger measures the manager's actual cold allocations. +static std::array live{}; +static std::array sizes{}; +static size_t allocated = 0, peak = 0; +static bool fail_next = false; +extern "C" void* __real_malloc(size_t); +extern "C" void __real_free(void*); +extern "C" void* __wrap_malloc(size_t bytes) { + if (fail_next) { fail_next = false; return nullptr; } + void* result = __real_malloc(bytes); + if (result) { + auto empty = std::find(live.begin(), live.end(), nullptr); + assert(empty != live.end()); + const size_t slot = empty - live.begin(); + live[slot] = result; sizes[slot] = bytes; + allocated += bytes; peak = std::max(peak, allocated); + } + return result; +} +extern "C" void __wrap_free(void* pointer) { + if (pointer) { + auto found = std::find(live.begin(), live.end(), pointer); + if (found != live.end()) { + const size_t slot = found - live.begin(); + allocated -= sizes[slot]; live[slot] = nullptr; + } + } + __real_free(pointer); +} + +static std::vector reference_proof(std::vector level, uint32_t index) { + std::vector proof; + while (level.size() > 4) { + const uint32_t count = level.size() / 4; + if (!((count & 1u) && index == count - 1)) { + const uint32_t sibling = index ^ 1u; + proof.insert(proof.end(), level.begin() + sibling * 4, level.begin() + sibling * 4 + 4); + } + std::vector next(((count + 1) / 2) * 4); + for (uint32_t i = 0; i < count; i += 2) { + if (i + 1 < count) merkle_combine(next.data() + (i / 2) * 4, + level.data() + i * 4, level.data() + (i + 1) * 4); + else memcpy(next.data() + (i / 2) * 4, level.data() + i * 4, 4); + } + level = std::move(next); index >>= 1; + } + return proof; +} +static void proofs() { + std::vector counts; + for (uint32_t n = 1; n <= 65; ++n) counts.push_back(n); + for (uint32_t n : {127,128,129,255,256,257,511,512,513,1023,1024,1025,2047,2048,4095,4096}) + counts.push_back(n); + for (uint32_t count : counts) { + std::vector leaves(count * 4); + for (uint32_t i = 0; i < count; ++i) { + uint8_t value[4]; wr_u32le(value, i * 8191u + count); + merkle_leaf(leaves.data() + i * 4, value, 4); + } + const auto original = leaves; + uint8_t root[4]; merkle_root(root, leaves.data(), count); + const uint32_t step = count <= 65 ? 1 : std::max(1u, count / 7); + for (uint32_t i = 0; i < count; i += step) { + uint8_t output[128], tiny_scratch = 0x5A; + const uint8_t length = merkle_gen_proof(leaves.data(), count, i, nullptr, output); + const auto expected = reference_proof(leaves, i); + assert(expected.size() == length * 4u); + assert(expected.empty() || !memcmp(output, expected.data(), expected.size())); + assert(merkle_verify_from_leaf(leaves.data() + i * 4, i, output, length, root, count)); + assert(merkle_gen_proof(leaves.data(), count, i, &tiny_scratch, output) == length); + assert(tiny_scratch == 0x5A && leaves == original); + } + } + uint8_t output[128]{}; + assert(!merkle_gen_proof(nullptr, 1, 0, nullptr, output)); + assert(!merkle_gen_proof(output, 1, 1, nullptr, output)); + assert(!merkle_gen_proof(output, 1, 0, nullptr, nullptr)); +} + +class Source : public MotaSource { +public: + std::vector mota; + MotaManifest manifest; + bool fail_read = false; + Source(uint32_t blocks, uint8_t value) { + const uint32_t total = 8 + MOTA_MFL + blocks * 4 + blocks * OTA_MAX_BLOCK + 5; + mota.resize(total); + memcpy(mota.data(), MOTA_MAGIC, 4); wr_u32le(mota.data() + 4, total); + uint8_t* mf = mota.data() + 8; + mf[0] = MOTA_FORMAT_VER; mf[1] = MFLAG_FULL; mf[2] = HASH_ALGO_SHA256; + wr_u32le(mf + 3, 123); wr_u32le(mf + 7, 1); + wr_u32le(mf + 11, blocks * OTA_MAX_BLOCK); wr_u32le(mf + 15, blocks * OTA_MAX_BLOCK); + mf[19] = 11; mf[56] = CODEC_FULL; + uint8_t* leaves = mf + MOTA_MFL; + uint8_t* payload = leaves + blocks * 4; + for (uint32_t i = 0; i < blocks; ++i) { + memset(payload + i * OTA_MAX_BLOCK, uint8_t(value + i), OTA_MAX_BLOCK); + merkle_leaf(leaves + i * 4, payload + i * OTA_MAX_BLOCK, OTA_MAX_BLOCK); + } + merkle_root(mf + 20, leaves, blocks); + memcpy(mota.data() + total - 5, MOTA_TRAILER, 5); + assert(mota_parse(mota.data(), mota.size(), manifest)); + } + uint8_t count() override { return 1; } + bool describe(uint8_t index, MotaDesc& d) override { + if (index) return false; + memcpy(d.mid, manifest.merkle_root, 4); d.target_id = 123; d.fw_version = 1; + d.codec_id = CODEC_FULL; d.flags = MFLAG_FULL; d.block_size_log2 = 11; + d.total_size = mota.size(); d.leaves_off = 8 + MOTA_MFL; + d.block_count = manifest.block_count; + d.payload_off = manifest.payload - mota.data(); d.payload_size = manifest.payload_size; + return true; + } + bool read(uint8_t index, uint32_t off, uint8_t* out, uint32_t len) override { + if (index || (uint64_t)off + len > mota.size()) return false; + if (fail_read) { if (len) out[0] = 0xEE; return false; } + memcpy(out, mota.data() + off, len); return true; + } +}; +struct Sent { + bool accept_data = true, accept_proof = true, accept_manifest = true; + std::vector> packets; +}; +static bool send(void* opaque, const uint8_t* data, uint16_t len, bool) { + auto& sent = *static_cast(opaque); + if ((data[0] == OTA_DATA && !sent.accept_data) || + (data[0] == OTA_PROOF && !sent.accept_proof) || + (data[0] == OTA_MANIFEST && !sent.accept_manifest)) return false; + sent.packets.emplace_back(data, data + len); return true; +} +static bool encode(void*, const uint8_t* src, uint16_t, uint8_t* dst, + uint16_t cap, uint16_t* len) { + assert(cap == OTA_MAX_BLOCK); memset(dst, src[0], 8); *len = 8; return true; +} +static bool encode_long(void*, const uint8_t* src, uint16_t, uint8_t* dst, + uint16_t cap, uint16_t* len) { + assert(cap == OTA_MAX_BLOCK); memset(dst, src[0], 400); *len = 400; return true; +} +static bool decode(void*, const uint8_t* src, uint16_t len, uint8_t* dst, + uint16_t cap, uint16_t* produced) { + if (len != 8) return false; + memset(dst, src[0], cap); *produced = cap; return true; +} +static bool request(OtaManager& manager, const Source& source, uint16_t block = 0, bool compressed = true) { + ReqMsg req{}; memcpy(req.manifest_id, source.manifest.merkle_root, 4); + req.block_idx = block; req.want_mask = compressed ? ota_req_make_v2(0xFFF, true, true) : 0xFFFF; + uint8_t wire[MAX_PACKET_PAYLOAD]; + return manager.on_message(wire, encode_req(wire, sizeof(wire), req)); +} +static bool manifest_request(OtaManager& manager, const Source& source) { + GetManifestMsg req{}; memcpy(req.manifest_id, source.manifest.merkle_root, 4); req.want_mask = 0xFFFF; + uint8_t wire[MAX_PACKET_PAYLOAD]; + return manager.on_message(wire, encode_get_manifest(wire, sizeof(wire), req)); +} +static void drain(OtaManager& manager, uint32_t start = 100000) { + for (unsigned turn = 0; turn < 100 && (manager.pendingServeJobs() || manager.pendingManifestJobs()); ++turn) { + manager.set_clock(start + turn * 10000u); manager.serviceEgress(); + } + assert(!manager.pendingServeJobs() && !manager.pendingManifestJobs()); +} +static void verify(const Sent& sent, const Source& source, bool compressed) { + std::vector bytes; + bool proof_seen = false; + for (const auto& wire : sent.packets) { + DataMsg data{}; ProofMsg proof{}; + if (decode_data(wire.data(), wire.size(), data) && !memcmp(data.manifest_id, source.manifest.merkle_root, 4)) { + if (compressed) { + uint8_t fragment; uint16_t encoded; bool deflated; + assert(ota_data_v2_unpack_extended(data.frag_off, OTA_MAX_BLOCK, fragment, encoded, deflated)); + assert(deflated && fragment == 0 && encoded == 8 && data.data_len == 12); + bytes.assign(OTA_MAX_BLOCK, data.data[4]); + } else bytes.insert(bytes.end(), data.data, data.data + data.data_len); + } else if (decode_proof(wire.data(), wire.size(), proof) && + !memcmp(proof.manifest_id, source.manifest.merkle_root, 4)) { + assert(merkle_verify(source.manifest.payload, OTA_MAX_BLOCK, 0, + proof.proof, proof.n_proof, source.manifest.merkle_root, source.manifest.block_count)); + proof_seen = true; + } + } + assert(proof_seen && bytes.size() == OTA_MAX_BLOCK); + assert(!memcmp(bytes.data(), source.manifest.payload, OTA_MAX_BLOCK)); +} + +static void lifetimes() { + Source source(9, 0x42); + Sent sent; OtaManager manager; + manager.begin(123, send, &sent); manager.set_transport_deflate_encoder(encode); + assert(manager.add_source(&source) && allocated == 0); + assert(request(manager, source) && allocated == 36); + sent.accept_data = false; + manager.serviceEgress(); assert(allocated == 36 + OTA_MAX_BLOCK); + manager.set_clock(OTA_SOURCE_BUFFER_IDLE_MS + 1000); manager.loop(); + assert(allocated == 36 + OTA_MAX_BLOCK && manager.pendingServeJobs()); + sent.accept_data = true; sent.accept_proof = false; + manager.serviceEgress(); manager.set_clock(100000); manager.serviceEgress(); + manager.loop(); assert(allocated == 36 + OTA_MAX_BLOCK && manager.pendingServeJobs()); + sent.accept_proof = true; drain(manager); verify(sent, source, true); + manager.set_clock(1000000); manager.loop(); + assert(allocated == 0 && manager.servedCount() == 1); // descriptors survive; reload on demand + sent.packets.clear(); assert(request(manager, source)); drain(manager); verify(sent, source, true); + assert(manager.remove_source(&source) && allocated == 0 && manager.servedCount() == 0); + assert(manager.add_source(&source) && request(manager, source)); + manager.clearPendingEgress(); assert(allocated == 0 && manager.servedCount() == 1); + assert(manifest_request(manager, source)); sent.accept_manifest = false; + manager.set_clock(2000000); manager.loop(); assert(allocated == 36); + manager.clear_sources(); assert(allocated == 0 && !manager.pendingManifestJobs()); +} +static void faults_and_capacity() { + Source small(3, 0x31), large(OTA_PROOFGEN_SCRATCH / 4, 0x52); + Sent sent; OtaManager manager; + manager.begin(123, send, &sent); manager.set_transport_deflate_encoder(encode); + assert(manager.add_source(&small)); + fail_next = true; assert(!request(manager, small) && allocated == 0 && !manager.pendingServeJobs()); + assert(request(manager, small) && allocated == 12); + fail_next = true; manager.serviceEgress(); + assert(allocated == 12); // encoder-output OOM selects raw v2 rather than dropping the transfer + drain(manager); + bool raw_seen = false; + for (const auto& wire : sent.packets) { + DataMsg data{}; + if (decode_data(wire.data(), wire.size(), data)) { + uint8_t f; uint16_t n; bool deflated; + assert(ota_data_v2_unpack_extended(data.frag_off, OTA_MAX_BLOCK, f, n, deflated)); + assert(!deflated); raw_seen = true; + } + } + assert(raw_seen); manager.clearPendingEgress(); assert(allocated == 0); + assert(manager.add_source(&large)); + peak = 0; assert(manifest_request(manager, large)); + assert(allocated == OTA_PROOFGEN_SCRATCH && peak == OTA_PROOFGEN_SCRATCH); + manager.clearPendingEgress(); assert(allocated == 0); + ReqProofMsg req{}; memcpy(req.manifest_id, large.manifest.merkle_root, 4); + req.block_idx = large.manifest.block_count - 1; + uint8_t wire[MAX_PACKET_PAYLOAD]; + sent.packets.clear(); fail_next = false; + assert(manager.on_message(wire, encode_req_proof(wire, sizeof(wire), req))); + fail_next = true; drain(manager); assert(fail_next); fail_next = false; + assert(allocated == OTA_PROOFGEN_SCRATCH); // maximum-capacity proof needs no output allocation + ProofMsg maximum{}; assert(sent.packets.size() == 1); + assert(decode_proof(sent.packets[0].data(), sent.packets[0].size(), maximum)); + assert(merkle_verify_from_leaf(large.manifest.leaves + req.block_idx * 4, req.block_idx, + maximum.proof, maximum.n_proof, large.manifest.merkle_root, + large.manifest.block_count)); + manager.clearPendingEgress(); assert(allocated == 0); + small.fail_read = true; assert(!manifest_request(manager, small)); + manager.set_clock(3000000); manager.loop(); assert(allocated == 0); + small.fail_read = false; assert(request(manager, small)); drain(manager); + assert(manager.serve(small.mota.data(), small.mota.size()) && allocated == 0); + fail_next = true; assert(request(manager, small, 0, false)); drain(manager); + assert(fail_next && allocated == 0); fail_next = false; // raw proof serving allocates nothing +} +static void concurrent_receive() { + Source outbound(5, 0x40), incoming(3, 0x70); + Sent sent; OtaManager manager; OtaStoreRam<16384> store; + manager.begin(123, send, &sent); manager.set_fetch_store(&store); + manager.set_transport_deflate_encoder(encode); manager.set_transport_deflate_decoder(decode); + assert(manager.add_source(&outbound) && request(manager, outbound)); + sent.accept_data = false; manager.serviceEgress(); + assert(allocated == 20 + OTA_MAX_BLOCK); + assert(manager.pull(incoming.manifest.merkle_root, 123) == OtaManager::PULL_STARTED); + for (unsigned f = 0; f < 2; ++f) { + ManifestMsg mf{}; memcpy(mf.manifest_id, incoming.manifest.merkle_root, 4); + mf.frag_idx = f; mf.frag_total = 2; mf.bytes = incoming.manifest.manifest_start + f * OTA_MF_FRAG; + mf.len = f == 0 ? OTA_MF_FRAG : MOTA_MFL - OTA_MF_FRAG; + uint8_t wire[MAX_PACKET_PAYLOAD]; manager.on_message(wire, encode_manifest(wire, sizeof(wire), mf)); + } + assert(manager.fetchState() == OtaManager::FETCHING); + for (uint16_t block = 0; block < 3; ++block) { + uint8_t representation[12]; memset(representation + 4, uint8_t(0x70 + block), 8); + mh4(representation, representation + 4, 8); + DataMsg data{}; memcpy(data.manifest_id, incoming.manifest.merkle_root, 4); data.block_idx = block; + assert(ota_data_v2_pack_extended(0, 8, data.frag_off)); data.data = representation; data.data_len = 12; + uint8_t wire[MAX_PACKET_PAYLOAD]; assert(manager.on_message(wire, encode_data(wire, sizeof(wire), data))); + uint8_t siblings[128]; ProofMsg proof{}; memcpy(proof.manifest_id, incoming.manifest.merkle_root, 4); + proof.block_idx = block; proof.n_proof = merkle_gen_proof(incoming.manifest.leaves, 3, block, nullptr, siblings); + proof.proof = siblings; assert(manager.on_message(wire, encode_proof(wire, sizeof(wire), proof))); + manager.set_clock(100000 + block * 10000); manager.loop(); + assert(allocated == 20 + OTA_MAX_BLOCK && manager.pendingServeJobs()); + } + assert(manager.fetchState() == OtaManager::COMPLETE); + MotaManifest staged; assert(mota_parse(store.data(), store.staged_size(), staged)); + assert(!memcmp(staged.payload, incoming.manifest.payload, incoming.manifest.payload_size)); + sent.accept_data = true; drain(manager, 200000); verify(sent, outbound, true); + manager.clear_sources(); assert(allocated == 0); +} +static void interleaved_manifest() { + Source data_source(9, 0x35), manifest_source(17, 0x75); + Sent sent; OtaManager manager; + manager.begin(123, send, &sent); manager.set_transport_deflate_encoder(encode_long); + assert(manager.add_source(&data_source) && manager.add_source(&manifest_source)); + assert(request(manager, data_source)); + manager.serviceEgress(); // first compressed fragment is admitted; later DATA stays pinned + assert(manager.pendingServeJobs() && allocated == 36 + OTA_MAX_BLOCK); + peak = allocated; + assert(manifest_request(manager, manifest_source)); + assert(allocated == 68 + OTA_MAX_BLOCK && peak == allocated); // old leaves were freed before growth + sent.accept_data = false; + manager.set_clock(100000); manager.serviceEgress(); + manager.set_clock(200000); manager.serviceEgress(); + assert(!manager.pendingManifestJobs() && manager.pendingServeJobs()); + manager.loop(); assert(allocated == 68 + OTA_MAX_BLOCK); + sent.accept_data = true; drain(manager, 300000); + std::vector encoded; + uint8_t identity[4]{}; bool proof_seen = false; + for (const auto& wire : sent.packets) { + DataMsg data{}; ProofMsg proof{}; + if (decode_data(wire.data(), wire.size(), data)) { + assert(!memcmp(data.manifest_id, data_source.manifest.merkle_root, 4)); + uint8_t fragment; uint16_t length; bool deflated; + assert(ota_data_v2_unpack_extended(data.frag_off, OTA_MAX_BLOCK, fragment, length, deflated)); + assert(deflated && length == 400 && fragment * OTA_FRAG_DATA_V2 == encoded.size()); + if (encoded.empty()) memcpy(identity, data.data, 4); + else assert(!memcmp(identity, data.data, 4)); + encoded.insert(encoded.end(), data.data + 4, data.data + data.data_len); + } else if (decode_proof(wire.data(), wire.size(), proof)) { + assert(merkle_verify(data_source.manifest.payload, OTA_MAX_BLOCK, 0, + proof.proof, proof.n_proof, data_source.manifest.merkle_root, + data_source.manifest.block_count)); + proof_seen = true; + } + } + assert(proof_seen && encoded == std::vector(400, 0x35)); + uint8_t expected[4]; mh4(expected, encoded.data(), encoded.size()); assert(!memcmp(identity, expected, 4)); + manager.clearPendingEgress(); assert(allocated == 0 && manager.servedCount() == 2); + sent.packets.clear(); assert(request(manager, data_source)); manager.serviceEgress(); + fail_next = true; assert(!manifest_request(manager, manifest_source)); + assert(allocated == OTA_MAX_BLOCK); // failed growth released old leaves, but pinned DATA survives + drain(manager, 900000); + encoded.clear(); + for (const auto& packet : sent.packets) { + DataMsg data{}; + if (decode_data(packet.data(), packet.size(), data)) + encoded.insert(encoded.end(), data.data + 4, data.data + data.data_len); + } + assert(encoded == std::vector(400, 0x35)); + manager.clearPendingEgress(); assert(allocated == 0); + manager.set_clock(0xFFFFFF00u); assert(request(manager, data_source)); drain(manager, 0xFFFFFF00u); + manager.set_clock(0xFFFFFF00u + 100000u); manager.loop(); assert(allocated == 0); +} +int main() { + proofs(); assert(allocated == 0); + lifetimes(); assert(allocated == 0); + faults_and_capacity(); assert(allocated == 0); + concurrent_receive(); assert(allocated == 0); + interleaved_manifest(); assert(allocated == 0); + puts("PASS: streaming proofs and OTA cold-buffer lifetimes"); +} diff --git a/test/test_client_acl_role.py b/test/test_client_acl_role.py new file mode 100644 index 000000000..263e7ddda --- /dev/null +++ b/test/test_client_acl_role.py @@ -0,0 +1,163 @@ +"""Role-safe ACL layout and unchanged production persistence, without PlatformIO.""" + +import configparser +import importlib.util +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +from types import SimpleNamespace +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("client_acl_role", ROOT / "scripts/client_acl_role.py") +POLICY = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(POLICY) +DEFINE = POLICY.DEFINITION +FIXTURE = ROOT / "test/fixtures/client_acl_spiffs" + + +class ClientAclRoleTests(unittest.TestCase): + def test_only_complete_repeater_role_qualifies(self): + for value in ("+<../examples/simple_repeater>", "+<../examples/simple_repeater/*.cpp>", + ["+<*.cpp>", "+<..\\examples\\simple_repeater\\*.cpp>"]): + self.assertEqual(POLICY.role_policy(value), {DEFINE: 1}) + for role in POLICY.OTHER_ROLES: + self.assertEqual(POLICY.role_policy("+<../examples/" + role + ">"), {}) + self.assertEqual(POLICY.role_policy("+<../examples/simple_repeater> " + "+<../examples/" + role + ">"), {}) + for value in ("", "+<*>", "+<../examples/simple_repeater/main.cpp>", + "+<../examples/simple_repeater> -<../examples/simple_repeater/MyMesh.cpp>"): + self.assertEqual(POLICY.role_policy(value), {}) + + def test_ordered_source_selection_and_explicit_overrides(self): + role = "+<../examples/simple_repeater>" + self.assertEqual(POLICY.role_policy(role + " -<../examples/simple_repeater>"), {}) + self.assertEqual(POLICY.role_policy(role + " -<../examples/simple_repeater> " + role), {DEFINE: 1}) + for flag in ("-D" + DEFINE + "=0", "-D" + DEFINE + "=1", "-U" + DEFINE): + self.assertEqual(POLICY.role_policy(role, flag), {}) + self.assertEqual(POLICY.role_policy(role, cppdefines=[(DEFINE, 1)]), {}) + self.assertEqual(POLICY.role_policy(role, "-DCOMPANION_RADIO_FULL=1"), {}) + for source in ("", "+<../examples/simple_room_server>", "+<*>"): + with self.assertRaises(ValueError): + POLICY.role_policy(source, "-D" + DEFINE + "=1") + for value in ("true", "2", "-1"): + with self.assertRaises(ValueError): + POLICY.role_policy(role, "-D" + DEFINE + "=" + value) + + def test_one_common_definition_not_header_local(self): + class Env(dict): + def AppendUnique(self, **options): self["appended"] = options + def GetProjectOption(self, *args): raise AssertionError("effective filter required") + env = Env(SRC_FILTER="+<../examples/simple_repeater>", BUILD_FLAGS="", CPPDEFINES=[]) + POLICY.install(env) + self.assertEqual(env["appended"], {"CPPDEFINES": [(DEFINE, 1)]}) + env["SRC_FILTER"] = "+<../examples/simple_sensor>" + del env["appended"] + POLICY.install(env) + self.assertNotIn("appended", env) + + def test_every_platform_base_installs_role_hook(self): + ini = (ROOT / "platformio.ini").read_text() + for name in ("esp32_base", "nrf52_base", "rp2040_base", "stm32_base"): + section = ini.split("[" + name + "]", 1)[1].split("\n[", 1)[0] + self.assertIn("pre:scripts/client_acl_role.py", section) + + def test_real_board_roles_and_inherited_hooks(self): + config = configparser.ConfigParser(interpolation=None, strict=False, + inline_comment_prefixes=(";",)) + config.read([str(ROOT / "platformio.ini")] + + [str(path) for path in sorted((ROOT / "variants").glob("*/platformio.ini"))]) + def option(section, key): + if config.has_option(section, key): + raw = config.get(section, key) + else: + parents = re.split(r"[,\n]", config.get(section, "extends", fallback="")) + raw = next((option(parent.strip(), key) for parent in parents + if parent.strip() and option(parent.strip(), key)), "") + return re.sub(r"\$\{([^}]+)\}", lambda match: option(*match[1].rsplit(".", 1)), raw) + platforms = set() + repeaters = 0 + for section in config.sections(): + if not section.startswith("env:") or section == "env:native": continue + filt = option(section, "build_src_filter") + definitions = POLICY.role_policy(filt, option(section, "build_flags")) + if definitions: + repeaters += 1 + platforms.add(option(section, "platform")) + self.assertIn("pre:scripts/client_acl_role.py", option(section, "extra_scripts"), section) + self.assertGreater(repeaters, 100) + self.assertGreaterEqual(len(platforms), 4) + + def test_compact_and_complete_acl_have_identical_persistence(self): + compiler = shutil.which("g++") or shutil.which("clang++") + self.assertIsNotNone(compiler) + source = r''' +#define main full_fixture_main +#include "test_client_acl_spiffs.cpp" +#undef main +#include +static_assert(std::is_trivially_copyable::value, "ACL remains copyable"); +#if MESH_CLIENT_REPEATER_ONLY +static_assert(sizeof(((ClientInfo*)0)->extra) == 4, "only persisted sync field retained"); +#else +static_assert(sizeof(((ClientInfo*)0)->extra.sensor.min_deltas) == 14, "sensor state retained"); +static_assert(sizeof(((ClientInfo*)0)->extra.room.pending_ack) == 4, "room state retained"); +#endif +int main() { + CHECK(full_fixture_main() == 0); // replay, corruption, OOM and rollback coverage + FakeFilesystem fs; + ClientACL acl; + acl.load(&fs, SELF); + for (int i = 0; i < MAX_CLIENTS; ++i) { + uint8_t key[PUB_KEY_SIZE] = {}; key[0] = uint8_t(i + 1); + auto* c = acl.putClient(mesh::Identity(key), PERM_ACL_ADMIN); + CHECK(c); + c->extra.room.sync_since = 0x12345678u + i; + c->last_timestamp = 100 + i; + c->out_path_is_persistable = true; + c->out_path_len = 1; c->out_path[0] = uint8_t(i + 3); + c->alt_path_len = 1; c->alt_path[0] = uint8_t(i + 5); +#if !MESH_CLIENT_REPEATER_ONLY + c->extra.room.pending_ack = 99; c->extra.room.push_failures = 3; +#endif + } + CHECK(acl.save(&fs)); + ClientACL restored; + restored.load(&fs, SELF); + CHECK(restored.getNumClients() == MAX_CLIENTS); + for (int i = 0; i < MAX_CLIENTS; ++i) { + const auto* c = restored.getClientByIdx(i); + CHECK(c->extra.room.sync_since == 0x12345678u + i); + CHECK(c->out_path_len == 1 && c->out_path[0] == i + 3); + CHECK(c->alt_path_len == 1 && c->alt_path[0] == i + 5); + CHECK(c->isAdmin()); + } + std::printf("SIZE:%zu\nCONTACTS:", sizeof(ClientInfo)); + for (auto byte : fs.files["/s_contacts"]) std::printf("%02x", unsigned(byte)); + std::puts(""); +} +''' + results = [] + with tempfile.TemporaryDirectory(prefix="compact-client-acl-") as directory: + work = Path(directory) + (work / "test.cpp").write_text(source) + for compact in (0, 1): + binary = work / ("acl-" + str(compact)) + command = [compiler, "-std=c++17", "-Wall", "-Wextra", "-DESP32=1", "-DESP32_PLATFORM=1", + "-D" + DEFINE + "=" + str(compact), "-I" + str(FIXTURE / "mocks"), + "-I" + str(FIXTURE), "-I" + str(ROOT / "src"), str(work / "test.cpp"), "-o", str(binary)] + command[1:1] = ["-fsanitize=address,undefined", "-fno-sanitize-recover=all", "-fno-pie", "-no-pie"] + built = subprocess.run(command, text=True, capture_output=True, timeout=60) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + run = subprocess.run([str(binary)], text=True, capture_output=True, timeout=30) + self.assertEqual(run.returncode, 0, run.stdout + run.stderr) + results.append(run.stdout) + full, compact = results + self.assertEqual(full.split("CONTACTS:")[1], compact.split("CONTACTS:")[1]) + sizes = [int(re.search(r"SIZE:(\d+)", result)[1]) for result in results] + self.assertGreaterEqual(sizes[0] - sizes[1], 36) + + +if __name__ == "__main__": unittest.main() diff --git a/test/test_esp32_tinyusb_cooperative_output.py b/test/test_esp32_tinyusb_cooperative_output.py index 540342445..ec0e5adba 100644 --- a/test/test_esp32_tinyusb_cooperative_output.py +++ b/test/test_esp32_tinyusb_cooperative_output.py @@ -99,6 +99,12 @@ struct SimpleMeshTables { int8_t snr_x4; }; std::vector rows; + static void copyRecentRepeaterInfo(RecentRepeaterInfo& out, + const RecentRepeaterInfo& in) { out = in; } + static void copyRecentRepeaterPrefix(uint8_t* out, + const RecentRepeaterInfo& in) { + memcpy(out, in.prefix, sizeof(in.prefix)); + } int getRecentRepeaterCount() const { return static_cast(rows.size()); } const RecentRepeaterInfo* getNextRecentRepeaterBySortKey( const RecentRepeaterInfo*, int previous, int& result) const { diff --git a/test/test_firmware_ram.py b/test/test_firmware_ram.py index 71680ae06..7cc651941 100644 --- a/test/test_firmware_ram.py +++ b/test/test_firmware_ram.py @@ -90,9 +90,11 @@ class FirmwareRamTest(unittest.TestCase): self.assertEqual("ota_encoder_workspace" in components, enabled) if enabled: self.assertEqual(components["ota_encoder_workspace"], 1024 + 16) - self.assertEqual("ota_self_source_scratch" in components, - enabled and platform == "NRF52_PLATFORM") - if enabled and platform == "NRF52_PLATFORM": + self_source = (platform == "NRF52_PLATFORM" and "ENABLE_OTA" in defines + and "OTA_SEEDER_ONLY" not in defines and any(key in defines for key in ( + "OTA_QSPI_STORE", "OTA_SD_STORE", "OTA_RAK_AUTO_STORE", "OTA_TOWER_AUTO_STORE"))) + self.assertEqual("ota_self_source_scratch" in components, self_source) + if self_source: self.assertEqual(components["ota_self_source_scratch"], 4096 + 32) def test_device_encoder_hash_override_is_budgeted_and_bounded(self): @@ -123,7 +125,7 @@ class FirmwareRamTest(unittest.TestCase): "ENABLE_OTA": 1, "OTA_HEAP_CONTEXT": 1, }, "GEPRC_Linkflow_900_repeater") self.assertEqual(policy["components"]["client_table"], 32 * 320 + 16) - self.assertEqual(policy["components"]["flood_filter_table"], 63 * 200 + 16) + self.assertEqual(policy["components"]["flood_filter_table"], 63 * 192 + 16) self.assertEqual(policy["components"]["ota_context"], 16384 + 16) self.assertGreaterEqual(policy["required_contiguous_bytes"], 16384 + 16) reduced = ram.requirements("STM32_PLATFORM", { @@ -201,13 +203,13 @@ class FirmwareRamTest(unittest.TestCase): }, "RAK_4631_repeater_unified_lora_ota") components = combined["components"] self.assertEqual(components["uart_bridge_and_driver"], 2560) - self.assertEqual(components["ethernet_or_external_ota_workspaces"], 5168) + self.assertEqual(components["ethernet_or_external_ota_workspaces"], 11344) self.assertNotIn("ota_encoder_workspace", components) self.assertNotIn("ota_self_source_scratch", components) self.assertEqual(components["display_pixels_and_driver"], 2048) self.assertEqual(combined["required_heap_bytes"] - legacy["required_heap_bytes"], 2560 - 2048) self.assertEqual(components["loop_and_callback_stacks"], 8192 + 3072) - self.assertEqual(components["flood_filter_table"], 63 * 200 + 16) + self.assertEqual(components["flood_filter_table"], 63 * 192 + 16) # An alternate port selects the same bridge instead of allocating two. alternate = ram.requirements("NRF52_PLATFORM", { **defines, "RAK4631_COMBINED_ETHERNET": 1, @@ -221,7 +223,7 @@ class FirmwareRamTest(unittest.TestCase): "RAK4631_COMBINED_ETHERNET": 1, "ENABLE_OTA": 1, "MESHCORE_OTA_DEVICE_DEFLATE": 1, } - for words, expected in ((1024, 5168), (1536, 6976), (2048, 9024)): + for words, expected in ((1024, 11344), (1536, 11344), (2048, 11344), (3072, 13120)): with self.subTest(words=words): policy = ram.requirements("NRF52_PLATFORM", { **defines, "RAK4631_ETHERNET_TASK_STACK_WORDS": words, @@ -231,21 +233,59 @@ class FirmwareRamTest(unittest.TestCase): no_encoder = ram.requirements("NRF52_PLATFORM", { **defines, "MESHCORE_OTA_DEVICE_DEFLATE": 0, }, "RAK_4631_sensor") - self.assertEqual(no_encoder["components"]["ethernet_or_external_ota_workspaces"], 4928) + self.assertEqual(no_encoder["components"]["ethernet_or_external_ota_workspaces"], 10304) for bits in (7, 8, 9, 10): with self.subTest(bits=bits): policy = ram.requirements("NRF52_PLATFORM", { **defines, "MESHCORE_OTA_DEFLATE_HASH_BITS": bits, }, "RAK_4631_sensor") self.assertEqual(policy["components"]["ethernet_or_external_ota_workspaces"], - max(4928, (1 << bits) * 2 + 16 + 4096 + 32)) - for capacity, expected in ((512, 5168), (1024, 5440), (2048, 6464)): + max(4928, (1 << bits) * 2 + 16 + 4096 + 32 + 6176)) + for capacity, expected in ((512, 11344), (1024, 11344), (2048, 11344)): with self.subTest(tx_capacity=capacity): policy = ram.requirements("NRF52_PLATFORM", { **defines, "ETHERNET_CLI_TX_BUFFER_BYTES": capacity, }, "RAK_4631_sensor") self.assertEqual(policy["components"]["ethernet_or_external_ota_workspaces"], expected) + def test_compact_tables_keep_capacity_and_reserve_active_ota_allocations(self): + for platform, leaf_limit in (("NRF52_PLATFORM", 4096), ("ESP32_PLATFORM", 16384), + ("RP2040_PLATFORM", 4096), ("STM32_PLATFORM", 4096)): + with self.subTest(platform=platform): + flags = {"ENABLE_OTA": 1, "MESH_CLIENT_REPEATER_ONLY": 1, + "MAX_RECENT_REPEATERS": 512} + compact = ram.requirements(platform, flags, "board_repeater")["components"] + self.assertEqual(compact["client_table"], 32 * 284 + 16) + self.assertEqual(compact["ota_generic_source_buffers"], leaf_limit + 2048 + 32) + full = ram.requirements(platform, {**flags, "MESH_CLIENT_REPEATER_ONLY": 0}, + "board_repeater")["components"] + self.assertEqual(full["client_table"] - compact["client_table"], 36 * 32) + if platform == "ESP32_PLATFORM": + self.assertEqual(compact["neighbor_history"], 512 * 9 + 16) + sd = ram.requirements("NRF52_PLATFORM", {"ENABLE_OTA": 1, "OTA_SD_STORE": 1}, + "board_repeater")["components"] + self.assertEqual(sd["ota_generic_source_buffers"], 8192 + 2048 + 32) + custom = ram.requirements("NRF52_PLATFORM", { + "ENABLE_OTA": 1, "OTA_PROOFGEN_SCRATCH": 12000, "OTA_MAX_BLOCK": 1024, + }, "board_repeater")["components"] + self.assertEqual(custom["ota_generic_source_buffers"], 12000 + 1024 + 32) + for default, expected in ((256, 256), (2048, 2048)): + parts = ram.requirements("ESP32_PLATFORM", { + "MESH_RAM_DEFAULT_RECENT_REPEATERS": default, + }, "board_repeater")["components"] + self.assertEqual(parts["neighbor_history"], expected * 9 + 16) + disabled = ram.requirements("ESP32_PLATFORM", {"MESH_ENABLE_RECENT_REPEATERS": 0}, + "board_repeater")["components"] + self.assertEqual(disabled["neighbor_history"], 0) + s3 = ram.requirements("ESP32_PLATFORM", {}, "board_repeater") + self.assertGreaterEqual(s3["required_contiguous_bytes"], 2048 * 9 + 16) + alias = ram.requirements("ESP32_PLATFORM", {"MESH_CLIENT_REPEATER_ONLY": 1}, + "custom_companion_alias") + self.assertEqual(alias["components"]["client_table"], 32 * 284 + 16) + self.assertEqual(alias["components"]["flood_filter_table"], 63 * 192 + 16) + self.assertEqual(alias["components"]["neighbor_history"], 2048 * 9 + 16) + self.assertEqual(alias["components"]["radio_packet_pool"], 10240) + def test_combined_rak_oled_bound_does_not_change_other_display_budgets(self): defines = { "RAK_4631": 1, "ETHERNET_ENABLED": 1, "OTA_RAK_AUTO_STORE": 1, diff --git a/test/test_flood_policy_capacity.py b/test/test_flood_policy_capacity.py index 030dbefb1..fc3a3bfa4 100644 --- a/test/test_flood_policy_capacity.py +++ b/test/test_flood_policy_capacity.py @@ -76,7 +76,31 @@ public: bool isFloodChannelDataRule(const FloodPacketFilterEntry&) const; }; ''' +ROOM_CLASS = r''' +class FloodRuleEngine { +public: + static constexpr uint8_t RULE_SLOTS = 31, NAME_LEN = 32, PATH_PREFIX_BYTES_MAX = 9; +@ENTRY@; + FILESYSTEM* _fs; + Entry storage[RULE_SLOTS]{}; + Entry* _entries = storage; + explicit FloodRuleEngine(MemoryFS& fs): _fs(&fs) {} + bool save(); +}; +static const char RULE_FILE[] = "/flood_filter"; +static const char RULE_TEMP_FILE[] = "/flood_filter.tmp"; +static const char RULE_BACKUP_FILE[] = "/flood_filter.bak"; +''' SCENARIOS = r''' +template void canonicalRule(Entry& row) { + row.active=true;row.payload_type=PAYLOAD_TYPE_ADVERT; + row.min_hops=2;row.max_hops=63;row.suspend_on_temp_radio=true; + std::strcpy(row.scope_name,"#Town");row.match_blacklisted_path=true; + row.scope_uses_slow_timing=true;row.incoming_scope_kind=FloodFilterPolicy::RULE_IN_ALLOWED; + row.path_hash_size=3;row.rate_limit_enabled=true;row.rate_per_minute=1234; + row.priority=7;row.stop_on_match=true;row.retry_on_match=true; + row.rate_window_started=0xfedcba98;row.rate_window_count=17;row.rate_window_active=true; +} std::vector fixture(uint8_t slots, uint8_t highest) { MemoryFS fs;MyMesh writer(fs,slots); for (uint8_t index : {uint8_t(0),highest}) { @@ -97,6 +121,41 @@ std::map> snapshot(const MemoryFS& fs) { return result; } int main() { + // Canonical FPF7 bytes remain independent of the compact in-memory layout. + // Exercise every persisted packed flag plus the addressable retry flag. + { + MemoryFS fs;MyMesh writer(fs,63);auto& row=writer.storage[0]; + canonicalRule(row); + std::vector expected={'F','P','F','7',1,1,PAYLOAD_TYPE_ADVERT,2,63,1}; + const auto zeroes=[&expected](size_t count) {expected.insert(expected.end(),count,0);}; + expected.insert(expected.end(),{'#','T','o','w','n'});zeroes(27); + expected.insert(expected.end(),{1,1,1,3});zeroes(32); + expected.push_back(0x80);zeroes(32+32); + expected.insert(expected.end(),{3,0});zeroes(9); + expected.insert(expected.end(),{0,1,0xd2,0x04});zeroes(32); + expected.insert(expected.end(),{7,1,'F','P','S','1',0xff,0xff,0,0,0}); + assert(writer.saveFloodPacketFilters());assert(fs.get(FLOOD_PACKET_FILTER_FILE)==expected); + MyMesh reader(fs,63);assert(reader.loadFloodPacketFilters());const auto& loaded=reader.storage[0]; + assert(loaded.active && loaded.suspend_on_temp_radio && loaded.match_blacklisted_path); + assert(loaded.scope_uses_slow_timing && !loaded.drop_on_match && loaded.rate_limit_enabled); + assert(loaded.stop_on_match && loaded.retry_on_match && loaded.rate_per_minute==1234); + assert(!loaded.rate_window_active && loaded.rate_window_started==0 && loaded.rate_window_count==0); + // Execute the real room serializer too. It retains its 31 padded slots, + // and the same canonical row must load through the repeater path. + MemoryFS room_fs;FloodRuleEngine room(room_fs);canonicalRule(room.storage[0]); + auto room_expected=expected;room_expected[4]=31; + const size_t row_stride=expected.size()-5-9; + room_expected.insert(room_expected.end()-9,30*row_stride,0); + assert(room.save());assert(room_fs.get(FLOOD_PACKET_FILTER_FILE)==room_expected); + MyMesh from_room(room_fs,63);assert(from_room.loadFloodPacketFilters()); + assert(from_room.storage[0].scope_uses_slow_timing && from_room.storage[0].stop_on_match); + assert(from_room.storage[0].retry_on_match && from_room.storage[0].rate_per_minute==1234); + // A drop rule exercises the remaining persisted packed flag without an + // incompatible scope/rate/retry action, while preserving the same format. + row={};row.active=true;row.payload_type=PAYLOAD_TYPE_ADVERT;row.max_hops=63;row.drop_on_match=true; + assert(writer.saveFloodPacketFilters());MyMesh drop(fs,63);assert(drop.loadFloodPacketFilters()); + assert(drop.storage[0].active && drop.storage[0].drop_on_match && !drop.storage[0].retry_on_match); + } const auto large=fixture(63,62),small=fixture(4,2); // Older writers stored every slot, including inactive trailing entries. // Build the exact production wire format and replace the final active row @@ -198,6 +257,8 @@ class FloodPolicyCapacityTests(unittest.TestCase): def test_saved_rules_survive_smaller_firmware_capacity(self): source=(ROOT/'examples/simple_repeater/MyMesh.cpp').read_text() header=(ROOT/'examples/simple_repeater/MyMesh.h').read_text() + room_source=(ROOT/'examples/simple_room_server/FloodRuleEngine.cpp').read_text() + room_header=(ROOT/'examples/simple_room_server/FloodRuleEngine.h').read_text() fs=HARNESS[:HARNESS.index('@STORE@')] fs=fs.replace('bool valid = false;', 'bool valid = false;\n size_t cursor=0;') fs=fs.replace('size_t write(const uint8_t*, size_t);', '''size_t write(const uint8_t*, size_t); @@ -216,6 +277,11 @@ class FloodPolicyCapacityTests(unittest.TestCase): 'bool MyMesh::isFloodChannelDataRule(', 'bool MyMesh::loadFloodPacketFilters(', 'bool MyMesh::saveFloodPacketFilters('): parts.append(extract_braced(source,marker)) + parts.append(ROOM_CLASS.replace('@ENTRY@',extract_braced(room_header,'struct Entry'))) + for marker in ('static File openRead(', 'static File openWrite(', + 'static uint32_t updateFileHash(', 'static bool verifyWrittenFile(', + 'bool FloodRuleEngine::save('): + parts.append(extract_braced(room_source,marker)) parts.append(SCENARIOS) with tempfile.TemporaryDirectory(prefix='flood-policy-capacity-') as temp: p=Path(temp);cpp=p/'test.cpp';cpp.write_text('\n'.join(parts)) diff --git a/test/test_flood_rule_layout.py b/test/test_flood_rule_layout.py new file mode 100644 index 000000000..af616a2e9 --- /dev/null +++ b/test/test_flood_rule_layout.py @@ -0,0 +1,238 @@ +"""Exercise production flood-entry layouts and independent packed flag states.""" +from pathlib import Path +import os +import subprocess +import tempfile +import unittest + +from test_replay_reset_integration import extract_braced + +ROOT = Path(__file__).resolve().parents[1] + +PRELUDE = r''' +#include +#include +#include +#include +#include +#include +#include +#define FLOOD_PACKET_FILTER_SCOPE_NAME_LEN 32 +#define FLOOD_GROUP_MODERATION_NAME_LEN 32 +#define FLOOD_PACKET_FILTER_PATH_PREFIX_BYTES_MAX 9 +static constexpr uint8_t NAME_LEN = 32, PATH_PREFIX_BYTES_MAX = 9; +''' + +SCENARIOS = r''' +static_assert(sizeof(RepeaterEntry) == 192 && sizeof(RoomEntry) == 192, + "63 repeater rows / 31 room rows must retain the 192-byte bound"); +static_assert(sizeof(LegacyEntry) == 40, "retain legacy STM32 rule layout"); +static_assert(alignof(RepeaterEntry) >= alignof(uint32_t)); +static_assert(alignof(RoomEntry) >= alignof(uint32_t)); +#define CHECK_ALIGNED(T) \ + static_assert(offsetof(T, rate_window_started) % alignof(uint32_t) == 0); \ + static_assert(offsetof(T, rate_window_count) % alignof(uint16_t) == 0); \ + static_assert(offsetof(T, rate_per_minute) % alignof(uint16_t) == 0); \ + static_assert(std::is_trivially_copyable::value) +CHECK_ALIGNED(RepeaterEntry); +CHECK_ALIGNED(RoomEntry); + +template void setFlags(Entry& e, unsigned flags) { + e.active = flags & 1; + e.suspend_on_temp_radio = flags & 2; + e.match_blacklisted_path = flags & 4; + e.scope_uses_slow_timing = flags & 8; + e.drop_on_match = flags & 16; + e.rate_limit_enabled = flags & 32; + e.stop_on_match = flags & 64; + e.retry_on_match = flags & 128; + e.rate_window_active = flags & 256; +} +template unsigned flagsOf(const Entry& e) { + return unsigned(e.active) | unsigned(e.suspend_on_temp_radio) * 2 + | unsigned(e.match_blacklisted_path) * 4 + | unsigned(e.scope_uses_slow_timing) * 8 | unsigned(e.drop_on_match) * 16 + | unsigned(e.rate_limit_enabled) * 32 | unsigned(e.stop_on_match) * 64 + | unsigned(e.retry_on_match) * 128 | unsigned(e.rate_window_active) * 256; +} +template void exerciseFlags() { + // Exhaustive combinations detect adjacent flag corruption; copy/clear are + // the same operations used by production rule load, delete, and replace. + for (unsigned flags = 0; flags < 512; ++flags) { + Entry e{}; + e.rate_window_started = 0xfedcba98; + e.rate_window_count = 0x7654; + e.rate_per_minute = 0x3210; + setFlags(e, flags); + assert(flagsOf(e) == flags); + assert(e.rate_window_started == 0xfedcba98); + assert(e.rate_window_count == 0x7654 && e.rate_per_minute == 0x3210); + Entry copied = e; + assert(flagsOf(copied) == flags); + std::memcpy(&copied, &e, sizeof(e)); + assert(flagsOf(copied) == flags); + // These actual decoders require ordinary bool references. Packing must + // preserve this API and must not alter any unrelated flag. + assert(FloodFilterPolicy::decodeStoredRuleActive(5, copied.active, + copied.transport_modes)); + assert(FloodFilterPolicy::decodeStoredRuleChannel(16, copied.channel_key_len, + copied.retry_on_match)); + assert(flagsOf(copied) == ((flags | 1) & ~128U)); + assert(copied.transport_modes == 4 && copied.channel_key_len == 16); + std::memset(&copied, 0, sizeof(copied)); + assert(flagsOf(copied) == 0 && copied.rate_window_started == 0); + } +} +template void exerciseConfigurationEquality() { + Entry original{}; + original.active = true; + original.rate_limit_enabled = true; + original.rate_per_minute = 5; + Entry live = original; + live.rate_window_started = 0xfedcba98; + live.rate_window_count = 3; + live.rate_window_active = true; + assert(FloodFilterPolicy::sameRuleConfiguration(original, live)); + // Bytewise struct comparisons are forbidden: neither alignment padding nor + // live counters identify a rule. Poison trailing padding to enforce this. + for (size_t offset = offsetof(Entry, transport_modes) + 1; offset < sizeof(Entry); ++offset) + reinterpret_cast(&live)[offset] = 0xa5; + assert(FloodFilterPolicy::sameRuleConfiguration(original, live)); + // Every configured field must still distinguish two rows. Arrays include + // their final byte so accidental prefix-only comparisons cannot pass. + for (unsigned field = 0; field < 26; ++field) { + Entry changed = original; + switch (field) { + case 0: changed.active = false; break; + case 1: changed.payload_type = 1; break; + case 2: changed.min_hops = 1; break; + case 3: changed.max_hops = 1; break; + case 4: changed.suspend_on_temp_radio = true; break; + case 5: changed.scope_name[sizeof(changed.scope_name)-1] = 1; break; + case 6: changed.match_blacklisted_path = true; break; + case 7: changed.scope_uses_slow_timing = true; break; + case 8: changed.incoming_scope_kind = 1; break; + case 9: changed.incoming_scope_name[sizeof(changed.incoming_scope_name)-1] = 1; break; + case 10: changed.channel_key_len = 1; break; + case 11: changed.channel_hash = 1; break; + case 12: changed.channel_secret[sizeof(changed.channel_secret)-1] = 1; break; + case 13: changed.channel_name[sizeof(changed.channel_name)-1] = 1; break; + case 14: changed.path_hash_size = 1; break; + case 15: changed.path_hops = 1; break; + case 16: changed.path[sizeof(changed.path)-1] = 1; break; + case 17: changed.target_region_name[sizeof(changed.target_region_name)-1] = 1; break; + case 18: changed.drop_on_match = true; break; + case 19: changed.rate_limit_enabled = false; break; + case 20: changed.rate_per_minute = 6; break; + case 21: changed.priority = 1; break; + case 22: changed.stop_on_match = true; break; + case 23: changed.retry_on_match = true; break; + case 24: changed.transport_modes = 2; break; + case 25: changed.scope_name[0] = '#'; break; + } + assert(!FloodFilterPolicy::sameRuleConfiguration(original, changed)); + assert(!FloodFilterPolicy::sameRuleConfiguration(changed, original)); + } +} +template void exerciseProductionSlotSelection() { + Rules rules; + Entry candidate{}; + candidate.active = true; candidate.payload_type = PAYLOAD_TYPE_ADVERT; + candidate.max_hops = 63; candidate.rate_limit_enabled = true; + candidate.rate_per_minute = 5; + assert(rules.select(candidate) == 0); + rules.storage[0] = candidate; + rules.storage[0].rate_window_started = 100; + rules.storage[0].rate_window_count = 3; + rules.storage[0].rate_window_active = true; + assert(rules.select(candidate) == 0); // Reuse despite a live rate window. + candidate.path_hash_size = 2; + assert(rules.select(candidate) == 1); // Distinct configuration needs a slot. + rules.storage[1] = candidate; + assert(rules.select(candidate) == 1); + assert(rules.select(candidate, 3) == 3); // Explicit index remains authoritative. +} +int main() { + exerciseFlags(); + exerciseFlags(); + exerciseConfigurationEquality(); + exerciseConfigurationEquality(); + exerciseProductionSlotSelection(); + exerciseProductionSlotSelection(); +} +''' + + +def production_layouts(): + repeater = extract_braced( + (ROOT / 'examples/simple_repeater/MyMesh.h').read_text(), + 'struct FloodPacketFilterEntry') + room = extract_braced( + (ROOT / 'examples/simple_room_server/FloodRuleEngine.h').read_text(), + 'struct Entry') + return ('#define MESH_ENABLE_FLOOD_RULE_ENGINE 1\n' + + repeater.replace('struct FloodPacketFilterEntry', 'struct RepeaterEntry') + + ';\n' + room.replace('struct Entry', 'struct RoomEntry') + ';\n' + + '#undef MESH_ENABLE_FLOOD_RULE_ENGINE\n' + + '#define MESH_ENABLE_FLOOD_RULE_ENGINE 0\n' + + repeater.replace('struct FloodPacketFilterEntry', 'struct LegacyEntry') + ';\n') + + +def production_slot_selection(): + bodies = [] + for role, marker, end, members in ( + ('simple_repeater', 'void MyMesh::setFloodPacketFilter(', + '\n if (slot < 0) {\n strcpy(reply, "Err - filter table full");', + 'using FloodPacketFilterEntry = RepeaterEntry;\n' + 'RepeaterEntry storage[63]{};\n' + 'RepeaterEntry* flood_packet_filters = storage;\n' + 'uint8_t flood_packet_filter_slots = 63, flood_channel_data_rule_slot = 0xff;'), + ('simple_room_server', 'void FloodRuleEngine::set(', + '\n if (slot < 0) {\n copyString(reply, "Err - filter table full", 160);', + 'using Entry = RoomEntry;\n' + 'static constexpr uint8_t RULE_SLOTS = 31;\n' + 'RoomEntry storage[RULE_SLOTS]{};\nEntry* _entries = storage;'), + ): + file = 'MyMesh.cpp' if role == 'simple_repeater' else 'FloodRuleEngine.cpp' + method = extract_braced((ROOT / 'examples' / role / file).read_text(), marker) + # Compile both real setter slot-selection paths, not a test rewrite. + body = method[method.index(' int slot = requested_slot;'):method.index(end)] + name = 'RepeaterRules' if role == 'simple_repeater' else 'RoomRules' + entry = 'RepeaterEntry' if role == 'simple_repeater' else 'RoomEntry' + bodies.append('struct ' + name + ' {\n' + members + + '\nint select(const ' + entry + '& candidate, int requested_slot=-1) {\n' + + body + '\nreturn slot;\n}\n};\n') + return ''.join(bodies) + + +class FloodRuleLayoutTests(unittest.TestCase): + def test_packed_flags_keep_all_states_and_aligned_counters(self): + with tempfile.TemporaryDirectory(prefix='flood-rule-layout-') as temp: + path = Path(temp) + source = path / 'test.cpp' + source.write_text(PRELUDE + production_layouts() + production_slot_selection() + SCENARIOS) + exe = path / 'test' + subprocess.run(['g++', '-std=c++17', '-O1', '-g', + '-fsanitize=address,undefined', '-fno-omit-frame-pointer', + '-no-pie', '-I' + str(ROOT / 'src'), str(source), + '-o', str(exe)], check=True) + subprocess.run([str(exe)], check=True, + env={**os.environ, 'ASAN_OPTIONS': 'detect_leaks=1'}) + + def test_arm_layouts_keep_the_same_bound_and_natural_alignment(self): + compilers = sorted((Path.home() / '.platformio/packages').glob( + 'toolchain-gccarmnoneeabi*/bin/arm-none-eabi-g++')) + if not compilers: + self.skipTest('ARM compiler is not installed') + # Syntax/type checks invoke the compiler directly, never PlatformIO. + for compiler in compilers: + with self.subTest(compiler=compiler.parent.parent.name): + subprocess.run([str(compiler), '-std=c++17', '-mcpu=cortex-m4', + '-mthumb', '-fsyntax-only', '-x', 'c++', + '-I' + str(ROOT / 'src'), '-'], check=True, + input=PRELUDE + production_layouts() + production_slot_selection() + SCENARIOS, + text=True) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_mesh_tables/test_simple_mesh_tables.cpp b/test/test_mesh_tables/test_simple_mesh_tables.cpp index cd6aac0c1..42d911b3d 100644 --- a/test/test_mesh_tables/test_simple_mesh_tables.cpp +++ b/test/test_mesh_tables/test_simple_mesh_tables.cpp @@ -1,9 +1,88 @@ #include +#include +#include +#include #define MAX_RECENT_REPEATERS 8 #include "helpers/SimpleMeshTables.h" using namespace mesh; +TEST(SimpleMeshTables, CompactRecentRowsPreserveFullMillisAtEveryByteAlignment) { + using Info = SimpleMeshTables::RecentRepeaterInfo; + static_assert(sizeof(Info) == 9 && alignof(Info) == 1, "compact row layout"); + static_assert(std::is_trivially_copyable::value, "memset/cursor copy support"); + constexpr uint32_t values[] = {0, 1, 0x0000ffff, 0x12345678, 0x80000000, + 0xfffffffe, 0xffffffff}; + for (size_t offset = 0; offset < alignof(uint32_t); ++offset) { + alignas(uint32_t) unsigned char bytes[sizeof(Info) + alignof(uint32_t)] = {}; + Info* info = new (bytes + offset) Info{}; + info->prefix[0] = 0x86; + info->prefix[1] = 0x0c; + info->prefix[2] = 0xca; + info->prefix_len = 3; + info->snr_x4 = -128; + for (uint32_t value : values) { + info->last_heard_millis = value; + EXPECT_EQ(value, uint32_t(info->last_heard_millis)); + Info cursor; + SimpleMeshTables::copyRecentRepeaterInfo(cursor, *info); + EXPECT_EQ(value, uint32_t(cursor.last_heard_millis)); + EXPECT_EQ(0x86, info->prefix[0]); + EXPECT_EQ(0x0c, info->prefix[1]); + EXPECT_EQ(0xca, info->prefix[2]); + EXPECT_EQ(3, info->prefix_len); + EXPECT_EQ(-128, info->snr_x4); + } + SimpleMeshTables::clearRecentRepeaterInfo(*info); + EXPECT_EQ(0u, uint32_t(info->last_heard_millis)); + } +} + +TEST(SimpleMeshTables, CompactRecentPrefixCopiesStayWithinThreeBytes) { + using Info = SimpleMeshTables::RecentRepeaterInfo; + alignas(uint32_t) unsigned char bytes[sizeof(Info) + alignof(uint32_t)] = {}; + for (size_t offset = 0; offset < alignof(uint32_t); ++offset) { + Info* info = new (bytes + offset) Info{}; + info->prefix[0] = 0x86; + info->prefix[1] = 0x0c; + info->prefix[2] = 0xca; + info->prefix_len = 3; + unsigned char out[] = {0xa5, 0xa5, 0xa5, 0xa5, 0xa5}; + SimpleMeshTables::copyRecentRepeaterPrefix(out + 1, *info); + EXPECT_EQ(0xa5, out[0]); + EXPECT_EQ(0x86, out[1]); + EXPECT_EQ(0x0c, out[2]); + EXPECT_EQ(0xca, out[3]); + EXPECT_EQ(0xa5, out[4]); + EXPECT_EQ(0, routeHashPrefixBytesCompare(info->prefix, out + 1, 3)); + out[3] = 0xc9; + EXPECT_GT(routeHashPrefixBytesCompare(info->prefix, out + 1, 3), 0); + EXPECT_LT(routeHashPrefixBytesCompare(out + 1, info->prefix, 3), 0); + } +} + +TEST(SimpleMeshTables, CompactRecentExpiryPreservesUnsignedMillisRollover) { + SimpleMeshTables::RecentRepeaterInfo storage[3]; + SimpleMeshTables table(storage, 3); + const uint8_t older[] = {0x10, 0x20, 0x30}; + const uint8_t boundary[] = {0x11, 0x21, 0x31}; + const uint8_t fresh[] = {0x12, 0x22, 0x32}; + ASSERT_TRUE(table.setRecentRepeater(older, 3, 12)); + ASSERT_TRUE(table.setRecentRepeater(boundary, 3, 8)); + ASSERT_TRUE(table.setRecentRepeater(fresh, 3, 4)); + storage[0].last_heard_millis = 0xffffffe0; + storage[1].last_heard_millis = 0xfffffff0; + storage[2].last_heard_millis = 8; + EXPECT_EQ(1, table.expireRecentRepeaters(16, 32)); + EXPECT_EQ(nullptr, table.findRecentRepeaterByHash(older, 3)); + ASSERT_NE(nullptr, table.findRecentRepeaterByHash(boundary, 3)); + ASSERT_NE(nullptr, table.findRecentRepeaterByHash(fresh, 3)); + EXPECT_EQ(0xfffffff0u, uint32_t(table.findRecentRepeaterByHash(boundary, 3)->last_heard_millis)); + EXPECT_EQ(8u, uint32_t(table.findRecentRepeaterByHash(fresh, 3)->last_heard_millis)); + EXPECT_EQ(2, table.getRecentRepeaterCount()); + EXPECT_EQ(0u, uint32_t(storage[2].last_heard_millis)); +} + // Build a packet that calculatePacketHash() distinguishes by payload content. // header selects ROUTE_TYPE_FLOOD so isRouteDirect() returns false. static Packet makeFloodPacket(uint8_t seed) { @@ -362,7 +441,7 @@ TEST(SimpleMeshTables, CooperativeRecentCursorMatchesUnchangedRankOrder) { rank == 0 ? nullptr : &cursor, cursor_index, found_index); ASSERT_NE(nullptr, found); EXPECT_EQ(t.getRecentRepeaterBySortedIdx(rank), found); - cursor = *found; + SimpleMeshTables::copyRecentRepeaterInfo(cursor, *found); cursor_index = found_index; } int found_index = 123; @@ -382,7 +461,8 @@ TEST(SimpleMeshTables, CooperativeRecentCursorSurvivesLiveStorageCompaction) { int cursor_index = -1; const auto* first_row = t.getNextRecentRepeaterBySortKey(nullptr, -1, cursor_index); ASSERT_NE(nullptr, first_row); - const auto cursor = *first_row; + SimpleMeshTables::RecentRepeaterInfo cursor; + SimpleMeshTables::copyRecentRepeaterInfo(cursor, *first_row); storage[0].last_heard_millis = 0; storage[1].last_heard_millis = 100; storage[2].last_heard_millis = 100; @@ -411,7 +491,7 @@ TEST(SimpleMeshTables, CooperativeRecentCursorTraverses2048Rows) { ASSERT_NE(nullptr, found); EXPECT_EQ(static_cast(i >> 8), found->prefix[1]); EXPECT_EQ(static_cast(i), found->prefix[2]); - cursor = *found; + SimpleMeshTables::copyRecentRepeaterInfo(cursor, *found); cursor_index = found_index; } EXPECT_EQ(nullptr, t.getNextRecentRepeaterBySortKey(&cursor, cursor_index, cursor_index)); diff --git a/test/test_ota_cold_buffers.py b/test/test_ota_cold_buffers.py new file mode 100644 index 000000000..c4bc777eb --- /dev/null +++ b/test/test_ota_cold_buffers.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Verify streaming proofs and real OTA cold-buffer lifetimes without PlatformIO.""" + +from pathlib import Path +import os +import shutil +import subprocess +import tempfile +import unittest + +from test_serial_wifi_sessions import SANITIZER_FLAGS + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = ROOT / "test/fixtures/ota_cold_buffers/test.cpp" + + +class OtaColdBufferTests(unittest.TestCase): + def test_proofs_allocation_failures_and_transfer_lifetimes(self): + compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++") + if not compiler: + self.skipTest("a host C++17 compiler is required") + for platform in ((), ("-DNRF52_PLATFORM=1",), + ("-DNRF52_PLATFORM=1", "-DOTA_SD_STORE=1"), + ("-DESP32_PLATFORM=1",)): + with self.subTest(platform=platform), tempfile.TemporaryDirectory(prefix="ota-cold-") as directory: + binary = Path(directory) / "cold-buffers" + sources = ("OtaManager.cpp", "OtaProtocol.cpp", "MotaContainer.cpp", "MerkleTree.cpp") + built = subprocess.run([ + compiler, "-std=c++17", "-O1", "-Wall", "-Wextra", "-Werror", + "-Wno-unused-parameter", "-Wno-sign-compare", *SANITIZER_FLAGS, *platform, + "-I", str(ROOT / "src"), "-I", str(ROOT / "test/mocks"), + str(FIXTURE), *[str(ROOT / "src/helpers/ota" / name) for name in sources], + str(ROOT / "src/Utils.cpp"), "-Wl,--wrap=malloc", "-Wl,--wrap=free", + "-o", str(binary), + ], capture_output=True, text=True, timeout=60) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + checked = subprocess.run([str(binary)], capture_output=True, text=True, timeout=60) + self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) + self.assertIn("PASS: streaming proofs and OTA cold-buffer lifetimes", checked.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_recent_repeater_storage.py b/test/test_recent_repeater_storage.py new file mode 100644 index 000000000..4582bf241 --- /dev/null +++ b/test/test_recent_repeater_storage.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Exercise real recent-table clock behavior and alignment-safe ARM access.""" + +from pathlib import Path +import os +import re +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + +ARDUINO_CASES = r''' +#include +#include +#include +#include +#include +#include +#include "helpers/SimpleMeshTables.h" + +using Info = SimpleMeshTables::RecentRepeaterInfo; +static_assert(sizeof(Info) == 9 && alignof(Info) == 1, "compact ARM-compatible rows"); +static_assert(std::is_trivially_copyable::value, "clear/compaction/cursor API"); + +int main() { + // A full table must evict the oldest learned time, including a millis wrap. + Info rows[3]; + SimpleMeshTables table(rows, 3); + const uint8_t a[] = {0x10, 0x20, 0x30}, b[] = {0x11, 0x21, 0x31}; + const uint8_t c[] = {0x12, 0x22, 0x32}, d[] = {0x13, 0x23, 0x33}; + g_mock_millis = 0xffffff00; + assert(table.setRecentRepeater(a, 3, -128)); + g_mock_millis = 0xffffff80; + assert(table.setRecentRepeater(b, 3, 127)); + g_mock_millis = 0x10; + assert(table.setRecentRepeater(c, 3, 4)); + g_mock_millis = 0x20; + assert(table.setRecentRepeater(d, 3, 8)); + assert(!table.findRecentRepeaterByHash(a, 3)); + assert(table.findRecentRepeaterByHash(b, 3)); + assert(table.findRecentRepeaterByHash(c, 3)); + assert(table.findRecentRepeaterByHash(d, 3)); + assert(table.getRecentRepeaterCount() == 3); + assert(uint32_t(table.findRecentRepeaterByHash(b, 3)->last_heard_millis) == 0xffffff80); + assert(uint32_t(table.findRecentRepeaterByHash(d, 3)->last_heard_millis) == 0x20); + + // An update must refresh the full timestamp and keep the SNR blending rule. + g_mock_millis = 0xfedcba98; + assert(table.setRecentRepeater(b, 3, -128)); + Info saved; + SimpleMeshTables::copyRecentRepeaterInfo(saved, *table.findRecentRepeaterByHash(b, 3)); + assert(uint32_t(saved.last_heard_millis) == 0xfedcba98); + assert(saved.snr_x4 == 64); // ceil((127*3 - 128)/4) + assert(table.expireRecentRepeaters(0xfedcbaa0, 8) == 2); + assert(table.getRecentRepeaterCount() == 1); + assert(uint32_t(rows[0].last_heard_millis) == 0xfedcba98); + assert(uint32_t(rows[1].last_heard_millis) == 0); + assert(uint32_t(rows[2].last_heard_millis) == 0); + assert(uint32_t(saved.last_heard_millis) == 0xfedcba98); + table.clearRecentRepeaters(); + assert(table.getRecentRepeaterCount() == 0); + assert(uint32_t(rows[0].last_heard_millis) == 0); + + // Every board capacity retains every unique row with no alignment padding. + for (const size_t capacity : {size_t(64), size_t(256), size_t(512), size_t(2048)}) { + Info* storage = new Info[capacity]; + SimpleMeshTables full(storage, capacity); + for (size_t i = 0; i < capacity; ++i) { + const uint8_t prefix[] = {0x80, uint8_t(i >> 8), uint8_t(i)}; + g_mock_millis = 0xff000001u + uint32_t(i); + assert(full.setRecentRepeater(prefix, 3, int8_t(i % 128))); + } + assert(full.getRecentRepeaterCount() == int(capacity)); + for (size_t i = 0; i < capacity; ++i) { + const uint8_t prefix[] = {0x80, uint8_t(i >> 8), uint8_t(i)}; + const Info* found = full.findRecentRepeaterByHash(prefix, 3); + assert(found && uint32_t(found->last_heard_millis) == 0xff000001u + uint32_t(i)); + assert(found->snr_x4 == int8_t(i % 128)); + } + delete[] storage; + } + + // Roles which do not supply a history keep the zero-capacity behavior. + SimpleMeshTables no_storage; + assert(!no_storage.setRecentRepeater(a, 3, 12)); + assert(!no_storage.findRecentRepeaterByHash(a, 3)); + assert(no_storage.expireRecentRepeaters(0xffffffff, 1) == 0); + assert(no_storage.getRecentRepeaterCount() == 0); +} +''' + + +def run_checked(case, args, **kwargs): + result = subprocess.run(args, capture_output=True, text=True, timeout=60, **kwargs) + case.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return result + + +class RecentRepeaterStorageTests(unittest.TestCase): + def test_real_table_arduino_clock_and_all_board_capacities(self): + with tempfile.TemporaryDirectory() as directory: + cpp = Path(directory) / "recent.cpp" + cpp.write_text(ARDUINO_CASES) + for sanitizer in (False, True): + with self.subTest(sanitizer=sanitizer): + binary = Path(directory) / ("recent-san" if sanitizer else "recent") + flags = (["-fsanitize=address,undefined", "-fno-sanitize-recover=all", + "-fno-pie", "-no-pie"] if sanitizer else []) + run_checked(self, [ + "g++", "-std=c++17", "-O2", "-Wall", "-Wextra", "-Werror", + "-Wno-unused-parameter", "-Wno-sign-compare", "-Wno-reorder", + "-DARDUINO=1", *flags, "-I", str(ROOT / "src"), + "-I", str(ROOT / "test/mocks"), str(cpp), + str(ROOT / "src/Packet.cpp"), "-o", str(binary)]) + run_checked(self, [str(binary)]) + + def test_actual_row_arm_copy_clear_prefix_and_timestamp_accesses_are_bytes(self): + compiler = shutil.which("arm-none-eabi-g++") + package_dir = Path(os.environ.get("PLATFORMIO_CORE_DIR", + str(Path.home() / ".platformio"))) / "packages" + compilers = set(package_dir.glob("toolchain-gccarmnoneeabi*/bin/arm-none-eabi-g++")) + if compiler: + compilers.add(Path(compiler)) + if not compilers: + if os.environ.get("MESHCORE_REQUIRE_ARM_COMPILER") == "1": + self.fail("ARM compiler required for compact recent-row alignment checks") + self.skipTest("ARM code generation additionally checked by firmware qualification") + header = (ROOT / "src/helpers/SimpleMeshTables.h").read_text() + start = header.index(" struct RecentRepeaterMillis {") + end = header.index("\nprivate:", start) + # Extract the actual definition: do not duplicate the production wrapper. + source = "#include \n#include \n#include \n#define MAX_ROUTE_HASH_BYTES 3\n" + prefix_start = header.index("inline int routeHashPrefixBytesCompare(") + prefix_end = header.index("\ninline bool routeHashPrefixesOverlap(", prefix_start) + source += header[prefix_start:prefix_end] + "\n" + source += "struct RecentRows {\n" + header[start:end] + "\n};\n" + source += r''' +extern "C" __attribute__((noinline)) uint32_t read_recent( + const RecentRows::RecentRepeaterInfo* rows, unsigned i) { + return rows[i].last_heard_millis; +} +extern "C" __attribute__((noinline)) void write_recent( + RecentRows::RecentRepeaterInfo* rows, unsigned i, uint32_t value) { + rows[i].last_heard_millis = value; +} +extern "C" __attribute__((noinline, flatten)) void copy_recent( + RecentRows::RecentRepeaterInfo* out, const RecentRows::RecentRepeaterInfo* in) { + RecentRows::copyRecentRepeaterInfo(*out, *in); +} +extern "C" __attribute__((noinline, flatten)) void clear_recent( + RecentRows::RecentRepeaterInfo* out) { + RecentRows::clearRecentRepeaterInfo(*out); +} +extern "C" __attribute__((noinline, flatten)) void clear_history( + RecentRows::RecentRepeaterInfo* out) { + for (unsigned i = 0; i < 512; ++i) RecentRows::clearRecentRepeaterInfo(out[i]); +} +extern "C" __attribute__((noinline, flatten)) void copy_prefix( + uint8_t* out, const RecentRows::RecentRepeaterInfo* in) { + RecentRows::copyRecentRepeaterPrefix(out, *in); +} +extern "C" __attribute__((noinline, flatten)) int compare_prefix( + const RecentRows::RecentRepeaterInfo* a, const RecentRows::RecentRepeaterInfo* b) { + return routeHashPrefixBytesCompare(a->prefix, b->prefix, 3); +} +extern "C" __attribute__((noinline, flatten)) uint8_t read_prefix_length( + const RecentRows::RecentRepeaterInfo* rows, unsigned i) { + return rows[i].prefix_len; +} +extern "C" __attribute__((noinline, flatten)) int8_t read_snr( + const RecentRows::RecentRepeaterInfo* rows, unsigned i) { + return rows[i].snr_x4; +} +RecentRows::RecentRepeaterInfo history[512]; +static_assert(sizeof(history) == 4608, "all 512 rows stay available"); +''' + # Prefix writes must also use the actual production implementation. + write_start = header.index(" volatile uint8_t* out_prefix = slot.prefix;") + write_end = header.index(" slot.prefix_len = prefix_len;", write_start) + source += 'extern "C" __attribute__((noinline, flatten)) void write_prefix(\n' + source += " RecentRows::RecentRepeaterInfo& slot, const uint8_t* prefix, uint8_t prefix_len) {\n" + source += header[write_start:write_end] + "}\n" + with tempfile.TemporaryDirectory() as directory: + cpp = Path(directory) / "arm-recent.cpp" + cpp.write_text(source) + for compiler_path in sorted(compilers): + for cpu in ("cortex-m0", "cortex-m4"): + for optimization in ("-Os", "-O2"): + with self.subTest(compiler=str(compiler_path), cpu=cpu, optimization=optimization): + asm = Path(directory) / "arm-recent.s" + run_checked(self, [str(compiler_path), "-std=c++11", "-mcpu=" + cpu, + "-mthumb", optimization, "-Wall", "-Wextra", "-Werror", + "-S", str(cpp), "-o", str(asm)]) + generated = asm.read_text() + for function, operation in (("read_recent", "ldrb"), ("write_recent", "strb")): + body = generated.split(function + ":", 1)[1].split(".size", 1)[0] + self.assertEqual(len(re.findall(r"\b" + operation + r"(?:\.w)?\s", body)), 4, + body) + self.assertNotRegex(body, r"\b(?:ldr|str|ldrh|strh|ldrd|strd|ldm\w*|stm\w*)(?:\.w)?\s") + for function, operation in (("copy_recent", "ldrb"), ("clear_recent", "strb"), + ("clear_history", "strb"), ("copy_prefix", "ldrb"), + ("compare_prefix", "ldrb"), ("write_prefix", "strb"), + ("read_prefix_length", "ldrb"), ("read_snr", "ldr(?:s)?b")): + body = generated.split(function + ":", 1)[1].split(".size", 1)[0] + # Literal-pool LDR and stack PUSH/POP are naturally + # aligned; loads/stores through row pointers must be bytes. + self.assertRegex(body, r"\b" + operation + r"(?:\.w)?\s", body) + self.assertNotRegex(body, r"\b(?:ldr|str|ldrh|strh|ldrd|strd)(?:\.w)?\s[^\n]*\[") + self.assertNotRegex(body, r"\b(?:ldm\w*|stm\w*)\s") + self.assertNotRegex(body, r"\b(?:memcpy|memset|memcmp)\b") + + def test_actual_compaction_and_console_callers_use_byte_helpers(self): + header = (ROOT / "src/helpers/SimpleMeshTables.h").read_text() + source = (ROOT / "examples/simple_repeater/MyMesh.cpp").read_text() + self.assertIn("copyRecentRepeaterInfo(info, _recent_repeaters[_recent_repeater_count])", header) + self.assertIn("clearRecentRepeaterInfo(_recent_repeaters[_recent_repeater_count])", header) + self.assertNotIn("memset(_recent_repeaters", header) + self.assertNotIn("memcpy(slot.prefix", header) + self.assertNotIn("memcmp(info->prefix", header) + self.assertIn("copyRecentRepeaterInfo(serial_recent_cursor, *info)", source) + self.assertNotIn("serial_recent_cursor = *info", source) + self.assertNotIn("toHex(prefix, info->prefix", source) + self.assertEqual(source.count("copyRecentRepeaterPrefix(recorded_prefix, *info)"), 4) + self.assertEqual(source.count("alignas(uint32_t) uint8_t recorded_prefix[MAX_ROUTE_HASH_BYTES]"), 4) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_transport_filter/test_transport_filter.cpp b/test/test_transport_filter/test_transport_filter.cpp index 6ac2f5055..a557b3b61 100644 --- a/test/test_transport_filter/test_transport_filter.cpp +++ b/test/test_transport_filter/test_transport_filter.cpp @@ -426,6 +426,37 @@ TEST_F(TransportFilter, ShorthandAddReusesLongRuleAndDeletePersists) { EXPECT_FALSE(rules.handleCommand(text, reply)); } +TEST_F(TransportFilter, UnnumberedRulesKeepDistinctConfigurations) { + command("del fr all"); + command("set fr any pb=1 d"); + ASSERT_EQ(0, strncmp(reply, "OK", 2)) << reply; + command("set fr any pb=2 d"); + ASSERT_EQ(0, strncmp(reply, "OK", 2)) << reply; + command("get fr.1"); EXPECT_STREQ("set fr.1 any pb=1 d", reply); + command("get fr.2"); EXPECT_STREQ("set fr.2 any pb=2 d", reply); + rules.begin(&fs, nullptr); + command("get fr.1"); EXPECT_STREQ("set fr.1 any pb=1 d", reply); + command("get fr.2"); EXPECT_STREQ("set fr.2 any pb=2 d", reply); +} + +TEST_F(TransportFilter, UnnumberedDuplicateReusesRuleWithLiveRateWindow) { + command("del fr all"); + command("set fr any pb=2 q=1"); + ASSERT_EQ(0, strncmp(reply, "OK", 2)) << reply; + auto p = packet();p.setPathHashSizeAndCount(2, 0); + const auto mask = rules.evaluate(&p, false, true, nullptr, RULE_MODE_RADIO); + ASSERT_NE(0U, mask); + EXPECT_FALSE(rules.shouldBlock(&p, mask, 100)); + rules.commitRates(&p, mask, 100); + EXPECT_TRUE(rules.shouldBlock(&p, mask, 100)); + const auto saved = fs.files["/flood_filter"]; + command("set fr any pb=2 q=1"); + ASSERT_EQ(0, strncmp(reply, "OK", 2)) << reply; + EXPECT_EQ(saved, fs.files["/flood_filter"]); + command("get fr.1"); EXPECT_STREQ("set fr.1 any pb=2 q=1", reply); + command("get fr.2"); EXPECT_EQ(0, strncmp(reply, "Err", 3)); +} + TEST_F(TransportFilter, OversizedCompactReplyNeverReturnsPartialSetter) { // A valid short input can fit in 192 command bytes but exceed a 160-byte reply. const std::string channel = "#" + std::string(30, 'b'), name(30, 'a');