diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 8789644f7..6fe065f1a 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -529,6 +529,7 @@ jobs: python3 -B test/test_mqtt_disconnect.py -v python3 -B test/test_mqtt_connection_admission.py -v python3 -B test/test_mqtt_publish_admission.py -v + python3 -B test/test_mqtt_payload_oom.py -v python3 -B test/test_mqtt_reassembly_buffer.py -v python3 -B test/test_webconfig_json_allocator.py -v python3 -B test/test_mqtt_handshake_scheduler.py -v @@ -1147,6 +1148,7 @@ jobs: - name: Verify nRF52 infrastructure trims and unchanged Companion scope run: | python3 -B test/test_nrf52_flash_trim.py -v + python3 -B test/test_nrf52_vl53l0x_const.py -v python3 -B test/test_nrf52_gnss_version_trim.py -v python3 -B test/test_nrf52_program_lifetime.py -v python3 -B test/test_nrf52_tft_font_trim.py -v diff --git a/build_legacy.sh b/build_legacy.sh index bb34e3e03..2d70f1c54 100755 --- a/build_legacy.sh +++ b/build_legacy.sh @@ -4120,6 +4120,12 @@ apply_lora_ota_no_external_sensors_profile() { for flag in $voltage_monitor_flags; do retain_overrides+=" -D${flag}=1" done + # Keep only this image's readable OTA name in the reduced RAK profiles. + # A non-OTA-prefixed marker is deliberately not copied from a reduced + # logical recipe into the Full sensor pass by apply_logical_ota_tuning_flags. + retain_overrides+=" -DMESH_OTA_LOCAL_TARGET_NAME_ONLY=1" + record_build_reduction \ + "ota target-name catalog omitted; local target name and numeric OTA matching retained" export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS}${omit_overrides}${retain_overrides}" if is_rak_gps_retaining_ota_target "$env_name"; then record_build_reduction \ @@ -4587,6 +4593,11 @@ apply_companion_radio_full_profile() { # Reuse idle queue storage during explicit OTA; retain all 224 normal slots. if [[ "${env_name,,}" != generic_espnow_companion_radio_full ]]; then export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DOTA_SHARED_COMPANION_QUEUE=1 -DONE_KEY_DM_COMPACT_HELD=1" + if [[ "${env_name,,}" == xiao_c3_companion_radio_full ]]; then + # Move 32 of the existing 128 retained slots into byte-accessible + # RTC FAST. Both the 224 normal and 128 active-OTA limits stay intact. + export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DCOMPANION_RTC_FAST_OFFLINE_FRAMES=32" + fi else # Keep all 224 frames during OTA too; RTC FAST holds 32 existing # slots without borrowing the queue or shrinking held-DM storage. diff --git a/docs/WiFi.md b/docs/WiFi.md index 007840204..69d083a56 100644 --- a/docs/WiFi.md +++ b/docs/WiFi.md @@ -94,6 +94,13 @@ normally use no more than two TLS/WSS brokers; some classic ESP32 targets are reliable with only one. Excess configured slots remain saved but show as inactive in `get mqtt.status`. +Five active brokers on a PSRAM board is an upper limit, not a qualified load +for every board. Adding a connection while another broker is connected must +leave contiguous memory for packet publishing as well as SDK startup +allocations. A saved slot can wait for memory while existing connections +continue forwarding. Recovery of the first broker uses a separate bootstrap +bound so this extra reservation cannot strand every disconnected slot. + The bridge has a bounded packet queue for broker or WiFi outages and reconnects automatically. A non-PSRAM build holds 6 packets and a PSRAM build holds 50. When the queue is full, the oldest item is replaced; after five minutes with no @@ -111,7 +118,8 @@ settings remain enabled. Failed requests, a WiFi association, and NTP replies alone do not restart the outage timer or keep a failed MQTT uplink awake. After 30 minutes, the firmware opens a recovery window lasting up to two -minutes. A broker acknowledgment or a connection that stays up for 30 seconds +minutes. A new broker acknowledgment, including one immediately followed by a +disconnect, or a connection that stays up for 30 seconds returns the node to normal operation. Otherwise WiFi is suspended for another 30 minutes. An explicit management request can ask for an earlier probe. Automatic NTP requests have a 15-second timeout; failed requests wait 30 minutes @@ -841,6 +849,7 @@ For an MQTT observer: get wifi.status get mqtt.enabled get mqtt.status +get mqtt.stats get mqtt1.diag get mqtt2.diag ``` @@ -850,6 +859,17 @@ configured slot's `pub_ack` count in `get mqttN.diag`. Accepted publishes are not proof of delivery: a QoS 1 publish is confirmed only when the broker acknowledges it. A quiet radio channel alone is not a failed MQTT connection. +Capture these outputs before rebooting, then repeat `mqtt.stats` and the slot +diagnostics after 60 seconds. `(ok)` reports connection state, not delivery. +In `mqtt.stats`, `drops=A/B` separates outbox-cap drops from packet publication +attempts skipped for memory pressure; a growing `B` can explain missing packet +reports while broker connections still look healthy. `DNS failed (0x8001)` +records a hostname lookup failure, which can follow an earlier disconnect; it +does not identify the reason for that first disconnect. `wifi.status` showing +`suspended` means the shared outage cooldown is active, not that WiFi settings +were erased. These diagnostics do not reveal passwords or tokens, but WiFi +status can include the SSID or local IP address. Redact them before posting. + The shared MQTT bridge recovers failed transports on its owning task, including errors without a subsequent disconnect callback and connection attempts that never finish. It also watches accepted QoS 1 publishes for missing broker @@ -859,6 +879,19 @@ See the [MQTT recovery regression](test-results/mqtt_tls_v4_regression.md#long-r for the deadlines, test coverage and hardware limitations. Older preview images do not gain these changes until their firmware is updated. +On PSRAM-capable boards, MQTT receive reassembly and WebConfig JSON workspaces +prefer PSRAM, preserving internal memory for WiFi, task stacks and DMA. Packet, +raw and status publishing use the same memory admission check. A failed status +send retries only that broker every 30 seconds; successful brokers keep their +normal status interval. Each slot has a stable, distinct client ID, including +when two slots point at the same broker. + +Outbox diagnostics are cached by the network owner, so `get mqtt.stats` does +not wait for an SDK mutex held by a network write. `rec:N/write` records a +failed-write recovery; `no errors` describes the current connected session, +and `first_disc` is the age of the first disconnect after boot. Neither is a +claim that no earlier failure occurred. + Common causes are: - no saved SSID or a changed password; diff --git a/docs/firmware_picker.md b/docs/firmware_picker.md index f9e1ea8c9..97e783693 100644 --- a/docs/firmware_picker.md +++ b/docs/firmware_picker.md @@ -581,6 +581,9 @@ The legacy `no_external_sensors` profile is labeled **Receives LoRa OTA - Reduced optional sensors** on the same button. On RAK3401 and RAK4631 this compact profile uses internal storage without an external storage board. The reduction does not disable generic I2C or unrelated board-integrated peripherals. +These reduced RAK profiles keep their own readable OTA target name and omit +the all-board name catalog. Updates for other targets display numeric IDs; +OTA compatibility checks and transfers are unchanged. Reduced RAK3401 and RAK4631 targets retain INA219/INA226/INA260/INA3221 as voltage/current entries in the optional sensor table. They are not the only I2C users: the SSD1306 OLED, supported autodiscovered RTCs, and RAK12500 GPS remain diff --git a/docs/research/firmware_memory_budget.md b/docs/research/firmware_memory_budget.md index 5c63cc821..46a615156 100644 --- a/docs/research/firmware_memory_budget.md +++ b/docs/research/firmware_memory_budget.md @@ -119,19 +119,34 @@ qualification of five simultaneous PSRAM connections or two internal TLS connections. The `mqtt` report also names the production runtime admission thresholds. -A cold SDK start requires at least 32 KiB of DMA-capable internal free heap -and an 8 KiB contiguous DMA block. This includes a separate 16 KiB allowance -for its SDK task stack/TCB, receive/transmit/reassembly buffers and transport -or configuration objects, alongside the 16 KiB AES reserve. Those ordinary -SDK allocations occur after admission and do not use the TLS allocation guard. -A retained initialized SDK handle with a stopped task requires 24 KiB free and -an 8 KiB contiguous DMA block. Its separate 8 KiB restart allowance covers the -pinned 6 KiB task stack plus TCB and allocator overhead. SDK stop retains its -receive/transmit buffers, configuration, transport objects and wrapper -reassembly buffer, so restarting that handle does not need to reserve their -first-initialization allocation again. A wrapper whose SDK initialization has -not completed still receives the full cold allowance. This distinction also -applies after WiFi loss stops the SDK tasks but keeps their handles. +Connection admission and the packet-publishing guard share a contiguous-block +floor: 4,096 DMA-capable bytes with detected PSRAM, or 8,000 internal bytes without it. Admitting +another broker while one is connected reserves its startup allocations beside +that floor, protecting existing packet forwarding. The first connection after +boot or an outage can bootstrap without that extra reservation, so the +additional-broker reserve cannot block every disconnected slot's recovery. + +A cold SDK start requires at least 32 KiB of DMA-capable internal free heap. +The separate 16 KiB allowance covers its SDK task stack/TCB, +receive/transmit/reassembly buffers and transport or configuration objects, +alongside the 16 KiB AES reserve. Those ordinary allocations occur after +admission and do not use the TLS allocation guard. Bootstrap needs a contiguous +8,192-byte block for plain MQTT. With another connected +broker, the contiguous minimum includes its publication floor plus the startup +allowance: 20,480 bytes with PSRAM or 24,384 without it, before the additional +internal TLS requirement below. + +A retained initialized SDK handle with a stopped task requires 24 KiB free. +Its separate 8 KiB restart allowance covers the pinned 6 KiB task stack plus +TCB and allocator overhead. Bootstrap uses the same 8,192-byte block +minimum as a cold plain-MQTT start. With another connected broker, its contiguous minimum +is 12,288 bytes with PSRAM or 16,192 without it, before the internal TLS +requirement below. SDK stop retains its receive/transmit buffers, +configuration, transport objects and wrapper reassembly buffer, so restarting +that handle does not need their first-initialization allocation again. A +wrapper whose SDK initialization has not completed still receives the full +cold allowance. This distinction also applies after WiFi loss stops SDK tasks +but keeps their handles. All ESP32 clients use 896-byte receive, transmit and wrapper reassembly buffers from first initialization. Applying configuration cannot grow the retained SDK @@ -143,15 +158,37 @@ and cold admission covers the ordinary allocations. WiFi recovery and explicit reconfiguration continue to retain clients. A warm reconnect keeps its running task and buffers; it requires at least -16 KiB free and a 4 KiB contiguous DMA block. The per-client SDK stack is -independent of `MQTT_TASK_STACK_SIZE`, which sizes the existing bridge worker. -A TLS attempt without PSRAM requires at least -60 KiB free and a 17,408-byte contiguous block. PSRAM allocation fallback must -leave the 16 KiB DMA reserve available. These thresholds describe free heap at -the time of admission; they are not additional linked capacity and the TLS -contiguous threshold is not a startup allocation requirement. Available linked -internal heap cannot establish how much DMA-capable heap remains or how it is -fragmented after boot. Only runtime observations can establish that. +16 KiB free and the same 4,096/8,000-byte publication block. The per-client SDK +stack is independent of `MQTT_TASK_STACK_SIZE`, which sizes the existing bridge +worker. TLS with detected PSRAM retains its separate 16,000-byte +connection block: incoming handshake records are not bounded by the outgoing +MQTT buffer. With another connected broker this remains 32,384 bytes for a cold +start, 24,192 for a stopped task, and 16,000 for a warm reconnect. TLS without +detected PSRAM retains its 60 KiB free-heap and 17,408-byte +record-block floors for bootstrap. With another connected broker, admission +reserves that record beside the publication block and ordinary startup +allocations: 41,792 contiguous bytes for a cold start, 33,600 for a stopped +task, or 25,408 for a warm reconnect. Both admission and publication use +detected PSRAM, including a board whose compiled-in PSRAM support failed to +initialize. + +PSRAM builds allocate WebConfig JSON workspaces and the MQTT wrapper's receive +reassembly bytes in PSRAM. Task stacks and SDK/DMA owners remain internal. +The shared packet, raw and status publish check preserves 16 KiB free DMA +memory and a 4 KiB DMA block. ESP-MQTT fragments synchronous QoS 0 writes into +its pinned 896-byte SDK buffer; the JSON scratch is already in PSRAM or on the +worker stack. Queued QoS 1 records can be written whole, so larger records +raise the required DMA block and total reserve by their size. Admission also +charges the queued record itself before enqueue, then resamples before the +next slot because small SDK outbox allocations can consume internal RAM. These bounds +follow the [pinned ESP-MQTT implementation](https://github.com/espressif/esp-mqtt/blob/bb9c8af9d552b608dd3aabf9617bde757a538ebe/mqtt_client.c). + +PSRAM allocation fallback must leave the 16 KiB DMA reserve available. These +conservative thresholds describe free heap at admission; they are not extra +linked capacity or a guarantee against later fragmentation or allocations by +other tasks. Available linked internal heap cannot establish how much +DMA-capable heap remains or how it is fragmented after boot. Only runtime +observations can establish that. Maximum-load qualification still needs physical boot and sustained reconnect, publish and configuration-change tests with the selected number of brokers, diff --git a/docs/test-results/mqtt_tls_v4_regression.md b/docs/test-results/mqtt_tls_v4_regression.md index 68ff349ad..0f9188010 100644 --- a/docs/test-results/mqtt_tls_v4_regression.md +++ b/docs/test-results/mqtt_tls_v4_regression.md @@ -265,3 +265,51 @@ measured with three live brokers or qualification of maximum simultaneous load. No new physical three-hour V4 soak or reporter-side retest has been performed as part of this repair. These tests establish the repaired recovery behavior, not the cause of the reporter's hardware failure or website-side availability. + +## Station G3 follow-up: early failures and network suspension + +A user supplied a failed-state console capture from `1.18.0.1` with five WSS +presets configured. It shows all five slots disconnected and WiFi suspended +with the 30-minute recovery schedule. The displayed uptime and first-disconnect +ages place the first two failures roughly 28 and 89 seconds after boot, before +the reported seven-to-eight-minute loss of reporting. + +The slot diagnostics record repeated completed transport recoveries and DNS +lookup failures (`0x8001`). This establishes active recovery followed by the +outage cooldown, rather than an indefinitely frozen worker in this capture. +The DNS errors are the latest failures, not proof of the original disconnect +cause. The stats also record 345 packet publication attempts skipped for +memory pressure (`drops=0/345`), an empty SDK outbox and a growing bridge queue. +These skips are attempts, including retries, rather than 345 distinct packets. +Heap readings after clients were stopped do not establish their earlier +connected-state headroom. + +Two source issues were identified: + +- Connection admission could permit another SDK client below the contiguous + memory threshold required by packet publishing. When another slot is + connected, the shared admission policy now protects that publishing + threshold and accounts for allocations made during SDK startup or restart. + It defers additional connections under pressure. First-broker recovery has + a separate bootstrap bound, retaining a way to reconnect when all slots + have disconnected. Neither path lowers the publishing safety threshold or + erases slots; admission bounds do not guarantee future allocations. +- A new PUBACK followed by DISCONNECTED between worker passes was consumed + without crediting the successful delivery to the outage timer. The worker + now credits that reply once, even if the connection has since dropped. + CONNACK-only flapping and old acknowledgment counts cannot renew the timer. + +These repairs do not establish why the user's DNS resolution failed. Physical +G3 reproduction and a reporter-side retest remain necessary before calling +that reported failure resolved. The shared changes apply to all ESP32 roles +that include this MQTT bridge. + +Validation passed: 85 MQTT Python host cases, four Companion MQTT host cases +and 57 firmware RAM policy cases. The new disconnected-PUBACK regression +fails with the old condition and passes with the fix, including timer wrap, +disabled slots and one-time acknowledgment consumption. Admission tests use +the captured 93,024-byte free DMA heap and 20,468-byte largest block: one +stopped broker can attempt recovery, while an additional broker waits when +another slot is connected. The Station G3 Full repeater compiled and passed +capability, dual-OTA partition and linked RAM checks. These are host/build +checks, not a physical five-broker G3 soak. diff --git a/examples/companion_radio/MyMesh.h b/examples/companion_radio/MyMesh.h index 68371f067..a15321283 100644 --- a/examples/companion_radio/MyMesh.h +++ b/examples/companion_radio/MyMesh.h @@ -1003,18 +1003,7 @@ private: int offline_queue_head; // Local onboarding only: RAM-backed like the receive queue, not RF traffic. uint8_t pending_emergency_notices[(MAX_GROUP_CHANNELS + 7) / 8] = {}; -#if defined(OTA_SHARED_COMPANION_QUEUE) - static constexpr unsigned OTA_RETAINED_OFFLINE_FRAMES = 128; - static_assert(OFFLINE_QUEUE_SIZE > OTA_RETAINED_OFFLINE_FRAMES, - "Shared mOTA queue must retain 128 frames and provide workspace storage"); - // The buffer also asserts that the complete context fits in the remaining - // frames. Constrained ESP32 recipes keep 224 normal slots; nRF52 and Paper keep 256. - mesh::BorrowableFrameBuffer offline_queue; - static mesh::ota::OtaContext* acquireOfflineQueueForOta(void* owner); - static uint16_t drainOfflineQueueForOta(void* owner); - static void releaseOfflineQueueFromOta(void* owner); -#elif COMPANION_RTC_FAST_OFFLINE_FRAMES || COMPANION_RTC_SLOW_OFFLINE_FRAMES +#if COMPANION_RTC_FAST_OFFLINE_FRAMES || COMPANION_RTC_SLOW_OFFLINE_FRAMES // Concrete definitions in MyMesh.cpp keep GCC from moving template-local // statics back into ordinary .bss despite the requested RTC attribute. #if COMPANION_RTC_FAST_OFFLINE_FRAMES @@ -1037,6 +1026,23 @@ private: return nullptr; #endif } +#endif +#if defined(OTA_SHARED_COMPANION_QUEUE) + static constexpr unsigned OTA_RETAINED_OFFLINE_FRAMES = 128; + static_assert(OFFLINE_QUEUE_SIZE > OTA_RETAINED_OFFLINE_FRAMES, + "Shared mOTA queue must retain 128 frames and provide workspace storage"); + // The buffer also asserts that the complete context fits in the remaining + // frames. Constrained ESP32 recipes keep 224 normal slots; nRF52 and Paper keep 256. + mesh::BorrowableFrameBuffer +#endif + > offline_queue; + static mesh::ota::OtaContext* acquireOfflineQueueForOta(void* owner); + static uint16_t drainOfflineQueueForOta(void* owner); + static void releaseOfflineQueueFromOta(void* owner); +#elif COMPANION_RTC_FAST_OFFLINE_FRAMES || COMPANION_RTC_SLOW_OFFLINE_FRAMES mesh::CompanionRtcQueue offline_queue; #elif defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM) enum { diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 5d770a7a2..00afb61de 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -60,9 +60,10 @@ mesh::FloodAdvertLimiter::Entry* repeaterRtcAdvertSlowBank() { #endif #include #include +#include #include #include // std::nothrow (heap-allocated flood rule table) -#include // for qsort() +#include #include #include #if defined(NRF52_POWER_MANAGEMENT) @@ -707,7 +708,7 @@ void MyMesh::sendDefaultPasswordWarning(ClientInfo* client, } } -// Comparison functions for qsort() - defined at file scope to avoid heap allocations +// Shared neighbour ordering; equally ranked entries have no guaranteed tie order. static int cmp_neighbours_newest_to_oldest(const void* a, const void* b) { const NeighbourInfo* na = *(const NeighbourInfo**)a; const NeighbourInfo* nb = *(const NeighbourInfo**)b; @@ -942,21 +943,20 @@ int MyMesh::handleRequest(ClientInfo *sender, uint32_t sender_timestamp, uint8_t } } - // Sort neighbours based on order using qsort() - standard C library function - // qsort() doesn't allocate heap memory (uses stack-based recursion) and is O(n log n) - // This matches the pattern used elsewhere in the codebase (e.g., BaseChatMesh) + // Share the bounded O(n log n) sorter instead of retaining separate C + // and C++ sort implementations on flash-constrained boards. if (order_by == 0) { // sort by newest to oldest - qsort(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_newest_to_oldest); + mesh::sortSmallTable(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_newest_to_oldest); } else if (order_by == 1) { // sort by oldest to newest - qsort(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_oldest_to_newest); + mesh::sortSmallTable(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_oldest_to_newest); } else if (order_by == 2) { // sort by strongest to weakest - qsort(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_strongest_to_weakest); + mesh::sortSmallTable(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_strongest_to_weakest); } else if (order_by == 3) { // sort by weakest to strongest - qsort(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_weakest_to_strongest); + mesh::sortSmallTable(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_weakest_to_strongest); } // build results buffer @@ -6023,9 +6023,8 @@ void MyMesh::formatNeighborsReply(char *reply) { } // sort neighbours newest to oldest - std::sort(sorted_neighbours, sorted_neighbours + neighbours_count, [](const NeighbourInfo* a, const NeighbourInfo* b) { - return a->heard_timestamp > b->heard_timestamp; // desc - }); + mesh::sortSmallTable(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), + cmp_neighbours_newest_to_oldest); for (int i = 0; i < neighbours_count && dp - reply < 134; i++) { NeighbourInfo *neighbour = sorted_neighbours[i]; diff --git a/scripts/check_firmware_ram.py b/scripts/check_firmware_ram.py index 092cb61fd..58b7746d7 100644 --- a/scripts/check_firmware_ram.py +++ b/scripts/check_firmware_ram.py @@ -67,6 +67,9 @@ def mqtt_requirements(defines): # slots are lazy and each attempt must pass the production admission guard. startup = MQTT_STARTUP_ALLOWANCE + max(0, worker_stack - MQTT_WORKER_STACK_BYTES) active = min(slots, 5 if psram else 2) + # Match the pinned SDK's bounded QoS0 wire-buffer/DMA scratch floor. + # Larger QoS1 outbox records have a separate per-payload runtime guard. + publish_contiguous = 4096 if psram else 8000 potential = startup + max(0, active - 2) * (MQTT_CLIENT_STACK_BYTES + MQTT_CLIENT_BUFFER_BYTES) return { "budget_scope": "startup minimum with runtime connection admission", @@ -82,18 +85,33 @@ def mqtt_requirements(defines): "maximum_slot_stack_buffer_allowance_bytes": potential, "cold_start_allocation_allowance_bytes": 16384, "cold_start_dma_admission_free_bytes": 32768, - "cold_start_dma_admission_contiguous_bytes": 8192, + "cold_start_dma_admission_contiguous_bytes": max(publish_contiguous, 8192), + "additional_broker_cold_start_dma_admission_contiguous_bytes": publish_contiguous + 16384, "restart_allocation_allowance_bytes": 8192, "restart_dma_admission_free_bytes": 24576, - "restart_dma_admission_contiguous_bytes": 8192, + "restart_dma_admission_contiguous_bytes": max(publish_contiguous, 8192), + "additional_broker_restart_dma_admission_contiguous_bytes": publish_contiguous + 8192, "dma_admission_free_bytes": 16384, - "dma_admission_contiguous_bytes": 4096, + "dma_admission_contiguous_bytes": publish_contiguous, + "publish_contiguous_bytes": publish_contiguous, + "publish_contiguous_without_psram_bytes": 8000, + "publish_contiguous_with_psram_bytes": 4096, + # Incoming TLS records are independent of bounded outgoing QoS0 chunks. + "tls_psram_admission_contiguous_bytes": 16000, + "additional_broker_cold_start_tls_psram_admission_contiguous_bytes": 16000 + 16384, + "additional_broker_restart_tls_psram_admission_contiguous_bytes": 16000 + 8192, + "additional_broker_tls_psram_admission_contiguous_bytes": 16000, "tls_internal_admission_free_bytes": 61440, + "tls_record_allocation_bytes": 17408, + "additional_broker_cold_start_tls_internal_admission_contiguous_bytes": 8000 + 16384 + 17408, + "additional_broker_restart_tls_internal_admission_contiguous_bytes": 8000 + 8192 + 17408, "tls_internal_admission_contiguous_bytes": 17408, + "additional_broker_tls_internal_admission_contiguous_bytes": 8000 + 17408, "maximum_runtime_load_qualified": False, "physical_validation_performed": False, "limitations": "Maximum TLS/WSS/outbox workload is not covered by the startup allowance; " - "runtime free DMA heap and fragmentation must satisfy connection admission.", + "runtime free DMA heap and fragmentation must satisfy connection admission; " + "additional brokers reserve publication headroom beside startup allocations.", } diff --git a/scripts/client_acl_role.py b/scripts/client_acl_role.py index e7dc98786..995884297 100644 --- a/scripts/client_acl_role.py +++ b/scripts/client_acl_role.py @@ -1,12 +1,14 @@ -"""Use the smaller ClientInfo layout only for an effective repeater source role. +"""Derive feature-preserving role definitions from effective application sources. -One shared compiler definition covers application and helper translation units. +Shared compiler definitions cover application and helper translation units. Unknown/custom roles keep the complete layout; room and sensor session fields -are never removed from builds that use them. +are never removed from builds that use them. Companions omit only unbound +forwarding-region callbacks because that role has no RegionMap. """ import importlib.util from pathlib import Path +import re SCRIPT_DIR = Path(__file__).resolve().parent if "__file__" in globals() else Path("scripts").resolve() @@ -18,6 +20,39 @@ DEFINITION = "MESH_CLIENT_REPEATER_ONLY" REPEATER_SOURCES = ("../examples/simple_repeater/main.cpp", "../examples/simple_repeater/MyMesh.cpp") OTHER_ROLES = ("simple_room_server", "simple_sensor", "companion_radio", "simple_secure_chat", "kiss_modem", "partition_expander", "esp32_partition_migrator", "esp32_partition_legacy_seed") +COMPANION_DEFINITION = "MESH_ROLE_COMPANION" +COMPANION_SOURCES = ("../examples/companion_radio/main.cpp", "../examples/companion_radio/MyMesh.cpp") + + +def companion_role_policy(src_filter, flags=(), cppdefines=()): + """Exclude unbound forwarding-scope callbacks, retaining all radio commands. + + Companions have no RegionMap. Select the complete exclusive source role, + never an environment-name alias, so every translation unit shares its ABI. + Unknown and mixed roles retain the general callback implementation. + """ + values = SOURCE_POLICY.flag_defines(flags, cppdefines) + selected = SOURCE_POLICY.source_selected + other_roles = ("simple_repeater", *(role for role in OTHER_ROLES + if role != "companion_radio")) + filter_text = src_filter if isinstance(src_filter, str) else " ".join(map(str, src_filter)) + example_includes = re.findall(r"\+<\.\./examples/([^>]+)>", filter_text.replace("\\", "/")) + known_companion_only = all(path == "companion_radio" or path.startswith("companion_radio/") + for path in example_includes) + qualified = (known_companion_only + and all(selected(src_filter, source) for source in COMPANION_SOURCES) + and not any(selected(src_filter, "../examples/" + role + "/" + source) + for role in other_roles + for source in ("main.cpp", "MyMesh.cpp", "SensorMesh.cpp"))) + if COMPANION_DEFINITION in values: + value = values[COMPANION_DEFINITION] + text = "0" if value is None else str(value).strip("\"'() ") + if text not in ("0", "1"): + raise ValueError(COMPANION_DEFINITION + " override must be 0 or 1") + if text == "1" and not qualified: + raise ValueError(COMPANION_DEFINITION + "=1 requires only the Companion source role") + return {} + return {COMPANION_DEFINITION: 1} if qualified else {} def role_policy(src_filter, flags=(), cppdefines=()): @@ -46,6 +81,8 @@ def install(build_env): src_filter = build_env.GetProjectOption("build_src_filter", "") definitions = role_policy(src_filter, build_env.get("BUILD_FLAGS", ()), build_env.get("CPPDEFINES", ())) + definitions.update(companion_role_policy(src_filter, build_env.get("BUILD_FLAGS", ()), + build_env.get("CPPDEFINES", ()))) if definitions: build_env.AppendUnique(CPPDEFINES=list(definitions.items())) diff --git a/scripts/nrf52_flash_trim.py b/scripts/nrf52_flash_trim.py index 4d1a19f4b..90a48c3f4 100644 --- a/scripts/nrf52_flash_trim.py +++ b/scripts/nrf52_flash_trim.py @@ -141,6 +141,20 @@ def trim_policy(platform, mcu, env_name, src_filter, flags=(), cppdefines=(), if ota_name_mode != "front-coded": raise ValueError("unsupported nRF52 OTA name trim policy: " + ota_name_mode) definitions = dict(TRIM_DEFINES, OTA_TARGET_NAME_FRONT_CODED=1) + if _enabled(values.get("MESH_OTA_LOCAL_TARGET_NAME_ONLY")): + # Release overlays can compile a different PIOENV from their deployed + # OTA identity. Keep the logical name/ID in that case; direct builds + # use the same PIOENV hash as the ordinary EndF/board identity code. + target_id = values.get("MOTA_TARGET_ID") + if target_id is None: + target_id = "0x%08x" % int.from_bytes( + hashlib.sha256(str(env_name).encode()).digest()[:4], "little") + definitions.update( + OTA_TARGET_NAME_TABLE=0, + OTA_LOCAL_TARGET_ID=target_id, + OTA_LOCAL_TARGET_NAME="OTA_VARIANT" if values.get("OTA_VARIANT") + else '"%s"' % env_name, + ) # Explicit board/user flags retain their value. In particular never undo # an opt-out or a board-specific USB configuration by re-defining it. return {name: value for name, value in definitions.items() if name not in values} @@ -241,10 +255,21 @@ def install(build_env): build_env.get("SRC_FILTER", ()), flags, cppdefines) if not definitions: return + local_name = definitions.get("OTA_LOCAL_TARGET_NAME", "") + if local_name.startswith('"') and local_name.endswith('"'): + definitions["OTA_LOCAL_TARGET_NAME"] = build_env.StringifyMacro(local_name[1:-1]) build_env["MESH_NRF52_FLASH_TRIM_ACTIVE"] = True build_env.AppendUnique(CPPDEFINES=list(definitions.items())) + hook_path = build_env.subst("$PROJECT_DIR/scripts/nrf52_vl53l0x_const.py") + hook_spec = importlib.util.spec_from_file_location("mesh_nrf52_vl53_const", hook_path) + hook = importlib.util.module_from_spec(hook_spec) + hook_spec.loader.exec_module(hook) + hook.install(build_env) if infrastructure: - print("nRF52 infrastructure flash trim: compact crypto, unused USB device classes/GPS cache/vendor splash, compact OTA target names") + names = ("local OTA target name only" if definitions.get("OTA_TARGET_NAME_TABLE") == 0 + or flag_defines(flags, cppdefines).get("OTA_TARGET_NAME_TABLE") == "0" + else "compact OTA target names") + print("nRF52 infrastructure flash trim: compact crypto, unused USB device classes/GPS cache/vendor splash, " + names) else: print("nRF52 Companion flash trim: small crypto table, unused USB device classes/GPS cache/vendor splash; fonts and Reader retained") hook_path = build_env.subst("$PROJECT_DIR/scripts/nrf52_gnss_version_trim.py") diff --git a/scripts/nrf52_vl53l0x_const.py b/scripts/nrf52_vl53l0x_const.py new file mode 100644 index 000000000..0d52ae433 --- /dev/null +++ b/scripts/nrf52_vl53l0x_const.py @@ -0,0 +1,81 @@ +"""Keep the reviewed VL53L0X tuning tables in nRF52 flash. + +The vendor tuning loader only reads these bytes and copies register writes +through a local buffer. Compile a private dependency copy with const inputs; +leave its public caller-owned tuning-buffer setter/getter unchanged. Unknown +library versions retain their ordinary implementation and RAM requirements. +""" + +import hashlib +from pathlib import Path +import shutil + + +FILES = { + "vl53l0x_tuning.h": "0e0331b7df1084d183248355f75d835b3446200b22f71765b5c63a470ff01c63", + "vl53l0x_interrupt_threshold_settings.h": "9e5e6c00af66497d03d6c6a13a269521c12b33c90455efcd3b8494487e1780c2", + "vl53l0x_api_core.h": "948de4a0363339facc58c4e1bc5781b954b99cca1c13a4a105eb3969aeba8653", + "core/src/vl53l0x_api_core.cpp": "d31da82edbad0f7fd89106340f7a006b61cd2e5b7c353460fb394978ce55953b", + "core/src/vl53l0x_api.cpp": "6cfa0784600bd797e372b511761f4be10e97264b55d5c7269dd41ba607b8edb7", +} +REPLACEMENTS = { + "vl53l0x_tuning.h": ("uint8_t DefaultTuningSettings[]", "const uint8_t DefaultTuningSettings[]"), + "vl53l0x_interrupt_threshold_settings.h": ( + "uint8_t InterruptThresholdSettings[]", "const uint8_t InterruptThresholdSettings[]"), + "vl53l0x_api_core.h": ("uint8_t* pTuningSettingBuffer);", "const uint8_t* pTuningSettingBuffer);"), + "core/src/vl53l0x_api_core.cpp": ( + "uint8_t* pTuningSettingBuffer) {", "const uint8_t* pTuningSettingBuffer) {"), + "core/src/vl53l0x_api.cpp": ("uint8_t* pTuningSettingBuffer;", "const uint8_t* pTuningSettingBuffer;"), +} + + +def digest(text): + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def patched_files(source_root): + result = {} + for name, expected in FILES.items(): + text = (source_root / name).read_text(encoding="utf-8").replace("\r\n", "\n") + old, new = REPLACEMENTS[name] + if digest(text) != expected or text.count(old) != 1: + raise RuntimeError("changed reviewed VL53L0X dependency: " + name) + result[name] = text.replace(old, new, 1) + return result + + +def write_if_changed(path, content): + if not path.exists() or path.read_text(encoding="utf-8") != content: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def replace_vl53_source(build_env, node): + source = Path(node.srcnode().get_abspath()) + source_root = next((parent for parent in source.parents + if parent.name == "src" and parent.parent.name == "Adafruit_VL53L0X"), None) + if source_root is None or source.suffix != ".cpp": + return node + try: + patches = patched_files(source_root) + except (OSError, UnicodeError, RuntimeError) as error: + print("nRF52 VL53L0X const tables: ordinary source retained (" + str(error) + ")") + return node + destination = Path(build_env.subst("$BUILD_DIR")) / "readonly-nrf52-vl53l0x" / "src" + # Keep every quoted sibling/relative include inside the same private copy. + # All compilation units therefore see the same const loader declaration. + for path in source_root.rglob("*"): + if not path.is_file(): + continue + relative = path.relative_to(source_root) + output = destination / relative + if relative.as_posix() in patches: + write_if_changed(output, patches[relative.as_posix()]) + elif not output.exists() or output.read_bytes() != path.read_bytes(): + output.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, output) + return build_env.File(str(destination / source.relative_to(source_root))) + + +def install(build_env): + build_env.AddBuildMiddleware(replace_vl53_source, "*Adafruit_VL53L0X*/*.cpp") diff --git a/src/helpers/BorrowableFrameBuffer.h b/src/helpers/BorrowableFrameBuffer.h index 16cdc1a8f..32fda417d 100644 --- a/src/helpers/BorrowableFrameBuffer.h +++ b/src/helpers/BorrowableFrameBuffer.h @@ -6,16 +6,28 @@ namespace mesh { +template class InlineRetainedFrames { + Frame frames[Count]; +public: + static constexpr size_t frame_count = Count; + Frame& at(size_t index) { return frames[index]; } +}; + // One owner at a time: an offline FIFO borrows the workspace's storage while // idle. Switching owners compacts the ring in place, without allocating RAM or // dropping unread frames. The caller owns the FIFO's length and head index. -template +// Qualified boards can supply a physically separate retained-frame bank. It +// replaces only existing retained slots, never the extra frames or workspace. +template> class BorrowableFrameBuffer { static_assert(Reserved > 0 && Reserved < Capacity, "Invalid shared queue capacities"); + static_assert(RetainedFrames::frame_count == Reserved, + "The retained backing must preserve every active-OTA slot"); struct ExtraFrames { Frame frames[Capacity - Reserved]; }; static_assert(sizeof(Workspace) <= sizeof(ExtraFrames), "mOTA exceeds borrowed queue storage"); - Frame _retained[Reserved]; + RetainedFrames _retained; union Extra { ExtraFrames queue; Workspace workspace; @@ -45,6 +57,7 @@ class BorrowableFrameBuffer { public: BorrowableFrameBuffer() = default; + BorrowableFrameBuffer(Frame* fast, Frame* slow) : _retained(fast, slow) {} BorrowableFrameBuffer(const BorrowableFrameBuffer&) = delete; BorrowableFrameBuffer& operator=(const BorrowableFrameBuffer&) = delete; ~BorrowableFrameBuffer() { @@ -53,7 +66,7 @@ public: size_t capacity() const { return _borrowed ? Reserved : Capacity; } Frame& at(size_t physical_index) { - return physical_index < Reserved ? _retained[physical_index] + return physical_index < Reserved ? _retained.at(physical_index) : _extra.queue.frames[physical_index - Reserved]; } diff --git a/src/helpers/CompanionRtcQueue.h b/src/helpers/CompanionRtcQueue.h index 4ac374675..692fe75ce 100644 --- a/src/helpers/CompanionRtcQueue.h +++ b/src/helpers/CompanionRtcQueue.h @@ -16,8 +16,7 @@ #if !defined(ESP32_PLATFORM) || !defined(COMPANION_RADIO_FULL) #error "RTC offline frames require a qualified ESP32 Full Companion recipe" #endif -#if defined(BOARD_HAS_PSRAM) || defined(OTA_SHARED_COMPANION_QUEUE) \ - || ONE_KEY_DM_SHARED_OFFLINE_QUEUE +#if defined(BOARD_HAS_PSRAM) || ONE_KEY_DM_SHARED_OFFLINE_QUEUE #error "RTC offline frames require the independent fixed-capacity queue" #endif #include @@ -36,6 +35,10 @@ #if COMPANION_RTC_SLOW_OFFLINE_FRAMES && !defined(CONFIG_IDF_TARGET_ESP32S3) #error "The separate RTC SLOW offline-frame bank requires ESP32-S3" #endif +#if defined(OTA_SHARED_COMPANION_QUEUE) \ + && (!defined(CONFIG_IDF_TARGET_ESP32C3) || COMPANION_RTC_SLOW_OFFLINE_FRAMES) +#error "RTC retained shared-OTA frames are qualified only on ESP32-C3 FAST memory" +#endif namespace mesh { namespace companion_rtc { @@ -75,6 +78,7 @@ template class CompanionRtcQueue { } public: + static constexpr size_t frame_count = Count; CompanionRtcQueue(Frame* fast, Frame* slow) : fast_frames(fast), slow_frames(slow) { if ((FastCount && (!fast || reinterpret_cast(fast) % 4)) || (SlowCount && (!slow || reinterpret_cast(slow) % 4))) abort(); diff --git a/src/helpers/MQTTConnectionAdmission.h b/src/helpers/MQTTConnectionAdmission.h index 1338e7157..915fd0f86 100644 --- a/src/helpers/MQTTConnectionAdmission.h +++ b/src/helpers/MQTTConnectionAdmission.h @@ -11,6 +11,18 @@ namespace MQTTConnectionAdmission { static const size_t kDmaReserveBytes = 16384; static const size_t kDmaLargestBytes = 4096; +// Keep the packet-publishing guard and connection admission on the same floor. +// A new optional broker must not consume the contiguous block that existing +// brokers need to continue forwarding packets. +// SDK QoS0 writes are fragmented into the pinned 896-byte wire buffer. +// PSRAM-backed JSON and TLS need DMA scratch, not another 16 KiB JSON block. +static const size_t kPsramPublishLargestBytes = 4096; +static_assert(kPsramPublishLargestBytes == kDmaLargestBytes, + "Connection and publish DMA scratch floors must agree"); +// Incoming TLS handshake/records are not bounded by our outgoing MQTT buffer. +// Preserve the existing TLS connection floor independently of publication. +static const size_t kPsramTlsLargestBytes = 16000; +static const size_t kInternalPublishLargestBytes = 8000; // A stopped SDK client needs a new internal task stack/TCB. First starts also // allocate receive/transmit/reassembly buffers and SDK transport/config objects // after admission. Budget that ordinary allocation separately from AES headroom. @@ -34,26 +46,50 @@ enum class ClientMemoryState : uint8_t { Started, }; +static inline size_t publishLargestBytes(bool psram) { + return psram ? kPsramPublishLargestBytes : kInternalPublishLargestBytes; +} + +static inline bool hasPublishHeadroom(bool psram, size_t largest) { + return largest >= publishLargestBytes(psram); +} + static inline bool canStart(bool psram, bool tls, ClientMemoryState state, - size_t dma_free, size_t dma_largest) { + size_t dma_free, size_t dma_largest, + bool protect_publish = false) { size_t allocation_allowance; - size_t largest_required; + size_t sdk_largest; switch (state) { case ClientMemoryState::Uninitialized: allocation_allowance = kColdStartAllowanceBytes; - largest_required = kColdStartLargestBytes; + sdk_largest = kColdStartLargestBytes; break; case ClientMemoryState::Stopped: allocation_allowance = kRestartAllowanceBytes; - largest_required = kRestartLargestBytes; + sdk_largest = kRestartLargestBytes; break; case ClientMemoryState::Started: allocation_allowance = 0; - largest_required = kDmaLargestBytes; + sdk_largest = kDmaLargestBytes; break; default: return false; } + const size_t publication_floor = publishLargestBytes(psram); + const size_t tls_record = tls ? (psram ? kPsramTlsLargestBytes : kTlsRecordAllocationBytes) : 0; + // Bootstrap/recover one broker without requiring spare capacity for another + // live connection: retained SDK buffers can fragment the heap after an outage. + // With a live peer, conservatively allow startup/TLS allocations to consume + // the largest block while leaving existing forwarding's publication floor. + // Other tasks and TLS allocations can still change/fragment this snapshot; + // these are admission bounds, not an allocation guarantee. + size_t largest_required = publication_floor > sdk_largest ? publication_floor : sdk_largest; + if (tls_record > largest_required) largest_required = tls_record; + if (protect_publish) { + largest_required = psram + ? (publication_floor > tls_record ? publication_floor : tls_record) + allocation_allowance + : publication_floor + allocation_allowance + tls_record; + } const size_t free_required = kDmaReserveBytes + allocation_allowance; if (dma_free < free_required || dma_largest < largest_required) { return false; diff --git a/src/helpers/MQTTPayloadBuilder.cpp b/src/helpers/MQTTPayloadBuilder.cpp index b102498aa..b48b875ab 100644 --- a/src/helpers/MQTTPayloadBuilder.cpp +++ b/src/helpers/MQTTPayloadBuilder.cpp @@ -7,8 +7,13 @@ namespace { -static int serializeComplete(JsonObject root, char* buffer, size_t buffer_size) { +static int serializeComplete(JsonDocument& doc, char* buffer, size_t buffer_size) { if (!buffer || buffer_size == 0) return 0; + buffer[0] = '\0'; + JsonObject root = doc.as(); + // A document can serialize successfully after some allocations failed. + // Never publish that incomplete object as a complete status or packet. + if (doc.overflowed() || root.isNull()) return 0; size_t written = serializeJson(root, buffer, buffer_size); // Preserve MQTTMessageBuilder's existing success criterion while clearing @@ -97,7 +102,7 @@ int MQTTPayloadBuilder::buildStatusMessage( if (conn_health.slots_breaker > 0) stats["mqtt_slots_breaker"] = conn_health.slots_breaker; } - return serializeComplete(root, buffer, buffer_size); + return serializeComplete(doc, buffer, buffer_size); } int MQTTPayloadBuilder::buildPacketMessage( @@ -182,7 +187,7 @@ int MQTTPayloadBuilder::buildPacketMessage( } } - return serializeComplete(root, buffer, buffer_size); + return serializeComplete(doc, buffer, buffer_size); } int MQTTPayloadBuilder::buildRawMessage( @@ -203,7 +208,7 @@ int MQTTPayloadBuilder::buildRawMessage( root["type"] = "RAW"; root["data"] = raw; - return serializeComplete(root, buffer, buffer_size); + return serializeComplete(doc, buffer, buffer_size); } static JsonArray buildNeighborsMessageBase( @@ -294,6 +299,7 @@ int MQTTPayloadBuilder::buildNeighborsMessage( bool truncated ) { if (!buffer || buffer_size == 0) return 0; + buffer[0] = '\0'; JsonArray arr = buildNeighborsMessageBase( doc, origin, origin_id, timestamp, self_scopes, self_default_scope, @@ -314,5 +320,5 @@ int MQTTPayloadBuilder::buildNeighborsMessage( } } - return serializeComplete(doc.as(), buffer, buffer_size); + return serializeComplete(doc, buffer, buffer_size); } diff --git a/src/helpers/MQTTPublishAdmission.h b/src/helpers/MQTTPublishAdmission.h index 0594ac401..1e6e1cfd8 100644 --- a/src/helpers/MQTTPublishAdmission.h +++ b/src/helpers/MQTTPublishAdmission.h @@ -8,7 +8,7 @@ namespace MQTTPublishAdmission { // QoS0 writes use the SDK's fixed buffer; QoS1 can resend an entire outbox // record. Keep the existing DMA reserve and account for larger queued records. -static const size_t kLegacyLargestBytes = 8000; +static const size_t kLegacyLargestBytes = MQTTConnectionAdmission::kInternalPublishLargestBytes; static const size_t kFramingAllowanceBytes = 128; static const uint32_t kSampleIntervalMs = 5000; @@ -23,7 +23,8 @@ inline bool canPublish(const Heap& heap, size_t payload_size = 0, size_t topic_size = 0, uint8_t qos = 0) { if (payload_size > SIZE_MAX - kFramingAllowanceBytes || topic_size > SIZE_MAX - kFramingAllowanceBytes - payload_size) return false; - size_t record = qos ? payload_size + topic_size + kFramingAllowanceBytes : 0; + const size_t queued_bytes = qos ? payload_size + topic_size + kFramingAllowanceBytes : 0; + size_t record = queued_bytes; if (record > MQTTConnectionAdmission::kTlsRecordAllocationBytes) record = MQTTConnectionAdmission::kTlsRecordAllocationBytes; if (!heap.psram) { @@ -34,7 +35,10 @@ inline bool canPublish(const Heap& heap, size_t payload_size = 0, ? record : MQTTConnectionAdmission::kDmaLargestBytes; const size_t free = MQTTConnectionAdmission::kDmaReserveBytes + largest - MQTTConnectionAdmission::kDmaLargestBytes; - return heap.dma_free >= free && heap.dma_largest >= largest; + // Small SDK outbox records prefer internal RAM. Leave the reserve after + // enqueue as well as room for the subsequent TLS/DMA write. + if (queued_bytes > SIZE_MAX - free) return false; + return heap.dma_free >= free + queued_bytes && heap.dma_largest >= largest; } struct State { diff --git a/src/helpers/RadioProfileCLI.cpp b/src/helpers/RadioProfileCLI.cpp index 5127ab7ea..4b790912b 100644 --- a/src/helpers/RadioProfileCLI.cpp +++ b/src/helpers/RadioProfileCLI.cpp @@ -92,7 +92,7 @@ bool secondaryOverlaps(uint32_t primary_hz, float secondary_mhz, uint32_t cleara return (primary_hz > secondary_hz ? primary_hz - secondary_hz : secondary_hz - primary_hz) <= clearance_hz; } -#if !defined(COMPANION_RADIO_FULL) +#if !defined(COMPANION_RADIO_FULL) && !MESH_ROLE_COMPANION bool isMediaMeshProfile(const RadioProfileConfig& config) { return config.mode != RadioProfileMode::Off && config.params.freq >= 911.1495f && config.params.freq <= 911.1505f @@ -102,7 +102,7 @@ bool isMediaMeshProfile(const RadioProfileConfig& config) { #endif } -#if !defined(COMPANION_RADIO_FULL) +#if !defined(COMPANION_RADIO_FULL) && !MESH_ROLE_COMPANION void RadioProfileCLI::setMediaMeshScopeHandler(MediaMeshScopeHandler handler, void* context) { media_scope_handler_ = handler; media_scope_context_ = context; diff --git a/src/helpers/RadioProfileCLI.h b/src/helpers/RadioProfileCLI.h index 41e1b1dbb..5fe0bae40 100644 --- a/src/helpers/RadioProfileCLI.h +++ b/src/helpers/RadioProfileCLI.h @@ -61,7 +61,7 @@ class RadioProfileCLI { // readable backup. Read/I/O failures and an intact backup remain read-only. bool recoverable_corrupt_store_ = false; bool publish_pending_ = false; -#if !defined(COMPANION_RADIO_FULL) +#if !defined(COMPANION_RADIO_FULL) && !MESH_ROLE_COMPANION MediaMeshScopeHandler media_scope_handler_ = nullptr; void* media_scope_context_ = nullptr; uint32_t media_scope_retry_ms_ = 0; @@ -71,7 +71,7 @@ class RadioProfileCLI { void retryMediaMeshScope(); bool mediaMeshScopePending() const { return media_scope_pending_; } #else - // Full Companions have no forwarding RegionMap. Do not retain unreachable + // Companions have no forwarding RegionMap. Do not retain unreachable // infrastructure callbacks, state or failure text in their tight images. bool mediaMeshScopeReady(const RadioProfileConfig&) const { return true; } void updateMediaMeshScope() {} @@ -141,7 +141,7 @@ class RadioProfileCLI { #endif void begin(FILESYSTEM* fs, Radio* radio, RTCClock* rtc, bool infrastructure_replies = false); // Bind only after RegionMap::load(), so boot reconciliation is not discarded. -#if !defined(COMPANION_RADIO_FULL) +#if !defined(COMPANION_RADIO_FULL) && !MESH_ROLE_COMPANION void setMediaMeshScopeHandler(MediaMeshScopeHandler handler, void* context); #else void setMediaMeshScopeHandler(MediaMeshScopeHandler, void*) {} diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 6e4966a3f..f854f20c3 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1681,10 +1681,10 @@ bool MQTTBridge::serviceNetworkPower(uint32_t now) { && acknowledgments != _network_publish_acks[i]; _network_publish_acks[i] = acknowledgments; // A flapping CONNACK must not buy another full outage window forever. - // A PUBACK proves delivery immediately; a quiet connection must stay up - // for 30 seconds, leaving time to prove recovery within the probe window. - if (_slots[i].enabled && _slots[i].connected - && (broker_replied || (now - static_cast(_slots[i].connected_at_ms) + // A PUBACK proves delivery even if DISCONNECTED arrives before this pass. + // Consume it once; a quiet connection must still stay up for 30 seconds. + if (_slots[i].enabled && (broker_replied || (_slots[i].connected + && now - static_cast(_slots[i].connected_at_ms) >= mesh::wifi::kNetworkHealthyMs))) { useful_network = true; } @@ -2324,9 +2324,16 @@ bool MQTTBridge::canStartSlotConnection(int index) const { ? ClientMemoryState::Uninitialized : (slot.client->isStarted() ? ClientMemoryState::Started : ClientMemoryState::Stopped); const uint32_t caps = MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA; + bool protect_publish = false; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; ++i) { + if (i != index && _slots[i].enabled && _slots[i].connected) { + protect_publish = true; + break; + } + } return MQTTConnectionAdmission::canStart( ESP.getPsramSize() > 0, tls, memory_state, heap_caps_get_free_size(caps), - heap_caps_get_largest_free_block(caps)); + heap_caps_get_largest_free_block(caps), protect_publish); #else return true; #endif @@ -2735,6 +2742,7 @@ bool MQTTBridge::recoverSlotTransport(int index, uint32_t now) { slot.last_reconnect_attempt = now; _last_slot_reconnect_ms = now; health.recovered(); + _publish_memory.invalidate(); updateSlotTransportDiag(index, now); updateCachedConnectionStatus(); return true; @@ -2746,6 +2754,7 @@ void MQTTBridge::serviceSlotTransportHealth(uint32_t now) { MQTTSlot& slot = _slots[i]; auto& health = slot.transport_health; if (_slot_health_reset_pending[i].exchange(false, std::memory_order_acq_rel)) { + _publish_memory.invalidate(); // A late CONNACK does not release an already-required SDK stop. Its // session clocks may reset, but only proven cleanup clears recovery. health.resetSession(); @@ -3214,6 +3223,7 @@ bool MQTTBridge::publishToSlot(int index, const char* topic, const char* payload // session latch BEFORE enqueueing it, or maintenance would erase its ACK // watch immediately afterwards. Only this owner task mutates health state. if (_slot_health_reset_pending[index].exchange(false, std::memory_order_acq_rel)) { + _publish_memory.invalidate(); slot.transport_health.resetSession(); } if (slot.transport_health.recovery_reason != MQTTTransportHealth::RecoveryReason::None || @@ -3269,6 +3279,7 @@ bool MQTTBridge::publishToSlot(int index, const char* topic, const char* payload } if (qos > 0) { slot.transport_health.acceptedQos1(static_cast(millis()), ack_before); + _publish_memory.invalidate(); // Also reflect an ACK delivered during enqueue, not just on the next tick. slot.transport_health.observeAck(static_cast(slot.client->getPublishAck())); updateSlotTransportDiag(index, static_cast(millis())); diff --git a/src/helpers/ota/OtaManager.cpp b/src/helpers/ota/OtaManager.cpp index e853d2691..5ef6f0295 100644 --- a/src/helpers/ota/OtaManager.cpp +++ b/src/helpers/ota/OtaManager.cpp @@ -334,10 +334,8 @@ void OtaManager::refresh_sources() { (uint64_t)d.payload_off + d.payload_size + 5 != d.total_size) continue; if (serveEntryIndex(d.mid) >= 0) continue; // already offered (e.g. our own fw in the folder) ServeEntry& e = _serve[_n_serve++]; - memcpy(e.mid, d.mid, 4); - e.target_id = d.target_id; e.fw_version = d.fw_version; - e.codec_id = d.codec_id; e.flags = d.flags; e.have_count = d.block_count; // a folder mota is fully held - e.is_self = false; e.src = src; e.src_idx = i; e.desc = d; + e.cacheDescriptor(d); + e.is_self = false; e.src = src; e.src_idx = i; _src_advertised[s]++; } } @@ -409,7 +407,7 @@ OtaManager::ServeView* OtaManager::resolve(const uint8_t* mid) { // builds also cache leaves here; bounded builds acquire proof metadata through // cooperative egress steps, without delaying request admission on a full table. bool OtaManager::loadSource(const ServeEntry& e) { - const MotaDesc& d = e.desc; + const MotaDesc d = e.descriptor(); if (d.leaves_off < 8) return false; if (!e.src) return false; uint16_t mfl = (uint16_t)(d.leaves_off - 8); diff --git a/src/helpers/ota/OtaManager.h b/src/helpers/ota/OtaManager.h index d7a32c5f1..1ec321ed8 100644 --- a/src/helpers/ota/OtaManager.h +++ b/src/helpers/ota/OtaManager.h @@ -2,6 +2,7 @@ #include #include +#include #include "OtaFormat.h" #include "OtaMemoryPolicy.h" #include "OtaTiming.h" @@ -327,19 +328,44 @@ public: }; // A lightweight catalog entry: what we advertise per mota + how to load its ServeView on demand. struct ServeEntry { - // Named fields only: this is not a wire/persistent layout. Group aligned - // values ahead of byte flags instead of paying three padding gaps per row. - MotaSource* src; // load from this external source ... - MotaDesc desc; // cached region offsets for the source entry - uint32_t target_id, fw_version; - uint32_t have_count; // blocks we currently hold (== block_count when complete) - uint8_t mid[4]; - uint8_t codec_id, flags; - bool is_self; // true => entry is view0 (our own fw / RAM mota) - uint8_t src_idx; // ... at this index + // This is not a wire/persistent layout. Source admission proves the fixed + // header/leaves/payload geometry, so cache each field once and reconstruct + // the three derived offsets/sizes instead of retaining another MotaDesc. + MotaSource* src = nullptr; // load from this external source ... + uint32_t target_id = 0, fw_version = 0; + uint32_t have_count = 0; // complete sources hold every block + uint32_t payload_size = 0; + uint8_t mid[4] = {}; + uint8_t codec_id = 0, flags = 0; + bool is_self = false; // true => entry is view0 (our own fw / RAM mota) + uint8_t src_idx = 0; // ... at this index + uint8_t block_size_log2 = 0, source_caps = 0; + + void cacheDescriptor(const MotaDesc& d) { + memcpy(mid, d.mid, sizeof(mid)); + target_id = d.target_id; fw_version = d.fw_version; + codec_id = d.codec_id; flags = d.flags; have_count = d.block_count; + payload_size = d.payload_size; + block_size_log2 = d.block_size_log2; source_caps = d.source_caps; + } + MotaDesc descriptor() const { + MotaDesc d; + if (!have_count) return d; + memcpy(d.mid, mid, sizeof(mid)); + d.target_id = target_id; d.fw_version = fw_version; + d.codec_id = codec_id; d.flags = flags; d.block_count = have_count; + d.payload_size = payload_size; + d.block_size_log2 = block_size_log2; d.source_caps = source_caps; + d.leaves_off = 8u + MOTA_MFL; + d.payload_off = d.leaves_off + have_count * 4u; + d.total_size = d.payload_off + payload_size + 5u; + return d; + } }; - static_assert(sizeof(ServeEntry) <= sizeof(MotaDesc) + sizeof(MotaSource*) + 20u, - "OTA source catalog must not add per-entry alignment padding"); + static_assert(sizeof(ServeEntry) == + ((sizeof(MotaSource*) + 28u + alignof(MotaSource*) - 1u) + / alignof(MotaSource*)) * alignof(MotaSource*), + "OTA source catalog must not duplicate validated descriptor fields"); // Context for the source-payload reader trampoline. struct SrcReadCtx { MotaSource* src; uint8_t idx; uint32_t payload_off; }; @@ -903,7 +929,9 @@ private: uint8_t _observed_path_transmissions = 0; // source + relays; 0 falls back to configured max_hops+1 uint32_t _packets_sent = 0; // OTA packets accepted by the radio adapter (wrap-safe) // multi-fragment manifest reassembly (a signed v2 manifest exceeds one packet) - uint8_t _mf_buf[OTA_MF_MAXFRAG * OTA_MF_FRAG]; // sized to the fragment cap so no valid manifest is silently dropped + // handleManifest admits only the fixed MOTA_MFL bytes and exact fragment + // shapes. The transport's larger fragment-count ceiling adds no valid data. + uint8_t _mf_buf[MOTA_MFL]; uint16_t _mf_retries = 0; // GET_MANIFEST retries while WANT_MANIFEST (give up after a cap) uint8_t _mf_total = 0; // frag_total of the manifest being reassembled (0 = none) uint16_t _mf_mask = 0; // received manifest-fragment bitmap diff --git a/test/fixtures/companion_delayed_reply_delivery/test.cpp b/test/fixtures/companion_delayed_reply_delivery/test.cpp index 1e7b1ff48..bd8b10820 100644 --- a/test/fixtures/companion_delayed_reply_delivery/test.cpp +++ b/test/fixtures/companion_delayed_reply_delivery/test.cpp @@ -17,6 +17,9 @@ #include #include +// This fixture models correlated RF replies. Direct USB OTA administration +// has a separate fixture and is disabled for every consumer of this harness. +#define COMPANION_FEATURE_DIRECT_OTA_ADMIN 0 #define MESH_DEBUG_PRINTLN(...) ((void)0) #include "production_constants.inc" #include "production_stream.inc" diff --git a/test/fixtures/morse_main_loop.cpp b/test/fixtures/morse_main_loop.cpp index d26ff575e..da6f1fb93 100644 --- a/test/fixtures/morse_main_loop.cpp +++ b/test/fixtures/morse_main_loop.cpp @@ -109,6 +109,7 @@ bool profileIdleSleepAllowed() { && (!the_mesh.isDualRadioActive() || (profileIdleQueryAllowed() && the_mesh.profile_idle_allowed)); } void serviceCommandInterfaces() { ++command_services; } +bool serviceCompanionHostKissUsb() { return false; } void serviceUsbLoggingOnlySession() { ++usb_services; } void serviceUsbTerminalHostSessionReset() { ++usb_services; } void serviceUsbAsciiSessionDefault() { ++usb_services; } diff --git a/test/fixtures/vl53_const/README.md b/test/fixtures/vl53_const/README.md new file mode 100644 index 000000000..3c281cb6f --- /dev/null +++ b/test/fixtures/vl53_const/README.md @@ -0,0 +1,11 @@ +These fixtures preserve Adafruit_VL53L0X 1.2.5's two tuning tables and the +complete `VL53L0X_load_tuning_settings` function, with their ST license notices. +The loader's hardware write function is replaced only in the native harness. + +Source: https://github.com/adafruit/Adafruit_VL53L0X/tree/1.2.5/src + +`test_nrf52_vl53l0x_const.py` compares every register write and internal tuning +parameter before and after const qualification, including hardware failures. +The production middleware fingerprints the five complete affected files and +compiles a private library copy. Unknown dependency revisions use their ordinary +source; the downloaded library is never modified. diff --git a/test/fixtures/vl53_const/tuning_loader.inc b/test/fixtures/vl53_const/tuning_loader.inc new file mode 100644 index 000000000..67a4b0a9e --- /dev/null +++ b/test/fixtures/vl53_const/tuning_loader.inc @@ -0,0 +1,109 @@ +/******************************************************************************* + Copyright 2016, STMicroelectronics International N.V. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of STMicroelectronics nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND + NON-INFRINGEMENT OF INTELLECTUAL PROPERTY RIGHTS ARE DISCLAIMED. + IN NO EVENT SHALL STMICROELECTRONICS INTERNATIONAL N.V. BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ + +VL53L0X_Error VL53L0X_load_tuning_settings(VL53L0X_DEV Dev, + uint8_t* pTuningSettingBuffer) { + VL53L0X_Error Status = VL53L0X_ERROR_NONE; + int i; + int Index; + uint8_t msb; + uint8_t lsb; + uint8_t SelectParam; + uint8_t NumberOfWrites; + uint8_t Address; + uint8_t localBuffer[4]; /* max */ + uint16_t Temp16; + + LOG_FUNCTION_START(""); + + Index = 0; + + while ((*(pTuningSettingBuffer + Index) != 0) && + (Status == VL53L0X_ERROR_NONE)) { + NumberOfWrites = *(pTuningSettingBuffer + Index); + Index++; + if (NumberOfWrites == 0xFF) { + /* internal parameters */ + SelectParam = *(pTuningSettingBuffer + Index); + Index++; + switch (SelectParam) { + case 0: /* uint16_t SigmaEstRefArray -> 2 bytes */ + msb = *(pTuningSettingBuffer + Index); + Index++; + lsb = *(pTuningSettingBuffer + Index); + Index++; + Temp16 = VL53L0X_MAKEUINT16(lsb, msb); + PALDevDataSet(Dev, SigmaEstRefArray, Temp16); + break; + case 1: /* uint16_t SigmaEstEffPulseWidth -> 2 bytes */ + msb = *(pTuningSettingBuffer + Index); + Index++; + lsb = *(pTuningSettingBuffer + Index); + Index++; + Temp16 = VL53L0X_MAKEUINT16(lsb, msb); + PALDevDataSet(Dev, SigmaEstEffPulseWidth, Temp16); + break; + case 2: /* uint16_t SigmaEstEffAmbWidth -> 2 bytes */ + msb = *(pTuningSettingBuffer + Index); + Index++; + lsb = *(pTuningSettingBuffer + Index); + Index++; + Temp16 = VL53L0X_MAKEUINT16(lsb, msb); + PALDevDataSet(Dev, SigmaEstEffAmbWidth, Temp16); + break; + case 3: /* uint16_t targetRefRate -> 2 bytes */ + msb = *(pTuningSettingBuffer + Index); + Index++; + lsb = *(pTuningSettingBuffer + Index); + Index++; + Temp16 = VL53L0X_MAKEUINT16(lsb, msb); + PALDevDataSet(Dev, targetRefRate, Temp16); + break; + default: /* invalid parameter */ + Status = VL53L0X_ERROR_INVALID_PARAMS; + } + + } else if (NumberOfWrites <= 4) { + Address = *(pTuningSettingBuffer + Index); + Index++; + + for (i = 0; i < NumberOfWrites; i++) { + localBuffer[i] = *(pTuningSettingBuffer + Index); + Index++; + } + + Status = VL53L0X_WriteMulti(Dev, Address, localBuffer, NumberOfWrites); + + } else { + Status = VL53L0X_ERROR_INVALID_PARAMS; + } + } + + LOG_FUNCTION_END(Status); + return Status; +} diff --git a/test/fixtures/vl53_const/vl53l0x_interrupt_threshold_settings.h b/test/fixtures/vl53_const/vl53l0x_interrupt_threshold_settings.h new file mode 100644 index 000000000..64aba5ecf --- /dev/null +++ b/test/fixtures/vl53_const/vl53l0x_interrupt_threshold_settings.h @@ -0,0 +1,81 @@ +/******************************************************************************* +Copyright 2016, STMicroelectronics International N.V. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of STMicroelectronics nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +NON-INFRINGEMENT OF INTELLECTUAL PROPERTY RIGHTS ARE DISCLAIMED. +IN NO EVENT SHALL STMICROELECTRONICS INTERNATIONAL N.V. BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#ifndef _VL53L0X_INTERRUPT_THRESHOLD_SETTINGS_H_ +#define _VL53L0X_INTERRUPT_THRESHOLD_SETTINGS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +uint8_t InterruptThresholdSettings[] = { + + /* Start of Interrupt Threshold Settings */ + 0x1, 0xff, 0x00, 0x1, 0x80, 0x01, 0x1, 0xff, 0x01, 0x1, 0x00, 0x00, + 0x1, 0xff, 0x01, 0x1, 0x4f, 0x02, 0x1, 0xFF, 0x0E, 0x1, 0x00, 0x03, + 0x1, 0x01, 0x84, 0x1, 0x02, 0x0A, 0x1, 0x03, 0x03, 0x1, 0x04, 0x08, + 0x1, 0x05, 0xC8, 0x1, 0x06, 0x03, 0x1, 0x07, 0x8D, 0x1, 0x08, 0x08, + 0x1, 0x09, 0xC6, 0x1, 0x0A, 0x01, 0x1, 0x0B, 0x02, 0x1, 0x0C, 0x00, + 0x1, 0x0D, 0xD5, 0x1, 0x0E, 0x18, 0x1, 0x0F, 0x12, 0x1, 0x10, 0x01, + 0x1, 0x11, 0x82, 0x1, 0x12, 0x00, 0x1, 0x13, 0xD5, 0x1, 0x14, 0x18, + 0x1, 0x15, 0x13, 0x1, 0x16, 0x03, 0x1, 0x17, 0x86, 0x1, 0x18, 0x0A, + 0x1, 0x19, 0x09, 0x1, 0x1A, 0x08, 0x1, 0x1B, 0xC2, 0x1, 0x1C, 0x03, + 0x1, 0x1D, 0x8F, 0x1, 0x1E, 0x0A, 0x1, 0x1F, 0x06, 0x1, 0x20, 0x01, + 0x1, 0x21, 0x02, 0x1, 0x22, 0x00, 0x1, 0x23, 0xD5, 0x1, 0x24, 0x18, + 0x1, 0x25, 0x22, 0x1, 0x26, 0x01, 0x1, 0x27, 0x82, 0x1, 0x28, 0x00, + 0x1, 0x29, 0xD5, 0x1, 0x2A, 0x18, 0x1, 0x2B, 0x0B, 0x1, 0x2C, 0x28, + 0x1, 0x2D, 0x78, 0x1, 0x2E, 0x28, 0x1, 0x2F, 0x91, 0x1, 0x30, 0x00, + 0x1, 0x31, 0x0B, 0x1, 0x32, 0x00, 0x1, 0x33, 0x0B, 0x1, 0x34, 0x00, + 0x1, 0x35, 0xA1, 0x1, 0x36, 0x00, 0x1, 0x37, 0xA0, 0x1, 0x38, 0x00, + 0x1, 0x39, 0x04, 0x1, 0x3A, 0x28, 0x1, 0x3B, 0x30, 0x1, 0x3C, 0x0C, + 0x1, 0x3D, 0x04, 0x1, 0x3E, 0x0F, 0x1, 0x3F, 0x79, 0x1, 0x40, 0x28, + 0x1, 0x41, 0x1E, 0x1, 0x42, 0x2F, 0x1, 0x43, 0x87, 0x1, 0x44, 0x00, + 0x1, 0x45, 0x0B, 0x1, 0x46, 0x00, 0x1, 0x47, 0x0B, 0x1, 0x48, 0x00, + 0x1, 0x49, 0xA7, 0x1, 0x4A, 0x00, 0x1, 0x4B, 0xA6, 0x1, 0x4C, 0x00, + 0x1, 0x4D, 0x04, 0x1, 0x4E, 0x01, 0x1, 0x4F, 0x00, 0x1, 0x50, 0x00, + 0x1, 0x51, 0x80, 0x1, 0x52, 0x09, 0x1, 0x53, 0x08, 0x1, 0x54, 0x01, + 0x1, 0x55, 0x00, 0x1, 0x56, 0x0F, 0x1, 0x57, 0x79, 0x1, 0x58, 0x09, + 0x1, 0x59, 0x05, 0x1, 0x5A, 0x00, 0x1, 0x5B, 0x60, 0x1, 0x5C, 0x05, + 0x1, 0x5D, 0xD1, 0x1, 0x5E, 0x0C, 0x1, 0x5F, 0x3C, 0x1, 0x60, 0x00, + 0x1, 0x61, 0xD0, 0x1, 0x62, 0x0B, 0x1, 0x63, 0x03, 0x1, 0x64, 0x28, + 0x1, 0x65, 0x10, 0x1, 0x66, 0x2A, 0x1, 0x67, 0x39, 0x1, 0x68, 0x0B, + 0x1, 0x69, 0x02, 0x1, 0x6A, 0x28, 0x1, 0x6B, 0x10, 0x1, 0x6C, 0x2A, + 0x1, 0x6D, 0x61, 0x1, 0x6E, 0x0C, 0x1, 0x6F, 0x00, 0x1, 0x70, 0x0F, + 0x1, 0x71, 0x79, 0x1, 0x72, 0x00, 0x1, 0x73, 0x0B, 0x1, 0x74, 0x00, + 0x1, 0x75, 0x0B, 0x1, 0x76, 0x00, 0x1, 0x77, 0xA1, 0x1, 0x78, 0x00, + 0x1, 0x79, 0xA0, 0x1, 0x7A, 0x00, 0x1, 0x7B, 0x04, 0x1, 0xFF, 0x04, + 0x1, 0x79, 0x1D, 0x1, 0x7B, 0x27, 0x1, 0x96, 0x0E, 0x1, 0x97, 0xFE, + 0x1, 0x98, 0x03, 0x1, 0x99, 0xEF, 0x1, 0x9A, 0x02, 0x1, 0x9B, 0x44, + 0x1, 0x73, 0x07, 0x1, 0x70, 0x01, 0x1, 0xff, 0x01, 0x1, 0x00, 0x01, + 0x1, 0xff, 0x00, 0x00, 0x00, 0x00}; + +#ifdef __cplusplus +} +#endif + +#endif /* _VL53L0X_INTERRUPT_THRESHOLD_SETTINGS_H_ */ diff --git a/test/fixtures/vl53_const/vl53l0x_tuning.h b/test/fixtures/vl53_const/vl53l0x_tuning.h new file mode 100644 index 000000000..94cbb7507 --- /dev/null +++ b/test/fixtures/vl53_const/vl53l0x_tuning.h @@ -0,0 +1,87 @@ +/******************************************************************************* +Copyright 2016, STMicroelectronics International N.V. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of STMicroelectronics nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +NON-INFRINGEMENT OF INTELLECTUAL PROPERTY RIGHTS ARE DISCLAIMED. +IN NO EVENT SHALL STMICROELECTRONICS INTERNATIONAL N.V. BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#ifndef _VL53L0X_TUNING_H_ +#define _VL53L0X_TUNING_H_ + +#include "vl53l0x_def.h" + +#ifdef __cplusplus +extern "C" { +#endif + +uint8_t DefaultTuningSettings[] = { + + /* update 02/11/2015_v36 */ + 0x01, 0xFF, 0x01, 0x01, 0x00, 0x00, + + 0x01, 0xFF, 0x00, 0x01, 0x09, 0x00, 0x01, 0x10, 0x00, 0x01, 0x11, 0x00, + + 0x01, 0x24, 0x01, 0x01, 0x25, 0xff, 0x01, 0x75, 0x00, + + 0x01, 0xFF, 0x01, 0x01, 0x4e, 0x2c, 0x01, 0x48, 0x00, 0x01, 0x30, 0x20, + + 0x01, 0xFF, 0x00, 0x01, 0x30, 0x09, /* mja changed from 0x64. */ + 0x01, 0x54, 0x00, 0x01, 0x31, 0x04, 0x01, 0x32, 0x03, 0x01, 0x40, 0x83, + 0x01, 0x46, 0x25, 0x01, 0x60, 0x00, 0x01, 0x27, 0x00, 0x01, 0x50, 0x06, + 0x01, 0x51, 0x00, 0x01, 0x52, 0x96, 0x01, 0x56, 0x08, 0x01, 0x57, 0x30, + 0x01, 0x61, 0x00, 0x01, 0x62, 0x00, 0x01, 0x64, 0x00, 0x01, 0x65, 0x00, + 0x01, 0x66, 0xa0, + + 0x01, 0xFF, 0x01, 0x01, 0x22, 0x32, 0x01, 0x47, 0x14, 0x01, 0x49, 0xff, + 0x01, 0x4a, 0x00, + + 0x01, 0xFF, 0x00, 0x01, 0x7a, 0x0a, 0x01, 0x7b, 0x00, 0x01, 0x78, 0x21, + + 0x01, 0xFF, 0x01, 0x01, 0x23, 0x34, 0x01, 0x42, 0x00, 0x01, 0x44, 0xff, + 0x01, 0x45, 0x26, 0x01, 0x46, 0x05, 0x01, 0x40, 0x40, 0x01, 0x0E, 0x06, + 0x01, 0x20, 0x1a, 0x01, 0x43, 0x40, + + 0x01, 0xFF, 0x00, 0x01, 0x34, 0x03, 0x01, 0x35, 0x44, + + 0x01, 0xFF, 0x01, 0x01, 0x31, 0x04, 0x01, 0x4b, 0x09, 0x01, 0x4c, 0x05, + 0x01, 0x4d, 0x04, + + 0x01, 0xFF, 0x00, 0x01, 0x44, 0x00, 0x01, 0x45, 0x20, 0x01, 0x47, 0x08, + 0x01, 0x48, 0x28, 0x01, 0x67, 0x00, 0x01, 0x70, 0x04, 0x01, 0x71, 0x01, + 0x01, 0x72, 0xfe, 0x01, 0x76, 0x00, 0x01, 0x77, 0x00, + + 0x01, 0xFF, 0x01, 0x01, 0x0d, 0x01, + + 0x01, 0xFF, 0x00, 0x01, 0x80, 0x01, 0x01, 0x01, 0xF8, + + 0x01, 0xFF, 0x01, 0x01, 0x8e, 0x01, 0x01, 0x00, 0x01, 0x01, 0xFF, 0x00, + 0x01, 0x80, 0x00, + + 0x00, 0x00, 0x00}; + +#ifdef __cplusplus +} +#endif + +#endif /* _VL53L0X_TUNING_H_ */ diff --git a/test/test_build_profiles.sh b/test/test_build_profiles.sh index 4b62241ce..05fd4c398 100644 --- a/test/test_build_profiles.sh +++ b/test/test_build_profiles.sh @@ -1900,11 +1900,17 @@ verify_reduced_sensor_flags() { [[ "$PLATFORMIO_BUILD_FLAGS" == *-UENV_INCLUDE_BME280* ]] \ || fail "$env_name did not omit environmental sensors" if [ "$expect_ina" = 1 ]; then + [[ "$PLATFORMIO_BUILD_FLAGS" == *-DMESH_OTA_LOCAL_TARGET_NAME_ONLY=1* ]] \ + || fail "$env_name retained the all-board OTA name catalog" + [[ " ${BUILD_REDUCTIONS[*]} " == *"ota target-name catalog omitted"* ]] \ + || fail "$env_name did not declare its OTA name reduction" [[ "$PLATFORMIO_BUILD_FLAGS" == *-DENV_INCLUDE_INA219=1* ]] \ || fail "$env_name did not retain INA monitor support" [[ "$PLATFORMIO_BUILD_FLAGS" != *-UENV_INCLUDE_INA219* ]] \ || fail "$env_name both retained and omitted INA monitor support" else + [[ "$PLATFORMIO_BUILD_FLAGS" != *MESH_OTA_LOCAL_TARGET_NAME_ONLY* ]] \ + || fail "$env_name lost OTA names outside the reduced RAK profile" [[ "$PLATFORMIO_BUILD_FLAGS" == *-UENV_INCLUDE_INA219* ]] \ || fail "$env_name incorrectly retained INA monitor support" fi @@ -1914,6 +1920,10 @@ verify_reduced_sensor_flags \ RAK_3401_repeater_lora_ota_no_external_sensors 1 verify_reduced_sensor_flags \ RAK_3401_room_server_lora_ota_no_external_sensors 1 +verify_reduced_sensor_flags \ + RAK_4631_repeater_lora_ota_no_external_sensors 1 +verify_reduced_sensor_flags \ + RAK_4631_sensor_lora_ota_no_external_sensors 1 verify_reduced_sensor_flags \ Heltec_t114_repeater_lora_ota_no_external_sensors 0 diff --git a/test/test_carrier_wave.py b/test/test_carrier_wave.py index 2d95018bc..6da7c13a0 100644 --- a/test/test_carrier_wave.py +++ b/test/test_carrier_wave.py @@ -141,6 +141,7 @@ struct RadioLibWrapper : mesh::Radio { uint32_t rssi_latency=0; unsigned rssi_reads=0; bool _params_valid=true, _rx_ps_armed=true, _rx_hold_continuous=false; + bool _primary_receive_preempt=false; bool _rx_boosted_gain_valid=false, _cur_rx_boosted_gain=false, _rx_boosted_gain_applied=false; bool rx_gain_apply_ok=true, physical_rx_gain=false; unsigned rx_gain_applies=0,rx_gain_order=0; @@ -212,6 +213,7 @@ struct RadioLibWrapper : mesh::Radio { #endif virtual bool supportsRadioDeepInit() const { return deep_supported; } uint8_t beginReconfigure(); + void noteReconfigureAdmission(const char*, int16_t) {} void endReconfigure(bool resume) { if (resume) startRecv(); } bool isChipBusy() { return busy; } bool isReceivingPacket() { return packet; } diff --git a/test/test_cli_settings_contract.py b/test/test_cli_settings_contract.py index 6dfe8c0c1..d660eed2c 100644 --- a/test/test_cli_settings_contract.py +++ b/test/test_cli_settings_contract.py @@ -269,9 +269,11 @@ namespace mesh { struct Utils { 'flash.health', # Read-only flash health diagnostics 'key.backup.transport', # Identity-backup transport capability 'mqtt', 'mqtt.running', # MQTT connection status + 'ota.direct', # Direct USB OTA session diagnostics 'password', 'prv.key', # Local secret/identity reads 'public.key', # Read-only public identity 'pwrmgt.bootreason', 'role', # Boot/build facts + 'radio.apply', # Pending/applied radio diagnostics 'radio.rxps.config', # Detail view of radio.rxps 'tx.status', # HIL-only applied-power verification 'wifi.status', 'wifi.ip', # Live WiFi link state diff --git a/test/test_client_acl_role.py b/test/test_client_acl_role.py index 7f23fa0dc..46be48bdc 100644 --- a/test/test_client_acl_role.py +++ b/test/test_client_acl_role.py @@ -19,6 +19,33 @@ FIXTURE = ROOT / "test/fixtures/client_acl_spiffs" class ClientAclRoleTests(unittest.TestCase): + def test_complete_exclusive_companion_role_uses_one_global_definition(self): + definition = POLICY.COMPANION_DEFINITION + for role in ("+<../examples/companion_radio>", + "+<../examples/companion_radio/*.cpp>", + ["+<*.cpp>", "+<..\\examples\\companion_radio\\*.cpp>"]): + self.assertEqual(POLICY.companion_role_policy(role), {definition: 1}) + for source in ("", "+<*>", "+<../examples/companion_radio/main.cpp>", + "+<../examples/companion_radio> -<../examples/companion_radio/MyMesh.cpp>", + "+<../examples/companion_radio> +<../examples/custom_role>", + "+<../examples/simple_repeater>"): + self.assertEqual(POLICY.companion_role_policy(source), {}) + for other in ("simple_repeater", *POLICY.OTHER_ROLES): + if other == "companion_radio": + continue + self.assertEqual(POLICY.companion_role_policy( + "+<../examples/companion_radio> +<../examples/" + other + ">"), {}) + role = "+<../examples/companion_radio>" + for flag in ("-D" + definition + "=0", "-D" + definition + "=1", "-U" + definition): + self.assertEqual(POLICY.companion_role_policy(role, flag), {}) + self.assertEqual(POLICY.companion_role_policy(role, cppdefines=[(definition, 1)]), {}) + for value in ("true", "2", "-1"): + with self.assertRaises(ValueError): + POLICY.companion_role_policy(role, "-D" + definition + "=" + value) + with self.assertRaises(ValueError): + POLICY.companion_role_policy("+<../examples/simple_repeater>", + "-D" + definition + "=1") + def test_only_complete_repeater_role_qualifies(self): for value in ("+<../examples/simple_repeater>", "+<../examples/simple_repeater/*.cpp>", ["+<*.cpp>", "+<..\\examples\\simple_repeater\\*.cpp>"]): @@ -57,6 +84,9 @@ class ClientAclRoleTests(unittest.TestCase): del env["appended"] POLICY.install(env) self.assertNotIn("appended", env) + env["SRC_FILTER"] = "+<../examples/companion_radio/*.cpp>" + POLICY.install(env) + self.assertEqual(env["appended"], {"CPPDEFINES": [(POLICY.COMPANION_DEFINITION, 1)]}) def test_every_platform_base_installs_role_hook(self): ini = (ROOT / "platformio.ini").read_text() diff --git a/test/test_companion_ota_includes.py b/test/test_companion_ota_includes.py index 9a01bc510..8d2039ebc 100644 --- a/test/test_companion_ota_includes.py +++ b/test/test_companion_ota_includes.py @@ -54,6 +54,14 @@ class CompanionOtaIncludeTest(unittest.TestCase): (directory / "MyMesh.h").write_text( '#pragma once\n#include "CompanionFeatures.h"\n' ) + # Host KISS declares its filesystem boundary in Full images. Keep + # that SDK type visible without pulling a board filesystem into + # this include-only test of the real OTA headers. + identity = directory / "helpers/IdentityStore.h" + identity.write_text( + "#pragma once\nclass IncludeTestFilesystem;\n" + "#define FILESYSTEM IncludeTestFilesystem\n" + ) if not (usb or ble): # Prove that small/non-OTA images do not gain these dependencies. for header in ("MotaSourceSerial.h", "OtaContext.h"): diff --git a/test/test_companion_rtc_queue.py b/test/test_companion_rtc_queue.py index 100918bf0..bec0754cf 100644 --- a/test/test_companion_rtc_queue.py +++ b/test/test_companion_rtc_queue.py @@ -196,7 +196,10 @@ class CompanionRtcQueueTests(unittest.TestCase): source = (ROOT / 'examples/companion_radio/MyMesh.cpp').read_text() start = header.index('#elif COMPANION_RTC_FAST_OFFLINE_FRAMES || COMPANION_RTC_SLOW_OFFLINE_FRAMES', header.index('static constexpr unsigned OTA_RETAINED_OFFLINE_FRAMES')) end = header.index('#elif defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM)', start) - storage = header[start:].split('\n', 1)[1][:end - start - len(header[start:].split('\n', 1)[0]) - 1] + common_start = header.index(' // Concrete definitions in MyMesh.cpp') + common_end = header.index('\n#endif\n#if defined(OTA_SHARED_COMPANION_QUEUE)', common_start) + storage = (header[common_start:common_end] + '\n' + + header[start:].split('\n', 1)[1][:end - start - len(header[start:].split('\n', 1)[0]) - 1]) banks_start = source.index('#if COMPANION_RTC_FAST_OFFLINE_FRAMES\n') banks_end = source.index('bool MyMesh::Frame::isChannelMsg()', banks_start) methods = '\n'.join(extract_braced(source, name) for name in ( @@ -287,7 +290,8 @@ class CompanionRtcQueueTests(unittest.TestCase): base = ['ESP32_PLATFORM=1', 'COMPANION_RADIO_FULL=1', 'CONFIG_IDF_TARGET_ESP32C3=1', 'CONFIG_ESP_SYSTEM_ALLOW_RTC_FAST_MEM_AS_HEAP=1', 'COMPANION_RTC_FAST_OFFLINE_FRAMES=32'] cases = ([value for value in base if not value.startswith('CONFIG_IDF_TARGET_')], - base + ['BOARD_HAS_PSRAM=1'], base + ['OTA_SHARED_COMPANION_QUEUE=1'], + base + ['BOARD_HAS_PSRAM=1'], [value for value in base if not value.startswith('CONFIG_IDF_TARGET_')] + + ['OTA_SHARED_COMPANION_QUEUE=1', 'CONFIG_IDF_TARGET_ESP32S3=1'], base + ['ONE_KEY_DM_SHARED_OFFLINE_QUEUE=1'], base + ['COMPANION_RTC_SLOW_OFFLINE_FRAMES=32'], [value for value in base if not value.startswith('COMPANION_RADIO_FULL')], @@ -299,7 +303,7 @@ class CompanionRtcQueueTests(unittest.TestCase): capture_output=True, text=True, timeout=15) self.assertNotEqual(result.returncode, 0) - def test_only_five_named_full_policies_select_banks_without_capacity_reductions(self): + def test_only_named_full_policies_select_banks_without_capacity_reductions(self): source = (ROOT / 'build_legacy.sh').read_text() start = source.index(' # BLE + WiFi exhaust internal DRAM') start = source.index(' case "${env_name,,}" in', start) @@ -310,6 +314,8 @@ class CompanionRtcQueueTests(unittest.TestCase): self.assertNotIn('COMPANION_RTC_SLOW_OFFLINE_FRAMES', prefix) for target, fast, slow, contacts, queue in ( ('Generic_ESPNOW_companion_radio_full', 32, 0, 100, 224), + ('Xiao_C3_companion_radio_full', 32, 0, 100, 224), + ('Heltec_CT62_companion_radio_full', 0, 0, 100, 224), ('Heltec_Wireless_Tracker_companion_radio_full', 32, 32, 150, 256), ('MKE_s3_companion_radio_full', 32, 32, 350, 256), ('ThinkNode_M2_companion_radio_full', 32, 32, 350, 256), @@ -328,7 +334,11 @@ class CompanionRtcQueueTests(unittest.TestCase): self.assertEqual(int(flags.get('COMPANION_RTC_SLOW_OFFLINE_FRAMES', 0)), slow) self.assertEqual(int(flags.get('MAX_CONTACTS', 350)), contacts) self.assertEqual(int(flags.get('OFFLINE_QUEUE_SIZE', 256)), queue) - self.assertNotIn('OTA_SHARED_COMPANION_QUEUE', flags) + if target in ('Xiao_C3_companion_radio_full', 'Heltec_CT62_companion_radio_full'): + self.assertEqual(flags.get('OTA_SHARED_COMPANION_QUEUE'), '1') + self.assertEqual(flags.get('ONE_KEY_DM_COMPACT_HELD'), '1') + else: + self.assertNotIn('OTA_SHARED_COMPANION_QUEUE', flags) if __name__ == '__main__': diff --git a/test/test_companion_rtc_terminal_input.py b/test/test_companion_rtc_terminal_input.py index 8f596ae07..96593c055 100644 --- a/test/test_companion_rtc_terminal_input.py +++ b/test/test_companion_rtc_terminal_input.py @@ -209,7 +209,10 @@ int main() { self.assertEqual(flags.get('COMPANION_RTC_TERMINAL_INPUT')=='1', selected) self.assertEqual(int(flags['MAX_CONTACTS']), contacts) self.assertEqual(int(flags.get('OFFLINE_QUEUE_SIZE', 256)), queue) - self.assertNotIn('COMPANION_RTC_FAST_OFFLINE_FRAMES', flags) + if target == 'Xiao_C3_companion_radio_full': + self.assertEqual(flags.get('COMPANION_RTC_FAST_OFFLINE_FRAMES'), '32') + else: + self.assertNotIn('COMPANION_RTC_FAST_OFFLINE_FRAMES', flags) self.assertNotIn('COMPANION_RTC_SLOW_OFFLINE_FRAMES', flags) diff --git a/test/test_companion_shared_ota_capacity.py b/test/test_companion_shared_ota_capacity.py index 788393cb7..9d247ed65 100644 --- a/test/test_companion_shared_ota_capacity.py +++ b/test/test_companion_shared_ota_capacity.py @@ -3,6 +3,9 @@ from pathlib import Path import re +import resource +import signal +import shutil import subprocess import tempfile import unittest @@ -22,24 +25,41 @@ def method(text, signature): class SharedCompanionOtaCapacityTest(unittest.TestCase): - def compile(self, text, *, run=False): + def compile(self, text, *, run=False, rtc=False, args=(), expected_returncode=0): with tempfile.TemporaryDirectory(prefix="meshcore-shared-c3-") as temp: binary = Path(temp) / "queue" + (Path(temp) / "sdkconfig.h").write_text("#pragma once\n") + (Path(temp) / "esp_attr.h").write_text( + '#pragma once\n#define RTC_FAST_ATTR __attribute__((section(".rtc.force_fast.test")))\n') command = ["c++", "-std=c++17", "-Wall", "-Wextra", "-Werror", - "-I", str(ROOT / "src"), "-x", "c++", "-", + "-I", str(ROOT / "src"), "-I", temp, "-x", "c++", "-", "-o", str(binary)] + if rtc: + command[1:1] = ["-DNDEBUG", "-DESP32_PLATFORM=1", "-DCOMPANION_RADIO_FULL=1", + "-DOTA_SHARED_COMPANION_QUEUE=1", "-DOFFLINE_QUEUE_SIZE=224", "-DCONFIG_IDF_TARGET_ESP32C3=1", + "-DCONFIG_ESP_SYSTEM_ALLOW_RTC_FAST_MEM_AS_HEAP=1", + "-DCOMPANION_RTC_FAST_OFFLINE_FRAMES=32"] if run: command[1:1] = ["-fsanitize=address,undefined", "-g"] result = subprocess.run(command, input=text, text=True, capture_output=True) if run: self.assertEqual(result.returncode, 0, result.stderr) - result = subprocess.run([str(binary)], text=True, - capture_output=True) - self.assertEqual(result.returncode, 0, result.stderr) + if rtc: + objdump = shutil.which('objdump') + self.assertIsNotNone(objdump, 'objdump is required for the physical RTC bank check') + sections = subprocess.check_output([objdump, '-h', str(binary)], text=True) + match = re.search(r'\.rtc\.force_fast\.test\s+([0-9a-fA-F]+)', sections) + self.assertIsNotNone(match, sections) + self.assertEqual(int(match[1], 16), 32 * 177, sections) + def no_core_dump(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + result = subprocess.run([str(binary), *args], text=True, + capture_output=True, preexec_fn=no_core_dump) + self.assertEqual(result.returncode, expected_returncode, result.stderr) return result - def test_224_normal_slots_and_128_retained_slots_preserve_fifo_and_ownership(self): + def queue_fixture(self, rtc=False): source = (ROOT / "examples/companion_radio/MyMesh.cpp").read_text() declaration = (ROOT / "examples/companion_radio/MyMesh.h").read_text() retained = re.search( @@ -52,10 +72,14 @@ class SharedCompanionOtaCapacityTest(unittest.TestCase): )) harness = r''' #include +#include +#include #include #include #include #include +#undef assert +#define assert(value) do { if (!(value)) std::abort(); } while (0) constexpr unsigned MAX_FRAME_SIZE = 176; namespace mesh { namespace ota { struct OtaContext { @@ -76,8 +100,7 @@ unsigned OtaContext::live = 0, OtaContext::constructions = 0, struct MyMesh { struct Frame { uint8_t len; uint8_t buf[MAX_FRAME_SIZE]; }; static constexpr unsigned OTA_RETAINED_OFFLINE_FRAMES = @RETAINED@; - mesh::BorrowableFrameBuffer offline_queue; + @STORAGE@ int offline_queue_len = 0, offline_queue_head = 0; unsigned next = 0, notifications = 0; std::deque expected; @@ -112,6 +135,7 @@ struct MyMesh { } void pop() { uint8_t bytes[MAX_FRAME_SIZE]; assert(getFromOfflineQueue(bytes)); } }; +@BANKS@ @METHODS@ int main() { using mesh::ota::OtaContext; @@ -148,7 +172,96 @@ int main() { } } '''.replace("@RETAINED@", retained).replace("@METHODS@", methods) - self.compile(harness, run=True) + if rtc: + start = declaration.index(' // Concrete definitions in MyMesh.cpp') + end = declaration.index('\n#endif\n#if defined(OTA_SHARED_COMPANION_QUEUE)', start) + shared_start = declaration.index(' static constexpr unsigned OTA_RETAINED_OFFLINE_FRAMES') + queue_start = declaration.index(' mesh::BorrowableFrameBuffer offline_queue;') + banks = '' + return harness.replace('@STORAGE@', storage).replace('@BANKS@', banks) + + def test_224_normal_slots_and_128_retained_slots_preserve_fifo_and_ownership(self): + self.compile(self.queue_fixture(), run=True) + + def test_rtc_backed_224_and_128_fifo_all_heads_use_actual_mesh_storage_and_callbacks(self): + self.compile(self.queue_fixture(rtc=True), run=True, rtc=True) + + def test_rtc_owner_reset_geometry_and_release_build_safeguards(self): + harness = self.queue_fixture(rtc=True).split('int main() {', 1)[0] + r''' +using Frame = MyMesh::Frame; +using Workspace = mesh::ota::OtaContext; +using Inline = mesh::BorrowableFrameBuffer; +using Retained = mesh::CompanionRtcQueue; +using Split = mesh::BorrowableFrameBuffer; +struct Legacy { + Frame retained[128]; + union Extra { Frame frames[96]; Workspace workspace; Extra() {} ~Extra() {} } extra; + bool borrowed; +}; +static_assert(sizeof(Inline) == sizeof(Legacy), "Ordinary targets keep the exact BSS layout"); +static_assert(alignof(Inline) == alignof(Legacy), "Ordinary targets keep their alignment"); +static_assert(sizeof(Inline) - sizeof(Split) >= 32 * sizeof(Frame) - 2 * sizeof(void*) - alignof(Split), + "The complete 32-slot bank must leave internal DRAM"); +static_assert(Retained::frame_count == 128, "All active-OTA slots remain independent of workspace"); +static_assert(!std::is_copy_constructible::value && !std::is_move_constructible::value, + "A bank lease must never be duplicated"); +int main(int argc, char** argv) { + alignas(4) Frame bank[32]; + memset(bank, 0xA5, sizeof bank); + if (argc > 1 && !strcmp(argv[1], "null")) { Split bad(nullptr, nullptr); return 1; } + if (argc > 1 && !strcmp(argv[1], "misaligned")) { + Split bad(reinterpret_cast(reinterpret_cast(bank) + 1), nullptr); return 1; + } + if (argc > 1 && !strcmp(argv[1], "range")) { + Retained owner(bank, nullptr); owner.at(128); return 1; + } + { + Split owner(bank, nullptr); + for (const auto& frame : bank) { + assert(frame.len == 0); + for (uint8_t byte : frame.buf) assert(byte == 0); + } + for (unsigned i = 0; i < 128; ++i) { + auto& frame = owner.at(i); assert(frame.len == 0); + for (uint8_t byte : frame.buf) assert(byte == 0); + } + if (argc > 1 && !strcmp(argv[1], "second")) { Split second(bank, nullptr); return 1; } + if (argc > 1 && !strcmp(argv[1], "other-count")) { + mesh::CompanionRtcQueue second(bank, nullptr); return 1; + } + if (argc > 1 && !strcmp(argv[1], "foreign")) { + // A stale or foreign lease must be rejected, even with NDEBUG. + mesh::companion_rtc::Lease::owner() = nullptr; + owner.at(0); return 1; + } + for (unsigned i = 0; i < 224; ++i) memset(&owner.at(i), 0xBB, sizeof(Frame)); + int head = 217; + assert(owner.acquire(128, head) && head == 0 && owner.capacity() == 128); + // Destruction must end an active workspace exactly once before releasing its bank. + } + assert(Workspace::live == 0 && Workspace::constructions == Workspace::destructions); + Split next(bank, nullptr); + for (unsigned i = 0; i < 224; ++i) { + const auto& frame = next.at(i); assert(frame.len == 0); + for (uint8_t byte : frame.buf) assert(byte == 0); + } +} +''' + harness = harness.replace('#include ', '#include \n#include ') + self.compile(harness, run=True, rtc=True) + for argument in ('null', 'misaligned', 'second', 'other-count', 'foreign', 'range'): + with self.subTest(argument=argument): + self.compile(harness, run=True, rtc=True, args=(argument,), + expected_returncode=-signal.SIGABRT) def test_insufficient_workspace_or_no_retained_region_cannot_compile(self): harness = r''' diff --git a/test/test_companion_usb_default.py b/test/test_companion_usb_default.py index 43eeba0c8..555d3bcc2 100644 --- a/test/test_companion_usb_default.py +++ b/test/test_companion_usb_default.py @@ -68,6 +68,7 @@ static constexpr unsigned USB_HOST_LOSS_EDGE_MS = 100, USB_HOST_LOSS_GRACE_MS = struct Stream { int pending = 0; int available() const { return pending; } + int peek() const { return pending > 0 ? 'x' : -1; } int read() { return pending-- > 0 ? 'x' : -1; } void print(const char*) {} } usb_stream; diff --git a/test/test_firmware_ram.py b/test/test_firmware_ram.py index b856edaa0..3809f20c9 100644 --- a/test/test_firmware_ram.py +++ b/test/test_firmware_ram.py @@ -595,8 +595,11 @@ int main() { # The potential five-slot allowance is reported separately: additional # clients allocate lazily and must pass the production runtime gate. self.assertEqual(plain["required_heap_bytes"], psram["required_heap_bytes"]) + self.assertEqual(plain["required_heap_bytes"], 119808) self.assertEqual(plain["components"]["mqtt_startup_allowance"], 24576) self.assertEqual(psram["components"]["mqtt_startup_allowance"], 24576) + self.assertEqual(plain["components"]["allocation_and_transient_margin"], 16384) + self.assertEqual(psram["components"]["allocation_and_transient_margin"], 16384) for policy, slots, active, potential in ((plain, 3, 2, 24576), (psram, 6, 5, 48384)): with self.subTest(active=active): profile = policy["mqtt"] @@ -707,16 +710,36 @@ int main() { ("client_task_stack_bytes", "kClientTaskStackBytes"), ("client_task_priority", "kClientTaskPriority"), ("cold_start_allocation_allowance_bytes", "kColdStartAllowanceBytes"), - ("cold_start_dma_admission_contiguous_bytes", "kColdStartLargestBytes"), ("restart_allocation_allowance_bytes", "kRestartAllowanceBytes"), - ("restart_dma_admission_contiguous_bytes", "kRestartLargestBytes"), ("dma_admission_free_bytes", "kDmaReserveBytes"), - ("dma_admission_contiguous_bytes", "kDmaLargestBytes"), + ("publish_contiguous_without_psram_bytes", "kInternalPublishLargestBytes"), + ("publish_contiguous_with_psram_bytes", "kPsramPublishLargestBytes"), + ("tls_psram_admission_contiguous_bytes", "kPsramTlsLargestBytes"), ("tls_internal_admission_free_bytes", "kTlsInternalFreeBytes"), - ("tls_internal_admission_contiguous_bytes", "kTlsRecordAllocationBytes"), + ("tls_record_allocation_bytes", "kTlsRecordAllocationBytes"), ): with self.subTest(key=key): self.assertIn(f"{name} = {profile[key]};", source) + for psram in (False, True): + flags = {"WITH_MQTT_BRIDGE": 1} + if psram: + flags["BOARD_HAS_PSRAM"] = 1 + actual = ram.requirements("ESP32_PLATFORM", flags, "v4_companion")["mqtt"] + floor = 4096 if psram else 8000 + self.assertEqual(actual["publish_contiguous_bytes"], floor) + self.assertEqual(actual["dma_admission_contiguous_bytes"], floor) + self.assertEqual(actual["cold_start_dma_admission_contiguous_bytes"], max(floor, 8192)) + self.assertEqual(actual["additional_broker_cold_start_dma_admission_contiguous_bytes"], floor + 16384) + self.assertEqual(actual["restart_dma_admission_contiguous_bytes"], max(floor, 8192)) + self.assertEqual(actual["additional_broker_restart_dma_admission_contiguous_bytes"], floor + 8192) + self.assertEqual(actual["tls_psram_admission_contiguous_bytes"], 16000) + self.assertEqual(actual["additional_broker_cold_start_tls_psram_admission_contiguous_bytes"], 32384) + self.assertEqual(actual["additional_broker_restart_tls_psram_admission_contiguous_bytes"], 24192) + self.assertEqual(actual["additional_broker_tls_psram_admission_contiguous_bytes"], 16000) + self.assertEqual(actual["additional_broker_cold_start_tls_internal_admission_contiguous_bytes"], 41792) + self.assertEqual(actual["additional_broker_restart_tls_internal_admission_contiguous_bytes"], 33600) + self.assertEqual(actual["tls_internal_admission_contiguous_bytes"], 17408) + self.assertEqual(actual["additional_broker_tls_internal_admission_contiguous_bytes"], 25408) self.assertEqual(profile["cold_start_dma_admission_free_bytes"], profile["dma_admission_free_bytes"] + profile["cold_start_allocation_allowance_bytes"]) self.assertEqual(profile["restart_dma_admission_free_bytes"], diff --git a/test/test_mqtt_connection_admission.py b/test/test_mqtt_connection_admission.py index 4a668d05c..199d1f697 100644 --- a/test/test_mqtt_connection_admission.py +++ b/test/test_mqtt_connection_admission.py @@ -14,6 +14,8 @@ POLICY_PREAMBLE = r''' #include #include #include +#include +#include #include "helpers/MQTTConnectionAdmission.h" #include "helpers/MQTTTransportHealth.h" namespace Admission = MQTTConnectionAdmission; @@ -62,6 +64,7 @@ struct Client { }; struct MQTTBridge { struct MQTTSlot { + bool enabled = true, connected = false; Client* client = nullptr; const Preset* preset = nullptr; char broker_uri[128] = {}, host[64] = "broker"; @@ -96,47 +99,134 @@ class MqttConnectionAdmissionTest(unittest.TestCase): def test_connection_admission_checks_both_total_and_contiguous_dma_heap(self): self.compile_run(POLICY_PREAMBLE + r''' int main() { - // A large aggregate heap cannot compensate for a fragmented DMA region. - assert(!Admission::canStart(true, true, MemoryState::Started, 100000, 4095)); - assert(!Admission::canStart(true, true, MemoryState::Started, 16383, 20000)); - assert(Admission::canStart(true, true, MemoryState::Started, 16384, 4096)); - assert(Admission::canStart(false, false, MemoryState::Started, 16384, 4096)); - assert(!Admission::canStart(false, false, MemoryState::Started, 16383, 4096)); - assert(!Admission::canStart(false, false, MemoryState::Started, 16384, 4095)); - - // Without PSRAM, TLS also needs a full record allocation and session budget. - const MemoryState states[] = {MemoryState::Uninitialized, MemoryState::Stopped, - MemoryState::Started}; - for (MemoryState state : states) { - assert(!Admission::canStart(false, true, state, 61439, 20000)); - assert(!Admission::canStart(false, true, state, 100000, 17407)); - assert(Admission::canStart(false, true, state, 61440, 17408)); - } - assert(Admission::canStart(true, false, MemoryState::Started, 16384, 4096)); - assert(!Admission::canStart(true, false, static_cast(255), SIZE_MAX, SIZE_MAX)); - - // A new SDK task needs its stack and ordinary startup allocations in - // addition to the DMA reserve, including plaintext and PSRAM-backed TLS. + assert(Admission::kPsramPublishLargestBytes == 4096); + assert(Admission::kPsramPublishLargestBytes == Admission::kDmaLargestBytes); + assert(Admission::kPsramTlsLargestBytes == 16000); + assert(Admission::kInternalPublishLargestBytes == 8000); + assert(Admission::kDmaReserveBytes == 16384); assert(Admission::kClientTaskStackBytes == 6144); assert(Admission::kClientTaskPriority == 5); assert(Admission::kColdStartAllowanceBytes == 16384); - assert(Admission::kColdStartLargestBytes == 8192); assert(Admission::kRestartAllowanceBytes == 8192); - assert(Admission::kRestartLargestBytes == 8192); - for (int psram = 0; psram <= 1; ++psram) { - for (int tls = 0; tls <= 1; ++tls) { - if (!psram && tls) continue; // Larger TLS thresholds checked above. - assert(!Admission::canStart(psram, tls, MemoryState::Uninitialized, 32767, 20000)); - assert(!Admission::canStart(psram, tls, MemoryState::Uninitialized, 100000, 8191)); - assert(Admission::canStart(psram, tls, MemoryState::Uninitialized, 32768, 8192)); - // An initialized handle retains its ordinary SDK buffers across a stop. - // Only the replacement task/TCB is charged beside the DMA reserve. - assert(!Admission::canStart(psram, tls, MemoryState::Stopped, 24575, 20000)); - assert(!Admission::canStart(psram, tls, MemoryState::Stopped, 100000, 8191)); - assert(Admission::canStart(psram, tls, MemoryState::Stopped, 24576, 8192)); + assert(Admission::kTlsInternalFreeBytes == 61440); + assert(Admission::kClientBufferBytes == 896); + const MemoryState states[] = {MemoryState::Uninitialized, MemoryState::Stopped, + MemoryState::Started}; + const size_t allowances[] = {Admission::kColdStartAllowanceBytes, + Admission::kRestartAllowanceBytes, 0}; + const size_t sdk_blocks[] = {Admission::kColdStartLargestBytes, + Admission::kRestartLargestBytes, Admission::kDmaLargestBytes}; + for (bool psram : {false, true}) { + const size_t floor = Admission::publishLargestBytes(psram); + for (bool tls : {false, true}) { + const size_t record = tls + ? (psram ? Admission::kPsramTlsLargestBytes : Admission::kTlsRecordAllocationBytes) : 0; + for (bool protect : {false, true}) { + for (size_t i = 0; i < 3; ++i) { + const size_t free = std::max(Admission::kDmaReserveBytes + allowances[i], + tls && !psram ? Admission::kTlsInternalFreeBytes : size_t(0)); + const size_t protected_largest = psram ? std::max(floor, record) + allowances[i] + : floor + allowances[i] + record; + const size_t largest = protect ? protected_largest + : std::max(std::max(floor, sdk_blocks[i]), record); + // Total and contiguous headroom are independent requirements, for + // bootstrap, stopped-task restart, and retained-task reconnect. + assert(!Admission::canStart(psram, tls, states[i], free - 1, 100000, protect)); + assert(!Admission::canStart(psram, tls, states[i], 100000, largest - 1, protect)); + assert(Admission::canStart(psram, tls, states[i], free, largest, protect)); + assert(Admission::canStart(psram, tls, states[i], free + 1, largest + 1, protect)); + } + assert(!Admission::canStart(psram, tls, static_cast(255), + SIZE_MAX, SIZE_MAX, protect)); + } } } } +''') + + def test_additional_broker_cannot_spend_live_packet_publishing_headroom(self): + self.compile_run(POLICY_PREAMBLE + r''' +int main() { + for (bool psram : {false, true}) { + const size_t floor = Admission::publishLargestBytes(psram); + assert(!Admission::hasPublishHeadroom(psram, floor - 1)); + assert(Admission::hasPublishHeadroom(psram, floor)); + // Previously admitted: 8 KiB largest was enough to start another client, + // even though publishing on a PSRAM board was already suppressed there. + assert(!Admission::canStart(psram, false, MemoryState::Uninitialized, 100000, 8192, true)); + assert(!Admission::canStart(psram, true, MemoryState::Stopped, 100000, 8192, true)); + const size_t allowances[] = {16384, 8192, 0}; + const MemoryState states[] = {MemoryState::Uninitialized, MemoryState::Stopped, + MemoryState::Started}; + for (size_t i = 0; i < 3; ++i) { + const size_t minimum = floor + allowances[i]; + assert(!Admission::canStart(psram, false, states[i], 100000, minimum - 1, true)); + assert(Admission::canStart(psram, false, states[i], 100000, minimum, true)); + // Spending the complete bounded SDK startup allowance still leaves a + // block accepted by the exact production packet-publishing policy. + assert(Admission::hasPublishHeadroom(psram, minimum - allowances[i])); + } + } +} +''') + + def test_actual_packet_publish_guard_uses_detected_psram_and_throttles_sampling(self): + self.compile_run(POLICY_PREAMBLE + r''' +#include "helpers/MQTTPublishAdmission.h" +#define ESP_PLATFORM 1 +#define BOARD_HAS_PSRAM 1 +static unsigned long now_ms = 0; +unsigned long millis() { return now_ms; } +constexpr uint32_t MALLOC_CAP_DMA = 2; +static size_t dma_free = Admission::kDmaReserveBytes; +static size_t dma_largest = Admission::kPsramPublishLargestBytes; +static size_t dma_queries = 0; +size_t heap_caps_get_free_size(uint32_t caps) { + assert(caps == MALLOC_CAP_DMA); ++dma_queries; return dma_free; +} +size_t heap_caps_get_largest_free_block(uint32_t caps) { + assert(caps == MALLOC_CAP_DMA); ++dma_queries; return dma_largest; +} +struct { + size_t psram = 2097152, largest = Admission::kInternalPublishLargestBytes, queries = 0; + size_t getPsramSize() const { return psram; } + size_t getMaxAllocHeap() { ++queries; return largest; } +} ESP; +struct MQTTBridge { + MQTTPublishAdmission::State _publish_memory; + int _skipped_publishes = 0; + bool hasPublishHeadroom(size_t = 0, size_t = 0, uint8_t = 0); +}; +''' + method("src/helpers/bridges/MQTTBridge.cpp", + "bool MQTTBridge::hasPublishHeadroom(") + r''' +int main() { + MQTTBridge bridge; + const uint32_t interval = MQTTPublishAdmission::kSampleIntervalMs; + assert(bridge.hasPublishHeadroom() && dma_queries == 2 && ESP.queries == 0); + --dma_free; + // Admission uses one captured snapshot, including its detected PSRAM flag. + // A later allocator/PSRAM change does not splice fresh fields into that snapshot. + ESP.psram = 0; + assert(bridge.hasPublishHeadroom() && dma_queries == 2 && ESP.queries == 0); + ESP.psram = 2097152; + now_ms += interval; + assert(!bridge.hasPublishHeadroom() && bridge._skipped_publishes == 1 && dma_queries == 4); + assert(!bridge.hasPublishHeadroom() && bridge._skipped_publishes == 2 && dma_queries == 4); + ++dma_free; --dma_largest; now_ms += interval; + assert(!bridge.hasPublishHeadroom() && bridge._skipped_publishes == 3 && dma_queries == 6); + ++dma_largest; now_ms += interval; + assert(bridge.hasPublishHeadroom() && dma_queries == 8); + assert(!bridge.hasPublishHeadroom(10240, 128, 1)); // Whole QoS1 outbox record. + assert(bridge.hasPublishHeadroom(10240, 128, 0)); // QoS0 uses fixed SDK chunks. + assert(dma_queries == 8); + // Compiling BOARD_HAS_PSRAM does not establish that PSRAM initialized. + ESP.psram = 0; now_ms += interval; + assert(bridge.hasPublishHeadroom() && ESP.queries == 1 && dma_queries == 8); + --ESP.largest; now_ms += interval; + assert(!bridge.hasPublishHeadroom() && ESP.queries == 2); + ++ESP.largest; now_ms += interval; + assert(bridge.hasPublishHeadroom() && ESP.queries == 3); +} ''') def test_internal_fallback_preserves_dma_reserve_and_rejects_fragmentation(self): @@ -289,35 +379,49 @@ int main() { assert(!bridge.canStartSlotConnection(0)); dma_free = 100000; dma_largest = 17407; assert(!bridge.canStartSlotConnection(0)); - ESP.size = 2097152; dma_free = 16384; dma_largest = 4096; + const size_t floor = std::max(Admission::publishLargestBytes(true), Admission::kPsramTlsLargestBytes); + const size_t bootstrap_cold = std::max(floor, Admission::kColdStartLargestBytes); + const size_t bootstrap_restart = std::max(floor, Admission::kRestartLargestBytes); + ESP.size = 2097152; dma_free = Admission::kDmaReserveBytes; dma_largest = bootstrap_cold; assert(!bridge.canStartSlotConnection(0)); Client client; bridge._slots[0].client = &client; assert(!bridge.canStartSlotConnection(0)); // Wrapper allocated; SDK not initialized. - dma_free = 32767; dma_largest = 20000; + dma_free = Admission::kDmaReserveBytes + Admission::kColdStartAllowanceBytes - 1; assert(!bridge.canStartSlotConnection(0)); - dma_free = 32768; dma_largest = 8191; + dma_free = Admission::kDmaReserveBytes + Admission::kColdStartAllowanceBytes; + dma_largest = bootstrap_cold - 1; assert(!bridge.canStartSlotConnection(0)); - dma_largest = 8192; + dma_largest = bootstrap_cold; assert(bridge.canStartSlotConnection(0)); + // An existing connected peer must keep its publication block beside the + // additional client's cold SDK allocations. + bridge._slots[1].connected = true; + dma_largest = floor + Admission::kColdStartAllowanceBytes - 1; + assert(!bridge.canStartSlotConnection(0)); + ++dma_largest; + assert(bridge.canStartSlotConnection(0)); + bridge._slots[1].connected = false; // An initialized but stopped handle needs its task again, not new SDK buffers. client.initialized = true; dma_free = 24575; dma_largest = 20000; assert(!bridge.canStartSlotConnection(0)); - dma_free = 24576; dma_largest = 8191; + dma_free = Admission::kDmaReserveBytes + Admission::kRestartAllowanceBytes; + dma_largest = bootstrap_restart - 1; assert(!bridge.canStartSlotConnection(0)); - dma_largest = 8192; + dma_largest = bootstrap_restart; assert(bridge.canStartSlotConnection(0)); // A warm reconnect retains the SDK task/buffers and uses the smaller floor. - client.started = true; dma_free = 16384; dma_largest = 4096; + client.started = true; dma_free = Admission::kDmaReserveBytes; dma_largest = floor; assert(bridge.canStartSlotConnection(0)); - dma_largest = 4095; + --dma_largest; assert(!bridge.canStartSlotConnection(0)); // An explicit plaintext URI overrides a TLS port. The effective URI wins // over the raw host; a preset's URI wins over both custom fields. - ESP.size = 0; dma_free = 16384; dma_largest = 4096; + ESP.size = 0; dma_free = Admission::kDmaReserveBytes; + dma_largest = Admission::kInternalPublishLargestBytes; strcpy(bridge._slots[0].host, "mqtt://broker:8883"); assert(bridge.canStartSlotConnection(0)); strcpy(bridge._slots[0].broker_uri, "wss://broker/mqtt"); @@ -329,8 +433,37 @@ int main() { bridge._slots[0].preset = &tls; assert(!bridge.canStartSlotConnection(0)); ESP.size = 2097152; + assert(!bridge.canStartSlotConnection(0)); // TLS still needs its incoming-record floor. + dma_largest = floor; assert(bridge.canStartSlotConnection(0)); + // Reproduce the reporter's failed-state heap after all SDK tasks stopped. + // One broker must be able to recover instead of waiting forever for a + // contiguous 24,192-byte block pinned apart by retained SDK buffers. + client.started = false; + dma_free = 93024; dma_largest = 20468; + assert(bridge.canStartSlotConnection(0)); + // Outgoing QoS0 chunks do not bound incoming TLS records. Keep the original + // additional TLS admission beside a live peer, even though packet publishing + // itself now uses a separate four-KiB DMA scratch floor. + bridge._slots[1].connected = true; + assert(!bridge.canStartSlotConnection(0)); + dma_largest = floor + Admission::kRestartAllowanceBytes - 1; + assert(!bridge.canStartSlotConnection(0)); + ++dma_largest; + assert(bridge.canStartSlotConnection(0)); + // Health reconciliation removing the other failed connection restores + // bootstrap admission. The target never counts itself as a live peer. + dma_largest = bootstrap_restart; + bridge._slots[1].connected = false; + bridge._slots[0].connected = true; + assert(bridge.canStartSlotConnection(0)); + bridge._slots[0].connected = false; + bridge._slots[1].enabled = false; + bridge._slots[1].connected = true; + assert(bridge.canStartSlotConnection(0)); + bridge._slots[1].connected = false; + // Every pending TLS attempt, including the target's own attempt, blocks all // other starts until an event callback clears it. No heap walk is needed. for (int pending = 0; pending < RUNTIME_MQTT_SLOTS; ++pending) { diff --git a/test/test_mqtt_handshake_scheduler.py b/test/test_mqtt_handshake_scheduler.py index 43682ae46..42228c99b 100644 --- a/test/test_mqtt_handshake_scheduler.py +++ b/test/test_mqtt_handshake_scheduler.py @@ -16,6 +16,7 @@ ROOT = Path(__file__).resolve().parents[1] PREAMBLE = r''' #include +#include #include #include #include @@ -32,6 +33,7 @@ PREAMBLE = r''' #include "helpers/MQTTTransportHealth.h" #include "helpers/MQTTStatusPublishPolicy.h" #include "helpers/WiFiPowerSave.h" +namespace Admission = MQTTConnectionAdmission; #define ESP_PLATFORM 1 #define MQTT_DEBUG_PRINTLN(...) ((void)0) using esp_err_t = int; @@ -56,7 +58,7 @@ struct esp_mqtt_error_codes { constexpr uint32_t MALLOC_CAP_INTERNAL = 1, MALLOC_CAP_DMA = 2; static uint32_t clock_ms = 1000; static unsigned long wall_seconds = 1800000000; -static size_t dma_free = 100000, dma_largest = 20000; +static size_t dma_free = 100000, dma_largest = 100000; static unsigned long s_wifi_connected_at = 0; uint32_t millis() { return clock_ms; } unsigned long fake_time(void*) { return wall_seconds; } @@ -157,7 +159,7 @@ struct PsychicMqttClient { void softDisconnect() { ++bounces; live = false; // Releasing the old internal TLS session creates room for its replacement. - dma_free += 44000; dma_largest = 20000; + dma_free += 44000; dma_largest = 50000; } void setCredentials(const char*, const char*) { ++credentials; } void serviceOutboxDiagnostics() {} @@ -429,15 +431,19 @@ int main() { self.compile_run(r''' int main() { MQTTBridge bridge; - bridge.only(0); clock_ms = 1000; dma_free = 32767; + bridge.only(0); clock_ms = 1000; + const size_t cold_free = Admission::kDmaReserveBytes + Admission::kColdStartAllowanceBytes; + const size_t cold_largest = std::max(Admission::kPsramTlsLargestBytes, + std::max(Admission::publishLargestBytes(true), Admission::kColdStartLargestBytes)); + dma_free = cold_free - 1; bridge.maintainSlotConnections(); assert(bridge.setups.empty() && bridge._slots[0].last_reconnect_attempt == 0); assert(bridge._slots[0].enabled && bridge.activatedSlotCount() == 0); - dma_free = 32768; dma_largest = 8191; + dma_free = cold_free; dma_largest = cold_largest - 1; bridge.maintainSlotConnections(); assert(bridge.setups.empty() && bridge._slots[0].last_reconnect_attempt == 0); assert(!bridge.hasPendingSlotConnection() && bridge.activatedSlotCount() == 0); - dma_largest = 8192; + dma_largest = cold_largest; bridge.maintainSlotConnections(); assert(bridge.setups == std::vector{0} && bridge.activatedSlotCount() == 1); assert(bridge.clients[0].starts == 1 && bridge.clients[0].reconnects == 0); @@ -450,7 +456,8 @@ int main() { MQTTBridge bridge; for (int i = 2; i < RUNTIME_MQTT_SLOTS; ++i) bridge._slots[i].enabled = false; clock_ms = 70000; - dma_free = 16384; dma_largest = 4096; + const size_t floor = std::max(Admission::publishLargestBytes(true), Admission::kPsramTlsLargestBytes); + dma_free = Admission::kDmaReserveBytes; dma_largest = floor; auto& stopped = bridge._slots[0]; auto& retained = bridge._slots[1]; stopped.client = &bridge.clients[0]; @@ -474,7 +481,10 @@ int main() { // with no extra delay introduced by the earlier memory-only deferral. bridge.complete(1); clock_ms += 15000; - dma_free = 24576; dma_largest = 8192; + dma_free = Admission::kDmaReserveBytes + Admission::kRestartAllowanceBytes; + dma_largest = floor + Admission::kRestartAllowanceBytes - 1; + assert(!bridge.canStartSlotConnection(0)); + ++dma_largest; bridge.maintainSlotConnections(); assert(stopped.client->starts == 1 && stopped.client->reconnects == 0); assert(stopped.client->isStarted() && bridge._slot_attempt_pending[0]); @@ -546,16 +556,19 @@ int main() { assert(!bridge.wifiOutage().down); // Restart admission still fails safely if either aggregate or contiguous // headroom is inadequate. Neither failure consumes retry/backoff state. - dma_free = 24575; + const size_t restart_free = Admission::kDmaReserveBytes + Admission::kRestartAllowanceBytes; + const size_t floor = std::max(Admission::publishLargestBytes(true), Admission::kPsramTlsLargestBytes); + const size_t bootstrap_largest = std::max(floor, Admission::kRestartLargestBytes); + dma_free = restart_free - 1; bridge.maintainSlotConnections(); - dma_free = 24576; dma_largest = 8191; + dma_free = restart_free; dma_largest = bootstrap_largest - 1; bridge.maintainSlotConnections(); for (int i = 0; i < 3; ++i) { assert(bridge.clients[i].starts == 0 && bridge._slots[i].last_reconnect_attempt == 0); assert(bridge._slots[i].reconnect_backoff == 0 && bridge._slots[i].start_failures == 0); } - dma_free = 42376; dma_largest = 22516; + dma_free = 42376; dma_largest = floor + Admission::kRestartAllowanceBytes; bridge.maintainSlotConnections(); assert(bridge.clients[0].starts == 1 && dma_free == 35840); assert(bridge.hasPendingSlotConnection()); @@ -845,14 +858,17 @@ int main() { assert(slot.client->stops == 1 && slot.reconnect_backoff == 3); assert(slot.max_backoff_failures == 1 && slot.connect_failures == 0); clock_ms += 120000; - dma_free = 24575; dma_largest = 20000; + const size_t restart_free = Admission::kDmaReserveBytes + Admission::kRestartAllowanceBytes; + const size_t restart_largest = std::max(Admission::kPsramTlsLargestBytes, + std::max(Admission::publishLargestBytes(true), Admission::kRestartLargestBytes)); + dma_free = restart_free - 1; dma_largest = 20000; bridge.maintainSlotConnections(); assert(slot.client->starts == 0 && slot.reconnect_backoff == 3); assert(slot.last_reconnect_attempt == 70000 && slot.start_failures == 0); - dma_free = 24576; dma_largest = 8191; + dma_free = restart_free; dma_largest = restart_largest - 1; bridge.maintainSlotConnections(); assert(slot.client->starts == 0 && slot.reconnect_backoff == 3); - dma_largest = 8192; + dma_largest = restart_largest; bridge.maintainSlotConnections(); assert(slot.client->starts == 1 && bridge._slot_attempt_pending[0]); assert(slot.reconnect_backoff == 4 && slot.max_backoff_failures == 1); diff --git a/test/test_mqtt_network_timeout.py b/test/test_mqtt_network_timeout.py index bef58c9ea..c0afc0365 100644 --- a/test/test_mqtt_network_timeout.py +++ b/test/test_mqtt_network_timeout.py @@ -112,7 +112,7 @@ void* heap_caps_malloc(size_t bytes, uint32_t caps) { return std::malloc(bytes); } void heap_caps_free(void* memory) { std::free(memory); } -static size_t dma_free = 100000, dma_largest = 20000; +static size_t dma_free = 100000, dma_largest = 100000; size_t heap_caps_get_free_size(uint32_t caps) { assert(caps == (MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA)); return dma_free; } @@ -232,11 +232,21 @@ int main() { assert(init_calls == 1 && destroy_calls == 0); // Reconfiguration retains the initialized client on both board profiles. // It needs the stopped-task floor and never silently shrinks its buffers. - dma_free = 24575; dma_largest = 20000; + dma_free = MQTTConnectionAdmission::kDmaReserveBytes + + MQTTConnectionAdmission::kRestartAllowanceBytes - 1; + dma_largest = 100000; bridge.applySlotPreset(0, "plain"); assert(slot.client && !slot.client->isStarted() && !slot.initial_connect_done); assert(init_calls == 1 && destroy_calls == 0); - dma_free = 24576; + dma_free = MQTTConnectionAdmission::kDmaReserveBytes + + MQTTConnectionAdmission::kRestartAllowanceBytes; + const bool psram = ESP.getPsramSize() > 0; + const size_t floor = MQTTConnectionAdmission::publishLargestBytes(psram); + const size_t bootstrap_largest = floor > MQTTConnectionAdmission::kRestartLargestBytes + ? floor : MQTTConnectionAdmission::kRestartLargestBytes; + dma_largest = bootstrap_largest - 1; + assert(!bridge.setupSlot(0)); + dma_largest = bootstrap_largest; assert(bridge.setupSlot(0)); assert(slot.client->_buffer_capacity == 896 && init_calls == 1); const int destructions_before_guard = destroy_calls; diff --git a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp index d901bf8e1..dc380adbd 100644 --- a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp +++ b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -19,6 +20,23 @@ struct RejectAllJsonAllocations : ArduinoJson::Allocator { void* reallocate(void*, size_t) override { return nullptr; } }; +struct FailAfterJsonAllocations : ArduinoJson::Allocator { + size_t remaining; + size_t denied = 0; + explicit FailAfterJsonAllocations(size_t allowed) : remaining(allowed) {} + void* allocate(size_t size) override { + if (!remaining) { ++denied; return nullptr; } + --remaining; + return std::malloc(size); + } + void deallocate(void* pointer) override { std::free(pointer); } + void* reallocate(void* pointer, size_t size) override { + if (!remaining) { ++denied; return nullptr; } + --remaining; + return std::realloc(pointer, size); + } +}; + static int buildMinimalStatus(JsonDocument& scratch, char* buffer, size_t buffer_size) { return MQTTPayloadBuilder::buildStatusMessage( scratch, "DEN Repeater", "0123456789ABCDEF", "Heltec V3", "v1.16.0", @@ -37,6 +55,51 @@ static int buildRepresentativePacket(JsonDocument& scratch, const char* directio path_hash_size, 64, buffer, buffer_size); } +static int buildMinimalPacket(JsonDocument& doc, char* buffer, size_t capacity) { + return buildRepresentativePacket(doc, "rx", 0.125f, nullptr, 0, 0, "AABB", buffer, capacity); +} + +static int buildMinimalRaw(JsonDocument& doc, char* buffer, size_t capacity) { + return MQTTPayloadBuilder::buildRawMessage(doc, "node", "id", kTimestamp, "AABB", buffer, capacity); +} + +TEST(MQTTPayloadBuilder, StatusPacketAndRawRejectAllAllocationFailures) { + for (auto build : {buildMinimalStatus, buildMinimalPacket, buildMinimalRaw}) { + RejectAllJsonAllocations allocator; + JsonDocument doc(&allocator); + char buffer[2048]; + memset(buffer, 'x', sizeof(buffer)); + EXPECT_EQ(0, build(doc, buffer, sizeof(buffer))); + EXPECT_TRUE(doc.overflowed()); + EXPECT_EQ('\0', buffer[0]); + } +} + +TEST(MQTTPayloadBuilder, StatusPacketAndRawRejectPartiallyPopulatedDocuments) { + for (auto build : {buildMinimalStatus, buildMinimalPacket, buildMinimalRaw}) { + bool found_partial_failure = false; + for (size_t allowed = 1; allowed < 32; ++allowed) { + FailAfterJsonAllocations allocator(allowed); + JsonDocument doc(&allocator); + char buffer[2048]; + memset(buffer, 'x', sizeof(buffer)); + const int result = build(doc, buffer, sizeof(buffer)); + if (allocator.denied && doc.size() > 0) { + found_partial_failure = true; + EXPECT_TRUE(doc.overflowed()); + EXPECT_EQ(0, result); + EXPECT_EQ('\0', buffer[0]); + // A document is reusable after memory becomes available again. + allocator.remaining = 64; + EXPECT_GT(build(doc, buffer, sizeof(buffer)), 0); + EXPECT_FALSE(doc.overflowed()); + break; + } + } + EXPECT_TRUE(found_partial_failure); + } +} + TEST(MQTTPayloadBuilder, MinimalStatusHasExactRequiredContract) { JsonDocument scratch; char buffer[768]; @@ -523,11 +586,13 @@ TEST(MQTTPayloadBuilder, NeighborsMessageFailsCleanlyOnAllocationFailure) { RejectAllJsonAllocations allocator; JsonDocument scratch(&allocator); char buffer[512]; + memset(buffer, 'x', sizeof(buffer)); EXPECT_EQ(0, MQTTPayloadBuilder::buildNeighborsMessage( scratch, "node", "id", kTimestamp, "DEN", "*", &neighbor, 1, buffer, sizeof(buffer), 1, 1, false)); + EXPECT_EQ('\0', buffer[0]); } TEST(MQTTPayloadBuilder, NeighborsMessageHandlesEmptyTableAndNullScopes) { diff --git a/test/test_mqtt_payload_oom.py b/test/test_mqtt_payload_oom.py new file mode 100644 index 000000000..2eed11fc4 --- /dev/null +++ b/test/test_mqtt_payload_oom.py @@ -0,0 +1,137 @@ +"""Reject incomplete real ArduinoJson documents before MQTT publication.""" + +from pathlib import Path +import os +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "src/helpers/MQTTPayloadBuilder.cpp" +HARNESS = r''' +#include +#include +#include +#include +#include +#include "helpers/MQTTPayloadBuilder.h" + +struct FailingAllocator : ArduinoJson::Allocator { + enum Mode { Healthy, RejectAll, RejectSecondPool } mode; + unsigned pools = 0, denied = 0; + explicit FailingAllocator(Mode value) : mode(value) {} + void* allocate(size_t bytes) override { + const size_t pool_bytes = ArduinoJson::detail::ResourceManager::slotSize * ARDUINOJSON_POOL_CAPACITY; + if (mode == RejectAll || (mode == RejectSecondPool && bytes == pool_bytes && pools++)) { + ++denied; + return nullptr; + } + return std::malloc(bytes); + } + void* reallocate(void* pointer, size_t bytes) override { + if (mode == RejectAll) { ++denied; return nullptr; } + return std::realloc(pointer, bytes); + } + void deallocate(void* pointer) override { std::free(pointer); } +}; + +static int build(unsigned kind, JsonDocument& doc, char* buffer, size_t capacity) { + if (kind == 0) return MQTTPayloadBuilder::buildStatusMessage(doc, + "node", "identity", "model", "firmware", "radio", "client", "online", "timestamp", + buffer, capacity, 4200, 12, 0, 1, -112, 2, 3, 0, 20000, 4, 5, "on"); + if (kind == 1) { + const uint8_t path[] = {0xAA, 0xBB}; + return MQTTPayloadBuilder::buildPacketMessage(doc, "node", "identity", "timestamp", + "rx", "12:34:56", "27/10/2026", 12, 4, "F", 2, "AABB", 2.5f, -85, + 0.125f, "hash", path, 2, 1, sizeof(path), buffer, capacity); + } + if (kind == 2) return MQTTPayloadBuilder::buildRawMessage(doc, + "node", "identity", "timestamp", "AABB", buffer, capacity); + const MQTTPayloadBuilder::NeighborsMessageEntry neighbor = { + "ABCDEF", 2.5f, 10, "region", "active", -85, false}; + return MQTTPayloadBuilder::buildNeighborsMessage(doc, "node", "identity", "timestamp", + "region", "*", &neighbor, 1, buffer, capacity, 1, 1, false); +} + +int main() { + static_assert(ARDUINOJSON_POOL_CAPACITY == 4, "Force real pool growth even for the five-field RAW message"); + for (unsigned kind = 0; kind < 4; ++kind) { + JsonDocument healthy; + char expected[2048]; + const int complete = build(kind, healthy, expected, sizeof(expected)); + assert(complete > 0 && !healthy.overflowed()); + JsonDocument parsed; + assert(!deserializeJson(parsed, expected)); + assert(parsed.size() >= 5); + + for (auto mode : {FailingAllocator::RejectAll, FailingAllocator::RejectSecondPool}) { + FailingAllocator allocator(mode); + JsonDocument doc(&allocator); + char buffer[2048]; memset(buffer, 'x', sizeof(buffer)); + const int written = build(kind, doc, buffer, sizeof(buffer)); + assert(allocator.denied && doc.overflowed()); + if (mode == FailingAllocator::RejectSecondPool) { + assert(allocator.pools > 1 && doc.size() > 0); + // The corrupt partial object itself still serializes to positive JSON. + // Only the production completeness guard prevents its publication. + char incomplete[2048]; + assert(serializeJson(doc, incomplete, sizeof(incomplete)) > 0); + assert(strcmp(incomplete, expected)); + } + assert(written == 0 && buffer[0] == '\0'); + // Recovering the same document clears its overflow state and restores + // every original byte, including packet metrics/path and neighbor data. + allocator.mode = FailingAllocator::Healthy; + assert(build(kind, doc, buffer, sizeof(buffer)) == complete); + assert(!doc.overflowed() && !strcmp(buffer, expected)); + assert(build(kind, doc, nullptr, sizeof(buffer)) == 0); + assert(build(kind, doc, buffer, 0) == 0); + } + } +} +''' + + +class MqttPayloadOomTests(unittest.TestCase): + def compile_run(self, source=None, reject=False): + compiler = shutil.which("g++") or shutil.which("clang++") + self.assertIsNotNone(compiler, "a native C++ compiler is required") + header = next((ROOT / ".pio/libdeps").glob("*/ArduinoJson/src/ArduinoJson.h"), None) + self.assertIsNotNone(header, "actual ArduinoJson headers are required") + with tempfile.TemporaryDirectory(prefix="mqtt-payload-oom-") as directory: + folder = Path(directory) + harness, binary = folder / "oom.cpp", folder / "oom" + harness.write_text(HARNESS) + production = SOURCE + if source is not None: + production = folder / "MQTTPayloadBuilder.cpp" + production.write_text(source) + flags = [] if os.name == "nt" else ["-fsanitize=address,undefined", "-fno-sanitize-recover=all", "-fno-pie", "-no-pie"] + built = subprocess.run([compiler, "-std=c++17", "-O1", "-Wall", "-Wextra", "-Werror", + "-DARDUINOJSON_POOL_CAPACITY=4", *flags, "-I" + str(header.parent), + "-I" + str(ROOT / "src"), "-I" + str(ROOT / "src/helpers"), + "-isystem", str(ROOT / "test/mocks"), + str(harness), str(production), str(ROOT / "src/helpers/TxtDataHelpers.cpp"), "-o", str(binary)], + capture_output=True, text=True, timeout=60) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + checked = subprocess.run([str(binary)], capture_output=True, text=True, timeout=30) + if reject: + self.assertNotEqual(checked.returncode, 0, checked.stdout + checked.stderr) + self.assertIn("Assertion", checked.stderr) + else: + self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr) + + def test_all_builders_reject_total_and_partial_pool_oom_and_recover(self): + self.compile_run() + + def test_previous_length_only_serialization_is_rejected(self): + source = SOURCE.read_text() + old = source.replace(" if (doc.overflowed() || root.isNull()) return 0;", "") + self.assertNotEqual(source, old) + self.compile_run(old, reject=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_mqtt_publish_admission.py b/test/test_mqtt_publish_admission.py index e4e2517ed..961c5bacc 100644 --- a/test/test_mqtt_publish_admission.py +++ b/test/test_mqtt_publish_admission.py @@ -79,11 +79,15 @@ int main(){ heap.dma_free+=5*897; // The five CPU-only reassembly buffers move to PSRAM. assert(canPublish(heap)); // No new 16K internal allocation is needed to write. heap.dma_free=16384;heap.dma_largest=4096; - assert(canPublish(heap,2048,128,0)&&canPublish(heap,1024,128,1)); + assert(canPublish(heap,2048,128,0)); + assert(!canPublish(heap,1024,128,1)); // Enqueue must also retain the reserve. --heap.dma_free;assert(!canPublish(heap));++heap.dma_free; --heap.dma_largest;assert(!canPublish(heap));++heap.dma_largest; + heap.dma_free=16384+1024+128+128; + assert(canPublish(heap,1024,128,1)); + --heap.dma_free;assert(!canPublish(heap,1024,128,1)); assert(!canPublish(heap,10240,128,1)); // Large outbox records need more space. - heap.dma_free=24000;heap.dma_largest=11000; + heap.dma_free=40000;heap.dma_largest=11000; assert(canPublish(heap,10240,128,1)); assert(!canPublish(heap,SIZE_MAX,1,1)&&!canPublish(heap,1,SIZE_MAX,0)); heap.psram=false;heap.internal_largest=7999;assert(!canPublish(heap)); diff --git a/test/test_mqtt_wifi_network_power.py b/test/test_mqtt_wifi_network_power.py index c9faadc4c..23cead856 100644 --- a/test/test_mqtt_wifi_network_power.py +++ b/test/test_mqtt_wifi_network_power.py @@ -183,6 +183,7 @@ static void fillNtpServerList(const Prefs*, const char* servers[], int& count) { @HELPERS@ struct MQTTBridge { + struct PublishMemory { void invalidate() {} } _publish_memory; static constexpr int kMaxNtpServers = 6; struct Slot { bool enabled = true, connected = false, circuit_breaker_tripped = true; @@ -613,6 +614,41 @@ static void publishAcknowledgmentProvesRecoveryImmediately() { "an old PUBACK repeatedly renewed a disconnected broker"); } +static void disconnectedPublishAcknowledgmentRenewsGraceOnce() { + for (uint32_t start : {100U, UINT32_MAX - 10000U}) { + resetEnvironment(); + MQTTBridge bridge; + require(bridge.serviceNetworkPower(start), "initial outage did not run"); + require(!bridge.serviceNetworkPower(start + grace), "outage did not suspend"); + const uint32_t resumed = start + grace + interval; + require(bridge.serviceNetworkPower(resumed), "probe did not resume"); + const uint32_t replied = resumed + probe - 1U; + // The SDK delivered PUBACK and then DISCONNECTED between owner passes. + // A real broker reply still proves service; a CONNACK by itself does not. + bridge.clients[0].publish_acks = 1U; + require(!bridge._slots[0].connected, "fixture unexpectedly connected"); + require(bridge.serviceNetworkPower(replied), "new PUBACK was discarded"); + require(bridge.serviceNetworkPower(resumed + probe), + "disconnect hid PUBACK and retained the expired probe deadline"); + require(bridge.serviceNetworkPower(replied + grace - 1U), + "disconnected PUBACK did not renew normal grace"); + require(!bridge.serviceNetworkPower(replied + grace), + "unchanged PUBACK kept a failed service awake forever"); + } + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + bridge._slots[0].enabled = false; + bridge.clients[0].publish_acks = 1U; + require(!bridge.serviceNetworkPower(100U + grace), + "a disabled broker renewed the outage budget"); + bridge._slots[0].enabled = true; + require(!bridge.serviceNetworkPower(101U + grace), + "enabling a broker replayed its already consumed PUBACK"); + } +} + static void quietConnectionHasExactThirtySecondHealthBoundary() { resetEnvironment(); { @@ -744,6 +780,7 @@ int main(int argc, char** argv) { else if (name == "channel-repair") deferredEspNowChannelRepair(); else if (name == "healthy") stableBrokerCountsButFlappingDoesNot(); else if (name == "puback") publishAcknowledgmentProvesRecoveryImmediately(); + else if (name == "disconnected-puback") disconnectedPublishAcknowledgmentRenewsGraceOnce(); else if (name == "quiet-boundary") quietConnectionHasExactThirtySecondHealthBoundary(); else if (name == "ntp-retry") sntpFailureAndBusyLeaseRetries(); else if (name == "ntp-rotation") everyNtpServerGetsTheLeadingTimeoutSlot(); @@ -892,6 +929,9 @@ class MqttWiFiNetworkPowerTests(unittest.TestCase): def test_publish_ack_proves_recovery_immediately_and_is_consumed_once(self): self.run_case("puback") + def test_puback_followed_by_disconnect_renews_grace_once_across_wrap(self): + self.run_case("disconnected-puback") + def test_quiet_broker_health_requires_exactly_thirty_seconds(self): self.run_case("quiet-boundary") diff --git a/test/test_nrf52_flash_trim.py b/test/test_nrf52_flash_trim.py index 402f3e5dd..d1cd69251 100644 --- a/test/test_nrf52_flash_trim.py +++ b/test/test_nrf52_flash_trim.py @@ -67,6 +67,26 @@ class Nrf52FlashTrimTest(unittest.TestCase): self.assertNotIn("SSD1306_NO_SPLASH", defines) self.assertEqual(TRIM.flag_defines("-DMESH_NRF52_FLASH_TRIM=1 -UMESH_NRF52_FLASH_TRIM")["MESH_NRF52_FLASH_TRIM"], None) + def test_local_names_honor_overlay_identity_and_explicit_defines(self): + flags = ('-DMESH_OTA_LOCAL_TARGET_NAME_ONLY=1 ' + '-DMOTA_TARGET_ID=0x29a0da19 ' + '-DOTA_VARIANT=\'"RAK_4631_repeater_lora_ota_no_external_sensors"\'') + defines = self.policy(flags=flags) + self.assertEqual(defines["OTA_TARGET_NAME_TABLE"], 0) + self.assertEqual(defines["OTA_LOCAL_TARGET_ID"], "0x29a0da19") + self.assertEqual(defines["OTA_LOCAL_TARGET_NAME"], "OTA_VARIANT") + explicit = self.policy(flags=flags + ' -DOTA_TARGET_NAME_TABLE=1 ' + '-DOTA_LOCAL_TARGET_ID=7 -DOTA_LOCAL_TARGET_NAME=custom_name') + for name in ("OTA_TARGET_NAME_TABLE", "OTA_LOCAL_TARGET_ID", "OTA_LOCAL_TARGET_NAME"): + self.assertNotIn(name, explicit) + for flags in ("", "-DMESH_OTA_LOCAL_TARGET_NAME_ONLY=0"): + self.assertNotIn("OTA_TARGET_NAME_TABLE", self.policy(flags=flags)) + + def test_local_names_do_not_change_companion_or_unqualified_roles(self): + for options in (dict(name="RAK_4631_companion_radio_full"), + dict(role="custom"), dict(platform="espressif32", mcu="esp32")): + self.assertEqual(self.policy(flags="-DMESH_OTA_LOCAL_TARGET_NAME_ONLY=1", **options), {}) + def test_cc310_archive_is_exact_and_requires_supported_mcu_and_enabled_crypto(self): with tempfile.TemporaryDirectory() as temp_dir: archive = Path(temp_dir) / "libraries/Adafruit_nRFCrypto" / TRIM.CC310_ARCHIVE @@ -134,6 +154,12 @@ class Nrf52FlashTrimTest(unittest.TestCase): def BuildSources(self, *args): self.setdefault("BUILT_SOURCES", []).append(args) + def subst(self, value): + return value.replace("$PROJECT_DIR", str(ROOT)) + + def AddBuildMiddleware(self, callback, pattern): + self.setdefault("MIDDLEWARE", []).append((callback, pattern)) + base = dict(BUILD_FLAGS=["-DUSE_CC310_HW_CRYPTO=1"], SRC_FILTER=["+<../examples/simple_repeater>"], CPPDEFINES=[]) excluded = BuildEnvironment(**base, PIOENV="RAK_4631_companion_radio_full") @@ -148,6 +174,7 @@ class Nrf52FlashTrimTest(unittest.TestCase): self.assertTrue(active["MESH_NRF52_FLASH_TRIM_ACTIVE"]) self.assertIn("-Wl,--wrap=CRYS_ECPKI_GetEcDomain", active["LINKFLAGS"]) self.assertEqual(active["BUILT_SOURCES"][0][-1], "+") + self.assertEqual(active["MIDDLEWARE"][0][1], "*Adafruit_VL53L0X*/*.cpp") @unittest.skipUnless(shutil.which("cc"), "a native C compiler is required") def test_wrapped_domain_preserves_p256_and_rejects_unused_curves(self): diff --git a/test/test_nrf52_vl53l0x_const.py b/test/test_nrf52_vl53l0x_const.py new file mode 100644 index 000000000..cacc1f095 --- /dev/null +++ b/test/test_nrf52_vl53l0x_const.py @@ -0,0 +1,190 @@ +"""Qualify flash-resident VL53L0X tables against the reviewed vendor loader.""" + +import contextlib +import importlib.util +import io +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = ROOT / "test/fixtures/vl53_const" +SPEC = importlib.util.spec_from_file_location("vl53_const", ROOT / "scripts/nrf52_vl53l0x_const.py") +TRIM = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(TRIM) + + +class Node: + def __init__(self, path): + self.path = Path(path) + + def srcnode(self): + return self + + def get_abspath(self): + return str(self.path) + + +class Environment: + def __init__(self, build): + self.build = Path(build) + + def subst(self, value): + assert value == "$BUILD_DIR" + return str(self.build) + + def File(self, path): + return Path(path) + + +class Vl53ConstTests(unittest.TestCase): + def test_private_copy_is_complete_idempotent_and_downloads_are_unchanged(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "libdeps/Adafruit_VL53L0X/src" + texts = {} + for name, (old, _) in TRIM.REPLACEMENTS.items(): + texts[name] = "// synthetic reviewed fixture\n" + old + "\n" + texts["vl53l0x_def.h"] = "// ordinary sibling header\n" + texts["core/src/vl53l0x_api_calibration.cpp"] = '#include "../../vl53l0x_api_core.h"\n' + for name, text in texts.items(): + path = source / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(text.replace("\n", "\r\n").encode()) + before = {name: (source / name).read_bytes() for name in texts} + node = Node(source / "core/src/vl53l0x_api_calibration.cpp") + env = Environment(root / "build") + hashes = {name: TRIM.digest(texts[name]) for name in TRIM.FILES} + with patch.object(TRIM, "FILES", hashes): + output = TRIM.replace_vl53_source(env, node) + repeated = TRIM.replace_vl53_source(env, node) + self.assertEqual(output, repeated) + copied = root / "build/readonly-nrf52-vl53l0x/src" + self.assertEqual(output, copied / "core/src/vl53l0x_api_calibration.cpp") + self.assertEqual({name: (source / name).read_bytes() for name in texts}, before) + for name, text in texts.items(): + if name in TRIM.REPLACEMENTS: + old, new = TRIM.REPLACEMENTS[name] + self.assertEqual((copied / name).read_text(), text.replace(old, new)) + else: + self.assertEqual((copied / name).read_bytes(), before[name]) + # One changed table, loader, or prototype keeps every original TU. + for changed in TRIM.FILES: + path = source / changed + saved = path.read_bytes() + path.write_bytes(saved + b"// custom revision\n") + with patch.object(TRIM, "FILES", hashes), contextlib.redirect_stdout(io.StringIO()): + self.assertIs(TRIM.replace_vl53_source(env, node), node) + path.write_bytes(saved) + + def test_real_vendor_tables_produce_identical_register_writes_and_parameters(self): + original = (FIXTURE / "tuning_loader.inc").read_text() + mutable = original.replace("VL53L0X_load_tuning_settings", "load_original") + readonly = original.replace("VL53L0X_load_tuning_settings", "load_readonly").replace( + *TRIM.REPLACEMENTS["core/src/vl53l0x_api_core.cpp"]) + tables = "\n".join((FIXTURE / name).read_text().replace(*TRIM.REPLACEMENTS[name]) + .replace('#include "vl53l0x_def.h"', '') + for name in ("vl53l0x_tuning.h", "vl53l0x_interrupt_threshold_settings.h")) + source = r''' +#include +#include +#include +#include +#include +struct Device { + unsigned SigmaEstRefArray=0, SigmaEstEffPulseWidth=0, SigmaEstEffAmbWidth=0, targetRefRate=0; + unsigned calls=0, fail=0; + std::vector writes; +}; +using VL53L0X_Error = int; +using VL53L0X_DEV = Device*; +#define VL53L0X_ERROR_NONE 0 +#define VL53L0X_ERROR_INVALID_PARAMS 1 +#define LOG_FUNCTION_START(...) +#define LOG_FUNCTION_END(...) +#define PALDevDataSet(device, member, value) ((device)->member = (value)) +#define VL53L0X_MAKEUINT16(low, high) (uint16_t(low) | (uint16_t(high) << 8)) +int VL53L0X_WriteMulti(Device* device, uint8_t address, uint8_t* bytes, unsigned count) { + device->writes.push_back(address); device->writes.push_back(uint8_t(count)); + device->writes.insert(device->writes.end(),bytes,bytes+count); + return ++device->calls == device->fail ? 2 : 0; +} +@TABLES@ +@MUTABLE@ +@READONLY@ +static_assert(std::is_const::type>::value, + "default tuning must live in readonly storage"); +int main() { + const struct { const uint8_t* bytes; unsigned size; } inputs[] = { + {DefaultTuningSettings,sizeof(DefaultTuningSettings)}, + {InterruptThresholdSettings,sizeof(InterruptThresholdSettings)}, + }; + assert(inputs[0].size == 243 && inputs[1].size == 438); + for (const auto& input:inputs) { + for(unsigned fail=0;fail copy(input.bytes,input.bytes+input.size); + Device before, after; before.fail=after.fail=fail; + assert(load_original(&before,copy.data()) == load_readonly(&after,input.bytes)); + assert(before.calls==after.calls && before.writes==after.writes); + assert(before.SigmaEstRefArray==after.SigmaEstRefArray); + assert(before.SigmaEstEffPulseWidth==after.SigmaEstEffPulseWidth); + assert(before.SigmaEstEffAmbWidth==after.SigmaEstEffAmbWidth); + assert(before.targetRefRate==after.targetRefRate); + assert(memcmp(copy.data(),input.bytes,input.size)==0); + } + } + // The caller-owned custom tuning API keeps accepting ordinary mutable data. + uint8_t custom[] = {0xFF,0,0x01,0x23,2,0x10,0xAA,0xBB,0}; + uint8_t snapshot[sizeof(custom)]; memcpy(snapshot,custom,sizeof(custom)); + Device before,after; + assert(load_original(&before,custom)==load_readonly(&after,custom)); + assert(before.SigmaEstRefArray==after.SigmaEstRefArray && before.writes==after.writes); + assert(memcmp(custom,snapshot,sizeof(custom))==0); +} +'''.replace("@TABLES@", tables).replace("@MUTABLE@", mutable).replace("@READONLY@", readonly) + with tempfile.TemporaryDirectory() as directory: + path, binary = Path(directory) / "loader.cpp", Path(directory) / "loader" + path.write_text(source, encoding="ascii") + built = subprocess.run(["g++", "-std=c++17", "-O1", "-Wall", "-Wextra", "-Werror", + "-fsanitize=address,undefined", "-fno-pie", "-no-pie", + str(path), "-o", str(binary)], capture_output=True, text=True, timeout=30) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + subprocess.run([str(binary)], check=True, timeout=10) + + def test_arm_table_bytes_move_to_readonly_without_changing_their_size(self): + compiler = shutil.which("arm-none-eabi-g++") + nm = shutil.which("arm-none-eabi-nm") + if not compiler or not nm: + self.skipTest("ARM toolchain required") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for readonly in (False, True): + text = "#include \n" + for name in ("vl53l0x_tuning.h", "vl53l0x_interrupt_threshold_settings.h"): + table = (FIXTURE / name).read_text().replace('#include "vl53l0x_def.h"', '') + if readonly: + table = table.replace(*TRIM.REPLACEMENTS[name]) + text += table + "\n" + text += ''' +extern "C" const uint8_t* default_table() { return DefaultTuningSettings; } +extern "C" const uint8_t* threshold_table() { return InterruptThresholdSettings; } +''' + path, obj = root / "tables.cpp", root / "tables.o" + path.write_text(text, encoding="ascii") + built = subprocess.run([compiler,"-std=c++17","-mcpu=cortex-m4","-mthumb","-Os", + "-c",str(path),"-o",str(obj)],capture_output=True,text=True,timeout=30) + self.assertEqual(built.returncode,0,built.stdout+built.stderr) + symbols = subprocess.check_output([nm,"-S","-C",str(obj)],text=True) + for name,size in (("DefaultTuningSettings",243),("InterruptThresholdSettings",438)): + line = next(line for line in symbols.splitlines() if line.endswith(" "+name)) + _, length, kind, _ = line.split() + self.assertEqual(int(length,16),size) + self.assertEqual(kind.lower(),"r" if readonly else "d") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_ota/test_ota_core.cpp b/test/test_ota/test_ota_core.cpp index 390a4b657..1d7cb28ed 100644 --- a/test/test_ota/test_ota_core.cpp +++ b/test/test_ota/test_ota_core.cpp @@ -4319,6 +4319,49 @@ TEST(OtaTransfer, RejectsShortNonFinalManifestFragment) { EXPECT_EQ(client.fetchState(), OtaManager::FETCHING); } +TEST(OtaTransfer, FixedManifestStoragePreservesEveryValidShapeAndRejectsOtherShapes) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA, SIM_MOTA_LEN, manifest)); + const uint8_t totals[] = {0,1,2,3,4,5,16,255}; + const uint8_t indexes[] = {0,1,2,3,4,15,254,255}; + for (uint8_t total : totals) for (uint8_t index : indexes) { + for (unsigned length = 0; length <= MAX_PACKET_PAYLOAD - 7; ++length) { + SCOPED_TRACE(::testing::Message() << "total=" << unsigned(total) + << " index=" << unsigned(index) << " length=" << length); + g_q.clear(); + OtaManager client; + OtaStoreRam<4096> store; + SendTo to_none{&client}; + client.begin(SIM_TARGET_ID, sim_send, &to_none); + client.set_fetch_store(&store); + ASSERT_EQ(client.pull(manifest.merkle_root, manifest.target_id), OtaManager::PULL_STARTED); + g_q.clear(); + const uint16_t final_length = MOTA_MFL - OTA_MF_FRAG; + const uint8_t other = index == 0 ? 1 : 0; + deliver_manifest_fragment(client, manifest.merkle_root, other, + manifest.manifest_start + (other ? OTA_MF_FRAG : 0), + other ? final_length : OTA_MF_FRAG); + ASSERT_EQ(client.fetchState(), OtaManager::WANT_MANIFEST); + uint8_t bytes[MAX_PACKET_PAYLOAD] = {}; + if (index == 0) memcpy(bytes, manifest.manifest_start, OTA_MF_FRAG); + else if (index == 1) memcpy(bytes, manifest.manifest_start + OTA_MF_FRAG, final_length); + ManifestMsg fragment{}; + memcpy(fragment.manifest_id, manifest.merkle_root, 4); + fragment.frag_idx = index; fragment.frag_total = total; + fragment.bytes = bytes; fragment.len = uint16_t(length); + uint8_t wire[MAX_PACKET_PAYLOAD]; + const auto wire_length = encode_manifest(wire, sizeof(wire), fragment); + ASSERT_GT(wire_length, 0); + client.on_message(wire, wire_length); + const bool valid = total == 2 && ((index == 0 && length == OTA_MF_FRAG) + || (index == 1 && length == final_length)); + EXPECT_EQ(client.fetchState(), valid ? OtaManager::FETCHING : OtaManager::WANT_MANIFEST); + if (!valid) EXPECT_EQ(store.staged_size(), 0u); + } + } + g_q.clear(); +} + TEST(OtaTransfer, RejectsManifestBlockExponentBeforeShift) { g_q.clear(); MotaManifest manifest; diff --git a/test/test_ota_folder_ready.py b/test/test_ota_folder_ready.py index eaac88266..f36f63409 100644 --- a/test/test_ota_folder_ready.py +++ b/test/test_ota_folder_ready.py @@ -21,6 +21,9 @@ class OtaFolderReadyTest(unittest.TestCase): self.assertIsNotNone(match) defines.append("#define " + name + " " + match[1]) source = (ROOT / "test/fixtures/ota_folder_ready.cpp").read_text(encoding="ascii") + # This folder-readiness fixture has no competing raw USB host session. + # Keep the real Arduino declaration visible to MotaSourceSerial.cpp. + source += "\nnamespace mesh { bool isUsbBinaryStreamOwned() { return false; } }\n" source = source.replace("// @PRODUCTION_DEFINES@", "\n".join(defines)) ready = extract_braced(production, "if (ota::ota_ctx().manager.takeSourceReadyAnnouncement())") source = source.replace("// @PRODUCTION_READY@", "" if missing_ready_trigger else ready) @@ -38,7 +41,8 @@ class OtaFolderReadyTest(unittest.TestCase): with self.subTest(definitions=definitions): sources = ("OtaManager.cpp", "OtaProtocol.cpp", "MotaContainer.cpp", "MerkleTree.cpp", "MotaSourceSerial.cpp") result = subprocess.run([compiler, "-std=c++17", "-O1", *sanitizers, - "-DENABLE_OTA=1", "-DOTA_SEEDER_ONLY=1", *["-D" + value for value in definitions], + "-DARDUINO=10819", "-DENABLE_OTA=1", "-DOTA_SEEDER_ONLY=1", + *["-D" + value for value in definitions], "-I", str(ROOT / "src"), "-I", str(ROOT / "test/mocks"), str(cpp), *[str(ROOT / "src/helpers/ota" / name) for name in sources], str(ROOT / "src/Utils.cpp"), "-o", str(executable)], capture_output=True, text=True, timeout=90) diff --git a/test/test_ota_memory_layout.py b/test/test_ota_memory_layout.py index b05c12895..f0643d3b3 100644 --- a/test/test_ota_memory_layout.py +++ b/test/test_ota_memory_layout.py @@ -21,25 +21,55 @@ class OtaMemoryLayoutTest(unittest.TestCase): #include #include using namespace mesh::ota; -static_assert(sizeof(OtaManager::ServeEntry) == (sizeof(void*) == 4 ? 60u : 64u), - "source entries must retain every field without old padding"); +static_assert(sizeof(OtaManager::ServeEntry) == (sizeof(void*) == 4 ? 32u : 40u), + "source entries must retain all independent descriptor fields once"); static_assert(alignof(OtaManager::ServeEntry) >= alignof(MotaSource*), "unaligned source pointer"); -static_assert(offsetof(OtaManager::ServeEntry, desc) % alignof(MotaDesc) == 0, "unaligned descriptor"); static_assert(offsetof(OtaManager::ServeEntry, target_id) % alignof(uint32_t) == 0, "unaligned word"); static_assert(sizeof(OtaStoreQspiNrf52) > 8192u, "both staging page buffers must remain"); int main() { OtaManager::ServeEntry row; - assert(row.desc.target_id == 0 && row.desc.fw_version == 0 && row.desc.source_caps == 0); - assert(row.desc.mid[0] == 0 && row.desc.mid[3] == 0 && row.desc.total_size == 0); + const auto empty = row.descriptor(); + assert(empty.target_id == 0 && empty.fw_version == 0 && empty.source_caps == 0); + assert(empty.mid[0] == 0 && empty.mid[3] == 0 && empty.total_size == 0); row.mid[0] = 17; row.target_id = 123; row.fw_version = 456; row.have_count = 789; row.src = nullptr; row.src_idx = 255; row.codec_id = 2; row.flags = 3; row.is_self = true; - row.desc.target_id = 987; row.desc.payload_size = 654; + row.payload_size = 654; row.block_size_log2 = 11; row.source_caps = 3; const OtaManager::ServeEntry copy = row; assert(copy.mid[0] == 17 && copy.target_id == 123 && copy.fw_version == 456); assert(copy.have_count == 789 && copy.src == nullptr && copy.src_idx == 255); assert(copy.codec_id == 2 && copy.flags == 3 && copy.is_self); - assert(copy.desc.target_id == 987 && copy.desc.payload_size == 654); + const auto descriptor = copy.descriptor(); + assert(descriptor.target_id == 123 && descriptor.payload_size == 654); + assert(descriptor.block_count == 789 && descriptor.block_size_log2 == 11 && descriptor.source_caps == 3); + assert(descriptor.leaves_off == 8 + MOTA_MFL); + assert(descriptor.payload_off == descriptor.leaves_off + 789 * 4); + assert(descriptor.total_size == descriptor.payload_off + 654 + 5); + // Every descriptor admitted by refresh_sources() has this canonical layout. + // Keep arbitrary metadata/caps/large payload values, not just current images. + uint32_t random = 0x6d76a345; + for (unsigned trial = 0; trial != 10000; ++trial) { + auto next = [&]() { random ^= random << 13; random ^= random >> 17; random ^= random << 5; return random; }; + MotaDesc original; + for (auto& byte : original.mid) byte = uint8_t(next()); + original.target_id = next(); original.fw_version = next(); + original.codec_id = uint8_t(next()); original.flags = uint8_t(next()); + original.block_size_log2 = uint8_t(next()); original.source_caps = uint8_t(next()); + original.block_count = 1 + next() % (UINT32_MAX / 8); + original.leaves_off = 8 + MOTA_MFL; + original.payload_off = original.leaves_off + original.block_count * 4; + original.payload_size = next() % (UINT32_MAX - original.payload_off - 5); + original.total_size = original.payload_off + original.payload_size + 5; + row.cacheDescriptor(original); + const auto roundtrip = row.descriptor(); + assert(memcmp(original.mid, roundtrip.mid, 4) == 0); + assert(original.target_id == roundtrip.target_id && original.fw_version == roundtrip.fw_version); + assert(original.codec_id == roundtrip.codec_id && original.flags == roundtrip.flags); + assert(original.block_size_log2 == roundtrip.block_size_log2 && original.source_caps == roundtrip.source_caps); + assert(original.block_count == roundtrip.block_count && original.payload_size == roundtrip.payload_size); + assert(original.leaves_off == roundtrip.leaves_off && original.payload_off == roundtrip.payload_off); + assert(original.total_size == roundtrip.total_size); + } } ''' with tempfile.TemporaryDirectory(prefix="ota-layout-") as directory: diff --git a/test/test_ota_source_proofs.py b/test/test_ota_source_proofs.py index 42cca597f..526fc5012 100644 --- a/test/test_ota_source_proofs.py +++ b/test/test_ota_source_proofs.py @@ -73,6 +73,8 @@ static_assert(OTA_PROOFGEN_SCRATCH / 4 == 4095, "odd byte limit lost usable leaf #include #include using namespace mesh::ota; +// This serial-source capacity test has no competing USB stream owner. +namespace mesh { bool isUsbBinaryStreamOwned() { return false; } } @FAKE@ class MtuSource : public SerialMotaSource { public: @@ -110,7 +112,7 @@ int main() { built = subprocess.run([ os.environ.get("CXX", "g++"), "-std=c++17", "-fsanitize=address,undefined", "-fno-sanitize-recover=all", "-fno-pie", "-no-pie", - "-DESP32_PLATFORM=1", "-DCOMPANION_RADIO_FULL=1", + "-DARDUINO=10819", "-DESP32_PLATFORM=1", "-DCOMPANION_RADIO_FULL=1", "-I", str(ROOT / "src"), "-I", str(ROOT / "test/mocks"), str(source), str(ROOT / "src/helpers/ota/MotaSourceSerial.cpp"), "-o", str(binary), ], capture_output=True, text=True, timeout=60) diff --git a/test/test_ota_target_names.py b/test/test_ota_target_names.py index 4405e8bf0..3e373ec2a 100644 --- a/test/test_ota_target_names.py +++ b/test/test_ota_target_names.py @@ -1,6 +1,8 @@ """All OTA names survive front coding; original Companion API stays unchanged.""" import re import os +import configparser +import importlib.util import shutil import subprocess import sys @@ -237,6 +239,29 @@ class OtaTargetNameTest(unittest.TestCase): f'-DOTA_LOCAL_TARGET_NAME="{local_name}"', ], ids), [local_name, "?", "?"]) + def test_reduced_rak_policy_compiles_with_exact_local_names_and_ids(self): + spec = importlib.util.spec_from_file_location("ota_name_trim", ROOT / "scripts/nrf52_flash_trim.py") + trim = importlib.util.module_from_spec(spec) + spec.loader.exec_module(trim) + for board, suffix in (("3401", ""), ("4631", ""), + ("4631", "_bridge_rs232_serial1"), ("4631", "_bridge_rs232_serial2")): + target = f"RAK_{board}_repeater{suffix}_lora_ota_no_external_sensors" + config = configparser.ConfigParser(interpolation=None, inline_comment_prefixes=(";",)) + config.read(ROOT / f"variants/rak{board}/platformio.ini") + flags = config["env:" + target]["build_flags"] + # Interpolated hardware flags do not influence the name policy. + flags = re.sub(r"\$\{[^}]+\}", "", flags) + target_id = gen_targets.ml.target_id_for_env(target) + definitions = trim.flag_defines(flags) + definitions.update(trim.trim_policy("nordicnrf52", "nrf52840", target, + "+<../examples/simple_repeater>", flags)) + name_flags = [f"-D{name}={value}" for name, value in definitions.items() + if name in ("OTA_TARGET_NAME_TABLE", "OTA_TARGET_NAME_FRONT_CODED", + "OTA_LOCAL_TARGET_ID", "OTA_LOCAL_TARGET_NAME")] + with self.subTest(target=target): + self.assertEqual(self.compile_and_lookup(name_flags, + [target_id, self.rows[0][0], 0]), [target, "?", "?"]) + def test_cpp_decoder_fails_closed_on_invalid_records_and_trailing_bytes(self): from zopfli_compress import raw_deflate_compress rows = gen_targets.target_rows(["hello"]) diff --git a/test/test_radio_profile_cli.py b/test/test_radio_profile_cli.py index 01d6d49b6..4166d7512 100644 --- a/test/test_radio_profile_cli.py +++ b/test/test_radio_profile_cli.py @@ -9,7 +9,7 @@ import unittest ROOT = Path(__file__).resolve().parents[1] class RadioProfileCLITest(unittest.TestCase): - def test_full_companion_keeps_rf_presets_without_unused_region_code(self): + def test_all_companions_keep_rf_presets_without_unused_region_code(self): compiler = shutil.which('g++') or shutil.which('clang++') self.assertIsNotNone(compiler) code = r''' @@ -23,7 +23,7 @@ int main() { f.cmd("set radio2 mediamesh"); f.advance(2000); assert(f.radio.p.enabled() && fabsf(f.radio.p.secondary.params.freq - 911.150f) < 0.0001f); -#if defined(COMPANION_RADIO_FULL) +#if defined(COMPANION_RADIO_FULL) || MESH_ROLE_COMPANION assert(!scope.commits && !scope.checks); #else assert(scope.enabled && scope.durable && scope.commits); @@ -37,9 +37,11 @@ int main() { with tempfile.TemporaryDirectory(prefix='profile-region-role-') as work: source = Path(work)/'role.cpp' source.write_text(code, encoding='ascii') - for full in (False, True): - binary = Path(work)/('companion' if full else 'infrastructure') - flags = ['-DCOMPANION_RADIO_FULL=1'] if full else [] + roles = {'infrastructure': [], 'full': ['-DCOMPANION_RADIO_FULL=1'], + 'ordinary': ['-DMESH_ROLE_COMPANION=1']} + for role, flags in roles.items(): + companion = role != 'infrastructure' + binary = Path(work)/role compiled = subprocess.run([ compiler, '-std=c++17', '-Os', '-ffunction-sections', '-fdata-sections', '-Wl,--gc-sections', @@ -53,11 +55,12 @@ int main() { self.assertEqual(compiled.returncode, 0, compiled.stderr) run = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10) self.assertEqual(run.returncode, 0, run.stdout+run.stderr) - sizes[full] = int(run.stdout) + sizes[role] = int(run.stdout) image = binary.read_bytes() for marker in (b'MediaMesh scope save pending', b'MediaMesh region unavailable or full'): - self.assertEqual(marker in image, not full) - self.assertLess(sizes[True], sizes[False]) + self.assertEqual(marker in image, not companion) + for role in ('full', 'ordinary'): + self.assertLess(sizes[role], sizes['infrastructure']) def test_commands_persistence_and_timers(self): compiler = shutil.which('g++') or shutil.which('clang++') diff --git a/test/test_rp2040_application_capabilities.py b/test/test_rp2040_application_capabilities.py index 7eb16fb48..50218305f 100644 --- a/test/test_rp2040_application_capabilities.py +++ b/test/test_rp2040_application_capabilities.py @@ -171,6 +171,7 @@ class RP2040ApplicationCapabilityTests(unittest.TestCase): set -euo pipefail source build.sh pio() { echo 'PlatformIO must not run' >&2; return 99; } +PIO_CONFIG_JSON='[]' PIO_BUILD_DIR_OVERRIDE=@BUILD@ OUTPUT_DIR=@OUTPUT@ BUILD_PROFILE_FOR_TARGET=auto diff --git a/test/test_shared_mota_queue.py b/test/test_shared_mota_queue.py index 9be466d99..3f8a0188b 100644 --- a/test/test_shared_mota_queue.py +++ b/test/test_shared_mota_queue.py @@ -17,7 +17,10 @@ class SharedMotaQueueTest(unittest.TestCase): def test_wireless_paper_wifi_listener_and_transfers_share_only_when_needed(self): self.run_transfer_test("ESP32_PLATFORM") - def run_transfer_test(self, platform): + def test_c3_rtc_retained_queue_preserves_real_ota_and_wifi_transfers(self): + self.run_transfer_test("ESP32_PLATFORM", rtc=True) + + def run_transfer_test(self, platform, rtc=False): with tempfile.TemporaryDirectory(prefix="meshcore-mota-queue-") as temp: binary = Path(temp) / "transfer" tinf = Path(temp) / "tinf.o" @@ -44,6 +47,30 @@ class SharedMotaQueueTest(unittest.TestCase): sources += ["src/helpers/esp32/WiFiOtaSeeder.cpp", "src/helpers/ota/MotaSourceSerial.cpp", "src/helpers/ota/FolderMotaStore.cpp"] + if rtc: + fixture = ROOT / "test/fixtures/shared_mota_queue" + source = (fixture / "test_shared_mota_queue.cpp").read_text() + source = source[:source.index(" Queue full;")] + "}\n" + source = source.replace("256", "224") + source = source.replace("struct Queue {", ( + "#include \n" + "alignas(4) RTC_FAST_ATTR Frame rtc_frames[32];\nstruct Queue {")) + source = source.replace( + "mesh::BorrowableFrameBuffer buffer;", + "mesh::BorrowableFrameBuffer> buffer;\n" + " Queue() : buffer(rtc_frames, nullptr) {}") + (Path(temp) / "test_shared_rtc.cpp").write_text(source) + (Path(temp) / "test_wifi_shared_queue.h").write_text( + (fixture / "test_wifi_shared_queue.h").read_text().replace("256", "224")) + (Path(temp) / "sdkconfig.h").write_text("#pragma once\n") + (Path(temp) / "esp_attr.h").write_text( + '#pragma once\n#define RTC_FAST_ATTR __attribute__((section(".rtc.force_fast.test")))\n') + sources[0] = str(Path(temp) / "test_shared_rtc.cpp") + flags += ["-DCOMPANION_RTC_FAST_OFFLINE_FRAMES=32", + "-DCONFIG_IDF_TARGET_ESP32C3=1", + "-DCONFIG_ESP_SYSTEM_ALLOW_RTC_FAST_MEM_AS_HEAP=1", + "-I", str(fixture)] built = subprocess.run([ "c++", "-std=c++17", "-fsanitize=address,undefined", "-g", *flags, "-DOTA_SEEDER_ONLY=1", diff --git a/test/test_small_table_sort.py b/test/test_small_table_sort.py index de868fb17..91b73f58c 100644 --- a/test/test_small_table_sort.py +++ b/test/test_small_table_sort.py @@ -8,6 +8,8 @@ import sys import tempfile import unittest +from cpp_source import body + ROOT = Path(__file__).resolve().parents[1] HARNESS = r''' @@ -140,8 +142,125 @@ void sortElsewhere(void* items, size_t count, size_t width, } ''' +NEIGHBOUR_HARNESS = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +struct NeighbourInfo { + uint32_t heard_timestamp; + int8_t snr; + uint16_t identity; +}; +@COMPARATORS@ +static void rpcSort(NeighbourInfo** sorted_neighbours, int16_t neighbours_count, + uint8_t order_by) { + @RPC_SORT@ +} +static void cliSort(NeighbourInfo** sorted_neighbours, int16_t neighbours_count) { + @CLI_SORT@ +} +static bool before(const NeighbourInfo* a, const NeighbourInfo* b, unsigned order) { + if (order == 0) return a->heard_timestamp > b->heard_timestamp; + if (order == 1) return a->heard_timestamp < b->heard_timestamp; + if (order == 2) return a->snr > b->snr; + return a->snr < b->snr; +} +int main() { + std::mt19937 random(0x18180002); + const uint32_t timestamps[] = {1, 42, 42, UINT32_MAX, 1, UINT32_MAX - 1}; + const int8_t snrs[] = {INT8_MIN, -16, -1, 0, 0, 7, 7, INT8_MAX}; + int (*const compare[])(const void*, const void*) = { + cmp_neighbours_newest_to_oldest, cmp_neighbours_oldest_to_newest, + cmp_neighbours_strongest_to_weakest, cmp_neighbours_weakest_to_strongest + }; + for (size_t count : {0U, 1U, 2U, 16U, 50U, 100U}) { + std::vector records(count); + for (size_t i = 0; i < count; ++i) { + records[i].heard_timestamp = timestamps[i % 6]; + records[i].snr = snrs[i % 8]; + records[i].identity = static_cast(i); + } + const auto original = records; + for (unsigned order = 0; order < 4; ++order) { + std::vector actual(count), expected(count); + for (size_t i = 0; i < count; ++i) actual[i] = &records[i]; + std::shuffle(actual.begin(), actual.end(), random); + expected = actual; + std::sort(expected.begin(), expected.end(), [order](const NeighbourInfo* a, + const NeighbourInfo* b) { + return before(a, b, order); + }); + rpcSort(actual.data(), static_cast(count), order); + std::vector seen(count); + for (size_t i = 0; i < count; ++i) { + const auto index = actual[i]->identity; + assert(index < count && !seen[index] && actual[i] == &records[index]); + seen[index] = true; + assert(!before(actual[i], expected[i], order)); + assert(!before(expected[i], actual[i], order)); + assert(compare[order](&actual[i], &expected[i]) == 0); + if (i) assert(!before(actual[i], actual[i - 1], order)); + } + // Unsupported order values retain the caller's original pointer order. + const auto unchanged = actual; + rpcSort(actual.data(), static_cast(count), 4); + assert(actual == unchanged); + cliSort(actual.data(), static_cast(count)); + for (size_t i = 1; i < count; ++i) { + assert(actual[i - 1]->heard_timestamp >= actual[i]->heard_timestamp); + } + for (size_t i = 0; i < count; ++i) { + assert(records[i].heard_timestamp == original[i].heard_timestamp); + assert(records[i].snr == original[i].snr); + assert(records[i].identity == original[i].identity); + } + } + } + puts("PASS: real neighbour RPC orders and CLI, duplicate keys, negative SNR and pointer records"); +} +''' + class SmallTableSortTests(unittest.TestCase): + def test_real_neighbour_rpc_and_cli_orderings(self): + compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++") + self.assertIsNotNone(compiler, "a host C++17 compiler is required") + production = (ROOT / "examples/simple_repeater/MyMesh.cpp").read_text() + names = ("newest_to_oldest", "oldest_to_newest", + "strongest_to_weakest", "weakest_to_strongest") + comparators = "\n".join(body(production, "static int cmp_neighbours_" + name + "(") + for name in names) + start = production.index("if (order_by == 0) {") + rpc_sort = production[start:production.index("// build results buffer", start)] + formatter = body(production, "void MyMesh::formatNeighborsReply(") + start = formatter.index("// sort neighbours newest to oldest") + cli_sort = formatter[start:formatter.index("for (int i = 0;", start)] + source = NEIGHBOUR_HARNESS.replace("@COMPARATORS@", comparators) + source = source.replace("@RPC_SORT@", rpc_sort).replace("@CLI_SORT@", cli_sort) + for platform in ("STM32_PLATFORM", "NRF52_PLATFORM", "ESP32_PLATFORM", "RP2040_PLATFORM"): + with self.subTest(platform=platform), tempfile.TemporaryDirectory( + prefix="neighbour-sort-") as directory: + work = Path(directory) + (work / "test.cpp").write_text(source) + binary = work / "sort" + command = [compiler, "-std=c++17", "-O2", "-Wall", "-Wextra", "-Werror", + "-D" + platform + "=1", "-I" + str(ROOT / "src"), + str(work / "test.cpp"), "-o", str(binary)] + if sys.platform.startswith("linux"): + command[1:1] = ["-fsanitize=address,undefined", "-fno-sanitize-recover=all", + "-fno-pie", "-no-pie"] + built = subprocess.run(command, capture_output=True, text=True, timeout=60) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + tested = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10) + self.assertEqual(tested.returncode, 0, tested.stdout + tested.stderr) + self.assertIn("negative SNR and pointer records", tested.stdout) + def test_sort_on_all_platforms(self): compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++") self.assertIsNotNone(compiler, "a host C++17 compiler is required") diff --git a/test/test_t096_message_footer_profile.py b/test/test_t096_message_footer_profile.py index af2ce85e0..43640586b 100644 --- a/test/test_t096_message_footer_profile.py +++ b/test/test_t096_message_footer_profile.py @@ -54,6 +54,7 @@ class T096MessageFooterProfileTest(unittest.TestCase): for display_class in ( "SH1106Display", "SSD1306Display", + "HeltecV4Display", "ST7735Display", "U8g2Display", ): @@ -103,6 +104,7 @@ class T096MessageFooterProfileTest(unittest.TestCase): for display_class in ( "SH1106Display", "SSD1306Display", + "HeltecV4Display", "ST7735Display", "U8g2Display", ): @@ -127,7 +129,7 @@ class T096MessageFooterProfileTest(unittest.TestCase): self.assertNotIn("MESHCORE_HAS_REAL_DISPLAY", null_macros) self.assertNotIn("MESHCORE_HAS_SMALL_DISPLAY", null_macros) - def test_v4_oled_is_compact_but_v4_tft_is_not(self): + def test_v4_display_aliases_share_the_unified_compact_driver(self): profile = V4_PROFILE.read_text(encoding="utf-8") usb = profile.split( "[env:heltec_v4_companion_radio_usb_femon]", 1 @@ -136,9 +138,9 @@ class T096MessageFooterProfileTest(unittest.TestCase): "[env:heltec_v4_tft_companion_radio_usb_femon]", 1 )[1].split("[", 1)[0] self.assertIn("extends = heltec_v4_oled", usb) - self.assertIn("DISPLAY_CLASS=SSD1306Display", usb) + self.assertIn("DISPLAY_CLASS=HeltecV4Display", usb) self.assertIn("extends = heltec_v4_tft", tft_usb) - self.assertIn("DISPLAY_CLASS=ST7789LCDDisplay", tft_usb) + self.assertIn("DISPLAY_CLASS=HeltecV4Display", tft_usb) if __name__ == "__main__": diff --git a/test/test_usb_logging_hwcdc_watchdog.py b/test/test_usb_logging_hwcdc_watchdog.py index 03cea3705..7fbf1414f 100644 --- a/test/test_usb_logging_hwcdc_watchdog.py +++ b/test/test_usb_logging_hwcdc_watchdog.py @@ -43,13 +43,24 @@ class HwcdcLoggingWatchdogTests(unittest.TestCase): cls.companion_methods = "\n".join(body(companion, signature) for signature in ( "static void clearUsbTerminalLine()", "static void cancelUsbSerialOperations()", "static void resetUsbTerminalHostSession()", "static void serviceUsbTerminalHostSessionReset()")) - cls.ordinary_methods = "" + # There is no functional text backlog in the peripheral fake. Keep + # the actual role calls and subsequent session-completion gate below. + cls.ordinary_methods = "namespace mesh { void serviceUsbTerminalPort() {} }\n" for role in ("simple_repeater", "simple_room_server", "simple_sensor"): text = (ROOT / "examples" / role / "main.cpp").read_text() - start = text.index(" if (mesh::takeUsbTerminalSessionReset())") + # The reset body now contains nested feature guards. Extract the + # complete compound statement, preserving every guarded operation. + reset = body(text, " if (mesh::takeUsbTerminalSessionReset())") + start = text.index(" if (usb_ready) mesh::serviceUsbTerminalPort();") stop = text.index("\n#endif", start) + completion = text[start:stop] + # This separate block follows optional KISS admission in the real + # loop. Do not silently truncate it if nested guards are added. + assert "#if" not in completion + assert "mesh::tryCompleteUsbTerminalSessionReset()" in completion cls.ordinary_methods += ("static void service_" + role + "() {\n" - " bool usb_ready = true;\n" + text[start:stop] + "\n (void)usb_ready;\n}\n") + " bool usb_ready = true;\n" + reset + "\n" + completion + + "\n (void)usb_ready;\n}\n") cls.binaries = {(backport, ota): cls.compile_backend(backport, ota) for backport in (0, 1) for ota in (False, True)} diff --git a/test/test_usb_logging_watchdog_roles.py b/test/test_usb_logging_watchdog_roles.py index 8b29ead50..84f3d5d4a 100644 --- a/test/test_usb_logging_watchdog_roles.py +++ b/test/test_usb_logging_watchdog_roles.py @@ -30,6 +30,10 @@ namespace mesh { struct Status {bool reader_connected=true, stalled=false;}; static Status status; Status usbLoggingStatus(){return status;} + namespace hostkiss { + static bool usb_active=false; + bool usbActive(){return usb_active;} + } } struct Store { bool dirty=false; @@ -91,6 +95,7 @@ bool isUsbTerminalDataConnected(){return usb_connected;} void reset(){ board=Board{};radio_driver=RadioDriver{};mesh::status=mesh::Status{}; + mesh::hostkiss::usb_active=false; the_mesh=RoleMesh{}; // Assignment copies the fixture's dependency pointers; point them back at // the live fixture objects, not the temporary used to reset scalar state. @@ -243,6 +248,13 @@ class UsbLoggingWatchdogRoleTests(unittest.TestCase): the_mesh._serial=&the_mesh.serial; ''' companion_checks = r''' + // A host KISS session owns the same physical USB transport regardless of + // the logging reader, terminal feature or ordinary Companion USB mode. + for(unsigned state=0;state<3;++state){ + reset();mesh::status.reader_connected=state!=1;mesh::status.stalled=state==2; + mesh::hostkiss::usb_active=true;assert(!usbLoggingRecoverySafe(nullptr)); + mesh::hostkiss::usb_active=false;assert(usbLoggingRecoverySafe(nullptr)); + } #if defined(ENABLE_USB_INTERFACE) // A live binary client or buffered protocol transaction is always protected, // even if the dedicated logging CDC is disconnected or its output is stuck. diff --git a/variants/rak3401/platformio.ini b/variants/rak3401/platformio.ini index aed1e4878..1b87797fd 100644 --- a/variants/rak3401/platformio.ini +++ b/variants/rak3401/platformio.ini @@ -89,7 +89,7 @@ build_flags = -D ENABLE_OTA=1 -D OTA_FLASH_STORE=1 -D OTA_FOLDER_SERIAL - -D OTA_TARGET_NAME_TABLE=0 ; IDs remain authoritative; omit the 540-entry display-name table + -D OTA_TARGET_NAME_TABLE=0 ; IDs remain authoritative; omit the all-board display-name catalog -D OTA_LOCAL_TARGET_ID=0x2fa509c1 -D OTA_LOCAL_TARGET_NAME='"RAK_3401_repeater_lora_ota_no_external_sensors"' -D OTA_FETCH_PIPELINE=4 ; adaptive one-packet flights grow 1 -> 2 -> 3 -> 4 on a clean link diff --git a/variants/rak4631/platformio.ini b/variants/rak4631/platformio.ini index a43f0225d..bf6221781 100644 --- a/variants/rak4631/platformio.ini +++ b/variants/rak4631/platformio.ini @@ -213,6 +213,7 @@ extra_scripts = ${rak4631_repeater_reduced_ota_base.extra_scripts} build_flags = ${rak4631_repeater_reduced_ota_base.build_flags} ${nrf52_rak4631_unified_ota.build_flags} + -D MESH_OTA_LOCAL_TARGET_NAME_ONLY=1 ; keep this target's name without the all-board catalog build_src_filter = ${rak4631_repeater_reduced_ota_base.build_src_filter} lib_deps = ${rak4631_repeater_reduced_ota_base.lib_deps} @@ -302,6 +303,7 @@ extends = rak4631 build_flags = ${rak4631.build_flags} ${nrf52_reduced_sensors_keep_ina.build_flags} + -D MESH_OTA_LOCAL_TARGET_NAME_ONLY=1 -D DISPLAY_CLASS=SSD1306Display -D ADVERT_NAME='"RS232 Bridge"' -D ADVERT_LAT=0.0 @@ -354,6 +356,7 @@ extends = rak4631 build_flags = ${rak4631.build_flags} ${nrf52_reduced_sensors_keep_ina_gps.build_flags} + -D MESH_OTA_LOCAL_TARGET_NAME_ONLY=1 -D DISPLAY_CLASS=SSD1306Display -D ADVERT_NAME='"RS232 Bridge"' -D ADVERT_LAT=0.0