From d1a2fd74ba90c5cc97d9ff9ee4bb62961e560e2c Mon Sep 17 00:00:00 2001 From: mikecarper Date: Mon, 26 Oct 2026 23:48:23 -0700 Subject: [PATCH 1/6] Protect MQTT publishing headroom and credit replies before disconnect --- docs/WiFi.md | 22 ++- docs/research/firmware_memory_budget.md | 66 +++++--- docs/test-results/mqtt_tls_v4_regression.md | 48 ++++++ scripts/check_firmware_ram.py | 19 ++- src/helpers/MQTTConnectionAdmission.h | 35 ++++- src/helpers/bridges/MQTTBridge.cpp | 29 ++-- test/test_firmware_ram.py | 29 +++- test/test_mqtt_connection_admission.py | 162 ++++++++++++++++---- test/test_mqtt_handshake_scheduler.py | 20 +-- test/test_mqtt_network_timeout.py | 4 +- test/test_mqtt_wifi_network_power.py | 39 +++++ 11 files changed, 377 insertions(+), 96 deletions(-) diff --git a/docs/WiFi.md b/docs/WiFi.md index 007840204..659f512a6 100644 --- a/docs/WiFi.md +++ b/docs/WiFi.md @@ -94,6 +94,13 @@ normally use no more than two TLS/WSS brokers; some classic ESP32 targets are reliable with only one. Excess configured slots remain saved but show as inactive in `get mqtt.status`. +Five active brokers on a PSRAM board is an upper limit, not a qualified load +for every board. Adding a connection while another broker is connected must +leave contiguous memory for packet publishing as well as SDK startup +allocations. A saved slot can wait for memory while existing connections +continue forwarding. Recovery of the first broker uses a separate bootstrap +bound so this extra reservation cannot strand every disconnected slot. + The bridge has a bounded packet queue for broker or WiFi outages and reconnects automatically. A non-PSRAM build holds 6 packets and a PSRAM build holds 50. When the queue is full, the oldest item is replaced; after five minutes with no @@ -111,7 +118,8 @@ settings remain enabled. Failed requests, a WiFi association, and NTP replies alone do not restart the outage timer or keep a failed MQTT uplink awake. After 30 minutes, the firmware opens a recovery window lasting up to two -minutes. A broker acknowledgment or a connection that stays up for 30 seconds +minutes. A new broker acknowledgment, including one immediately followed by a +disconnect, or a connection that stays up for 30 seconds returns the node to normal operation. Otherwise WiFi is suspended for another 30 minutes. An explicit management request can ask for an earlier probe. Automatic NTP requests have a 15-second timeout; failed requests wait 30 minutes @@ -841,6 +849,7 @@ For an MQTT observer: get wifi.status get mqtt.enabled get mqtt.status +get mqtt.stats get mqtt1.diag get mqtt2.diag ``` @@ -850,6 +859,17 @@ configured slot's `pub_ack` count in `get mqttN.diag`. Accepted publishes are not proof of delivery: a QoS 1 publish is confirmed only when the broker acknowledges it. A quiet radio channel alone is not a failed MQTT connection. +Capture these outputs before rebooting, then repeat `mqtt.stats` and the slot +diagnostics after 60 seconds. `(ok)` reports connection state, not delivery. +In `mqtt.stats`, `drops=A/B` separates outbox-cap drops from packet publication +attempts skipped for memory pressure; a growing `B` can explain missing packet +reports while broker connections still look healthy. `DNS failed (0x8001)` +records a hostname lookup failure, which can follow an earlier disconnect; it +does not identify the reason for that first disconnect. `wifi.status` showing +`suspended` means the shared outage cooldown is active, not that WiFi settings +were erased. These diagnostics do not reveal passwords or tokens, but WiFi +status can include the SSID or local IP address. Redact them before posting. + The shared MQTT bridge recovers failed transports on its owning task, including errors without a subsequent disconnect callback and connection attempts that never finish. It also watches accepted QoS 1 publishes for missing broker diff --git a/docs/research/firmware_memory_budget.md b/docs/research/firmware_memory_budget.md index 5c63cc821..a2ec4343e 100644 --- a/docs/research/firmware_memory_budget.md +++ b/docs/research/firmware_memory_budget.md @@ -119,19 +119,34 @@ qualification of five simultaneous PSRAM connections or two internal TLS connections. The `mqtt` report also names the production runtime admission thresholds. -A cold SDK start requires at least 32 KiB of DMA-capable internal free heap -and an 8 KiB contiguous DMA block. This includes a separate 16 KiB allowance -for its SDK task stack/TCB, receive/transmit/reassembly buffers and transport -or configuration objects, alongside the 16 KiB AES reserve. Those ordinary -SDK allocations occur after admission and do not use the TLS allocation guard. -A retained initialized SDK handle with a stopped task requires 24 KiB free and -an 8 KiB contiguous DMA block. Its separate 8 KiB restart allowance covers the -pinned 6 KiB task stack plus TCB and allocator overhead. SDK stop retains its -receive/transmit buffers, configuration, transport objects and wrapper -reassembly buffer, so restarting that handle does not need to reserve their -first-initialization allocation again. A wrapper whose SDK initialization has -not completed still receives the full cold allowance. This distinction also -applies after WiFi loss stops the SDK tasks but keeps their handles. +Connection admission and the packet-publishing guard share a contiguous-block +floor: 16,000 bytes with detected PSRAM, or 8,000 bytes without it. Admitting +another broker while one is connected reserves its startup allocations beside +that floor, protecting existing packet forwarding. The first connection after +boot or an outage can bootstrap without that extra reservation, so the +additional-broker reserve cannot block every disconnected slot's recovery. + +A cold SDK start requires at least 32 KiB of DMA-capable internal free heap. +The separate 16 KiB allowance covers its SDK task stack/TCB, +receive/transmit/reassembly buffers and transport or configuration objects, +alongside the 16 KiB AES reserve. Those ordinary allocations occur after +admission and do not use the TLS allocation guard. Bootstrap needs a contiguous +16,000-byte block with PSRAM or 8,192 bytes without it. With another connected +broker, the contiguous minimum includes its publication floor plus the startup +allowance: 32,384 bytes with PSRAM or 24,384 without it, before the additional +internal TLS requirement below. + +A retained initialized SDK handle with a stopped task requires 24 KiB free. +Its separate 8 KiB restart allowance covers the pinned 6 KiB task stack plus +TCB and allocator overhead. Bootstrap uses the same 16,000/8,192-byte block +minima as a cold start. With another connected broker, its contiguous minimum +is 24,192 bytes with PSRAM or 16,192 without it, before the internal TLS +requirement below. SDK stop retains its receive/transmit buffers, +configuration, transport objects and wrapper reassembly buffer, so restarting +that handle does not need their first-initialization allocation again. A +wrapper whose SDK initialization has not completed still receives the full +cold allowance. This distinction also applies after WiFi loss stops SDK tasks +but keeps their handles. All ESP32 clients use 896-byte receive, transmit and wrapper reassembly buffers from first initialization. Applying configuration cannot grow the retained SDK @@ -143,15 +158,22 @@ and cold admission covers the ordinary allocations. WiFi recovery and explicit reconfiguration continue to retain clients. A warm reconnect keeps its running task and buffers; it requires at least -16 KiB free and a 4 KiB contiguous DMA block. The per-client SDK stack is -independent of `MQTT_TASK_STACK_SIZE`, which sizes the existing bridge worker. -A TLS attempt without PSRAM requires at least -60 KiB free and a 17,408-byte contiguous block. PSRAM allocation fallback must -leave the 16 KiB DMA reserve available. These thresholds describe free heap at -the time of admission; they are not additional linked capacity and the TLS -contiguous threshold is not a startup allocation requirement. Available linked -internal heap cannot establish how much DMA-capable heap remains or how it is -fragmented after boot. Only runtime observations can establish that. +16 KiB free and the same 16,000/8,000-byte publication block. The per-client SDK +stack is independent of `MQTT_TASK_STACK_SIZE`, which sizes the existing bridge +worker. TLS without detected PSRAM retains its 60 KiB free-heap and 17,408-byte +record-block floors for bootstrap. With another connected broker, admission +reserves that record beside the publication block and ordinary startup +allocations: 41,792 contiguous bytes for a cold start, 33,600 for a stopped +task, or 25,408 for a warm reconnect. Both admission and publication use +detected PSRAM, including a board whose compiled-in PSRAM support failed to +initialize. + +PSRAM allocation fallback must leave the 16 KiB DMA reserve available. These +conservative thresholds describe free heap at admission; they are not extra +linked capacity or a guarantee against later fragmentation or allocations by +other tasks. Available linked internal heap cannot establish how much +DMA-capable heap remains or how it is fragmented after boot. Only runtime +observations can establish that. Maximum-load qualification still needs physical boot and sustained reconnect, publish and configuration-change tests with the selected number of brokers, diff --git a/docs/test-results/mqtt_tls_v4_regression.md b/docs/test-results/mqtt_tls_v4_regression.md index 68ff349ad..0f9188010 100644 --- a/docs/test-results/mqtt_tls_v4_regression.md +++ b/docs/test-results/mqtt_tls_v4_regression.md @@ -265,3 +265,51 @@ measured with three live brokers or qualification of maximum simultaneous load. No new physical three-hour V4 soak or reporter-side retest has been performed as part of this repair. These tests establish the repaired recovery behavior, not the cause of the reporter's hardware failure or website-side availability. + +## Station G3 follow-up: early failures and network suspension + +A user supplied a failed-state console capture from `1.18.0.1` with five WSS +presets configured. It shows all five slots disconnected and WiFi suspended +with the 30-minute recovery schedule. The displayed uptime and first-disconnect +ages place the first two failures roughly 28 and 89 seconds after boot, before +the reported seven-to-eight-minute loss of reporting. + +The slot diagnostics record repeated completed transport recoveries and DNS +lookup failures (`0x8001`). This establishes active recovery followed by the +outage cooldown, rather than an indefinitely frozen worker in this capture. +The DNS errors are the latest failures, not proof of the original disconnect +cause. The stats also record 345 packet publication attempts skipped for +memory pressure (`drops=0/345`), an empty SDK outbox and a growing bridge queue. +These skips are attempts, including retries, rather than 345 distinct packets. +Heap readings after clients were stopped do not establish their earlier +connected-state headroom. + +Two source issues were identified: + +- Connection admission could permit another SDK client below the contiguous + memory threshold required by packet publishing. When another slot is + connected, the shared admission policy now protects that publishing + threshold and accounts for allocations made during SDK startup or restart. + It defers additional connections under pressure. First-broker recovery has + a separate bootstrap bound, retaining a way to reconnect when all slots + have disconnected. Neither path lowers the publishing safety threshold or + erases slots; admission bounds do not guarantee future allocations. +- A new PUBACK followed by DISCONNECTED between worker passes was consumed + without crediting the successful delivery to the outage timer. The worker + now credits that reply once, even if the connection has since dropped. + CONNACK-only flapping and old acknowledgment counts cannot renew the timer. + +These repairs do not establish why the user's DNS resolution failed. Physical +G3 reproduction and a reporter-side retest remain necessary before calling +that reported failure resolved. The shared changes apply to all ESP32 roles +that include this MQTT bridge. + +Validation passed: 85 MQTT Python host cases, four Companion MQTT host cases +and 57 firmware RAM policy cases. The new disconnected-PUBACK regression +fails with the old condition and passes with the fix, including timer wrap, +disabled slots and one-time acknowledgment consumption. Admission tests use +the captured 93,024-byte free DMA heap and 20,468-byte largest block: one +stopped broker can attempt recovery, while an additional broker waits when +another slot is connected. The Station G3 Full repeater compiled and passed +capability, dual-OTA partition and linked RAM checks. These are host/build +checks, not a physical five-broker G3 soak. diff --git a/scripts/check_firmware_ram.py b/scripts/check_firmware_ram.py index 092cb61fd..38e168a89 100644 --- a/scripts/check_firmware_ram.py +++ b/scripts/check_firmware_ram.py @@ -67,6 +67,7 @@ def mqtt_requirements(defines): # slots are lazy and each attempt must pass the production admission guard. startup = MQTT_STARTUP_ALLOWANCE + max(0, worker_stack - MQTT_WORKER_STACK_BYTES) active = min(slots, 5 if psram else 2) + publish_contiguous = 16000 if psram else 8000 potential = startup + max(0, active - 2) * (MQTT_CLIENT_STACK_BYTES + MQTT_CLIENT_BUFFER_BYTES) return { "budget_scope": "startup minimum with runtime connection admission", @@ -82,18 +83,28 @@ def mqtt_requirements(defines): "maximum_slot_stack_buffer_allowance_bytes": potential, "cold_start_allocation_allowance_bytes": 16384, "cold_start_dma_admission_free_bytes": 32768, - "cold_start_dma_admission_contiguous_bytes": 8192, + "cold_start_dma_admission_contiguous_bytes": max(publish_contiguous, 8192), + "additional_broker_cold_start_dma_admission_contiguous_bytes": publish_contiguous + 16384, "restart_allocation_allowance_bytes": 8192, "restart_dma_admission_free_bytes": 24576, - "restart_dma_admission_contiguous_bytes": 8192, + "restart_dma_admission_contiguous_bytes": max(publish_contiguous, 8192), + "additional_broker_restart_dma_admission_contiguous_bytes": publish_contiguous + 8192, "dma_admission_free_bytes": 16384, - "dma_admission_contiguous_bytes": 4096, + "dma_admission_contiguous_bytes": publish_contiguous, + "publish_contiguous_bytes": publish_contiguous, + "publish_contiguous_without_psram_bytes": 8000, + "publish_contiguous_with_psram_bytes": 16000, "tls_internal_admission_free_bytes": 61440, + "tls_record_allocation_bytes": 17408, + "additional_broker_cold_start_tls_internal_admission_contiguous_bytes": 8000 + 16384 + 17408, + "additional_broker_restart_tls_internal_admission_contiguous_bytes": 8000 + 8192 + 17408, "tls_internal_admission_contiguous_bytes": 17408, + "additional_broker_tls_internal_admission_contiguous_bytes": 8000 + 17408, "maximum_runtime_load_qualified": False, "physical_validation_performed": False, "limitations": "Maximum TLS/WSS/outbox workload is not covered by the startup allowance; " - "runtime free DMA heap and fragmentation must satisfy connection admission.", + "runtime free DMA heap and fragmentation must satisfy connection admission; " + "additional brokers reserve publication headroom beside startup allocations.", } diff --git a/src/helpers/MQTTConnectionAdmission.h b/src/helpers/MQTTConnectionAdmission.h index e69ed8d3f..590f21946 100644 --- a/src/helpers/MQTTConnectionAdmission.h +++ b/src/helpers/MQTTConnectionAdmission.h @@ -11,6 +11,11 @@ namespace MQTTConnectionAdmission { static const size_t kDmaReserveBytes = 16384; static const size_t kDmaLargestBytes = 4096; +// Keep the packet-publishing guard and connection admission on the same floor. +// A new optional broker must not consume the contiguous block that existing +// brokers need to continue forwarding packets. +static const size_t kPsramPublishLargestBytes = 16000; +static const size_t kInternalPublishLargestBytes = 8000; // A stopped SDK client needs a new internal task stack/TCB. First starts also // allocate receive/transmit/reassembly buffers and SDK transport/config objects // after admission. Budget that ordinary allocation separately from AES headroom. @@ -33,26 +38,46 @@ enum class ClientMemoryState : uint8_t { Started, }; +static inline size_t publishLargestBytes(bool psram) { + return psram ? kPsramPublishLargestBytes : kInternalPublishLargestBytes; +} + +static inline bool hasPublishHeadroom(bool psram, size_t largest) { + return largest >= publishLargestBytes(psram); +} + static inline bool canStart(bool psram, bool tls, ClientMemoryState state, - size_t dma_free, size_t dma_largest) { + size_t dma_free, size_t dma_largest, + bool protect_publish = false) { size_t allocation_allowance; - size_t largest_required; + size_t sdk_largest; switch (state) { case ClientMemoryState::Uninitialized: allocation_allowance = kColdStartAllowanceBytes; - largest_required = kColdStartLargestBytes; + sdk_largest = kColdStartLargestBytes; break; case ClientMemoryState::Stopped: allocation_allowance = kRestartAllowanceBytes; - largest_required = kRestartLargestBytes; + sdk_largest = kRestartLargestBytes; break; case ClientMemoryState::Started: allocation_allowance = 0; - largest_required = kDmaLargestBytes; + sdk_largest = kDmaLargestBytes; break; default: return false; } + const size_t publication_floor = publishLargestBytes(psram); + const size_t tls_record = tls && !psram ? kTlsRecordAllocationBytes : 0; + // Bootstrap/recover one broker without requiring spare capacity for another + // live connection: retained SDK buffers can fragment the heap after an outage. + // With a live peer, conservatively allow startup/TLS allocations to consume + // the largest block while leaving existing forwarding's publication floor. + // Other tasks and TLS allocations can still change/fragment this snapshot; + // these are admission bounds, not an allocation guarantee. + size_t largest_required = publication_floor > sdk_largest ? publication_floor : sdk_largest; + if (tls_record > largest_required) largest_required = tls_record; + if (protect_publish) largest_required = publication_floor + allocation_allowance + tls_record; const size_t free_required = kDmaReserveBytes + allocation_allowance; if (dma_free < free_required || dma_largest < largest_required) { return false; diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 7c8429c2f..e2ff244c7 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -1671,10 +1671,10 @@ bool MQTTBridge::serviceNetworkPower(uint32_t now) { && acknowledgments != _network_publish_acks[i]; _network_publish_acks[i] = acknowledgments; // A flapping CONNACK must not buy another full outage window forever. - // A PUBACK proves delivery immediately; a quiet connection must stay up - // for 30 seconds, leaving time to prove recovery within the probe window. - if (_slots[i].enabled && _slots[i].connected - && (broker_replied || (now - static_cast(_slots[i].connected_at_ms) + // A PUBACK proves delivery even if DISCONNECTED arrives before this pass. + // Consume it once; a quiet connection must still stay up for 30 seconds. + if (_slots[i].enabled && (broker_replied || (_slots[i].connected + && now - static_cast(_slots[i].connected_at_ms) >= mesh::wifi::kNetworkHealthyMs))) { useful_network = true; } @@ -2381,9 +2381,16 @@ bool MQTTBridge::canStartSlotConnection(int index) const { ? ClientMemoryState::Uninitialized : (slot.client->isStarted() ? ClientMemoryState::Started : ClientMemoryState::Stopped); const uint32_t caps = MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA; + bool protect_publish = false; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; ++i) { + if (i != index && _slots[i].enabled && _slots[i].connected) { + protect_publish = true; + break; + } + } return MQTTConnectionAdmission::canStart( ESP.getPsramSize() > 0, tls, memory_state, heap_caps_get_free_size(caps), - heap_caps_get_largest_free_block(caps)); + heap_caps_get_largest_free_block(caps), protect_publish); #else return true; #endif @@ -4503,11 +4510,6 @@ bool MQTTBridge::publishPacket(mesh::Packet* packet, bool is_tx, // at PUBLISH_JSON_BUFFER_SIZE (2 KB); the threshold includes frame overhead // and transient TCP buffers. #ifdef ESP32 - #if defined(BOARD_HAS_PSRAM) - static const size_t PUBLISH_SKIP_MAX_ALLOC_THRESHOLD = 16000; - #else - static const size_t PUBLISH_SKIP_MAX_ALLOC_THRESHOLD = 8000; - #endif unsigned long now = millis(); // Re-sample max-alloc at most once per interval and cache the verdict. // getMaxAllocHeap() walks the heap free-list, so it must not run per packet. @@ -4519,10 +4521,13 @@ bool MQTTBridge::publishPacket(mesh::Packet* packet, bool is_tx, if (now - _last_memory_check > 5000) { _last_memory_check = now; size_t max_alloc = ESP.getMaxAllocHeap(); - _memory_pressure = (max_alloc < PUBLISH_SKIP_MAX_ALLOC_THRESHOLD); + // Match connection admission's detected-PSRAM decision, including a board + // whose optional PSRAM failed initialization. + const bool psram = ESP.getPsramSize() > 0; + _memory_pressure = !MQTTConnectionAdmission::hasPublishHeadroom(psram, max_alloc); if (_memory_pressure) { MQTT_DEBUG_PRINTLN("MQTT: memory pressure, skipping publishes (Max alloc: %d, threshold: %d, skipped: %d)", - (int)max_alloc, (int)PUBLISH_SKIP_MAX_ALLOC_THRESHOLD, _skipped_publishes); + (int)max_alloc, (int)MQTTConnectionAdmission::publishLargestBytes(psram), _skipped_publishes); } } if (_memory_pressure) { diff --git a/test/test_firmware_ram.py b/test/test_firmware_ram.py index b856edaa0..e4988f5e6 100644 --- a/test/test_firmware_ram.py +++ b/test/test_firmware_ram.py @@ -607,10 +607,12 @@ int main() { self.assertEqual(profile["maximum_slot_stack_buffer_allowance_bytes"], potential) self.assertEqual(profile["cold_start_allocation_allowance_bytes"], 16384) self.assertEqual(profile["cold_start_dma_admission_free_bytes"], 32768) - self.assertEqual(profile["cold_start_dma_admission_contiguous_bytes"], 8192) + self.assertEqual(profile["cold_start_dma_admission_contiguous_bytes"], + 16000 if active == 5 else 8192) self.assertEqual(profile["restart_allocation_allowance_bytes"], 8192) self.assertEqual(profile["restart_dma_admission_free_bytes"], 24576) - self.assertEqual(profile["restart_dma_admission_contiguous_bytes"], 8192) + self.assertEqual(profile["restart_dma_admission_contiguous_bytes"], + 16000 if active == 5 else 8192) self.assertFalse(profile["maximum_runtime_load_qualified"]) self.assertFalse(profile["physical_validation_performed"]) ordinary = ram.requirements("ESP32_PLATFORM", {"BOARD_HAS_PSRAM": 1}, "v4_companion") @@ -707,16 +709,31 @@ int main() { ("client_task_stack_bytes", "kClientTaskStackBytes"), ("client_task_priority", "kClientTaskPriority"), ("cold_start_allocation_allowance_bytes", "kColdStartAllowanceBytes"), - ("cold_start_dma_admission_contiguous_bytes", "kColdStartLargestBytes"), ("restart_allocation_allowance_bytes", "kRestartAllowanceBytes"), - ("restart_dma_admission_contiguous_bytes", "kRestartLargestBytes"), ("dma_admission_free_bytes", "kDmaReserveBytes"), - ("dma_admission_contiguous_bytes", "kDmaLargestBytes"), + ("publish_contiguous_without_psram_bytes", "kInternalPublishLargestBytes"), + ("publish_contiguous_with_psram_bytes", "kPsramPublishLargestBytes"), ("tls_internal_admission_free_bytes", "kTlsInternalFreeBytes"), - ("tls_internal_admission_contiguous_bytes", "kTlsRecordAllocationBytes"), + ("tls_record_allocation_bytes", "kTlsRecordAllocationBytes"), ): with self.subTest(key=key): self.assertIn(f"{name} = {profile[key]};", source) + for psram in (False, True): + flags = {"WITH_MQTT_BRIDGE": 1} + if psram: + flags["BOARD_HAS_PSRAM"] = 1 + actual = ram.requirements("ESP32_PLATFORM", flags, "v4_companion")["mqtt"] + floor = 16000 if psram else 8000 + self.assertEqual(actual["publish_contiguous_bytes"], floor) + self.assertEqual(actual["dma_admission_contiguous_bytes"], floor) + self.assertEqual(actual["cold_start_dma_admission_contiguous_bytes"], max(floor, 8192)) + self.assertEqual(actual["additional_broker_cold_start_dma_admission_contiguous_bytes"], floor + 16384) + self.assertEqual(actual["restart_dma_admission_contiguous_bytes"], max(floor, 8192)) + self.assertEqual(actual["additional_broker_restart_dma_admission_contiguous_bytes"], floor + 8192) + self.assertEqual(actual["additional_broker_cold_start_tls_internal_admission_contiguous_bytes"], 41792) + self.assertEqual(actual["additional_broker_restart_tls_internal_admission_contiguous_bytes"], 33600) + self.assertEqual(actual["tls_internal_admission_contiguous_bytes"], 17408) + self.assertEqual(actual["additional_broker_tls_internal_admission_contiguous_bytes"], 25408) self.assertEqual(profile["cold_start_dma_admission_free_bytes"], profile["dma_admission_free_bytes"] + profile["cold_start_allocation_allowance_bytes"]) self.assertEqual(profile["restart_dma_admission_free_bytes"], diff --git a/test/test_mqtt_connection_admission.py b/test/test_mqtt_connection_admission.py index e47334027..5ae4da884 100644 --- a/test/test_mqtt_connection_admission.py +++ b/test/test_mqtt_connection_admission.py @@ -14,6 +14,7 @@ POLICY_PREAMBLE = r''' #include #include #include +#include #include "helpers/MQTTConnectionAdmission.h" #include "helpers/MQTTTransportHealth.h" namespace Admission = MQTTConnectionAdmission; @@ -62,6 +63,7 @@ struct Client { }; struct MQTTBridge { struct MQTTSlot { + bool enabled = true, connected = false; Client* client = nullptr; const Preset* preset = nullptr; char broker_uri[128] = {}, host[64] = "broker"; @@ -97,46 +99,112 @@ class MqttConnectionAdmissionTest(unittest.TestCase): self.compile_run(POLICY_PREAMBLE + r''' int main() { // A large aggregate heap cannot compensate for a fragmented DMA region. - assert(!Admission::canStart(true, true, MemoryState::Started, 100000, 4095)); - assert(!Admission::canStart(true, true, MemoryState::Started, 16383, 20000)); - assert(Admission::canStart(true, true, MemoryState::Started, 16384, 4096)); - assert(Admission::canStart(false, false, MemoryState::Started, 16384, 4096)); - assert(!Admission::canStart(false, false, MemoryState::Started, 16383, 4096)); - assert(!Admission::canStart(false, false, MemoryState::Started, 16384, 4095)); + assert(!Admission::canStart(true, true, MemoryState::Started, 100000, 15999, true)); + assert(!Admission::canStart(true, true, MemoryState::Started, 16383, 16000, true)); + assert(Admission::canStart(true, true, MemoryState::Started, 16384, 16000, true)); + assert(Admission::canStart(false, false, MemoryState::Started, 16384, 8000, true)); + assert(!Admission::canStart(false, false, MemoryState::Started, 16383, 8000, true)); + assert(!Admission::canStart(false, false, MemoryState::Started, 16384, 7999, true)); - // Without PSRAM, TLS also needs a full record allocation and session budget. + // Internal TLS must leave a publication block beside its new TLS record and + // ordinary SDK startup allocations, even for a retained running task. const MemoryState states[] = {MemoryState::Uninitialized, MemoryState::Stopped, MemoryState::Started}; - for (MemoryState state : states) { - assert(!Admission::canStart(false, true, state, 61439, 20000)); - assert(!Admission::canStart(false, true, state, 100000, 17407)); - assert(Admission::canStart(false, true, state, 61440, 17408)); + const size_t internal_tls_largest[] = {41792, 33600, 25408}; + for (size_t i = 0; i < 3; ++i) { + assert(!Admission::canStart(false, true, states[i], 61439, 100000, true)); + assert(!Admission::canStart(false, true, states[i], 100000, internal_tls_largest[i] - 1, true)); + assert(Admission::canStart(false, true, states[i], 61440, internal_tls_largest[i], true)); } - assert(Admission::canStart(true, false, MemoryState::Started, 16384, 4096)); - assert(!Admission::canStart(true, false, static_cast(255), SIZE_MAX, SIZE_MAX)); + assert(Admission::canStart(true, false, MemoryState::Started, 16384, 16000, true)); + assert(!Admission::canStart(true, false, static_cast(255), SIZE_MAX, SIZE_MAX, true)); - // A new SDK task needs its stack and ordinary startup allocations in - // addition to the DMA reserve, including plaintext and PSRAM-backed TLS. assert(Admission::kClientTaskStackBytes == 6144); assert(Admission::kClientTaskPriority == 5); assert(Admission::kColdStartAllowanceBytes == 16384); - assert(Admission::kColdStartLargestBytes == 8192); assert(Admission::kRestartAllowanceBytes == 8192); - assert(Admission::kRestartLargestBytes == 8192); for (int psram = 0; psram <= 1; ++psram) { + const size_t floor = psram ? 16000 : 8000; for (int tls = 0; tls <= 1; ++tls) { if (!psram && tls) continue; // Larger TLS thresholds checked above. - assert(!Admission::canStart(psram, tls, MemoryState::Uninitialized, 32767, 20000)); - assert(!Admission::canStart(psram, tls, MemoryState::Uninitialized, 100000, 8191)); - assert(Admission::canStart(psram, tls, MemoryState::Uninitialized, 32768, 8192)); - // An initialized handle retains its ordinary SDK buffers across a stop. - // Only the replacement task/TCB is charged beside the DMA reserve. - assert(!Admission::canStart(psram, tls, MemoryState::Stopped, 24575, 20000)); - assert(!Admission::canStart(psram, tls, MemoryState::Stopped, 100000, 8191)); - assert(Admission::canStart(psram, tls, MemoryState::Stopped, 24576, 8192)); + assert(!Admission::canStart(psram, tls, MemoryState::Uninitialized, 32767, 100000, true)); + assert(!Admission::canStart(psram, tls, MemoryState::Uninitialized, 100000, floor + 16383, true)); + assert(Admission::canStart(psram, tls, MemoryState::Uninitialized, 32768, floor + 16384, true)); + // An initialized handle retains its buffers, but recreates its task. + assert(!Admission::canStart(psram, tls, MemoryState::Stopped, 24575, 100000, true)); + assert(!Admission::canStart(psram, tls, MemoryState::Stopped, 100000, floor + 8191, true)); + assert(Admission::canStart(psram, tls, MemoryState::Stopped, 24576, floor + 8192, true)); } } } +''') + + def test_additional_broker_cannot_spend_live_packet_publishing_headroom(self): + self.compile_run(POLICY_PREAMBLE + r''' +int main() { + for (bool psram : {false, true}) { + const size_t floor = psram ? 16000 : 8000; + assert(!Admission::hasPublishHeadroom(psram, floor - 1)); + assert(Admission::hasPublishHeadroom(psram, floor)); + // Previously admitted: 8 KiB largest was enough to start another client, + // even though publishing on a PSRAM board was already suppressed there. + assert(!Admission::canStart(psram, false, MemoryState::Uninitialized, 100000, 8192, true)); + assert(!Admission::canStart(psram, true, MemoryState::Stopped, 100000, 8192, true)); + const size_t allowances[] = {16384, 8192, 0}; + const MemoryState states[] = {MemoryState::Uninitialized, MemoryState::Stopped, + MemoryState::Started}; + for (size_t i = 0; i < 3; ++i) { + const size_t minimum = floor + allowances[i]; + assert(!Admission::canStart(psram, false, states[i], 100000, minimum - 1, true)); + assert(Admission::canStart(psram, false, states[i], 100000, minimum, true)); + // Spending the complete bounded SDK startup allowance still leaves a + // block accepted by the exact production packet-publishing policy. + assert(Admission::hasPublishHeadroom(psram, minimum - allowances[i])); + } + } +} +''') + + def test_actual_packet_publish_guard_uses_detected_psram_and_throttles_sampling(self): + source = (ROOT / "src/helpers/bridges/MQTTBridge.cpp").read_text() + start = source.index(" #ifdef ESP32\n", source.index("bool MQTTBridge::publishPacket(")) + end = source.index(" #endif\n", start) + len(" #endif\n") + guard = source[start:end] + self.compile_run(POLICY_PREAMBLE + r''' +#define ESP32 1 +#define BOARD_HAS_PSRAM 1 +#define MQTT_DEBUG_PRINTLN(...) ((void)0) +static unsigned long now_ms = 6000; +unsigned long millis() { return now_ms; } +struct { + size_t psram = 2097152, largest = 16000, queries = 0; + size_t getPsramSize() const { return psram; } + size_t getMaxAllocHeap() { ++queries; return largest; } +} ESP; +struct Bridge { + unsigned long _last_memory_check = 0; + bool _memory_pressure = false; + int _skipped_publishes = 0; + bool publish() { +''' + guard + r''' + return true; + } +}; +int main() { + Bridge bridge; + assert(bridge.publish() && ESP.queries == 1); + ESP.largest = 15999; + now_ms += 5001; + assert(!bridge.publish() && bridge._skipped_publishes == 1 && ESP.queries == 2); + assert(!bridge.publish() && bridge._skipped_publishes == 2 && ESP.queries == 2); + now_ms += 5001; ESP.largest = 16000; + assert(bridge.publish() && ESP.queries == 3); + // Compiling BOARD_HAS_PSRAM does not establish that PSRAM initialized. + now_ms += 5001; ESP.psram = 0; ESP.largest = 8000; + assert(bridge.publish() && ESP.queries == 4); + now_ms += 5001; ESP.largest = 7999; + assert(!bridge.publish() && bridge._skipped_publishes == 3); +} ''') def test_internal_fallback_preserves_dma_reserve_and_rejects_fragmentation(self): @@ -292,35 +360,35 @@ int main() { assert(!bridge.canStartSlotConnection(0)); dma_free = 100000; dma_largest = 17407; assert(!bridge.canStartSlotConnection(0)); - ESP.size = 2097152; dma_free = 16384; dma_largest = 4096; + ESP.size = 2097152; dma_free = 16384; dma_largest = 16000; assert(!bridge.canStartSlotConnection(0)); Client client; bridge._slots[0].client = &client; assert(!bridge.canStartSlotConnection(0)); // Wrapper allocated; SDK not initialized. - dma_free = 32767; dma_largest = 20000; + dma_free = 32767; dma_largest = 16000; assert(!bridge.canStartSlotConnection(0)); - dma_free = 32768; dma_largest = 8191; + dma_free = 32768; dma_largest = 15999; assert(!bridge.canStartSlotConnection(0)); - dma_largest = 8192; + dma_largest = 16000; assert(bridge.canStartSlotConnection(0)); // An initialized but stopped handle needs its task again, not new SDK buffers. client.initialized = true; dma_free = 24575; dma_largest = 20000; assert(!bridge.canStartSlotConnection(0)); - dma_free = 24576; dma_largest = 8191; + dma_free = 24576; dma_largest = 15999; assert(!bridge.canStartSlotConnection(0)); - dma_largest = 8192; + dma_largest = 16000; assert(bridge.canStartSlotConnection(0)); // A warm reconnect retains the SDK task/buffers and uses the smaller floor. - client.started = true; dma_free = 16384; dma_largest = 4096; + client.started = true; dma_free = 16384; dma_largest = 16000; assert(bridge.canStartSlotConnection(0)); - dma_largest = 4095; + dma_largest = 15999; assert(!bridge.canStartSlotConnection(0)); // An explicit plaintext URI overrides a TLS port. The effective URI wins // over the raw host; a preset's URI wins over both custom fields. - ESP.size = 0; dma_free = 16384; dma_largest = 4096; + ESP.size = 0; dma_free = 16384; dma_largest = 16000; strcpy(bridge._slots[0].host, "mqtt://broker:8883"); assert(bridge.canStartSlotConnection(0)); strcpy(bridge._slots[0].broker_uri, "wss://broker/mqtt"); @@ -334,6 +402,32 @@ int main() { ESP.size = 2097152; assert(bridge.canStartSlotConnection(0)); + // Reproduce the reporter's failed-state heap after all SDK tasks stopped. + // One broker must be able to recover instead of waiting forever for a + // contiguous 24,192-byte block pinned apart by retained SDK buffers. + client.started = false; + dma_free = 93024; dma_largest = 20468; + assert(bridge.canStartSlotConnection(0)); + // Once another broker is connected, an additional task must leave its + // publication headroom beside the restart allowance. + bridge._slots[1].connected = true; + assert(!bridge.canStartSlotConnection(0)); + dma_largest = 24191; + assert(!bridge.canStartSlotConnection(0)); + dma_largest = 24192; + assert(bridge.canStartSlotConnection(0)); + // Health reconciliation removing the other failed connection restores + // bootstrap admission. The target never counts itself as a live peer. + dma_largest = 20468; + bridge._slots[1].connected = false; + bridge._slots[0].connected = true; + assert(bridge.canStartSlotConnection(0)); + bridge._slots[0].connected = false; + bridge._slots[1].enabled = false; + bridge._slots[1].connected = true; + assert(bridge.canStartSlotConnection(0)); + bridge._slots[1].connected = false; + // Every pending TLS attempt, including the target's own attempt, blocks all // other starts until an event callback clears it. No heap walk is needed. for (int pending = 0; pending < RUNTIME_MQTT_SLOTS; ++pending) { diff --git a/test/test_mqtt_handshake_scheduler.py b/test/test_mqtt_handshake_scheduler.py index c48ec1d55..f8d972f14 100644 --- a/test/test_mqtt_handshake_scheduler.py +++ b/test/test_mqtt_handshake_scheduler.py @@ -54,7 +54,7 @@ struct esp_mqtt_error_codes { constexpr uint32_t MALLOC_CAP_INTERNAL = 1, MALLOC_CAP_DMA = 2; static uint32_t clock_ms = 1000; static unsigned long wall_seconds = 1800000000; -static size_t dma_free = 100000, dma_largest = 20000; +static size_t dma_free = 100000, dma_largest = 100000; static unsigned long s_wifi_connected_at = 0; uint32_t millis() { return clock_ms; } unsigned long fake_time(void*) { return wall_seconds; } @@ -155,7 +155,7 @@ struct PsychicMqttClient { void softDisconnect() { ++bounces; live = false; // Releasing the old internal TLS session creates room for its replacement. - dma_free += 44000; dma_largest = 20000; + dma_free += 44000; dma_largest = 50000; } void setCredentials(const char*, const char*) { ++credentials; } }; @@ -421,11 +421,11 @@ int main() { bridge.maintainSlotConnections(); assert(bridge.setups.empty() && bridge._slots[0].last_reconnect_attempt == 0); assert(bridge._slots[0].enabled && bridge.activatedSlotCount() == 0); - dma_free = 32768; dma_largest = 8191; + dma_free = 32768; dma_largest = 15999; bridge.maintainSlotConnections(); assert(bridge.setups.empty() && bridge._slots[0].last_reconnect_attempt == 0); assert(!bridge.hasPendingSlotConnection() && bridge.activatedSlotCount() == 0); - dma_largest = 8192; + dma_largest = 16000; bridge.maintainSlotConnections(); assert(bridge.setups == std::vector{0} && bridge.activatedSlotCount() == 1); assert(bridge.clients[0].starts == 1 && bridge.clients[0].reconnects == 0); @@ -438,7 +438,7 @@ int main() { MQTTBridge bridge; for (int i = 2; i < RUNTIME_MQTT_SLOTS; ++i) bridge._slots[i].enabled = false; clock_ms = 70000; - dma_free = 16384; dma_largest = 4096; + dma_free = 16384; dma_largest = 16000; auto& stopped = bridge._slots[0]; auto& retained = bridge._slots[1]; stopped.client = &bridge.clients[0]; @@ -462,7 +462,7 @@ int main() { // with no extra delay introduced by the earlier memory-only deferral. bridge.complete(1); clock_ms += 15000; - dma_free = 24576; dma_largest = 8192; + dma_free = 24576; dma_largest = 24192; bridge.maintainSlotConnections(); assert(stopped.client->starts == 1 && stopped.client->reconnects == 0); assert(stopped.client->isStarted() && bridge._slot_attempt_pending[0]); @@ -536,14 +536,14 @@ int main() { // headroom is inadequate. Neither failure consumes retry/backoff state. dma_free = 24575; bridge.maintainSlotConnections(); - dma_free = 24576; dma_largest = 8191; + dma_free = 24576; dma_largest = 15999; bridge.maintainSlotConnections(); for (int i = 0; i < 3; ++i) { assert(bridge.clients[i].starts == 0 && bridge._slots[i].last_reconnect_attempt == 0); assert(bridge._slots[i].reconnect_backoff == 0 && bridge._slots[i].start_failures == 0); } - dma_free = 42376; dma_largest = 22516; + dma_free = 42376; dma_largest = 24192; bridge.maintainSlotConnections(); assert(bridge.clients[0].starts == 1 && dma_free == 35840); assert(bridge.hasPendingSlotConnection()); @@ -837,10 +837,10 @@ int main() { bridge.maintainSlotConnections(); assert(slot.client->starts == 0 && slot.reconnect_backoff == 3); assert(slot.last_reconnect_attempt == 70000 && slot.start_failures == 0); - dma_free = 24576; dma_largest = 8191; + dma_free = 24576; dma_largest = 15999; bridge.maintainSlotConnections(); assert(slot.client->starts == 0 && slot.reconnect_backoff == 3); - dma_largest = 8192; + dma_largest = 16000; bridge.maintainSlotConnections(); assert(slot.client->starts == 1 && bridge._slot_attempt_pending[0]); assert(slot.reconnect_backoff == 4 && slot.max_backoff_failures == 1); diff --git a/test/test_mqtt_network_timeout.py b/test/test_mqtt_network_timeout.py index 5d3fc1a20..28eb9ec56 100644 --- a/test/test_mqtt_network_timeout.py +++ b/test/test_mqtt_network_timeout.py @@ -104,7 +104,7 @@ RECONFIGURE_PREAMBLE = r''' #define pdMS_TO_TICKS(value) (value) constexpr int RUNTIME_MQTT_SLOTS = 2, MQTT_AUTH_JWT = 1; constexpr uint32_t MALLOC_CAP_INTERNAL = 1, MALLOC_CAP_DMA = 2; -static size_t dma_free = 100000, dma_largest = 20000; +static size_t dma_free = 100000, dma_largest = 100000; size_t heap_caps_get_free_size(uint32_t caps) { assert(caps == (MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA)); return dma_free; } @@ -228,7 +228,7 @@ int main() { bridge.applySlotPreset(0, "plain"); assert(slot.client && !slot.client->isStarted() && !slot.initial_connect_done); assert(init_calls == 1 && destroy_calls == 0); - dma_free = 24576; + dma_free = 24576; dma_largest = 24192; assert(bridge.setupSlot(0)); assert(slot.client->_buffer_capacity == 896 && init_calls == 1); const int destructions_before_guard = destroy_calls; diff --git a/test/test_mqtt_wifi_network_power.py b/test/test_mqtt_wifi_network_power.py index d3c23129f..9d2731d61 100644 --- a/test/test_mqtt_wifi_network_power.py +++ b/test/test_mqtt_wifi_network_power.py @@ -612,6 +612,41 @@ static void publishAcknowledgmentProvesRecoveryImmediately() { "an old PUBACK repeatedly renewed a disconnected broker"); } +static void disconnectedPublishAcknowledgmentRenewsGraceOnce() { + for (uint32_t start : {100U, UINT32_MAX - 10000U}) { + resetEnvironment(); + MQTTBridge bridge; + require(bridge.serviceNetworkPower(start), "initial outage did not run"); + require(!bridge.serviceNetworkPower(start + grace), "outage did not suspend"); + const uint32_t resumed = start + grace + interval; + require(bridge.serviceNetworkPower(resumed), "probe did not resume"); + const uint32_t replied = resumed + probe - 1U; + // The SDK delivered PUBACK and then DISCONNECTED between owner passes. + // A real broker reply still proves service; a CONNACK by itself does not. + bridge.clients[0].publish_acks = 1U; + require(!bridge._slots[0].connected, "fixture unexpectedly connected"); + require(bridge.serviceNetworkPower(replied), "new PUBACK was discarded"); + require(bridge.serviceNetworkPower(resumed + probe), + "disconnect hid PUBACK and retained the expired probe deadline"); + require(bridge.serviceNetworkPower(replied + grace - 1U), + "disconnected PUBACK did not renew normal grace"); + require(!bridge.serviceNetworkPower(replied + grace), + "unchanged PUBACK kept a failed service awake forever"); + } + resetEnvironment(); + { + MQTTBridge bridge; + bridge.serviceNetworkPower(100U); + bridge._slots[0].enabled = false; + bridge.clients[0].publish_acks = 1U; + require(!bridge.serviceNetworkPower(100U + grace), + "a disabled broker renewed the outage budget"); + bridge._slots[0].enabled = true; + require(!bridge.serviceNetworkPower(101U + grace), + "enabling a broker replayed its already consumed PUBACK"); + } +} + static void quietConnectionHasExactThirtySecondHealthBoundary() { resetEnvironment(); { @@ -743,6 +778,7 @@ int main(int argc, char** argv) { else if (name == "channel-repair") deferredEspNowChannelRepair(); else if (name == "healthy") stableBrokerCountsButFlappingDoesNot(); else if (name == "puback") publishAcknowledgmentProvesRecoveryImmediately(); + else if (name == "disconnected-puback") disconnectedPublishAcknowledgmentRenewsGraceOnce(); else if (name == "quiet-boundary") quietConnectionHasExactThirtySecondHealthBoundary(); else if (name == "ntp-retry") sntpFailureAndBusyLeaseRetries(); else if (name == "ntp-rotation") everyNtpServerGetsTheLeadingTimeoutSlot(); @@ -891,6 +927,9 @@ class MqttWiFiNetworkPowerTests(unittest.TestCase): def test_publish_ack_proves_recovery_immediately_and_is_consumed_once(self): self.run_case("puback") + def test_puback_followed_by_disconnect_renews_grace_once_across_wrap(self): + self.run_case("disconnected-puback") + def test_quiet_broker_health_requires_exactly_thirty_seconds(self): self.run_case("quiet-boundary") From cfcfc832b136855a13cd82857b96b16f677f8126 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Tue, 27 Oct 2026 08:17:23 -0700 Subject: [PATCH 2/6] Keep Full C3 Companion queues within the DRAM budget Place 32 existing retained frames in RTC FAST without changing the 224 normal or 128 active-OTA queue capacities. Keep the default backing layout unchanged for other boards and cover ownership, wraparound and OTA transfers. --- build_legacy.sh | 5 + examples/companion_radio/MyMesh.h | 30 +++-- src/helpers/BorrowableFrameBuffer.h | 19 ++- src/helpers/CompanionRtcQueue.h | 8 +- test/test_companion_rtc_queue.py | 18 ++- test/test_companion_rtc_terminal_input.py | 5 +- test/test_companion_shared_ota_capacity.py | 131 +++++++++++++++++++-- test/test_shared_mota_queue.py | 29 ++++- 8 files changed, 213 insertions(+), 32 deletions(-) diff --git a/build_legacy.sh b/build_legacy.sh index bb34e3e03..ff3d69999 100755 --- a/build_legacy.sh +++ b/build_legacy.sh @@ -4587,6 +4587,11 @@ apply_companion_radio_full_profile() { # Reuse idle queue storage during explicit OTA; retain all 224 normal slots. if [[ "${env_name,,}" != generic_espnow_companion_radio_full ]]; then export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DOTA_SHARED_COMPANION_QUEUE=1 -DONE_KEY_DM_COMPACT_HELD=1" + if [[ "${env_name,,}" == xiao_c3_companion_radio_full ]]; then + # Move 32 of the existing 128 retained slots into byte-accessible + # RTC FAST. Both the 224 normal and 128 active-OTA limits stay intact. + export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DCOMPANION_RTC_FAST_OFFLINE_FRAMES=32" + fi else # Keep all 224 frames during OTA too; RTC FAST holds 32 existing # slots without borrowing the queue or shrinking held-DM storage. diff --git a/examples/companion_radio/MyMesh.h b/examples/companion_radio/MyMesh.h index 68371f067..a15321283 100644 --- a/examples/companion_radio/MyMesh.h +++ b/examples/companion_radio/MyMesh.h @@ -1003,18 +1003,7 @@ private: int offline_queue_head; // Local onboarding only: RAM-backed like the receive queue, not RF traffic. uint8_t pending_emergency_notices[(MAX_GROUP_CHANNELS + 7) / 8] = {}; -#if defined(OTA_SHARED_COMPANION_QUEUE) - static constexpr unsigned OTA_RETAINED_OFFLINE_FRAMES = 128; - static_assert(OFFLINE_QUEUE_SIZE > OTA_RETAINED_OFFLINE_FRAMES, - "Shared mOTA queue must retain 128 frames and provide workspace storage"); - // The buffer also asserts that the complete context fits in the remaining - // frames. Constrained ESP32 recipes keep 224 normal slots; nRF52 and Paper keep 256. - mesh::BorrowableFrameBuffer offline_queue; - static mesh::ota::OtaContext* acquireOfflineQueueForOta(void* owner); - static uint16_t drainOfflineQueueForOta(void* owner); - static void releaseOfflineQueueFromOta(void* owner); -#elif COMPANION_RTC_FAST_OFFLINE_FRAMES || COMPANION_RTC_SLOW_OFFLINE_FRAMES +#if COMPANION_RTC_FAST_OFFLINE_FRAMES || COMPANION_RTC_SLOW_OFFLINE_FRAMES // Concrete definitions in MyMesh.cpp keep GCC from moving template-local // statics back into ordinary .bss despite the requested RTC attribute. #if COMPANION_RTC_FAST_OFFLINE_FRAMES @@ -1037,6 +1026,23 @@ private: return nullptr; #endif } +#endif +#if defined(OTA_SHARED_COMPANION_QUEUE) + static constexpr unsigned OTA_RETAINED_OFFLINE_FRAMES = 128; + static_assert(OFFLINE_QUEUE_SIZE > OTA_RETAINED_OFFLINE_FRAMES, + "Shared mOTA queue must retain 128 frames and provide workspace storage"); + // The buffer also asserts that the complete context fits in the remaining + // frames. Constrained ESP32 recipes keep 224 normal slots; nRF52 and Paper keep 256. + mesh::BorrowableFrameBuffer +#endif + > offline_queue; + static mesh::ota::OtaContext* acquireOfflineQueueForOta(void* owner); + static uint16_t drainOfflineQueueForOta(void* owner); + static void releaseOfflineQueueFromOta(void* owner); +#elif COMPANION_RTC_FAST_OFFLINE_FRAMES || COMPANION_RTC_SLOW_OFFLINE_FRAMES mesh::CompanionRtcQueue offline_queue; #elif defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM) enum { diff --git a/src/helpers/BorrowableFrameBuffer.h b/src/helpers/BorrowableFrameBuffer.h index 16cdc1a8f..32fda417d 100644 --- a/src/helpers/BorrowableFrameBuffer.h +++ b/src/helpers/BorrowableFrameBuffer.h @@ -6,16 +6,28 @@ namespace mesh { +template class InlineRetainedFrames { + Frame frames[Count]; +public: + static constexpr size_t frame_count = Count; + Frame& at(size_t index) { return frames[index]; } +}; + // One owner at a time: an offline FIFO borrows the workspace's storage while // idle. Switching owners compacts the ring in place, without allocating RAM or // dropping unread frames. The caller owns the FIFO's length and head index. -template +// Qualified boards can supply a physically separate retained-frame bank. It +// replaces only existing retained slots, never the extra frames or workspace. +template> class BorrowableFrameBuffer { static_assert(Reserved > 0 && Reserved < Capacity, "Invalid shared queue capacities"); + static_assert(RetainedFrames::frame_count == Reserved, + "The retained backing must preserve every active-OTA slot"); struct ExtraFrames { Frame frames[Capacity - Reserved]; }; static_assert(sizeof(Workspace) <= sizeof(ExtraFrames), "mOTA exceeds borrowed queue storage"); - Frame _retained[Reserved]; + RetainedFrames _retained; union Extra { ExtraFrames queue; Workspace workspace; @@ -45,6 +57,7 @@ class BorrowableFrameBuffer { public: BorrowableFrameBuffer() = default; + BorrowableFrameBuffer(Frame* fast, Frame* slow) : _retained(fast, slow) {} BorrowableFrameBuffer(const BorrowableFrameBuffer&) = delete; BorrowableFrameBuffer& operator=(const BorrowableFrameBuffer&) = delete; ~BorrowableFrameBuffer() { @@ -53,7 +66,7 @@ public: size_t capacity() const { return _borrowed ? Reserved : Capacity; } Frame& at(size_t physical_index) { - return physical_index < Reserved ? _retained[physical_index] + return physical_index < Reserved ? _retained.at(physical_index) : _extra.queue.frames[physical_index - Reserved]; } diff --git a/src/helpers/CompanionRtcQueue.h b/src/helpers/CompanionRtcQueue.h index 4ac374675..692fe75ce 100644 --- a/src/helpers/CompanionRtcQueue.h +++ b/src/helpers/CompanionRtcQueue.h @@ -16,8 +16,7 @@ #if !defined(ESP32_PLATFORM) || !defined(COMPANION_RADIO_FULL) #error "RTC offline frames require a qualified ESP32 Full Companion recipe" #endif -#if defined(BOARD_HAS_PSRAM) || defined(OTA_SHARED_COMPANION_QUEUE) \ - || ONE_KEY_DM_SHARED_OFFLINE_QUEUE +#if defined(BOARD_HAS_PSRAM) || ONE_KEY_DM_SHARED_OFFLINE_QUEUE #error "RTC offline frames require the independent fixed-capacity queue" #endif #include @@ -36,6 +35,10 @@ #if COMPANION_RTC_SLOW_OFFLINE_FRAMES && !defined(CONFIG_IDF_TARGET_ESP32S3) #error "The separate RTC SLOW offline-frame bank requires ESP32-S3" #endif +#if defined(OTA_SHARED_COMPANION_QUEUE) \ + && (!defined(CONFIG_IDF_TARGET_ESP32C3) || COMPANION_RTC_SLOW_OFFLINE_FRAMES) +#error "RTC retained shared-OTA frames are qualified only on ESP32-C3 FAST memory" +#endif namespace mesh { namespace companion_rtc { @@ -75,6 +78,7 @@ template class CompanionRtcQueue { } public: + static constexpr size_t frame_count = Count; CompanionRtcQueue(Frame* fast, Frame* slow) : fast_frames(fast), slow_frames(slow) { if ((FastCount && (!fast || reinterpret_cast(fast) % 4)) || (SlowCount && (!slow || reinterpret_cast(slow) % 4))) abort(); diff --git a/test/test_companion_rtc_queue.py b/test/test_companion_rtc_queue.py index 100918bf0..bec0754cf 100644 --- a/test/test_companion_rtc_queue.py +++ b/test/test_companion_rtc_queue.py @@ -196,7 +196,10 @@ class CompanionRtcQueueTests(unittest.TestCase): source = (ROOT / 'examples/companion_radio/MyMesh.cpp').read_text() start = header.index('#elif COMPANION_RTC_FAST_OFFLINE_FRAMES || COMPANION_RTC_SLOW_OFFLINE_FRAMES', header.index('static constexpr unsigned OTA_RETAINED_OFFLINE_FRAMES')) end = header.index('#elif defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM)', start) - storage = header[start:].split('\n', 1)[1][:end - start - len(header[start:].split('\n', 1)[0]) - 1] + common_start = header.index(' // Concrete definitions in MyMesh.cpp') + common_end = header.index('\n#endif\n#if defined(OTA_SHARED_COMPANION_QUEUE)', common_start) + storage = (header[common_start:common_end] + '\n' + + header[start:].split('\n', 1)[1][:end - start - len(header[start:].split('\n', 1)[0]) - 1]) banks_start = source.index('#if COMPANION_RTC_FAST_OFFLINE_FRAMES\n') banks_end = source.index('bool MyMesh::Frame::isChannelMsg()', banks_start) methods = '\n'.join(extract_braced(source, name) for name in ( @@ -287,7 +290,8 @@ class CompanionRtcQueueTests(unittest.TestCase): base = ['ESP32_PLATFORM=1', 'COMPANION_RADIO_FULL=1', 'CONFIG_IDF_TARGET_ESP32C3=1', 'CONFIG_ESP_SYSTEM_ALLOW_RTC_FAST_MEM_AS_HEAP=1', 'COMPANION_RTC_FAST_OFFLINE_FRAMES=32'] cases = ([value for value in base if not value.startswith('CONFIG_IDF_TARGET_')], - base + ['BOARD_HAS_PSRAM=1'], base + ['OTA_SHARED_COMPANION_QUEUE=1'], + base + ['BOARD_HAS_PSRAM=1'], [value for value in base if not value.startswith('CONFIG_IDF_TARGET_')] + + ['OTA_SHARED_COMPANION_QUEUE=1', 'CONFIG_IDF_TARGET_ESP32S3=1'], base + ['ONE_KEY_DM_SHARED_OFFLINE_QUEUE=1'], base + ['COMPANION_RTC_SLOW_OFFLINE_FRAMES=32'], [value for value in base if not value.startswith('COMPANION_RADIO_FULL')], @@ -299,7 +303,7 @@ class CompanionRtcQueueTests(unittest.TestCase): capture_output=True, text=True, timeout=15) self.assertNotEqual(result.returncode, 0) - def test_only_five_named_full_policies_select_banks_without_capacity_reductions(self): + def test_only_named_full_policies_select_banks_without_capacity_reductions(self): source = (ROOT / 'build_legacy.sh').read_text() start = source.index(' # BLE + WiFi exhaust internal DRAM') start = source.index(' case "${env_name,,}" in', start) @@ -310,6 +314,8 @@ class CompanionRtcQueueTests(unittest.TestCase): self.assertNotIn('COMPANION_RTC_SLOW_OFFLINE_FRAMES', prefix) for target, fast, slow, contacts, queue in ( ('Generic_ESPNOW_companion_radio_full', 32, 0, 100, 224), + ('Xiao_C3_companion_radio_full', 32, 0, 100, 224), + ('Heltec_CT62_companion_radio_full', 0, 0, 100, 224), ('Heltec_Wireless_Tracker_companion_radio_full', 32, 32, 150, 256), ('MKE_s3_companion_radio_full', 32, 32, 350, 256), ('ThinkNode_M2_companion_radio_full', 32, 32, 350, 256), @@ -328,7 +334,11 @@ class CompanionRtcQueueTests(unittest.TestCase): self.assertEqual(int(flags.get('COMPANION_RTC_SLOW_OFFLINE_FRAMES', 0)), slow) self.assertEqual(int(flags.get('MAX_CONTACTS', 350)), contacts) self.assertEqual(int(flags.get('OFFLINE_QUEUE_SIZE', 256)), queue) - self.assertNotIn('OTA_SHARED_COMPANION_QUEUE', flags) + if target in ('Xiao_C3_companion_radio_full', 'Heltec_CT62_companion_radio_full'): + self.assertEqual(flags.get('OTA_SHARED_COMPANION_QUEUE'), '1') + self.assertEqual(flags.get('ONE_KEY_DM_COMPACT_HELD'), '1') + else: + self.assertNotIn('OTA_SHARED_COMPANION_QUEUE', flags) if __name__ == '__main__': diff --git a/test/test_companion_rtc_terminal_input.py b/test/test_companion_rtc_terminal_input.py index 8f596ae07..96593c055 100644 --- a/test/test_companion_rtc_terminal_input.py +++ b/test/test_companion_rtc_terminal_input.py @@ -209,7 +209,10 @@ int main() { self.assertEqual(flags.get('COMPANION_RTC_TERMINAL_INPUT')=='1', selected) self.assertEqual(int(flags['MAX_CONTACTS']), contacts) self.assertEqual(int(flags.get('OFFLINE_QUEUE_SIZE', 256)), queue) - self.assertNotIn('COMPANION_RTC_FAST_OFFLINE_FRAMES', flags) + if target == 'Xiao_C3_companion_radio_full': + self.assertEqual(flags.get('COMPANION_RTC_FAST_OFFLINE_FRAMES'), '32') + else: + self.assertNotIn('COMPANION_RTC_FAST_OFFLINE_FRAMES', flags) self.assertNotIn('COMPANION_RTC_SLOW_OFFLINE_FRAMES', flags) diff --git a/test/test_companion_shared_ota_capacity.py b/test/test_companion_shared_ota_capacity.py index 788393cb7..9d247ed65 100644 --- a/test/test_companion_shared_ota_capacity.py +++ b/test/test_companion_shared_ota_capacity.py @@ -3,6 +3,9 @@ from pathlib import Path import re +import resource +import signal +import shutil import subprocess import tempfile import unittest @@ -22,24 +25,41 @@ def method(text, signature): class SharedCompanionOtaCapacityTest(unittest.TestCase): - def compile(self, text, *, run=False): + def compile(self, text, *, run=False, rtc=False, args=(), expected_returncode=0): with tempfile.TemporaryDirectory(prefix="meshcore-shared-c3-") as temp: binary = Path(temp) / "queue" + (Path(temp) / "sdkconfig.h").write_text("#pragma once\n") + (Path(temp) / "esp_attr.h").write_text( + '#pragma once\n#define RTC_FAST_ATTR __attribute__((section(".rtc.force_fast.test")))\n') command = ["c++", "-std=c++17", "-Wall", "-Wextra", "-Werror", - "-I", str(ROOT / "src"), "-x", "c++", "-", + "-I", str(ROOT / "src"), "-I", temp, "-x", "c++", "-", "-o", str(binary)] + if rtc: + command[1:1] = ["-DNDEBUG", "-DESP32_PLATFORM=1", "-DCOMPANION_RADIO_FULL=1", + "-DOTA_SHARED_COMPANION_QUEUE=1", "-DOFFLINE_QUEUE_SIZE=224", "-DCONFIG_IDF_TARGET_ESP32C3=1", + "-DCONFIG_ESP_SYSTEM_ALLOW_RTC_FAST_MEM_AS_HEAP=1", + "-DCOMPANION_RTC_FAST_OFFLINE_FRAMES=32"] if run: command[1:1] = ["-fsanitize=address,undefined", "-g"] result = subprocess.run(command, input=text, text=True, capture_output=True) if run: self.assertEqual(result.returncode, 0, result.stderr) - result = subprocess.run([str(binary)], text=True, - capture_output=True) - self.assertEqual(result.returncode, 0, result.stderr) + if rtc: + objdump = shutil.which('objdump') + self.assertIsNotNone(objdump, 'objdump is required for the physical RTC bank check') + sections = subprocess.check_output([objdump, '-h', str(binary)], text=True) + match = re.search(r'\.rtc\.force_fast\.test\s+([0-9a-fA-F]+)', sections) + self.assertIsNotNone(match, sections) + self.assertEqual(int(match[1], 16), 32 * 177, sections) + def no_core_dump(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + result = subprocess.run([str(binary), *args], text=True, + capture_output=True, preexec_fn=no_core_dump) + self.assertEqual(result.returncode, expected_returncode, result.stderr) return result - def test_224_normal_slots_and_128_retained_slots_preserve_fifo_and_ownership(self): + def queue_fixture(self, rtc=False): source = (ROOT / "examples/companion_radio/MyMesh.cpp").read_text() declaration = (ROOT / "examples/companion_radio/MyMesh.h").read_text() retained = re.search( @@ -52,10 +72,14 @@ class SharedCompanionOtaCapacityTest(unittest.TestCase): )) harness = r''' #include +#include +#include #include #include #include #include +#undef assert +#define assert(value) do { if (!(value)) std::abort(); } while (0) constexpr unsigned MAX_FRAME_SIZE = 176; namespace mesh { namespace ota { struct OtaContext { @@ -76,8 +100,7 @@ unsigned OtaContext::live = 0, OtaContext::constructions = 0, struct MyMesh { struct Frame { uint8_t len; uint8_t buf[MAX_FRAME_SIZE]; }; static constexpr unsigned OTA_RETAINED_OFFLINE_FRAMES = @RETAINED@; - mesh::BorrowableFrameBuffer offline_queue; + @STORAGE@ int offline_queue_len = 0, offline_queue_head = 0; unsigned next = 0, notifications = 0; std::deque expected; @@ -112,6 +135,7 @@ struct MyMesh { } void pop() { uint8_t bytes[MAX_FRAME_SIZE]; assert(getFromOfflineQueue(bytes)); } }; +@BANKS@ @METHODS@ int main() { using mesh::ota::OtaContext; @@ -148,7 +172,96 @@ int main() { } } '''.replace("@RETAINED@", retained).replace("@METHODS@", methods) - self.compile(harness, run=True) + if rtc: + start = declaration.index(' // Concrete definitions in MyMesh.cpp') + end = declaration.index('\n#endif\n#if defined(OTA_SHARED_COMPANION_QUEUE)', start) + shared_start = declaration.index(' static constexpr unsigned OTA_RETAINED_OFFLINE_FRAMES') + queue_start = declaration.index(' mesh::BorrowableFrameBuffer offline_queue;') + banks = '' + return harness.replace('@STORAGE@', storage).replace('@BANKS@', banks) + + def test_224_normal_slots_and_128_retained_slots_preserve_fifo_and_ownership(self): + self.compile(self.queue_fixture(), run=True) + + def test_rtc_backed_224_and_128_fifo_all_heads_use_actual_mesh_storage_and_callbacks(self): + self.compile(self.queue_fixture(rtc=True), run=True, rtc=True) + + def test_rtc_owner_reset_geometry_and_release_build_safeguards(self): + harness = self.queue_fixture(rtc=True).split('int main() {', 1)[0] + r''' +using Frame = MyMesh::Frame; +using Workspace = mesh::ota::OtaContext; +using Inline = mesh::BorrowableFrameBuffer; +using Retained = mesh::CompanionRtcQueue; +using Split = mesh::BorrowableFrameBuffer; +struct Legacy { + Frame retained[128]; + union Extra { Frame frames[96]; Workspace workspace; Extra() {} ~Extra() {} } extra; + bool borrowed; +}; +static_assert(sizeof(Inline) == sizeof(Legacy), "Ordinary targets keep the exact BSS layout"); +static_assert(alignof(Inline) == alignof(Legacy), "Ordinary targets keep their alignment"); +static_assert(sizeof(Inline) - sizeof(Split) >= 32 * sizeof(Frame) - 2 * sizeof(void*) - alignof(Split), + "The complete 32-slot bank must leave internal DRAM"); +static_assert(Retained::frame_count == 128, "All active-OTA slots remain independent of workspace"); +static_assert(!std::is_copy_constructible::value && !std::is_move_constructible::value, + "A bank lease must never be duplicated"); +int main(int argc, char** argv) { + alignas(4) Frame bank[32]; + memset(bank, 0xA5, sizeof bank); + if (argc > 1 && !strcmp(argv[1], "null")) { Split bad(nullptr, nullptr); return 1; } + if (argc > 1 && !strcmp(argv[1], "misaligned")) { + Split bad(reinterpret_cast(reinterpret_cast(bank) + 1), nullptr); return 1; + } + if (argc > 1 && !strcmp(argv[1], "range")) { + Retained owner(bank, nullptr); owner.at(128); return 1; + } + { + Split owner(bank, nullptr); + for (const auto& frame : bank) { + assert(frame.len == 0); + for (uint8_t byte : frame.buf) assert(byte == 0); + } + for (unsigned i = 0; i < 128; ++i) { + auto& frame = owner.at(i); assert(frame.len == 0); + for (uint8_t byte : frame.buf) assert(byte == 0); + } + if (argc > 1 && !strcmp(argv[1], "second")) { Split second(bank, nullptr); return 1; } + if (argc > 1 && !strcmp(argv[1], "other-count")) { + mesh::CompanionRtcQueue second(bank, nullptr); return 1; + } + if (argc > 1 && !strcmp(argv[1], "foreign")) { + // A stale or foreign lease must be rejected, even with NDEBUG. + mesh::companion_rtc::Lease::owner() = nullptr; + owner.at(0); return 1; + } + for (unsigned i = 0; i < 224; ++i) memset(&owner.at(i), 0xBB, sizeof(Frame)); + int head = 217; + assert(owner.acquire(128, head) && head == 0 && owner.capacity() == 128); + // Destruction must end an active workspace exactly once before releasing its bank. + } + assert(Workspace::live == 0 && Workspace::constructions == Workspace::destructions); + Split next(bank, nullptr); + for (unsigned i = 0; i < 224; ++i) { + const auto& frame = next.at(i); assert(frame.len == 0); + for (uint8_t byte : frame.buf) assert(byte == 0); + } +} +''' + harness = harness.replace('#include ', '#include \n#include ') + self.compile(harness, run=True, rtc=True) + for argument in ('null', 'misaligned', 'second', 'other-count', 'foreign', 'range'): + with self.subTest(argument=argument): + self.compile(harness, run=True, rtc=True, args=(argument,), + expected_returncode=-signal.SIGABRT) def test_insufficient_workspace_or_no_retained_region_cannot_compile(self): harness = r''' diff --git a/test/test_shared_mota_queue.py b/test/test_shared_mota_queue.py index 9be466d99..3f8a0188b 100644 --- a/test/test_shared_mota_queue.py +++ b/test/test_shared_mota_queue.py @@ -17,7 +17,10 @@ class SharedMotaQueueTest(unittest.TestCase): def test_wireless_paper_wifi_listener_and_transfers_share_only_when_needed(self): self.run_transfer_test("ESP32_PLATFORM") - def run_transfer_test(self, platform): + def test_c3_rtc_retained_queue_preserves_real_ota_and_wifi_transfers(self): + self.run_transfer_test("ESP32_PLATFORM", rtc=True) + + def run_transfer_test(self, platform, rtc=False): with tempfile.TemporaryDirectory(prefix="meshcore-mota-queue-") as temp: binary = Path(temp) / "transfer" tinf = Path(temp) / "tinf.o" @@ -44,6 +47,30 @@ class SharedMotaQueueTest(unittest.TestCase): sources += ["src/helpers/esp32/WiFiOtaSeeder.cpp", "src/helpers/ota/MotaSourceSerial.cpp", "src/helpers/ota/FolderMotaStore.cpp"] + if rtc: + fixture = ROOT / "test/fixtures/shared_mota_queue" + source = (fixture / "test_shared_mota_queue.cpp").read_text() + source = source[:source.index(" Queue full;")] + "}\n" + source = source.replace("256", "224") + source = source.replace("struct Queue {", ( + "#include \n" + "alignas(4) RTC_FAST_ATTR Frame rtc_frames[32];\nstruct Queue {")) + source = source.replace( + "mesh::BorrowableFrameBuffer buffer;", + "mesh::BorrowableFrameBuffer> buffer;\n" + " Queue() : buffer(rtc_frames, nullptr) {}") + (Path(temp) / "test_shared_rtc.cpp").write_text(source) + (Path(temp) / "test_wifi_shared_queue.h").write_text( + (fixture / "test_wifi_shared_queue.h").read_text().replace("256", "224")) + (Path(temp) / "sdkconfig.h").write_text("#pragma once\n") + (Path(temp) / "esp_attr.h").write_text( + '#pragma once\n#define RTC_FAST_ATTR __attribute__((section(".rtc.force_fast.test")))\n') + sources[0] = str(Path(temp) / "test_shared_rtc.cpp") + flags += ["-DCOMPANION_RTC_FAST_OFFLINE_FRAMES=32", + "-DCONFIG_IDF_TARGET_ESP32C3=1", + "-DCONFIG_ESP_SYSTEM_ALLOW_RTC_FAST_MEM_AS_HEAP=1", + "-I", str(fixture)] built = subprocess.run([ "c++", "-std=c++17", "-fsanitize=address,undefined", "-g", *flags, "-DOTA_SEEDER_ONLY=1", From 2b55ff6a621584f661e9d6d3b3f3ab48fb3639de Mon Sep 17 00:00:00 2001 From: mikecarper Date: Tue, 27 Oct 2026 08:17:23 -0700 Subject: [PATCH 3/6] Recover RAK RAM without reducing OTA or sensor capacities Cache each validated OTA descriptor field once and size manifest reassembly to the admitted wire format. Compile reviewed read-only VL53L0X tuning data in flash through a private dependency copy, retaining the original implementation for unknown revisions. --- .github/workflows/run-unit-tests.yml | 1 + scripts/nrf52_flash_trim.py | 5 + scripts/nrf52_vl53l0x_const.py | 81 ++++++++ src/helpers/ota/OtaManager.cpp | 8 +- src/helpers/ota/OtaManager.h | 54 +++-- test/fixtures/vl53_const/README.md | 11 + test/fixtures/vl53_const/tuning_loader.inc | 109 ++++++++++ .../vl53l0x_interrupt_threshold_settings.h | 81 ++++++++ test/fixtures/vl53_const/vl53l0x_tuning.h | 87 ++++++++ test/test_nrf52_flash_trim.py | 7 + test/test_nrf52_vl53l0x_const.py | 190 ++++++++++++++++++ test/test_ota/test_ota_core.cpp | 43 ++++ test/test_ota_memory_layout.py | 44 +++- 13 files changed, 696 insertions(+), 25 deletions(-) create mode 100644 scripts/nrf52_vl53l0x_const.py create mode 100644 test/fixtures/vl53_const/README.md create mode 100644 test/fixtures/vl53_const/tuning_loader.inc create mode 100644 test/fixtures/vl53_const/vl53l0x_interrupt_threshold_settings.h create mode 100644 test/fixtures/vl53_const/vl53l0x_tuning.h create mode 100644 test/test_nrf52_vl53l0x_const.py diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index ca217bc92..d89db099e 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -1143,6 +1143,7 @@ jobs: - name: Verify nRF52 infrastructure trims and unchanged Companion scope run: | python3 -B test/test_nrf52_flash_trim.py -v + python3 -B test/test_nrf52_vl53l0x_const.py -v python3 -B test/test_nrf52_gnss_version_trim.py -v python3 -B test/test_nrf52_program_lifetime.py -v python3 -B test/test_nrf52_tft_font_trim.py -v diff --git a/scripts/nrf52_flash_trim.py b/scripts/nrf52_flash_trim.py index 4d1a19f4b..a7116487f 100644 --- a/scripts/nrf52_flash_trim.py +++ b/scripts/nrf52_flash_trim.py @@ -243,6 +243,11 @@ def install(build_env): return build_env["MESH_NRF52_FLASH_TRIM_ACTIVE"] = True build_env.AppendUnique(CPPDEFINES=list(definitions.items())) + hook_path = build_env.subst("$PROJECT_DIR/scripts/nrf52_vl53l0x_const.py") + hook_spec = importlib.util.spec_from_file_location("mesh_nrf52_vl53_const", hook_path) + hook = importlib.util.module_from_spec(hook_spec) + hook_spec.loader.exec_module(hook) + hook.install(build_env) if infrastructure: print("nRF52 infrastructure flash trim: compact crypto, unused USB device classes/GPS cache/vendor splash, compact OTA target names") else: diff --git a/scripts/nrf52_vl53l0x_const.py b/scripts/nrf52_vl53l0x_const.py new file mode 100644 index 000000000..0d52ae433 --- /dev/null +++ b/scripts/nrf52_vl53l0x_const.py @@ -0,0 +1,81 @@ +"""Keep the reviewed VL53L0X tuning tables in nRF52 flash. + +The vendor tuning loader only reads these bytes and copies register writes +through a local buffer. Compile a private dependency copy with const inputs; +leave its public caller-owned tuning-buffer setter/getter unchanged. Unknown +library versions retain their ordinary implementation and RAM requirements. +""" + +import hashlib +from pathlib import Path +import shutil + + +FILES = { + "vl53l0x_tuning.h": "0e0331b7df1084d183248355f75d835b3446200b22f71765b5c63a470ff01c63", + "vl53l0x_interrupt_threshold_settings.h": "9e5e6c00af66497d03d6c6a13a269521c12b33c90455efcd3b8494487e1780c2", + "vl53l0x_api_core.h": "948de4a0363339facc58c4e1bc5781b954b99cca1c13a4a105eb3969aeba8653", + "core/src/vl53l0x_api_core.cpp": "d31da82edbad0f7fd89106340f7a006b61cd2e5b7c353460fb394978ce55953b", + "core/src/vl53l0x_api.cpp": "6cfa0784600bd797e372b511761f4be10e97264b55d5c7269dd41ba607b8edb7", +} +REPLACEMENTS = { + "vl53l0x_tuning.h": ("uint8_t DefaultTuningSettings[]", "const uint8_t DefaultTuningSettings[]"), + "vl53l0x_interrupt_threshold_settings.h": ( + "uint8_t InterruptThresholdSettings[]", "const uint8_t InterruptThresholdSettings[]"), + "vl53l0x_api_core.h": ("uint8_t* pTuningSettingBuffer);", "const uint8_t* pTuningSettingBuffer);"), + "core/src/vl53l0x_api_core.cpp": ( + "uint8_t* pTuningSettingBuffer) {", "const uint8_t* pTuningSettingBuffer) {"), + "core/src/vl53l0x_api.cpp": ("uint8_t* pTuningSettingBuffer;", "const uint8_t* pTuningSettingBuffer;"), +} + + +def digest(text): + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def patched_files(source_root): + result = {} + for name, expected in FILES.items(): + text = (source_root / name).read_text(encoding="utf-8").replace("\r\n", "\n") + old, new = REPLACEMENTS[name] + if digest(text) != expected or text.count(old) != 1: + raise RuntimeError("changed reviewed VL53L0X dependency: " + name) + result[name] = text.replace(old, new, 1) + return result + + +def write_if_changed(path, content): + if not path.exists() or path.read_text(encoding="utf-8") != content: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def replace_vl53_source(build_env, node): + source = Path(node.srcnode().get_abspath()) + source_root = next((parent for parent in source.parents + if parent.name == "src" and parent.parent.name == "Adafruit_VL53L0X"), None) + if source_root is None or source.suffix != ".cpp": + return node + try: + patches = patched_files(source_root) + except (OSError, UnicodeError, RuntimeError) as error: + print("nRF52 VL53L0X const tables: ordinary source retained (" + str(error) + ")") + return node + destination = Path(build_env.subst("$BUILD_DIR")) / "readonly-nrf52-vl53l0x" / "src" + # Keep every quoted sibling/relative include inside the same private copy. + # All compilation units therefore see the same const loader declaration. + for path in source_root.rglob("*"): + if not path.is_file(): + continue + relative = path.relative_to(source_root) + output = destination / relative + if relative.as_posix() in patches: + write_if_changed(output, patches[relative.as_posix()]) + elif not output.exists() or output.read_bytes() != path.read_bytes(): + output.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, output) + return build_env.File(str(destination / source.relative_to(source_root))) + + +def install(build_env): + build_env.AddBuildMiddleware(replace_vl53_source, "*Adafruit_VL53L0X*/*.cpp") diff --git a/src/helpers/ota/OtaManager.cpp b/src/helpers/ota/OtaManager.cpp index e853d2691..5ef6f0295 100644 --- a/src/helpers/ota/OtaManager.cpp +++ b/src/helpers/ota/OtaManager.cpp @@ -334,10 +334,8 @@ void OtaManager::refresh_sources() { (uint64_t)d.payload_off + d.payload_size + 5 != d.total_size) continue; if (serveEntryIndex(d.mid) >= 0) continue; // already offered (e.g. our own fw in the folder) ServeEntry& e = _serve[_n_serve++]; - memcpy(e.mid, d.mid, 4); - e.target_id = d.target_id; e.fw_version = d.fw_version; - e.codec_id = d.codec_id; e.flags = d.flags; e.have_count = d.block_count; // a folder mota is fully held - e.is_self = false; e.src = src; e.src_idx = i; e.desc = d; + e.cacheDescriptor(d); + e.is_self = false; e.src = src; e.src_idx = i; _src_advertised[s]++; } } @@ -409,7 +407,7 @@ OtaManager::ServeView* OtaManager::resolve(const uint8_t* mid) { // builds also cache leaves here; bounded builds acquire proof metadata through // cooperative egress steps, without delaying request admission on a full table. bool OtaManager::loadSource(const ServeEntry& e) { - const MotaDesc& d = e.desc; + const MotaDesc d = e.descriptor(); if (d.leaves_off < 8) return false; if (!e.src) return false; uint16_t mfl = (uint16_t)(d.leaves_off - 8); diff --git a/src/helpers/ota/OtaManager.h b/src/helpers/ota/OtaManager.h index d7a32c5f1..1ec321ed8 100644 --- a/src/helpers/ota/OtaManager.h +++ b/src/helpers/ota/OtaManager.h @@ -2,6 +2,7 @@ #include #include +#include #include "OtaFormat.h" #include "OtaMemoryPolicy.h" #include "OtaTiming.h" @@ -327,19 +328,44 @@ public: }; // A lightweight catalog entry: what we advertise per mota + how to load its ServeView on demand. struct ServeEntry { - // Named fields only: this is not a wire/persistent layout. Group aligned - // values ahead of byte flags instead of paying three padding gaps per row. - MotaSource* src; // load from this external source ... - MotaDesc desc; // cached region offsets for the source entry - uint32_t target_id, fw_version; - uint32_t have_count; // blocks we currently hold (== block_count when complete) - uint8_t mid[4]; - uint8_t codec_id, flags; - bool is_self; // true => entry is view0 (our own fw / RAM mota) - uint8_t src_idx; // ... at this index + // This is not a wire/persistent layout. Source admission proves the fixed + // header/leaves/payload geometry, so cache each field once and reconstruct + // the three derived offsets/sizes instead of retaining another MotaDesc. + MotaSource* src = nullptr; // load from this external source ... + uint32_t target_id = 0, fw_version = 0; + uint32_t have_count = 0; // complete sources hold every block + uint32_t payload_size = 0; + uint8_t mid[4] = {}; + uint8_t codec_id = 0, flags = 0; + bool is_self = false; // true => entry is view0 (our own fw / RAM mota) + uint8_t src_idx = 0; // ... at this index + uint8_t block_size_log2 = 0, source_caps = 0; + + void cacheDescriptor(const MotaDesc& d) { + memcpy(mid, d.mid, sizeof(mid)); + target_id = d.target_id; fw_version = d.fw_version; + codec_id = d.codec_id; flags = d.flags; have_count = d.block_count; + payload_size = d.payload_size; + block_size_log2 = d.block_size_log2; source_caps = d.source_caps; + } + MotaDesc descriptor() const { + MotaDesc d; + if (!have_count) return d; + memcpy(d.mid, mid, sizeof(mid)); + d.target_id = target_id; d.fw_version = fw_version; + d.codec_id = codec_id; d.flags = flags; d.block_count = have_count; + d.payload_size = payload_size; + d.block_size_log2 = block_size_log2; d.source_caps = source_caps; + d.leaves_off = 8u + MOTA_MFL; + d.payload_off = d.leaves_off + have_count * 4u; + d.total_size = d.payload_off + payload_size + 5u; + return d; + } }; - static_assert(sizeof(ServeEntry) <= sizeof(MotaDesc) + sizeof(MotaSource*) + 20u, - "OTA source catalog must not add per-entry alignment padding"); + static_assert(sizeof(ServeEntry) == + ((sizeof(MotaSource*) + 28u + alignof(MotaSource*) - 1u) + / alignof(MotaSource*)) * alignof(MotaSource*), + "OTA source catalog must not duplicate validated descriptor fields"); // Context for the source-payload reader trampoline. struct SrcReadCtx { MotaSource* src; uint8_t idx; uint32_t payload_off; }; @@ -903,7 +929,9 @@ private: uint8_t _observed_path_transmissions = 0; // source + relays; 0 falls back to configured max_hops+1 uint32_t _packets_sent = 0; // OTA packets accepted by the radio adapter (wrap-safe) // multi-fragment manifest reassembly (a signed v2 manifest exceeds one packet) - uint8_t _mf_buf[OTA_MF_MAXFRAG * OTA_MF_FRAG]; // sized to the fragment cap so no valid manifest is silently dropped + // handleManifest admits only the fixed MOTA_MFL bytes and exact fragment + // shapes. The transport's larger fragment-count ceiling adds no valid data. + uint8_t _mf_buf[MOTA_MFL]; uint16_t _mf_retries = 0; // GET_MANIFEST retries while WANT_MANIFEST (give up after a cap) uint8_t _mf_total = 0; // frag_total of the manifest being reassembled (0 = none) uint16_t _mf_mask = 0; // received manifest-fragment bitmap diff --git a/test/fixtures/vl53_const/README.md b/test/fixtures/vl53_const/README.md new file mode 100644 index 000000000..3c281cb6f --- /dev/null +++ b/test/fixtures/vl53_const/README.md @@ -0,0 +1,11 @@ +These fixtures preserve Adafruit_VL53L0X 1.2.5's two tuning tables and the +complete `VL53L0X_load_tuning_settings` function, with their ST license notices. +The loader's hardware write function is replaced only in the native harness. + +Source: https://github.com/adafruit/Adafruit_VL53L0X/tree/1.2.5/src + +`test_nrf52_vl53l0x_const.py` compares every register write and internal tuning +parameter before and after const qualification, including hardware failures. +The production middleware fingerprints the five complete affected files and +compiles a private library copy. Unknown dependency revisions use their ordinary +source; the downloaded library is never modified. diff --git a/test/fixtures/vl53_const/tuning_loader.inc b/test/fixtures/vl53_const/tuning_loader.inc new file mode 100644 index 000000000..67a4b0a9e --- /dev/null +++ b/test/fixtures/vl53_const/tuning_loader.inc @@ -0,0 +1,109 @@ +/******************************************************************************* + Copyright 2016, STMicroelectronics International N.V. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of STMicroelectronics nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND + NON-INFRINGEMENT OF INTELLECTUAL PROPERTY RIGHTS ARE DISCLAIMED. + IN NO EVENT SHALL STMICROELECTRONICS INTERNATIONAL N.V. BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + ******************************************************************************/ + +VL53L0X_Error VL53L0X_load_tuning_settings(VL53L0X_DEV Dev, + uint8_t* pTuningSettingBuffer) { + VL53L0X_Error Status = VL53L0X_ERROR_NONE; + int i; + int Index; + uint8_t msb; + uint8_t lsb; + uint8_t SelectParam; + uint8_t NumberOfWrites; + uint8_t Address; + uint8_t localBuffer[4]; /* max */ + uint16_t Temp16; + + LOG_FUNCTION_START(""); + + Index = 0; + + while ((*(pTuningSettingBuffer + Index) != 0) && + (Status == VL53L0X_ERROR_NONE)) { + NumberOfWrites = *(pTuningSettingBuffer + Index); + Index++; + if (NumberOfWrites == 0xFF) { + /* internal parameters */ + SelectParam = *(pTuningSettingBuffer + Index); + Index++; + switch (SelectParam) { + case 0: /* uint16_t SigmaEstRefArray -> 2 bytes */ + msb = *(pTuningSettingBuffer + Index); + Index++; + lsb = *(pTuningSettingBuffer + Index); + Index++; + Temp16 = VL53L0X_MAKEUINT16(lsb, msb); + PALDevDataSet(Dev, SigmaEstRefArray, Temp16); + break; + case 1: /* uint16_t SigmaEstEffPulseWidth -> 2 bytes */ + msb = *(pTuningSettingBuffer + Index); + Index++; + lsb = *(pTuningSettingBuffer + Index); + Index++; + Temp16 = VL53L0X_MAKEUINT16(lsb, msb); + PALDevDataSet(Dev, SigmaEstEffPulseWidth, Temp16); + break; + case 2: /* uint16_t SigmaEstEffAmbWidth -> 2 bytes */ + msb = *(pTuningSettingBuffer + Index); + Index++; + lsb = *(pTuningSettingBuffer + Index); + Index++; + Temp16 = VL53L0X_MAKEUINT16(lsb, msb); + PALDevDataSet(Dev, SigmaEstEffAmbWidth, Temp16); + break; + case 3: /* uint16_t targetRefRate -> 2 bytes */ + msb = *(pTuningSettingBuffer + Index); + Index++; + lsb = *(pTuningSettingBuffer + Index); + Index++; + Temp16 = VL53L0X_MAKEUINT16(lsb, msb); + PALDevDataSet(Dev, targetRefRate, Temp16); + break; + default: /* invalid parameter */ + Status = VL53L0X_ERROR_INVALID_PARAMS; + } + + } else if (NumberOfWrites <= 4) { + Address = *(pTuningSettingBuffer + Index); + Index++; + + for (i = 0; i < NumberOfWrites; i++) { + localBuffer[i] = *(pTuningSettingBuffer + Index); + Index++; + } + + Status = VL53L0X_WriteMulti(Dev, Address, localBuffer, NumberOfWrites); + + } else { + Status = VL53L0X_ERROR_INVALID_PARAMS; + } + } + + LOG_FUNCTION_END(Status); + return Status; +} diff --git a/test/fixtures/vl53_const/vl53l0x_interrupt_threshold_settings.h b/test/fixtures/vl53_const/vl53l0x_interrupt_threshold_settings.h new file mode 100644 index 000000000..64aba5ecf --- /dev/null +++ b/test/fixtures/vl53_const/vl53l0x_interrupt_threshold_settings.h @@ -0,0 +1,81 @@ +/******************************************************************************* +Copyright 2016, STMicroelectronics International N.V. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of STMicroelectronics nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +NON-INFRINGEMENT OF INTELLECTUAL PROPERTY RIGHTS ARE DISCLAIMED. +IN NO EVENT SHALL STMICROELECTRONICS INTERNATIONAL N.V. BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#ifndef _VL53L0X_INTERRUPT_THRESHOLD_SETTINGS_H_ +#define _VL53L0X_INTERRUPT_THRESHOLD_SETTINGS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +uint8_t InterruptThresholdSettings[] = { + + /* Start of Interrupt Threshold Settings */ + 0x1, 0xff, 0x00, 0x1, 0x80, 0x01, 0x1, 0xff, 0x01, 0x1, 0x00, 0x00, + 0x1, 0xff, 0x01, 0x1, 0x4f, 0x02, 0x1, 0xFF, 0x0E, 0x1, 0x00, 0x03, + 0x1, 0x01, 0x84, 0x1, 0x02, 0x0A, 0x1, 0x03, 0x03, 0x1, 0x04, 0x08, + 0x1, 0x05, 0xC8, 0x1, 0x06, 0x03, 0x1, 0x07, 0x8D, 0x1, 0x08, 0x08, + 0x1, 0x09, 0xC6, 0x1, 0x0A, 0x01, 0x1, 0x0B, 0x02, 0x1, 0x0C, 0x00, + 0x1, 0x0D, 0xD5, 0x1, 0x0E, 0x18, 0x1, 0x0F, 0x12, 0x1, 0x10, 0x01, + 0x1, 0x11, 0x82, 0x1, 0x12, 0x00, 0x1, 0x13, 0xD5, 0x1, 0x14, 0x18, + 0x1, 0x15, 0x13, 0x1, 0x16, 0x03, 0x1, 0x17, 0x86, 0x1, 0x18, 0x0A, + 0x1, 0x19, 0x09, 0x1, 0x1A, 0x08, 0x1, 0x1B, 0xC2, 0x1, 0x1C, 0x03, + 0x1, 0x1D, 0x8F, 0x1, 0x1E, 0x0A, 0x1, 0x1F, 0x06, 0x1, 0x20, 0x01, + 0x1, 0x21, 0x02, 0x1, 0x22, 0x00, 0x1, 0x23, 0xD5, 0x1, 0x24, 0x18, + 0x1, 0x25, 0x22, 0x1, 0x26, 0x01, 0x1, 0x27, 0x82, 0x1, 0x28, 0x00, + 0x1, 0x29, 0xD5, 0x1, 0x2A, 0x18, 0x1, 0x2B, 0x0B, 0x1, 0x2C, 0x28, + 0x1, 0x2D, 0x78, 0x1, 0x2E, 0x28, 0x1, 0x2F, 0x91, 0x1, 0x30, 0x00, + 0x1, 0x31, 0x0B, 0x1, 0x32, 0x00, 0x1, 0x33, 0x0B, 0x1, 0x34, 0x00, + 0x1, 0x35, 0xA1, 0x1, 0x36, 0x00, 0x1, 0x37, 0xA0, 0x1, 0x38, 0x00, + 0x1, 0x39, 0x04, 0x1, 0x3A, 0x28, 0x1, 0x3B, 0x30, 0x1, 0x3C, 0x0C, + 0x1, 0x3D, 0x04, 0x1, 0x3E, 0x0F, 0x1, 0x3F, 0x79, 0x1, 0x40, 0x28, + 0x1, 0x41, 0x1E, 0x1, 0x42, 0x2F, 0x1, 0x43, 0x87, 0x1, 0x44, 0x00, + 0x1, 0x45, 0x0B, 0x1, 0x46, 0x00, 0x1, 0x47, 0x0B, 0x1, 0x48, 0x00, + 0x1, 0x49, 0xA7, 0x1, 0x4A, 0x00, 0x1, 0x4B, 0xA6, 0x1, 0x4C, 0x00, + 0x1, 0x4D, 0x04, 0x1, 0x4E, 0x01, 0x1, 0x4F, 0x00, 0x1, 0x50, 0x00, + 0x1, 0x51, 0x80, 0x1, 0x52, 0x09, 0x1, 0x53, 0x08, 0x1, 0x54, 0x01, + 0x1, 0x55, 0x00, 0x1, 0x56, 0x0F, 0x1, 0x57, 0x79, 0x1, 0x58, 0x09, + 0x1, 0x59, 0x05, 0x1, 0x5A, 0x00, 0x1, 0x5B, 0x60, 0x1, 0x5C, 0x05, + 0x1, 0x5D, 0xD1, 0x1, 0x5E, 0x0C, 0x1, 0x5F, 0x3C, 0x1, 0x60, 0x00, + 0x1, 0x61, 0xD0, 0x1, 0x62, 0x0B, 0x1, 0x63, 0x03, 0x1, 0x64, 0x28, + 0x1, 0x65, 0x10, 0x1, 0x66, 0x2A, 0x1, 0x67, 0x39, 0x1, 0x68, 0x0B, + 0x1, 0x69, 0x02, 0x1, 0x6A, 0x28, 0x1, 0x6B, 0x10, 0x1, 0x6C, 0x2A, + 0x1, 0x6D, 0x61, 0x1, 0x6E, 0x0C, 0x1, 0x6F, 0x00, 0x1, 0x70, 0x0F, + 0x1, 0x71, 0x79, 0x1, 0x72, 0x00, 0x1, 0x73, 0x0B, 0x1, 0x74, 0x00, + 0x1, 0x75, 0x0B, 0x1, 0x76, 0x00, 0x1, 0x77, 0xA1, 0x1, 0x78, 0x00, + 0x1, 0x79, 0xA0, 0x1, 0x7A, 0x00, 0x1, 0x7B, 0x04, 0x1, 0xFF, 0x04, + 0x1, 0x79, 0x1D, 0x1, 0x7B, 0x27, 0x1, 0x96, 0x0E, 0x1, 0x97, 0xFE, + 0x1, 0x98, 0x03, 0x1, 0x99, 0xEF, 0x1, 0x9A, 0x02, 0x1, 0x9B, 0x44, + 0x1, 0x73, 0x07, 0x1, 0x70, 0x01, 0x1, 0xff, 0x01, 0x1, 0x00, 0x01, + 0x1, 0xff, 0x00, 0x00, 0x00, 0x00}; + +#ifdef __cplusplus +} +#endif + +#endif /* _VL53L0X_INTERRUPT_THRESHOLD_SETTINGS_H_ */ diff --git a/test/fixtures/vl53_const/vl53l0x_tuning.h b/test/fixtures/vl53_const/vl53l0x_tuning.h new file mode 100644 index 000000000..94cbb7507 --- /dev/null +++ b/test/fixtures/vl53_const/vl53l0x_tuning.h @@ -0,0 +1,87 @@ +/******************************************************************************* +Copyright 2016, STMicroelectronics International N.V. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of STMicroelectronics nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND +NON-INFRINGEMENT OF INTELLECTUAL PROPERTY RIGHTS ARE DISCLAIMED. +IN NO EVENT SHALL STMICROELECTRONICS INTERNATIONAL N.V. BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*******************************************************************************/ + +#ifndef _VL53L0X_TUNING_H_ +#define _VL53L0X_TUNING_H_ + +#include "vl53l0x_def.h" + +#ifdef __cplusplus +extern "C" { +#endif + +uint8_t DefaultTuningSettings[] = { + + /* update 02/11/2015_v36 */ + 0x01, 0xFF, 0x01, 0x01, 0x00, 0x00, + + 0x01, 0xFF, 0x00, 0x01, 0x09, 0x00, 0x01, 0x10, 0x00, 0x01, 0x11, 0x00, + + 0x01, 0x24, 0x01, 0x01, 0x25, 0xff, 0x01, 0x75, 0x00, + + 0x01, 0xFF, 0x01, 0x01, 0x4e, 0x2c, 0x01, 0x48, 0x00, 0x01, 0x30, 0x20, + + 0x01, 0xFF, 0x00, 0x01, 0x30, 0x09, /* mja changed from 0x64. */ + 0x01, 0x54, 0x00, 0x01, 0x31, 0x04, 0x01, 0x32, 0x03, 0x01, 0x40, 0x83, + 0x01, 0x46, 0x25, 0x01, 0x60, 0x00, 0x01, 0x27, 0x00, 0x01, 0x50, 0x06, + 0x01, 0x51, 0x00, 0x01, 0x52, 0x96, 0x01, 0x56, 0x08, 0x01, 0x57, 0x30, + 0x01, 0x61, 0x00, 0x01, 0x62, 0x00, 0x01, 0x64, 0x00, 0x01, 0x65, 0x00, + 0x01, 0x66, 0xa0, + + 0x01, 0xFF, 0x01, 0x01, 0x22, 0x32, 0x01, 0x47, 0x14, 0x01, 0x49, 0xff, + 0x01, 0x4a, 0x00, + + 0x01, 0xFF, 0x00, 0x01, 0x7a, 0x0a, 0x01, 0x7b, 0x00, 0x01, 0x78, 0x21, + + 0x01, 0xFF, 0x01, 0x01, 0x23, 0x34, 0x01, 0x42, 0x00, 0x01, 0x44, 0xff, + 0x01, 0x45, 0x26, 0x01, 0x46, 0x05, 0x01, 0x40, 0x40, 0x01, 0x0E, 0x06, + 0x01, 0x20, 0x1a, 0x01, 0x43, 0x40, + + 0x01, 0xFF, 0x00, 0x01, 0x34, 0x03, 0x01, 0x35, 0x44, + + 0x01, 0xFF, 0x01, 0x01, 0x31, 0x04, 0x01, 0x4b, 0x09, 0x01, 0x4c, 0x05, + 0x01, 0x4d, 0x04, + + 0x01, 0xFF, 0x00, 0x01, 0x44, 0x00, 0x01, 0x45, 0x20, 0x01, 0x47, 0x08, + 0x01, 0x48, 0x28, 0x01, 0x67, 0x00, 0x01, 0x70, 0x04, 0x01, 0x71, 0x01, + 0x01, 0x72, 0xfe, 0x01, 0x76, 0x00, 0x01, 0x77, 0x00, + + 0x01, 0xFF, 0x01, 0x01, 0x0d, 0x01, + + 0x01, 0xFF, 0x00, 0x01, 0x80, 0x01, 0x01, 0x01, 0xF8, + + 0x01, 0xFF, 0x01, 0x01, 0x8e, 0x01, 0x01, 0x00, 0x01, 0x01, 0xFF, 0x00, + 0x01, 0x80, 0x00, + + 0x00, 0x00, 0x00}; + +#ifdef __cplusplus +} +#endif + +#endif /* _VL53L0X_TUNING_H_ */ diff --git a/test/test_nrf52_flash_trim.py b/test/test_nrf52_flash_trim.py index 402f3e5dd..09e8ae260 100644 --- a/test/test_nrf52_flash_trim.py +++ b/test/test_nrf52_flash_trim.py @@ -134,6 +134,12 @@ class Nrf52FlashTrimTest(unittest.TestCase): def BuildSources(self, *args): self.setdefault("BUILT_SOURCES", []).append(args) + def subst(self, value): + return value.replace("$PROJECT_DIR", str(ROOT)) + + def AddBuildMiddleware(self, callback, pattern): + self.setdefault("MIDDLEWARE", []).append((callback, pattern)) + base = dict(BUILD_FLAGS=["-DUSE_CC310_HW_CRYPTO=1"], SRC_FILTER=["+<../examples/simple_repeater>"], CPPDEFINES=[]) excluded = BuildEnvironment(**base, PIOENV="RAK_4631_companion_radio_full") @@ -148,6 +154,7 @@ class Nrf52FlashTrimTest(unittest.TestCase): self.assertTrue(active["MESH_NRF52_FLASH_TRIM_ACTIVE"]) self.assertIn("-Wl,--wrap=CRYS_ECPKI_GetEcDomain", active["LINKFLAGS"]) self.assertEqual(active["BUILT_SOURCES"][0][-1], "+") + self.assertEqual(active["MIDDLEWARE"][0][1], "*Adafruit_VL53L0X*/*.cpp") @unittest.skipUnless(shutil.which("cc"), "a native C compiler is required") def test_wrapped_domain_preserves_p256_and_rejects_unused_curves(self): diff --git a/test/test_nrf52_vl53l0x_const.py b/test/test_nrf52_vl53l0x_const.py new file mode 100644 index 000000000..cacc1f095 --- /dev/null +++ b/test/test_nrf52_vl53l0x_const.py @@ -0,0 +1,190 @@ +"""Qualify flash-resident VL53L0X tables against the reviewed vendor loader.""" + +import contextlib +import importlib.util +import io +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = ROOT / "test/fixtures/vl53_const" +SPEC = importlib.util.spec_from_file_location("vl53_const", ROOT / "scripts/nrf52_vl53l0x_const.py") +TRIM = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(TRIM) + + +class Node: + def __init__(self, path): + self.path = Path(path) + + def srcnode(self): + return self + + def get_abspath(self): + return str(self.path) + + +class Environment: + def __init__(self, build): + self.build = Path(build) + + def subst(self, value): + assert value == "$BUILD_DIR" + return str(self.build) + + def File(self, path): + return Path(path) + + +class Vl53ConstTests(unittest.TestCase): + def test_private_copy_is_complete_idempotent_and_downloads_are_unchanged(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "libdeps/Adafruit_VL53L0X/src" + texts = {} + for name, (old, _) in TRIM.REPLACEMENTS.items(): + texts[name] = "// synthetic reviewed fixture\n" + old + "\n" + texts["vl53l0x_def.h"] = "// ordinary sibling header\n" + texts["core/src/vl53l0x_api_calibration.cpp"] = '#include "../../vl53l0x_api_core.h"\n' + for name, text in texts.items(): + path = source / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(text.replace("\n", "\r\n").encode()) + before = {name: (source / name).read_bytes() for name in texts} + node = Node(source / "core/src/vl53l0x_api_calibration.cpp") + env = Environment(root / "build") + hashes = {name: TRIM.digest(texts[name]) for name in TRIM.FILES} + with patch.object(TRIM, "FILES", hashes): + output = TRIM.replace_vl53_source(env, node) + repeated = TRIM.replace_vl53_source(env, node) + self.assertEqual(output, repeated) + copied = root / "build/readonly-nrf52-vl53l0x/src" + self.assertEqual(output, copied / "core/src/vl53l0x_api_calibration.cpp") + self.assertEqual({name: (source / name).read_bytes() for name in texts}, before) + for name, text in texts.items(): + if name in TRIM.REPLACEMENTS: + old, new = TRIM.REPLACEMENTS[name] + self.assertEqual((copied / name).read_text(), text.replace(old, new)) + else: + self.assertEqual((copied / name).read_bytes(), before[name]) + # One changed table, loader, or prototype keeps every original TU. + for changed in TRIM.FILES: + path = source / changed + saved = path.read_bytes() + path.write_bytes(saved + b"// custom revision\n") + with patch.object(TRIM, "FILES", hashes), contextlib.redirect_stdout(io.StringIO()): + self.assertIs(TRIM.replace_vl53_source(env, node), node) + path.write_bytes(saved) + + def test_real_vendor_tables_produce_identical_register_writes_and_parameters(self): + original = (FIXTURE / "tuning_loader.inc").read_text() + mutable = original.replace("VL53L0X_load_tuning_settings", "load_original") + readonly = original.replace("VL53L0X_load_tuning_settings", "load_readonly").replace( + *TRIM.REPLACEMENTS["core/src/vl53l0x_api_core.cpp"]) + tables = "\n".join((FIXTURE / name).read_text().replace(*TRIM.REPLACEMENTS[name]) + .replace('#include "vl53l0x_def.h"', '') + for name in ("vl53l0x_tuning.h", "vl53l0x_interrupt_threshold_settings.h")) + source = r''' +#include +#include +#include +#include +#include +struct Device { + unsigned SigmaEstRefArray=0, SigmaEstEffPulseWidth=0, SigmaEstEffAmbWidth=0, targetRefRate=0; + unsigned calls=0, fail=0; + std::vector writes; +}; +using VL53L0X_Error = int; +using VL53L0X_DEV = Device*; +#define VL53L0X_ERROR_NONE 0 +#define VL53L0X_ERROR_INVALID_PARAMS 1 +#define LOG_FUNCTION_START(...) +#define LOG_FUNCTION_END(...) +#define PALDevDataSet(device, member, value) ((device)->member = (value)) +#define VL53L0X_MAKEUINT16(low, high) (uint16_t(low) | (uint16_t(high) << 8)) +int VL53L0X_WriteMulti(Device* device, uint8_t address, uint8_t* bytes, unsigned count) { + device->writes.push_back(address); device->writes.push_back(uint8_t(count)); + device->writes.insert(device->writes.end(),bytes,bytes+count); + return ++device->calls == device->fail ? 2 : 0; +} +@TABLES@ +@MUTABLE@ +@READONLY@ +static_assert(std::is_const::type>::value, + "default tuning must live in readonly storage"); +int main() { + const struct { const uint8_t* bytes; unsigned size; } inputs[] = { + {DefaultTuningSettings,sizeof(DefaultTuningSettings)}, + {InterruptThresholdSettings,sizeof(InterruptThresholdSettings)}, + }; + assert(inputs[0].size == 243 && inputs[1].size == 438); + for (const auto& input:inputs) { + for(unsigned fail=0;fail copy(input.bytes,input.bytes+input.size); + Device before, after; before.fail=after.fail=fail; + assert(load_original(&before,copy.data()) == load_readonly(&after,input.bytes)); + assert(before.calls==after.calls && before.writes==after.writes); + assert(before.SigmaEstRefArray==after.SigmaEstRefArray); + assert(before.SigmaEstEffPulseWidth==after.SigmaEstEffPulseWidth); + assert(before.SigmaEstEffAmbWidth==after.SigmaEstEffAmbWidth); + assert(before.targetRefRate==after.targetRefRate); + assert(memcmp(copy.data(),input.bytes,input.size)==0); + } + } + // The caller-owned custom tuning API keeps accepting ordinary mutable data. + uint8_t custom[] = {0xFF,0,0x01,0x23,2,0x10,0xAA,0xBB,0}; + uint8_t snapshot[sizeof(custom)]; memcpy(snapshot,custom,sizeof(custom)); + Device before,after; + assert(load_original(&before,custom)==load_readonly(&after,custom)); + assert(before.SigmaEstRefArray==after.SigmaEstRefArray && before.writes==after.writes); + assert(memcmp(custom,snapshot,sizeof(custom))==0); +} +'''.replace("@TABLES@", tables).replace("@MUTABLE@", mutable).replace("@READONLY@", readonly) + with tempfile.TemporaryDirectory() as directory: + path, binary = Path(directory) / "loader.cpp", Path(directory) / "loader" + path.write_text(source, encoding="ascii") + built = subprocess.run(["g++", "-std=c++17", "-O1", "-Wall", "-Wextra", "-Werror", + "-fsanitize=address,undefined", "-fno-pie", "-no-pie", + str(path), "-o", str(binary)], capture_output=True, text=True, timeout=30) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + subprocess.run([str(binary)], check=True, timeout=10) + + def test_arm_table_bytes_move_to_readonly_without_changing_their_size(self): + compiler = shutil.which("arm-none-eabi-g++") + nm = shutil.which("arm-none-eabi-nm") + if not compiler or not nm: + self.skipTest("ARM toolchain required") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for readonly in (False, True): + text = "#include \n" + for name in ("vl53l0x_tuning.h", "vl53l0x_interrupt_threshold_settings.h"): + table = (FIXTURE / name).read_text().replace('#include "vl53l0x_def.h"', '') + if readonly: + table = table.replace(*TRIM.REPLACEMENTS[name]) + text += table + "\n" + text += ''' +extern "C" const uint8_t* default_table() { return DefaultTuningSettings; } +extern "C" const uint8_t* threshold_table() { return InterruptThresholdSettings; } +''' + path, obj = root / "tables.cpp", root / "tables.o" + path.write_text(text, encoding="ascii") + built = subprocess.run([compiler,"-std=c++17","-mcpu=cortex-m4","-mthumb","-Os", + "-c",str(path),"-o",str(obj)],capture_output=True,text=True,timeout=30) + self.assertEqual(built.returncode,0,built.stdout+built.stderr) + symbols = subprocess.check_output([nm,"-S","-C",str(obj)],text=True) + for name,size in (("DefaultTuningSettings",243),("InterruptThresholdSettings",438)): + line = next(line for line in symbols.splitlines() if line.endswith(" "+name)) + _, length, kind, _ = line.split() + self.assertEqual(int(length,16),size) + self.assertEqual(kind.lower(),"r" if readonly else "d") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_ota/test_ota_core.cpp b/test/test_ota/test_ota_core.cpp index 390a4b657..1d7cb28ed 100644 --- a/test/test_ota/test_ota_core.cpp +++ b/test/test_ota/test_ota_core.cpp @@ -4319,6 +4319,49 @@ TEST(OtaTransfer, RejectsShortNonFinalManifestFragment) { EXPECT_EQ(client.fetchState(), OtaManager::FETCHING); } +TEST(OtaTransfer, FixedManifestStoragePreservesEveryValidShapeAndRejectsOtherShapes) { + MotaManifest manifest; + ASSERT_TRUE(mota_parse(SIM_MOTA, SIM_MOTA_LEN, manifest)); + const uint8_t totals[] = {0,1,2,3,4,5,16,255}; + const uint8_t indexes[] = {0,1,2,3,4,15,254,255}; + for (uint8_t total : totals) for (uint8_t index : indexes) { + for (unsigned length = 0; length <= MAX_PACKET_PAYLOAD - 7; ++length) { + SCOPED_TRACE(::testing::Message() << "total=" << unsigned(total) + << " index=" << unsigned(index) << " length=" << length); + g_q.clear(); + OtaManager client; + OtaStoreRam<4096> store; + SendTo to_none{&client}; + client.begin(SIM_TARGET_ID, sim_send, &to_none); + client.set_fetch_store(&store); + ASSERT_EQ(client.pull(manifest.merkle_root, manifest.target_id), OtaManager::PULL_STARTED); + g_q.clear(); + const uint16_t final_length = MOTA_MFL - OTA_MF_FRAG; + const uint8_t other = index == 0 ? 1 : 0; + deliver_manifest_fragment(client, manifest.merkle_root, other, + manifest.manifest_start + (other ? OTA_MF_FRAG : 0), + other ? final_length : OTA_MF_FRAG); + ASSERT_EQ(client.fetchState(), OtaManager::WANT_MANIFEST); + uint8_t bytes[MAX_PACKET_PAYLOAD] = {}; + if (index == 0) memcpy(bytes, manifest.manifest_start, OTA_MF_FRAG); + else if (index == 1) memcpy(bytes, manifest.manifest_start + OTA_MF_FRAG, final_length); + ManifestMsg fragment{}; + memcpy(fragment.manifest_id, manifest.merkle_root, 4); + fragment.frag_idx = index; fragment.frag_total = total; + fragment.bytes = bytes; fragment.len = uint16_t(length); + uint8_t wire[MAX_PACKET_PAYLOAD]; + const auto wire_length = encode_manifest(wire, sizeof(wire), fragment); + ASSERT_GT(wire_length, 0); + client.on_message(wire, wire_length); + const bool valid = total == 2 && ((index == 0 && length == OTA_MF_FRAG) + || (index == 1 && length == final_length)); + EXPECT_EQ(client.fetchState(), valid ? OtaManager::FETCHING : OtaManager::WANT_MANIFEST); + if (!valid) EXPECT_EQ(store.staged_size(), 0u); + } + } + g_q.clear(); +} + TEST(OtaTransfer, RejectsManifestBlockExponentBeforeShift) { g_q.clear(); MotaManifest manifest; diff --git a/test/test_ota_memory_layout.py b/test/test_ota_memory_layout.py index b05c12895..f0643d3b3 100644 --- a/test/test_ota_memory_layout.py +++ b/test/test_ota_memory_layout.py @@ -21,25 +21,55 @@ class OtaMemoryLayoutTest(unittest.TestCase): #include #include using namespace mesh::ota; -static_assert(sizeof(OtaManager::ServeEntry) == (sizeof(void*) == 4 ? 60u : 64u), - "source entries must retain every field without old padding"); +static_assert(sizeof(OtaManager::ServeEntry) == (sizeof(void*) == 4 ? 32u : 40u), + "source entries must retain all independent descriptor fields once"); static_assert(alignof(OtaManager::ServeEntry) >= alignof(MotaSource*), "unaligned source pointer"); -static_assert(offsetof(OtaManager::ServeEntry, desc) % alignof(MotaDesc) == 0, "unaligned descriptor"); static_assert(offsetof(OtaManager::ServeEntry, target_id) % alignof(uint32_t) == 0, "unaligned word"); static_assert(sizeof(OtaStoreQspiNrf52) > 8192u, "both staging page buffers must remain"); int main() { OtaManager::ServeEntry row; - assert(row.desc.target_id == 0 && row.desc.fw_version == 0 && row.desc.source_caps == 0); - assert(row.desc.mid[0] == 0 && row.desc.mid[3] == 0 && row.desc.total_size == 0); + const auto empty = row.descriptor(); + assert(empty.target_id == 0 && empty.fw_version == 0 && empty.source_caps == 0); + assert(empty.mid[0] == 0 && empty.mid[3] == 0 && empty.total_size == 0); row.mid[0] = 17; row.target_id = 123; row.fw_version = 456; row.have_count = 789; row.src = nullptr; row.src_idx = 255; row.codec_id = 2; row.flags = 3; row.is_self = true; - row.desc.target_id = 987; row.desc.payload_size = 654; + row.payload_size = 654; row.block_size_log2 = 11; row.source_caps = 3; const OtaManager::ServeEntry copy = row; assert(copy.mid[0] == 17 && copy.target_id == 123 && copy.fw_version == 456); assert(copy.have_count == 789 && copy.src == nullptr && copy.src_idx == 255); assert(copy.codec_id == 2 && copy.flags == 3 && copy.is_self); - assert(copy.desc.target_id == 987 && copy.desc.payload_size == 654); + const auto descriptor = copy.descriptor(); + assert(descriptor.target_id == 123 && descriptor.payload_size == 654); + assert(descriptor.block_count == 789 && descriptor.block_size_log2 == 11 && descriptor.source_caps == 3); + assert(descriptor.leaves_off == 8 + MOTA_MFL); + assert(descriptor.payload_off == descriptor.leaves_off + 789 * 4); + assert(descriptor.total_size == descriptor.payload_off + 654 + 5); + // Every descriptor admitted by refresh_sources() has this canonical layout. + // Keep arbitrary metadata/caps/large payload values, not just current images. + uint32_t random = 0x6d76a345; + for (unsigned trial = 0; trial != 10000; ++trial) { + auto next = [&]() { random ^= random << 13; random ^= random >> 17; random ^= random << 5; return random; }; + MotaDesc original; + for (auto& byte : original.mid) byte = uint8_t(next()); + original.target_id = next(); original.fw_version = next(); + original.codec_id = uint8_t(next()); original.flags = uint8_t(next()); + original.block_size_log2 = uint8_t(next()); original.source_caps = uint8_t(next()); + original.block_count = 1 + next() % (UINT32_MAX / 8); + original.leaves_off = 8 + MOTA_MFL; + original.payload_off = original.leaves_off + original.block_count * 4; + original.payload_size = next() % (UINT32_MAX - original.payload_off - 5); + original.total_size = original.payload_off + original.payload_size + 5; + row.cacheDescriptor(original); + const auto roundtrip = row.descriptor(); + assert(memcmp(original.mid, roundtrip.mid, 4) == 0); + assert(original.target_id == roundtrip.target_id && original.fw_version == roundtrip.fw_version); + assert(original.codec_id == roundtrip.codec_id && original.flags == roundtrip.flags); + assert(original.block_size_log2 == roundtrip.block_size_log2 && original.source_caps == roundtrip.source_caps); + assert(original.block_count == roundtrip.block_count && original.payload_size == roundtrip.payload_size); + assert(original.leaves_off == roundtrip.leaves_off && original.payload_off == roundtrip.payload_off); + assert(original.total_size == roundtrip.total_size); + } } ''' with tempfile.TemporaryDirectory(prefix="ota-layout-") as directory: From c650278fa7a680d097d73cdae5e7ede8454cb898 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Tue, 27 Oct 2026 08:17:23 -0700 Subject: [PATCH 4/6] Share constrained STM32 code while preserving firmware features Omit unbound forwarding-region callbacks only for exclusive Companion source roles. Reuse the compact neighbor sorter in repeaters for every RPC order and the CLI list, removing duplicate sort implementations while preserving storage and size gates. --- examples/simple_repeater/MyMesh.cpp | 23 +++--- scripts/client_acl_role.py | 43 +++++++++- src/helpers/RadioProfileCLI.cpp | 4 +- src/helpers/RadioProfileCLI.h | 6 +- test/test_client_acl_role.py | 30 +++++++ test/test_radio_profile_cli.py | 19 +++-- test/test_small_table_sort.py | 119 ++++++++++++++++++++++++++++ 7 files changed, 216 insertions(+), 28 deletions(-) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 5d770a7a2..00afb61de 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -60,9 +60,10 @@ mesh::FloodAdvertLimiter::Entry* repeaterRtcAdvertSlowBank() { #endif #include #include +#include #include #include // std::nothrow (heap-allocated flood rule table) -#include // for qsort() +#include #include #include #if defined(NRF52_POWER_MANAGEMENT) @@ -707,7 +708,7 @@ void MyMesh::sendDefaultPasswordWarning(ClientInfo* client, } } -// Comparison functions for qsort() - defined at file scope to avoid heap allocations +// Shared neighbour ordering; equally ranked entries have no guaranteed tie order. static int cmp_neighbours_newest_to_oldest(const void* a, const void* b) { const NeighbourInfo* na = *(const NeighbourInfo**)a; const NeighbourInfo* nb = *(const NeighbourInfo**)b; @@ -942,21 +943,20 @@ int MyMesh::handleRequest(ClientInfo *sender, uint32_t sender_timestamp, uint8_t } } - // Sort neighbours based on order using qsort() - standard C library function - // qsort() doesn't allocate heap memory (uses stack-based recursion) and is O(n log n) - // This matches the pattern used elsewhere in the codebase (e.g., BaseChatMesh) + // Share the bounded O(n log n) sorter instead of retaining separate C + // and C++ sort implementations on flash-constrained boards. if (order_by == 0) { // sort by newest to oldest - qsort(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_newest_to_oldest); + mesh::sortSmallTable(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_newest_to_oldest); } else if (order_by == 1) { // sort by oldest to newest - qsort(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_oldest_to_newest); + mesh::sortSmallTable(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_oldest_to_newest); } else if (order_by == 2) { // sort by strongest to weakest - qsort(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_strongest_to_weakest); + mesh::sortSmallTable(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_strongest_to_weakest); } else if (order_by == 3) { // sort by weakest to strongest - qsort(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_weakest_to_strongest); + mesh::sortSmallTable(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), cmp_neighbours_weakest_to_strongest); } // build results buffer @@ -6023,9 +6023,8 @@ void MyMesh::formatNeighborsReply(char *reply) { } // sort neighbours newest to oldest - std::sort(sorted_neighbours, sorted_neighbours + neighbours_count, [](const NeighbourInfo* a, const NeighbourInfo* b) { - return a->heard_timestamp > b->heard_timestamp; // desc - }); + mesh::sortSmallTable(sorted_neighbours, neighbours_count, sizeof(NeighbourInfo*), + cmp_neighbours_newest_to_oldest); for (int i = 0; i < neighbours_count && dp - reply < 134; i++) { NeighbourInfo *neighbour = sorted_neighbours[i]; diff --git a/scripts/client_acl_role.py b/scripts/client_acl_role.py index e7dc98786..995884297 100644 --- a/scripts/client_acl_role.py +++ b/scripts/client_acl_role.py @@ -1,12 +1,14 @@ -"""Use the smaller ClientInfo layout only for an effective repeater source role. +"""Derive feature-preserving role definitions from effective application sources. -One shared compiler definition covers application and helper translation units. +Shared compiler definitions cover application and helper translation units. Unknown/custom roles keep the complete layout; room and sensor session fields -are never removed from builds that use them. +are never removed from builds that use them. Companions omit only unbound +forwarding-region callbacks because that role has no RegionMap. """ import importlib.util from pathlib import Path +import re SCRIPT_DIR = Path(__file__).resolve().parent if "__file__" in globals() else Path("scripts").resolve() @@ -18,6 +20,39 @@ DEFINITION = "MESH_CLIENT_REPEATER_ONLY" REPEATER_SOURCES = ("../examples/simple_repeater/main.cpp", "../examples/simple_repeater/MyMesh.cpp") OTHER_ROLES = ("simple_room_server", "simple_sensor", "companion_radio", "simple_secure_chat", "kiss_modem", "partition_expander", "esp32_partition_migrator", "esp32_partition_legacy_seed") +COMPANION_DEFINITION = "MESH_ROLE_COMPANION" +COMPANION_SOURCES = ("../examples/companion_radio/main.cpp", "../examples/companion_radio/MyMesh.cpp") + + +def companion_role_policy(src_filter, flags=(), cppdefines=()): + """Exclude unbound forwarding-scope callbacks, retaining all radio commands. + + Companions have no RegionMap. Select the complete exclusive source role, + never an environment-name alias, so every translation unit shares its ABI. + Unknown and mixed roles retain the general callback implementation. + """ + values = SOURCE_POLICY.flag_defines(flags, cppdefines) + selected = SOURCE_POLICY.source_selected + other_roles = ("simple_repeater", *(role for role in OTHER_ROLES + if role != "companion_radio")) + filter_text = src_filter if isinstance(src_filter, str) else " ".join(map(str, src_filter)) + example_includes = re.findall(r"\+<\.\./examples/([^>]+)>", filter_text.replace("\\", "/")) + known_companion_only = all(path == "companion_radio" or path.startswith("companion_radio/") + for path in example_includes) + qualified = (known_companion_only + and all(selected(src_filter, source) for source in COMPANION_SOURCES) + and not any(selected(src_filter, "../examples/" + role + "/" + source) + for role in other_roles + for source in ("main.cpp", "MyMesh.cpp", "SensorMesh.cpp"))) + if COMPANION_DEFINITION in values: + value = values[COMPANION_DEFINITION] + text = "0" if value is None else str(value).strip("\"'() ") + if text not in ("0", "1"): + raise ValueError(COMPANION_DEFINITION + " override must be 0 or 1") + if text == "1" and not qualified: + raise ValueError(COMPANION_DEFINITION + "=1 requires only the Companion source role") + return {} + return {COMPANION_DEFINITION: 1} if qualified else {} def role_policy(src_filter, flags=(), cppdefines=()): @@ -46,6 +81,8 @@ def install(build_env): src_filter = build_env.GetProjectOption("build_src_filter", "") definitions = role_policy(src_filter, build_env.get("BUILD_FLAGS", ()), build_env.get("CPPDEFINES", ())) + definitions.update(companion_role_policy(src_filter, build_env.get("BUILD_FLAGS", ()), + build_env.get("CPPDEFINES", ()))) if definitions: build_env.AppendUnique(CPPDEFINES=list(definitions.items())) diff --git a/src/helpers/RadioProfileCLI.cpp b/src/helpers/RadioProfileCLI.cpp index 5127ab7ea..4b790912b 100644 --- a/src/helpers/RadioProfileCLI.cpp +++ b/src/helpers/RadioProfileCLI.cpp @@ -92,7 +92,7 @@ bool secondaryOverlaps(uint32_t primary_hz, float secondary_mhz, uint32_t cleara return (primary_hz > secondary_hz ? primary_hz - secondary_hz : secondary_hz - primary_hz) <= clearance_hz; } -#if !defined(COMPANION_RADIO_FULL) +#if !defined(COMPANION_RADIO_FULL) && !MESH_ROLE_COMPANION bool isMediaMeshProfile(const RadioProfileConfig& config) { return config.mode != RadioProfileMode::Off && config.params.freq >= 911.1495f && config.params.freq <= 911.1505f @@ -102,7 +102,7 @@ bool isMediaMeshProfile(const RadioProfileConfig& config) { #endif } -#if !defined(COMPANION_RADIO_FULL) +#if !defined(COMPANION_RADIO_FULL) && !MESH_ROLE_COMPANION void RadioProfileCLI::setMediaMeshScopeHandler(MediaMeshScopeHandler handler, void* context) { media_scope_handler_ = handler; media_scope_context_ = context; diff --git a/src/helpers/RadioProfileCLI.h b/src/helpers/RadioProfileCLI.h index 41e1b1dbb..5fe0bae40 100644 --- a/src/helpers/RadioProfileCLI.h +++ b/src/helpers/RadioProfileCLI.h @@ -61,7 +61,7 @@ class RadioProfileCLI { // readable backup. Read/I/O failures and an intact backup remain read-only. bool recoverable_corrupt_store_ = false; bool publish_pending_ = false; -#if !defined(COMPANION_RADIO_FULL) +#if !defined(COMPANION_RADIO_FULL) && !MESH_ROLE_COMPANION MediaMeshScopeHandler media_scope_handler_ = nullptr; void* media_scope_context_ = nullptr; uint32_t media_scope_retry_ms_ = 0; @@ -71,7 +71,7 @@ class RadioProfileCLI { void retryMediaMeshScope(); bool mediaMeshScopePending() const { return media_scope_pending_; } #else - // Full Companions have no forwarding RegionMap. Do not retain unreachable + // Companions have no forwarding RegionMap. Do not retain unreachable // infrastructure callbacks, state or failure text in their tight images. bool mediaMeshScopeReady(const RadioProfileConfig&) const { return true; } void updateMediaMeshScope() {} @@ -141,7 +141,7 @@ class RadioProfileCLI { #endif void begin(FILESYSTEM* fs, Radio* radio, RTCClock* rtc, bool infrastructure_replies = false); // Bind only after RegionMap::load(), so boot reconciliation is not discarded. -#if !defined(COMPANION_RADIO_FULL) +#if !defined(COMPANION_RADIO_FULL) && !MESH_ROLE_COMPANION void setMediaMeshScopeHandler(MediaMeshScopeHandler handler, void* context); #else void setMediaMeshScopeHandler(MediaMeshScopeHandler, void*) {} diff --git a/test/test_client_acl_role.py b/test/test_client_acl_role.py index 7f23fa0dc..46be48bdc 100644 --- a/test/test_client_acl_role.py +++ b/test/test_client_acl_role.py @@ -19,6 +19,33 @@ FIXTURE = ROOT / "test/fixtures/client_acl_spiffs" class ClientAclRoleTests(unittest.TestCase): + def test_complete_exclusive_companion_role_uses_one_global_definition(self): + definition = POLICY.COMPANION_DEFINITION + for role in ("+<../examples/companion_radio>", + "+<../examples/companion_radio/*.cpp>", + ["+<*.cpp>", "+<..\\examples\\companion_radio\\*.cpp>"]): + self.assertEqual(POLICY.companion_role_policy(role), {definition: 1}) + for source in ("", "+<*>", "+<../examples/companion_radio/main.cpp>", + "+<../examples/companion_radio> -<../examples/companion_radio/MyMesh.cpp>", + "+<../examples/companion_radio> +<../examples/custom_role>", + "+<../examples/simple_repeater>"): + self.assertEqual(POLICY.companion_role_policy(source), {}) + for other in ("simple_repeater", *POLICY.OTHER_ROLES): + if other == "companion_radio": + continue + self.assertEqual(POLICY.companion_role_policy( + "+<../examples/companion_radio> +<../examples/" + other + ">"), {}) + role = "+<../examples/companion_radio>" + for flag in ("-D" + definition + "=0", "-D" + definition + "=1", "-U" + definition): + self.assertEqual(POLICY.companion_role_policy(role, flag), {}) + self.assertEqual(POLICY.companion_role_policy(role, cppdefines=[(definition, 1)]), {}) + for value in ("true", "2", "-1"): + with self.assertRaises(ValueError): + POLICY.companion_role_policy(role, "-D" + definition + "=" + value) + with self.assertRaises(ValueError): + POLICY.companion_role_policy("+<../examples/simple_repeater>", + "-D" + definition + "=1") + def test_only_complete_repeater_role_qualifies(self): for value in ("+<../examples/simple_repeater>", "+<../examples/simple_repeater/*.cpp>", ["+<*.cpp>", "+<..\\examples\\simple_repeater\\*.cpp>"]): @@ -57,6 +84,9 @@ class ClientAclRoleTests(unittest.TestCase): del env["appended"] POLICY.install(env) self.assertNotIn("appended", env) + env["SRC_FILTER"] = "+<../examples/companion_radio/*.cpp>" + POLICY.install(env) + self.assertEqual(env["appended"], {"CPPDEFINES": [(POLICY.COMPANION_DEFINITION, 1)]}) def test_every_platform_base_installs_role_hook(self): ini = (ROOT / "platformio.ini").read_text() diff --git a/test/test_radio_profile_cli.py b/test/test_radio_profile_cli.py index 01d6d49b6..4166d7512 100644 --- a/test/test_radio_profile_cli.py +++ b/test/test_radio_profile_cli.py @@ -9,7 +9,7 @@ import unittest ROOT = Path(__file__).resolve().parents[1] class RadioProfileCLITest(unittest.TestCase): - def test_full_companion_keeps_rf_presets_without_unused_region_code(self): + def test_all_companions_keep_rf_presets_without_unused_region_code(self): compiler = shutil.which('g++') or shutil.which('clang++') self.assertIsNotNone(compiler) code = r''' @@ -23,7 +23,7 @@ int main() { f.cmd("set radio2 mediamesh"); f.advance(2000); assert(f.radio.p.enabled() && fabsf(f.radio.p.secondary.params.freq - 911.150f) < 0.0001f); -#if defined(COMPANION_RADIO_FULL) +#if defined(COMPANION_RADIO_FULL) || MESH_ROLE_COMPANION assert(!scope.commits && !scope.checks); #else assert(scope.enabled && scope.durable && scope.commits); @@ -37,9 +37,11 @@ int main() { with tempfile.TemporaryDirectory(prefix='profile-region-role-') as work: source = Path(work)/'role.cpp' source.write_text(code, encoding='ascii') - for full in (False, True): - binary = Path(work)/('companion' if full else 'infrastructure') - flags = ['-DCOMPANION_RADIO_FULL=1'] if full else [] + roles = {'infrastructure': [], 'full': ['-DCOMPANION_RADIO_FULL=1'], + 'ordinary': ['-DMESH_ROLE_COMPANION=1']} + for role, flags in roles.items(): + companion = role != 'infrastructure' + binary = Path(work)/role compiled = subprocess.run([ compiler, '-std=c++17', '-Os', '-ffunction-sections', '-fdata-sections', '-Wl,--gc-sections', @@ -53,11 +55,12 @@ int main() { self.assertEqual(compiled.returncode, 0, compiled.stderr) run = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10) self.assertEqual(run.returncode, 0, run.stdout+run.stderr) - sizes[full] = int(run.stdout) + sizes[role] = int(run.stdout) image = binary.read_bytes() for marker in (b'MediaMesh scope save pending', b'MediaMesh region unavailable or full'): - self.assertEqual(marker in image, not full) - self.assertLess(sizes[True], sizes[False]) + self.assertEqual(marker in image, not companion) + for role in ('full', 'ordinary'): + self.assertLess(sizes[role], sizes['infrastructure']) def test_commands_persistence_and_timers(self): compiler = shutil.which('g++') or shutil.which('clang++') diff --git a/test/test_small_table_sort.py b/test/test_small_table_sort.py index de868fb17..91b73f58c 100644 --- a/test/test_small_table_sort.py +++ b/test/test_small_table_sort.py @@ -8,6 +8,8 @@ import sys import tempfile import unittest +from cpp_source import body + ROOT = Path(__file__).resolve().parents[1] HARNESS = r''' @@ -140,8 +142,125 @@ void sortElsewhere(void* items, size_t count, size_t width, } ''' +NEIGHBOUR_HARNESS = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +struct NeighbourInfo { + uint32_t heard_timestamp; + int8_t snr; + uint16_t identity; +}; +@COMPARATORS@ +static void rpcSort(NeighbourInfo** sorted_neighbours, int16_t neighbours_count, + uint8_t order_by) { + @RPC_SORT@ +} +static void cliSort(NeighbourInfo** sorted_neighbours, int16_t neighbours_count) { + @CLI_SORT@ +} +static bool before(const NeighbourInfo* a, const NeighbourInfo* b, unsigned order) { + if (order == 0) return a->heard_timestamp > b->heard_timestamp; + if (order == 1) return a->heard_timestamp < b->heard_timestamp; + if (order == 2) return a->snr > b->snr; + return a->snr < b->snr; +} +int main() { + std::mt19937 random(0x18180002); + const uint32_t timestamps[] = {1, 42, 42, UINT32_MAX, 1, UINT32_MAX - 1}; + const int8_t snrs[] = {INT8_MIN, -16, -1, 0, 0, 7, 7, INT8_MAX}; + int (*const compare[])(const void*, const void*) = { + cmp_neighbours_newest_to_oldest, cmp_neighbours_oldest_to_newest, + cmp_neighbours_strongest_to_weakest, cmp_neighbours_weakest_to_strongest + }; + for (size_t count : {0U, 1U, 2U, 16U, 50U, 100U}) { + std::vector records(count); + for (size_t i = 0; i < count; ++i) { + records[i].heard_timestamp = timestamps[i % 6]; + records[i].snr = snrs[i % 8]; + records[i].identity = static_cast(i); + } + const auto original = records; + for (unsigned order = 0; order < 4; ++order) { + std::vector actual(count), expected(count); + for (size_t i = 0; i < count; ++i) actual[i] = &records[i]; + std::shuffle(actual.begin(), actual.end(), random); + expected = actual; + std::sort(expected.begin(), expected.end(), [order](const NeighbourInfo* a, + const NeighbourInfo* b) { + return before(a, b, order); + }); + rpcSort(actual.data(), static_cast(count), order); + std::vector seen(count); + for (size_t i = 0; i < count; ++i) { + const auto index = actual[i]->identity; + assert(index < count && !seen[index] && actual[i] == &records[index]); + seen[index] = true; + assert(!before(actual[i], expected[i], order)); + assert(!before(expected[i], actual[i], order)); + assert(compare[order](&actual[i], &expected[i]) == 0); + if (i) assert(!before(actual[i], actual[i - 1], order)); + } + // Unsupported order values retain the caller's original pointer order. + const auto unchanged = actual; + rpcSort(actual.data(), static_cast(count), 4); + assert(actual == unchanged); + cliSort(actual.data(), static_cast(count)); + for (size_t i = 1; i < count; ++i) { + assert(actual[i - 1]->heard_timestamp >= actual[i]->heard_timestamp); + } + for (size_t i = 0; i < count; ++i) { + assert(records[i].heard_timestamp == original[i].heard_timestamp); + assert(records[i].snr == original[i].snr); + assert(records[i].identity == original[i].identity); + } + } + } + puts("PASS: real neighbour RPC orders and CLI, duplicate keys, negative SNR and pointer records"); +} +''' + class SmallTableSortTests(unittest.TestCase): + def test_real_neighbour_rpc_and_cli_orderings(self): + compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++") + self.assertIsNotNone(compiler, "a host C++17 compiler is required") + production = (ROOT / "examples/simple_repeater/MyMesh.cpp").read_text() + names = ("newest_to_oldest", "oldest_to_newest", + "strongest_to_weakest", "weakest_to_strongest") + comparators = "\n".join(body(production, "static int cmp_neighbours_" + name + "(") + for name in names) + start = production.index("if (order_by == 0) {") + rpc_sort = production[start:production.index("// build results buffer", start)] + formatter = body(production, "void MyMesh::formatNeighborsReply(") + start = formatter.index("// sort neighbours newest to oldest") + cli_sort = formatter[start:formatter.index("for (int i = 0;", start)] + source = NEIGHBOUR_HARNESS.replace("@COMPARATORS@", comparators) + source = source.replace("@RPC_SORT@", rpc_sort).replace("@CLI_SORT@", cli_sort) + for platform in ("STM32_PLATFORM", "NRF52_PLATFORM", "ESP32_PLATFORM", "RP2040_PLATFORM"): + with self.subTest(platform=platform), tempfile.TemporaryDirectory( + prefix="neighbour-sort-") as directory: + work = Path(directory) + (work / "test.cpp").write_text(source) + binary = work / "sort" + command = [compiler, "-std=c++17", "-O2", "-Wall", "-Wextra", "-Werror", + "-D" + platform + "=1", "-I" + str(ROOT / "src"), + str(work / "test.cpp"), "-o", str(binary)] + if sys.platform.startswith("linux"): + command[1:1] = ["-fsanitize=address,undefined", "-fno-sanitize-recover=all", + "-fno-pie", "-no-pie"] + built = subprocess.run(command, capture_output=True, text=True, timeout=60) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + tested = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10) + self.assertEqual(tested.returncode, 0, tested.stdout + tested.stderr) + self.assertIn("negative SNR and pointer records", tested.stdout) + def test_sort_on_all_platforms(self): compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++") self.assertIsNotNone(compiler, "a host C++17 compiler is required") From 0cba0c3afac365539400f481670d6e2c5516d391 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Tue, 27 Oct 2026 08:17:24 -0700 Subject: [PATCH 5/6] Refresh host fixtures for current Companion and USB interfaces Declare fixture feature boundaries and SDK mocks explicitly, retain USB recovery ownership checks, and recognize the unified V4 display and read-only diagnostics in contract tests. --- .../companion_delayed_reply_delivery/test.cpp | 3 +++ test/fixtures/morse_main_loop.cpp | 1 + test/test_carrier_wave.py | 2 ++ test/test_cli_settings_contract.py | 2 ++ test/test_companion_ota_includes.py | 8 ++++++++ test/test_companion_usb_default.py | 1 + test/test_ota_folder_ready.py | 6 +++++- test/test_ota_source_proofs.py | 4 +++- test/test_rp2040_application_capabilities.py | 1 + test/test_t096_message_footer_profile.py | 8 +++++--- test/test_usb_logging_hwcdc_watchdog.py | 17 ++++++++++++++--- test/test_usb_logging_watchdog_roles.py | 12 ++++++++++++ 12 files changed, 57 insertions(+), 8 deletions(-) diff --git a/test/fixtures/companion_delayed_reply_delivery/test.cpp b/test/fixtures/companion_delayed_reply_delivery/test.cpp index 1e7b1ff48..bd8b10820 100644 --- a/test/fixtures/companion_delayed_reply_delivery/test.cpp +++ b/test/fixtures/companion_delayed_reply_delivery/test.cpp @@ -17,6 +17,9 @@ #include #include +// This fixture models correlated RF replies. Direct USB OTA administration +// has a separate fixture and is disabled for every consumer of this harness. +#define COMPANION_FEATURE_DIRECT_OTA_ADMIN 0 #define MESH_DEBUG_PRINTLN(...) ((void)0) #include "production_constants.inc" #include "production_stream.inc" diff --git a/test/fixtures/morse_main_loop.cpp b/test/fixtures/morse_main_loop.cpp index d26ff575e..da6f1fb93 100644 --- a/test/fixtures/morse_main_loop.cpp +++ b/test/fixtures/morse_main_loop.cpp @@ -109,6 +109,7 @@ bool profileIdleSleepAllowed() { && (!the_mesh.isDualRadioActive() || (profileIdleQueryAllowed() && the_mesh.profile_idle_allowed)); } void serviceCommandInterfaces() { ++command_services; } +bool serviceCompanionHostKissUsb() { return false; } void serviceUsbLoggingOnlySession() { ++usb_services; } void serviceUsbTerminalHostSessionReset() { ++usb_services; } void serviceUsbAsciiSessionDefault() { ++usb_services; } diff --git a/test/test_carrier_wave.py b/test/test_carrier_wave.py index 2d95018bc..6da7c13a0 100644 --- a/test/test_carrier_wave.py +++ b/test/test_carrier_wave.py @@ -141,6 +141,7 @@ struct RadioLibWrapper : mesh::Radio { uint32_t rssi_latency=0; unsigned rssi_reads=0; bool _params_valid=true, _rx_ps_armed=true, _rx_hold_continuous=false; + bool _primary_receive_preempt=false; bool _rx_boosted_gain_valid=false, _cur_rx_boosted_gain=false, _rx_boosted_gain_applied=false; bool rx_gain_apply_ok=true, physical_rx_gain=false; unsigned rx_gain_applies=0,rx_gain_order=0; @@ -212,6 +213,7 @@ struct RadioLibWrapper : mesh::Radio { #endif virtual bool supportsRadioDeepInit() const { return deep_supported; } uint8_t beginReconfigure(); + void noteReconfigureAdmission(const char*, int16_t) {} void endReconfigure(bool resume) { if (resume) startRecv(); } bool isChipBusy() { return busy; } bool isReceivingPacket() { return packet; } diff --git a/test/test_cli_settings_contract.py b/test/test_cli_settings_contract.py index 6dfe8c0c1..d660eed2c 100644 --- a/test/test_cli_settings_contract.py +++ b/test/test_cli_settings_contract.py @@ -269,9 +269,11 @@ namespace mesh { struct Utils { 'flash.health', # Read-only flash health diagnostics 'key.backup.transport', # Identity-backup transport capability 'mqtt', 'mqtt.running', # MQTT connection status + 'ota.direct', # Direct USB OTA session diagnostics 'password', 'prv.key', # Local secret/identity reads 'public.key', # Read-only public identity 'pwrmgt.bootreason', 'role', # Boot/build facts + 'radio.apply', # Pending/applied radio diagnostics 'radio.rxps.config', # Detail view of radio.rxps 'tx.status', # HIL-only applied-power verification 'wifi.status', 'wifi.ip', # Live WiFi link state diff --git a/test/test_companion_ota_includes.py b/test/test_companion_ota_includes.py index 9a01bc510..8d2039ebc 100644 --- a/test/test_companion_ota_includes.py +++ b/test/test_companion_ota_includes.py @@ -54,6 +54,14 @@ class CompanionOtaIncludeTest(unittest.TestCase): (directory / "MyMesh.h").write_text( '#pragma once\n#include "CompanionFeatures.h"\n' ) + # Host KISS declares its filesystem boundary in Full images. Keep + # that SDK type visible without pulling a board filesystem into + # this include-only test of the real OTA headers. + identity = directory / "helpers/IdentityStore.h" + identity.write_text( + "#pragma once\nclass IncludeTestFilesystem;\n" + "#define FILESYSTEM IncludeTestFilesystem\n" + ) if not (usb or ble): # Prove that small/non-OTA images do not gain these dependencies. for header in ("MotaSourceSerial.h", "OtaContext.h"): diff --git a/test/test_companion_usb_default.py b/test/test_companion_usb_default.py index 43eeba0c8..555d3bcc2 100644 --- a/test/test_companion_usb_default.py +++ b/test/test_companion_usb_default.py @@ -68,6 +68,7 @@ static constexpr unsigned USB_HOST_LOSS_EDGE_MS = 100, USB_HOST_LOSS_GRACE_MS = struct Stream { int pending = 0; int available() const { return pending; } + int peek() const { return pending > 0 ? 'x' : -1; } int read() { return pending-- > 0 ? 'x' : -1; } void print(const char*) {} } usb_stream; diff --git a/test/test_ota_folder_ready.py b/test/test_ota_folder_ready.py index eaac88266..f36f63409 100644 --- a/test/test_ota_folder_ready.py +++ b/test/test_ota_folder_ready.py @@ -21,6 +21,9 @@ class OtaFolderReadyTest(unittest.TestCase): self.assertIsNotNone(match) defines.append("#define " + name + " " + match[1]) source = (ROOT / "test/fixtures/ota_folder_ready.cpp").read_text(encoding="ascii") + # This folder-readiness fixture has no competing raw USB host session. + # Keep the real Arduino declaration visible to MotaSourceSerial.cpp. + source += "\nnamespace mesh { bool isUsbBinaryStreamOwned() { return false; } }\n" source = source.replace("// @PRODUCTION_DEFINES@", "\n".join(defines)) ready = extract_braced(production, "if (ota::ota_ctx().manager.takeSourceReadyAnnouncement())") source = source.replace("// @PRODUCTION_READY@", "" if missing_ready_trigger else ready) @@ -38,7 +41,8 @@ class OtaFolderReadyTest(unittest.TestCase): with self.subTest(definitions=definitions): sources = ("OtaManager.cpp", "OtaProtocol.cpp", "MotaContainer.cpp", "MerkleTree.cpp", "MotaSourceSerial.cpp") result = subprocess.run([compiler, "-std=c++17", "-O1", *sanitizers, - "-DENABLE_OTA=1", "-DOTA_SEEDER_ONLY=1", *["-D" + value for value in definitions], + "-DARDUINO=10819", "-DENABLE_OTA=1", "-DOTA_SEEDER_ONLY=1", + *["-D" + value for value in definitions], "-I", str(ROOT / "src"), "-I", str(ROOT / "test/mocks"), str(cpp), *[str(ROOT / "src/helpers/ota" / name) for name in sources], str(ROOT / "src/Utils.cpp"), "-o", str(executable)], capture_output=True, text=True, timeout=90) diff --git a/test/test_ota_source_proofs.py b/test/test_ota_source_proofs.py index 42cca597f..526fc5012 100644 --- a/test/test_ota_source_proofs.py +++ b/test/test_ota_source_proofs.py @@ -73,6 +73,8 @@ static_assert(OTA_PROOFGEN_SCRATCH / 4 == 4095, "odd byte limit lost usable leaf #include #include using namespace mesh::ota; +// This serial-source capacity test has no competing USB stream owner. +namespace mesh { bool isUsbBinaryStreamOwned() { return false; } } @FAKE@ class MtuSource : public SerialMotaSource { public: @@ -110,7 +112,7 @@ int main() { built = subprocess.run([ os.environ.get("CXX", "g++"), "-std=c++17", "-fsanitize=address,undefined", "-fno-sanitize-recover=all", "-fno-pie", "-no-pie", - "-DESP32_PLATFORM=1", "-DCOMPANION_RADIO_FULL=1", + "-DARDUINO=10819", "-DESP32_PLATFORM=1", "-DCOMPANION_RADIO_FULL=1", "-I", str(ROOT / "src"), "-I", str(ROOT / "test/mocks"), str(source), str(ROOT / "src/helpers/ota/MotaSourceSerial.cpp"), "-o", str(binary), ], capture_output=True, text=True, timeout=60) diff --git a/test/test_rp2040_application_capabilities.py b/test/test_rp2040_application_capabilities.py index 7eb16fb48..50218305f 100644 --- a/test/test_rp2040_application_capabilities.py +++ b/test/test_rp2040_application_capabilities.py @@ -171,6 +171,7 @@ class RP2040ApplicationCapabilityTests(unittest.TestCase): set -euo pipefail source build.sh pio() { echo 'PlatformIO must not run' >&2; return 99; } +PIO_CONFIG_JSON='[]' PIO_BUILD_DIR_OVERRIDE=@BUILD@ OUTPUT_DIR=@OUTPUT@ BUILD_PROFILE_FOR_TARGET=auto diff --git a/test/test_t096_message_footer_profile.py b/test/test_t096_message_footer_profile.py index af2ce85e0..43640586b 100644 --- a/test/test_t096_message_footer_profile.py +++ b/test/test_t096_message_footer_profile.py @@ -54,6 +54,7 @@ class T096MessageFooterProfileTest(unittest.TestCase): for display_class in ( "SH1106Display", "SSD1306Display", + "HeltecV4Display", "ST7735Display", "U8g2Display", ): @@ -103,6 +104,7 @@ class T096MessageFooterProfileTest(unittest.TestCase): for display_class in ( "SH1106Display", "SSD1306Display", + "HeltecV4Display", "ST7735Display", "U8g2Display", ): @@ -127,7 +129,7 @@ class T096MessageFooterProfileTest(unittest.TestCase): self.assertNotIn("MESHCORE_HAS_REAL_DISPLAY", null_macros) self.assertNotIn("MESHCORE_HAS_SMALL_DISPLAY", null_macros) - def test_v4_oled_is_compact_but_v4_tft_is_not(self): + def test_v4_display_aliases_share_the_unified_compact_driver(self): profile = V4_PROFILE.read_text(encoding="utf-8") usb = profile.split( "[env:heltec_v4_companion_radio_usb_femon]", 1 @@ -136,9 +138,9 @@ class T096MessageFooterProfileTest(unittest.TestCase): "[env:heltec_v4_tft_companion_radio_usb_femon]", 1 )[1].split("[", 1)[0] self.assertIn("extends = heltec_v4_oled", usb) - self.assertIn("DISPLAY_CLASS=SSD1306Display", usb) + self.assertIn("DISPLAY_CLASS=HeltecV4Display", usb) self.assertIn("extends = heltec_v4_tft", tft_usb) - self.assertIn("DISPLAY_CLASS=ST7789LCDDisplay", tft_usb) + self.assertIn("DISPLAY_CLASS=HeltecV4Display", tft_usb) if __name__ == "__main__": diff --git a/test/test_usb_logging_hwcdc_watchdog.py b/test/test_usb_logging_hwcdc_watchdog.py index 03cea3705..7fbf1414f 100644 --- a/test/test_usb_logging_hwcdc_watchdog.py +++ b/test/test_usb_logging_hwcdc_watchdog.py @@ -43,13 +43,24 @@ class HwcdcLoggingWatchdogTests(unittest.TestCase): cls.companion_methods = "\n".join(body(companion, signature) for signature in ( "static void clearUsbTerminalLine()", "static void cancelUsbSerialOperations()", "static void resetUsbTerminalHostSession()", "static void serviceUsbTerminalHostSessionReset()")) - cls.ordinary_methods = "" + # There is no functional text backlog in the peripheral fake. Keep + # the actual role calls and subsequent session-completion gate below. + cls.ordinary_methods = "namespace mesh { void serviceUsbTerminalPort() {} }\n" for role in ("simple_repeater", "simple_room_server", "simple_sensor"): text = (ROOT / "examples" / role / "main.cpp").read_text() - start = text.index(" if (mesh::takeUsbTerminalSessionReset())") + # The reset body now contains nested feature guards. Extract the + # complete compound statement, preserving every guarded operation. + reset = body(text, " if (mesh::takeUsbTerminalSessionReset())") + start = text.index(" if (usb_ready) mesh::serviceUsbTerminalPort();") stop = text.index("\n#endif", start) + completion = text[start:stop] + # This separate block follows optional KISS admission in the real + # loop. Do not silently truncate it if nested guards are added. + assert "#if" not in completion + assert "mesh::tryCompleteUsbTerminalSessionReset()" in completion cls.ordinary_methods += ("static void service_" + role + "() {\n" - " bool usb_ready = true;\n" + text[start:stop] + "\n (void)usb_ready;\n}\n") + " bool usb_ready = true;\n" + reset + "\n" + completion + + "\n (void)usb_ready;\n}\n") cls.binaries = {(backport, ota): cls.compile_backend(backport, ota) for backport in (0, 1) for ota in (False, True)} diff --git a/test/test_usb_logging_watchdog_roles.py b/test/test_usb_logging_watchdog_roles.py index 8b29ead50..84f3d5d4a 100644 --- a/test/test_usb_logging_watchdog_roles.py +++ b/test/test_usb_logging_watchdog_roles.py @@ -30,6 +30,10 @@ namespace mesh { struct Status {bool reader_connected=true, stalled=false;}; static Status status; Status usbLoggingStatus(){return status;} + namespace hostkiss { + static bool usb_active=false; + bool usbActive(){return usb_active;} + } } struct Store { bool dirty=false; @@ -91,6 +95,7 @@ bool isUsbTerminalDataConnected(){return usb_connected;} void reset(){ board=Board{};radio_driver=RadioDriver{};mesh::status=mesh::Status{}; + mesh::hostkiss::usb_active=false; the_mesh=RoleMesh{}; // Assignment copies the fixture's dependency pointers; point them back at // the live fixture objects, not the temporary used to reset scalar state. @@ -243,6 +248,13 @@ class UsbLoggingWatchdogRoleTests(unittest.TestCase): the_mesh._serial=&the_mesh.serial; ''' companion_checks = r''' + // A host KISS session owns the same physical USB transport regardless of + // the logging reader, terminal feature or ordinary Companion USB mode. + for(unsigned state=0;state<3;++state){ + reset();mesh::status.reader_connected=state!=1;mesh::status.stalled=state==2; + mesh::hostkiss::usb_active=true;assert(!usbLoggingRecoverySafe(nullptr)); + mesh::hostkiss::usb_active=false;assert(usbLoggingRecoverySafe(nullptr)); + } #if defined(ENABLE_USB_INTERFACE) // A live binary client or buffered protocol transaction is always protected, // even if the dedicated logging CDC is disconnected or its output is stuck. From 308fec1162e77134d15206b819dd59e898ed910b Mon Sep 17 00:00:00 2001 From: mikecarper Date: Tue, 27 Oct 2026 10:45:24 -0700 Subject: [PATCH 6/6] Keep only local OTA target names in reduced RAK builds --- build_legacy.sh | 6 ++++++ docs/firmware_picker.md | 3 +++ scripts/nrf52_flash_trim.py | 22 +++++++++++++++++++++- test/test_build_profiles.sh | 10 ++++++++++ test/test_nrf52_flash_trim.py | 20 ++++++++++++++++++++ test/test_ota_target_names.py | 25 +++++++++++++++++++++++++ variants/rak3401/platformio.ini | 2 +- variants/rak4631/platformio.ini | 3 +++ 8 files changed, 89 insertions(+), 2 deletions(-) diff --git a/build_legacy.sh b/build_legacy.sh index ff3d69999..2d70f1c54 100755 --- a/build_legacy.sh +++ b/build_legacy.sh @@ -4120,6 +4120,12 @@ apply_lora_ota_no_external_sensors_profile() { for flag in $voltage_monitor_flags; do retain_overrides+=" -D${flag}=1" done + # Keep only this image's readable OTA name in the reduced RAK profiles. + # A non-OTA-prefixed marker is deliberately not copied from a reduced + # logical recipe into the Full sensor pass by apply_logical_ota_tuning_flags. + retain_overrides+=" -DMESH_OTA_LOCAL_TARGET_NAME_ONLY=1" + record_build_reduction \ + "ota target-name catalog omitted; local target name and numeric OTA matching retained" export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS}${omit_overrides}${retain_overrides}" if is_rak_gps_retaining_ota_target "$env_name"; then record_build_reduction \ diff --git a/docs/firmware_picker.md b/docs/firmware_picker.md index f9e1ea8c9..97e783693 100644 --- a/docs/firmware_picker.md +++ b/docs/firmware_picker.md @@ -581,6 +581,9 @@ The legacy `no_external_sensors` profile is labeled **Receives LoRa OTA - Reduced optional sensors** on the same button. On RAK3401 and RAK4631 this compact profile uses internal storage without an external storage board. The reduction does not disable generic I2C or unrelated board-integrated peripherals. +These reduced RAK profiles keep their own readable OTA target name and omit +the all-board name catalog. Updates for other targets display numeric IDs; +OTA compatibility checks and transfers are unchanged. Reduced RAK3401 and RAK4631 targets retain INA219/INA226/INA260/INA3221 as voltage/current entries in the optional sensor table. They are not the only I2C users: the SSD1306 OLED, supported autodiscovered RTCs, and RAK12500 GPS remain diff --git a/scripts/nrf52_flash_trim.py b/scripts/nrf52_flash_trim.py index a7116487f..90a48c3f4 100644 --- a/scripts/nrf52_flash_trim.py +++ b/scripts/nrf52_flash_trim.py @@ -141,6 +141,20 @@ def trim_policy(platform, mcu, env_name, src_filter, flags=(), cppdefines=(), if ota_name_mode != "front-coded": raise ValueError("unsupported nRF52 OTA name trim policy: " + ota_name_mode) definitions = dict(TRIM_DEFINES, OTA_TARGET_NAME_FRONT_CODED=1) + if _enabled(values.get("MESH_OTA_LOCAL_TARGET_NAME_ONLY")): + # Release overlays can compile a different PIOENV from their deployed + # OTA identity. Keep the logical name/ID in that case; direct builds + # use the same PIOENV hash as the ordinary EndF/board identity code. + target_id = values.get("MOTA_TARGET_ID") + if target_id is None: + target_id = "0x%08x" % int.from_bytes( + hashlib.sha256(str(env_name).encode()).digest()[:4], "little") + definitions.update( + OTA_TARGET_NAME_TABLE=0, + OTA_LOCAL_TARGET_ID=target_id, + OTA_LOCAL_TARGET_NAME="OTA_VARIANT" if values.get("OTA_VARIANT") + else '"%s"' % env_name, + ) # Explicit board/user flags retain their value. In particular never undo # an opt-out or a board-specific USB configuration by re-defining it. return {name: value for name, value in definitions.items() if name not in values} @@ -241,6 +255,9 @@ def install(build_env): build_env.get("SRC_FILTER", ()), flags, cppdefines) if not definitions: return + local_name = definitions.get("OTA_LOCAL_TARGET_NAME", "") + if local_name.startswith('"') and local_name.endswith('"'): + definitions["OTA_LOCAL_TARGET_NAME"] = build_env.StringifyMacro(local_name[1:-1]) build_env["MESH_NRF52_FLASH_TRIM_ACTIVE"] = True build_env.AppendUnique(CPPDEFINES=list(definitions.items())) hook_path = build_env.subst("$PROJECT_DIR/scripts/nrf52_vl53l0x_const.py") @@ -249,7 +266,10 @@ def install(build_env): hook_spec.loader.exec_module(hook) hook.install(build_env) if infrastructure: - print("nRF52 infrastructure flash trim: compact crypto, unused USB device classes/GPS cache/vendor splash, compact OTA target names") + names = ("local OTA target name only" if definitions.get("OTA_TARGET_NAME_TABLE") == 0 + or flag_defines(flags, cppdefines).get("OTA_TARGET_NAME_TABLE") == "0" + else "compact OTA target names") + print("nRF52 infrastructure flash trim: compact crypto, unused USB device classes/GPS cache/vendor splash, " + names) else: print("nRF52 Companion flash trim: small crypto table, unused USB device classes/GPS cache/vendor splash; fonts and Reader retained") hook_path = build_env.subst("$PROJECT_DIR/scripts/nrf52_gnss_version_trim.py") diff --git a/test/test_build_profiles.sh b/test/test_build_profiles.sh index 4b62241ce..05fd4c398 100644 --- a/test/test_build_profiles.sh +++ b/test/test_build_profiles.sh @@ -1900,11 +1900,17 @@ verify_reduced_sensor_flags() { [[ "$PLATFORMIO_BUILD_FLAGS" == *-UENV_INCLUDE_BME280* ]] \ || fail "$env_name did not omit environmental sensors" if [ "$expect_ina" = 1 ]; then + [[ "$PLATFORMIO_BUILD_FLAGS" == *-DMESH_OTA_LOCAL_TARGET_NAME_ONLY=1* ]] \ + || fail "$env_name retained the all-board OTA name catalog" + [[ " ${BUILD_REDUCTIONS[*]} " == *"ota target-name catalog omitted"* ]] \ + || fail "$env_name did not declare its OTA name reduction" [[ "$PLATFORMIO_BUILD_FLAGS" == *-DENV_INCLUDE_INA219=1* ]] \ || fail "$env_name did not retain INA monitor support" [[ "$PLATFORMIO_BUILD_FLAGS" != *-UENV_INCLUDE_INA219* ]] \ || fail "$env_name both retained and omitted INA monitor support" else + [[ "$PLATFORMIO_BUILD_FLAGS" != *MESH_OTA_LOCAL_TARGET_NAME_ONLY* ]] \ + || fail "$env_name lost OTA names outside the reduced RAK profile" [[ "$PLATFORMIO_BUILD_FLAGS" == *-UENV_INCLUDE_INA219* ]] \ || fail "$env_name incorrectly retained INA monitor support" fi @@ -1914,6 +1920,10 @@ verify_reduced_sensor_flags \ RAK_3401_repeater_lora_ota_no_external_sensors 1 verify_reduced_sensor_flags \ RAK_3401_room_server_lora_ota_no_external_sensors 1 +verify_reduced_sensor_flags \ + RAK_4631_repeater_lora_ota_no_external_sensors 1 +verify_reduced_sensor_flags \ + RAK_4631_sensor_lora_ota_no_external_sensors 1 verify_reduced_sensor_flags \ Heltec_t114_repeater_lora_ota_no_external_sensors 0 diff --git a/test/test_nrf52_flash_trim.py b/test/test_nrf52_flash_trim.py index 09e8ae260..d1cd69251 100644 --- a/test/test_nrf52_flash_trim.py +++ b/test/test_nrf52_flash_trim.py @@ -67,6 +67,26 @@ class Nrf52FlashTrimTest(unittest.TestCase): self.assertNotIn("SSD1306_NO_SPLASH", defines) self.assertEqual(TRIM.flag_defines("-DMESH_NRF52_FLASH_TRIM=1 -UMESH_NRF52_FLASH_TRIM")["MESH_NRF52_FLASH_TRIM"], None) + def test_local_names_honor_overlay_identity_and_explicit_defines(self): + flags = ('-DMESH_OTA_LOCAL_TARGET_NAME_ONLY=1 ' + '-DMOTA_TARGET_ID=0x29a0da19 ' + '-DOTA_VARIANT=\'"RAK_4631_repeater_lora_ota_no_external_sensors"\'') + defines = self.policy(flags=flags) + self.assertEqual(defines["OTA_TARGET_NAME_TABLE"], 0) + self.assertEqual(defines["OTA_LOCAL_TARGET_ID"], "0x29a0da19") + self.assertEqual(defines["OTA_LOCAL_TARGET_NAME"], "OTA_VARIANT") + explicit = self.policy(flags=flags + ' -DOTA_TARGET_NAME_TABLE=1 ' + '-DOTA_LOCAL_TARGET_ID=7 -DOTA_LOCAL_TARGET_NAME=custom_name') + for name in ("OTA_TARGET_NAME_TABLE", "OTA_LOCAL_TARGET_ID", "OTA_LOCAL_TARGET_NAME"): + self.assertNotIn(name, explicit) + for flags in ("", "-DMESH_OTA_LOCAL_TARGET_NAME_ONLY=0"): + self.assertNotIn("OTA_TARGET_NAME_TABLE", self.policy(flags=flags)) + + def test_local_names_do_not_change_companion_or_unqualified_roles(self): + for options in (dict(name="RAK_4631_companion_radio_full"), + dict(role="custom"), dict(platform="espressif32", mcu="esp32")): + self.assertEqual(self.policy(flags="-DMESH_OTA_LOCAL_TARGET_NAME_ONLY=1", **options), {}) + def test_cc310_archive_is_exact_and_requires_supported_mcu_and_enabled_crypto(self): with tempfile.TemporaryDirectory() as temp_dir: archive = Path(temp_dir) / "libraries/Adafruit_nRFCrypto" / TRIM.CC310_ARCHIVE diff --git a/test/test_ota_target_names.py b/test/test_ota_target_names.py index 4405e8bf0..3e373ec2a 100644 --- a/test/test_ota_target_names.py +++ b/test/test_ota_target_names.py @@ -1,6 +1,8 @@ """All OTA names survive front coding; original Companion API stays unchanged.""" import re import os +import configparser +import importlib.util import shutil import subprocess import sys @@ -237,6 +239,29 @@ class OtaTargetNameTest(unittest.TestCase): f'-DOTA_LOCAL_TARGET_NAME="{local_name}"', ], ids), [local_name, "?", "?"]) + def test_reduced_rak_policy_compiles_with_exact_local_names_and_ids(self): + spec = importlib.util.spec_from_file_location("ota_name_trim", ROOT / "scripts/nrf52_flash_trim.py") + trim = importlib.util.module_from_spec(spec) + spec.loader.exec_module(trim) + for board, suffix in (("3401", ""), ("4631", ""), + ("4631", "_bridge_rs232_serial1"), ("4631", "_bridge_rs232_serial2")): + target = f"RAK_{board}_repeater{suffix}_lora_ota_no_external_sensors" + config = configparser.ConfigParser(interpolation=None, inline_comment_prefixes=(";",)) + config.read(ROOT / f"variants/rak{board}/platformio.ini") + flags = config["env:" + target]["build_flags"] + # Interpolated hardware flags do not influence the name policy. + flags = re.sub(r"\$\{[^}]+\}", "", flags) + target_id = gen_targets.ml.target_id_for_env(target) + definitions = trim.flag_defines(flags) + definitions.update(trim.trim_policy("nordicnrf52", "nrf52840", target, + "+<../examples/simple_repeater>", flags)) + name_flags = [f"-D{name}={value}" for name, value in definitions.items() + if name in ("OTA_TARGET_NAME_TABLE", "OTA_TARGET_NAME_FRONT_CODED", + "OTA_LOCAL_TARGET_ID", "OTA_LOCAL_TARGET_NAME")] + with self.subTest(target=target): + self.assertEqual(self.compile_and_lookup(name_flags, + [target_id, self.rows[0][0], 0]), [target, "?", "?"]) + def test_cpp_decoder_fails_closed_on_invalid_records_and_trailing_bytes(self): from zopfli_compress import raw_deflate_compress rows = gen_targets.target_rows(["hello"]) diff --git a/variants/rak3401/platformio.ini b/variants/rak3401/platformio.ini index aed1e4878..1b87797fd 100644 --- a/variants/rak3401/platformio.ini +++ b/variants/rak3401/platformio.ini @@ -89,7 +89,7 @@ build_flags = -D ENABLE_OTA=1 -D OTA_FLASH_STORE=1 -D OTA_FOLDER_SERIAL - -D OTA_TARGET_NAME_TABLE=0 ; IDs remain authoritative; omit the 540-entry display-name table + -D OTA_TARGET_NAME_TABLE=0 ; IDs remain authoritative; omit the all-board display-name catalog -D OTA_LOCAL_TARGET_ID=0x2fa509c1 -D OTA_LOCAL_TARGET_NAME='"RAK_3401_repeater_lora_ota_no_external_sensors"' -D OTA_FETCH_PIPELINE=4 ; adaptive one-packet flights grow 1 -> 2 -> 3 -> 4 on a clean link diff --git a/variants/rak4631/platformio.ini b/variants/rak4631/platformio.ini index a43f0225d..bf6221781 100644 --- a/variants/rak4631/platformio.ini +++ b/variants/rak4631/platformio.ini @@ -213,6 +213,7 @@ extra_scripts = ${rak4631_repeater_reduced_ota_base.extra_scripts} build_flags = ${rak4631_repeater_reduced_ota_base.build_flags} ${nrf52_rak4631_unified_ota.build_flags} + -D MESH_OTA_LOCAL_TARGET_NAME_ONLY=1 ; keep this target's name without the all-board catalog build_src_filter = ${rak4631_repeater_reduced_ota_base.build_src_filter} lib_deps = ${rak4631_repeater_reduced_ota_base.lib_deps} @@ -302,6 +303,7 @@ extends = rak4631 build_flags = ${rak4631.build_flags} ${nrf52_reduced_sensors_keep_ina.build_flags} + -D MESH_OTA_LOCAL_TARGET_NAME_ONLY=1 -D DISPLAY_CLASS=SSD1306Display -D ADVERT_NAME='"RS232 Bridge"' -D ADVERT_LAT=0.0 @@ -354,6 +356,7 @@ extends = rak4631 build_flags = ${rak4631.build_flags} ${nrf52_reduced_sensors_keep_ina_gps.build_flags} + -D MESH_OTA_LOCAL_TARGET_NAME_ONLY=1 -D DISPLAY_CLASS=SSD1306Display -D ADVERT_NAME='"RS232 Bridge"' -D ADVERT_LAT=0.0