diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 6fe065f1a..112fd5360 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -391,6 +391,9 @@ jobs: python3 -B test/test_flood_retry_bucket_cli.py -v python3 -B test/test_flood_advert_rtc.py -v python3 -B test/test_flood_advert_iram.py -v + python3 -B test/test_flood_advert_psram.py -v + python3 -B test/test_repeater_scope_psram.py -v + python3 -B test/test_ws_transport_psram.py -v python3 -B test/test_ota_security.py -v python3 -B test/test_companion_offline_delivery.py -v python3 -B test/test_room_receive_admission.py -v diff --git a/docs/WiFi.md b/docs/WiFi.md index 69d083a56..38ab3b4f0 100644 --- a/docs/WiFi.md +++ b/docs/WiFi.md @@ -886,6 +886,12 @@ send retries only that broker every 30 seconds; successful brokers keep their normal status interval. Each slot has a stable, distinct client ID, including when two slots point at the same broker. +G3 also places the SDK's two padded WebSocket buffers per initialized MQTT +client in PSRAM, saving 2,050 internal bytes per client when PSRAM allocation +succeeds. G3 repeaters move their complete scope tables and advert-limit +history there too. Capacities, saved formats, task stacks and DMA reserves +remain unchanged; safe internal fallback is retained. + Outbox diagnostics are cached by the network owner, so `get mqtt.stats` does not wait for an SDK mutex held by a network write. `rec:N/write` records a failed-write recovery; `no errors` describes the current connected session, diff --git a/docs/research/firmware_memory_budget.md b/docs/research/firmware_memory_budget.md index 46a615156..817c8b80e 100644 --- a/docs/research/firmware_memory_budget.md +++ b/docs/research/firmware_memory_budget.md @@ -190,6 +190,23 @@ other tasks. Available linked internal heap cannot establish how much DMA-capable heap remains or how it is fragmented after boot. Only runtime observations can establish that. +G3 repeater builds also place their channel/scope tables and full advert-source +history in PSRAM. Their capacities and persisted formats remain unchanged. +The RAM check still reserves internal fallback for both complete tables; +moving them out of BSS does not relax startup admission or the DMA reserve. +If neither heap can safely hold a complete table, startup reports the failed +allocation and stops rather than running with reduced filtering protection. +Other targets keep their existing storage. `MESH_REPEATER_PSRAM_TABLES=0` +restores inline storage for a custom G3 build. + +G3's IDF 4 WebSocket wrapper also prefers PSRAM for its two padded 1,025-byte +SDK buffers per initialized MQTT client, including clients using plain MQTT. +Five initialized clients move 10,250 bytes of payload from internal RAM. +Transport objects, task stacks, DMA buffers and radio packet queues keep their +existing placement. The existing extra-byte overflow fix and guarded internal +fallback remain in force. These figures describe allocation placement, not a +measurement of running free heap or qualification of five simultaneous brokers. + Maximum-load qualification still needs physical boot and sustained reconnect, publish and configuration-change tests with the selected number of brokers, transport schemes, certificates and enabled UART/browser/OTA features. Record diff --git a/docs/test-results/g3_psram_storage_2026-10-27.md b/docs/test-results/g3_psram_storage_2026-10-27.md new file mode 100644 index 000000000..4f3a12e93 --- /dev/null +++ b/docs/test-results/g3_psram_storage_2026-10-27.md @@ -0,0 +1,83 @@ + +

View this page on MeshCore Docs.

+ + +# G3 repeater PSRAM storage, 2026-10-27 + +The G3 Full repeater frees **36,776 bytes of static internal RAM** by moving +its channel/scope tables and advert-limit history to PSRAM. The MQTT SDK +WebSocket buffers also prefer PSRAM: **2,050 bytes per initialized client**, +or 10,250 bytes with five initialized clients. Capacities, saved formats and +radio queues remain unchanged. + +These are linked measurements and verified allocation sizes. No G3 was +flashed; running heap, RF timing and sustained broker load were not measured. +The [machine-readable evidence](g3_psram_storage_2026-10-27.safe.json) records +the matched builds, hashes and checks. + +## Matched builds + +Baseline: `88bfaebb5`. Final: the production changes accompanying this report. +Both use the same G3 Full recipe, version, Cascade profile and USA preset. + +| Measurement | Baseline | Final | +| --- | ---: | ---: | +| Internal BSS bytes | 142,088 | 105,312 | +| Internal data bytes | 23,980 | 23,980 | +| Linked internal heap capacity | 199,976 | 236,752 | +| Modeled internal requirement | 197,808 | 234,636 | +| Conservative admission margin | 2,168 | 2,116 | +| Packaged application bytes | 2,156,504 | 2,155,752 | +| OTA application-slot bytes | 6,553,600 | 6,553,600 | + +The model still reserves complete internal fallback for both tables: 18,892 +bytes for scope storage and 17,936 for advert history, including overhead. +That explains why the conservative admission margin does not rise with the +normal PSRAM-backed free heap. The 16 KiB DMA reserve and existing startup +requirements are unchanged. All 16 capability checks pass, with no declared +feature reductions; WiFi/LoRa OTA, MQTT, logging, WebConfig, bridges and host +KISS remain compiled in. + +## Allocation behavior + +- G3 repeater defaults select PSRAM for all 255 scope entries, 32 direct-scope + names, 255 scope requirements and 128 advert-source histories. +- Allocation happens once during construction. Array references preserve + addresses, complete-array `sizeof`, file formats and rollback behavior. +- The pinned ESP32-S3 SDK initializes PSRAM heaps before global constructors; + Arduino's later `psramFound()` flag is not used as a constructor-time gate. +- Complete internal fallback is allowed only while preserving the existing + DMA reserve. Otherwise startup reports the allocation failure and stops. + It never silently reduces filter or advert-protection capacity. +- `MESH_REPEATER_PSRAM_TABLES=0` restores inline storage. Other boards retain + their existing default storage, including RTC/IRAM paths. +- The IDF 4 SDK initializes both WS and WSS transports for each MQTT client, + even plain MQTT. Both buffers retain the existing 1,025-byte overflow fix. + A failed padded allocation destroys that transport safely instead of + returning its original undersized buffer. IDF 5 remains a passthrough. + +## Verification + +- Scope storage: three ASan/UBSan tests cover full capacities, identical 18,885 + persisted bytes, rollback on short writes/rename failures, lifetime, guarded + fallback, fatal allocation failure and platform/SDK gates. +- Advert history: four ASan/UBSan tests compare all 128 entries against inline + storage, including quota, duplicate, badlist and rollover behavior. Existing + RTC (eight), IRAM (seven) and native limiter (33 per backend) tests pass. +- WebSocket wrapper: three ASan/UBSan tests exercise 17 scenarios across G3, + other PSRAM boards, internal-only boards and IDF 4/5, including the extra + byte, cleanup, parent lifetime and fallback boundaries. +- RAM policy: all 63 tests pass, including six new fallback regressions. + Existing policy capacity, rule layout, moderation and CLI tests pass. +- Canonical G3 Full package: compile, internal-RAM admission, dual-OTA fit and + capability checks pass. PlatformIO ran one process at a time. + +Reproduce the final build with: + +```sh +bash build.sh build-firmware Station_G3_ESP32_repeater --full-exact --firmware-version v1.18.0.2-psram-test --profile cascade --radio-preset usa-cascade-fixed --require-ota +``` + +Physical acceptance remains a G3 boot, persistence, radio traffic and MQTT +reconnect/load run with internal/DMA heap monitoring. The maximum five-broker +TLS/WSS/outbox workload remains unqualified by these software checks. diff --git a/docs/test-results/g3_psram_storage_2026-10-27.safe.json b/docs/test-results/g3_psram_storage_2026-10-27.safe.json new file mode 100644 index 000000000..cfad93728 --- /dev/null +++ b/docs/test-results/g3_psram_storage_2026-10-27.safe.json @@ -0,0 +1,512 @@ +{ + "schema_version": 1, + "baseline_commit": "88bfaebb50cd932bca126675a22be72ee57c0f2d", + "target": "Station_G3_ESP32_repeater", + "physical_validation_performed": false, + "maximum_runtime_load_qualified": false, + "command": "bash build.sh build-firmware Station_G3_ESP32_repeater --full-exact --firmware-version v1.18.0.2-psram-test --profile cascade --radio-preset usa-cascade-fixed --require-ota", + "net_internal_bss_bytes_freed": 36776, + "websocket_payload_bytes_per_initialized_client": 2050, + "websocket_payload_bytes_for_five_initialized_clients": 10250, + "limitations": [ + "Linked capacity is not running free heap.", + "No G3 hardware flashed or measured.", + "No physical RF timing, persistence or broker-soak qualification." + ], + "builds": { + "baseline": { + "memory": { + "schema_version": 1, + "target": "Station_G3_ESP32_repeater", + "platform": "ESP32_PLATFORM", + "mcu": "esp32s3", + "elf_sha256": "f434ffa380fd55b77abfa67bf0195cd6af5ae0f5409f91a010c31b28cfe6251a", + "passed": true, + "available_internal_bytes": 199976, + "largest_internal_region_bytes": 167208, + "required_heap_bytes": 197808, + "required_contiguous_bytes": 19168, + "components": { + "core_tasks_usb_filesystems": 24576, + "wireless_stacks": 49152, + "mqtt_startup_allowance": 24576, + "udp_bridge_and_socket": 5120, + "ota_source_scratch": 8192, + "browser_terminal_session": 2048, + "neighbor_history": 18448, + "host_kiss_codec": 2048, + "radio_packet_pool": 10240, + "client_table": 9104, + "flood_filter_table": 19168, + "ota_generic_source_buffers": 3616, + "ota_encoder_workspace": 1040, + "display_pixels_and_driver": 2048, + "screen_objects_and_history": 2048, + "allocation_and_transient_margin": 16384 + }, + "display": "SH1106Display", + "full_companion": false, + "mqtt": { + "budget_scope": "startup minimum with runtime connection admission", + "startup_allowance_bytes": 24576, + "startup_active_slot_allowance": 2, + "runtime_slot_count": 6, + "maximum_active_slots": 5, + "maximum_active_slots_without_psram": 2, + "worker_stack_bytes": 8192, + "client_task_stack_bytes": 6144, + "client_task_priority": 5, + "client_rx_tx_buffer_bytes": 1792, + "maximum_slot_stack_buffer_allowance_bytes": 48384, + "cold_start_allocation_allowance_bytes": 16384, + "cold_start_dma_admission_free_bytes": 32768, + "cold_start_dma_admission_contiguous_bytes": 8192, + "additional_broker_cold_start_dma_admission_contiguous_bytes": 20480, + "restart_allocation_allowance_bytes": 8192, + "restart_dma_admission_free_bytes": 24576, + "restart_dma_admission_contiguous_bytes": 8192, + "additional_broker_restart_dma_admission_contiguous_bytes": 12288, + "dma_admission_free_bytes": 16384, + "dma_admission_contiguous_bytes": 4096, + "publish_contiguous_bytes": 4096, + "publish_contiguous_without_psram_bytes": 8000, + "publish_contiguous_with_psram_bytes": 4096, + "tls_psram_admission_contiguous_bytes": 16000, + "additional_broker_cold_start_tls_psram_admission_contiguous_bytes": 32384, + "additional_broker_restart_tls_psram_admission_contiguous_bytes": 24192, + "additional_broker_tls_psram_admission_contiguous_bytes": 16000, + "tls_internal_admission_free_bytes": 61440, + "tls_record_allocation_bytes": 17408, + "additional_broker_cold_start_tls_internal_admission_contiguous_bytes": 41792, + "additional_broker_restart_tls_internal_admission_contiguous_bytes": 33600, + "tls_internal_admission_contiguous_bytes": 17408, + "additional_broker_tls_internal_admission_contiguous_bytes": 25408, + "maximum_runtime_load_qualified": false, + "physical_validation_performed": false, + "limitations": "Maximum TLS/WSS/outbox workload is not covered by the startup allowance; runtime free DMA heap and fragmentation must satisfy connection admission; additional brokers reserve publication headroom beside startup allocations." + }, + "regions": [ + { + "start": 1070336488, + "end": 1070503696 + }, + { + "start": 1070530560, + "end": 1070563328 + } + ], + "scope": "Linked capacity before runtime allocation; physical boot/load/soak validation remains required.", + "artifact_target": "Station_G3_ESP32_repeater", + "pio_environment": "Station_G3_ESP32_repeater_observer_mqtt", + "files": { + "Station_G3_ESP32_repeater-full-usb-wifi-ota-v1.18.0.2-psram-test-88bfaebb.bin": "bf8b4130a79beba18faa95e5f0966d07112c2131c9bcb87915e8b070b387adae", + "Station_G3_ESP32_repeater-full-usb-wifi-ota-v1.18.0.2-psram-test-88bfaebb-merged.bin": "defb82a9007799ebb03912536948cc99c3d69f9366bd24898c679c7aa3e8d7c0", + "Station_G3_ESP32_repeater-full-usb-wifi-ota-v1.18.0.2-psram-test-88bfaebb.capabilities.json": "b2d1879bd5cbed1459cbd56f066af9a2951493263aa782a57fe4610222400e70", + "Station_G3_ESP32_repeater-full-usb-wifi-ota-v1.18.0.2-psram-test-88bfaebb.meshtext-license.txt": "c37d0a453a2491514bac33ed835c1877f207390f134ae439a97b06b004e236d2" + } + }, + "capabilities": { + "artifact_target": "Station_G3_ESP32_repeater", + "build_profile": "full", + "build_recipe": { + "schema_version": 1, + "sha256": "2db8b799c57e5b9ff224b8fec01bbd1ea1b425188e80d6a9e9b75a2b5658e3ff" + }, + "capabilities": [ + "bridge.espnow", + "bridge.udp", + "profile.full", + "release.full.same_partition_successor", + "radio2.kiss.host", + "radio2.kiss.usb", + "radio2.kiss.tcp", + "ota.update.wifi", + "ota.update.lora", + "cli.retry_preset", + "telemetry.history", + "ota.cli", + "web.webconfig", + "logging.usb.packets", + "logging.usb.control", + "logging.usb.output", + "bridge.udp.configuration", + "radio.station_g3.auto_slots" + ], + "ota_update_evidence": "firmware fits both OTA application slots; otadata present", + "ota_update_methods": [ + "wifi", + "lora" + ], + "ota_update_verified": true, + "platform": "ESP32_PLATFORM", + "platformio_env": "Station_G3_ESP32_repeater_observer_mqtt", + "reductions": [], + "schema_version": 2, + "target": "Station_G3_ESP32_repeater", + "third_party_notices": [ + { + "component": "espp-meshtext", + "file": "Station_G3_ESP32_repeater-full-usb-wifi-ota-v1.18.0.2-psram-test-88bfaebb.meshtext-license.txt", + "license": "MIT", + "sha256": "c37d0a453a2491514bac33ed835c1877f207390f134ae439a97b06b004e236d2" + } + ], + "verification": [ + { + "capability": "radio2.kiss.host", + "evidence": "_ZN4mesh8hostkiss5Codec4feedEh; _ZN4mesh8hostkiss5Codec11queueOutputEhPKhj; _ZN4mesh10Dispatcher22queueForeignRadioFrameEPKhjhjNS_20RadioProfileProtocolE", + "present": true, + "source": "linked ELF symbols" + }, + { + "capability": "radio2.kiss.usb", + "evidence": "Error: use set kiss on|off; Error: use set kiss.transport usb|tcp; USB: send KISS FEND to start", + "present": true, + "source": "packaged application" + }, + { + "capability": "radio2.kiss.tcp", + "evidence": "station TCP 5004", + "present": true, + "source": "packaged application" + }, + { + "capability": "bridge.espnow", + "evidence": "_ZN12ESPNowBridge5beginEv", + "present": true, + "source": "linked image" + }, + { + "capability": "bridge.udp", + "evidence": "_ZN9UDPBridge5beginEv", + "present": true, + "source": "linked image" + }, + { + "capability": "ota.update.wifi", + "evidence": "Started: http://%s/update", + "present": true, + "source": "linked image" + }, + { + "capability": "ota.update.lora", + "evidence": "image_hash MISMATCH after decode", + "present": true, + "source": "linked image" + }, + { + "capability": "cli.retry_preset", + "evidence": "retry.preset", + "present": true, + "source": "linked image" + }, + { + "capability": "telemetry.history", + "evidence": "telemetry.volt.i2c", + "present": true, + "source": "linked image" + }, + { + "capability": "ota.cli", + "evidence": "OTA: status", + "present": true, + "source": "linked image" + }, + { + "capability": "web.webconfig", + "evidence": "start webconfig", + "present": true, + "source": "linked image" + }, + { + "capability": "logging.usb.packets", + "evidence": "%s: %s, len=%d (type=%d, route=%s, payload_len=%d)", + "present": true, + "source": "packaged application" + }, + { + "capability": "logging.usb.control", + "evidence": "OK - USB logging %s (saved)", + "present": true, + "source": "packaged application" + }, + { + "capability": "logging.usb.output", + "evidence": "OK - logging.output %s (saved)", + "present": true, + "source": "packaged application" + }, + { + "capability": "bridge.udp.configuration", + "evidence": "udp.enabled", + "present": true, + "source": "packaged application" + }, + { + "capability": "radio.station_g3.auto_slots", + "evidence": "meshcore.capability.station_g3.auto_slots.v1", + "present": true, + "source": "packaged application" + } + ], + "verified": true + }, + "application_bytes": 2156504, + "application_slot_bytes": 6553600, + "dram_bss_bytes": 142088, + "dram_data_bytes": 23980 + }, + "final": { + "memory": { + "schema_version": 1, + "target": "Station_G3_ESP32_repeater", + "platform": "ESP32_PLATFORM", + "mcu": "esp32s3", + "elf_sha256": "96c94f1cafd78cbb9ef8429dbef746225ab0f32aa5b23f130fb42c88ea762fda", + "passed": true, + "available_internal_bytes": 236752, + "largest_internal_region_bytes": 203984, + "required_heap_bytes": 234636, + "required_contiguous_bytes": 19168, + "components": { + "core_tasks_usb_filesystems": 24576, + "wireless_stacks": 49152, + "mqtt_startup_allowance": 24576, + "udp_bridge_and_socket": 5120, + "ota_source_scratch": 8192, + "browser_terminal_session": 2048, + "neighbor_history": 18448, + "host_kiss_codec": 2048, + "radio_packet_pool": 10240, + "client_table": 9104, + "flood_filter_table": 19168, + "repeater_scope_storage_fallback": 18892, + "repeater_advert_storage_fallback": 17936, + "ota_generic_source_buffers": 3616, + "ota_encoder_workspace": 1040, + "display_pixels_and_driver": 2048, + "screen_objects_and_history": 2048, + "allocation_and_transient_margin": 16384 + }, + "display": "SH1106Display", + "full_companion": false, + "mqtt": { + "budget_scope": "startup minimum with runtime connection admission", + "startup_allowance_bytes": 24576, + "startup_active_slot_allowance": 2, + "runtime_slot_count": 6, + "maximum_active_slots": 5, + "maximum_active_slots_without_psram": 2, + "worker_stack_bytes": 8192, + "client_task_stack_bytes": 6144, + "client_task_priority": 5, + "client_rx_tx_buffer_bytes": 1792, + "maximum_slot_stack_buffer_allowance_bytes": 48384, + "cold_start_allocation_allowance_bytes": 16384, + "cold_start_dma_admission_free_bytes": 32768, + "cold_start_dma_admission_contiguous_bytes": 8192, + "additional_broker_cold_start_dma_admission_contiguous_bytes": 20480, + "restart_allocation_allowance_bytes": 8192, + "restart_dma_admission_free_bytes": 24576, + "restart_dma_admission_contiguous_bytes": 8192, + "additional_broker_restart_dma_admission_contiguous_bytes": 12288, + "dma_admission_free_bytes": 16384, + "dma_admission_contiguous_bytes": 4096, + "publish_contiguous_bytes": 4096, + "publish_contiguous_without_psram_bytes": 8000, + "publish_contiguous_with_psram_bytes": 4096, + "tls_psram_admission_contiguous_bytes": 16000, + "additional_broker_cold_start_tls_psram_admission_contiguous_bytes": 32384, + "additional_broker_restart_tls_psram_admission_contiguous_bytes": 24192, + "additional_broker_tls_psram_admission_contiguous_bytes": 16000, + "tls_internal_admission_free_bytes": 61440, + "tls_record_allocation_bytes": 17408, + "additional_broker_cold_start_tls_internal_admission_contiguous_bytes": 41792, + "additional_broker_restart_tls_internal_admission_contiguous_bytes": 33600, + "tls_internal_admission_contiguous_bytes": 17408, + "additional_broker_tls_internal_admission_contiguous_bytes": 25408, + "maximum_runtime_load_qualified": false, + "physical_validation_performed": false, + "limitations": "Maximum TLS/WSS/outbox workload is not covered by the startup allowance; runtime free DMA heap and fragmentation must satisfy connection admission; additional brokers reserve publication headroom beside startup allocations." + }, + "regions": [ + { + "start": 1070299712, + "end": 1070503696 + }, + { + "start": 1070530560, + "end": 1070563328 + } + ], + "scope": "Linked capacity before runtime allocation; physical boot/load/soak validation remains required.", + "artifact_target": "Station_G3_ESP32_repeater", + "pio_environment": "Station_G3_ESP32_repeater_observer_mqtt", + "files": { + "Station_G3_ESP32_repeater-full-usb-wifi-ota-v1.18.0.2-psram-test-88bfaebb.bin": "d3a941a6566d66e76b9315f16f76ec7fbb7d233734dbe5c74ec2a96a3b8344b9", + "Station_G3_ESP32_repeater-full-usb-wifi-ota-v1.18.0.2-psram-test-88bfaebb-merged.bin": "b4219d24cac25914b6d8f4b09c31868992c310aca21a5e40d03c4148dd0ef402", + "Station_G3_ESP32_repeater-full-usb-wifi-ota-v1.18.0.2-psram-test-88bfaebb.capabilities.json": "8efba981bbc219c81fedc78dbf04f4f358d77cf368747c6310ef7bc0a88d6d77", + "Station_G3_ESP32_repeater-full-usb-wifi-ota-v1.18.0.2-psram-test-88bfaebb.meshtext-license.txt": "c37d0a453a2491514bac33ed835c1877f207390f134ae439a97b06b004e236d2" + } + }, + "capabilities": { + "artifact_target": "Station_G3_ESP32_repeater", + "build_profile": "full", + "build_recipe": { + "schema_version": 1, + "sha256": "8ffd9cd9ad1dd47a035110d7108ea8a178ad4b9ea7c5552bca41f0d9c33fcef3" + }, + "capabilities": [ + "bridge.espnow", + "bridge.udp", + "profile.full", + "release.full.same_partition_successor", + "radio2.kiss.host", + "radio2.kiss.usb", + "radio2.kiss.tcp", + "ota.update.wifi", + "ota.update.lora", + "cli.retry_preset", + "telemetry.history", + "ota.cli", + "web.webconfig", + "logging.usb.packets", + "logging.usb.control", + "logging.usb.output", + "bridge.udp.configuration", + "radio.station_g3.auto_slots" + ], + "ota_update_evidence": "firmware fits both OTA application slots; otadata present", + "ota_update_methods": [ + "wifi", + "lora" + ], + "ota_update_verified": true, + "platform": "ESP32_PLATFORM", + "platformio_env": "Station_G3_ESP32_repeater_observer_mqtt", + "reductions": [], + "schema_version": 2, + "target": "Station_G3_ESP32_repeater", + "third_party_notices": [ + { + "component": "espp-meshtext", + "file": "Station_G3_ESP32_repeater-full-usb-wifi-ota-v1.18.0.2-psram-test-88bfaebb.meshtext-license.txt", + "license": "MIT", + "sha256": "c37d0a453a2491514bac33ed835c1877f207390f134ae439a97b06b004e236d2" + } + ], + "verification": [ + { + "capability": "radio2.kiss.host", + "evidence": "_ZN4mesh8hostkiss5Codec4feedEh; _ZN4mesh8hostkiss5Codec11queueOutputEhPKhj; _ZN4mesh10Dispatcher22queueForeignRadioFrameEPKhjhjNS_20RadioProfileProtocolE", + "present": true, + "source": "linked ELF symbols" + }, + { + "capability": "radio2.kiss.usb", + "evidence": "Error: use set kiss on|off; Error: use set kiss.transport usb|tcp; USB: send KISS FEND to start", + "present": true, + "source": "packaged application" + }, + { + "capability": "radio2.kiss.tcp", + "evidence": "station TCP 5004", + "present": true, + "source": "packaged application" + }, + { + "capability": "bridge.espnow", + "evidence": "_ZN12ESPNowBridge5beginEv", + "present": true, + "source": "linked image" + }, + { + "capability": "bridge.udp", + "evidence": "_ZN9UDPBridge5beginEv", + "present": true, + "source": "linked image" + }, + { + "capability": "ota.update.wifi", + "evidence": "Started: http://%s/update", + "present": true, + "source": "linked image" + }, + { + "capability": "ota.update.lora", + "evidence": "image_hash MISMATCH after decode", + "present": true, + "source": "linked image" + }, + { + "capability": "cli.retry_preset", + "evidence": "retry.preset", + "present": true, + "source": "linked image" + }, + { + "capability": "telemetry.history", + "evidence": "telemetry.volt.i2c", + "present": true, + "source": "linked image" + }, + { + "capability": "ota.cli", + "evidence": "OTA: status", + "present": true, + "source": "linked image" + }, + { + "capability": "web.webconfig", + "evidence": "start webconfig", + "present": true, + "source": "linked image" + }, + { + "capability": "logging.usb.packets", + "evidence": "%s: %s, len=%d (type=%d, route=%s, payload_len=%d)", + "present": true, + "source": "packaged application" + }, + { + "capability": "logging.usb.control", + "evidence": "OK - USB logging %s (saved)", + "present": true, + "source": "packaged application" + }, + { + "capability": "logging.usb.output", + "evidence": "OK - logging.output %s (saved)", + "present": true, + "source": "packaged application" + }, + { + "capability": "bridge.udp.configuration", + "evidence": "udp.enabled", + "present": true, + "source": "packaged application" + }, + { + "capability": "radio.station_g3.auto_slots", + "evidence": "meshcore.capability.station_g3.auto_slots.v1", + "present": true, + "source": "packaged application" + } + ], + "verified": true + }, + "application_bytes": 2155752, + "application_slot_bytes": 6553600, + "dram_bss_bytes": 105312, + "dram_data_bytes": 23980 + } + }, + "source_sha256": { + "src/helpers/RepeaterPsramStorage.h": "440d6c0e9ea82575ddb34ae7fe17592b86c8e37293fde7480f42a67927441cb7", + "src/helpers/FloodAdvertLimiter.h": "08fa87e74a8574f4784cac2a9c4327726fc38a5bfa368e5173c2958e48cfeef7", + "src/helpers/ESP32WsTransportFix.cpp": "7d90e2e478aae855f4546e7b9dfcbb9b3d9ad725a13979f099f596cbab89760c", + "examples/simple_repeater/MyMesh.h": "0b8aaa704953bd9654840287422010e33193998e7123aaa4e8dd087bec86a335", + "examples/simple_repeater/MyMesh.cpp": "f901dbe36021dc16b475785da63caf74889a71c075a19d4bcd1c368df8df0d77", + "scripts/check_firmware_ram.py": "0a4016444a95f87e0a6ac82159f8f122ab5609067e16aea525f53ece82f362b3" + } +} diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 00afb61de..e99837958 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -3707,6 +3707,11 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc : mesh::Mesh(radio, ms, rng, rtc, *createObserverPacketManager(32), tables), region_map(key_store), temp_map(key_store), _cli(board, rtc, sensors, region_map, acl, &_prefs, this), +#if MESH_REPEATER_PSRAM_TABLES + flood_channel_scopes(flood_channel_storage.require("channel scope tables").scopes), + flood_channel_direct_scopes(flood_channel_storage.require("channel scope tables").direct), + flood_channel_scope_requirements(flood_channel_storage.require("channel scope tables").requirements), +#endif telemetry(MAX_PACKET_PAYLOAD - 4), discover_limiter(4, 120), // max 4 every 2 minutes anon_limiter(4, 180) // max 4 every 3 minutes diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index a6efa7300..6e6ee587c 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -9,6 +9,7 @@ #include #include #include +#include #if defined(ENABLE_OTA) #include #endif @@ -558,11 +559,34 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks uint8_t flood_packet_filter_blacklist_count; uint8_t flood_packet_filter_blacklist[FLOOD_PACKET_FILTER_BLACKLIST_MAX] [FLOOD_PACKET_FILTER_PATH_ID_SIZE]; +#if MESH_REPEATER_PSRAM_TABLES + // Keep real array references: persistence, resets and rollback use sizeof + // these complete tables, and routing retains addresses into them. + struct FloodChannelStorage { + FloodChannelScopeEntry scopes[FLOOD_CHANNEL_SCOPE_SLOTS]; + char direct[FLOOD_CHANNEL_DIRECT_SCOPE_SLOTS][FLOOD_PACKET_FILTER_SCOPE_NAME_LEN]; + FloodChannelScopeRequireEntry requirements[FLOOD_CHANNEL_SCOPE_REQUIRE_SLOTS]; + }; + static_assert(sizeof(FloodChannelScopeEntry) == 36 + && sizeof(FloodChannelScopeRequireEntry) == 34 + && sizeof(FloodChannelStorage) + <= FLOOD_CHANNEL_SCOPE_SLOTS * 36U + + FLOOD_CHANNEL_DIRECT_SCOPE_SLOTS * 32U + + FLOOD_CHANNEL_SCOPE_REQUIRE_SLOTS * 34U + 2U, + "Update the repeater PSRAM fallback budget in check_firmware_ram.py"); + mesh::RepeaterPsramStorage flood_channel_storage; + FloodChannelScopeEntry (&flood_channel_scopes)[FLOOD_CHANNEL_SCOPE_SLOTS]; + char (&flood_channel_direct_scopes)[FLOOD_CHANNEL_DIRECT_SCOPE_SLOTS] + [FLOOD_PACKET_FILTER_SCOPE_NAME_LEN]; + FloodChannelScopeRequireEntry + (&flood_channel_scope_requirements)[FLOOD_CHANNEL_SCOPE_REQUIRE_SLOTS]; +#else FloodChannelScopeEntry flood_channel_scopes[FLOOD_CHANNEL_SCOPE_SLOTS]; char flood_channel_direct_scopes[FLOOD_CHANNEL_DIRECT_SCOPE_SLOTS] [FLOOD_PACKET_FILTER_SCOPE_NAME_LEN]; FloodChannelScopeRequireEntry flood_channel_scope_requirements[FLOOD_CHANNEL_SCOPE_REQUIRE_SLOTS]; +#endif #if MESH_ENABLE_FLOOD_GROUP_MODERATION FloodGroupModerationEntry flood_group_moderation[FLOOD_GROUP_MODERATION_SLOTS]; #endif diff --git a/scripts/check_firmware_ram.py b/scripts/check_firmware_ram.py index 58b7746d7..8624b6e5a 100644 --- a/scripts/check_firmware_ram.py +++ b/scripts/check_firmware_ram.py @@ -354,6 +354,27 @@ def requirements(platform, defines, target): engine = integer(defines, "MESH_ENABLE_FLOOD_RULE_ENGINE", int(platform != "STM32_PLATFORM")) slots = integer(defines, "FLOOD_PACKET_FILTER_SLOTS", 63 if engine else 16) parts["flood_filter_table"] = slots * (304 if engine else 40) + 16 + psram_tables_default = int(platform == "ESP32_PLATFORM" and qualified_repeater + and "STATION_G3_ESP32" in defines + and "BOARD_HAS_PSRAM" in defines) + if platform == "ESP32_PLATFORM" and integer( + defines, "MESH_REPEATER_PSRAM_TABLES", psram_tables_default): + # The data normally lives in PSRAM, but all capacities also need + # safe internal fallback. Moving it out of BSS cannot erase that + # requirement or relax startup admission when PSRAM is missing. + scopes = integer(defines, "FLOOD_CHANNEL_SCOPE_SLOTS", + 31 if "CONFIG_IDF_TARGET_ESP32" in defines else 255) + required = integer(defines, "FLOOD_CHANNEL_SCOPE_REQUIRE_SLOTS", scopes) + direct = integer(defines, "FLOOD_CHANNEL_DIRECT_SCOPE_SLOTS", + min(scopes, integer(defines, "MAX_REGION_ENTRIES", 32))) + parts["repeater_scope_storage_fallback"] = ( + scopes * 36 + required * 34 + direct * 32 + 2 + 16) + if ("PORTABLE_MQTT_OBSERVER" not in defines + and not integer(defines, "FLOOD_ADVERT_IRAM_WORD_STORAGE", 0) + and not integer(defines, "FLOOD_ADVERT_RTC_FAST_SLOTS", 0) + and not integer(defines, "FLOOD_ADVERT_RTC_SLOW_SLOTS", 0)): + parts["repeater_advert_storage_fallback"] = ( + integer(defines, "FLOOD_ADVERT_SOURCE_SLOTS", 128) * 140 + 16) if "room_server" in target.lower() and integer( defines, "MESH_ENABLE_ROOM_FLOOD_RULE_ENGINE", 0): # FloodRuleEngine keeps all 31 persisted rules in one startup heap @@ -473,6 +494,7 @@ def requirements(platform, defines, target): if "expanded_message_previews" in parts: largest = max(largest, 8192 + parts["expanded_message_previews"]) for name in ("client_table", "flood_filter_table", "room_flood_rule_table", "neighbor_history", + "repeater_scope_storage_fallback", "repeater_advert_storage_fallback", "solarxiao_voltage_history", "ota_context", "screen_objects_and_history", "ota_encoder_workspace", "ota_generic_source_buffers", "ethernet_or_external_ota_workspaces", "ota_target_name_lookup", diff --git a/src/helpers/ESP32WsTransportFix.cpp b/src/helpers/ESP32WsTransportFix.cpp index 37c2e2530..a4702cae4 100644 --- a/src/helpers/ESP32WsTransportFix.cpp +++ b/src/helpers/ESP32WsTransportFix.cpp @@ -40,7 +40,7 @@ // the fork moves to Arduino core 3.x. // // Scope: this stops the heap corruption and NOTHING else. It does not make an -// oversized response fail the handshake — v4.4's read loop also exits on +// oversized response fail the handshake; v4.4's read loop also exits on // `header_len < WS_BUFFER_SIZE` going false, and then still accepts the // connection if "Sec-WebSocket-Accept:" was inside those first bytes (it comes // early, so it usually is). ws_connect() therefore returns success on a partial @@ -67,6 +67,11 @@ #include "esp_transport.h" #include "esp_transport_ws.h" +#if defined(STATION_G3_ESP32) && defined(BOARD_HAS_PSRAM) +#include "esp_heap_caps.h" +#include "MQTTConnectionAdmission.h" +#endif + #ifndef CONFIG_WS_BUFFER_SIZE #define CONFIG_WS_BUFFER_SIZE 1024 #endif @@ -103,13 +108,30 @@ esp_transport_handle_t __wrap_esp_transport_ws_init(esp_transport_handle_t paren if (ws != nullptr && ws->buffer != nullptr) { // The buffer is untouched at this point (allocated moments ago inside // __real_esp_transport_ws_init), so a swap is safe. - char* padded = (char*)malloc(CONFIG_WS_BUFFER_SIZE + 1); - if (padded != nullptr) { - free(ws->buffer); - ws->buffer = padded; + const size_t bytes = CONFIG_WS_BUFFER_SIZE + 1; +#if defined(STATION_G3_ESP32) && defined(BOARD_HAS_PSRAM) + // Upgrade request/response text is CPU-only. Retain this padded buffer + // until the SDK destroys the transport; its ordinary free accepts PSRAM. + char* padded = (char*)heap_caps_malloc(bytes, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (padded == nullptr) { + const uint32_t caps = MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA; + if (MQTTConnectionAdmission::canFallback(bytes, + heap_caps_get_free_size(caps), heap_caps_get_largest_free_block(caps))) { + padded = (char*)heap_caps_malloc(bytes, caps | MALLOC_CAP_8BIT); + } } - // On alloc failure keep the original buffer: same behavior as before - // this fix, which is still strictly better than failing init here. +#else + char* padded = (char*)malloc(bytes); +#endif + if (padded == nullptr) { + // Returning the original 1024-byte buffer would reintroduce the + // overflow. This transport has not entered the client's list yet; + // destroy releases its context/buffer without destroying its parent. + esp_transport_destroy(t); + return nullptr; + } + free(ws->buffer); + ws->buffer = padded; } } return t; diff --git a/src/helpers/FloodAdvertLimiter.h b/src/helpers/FloodAdvertLimiter.h index eafd65722..abe3ad1fa 100644 --- a/src/helpers/FloodAdvertLimiter.h +++ b/src/helpers/FloodAdvertLimiter.h @@ -4,6 +4,7 @@ #include #include "FloodAdvertRtcPolicy.h" #include "FloodAdvertIramPolicy.h" +#include "RepeaterPsramStorage.h" #ifndef FLOOD_ADVERT_SOURCE_SLOTS #if defined(STM32_PLATFORM) @@ -89,6 +90,14 @@ protected: } #else FloodAdvertLimiter(Entry* entries, size_t capacity) : _entries(entries), _capacity(capacity) {} +#if MESH_REPEATER_PSRAM_TABLES + // The derived storage owner is initialized after this base. Bind its full + // table before reset() or any limiter operation can inspect the entries. + void bindStorage(Entry* entries) { + if (!entries || reinterpret_cast(entries) % alignof(Entry)) abort(); + _entries = entries; + } +#endif #endif FloodAdvertLimiter(const FloodAdvertLimiter&) = delete; FloodAdvertLimiter& operator=(const FloodAdvertLimiter&) = delete; @@ -196,6 +205,19 @@ public: public: StaticFloodAdvertLimiter(Entry* fast, Entry* slow) : FloodAdvertLimiter(_storage, Capacity, fast, slow) { reset(); } +#elif MESH_REPEATER_PSRAM_TABLES + static_assert(sizeof(Entry) == 140, + "Update the advert PSRAM fallback budget in check_firmware_ram.py"); + struct Storage { Entry entries[Capacity]; }; + RepeaterPsramStorage _storage; +public: + StaticFloodAdvertLimiter() : FloodAdvertLimiter(nullptr, Capacity) { + // Exhaustion must stop startup rather than disabling advert protection + // or reducing its source/hash history. No full internal array is retained. + if (!_storage.begin("advert source history")) abort(); + bindStorage(_storage.data()->entries); + reset(); + } #else Entry _storage[Capacity]; public: diff --git a/src/helpers/RepeaterPsramStorage.h b/src/helpers/RepeaterPsramStorage.h new file mode 100644 index 000000000..5993f1c88 --- /dev/null +++ b/src/helpers/RepeaterPsramStorage.h @@ -0,0 +1,79 @@ +#pragma once + +#ifndef MESH_REPEATER_PSRAM_TABLES + #if defined(STATION_G3_ESP32) && defined(ESP32_PLATFORM) && defined(BOARD_HAS_PSRAM) \ + && defined(MESH_CLIENT_REPEATER_ONLY) && MESH_CLIENT_REPEATER_ONLY + #define MESH_REPEATER_PSRAM_TABLES 1 + #else + #define MESH_REPEATER_PSRAM_TABLES 0 + #endif +#endif + +#if MESH_REPEATER_PSRAM_TABLES != 0 && MESH_REPEATER_PSRAM_TABLES != 1 +#error "MESH_REPEATER_PSRAM_TABLES must be 0 or 1" +#endif + +#if MESH_REPEATER_PSRAM_TABLES +#if !defined(ESP32_PLATFORM) || !defined(BOARD_HAS_PSRAM) +#error "Repeater PSRAM tables require an ESP32 target with PSRAM support" +#endif + +#include +#if !defined(CONFIG_IDF_TARGET_ESP32S3) || !CONFIG_IDF_TARGET_ESP32S3 \ + || !defined(CONFIG_SPIRAM_BOOT_INIT) || !CONFIG_SPIRAM_BOOT_INIT \ + || !defined(CONFIG_SPIRAM_USE_MALLOC) || !CONFIG_SPIRAM_USE_MALLOC +#error "Repeater PSRAM tables require ESP32-S3 with boot-initialized PSRAM heaps" +#endif + +#include +#include +#include +#include +#include + +namespace mesh { + +// Own only CPU data here. ISR state, locks, task stacks and DMA buffers retain +// their existing internal storage. G3's pinned SDK initializes PSRAM-backed +// heaps before global constructors, so tables can keep their startup lifetime. +template class RepeaterPsramStorage { + static_assert(std::is_trivial::value, + "Repeater PSRAM tables require zero-initialized trivial data"); + T* value_ = nullptr; +public: + RepeaterPsramStorage() = default; + ~RepeaterPsramStorage() { if (value_) heap_caps_free(value_); } + RepeaterPsramStorage(const RepeaterPsramStorage&) = delete; + RepeaterPsramStorage& operator=(const RepeaterPsramStorage&) = delete; + + bool begin(const char* label = "repeater table") { + if (value_) return true; + value_ = static_cast(heap_caps_calloc( + 1, sizeof(T), MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (!value_) { + const uint32_t caps = MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA; + if (MQTTConnectionAdmission::canFallback(sizeof(T), + heap_caps_get_free_size(caps), heap_caps_get_largest_free_block(caps))) { + value_ = static_cast(heap_caps_calloc( + 1, sizeof(T), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT)); + } + } + if (!value_) { + fprintf(stderr, "Cannot allocate %s (%u bytes); full capacity is required\n", + label, static_cast(sizeof(T))); + } + return value_ != nullptr; + } + + T* data() { return value_; } + const T* data() const { return value_; } + T& require(const char* label) { + // Used only while binding the owner's references during construction. + // Allocation never moves after pointers escape to routing or persistence. + if (!begin(label)) abort(); + return *value_; + } +}; + +} // namespace mesh +#endif diff --git a/test/test_firmware_ram.py b/test/test_firmware_ram.py index 3809f20c9..6578b768e 100644 --- a/test/test_firmware_ram.py +++ b/test/test_firmware_ram.py @@ -453,6 +453,149 @@ int main() { sensor = ram.requirements("NRF52_PLATFORM", {}, "t114_sensor") self.assertEqual(sensor["components"]["client_table"], 32 * 320 + 16) + def test_g3_psram_tables_reserve_full_internal_fallback_without_cutting_other_owners(self): + flags = {"STATION_G3_ESP32": 1, "BOARD_HAS_PSRAM": 1, + "MESH_CLIENT_REPEATER_ONLY": 1, "WITH_MQTT_BRIDGE": 1, + "DISPLAY_CLASS": "SH1106Display", "ADMIN_PASSWORD": "test", + "ENABLE_OTA": 1, "WITH_UDP_BRIDGE": 1, "WITH_RS232_BRIDGE": "Serial2"} + target = "Station_G3_ESP32_repeater" + original = ram.requirements("ESP32_PLATFORM", { + **flags, "MESH_REPEATER_PSRAM_TABLES": 0}, target) + selected = ram.requirements("ESP32_PLATFORM", flags, target) + scope = 255 * 36 + 255 * 34 + 32 * 32 + 2 + 16 + advert = 128 * 140 + 16 + self.assertEqual(selected["components"], { + **original["components"], "repeater_scope_storage_fallback": scope, + "repeater_advert_storage_fallback": advert}) + self.assertEqual(selected["required_heap_bytes"], + original["required_heap_bytes"] + scope + advert) + self.assertEqual(selected["mqtt"], original["mqtt"]) + self.assertEqual(selected["components"]["allocation_and_transient_margin"], 16384) + self.assertEqual(selected["components"]["wireless_stacks"], 49152) + self.assertEqual(ram.requirements("ESP32_PLATFORM", { + **flags, "MESH_REPEATER_PSRAM_TABLES": 1}, target), selected) + + def test_g3_psram_table_defaults_follow_board_psram_platform_and_source_role_gates(self): + flags = {"STATION_G3_ESP32": 1, "BOARD_HAS_PSRAM": 1, + "MESH_CLIENT_REPEATER_ONLY": 1} + cases = [ + ("ESP32_PLATFORM", {k: v for k, v in flags.items() if k != "STATION_G3_ESP32"}, + "other_s3_repeater"), + ("ESP32_PLATFORM", {k: v for k, v in flags.items() if k != "BOARD_HAS_PSRAM"}, + "Station_G3_ESP32_repeater"), + ("ESP32_PLATFORM", {k: v for k, v in flags.items() if k != "MESH_CLIENT_REPEATER_ONLY"}, + "Station_G3_ESP32_repeater"), + ("ESP32_PLATFORM", {**flags, "MESH_CLIENT_REPEATER_ONLY": 0}, + "Station_G3_ESP32_repeater"), + ("ESP32_PLATFORM", {**flags, "MESH_CLIENT_REPEATER_ONLY": 0, + "COMPANION_RADIO_FULL": 1}, + "Station_G3_ESP32_companion_radio_full"), + ("ESP32_PLATFORM", {**flags, "MESH_CLIENT_REPEATER_ONLY": 0}, + "Station_G3_ESP32_room_server"), + ] + for platform in ("NRF52_PLATFORM", "RP2040_PLATFORM", "STM32_PLATFORM"): + cases.extend((platform, {**flags, **extra}, "other_repeater") + for extra in ({}, {"MESH_REPEATER_PSRAM_TABLES": 1})) + for platform, defines, target in cases: + with self.subTest(platform=platform, defines=defines, target=target): + policy = ram.requirements(platform, defines, target) + self.assertNotIn("repeater_scope_storage_fallback", policy["components"]) + self.assertNotIn("repeater_advert_storage_fallback", policy["components"]) + self.assertEqual(policy, ram.requirements(platform, { + **defines, "MESH_REPEATER_PSRAM_TABLES": 0}, target)) + # The source role, rather than a release filename, selects MyMesh. + renamed = ram.requirements("ESP32_PLATFORM", flags, "custom_board_forwarder") + self.assertIn("repeater_scope_storage_fallback", renamed["components"]) + self.assertIn("repeater_advert_storage_fallback", renamed["components"]) + + def test_explicit_psram_repeater_table_opt_in_and_capacity_overrides_are_fully_budgeted(self): + flags = {"BOARD_HAS_PSRAM": 1, "MESH_CLIENT_REPEATER_ONLY": 1, + "MESH_REPEATER_PSRAM_TABLES": 1} + cases = ( + ({}, 255, 255, 32, 128), + ({"FLOOD_CHANNEL_SCOPE_SLOTS": 5, "MAX_REGION_ENTRIES": 2}, 5, 5, 2, 128), + ({"FLOOD_CHANNEL_SCOPE_SLOTS": 7, "FLOOD_CHANNEL_SCOPE_REQUIRE_SLOTS": 11, + "FLOOD_CHANNEL_DIRECT_SCOPE_SLOTS": 3, "FLOOD_ADVERT_SOURCE_SLOTS": 17}, + 7, 11, 3, 17), + ({"FLOOD_CHANNEL_SCOPE_SLOTS": 3, "MAX_REGION_ENTRIES": 2, + "FLOOD_CHANNEL_DIRECT_SCOPE_SLOTS": 9}, 3, 3, 9, 128), + ({"MAX_REGION_ENTRIES": 64, "FLOOD_ADVERT_SOURCE_SLOTS": 200}, + 255, 255, 64, 200), + ) + for extra, scopes, required, direct, adverts in cases: + with self.subTest(extra=extra): + defines = {**flags, **extra} + selected = ram.requirements("ESP32_PLATFORM", defines, "other_s3_repeater") + original = ram.requirements("ESP32_PLATFORM", { + **defines, "MESH_REPEATER_PSRAM_TABLES": 0}, "other_s3_repeater") + scope = scopes * 36 + required * 34 + direct * 32 + 2 + 16 + advert = adverts * 140 + 16 + self.assertEqual(selected["components"], { + **original["components"], "repeater_scope_storage_fallback": scope, + "repeater_advert_storage_fallback": advert}) + self.assertEqual(selected["required_heap_bytes"], + original["required_heap_bytes"] + scope + advert) + + def test_psram_advert_budget_excludes_portable_iram_and_rtc_storage_only(self): + flags = {"STATION_G3_ESP32": 1, "BOARD_HAS_PSRAM": 1, + "MESH_CLIENT_REPEATER_ONLY": 1} + target = "Station_G3_ESP32_repeater" + original = ram.requirements("ESP32_PLATFORM", flags, target) + advert = original["components"]["repeater_advert_storage_fallback"] + for extra in ({"PORTABLE_MQTT_OBSERVER": 1}, {"PORTABLE_MQTT_OBSERVER": 0}, + {"FLOOD_ADVERT_IRAM_WORD_STORAGE": 1}, + {"FLOOD_ADVERT_RTC_FAST_SLOTS": 16}, + {"FLOOD_ADVERT_RTC_SLOW_SLOTS": 50}, + {"FLOOD_ADVERT_RTC_FAST_SLOTS": 50, "FLOOD_ADVERT_RTC_SLOW_SLOTS": 50}): + with self.subTest(extra=extra): + policy = ram.requirements("ESP32_PLATFORM", {**flags, **extra}, target) + self.assertEqual(policy["components"], { + k: v for k, v in original["components"].items() + if k != "repeater_advert_storage_fallback"}) + self.assertEqual(policy["required_heap_bytes"], + original["required_heap_bytes"] - advert) + disabled_backends = {"FLOOD_ADVERT_IRAM_WORD_STORAGE": 0, + "FLOOD_ADVERT_RTC_FAST_SLOTS": 0, "FLOOD_ADVERT_RTC_SLOW_SLOTS": 0} + self.assertEqual(ram.requirements("ESP32_PLATFORM", { + **flags, **disabled_backends}, target), original) + + def test_each_psram_table_fallback_can_set_the_contiguous_allocation_bound(self): + flags = {"STATION_G3_ESP32": 1, "BOARD_HAS_PSRAM": 1, + "MESH_CLIENT_REPEATER_ONLY": 1, "MAX_CLIENTS": 1, + "FLOOD_PACKET_FILTER_SLOTS": 1, "MAX_RECENT_REPEATERS": 0} + for extra, name in (({}, "repeater_scope_storage_fallback"), + ({"FLOOD_ADVERT_SOURCE_SLOTS": 200}, + "repeater_advert_storage_fallback")): + with self.subTest(extra=extra): + policy = ram.requirements("ESP32_PLATFORM", {**flags, **extra}, "g3_repeater") + self.assertEqual(policy["required_contiguous_bytes"], policy["components"][name]) + self.assertGreater(policy["required_contiguous_bytes"], 16384) + + def test_g3_psram_fallback_elf_gate_rejects_old_budget_and_enforces_exact_total(self): + flags = {"STATION_G3_ESP32": 1, "BOARD_HAS_PSRAM": 1, + "MESH_CLIENT_REPEATER_ONLY": 1, "WITH_MQTT_BRIDGE": 1} + target = "Station_G3_ESP32_repeater" + previous = {**flags, "MESH_REPEATER_PSRAM_TABLES": 0} + old = ram.requirements("ESP32_PLATFORM", previous, target)["required_heap_bytes"] + required = ram.requirements("ESP32_PLATFORM", flags, target)["required_heap_bytes"] + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "firmware.elf" + report = Path(temporary) / "firmware.memory.json" + for capacity, accepted in ((old, False), (required - 1, False), (required, True)): + with self.subTest(capacity=capacity), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + esp_fixture(path, modern=True, internal_length=capacity + 0x4000) + self.assertEqual(ram.check_firmware(path, "ESP32_PLATFORM", "esp32s3", + previous, target), 0) + status = ram.check_firmware(path, "ESP32_PLATFORM", "esp32s3", + flags, target, report) + result = json.loads(report.read_text()) + self.assertEqual(status == 0, accepted) + self.assertEqual(result["passed"], accepted) + self.assertEqual(result["available_internal_bytes"], capacity) + self.assertEqual(result["required_heap_bytes"], required) + self.assertEqual(result["components"]["repeater_scope_storage_fallback"], 18892) + self.assertEqual(result["components"]["repeater_advert_storage_fallback"], 17936) + def test_browser_terminal_reserves_internal_session_and_psram_aware_scrollback(self): defines = {"ENABLE_USB_INTERFACE": 1, "WIFI_SSID": "", "DISPLAY_CLASS": "SSD1306Display"} base = ram.requirements("ESP32_PLATFORM", {**defines, "WEBCONFIG_DISABLED": 1}, "v4_companion") diff --git a/test/test_flood_advert_psram.py b/test/test_flood_advert_psram.py new file mode 100644 index 000000000..50c34969a --- /dev/null +++ b/test/test_flood_advert_psram.py @@ -0,0 +1,209 @@ +"""Keep every advert-history decision while moving its complete table to PSRAM.""" +from pathlib import Path +import os +import resource +import signal +import subprocess +import tempfile +import unittest + +from test_flood_advert_rtc import HARNESS + +ROOT = Path(__file__).resolve().parents[1] + +HEAP = r''' +#pragma once +#include +#include +#include +#include +#define HEAP_CHECK(value) do { if (!(value)) std::abort(); } while (0) +enum { MALLOC_CAP_INTERNAL=1, MALLOC_CAP_DMA=2, + MALLOC_CAP_SPIRAM=4, MALLOC_CAP_8BIT=8 }; +namespace heap_test { +struct Block { size_t bytes; unsigned caps; }; +struct Heap { + bool psram = true, internal = true; + size_t dma_free = 100000, dma_largest = 100000; + unsigned external_calls = 0, internal_calls = 0, frees = 0; + size_t last_bytes = 0; + std::map blocks; +}; +inline Heap& heap() { static Heap value; return value; } +} +inline void* heap_caps_malloc(size_t size, unsigned caps) { + using namespace heap_test; + Heap& state = heap(); + state.last_bytes = size; + const bool external = (caps & MALLOC_CAP_SPIRAM) != 0; + if (external) ++state.external_calls; else ++state.internal_calls; + if (external ? !state.psram : !state.internal) return nullptr; + if (!external) { + HEAP_CHECK(caps & MALLOC_CAP_INTERNAL); + HEAP_CHECK(size <= state.dma_largest && size <= state.dma_free - 16384); + state.dma_free -= size; + } + void* memory = std::malloc(size); + HEAP_CHECK(memory); + std::memset(memory, 0xa5, size); + state.blocks[memory] = {size, caps}; + return memory; +} +inline void* heap_caps_calloc(size_t count, size_t size, unsigned caps) { + void* memory = heap_caps_malloc(count * size, caps); + if (memory) std::memset(memory, 0, count * size); + return memory; +} +inline void heap_caps_free(void* memory) { + if (!memory) return; + using namespace heap_test; + Heap& state = heap(); + const auto found = state.blocks.find(memory); + HEAP_CHECK(found != state.blocks.end()); + if (!(found->second.caps & MALLOC_CAP_SPIRAM)) + state.dma_free += found->second.bytes; + state.blocks.erase(found); + ++state.frees; + std::free(memory); +} +inline size_t heap_caps_get_free_size(unsigned caps) { + HEAP_CHECK(caps == (MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA)); + return heap_test::heap().dma_free; +} +inline size_t heap_caps_get_largest_free_block(unsigned caps) { + HEAP_CHECK(caps == (MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA)); + return heap_test::heap().dma_largest; +} +''' + +WRAPPER = r''' +int main(int argc, char** argv) { + auto& heap = heap_test::heap(); + const char* mode = argc > 1 ? argv[1] : "trace"; + const bool fallback = !std::strcmp(mode, "fallback"); + const bool lifetime = !std::strcmp(mode, "lifetime"); + if (fallback || !std::strcmp(mode, "no-memory") + || !std::strcmp(mode, "reserve") || !std::strcmp(mode, "fragmented")) + heap.psram = false; + if (!std::strcmp(mode, "no-memory")) heap.internal = false; + if (!std::strcmp(mode, "reserve")) heap.dma_free = 16384 + 17920 - 1; + if (!std::strcmp(mode, "fragmented")) heap.dma_largest = 17920 - 1; + char trace[] = "trace"; + char life[] = "lifetime"; + char* args[] = {argv[0], lifetime ? life : trace, nullptr}; + const size_t dma_before = heap.dma_free; + const int result = traceMain(2, args); + CHECK(heap.blocks.empty() && heap.dma_free == dma_before); +#if MESH_REPEATER_PSRAM_TABLES + CHECK(heap.last_bytes == 128 * sizeof(mesh::FloodAdvertLimiter::Entry)); + CHECK(heap.external_calls == (lifetime ? 3u : 1u)); + CHECK(heap.internal_calls == (fallback ? 1u : 0u)); + CHECK(heap.frees == (lifetime ? 3u : 1u)); +#else + CHECK(heap.external_calls == 0 && heap.internal_calls == 0 && heap.frees == 0); +#endif + return result; +} +''' + + +class FloodAdvertPsramTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temp = tempfile.TemporaryDirectory(prefix="advert-psram-") + cls.work = Path(cls.temp.name) + (cls.work / "esp_heap_caps.h").write_text(HEAP) + (cls.work / "sdkconfig.h").write_text( + "#define CONFIG_IDF_TARGET_ESP32S3 1\n" + "#define CONFIG_SPIRAM_BOOT_INIT 1\n" + "#define CONFIG_SPIRAM_USE_MALLOC 1\n") + cls.compiler = [os.environ.get("CXX", "g++")] + cls.binaries = {} + + @classmethod + def tearDownClass(cls): + cls.temp.cleanup() + + def build(self, enabled, collisions=False): + key = (enabled, collisions) + if key in self.binaries: + return self.binaries[key] + name = "probe-%d-%d" % key + source = self.work / (name + ".cpp") + binary = self.work / name + code = HARNESS.replace("@BANKS@", "").replace("int main(", "int traceMain(") + # Release builds of the limiter are exercised while harness assertions + # stay active. The trace also checks exact full-key prefix collisions. + code = code.rstrip()[:-1] + " return 0;\n}\n" + if collisions: + code = code.replace("key[0] = uint8_t(i); key[1] = uint8_t(i >> 8);", + "key[0] = uint8_t(i / 2); key[1] = uint8_t(i >> 8);") + source.write_text("#include \n" + code + WRAPPER) + args = ["-std=c++11", "-Os", "-DNDEBUG", "-Wall", "-Wextra", "-Werror", + "-fsanitize=address,undefined", "-fno-sanitize-recover=all", + "-fno-pie", "-no-pie", "-DESP32_PLATFORM=1", "-DESP_PLATFORM=1", + "-DBOARD_HAS_PSRAM=1", "-DSTATION_G3_ESP32=1", + "-DMESH_CLIENT_REPEATER_ONLY=1", + "-DMESH_REPEATER_PSRAM_TABLES=" + str(enabled), + "-I", str(self.work), "-isystem", str(ROOT / "src"), + "-isystem", str(ROOT / "test/mocks")] + result = subprocess.run( + self.compiler + args + [str(source), + str(ROOT / "src/helpers/FloodAdvertLimiter.cpp"), "-o", str(binary)], + capture_output=True, text=True, timeout=30) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.binaries[key] = binary + return binary + + def run_probe(self, enabled, mode="trace", collisions=False): + return subprocess.run([str(self.build(enabled, collisions)), mode], + capture_output=True, text=True, timeout=20) + + def test_full_capacity_quota_duplicate_badlist_and_rollover_traces_match(self): + for collisions in (False, True): + with self.subTest(collisions=collisions): + baseline = self.run_probe(0, collisions=collisions) + self.assertEqual(baseline.returncode, 0, baseline.stderr) + self.assertIn("limited=128", baseline.stdout) + for mode in ("trace", "fallback"): + result = self.run_probe(1, mode, collisions) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, baseline.stdout) + + def test_owner_recreation_clears_history_and_releases_every_allocation(self): + baseline = self.run_probe(0, "lifetime") + result = self.run_probe(1, "lifetime") + self.assertEqual(baseline.returncode, 0, baseline.stderr) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, baseline.stdout) + + def test_exhausted_or_reserved_internal_memory_stops_startup(self): + def no_core(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + binary = self.build(1) + for mode in ("no-memory", "reserve", "fragmented"): + with self.subTest(mode=mode): + result = subprocess.run([str(binary), mode], capture_output=True, + text=True, preexec_fn=no_core, timeout=20) + self.assertEqual(result.returncode, -signal.SIGABRT, result.stderr) + self.assertIn("Cannot allocate advert source history (17920 bytes)", + result.stderr) + self.assertNotIn("runtime error:", result.stderr) + self.assertNotIn("AddressSanitizer", result.stderr) + + def test_selected_owner_retains_no_full_internal_fallback(self): + source = self.work / "layout.cpp" + source.write_text('#include \n' + 'static_assert(sizeof(mesh::StaticFloodAdvertLimiter<>) < 256, ' + '"advert owner must not retain a full inline table");\n') + result = subprocess.run(self.compiler + ["-std=c++11", "-fsyntax-only", + "-DESP32_PLATFORM=1", "-DESP_PLATFORM=1", "-DBOARD_HAS_PSRAM=1", + "-DSTATION_G3_ESP32=1", "-DMESH_CLIENT_REPEATER_ONLY=1", + "-DMESH_REPEATER_PSRAM_TABLES=1", "-I", str(self.work), + "-isystem", str(ROOT / "src"), "-isystem", str(ROOT / "test/mocks"), + str(source)], capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_repeater_scope_psram.py b/test/test_repeater_scope_psram.py new file mode 100644 index 000000000..6a6b1b776 --- /dev/null +++ b/test/test_repeater_scope_psram.py @@ -0,0 +1,375 @@ +"""Exercise actual scope table owners, persistence and rollback in both RAM backends.""" +from pathlib import Path +import os +import resource +import signal +import subprocess +import tempfile +import unittest + +from cpp_source import body +from test_flood_advert_psram import HEAP + + +ROOT = Path(__file__).resolve().parents[1] +SDK = ("#define CONFIG_IDF_TARGET_ESP32S3 1\n" + "#define CONFIG_SPIRAM_BOOT_INIT 1\n" + "#define CONFIG_SPIRAM_USE_MALLOC 1\n") +DEFINES = ["ESP32_PLATFORM=1", "BOARD_HAS_PSRAM=1", "STATION_G3_ESP32=1", + "MESH_CLIENT_REPEATER_ONLY=1"] + +PRELUDE = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define CHECK(value) do { if (!(value)) std::abort(); } while (0) +#define FLOOD_CHANNEL_SCOPE_SLOTS 255 +#define FLOOD_CHANNEL_DIRECT_SCOPE_SLOTS 32 +#define FLOOD_CHANNEL_SCOPE_REQUIRE_SLOTS 255 +#define FLOOD_PACKET_FILTER_SCOPE_NAME_LEN 32 +#define FLOOD_CHANNEL_SCOPE_TXT_ANY 0 +#define FLOOD_CHANNEL_SCOPE_OTHER_ANY 2 +#define MESH_ENABLE_FLOOD_RULE_ENGINE 0 +#define FLOOD_CHANNEL_SCOPE_FILE "/scopes" +#define FLOOD_CHANNEL_SCOPE_TEMP_FILE "/scopes.tmp" +#define FLOOD_CHANNEL_SCOPE_REQUIRE_FILE "/require" +#define FLOOD_CHANNEL_SCOPE_REQUIRE_TEMP_FILE "/require.tmp" +struct RegionMap { + static bool is_name_char(uint8_t c) { return std::isalnum(c) || c == '_' || c == '-'; } +}; +namespace mesh { struct Utils { + static void sha256(void* out, size_t size, const uint8_t*, size_t length) { + CHECK(size == 1 && (length == 16 || length == 32)); + std::memset(out, 0, size); + } +}; } +static unsigned external_io = 0; +static void inspectSpan(const void* data, size_t size) { + const uintptr_t address = reinterpret_cast(data); + for (const auto& block : heap_test::heap().blocks) { + const uintptr_t start = reinterpret_cast(block.first); + if (address >= start && address < start + block.second.bytes) { + CHECK(size <= start + block.second.bytes - address); + if (block.second.caps & MALLOC_CAP_SPIRAM) ++external_io; + } + } +} +struct MemoryFS; +struct File { + MemoryFS* fs = nullptr; + std::string path; + size_t cursor = 0; + bool valid = false; + explicit operator bool() const { return valid; } + size_t read(uint8_t*, size_t); + size_t write(const uint8_t*, size_t); + void close() { valid = false; } +}; +struct MemoryFS { + std::map> files; + bool fail_rename = false, short_write = false; + bool exists(const char* path) { return files.count(path); } + bool remove(const char* path) { return files.erase(path); } + bool rename(const char* from, const char* to) { + if (fail_rename || !files.count(from)) return false; + files[to] = files[from]; files.erase(from); return true; + } +}; +size_t File::read(uint8_t* out, size_t size) { + if (!valid) return 0; + const auto& bytes = fs->files[path]; + size = std::min(size, bytes.size() - cursor); + inspectSpan(out, size); + if (size) std::memcpy(out, bytes.data() + cursor, size); + cursor += size; return size; +} +size_t File::write(const uint8_t* data, size_t size) { + if (!valid) return 0; + if (fs->short_write && size) --size; + inspectSpan(data, size); + auto& bytes = fs->files[path]; + bytes.insert(bytes.end(), data, data + size); return size; +} +static File openFloodSettingsRead(MemoryFS* fs, const char* path) { + return File{fs, path, 0, fs->exists(path)}; +} +static File openFloodSettingsWrite(MemoryFS* fs, const char* path) { + fs->files[path].clear(); return File{fs, path, 0, true}; +} +''' + +SCENARIOS = r''' +static std::vector snapshot(const MyMesh& mesh) { + std::vector bytes; + const auto add = [&](const void* data, size_t size) { + const auto first = static_cast(data); + bytes.insert(bytes.end(), first, first + size); + }; + add(mesh.flood_channel_scopes, sizeof(mesh.flood_channel_scopes)); + add(mesh.flood_channel_direct_scopes, sizeof(mesh.flood_channel_direct_scopes)); + add(mesh.flood_channel_scope_requirements, sizeof(mesh.flood_channel_scope_requirements)); + return bytes; +} +static void populate(MyMesh& mesh) { + for (unsigned i = 0; i < 32; ++i) + std::snprintf(mesh.flood_channel_direct_scopes[i], 32, "#Scope%u", i); + for (unsigned i = 0; i < 255; ++i) { + auto& row = mesh.flood_channel_scopes[i]; + row.target_id = i % 32 + 1; + row.selector = FloodFilterPolicy::encodeChannelScopeTargetSelector(0, true); + auto& requirement = mesh.flood_channel_scope_requirements[i]; + requirement.key_len = 32; + std::memset(requirement.secret, uint8_t(i + 1), sizeof(requirement.secret)); + } +} +static void exercise(const char* mode) { + MemoryFS disk; + { + MyMesh writer(disk); + static_assert(sizeof(writer.flood_channel_scopes) == 255 * 36, "all scope slots"); + static_assert(sizeof(writer.flood_channel_direct_scopes) == 32 * 32, "all direct names"); + static_assert(sizeof(writer.flood_channel_scope_requirements) == 255 * 34, "all requirements"); +#if MESH_REPEATER_PSRAM_TABLES + static_assert(sizeof(MyMesh) < 256, "no inline fallback arrays"); + auto& heap = heap_test::heap(); + CHECK(heap.blocks.size() == 1); + CHECK(heap.last_bytes == 18874); + const auto caps = heap.blocks.begin()->second.caps; + CHECK(caps == (!std::strcmp(mode, "fallback") + ? (MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT) + : (MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT))); + const unsigned calls = heap.external_calls + heap.internal_calls; + CHECK(writer.flood_channel_storage.begin()); + CHECK(heap.external_calls + heap.internal_calls == calls); +#else + CHECK(heap_test::heap().blocks.empty()); + (void)mode; +#endif + CHECK(snapshot(writer) == std::vector(18874, 0)); + const void* scopes = writer.flood_channel_scopes; + const void* direct = writer.flood_channel_direct_scopes; + const void* require = writer.flood_channel_scope_requirements; + populate(writer); + const auto initial = snapshot(writer); + CHECK(writer.saveFloodChannelScopes()); + CHECK(writer.saveFloodChannelScopeRequirements()); + CHECK(disk.files["/scopes"].size() == 10210); + CHECK(disk.files["/require"].size() == 8675); + std::vector expected = {'F','C','S','5',255}; + const auto* start = reinterpret_cast(writer.flood_channel_scopes); + expected.insert(expected.end(), start, start + 9180); + expected.push_back(32); + start = reinterpret_cast(writer.flood_channel_direct_scopes); + expected.insert(expected.end(), start, start + 1024); + CHECK(disk.files["/scopes"] == expected); + expected = {'F','C','R','1',255}; + start = reinterpret_cast(writer.flood_channel_scope_requirements); + expected.insert(expected.end(), start, start + 8670); + CHECK(disk.files["/require"] == expected); + const auto saved = disk.files; + char reply[160]; + for (bool short_write : {false, true}) { + disk.fail_rename = !short_write; disk.short_write = short_write; + writer.deleteFloodChannelScope(".255", reply); + CHECK(!std::strcmp(reply, "Err - save failed")); + CHECK(snapshot(writer) == initial && disk.files == saved); + writer.deleteFloodChannelScopeRequirement(".255", reply); + CHECK(!std::strcmp(reply, "Err - save failed")); + CHECK(snapshot(writer) == initial && disk.files == saved); + writer.deleteFloodChannelScope("all", reply); + CHECK(!std::strcmp(reply, "Err - save failed")); + CHECK(snapshot(writer) == initial && disk.files == saved); + writer.deleteFloodChannelScopeRequirement("all", reply); + CHECK(!std::strcmp(reply, "Err - save failed")); + CHECK(snapshot(writer) == initial && disk.files == saved); + } + disk.fail_rename = disk.short_write = false; + std::memset(writer.flood_channel_scopes, 0xa5, sizeof(writer.flood_channel_scopes)); + std::memset(writer.flood_channel_direct_scopes, 0xa5, sizeof(writer.flood_channel_direct_scopes)); + std::memset(writer.flood_channel_scope_requirements, 0xa5, sizeof(writer.flood_channel_scope_requirements)); + CHECK(writer.loadFloodChannelScopes()); + writer.loadFloodChannelScopeRequirements(); + CHECK(snapshot(writer) == initial); + disk.files["/scopes"].pop_back(); + disk.files["/require"].pop_back(); + CHECK(!writer.loadFloodChannelScopes()); + writer.loadFloodChannelScopeRequirements(); + CHECK(snapshot(writer) == std::vector(18874, 0)); + disk.files = saved; + CHECK(writer.loadFloodChannelScopes()); + writer.loadFloodChannelScopeRequirements(); + CHECK(snapshot(writer) == initial); + CHECK(scopes == writer.flood_channel_scopes && direct == writer.flood_channel_direct_scopes + && require == writer.flood_channel_scope_requirements); + // Compare the complete production file payloads between inline and PSRAM runs. + for (const char* path : {"/scopes", "/require"}) { + const auto& bytes = disk.files[path]; + CHECK(std::fwrite(bytes.data(), 1, bytes.size(), stdout) == bytes.size()); + } + writer.deleteFloodChannelScope(".255", reply); + CHECK(!std::strcmp(reply, "OK") && writer.flood_channel_scopes[254].target_id == 0); + writer.deleteFloodChannelScopeRequirement(".255", reply); + CHECK(!std::strcmp(reply, "OK") && writer.flood_channel_scope_requirements[254].key_len == 0); + writer.deleteFloodChannelScope("all", reply); + CHECK(!std::strcmp(reply, "OK")); + writer.deleteFloodChannelScopeRequirement("all", reply); + CHECK(!std::strcmp(reply, "OK")); + CHECK(snapshot(writer) == std::vector(18874, 0)); + } + CHECK(heap_test::heap().blocks.empty()); + MyMesh recreated(disk); + CHECK(snapshot(recreated) == std::vector(18874, 0)); +} +int main(int argc, char** argv) { + const char* mode = argc > 1 ? argv[1] : "trace"; + auto& heap = heap_test::heap(); + if (std::strcmp(mode, "trace")) heap.psram = false; + if (!std::strcmp(mode, "no-memory")) heap.internal = false; + if (!std::strcmp(mode, "reserve")) heap.dma_free = 16384 + 18874 - 1; + if (!std::strcmp(mode, "fragmented")) heap.dma_largest = 18874 - 1; + if (!std::strcmp(mode, "boundary")) { + heap.dma_free = 16384 + 18874; + heap.dma_largest = 18874; + mode = "fallback"; + } + const size_t free_before = heap.dma_free; + exercise(mode); + CHECK(heap.blocks.empty() && heap.dma_free == free_before); +#if MESH_REPEATER_PSRAM_TABLES + CHECK(heap.external_calls == 2 && heap.frees == 2); + CHECK(heap.internal_calls == (!std::strcmp(mode, "fallback") ? 2u : 0u)); + CHECK(external_io > 0 || !std::strcmp(mode, "fallback")); +#else + CHECK(heap.external_calls == 0 && heap.internal_calls == 0 && heap.frees == 0); +#endif +} +''' + + +def production_program(): + header = (ROOT / "examples/simple_repeater/MyMesh.h").read_text(encoding="ascii") + source = (ROOT / "examples/simple_repeater/MyMesh.cpp").read_text(encoding="ascii") + location = header.index(" struct FloodChannelStorage {") + start = header.rfind("#if MESH_REPEATER_PSRAM_TABLES", 0, location) + declarations = header[start:header.index("#if MESH_ENABLE_FLOOD_GROUP_MODERATION", location)] + constructor = source.index("MyMesh::MyMesh(") + start = source.index("#if MESH_REPEATER_PSRAM_TABLES", constructor) + binders = source[start:source.index("#endif", start) + len("#endif")] + start = source.index(" memset(flood_channel_scopes, 0, sizeof(flood_channel_scopes));", constructor) + resets = source[start:source.index("#if MESH_ENABLE_FLOOD_GROUP_MODERATION", start)] + methods = [ + "bool MyMesh::loadFloodChannelScopes(", "bool MyMesh::saveFloodChannelScopes(", + "void MyMesh::loadFloodChannelScopeRequirements(", + "bool MyMesh::saveFloodChannelScopeRequirements(", + "void MyMesh::deleteFloodChannelScope(", + "void MyMesh::deleteFloodChannelScopeRequirement(", + ] + helpers = [ + "static const char* skipFloodFilterSpaces(", "static bool floodFilterAsciiEqual(", + "static bool parseFloodFilterUnsigned(", "static bool parseFloodChannelScopeIndex(", + "static bool isValidStoredFloodFilterScopeName(", + "static bool isExactFloodChannelScopeSelector(", + ] + owner = "class MyMesh {\npublic:\nMemoryFS* _fs;\n" + for name in ("FloodChannelScopeEntry", "FloodChannelScopeRequireEntry"): + owner += body(header, "struct " + name + " {") + ";\n" + owner += declarations + "\nint end_marker;\n" + owner += "explicit MyMesh(MemoryFS& fs) : _fs(&fs),\n" + binders + owner += "\nend_marker(0) {\n" + resets + "}\n" + owner += ("bool loadFloodChannelScopes();\n" + "bool saveFloodChannelScopes(bool empty_table=false);\n" + "void loadFloodChannelScopeRequirements();\n" + "bool saveFloodChannelScopeRequirements(bool empty_table=false);\n" + "void deleteFloodChannelScope(const char*, char*);\n" + "void deleteFloodChannelScopeRequirement(const char*, char*);\n};\n") + return "\n".join([PRELUDE, owner] + [body(source, item) for item in helpers + methods] + [SCENARIOS]) + + +class RepeaterScopePsramTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temporary = tempfile.TemporaryDirectory(prefix="repeater-scope-psram-") + cls.work = Path(cls.temporary.name) + (cls.work / "esp_heap_caps.h").write_text(HEAP, encoding="ascii") + (cls.work / "sdkconfig.h").write_text(SDK, encoding="ascii") + (cls.work / "scope.cpp").write_text(production_program(), encoding="ascii") + cls.compiler = [os.environ.get("CXX", "g++")] + cls.binaries = {} + + @classmethod + def tearDownClass(cls): + cls.temporary.cleanup() + + def build(self, enabled): + if enabled not in self.binaries: + binary = self.work / ("scope-%d" % enabled) + command = self.compiler + ["-std=c++17", "-O1", "-g", "-DNDEBUG", + "-Wall", "-Wextra", "-Werror", "-Wno-sign-compare", + "-fsanitize=address,undefined", "-fno-sanitize-recover=all", + "-fno-pie", "-no-pie", "-I", str(self.work), + "-isystem", str(ROOT / "src"), "-isystem", str(ROOT / "test/mocks")] + command += ["-D" + define for define in DEFINES] + command += ["-DMESH_REPEATER_PSRAM_TABLES=%d" % enabled, + str(self.work / "scope.cpp"), "-o", str(binary)] + result = subprocess.run(command, capture_output=True, text=True, timeout=60) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.binaries[enabled] = binary + return self.binaries[enabled] + + def test_full_table_file_bytes_loads_resets_lifetime_and_rollback_match(self): + baseline = subprocess.run([str(self.build(0))], capture_output=True, timeout=20) + self.assertEqual(baseline.returncode, 0, baseline.stderr.decode()) + self.assertEqual(len(baseline.stdout), 18885) + for mode in ("trace", "fallback", "boundary"): + with self.subTest(mode=mode): + result = subprocess.run([str(self.build(1)), mode], capture_output=True, timeout=20) + self.assertEqual(result.returncode, 0, result.stderr.decode()) + self.assertEqual(result.stdout, baseline.stdout) + + def test_no_capacity_reduction_or_null_references_on_allocation_failure(self): + def no_core(): + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + for mode in ("no-memory", "reserve", "fragmented"): + with self.subTest(mode=mode): + result = subprocess.run([str(self.build(1)), mode], capture_output=True, + text=True, timeout=20, preexec_fn=no_core) + self.assertEqual(result.returncode, -signal.SIGABRT, result.stderr) + self.assertIn("Cannot allocate channel scope tables (18874 bytes)", result.stderr) + self.assertNotIn("runtime error:", result.stderr) + self.assertNotIn("AddressSanitizer", result.stderr) + + def test_default_selection_and_explicit_sdk_prerequisites(self): + probe = self.work / "guard.cpp" + probe.write_text('#include \n' + 'static_assert(MESH_REPEATER_PSRAM_TABLES == EXPECTED, "selection");\n', + encoding="ascii") + cases = [([], 0, True, SDK), (["ESP32_PLATFORM=1", "STATION_G3_ESP32=1"], 0, True, SDK), + (["ESP32_PLATFORM=1", "BOARD_HAS_PSRAM=1"], 0, True, SDK), + (DEFINES[:-1], 0, True, SDK), + (DEFINES, 1, True, SDK), (DEFINES + ["MESH_REPEATER_PSRAM_TABLES=0"], 0, True, ""), + (["MESH_REPEATER_PSRAM_TABLES=1"], 1, False, SDK), + (DEFINES + ["MESH_REPEATER_PSRAM_TABLES=2"], 2, False, SDK), + (DEFINES, 1, False, SDK.replace("CONFIG_IDF_TARGET_ESP32S3 1", "CONFIG_IDF_TARGET_ESP32 1")), + (DEFINES, 1, False, SDK.replace("CONFIG_SPIRAM_BOOT_INIT 1", "CONFIG_SPIRAM_BOOT_INIT 0")), + (DEFINES, 1, False, SDK.replace("CONFIG_SPIRAM_USE_MALLOC 1", "CONFIG_SPIRAM_USE_MALLOC 0"))] + for index, (defines, expected, success, config) in enumerate(cases): + with self.subTest(index=index, defines=defines): + config_dir = self.work / ("guard-%d" % index) + config_dir.mkdir() + (config_dir / "sdkconfig.h").write_text(config, encoding="ascii") + command = self.compiler + ["-std=c++17", "-fsyntax-only", "-I", str(config_dir), + "-I", str(self.work), "-I", str(ROOT / "src"), "-DEXPECTED=%d" % expected] + command += ["-D" + define for define in defines] + [str(probe)] + result = subprocess.run(command, capture_output=True, text=True, timeout=20) + self.assertEqual(result.returncode == 0, success, result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_ws_transport_psram.py b/test/test_ws_transport_psram.py new file mode 100644 index 000000000..717025d74 --- /dev/null +++ b/test/test_ws_transport_psram.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +"""Exercise the actual WS wrapper with SDK allocation/lifetime stubs.""" + +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + +HEADERS = { + "esp_idf_version.h": "#pragma once\n", + "sdkconfig.h": "#pragma once\n#define CONFIG_WS_BUFFER_SIZE 1024\n", + "esp_transport.h": """ +#pragma once +#include +struct TestTransport; +typedef TestTransport* esp_transport_handle_t; +extern "C" void* esp_transport_get_context_data(esp_transport_handle_t); +extern "C" int esp_transport_destroy(esp_transport_handle_t); +""", + "esp_transport_ws.h": '#pragma once\n#include "esp_transport.h"\n', + "esp_heap_caps.h": """ +#pragma once +#include +#include +#define MALLOC_CAP_SPIRAM 1U +#define MALLOC_CAP_8BIT 2U +#define MALLOC_CAP_INTERNAL 4U +#define MALLOC_CAP_DMA 8U +void* heap_caps_malloc(size_t, uint32_t); +size_t heap_caps_get_free_size(uint32_t); +size_t heap_caps_get_largest_free_block(uint32_t); +""", +} + +HARNESS = r""" +#include +#include +#include +#include +#include +#include "esp_heap_caps.h" + +static void* padded_malloc(size_t); +static void sdk_free(void*); +#define malloc padded_malloc +#define free sdk_free +#include "src/helpers/ESP32WsTransportFix.cpp" +#undef free +#undef malloc + +struct TestTransport { + void* context; +}; + +#if ESP_IDF_VERSION_MAJOR == 4 +using TestWs = transport_ws_t; +#else +struct TestWs { + char* path; + char* buffer; + char* sub_protocol; + char* user_agent; + char* headers; + esp_transport_handle_t parent; +}; +#endif + +enum Kind { SDK, GENERIC, EXTERNAL, INTERNAL }; +struct Allocation { + void* ptr; + size_t size; + Kind kind; +}; +static Allocation allocations[32]; +static size_t allocation_count; +static unsigned destroy_calls; +static unsigned original_frees; +static unsigned generic_calls; +static unsigned external_calls; +static unsigned internal_calls; +static unsigned free_queries; +static unsigned largest_queries; +static bool fail_sdk; +static bool fail_external; +static bool fail_internal; +static bool fail_generic; +static size_t dma_free = 65536; +static size_t dma_largest = 8192; +static char* original; +static TestTransport parent; + +static void* allocate(size_t bytes, Kind kind) { + assert(allocation_count < 32); + void* p = std::calloc(1, bytes); + assert(p); + allocations[allocation_count++] = {p, bytes, kind}; + return p; +} + +static Allocation lookup(void* ptr) { + for (size_t i = 0; i < allocation_count; ++i) { + if (allocations[i].ptr == ptr) return allocations[i]; + } + std::abort(); +} + +static void sdk_free(void* ptr) { + if (!ptr) return; + for (size_t i = 0; i < allocation_count; ++i) { + if (allocations[i].ptr != ptr) continue; + if (ptr == original) ++original_frees; + allocations[i] = allocations[--allocation_count]; + std::free(ptr); + return; + } + std::abort(); // Double free, wrong allocator, or parent destruction. +} + +static void* padded_malloc(size_t bytes) { + ++generic_calls; + assert(bytes == 1025); + return fail_generic ? nullptr : allocate(bytes, GENERIC); +} + +void* heap_caps_malloc(size_t bytes, uint32_t caps) { + assert(bytes == 1025); + if (caps == (MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)) { + ++external_calls; + return fail_external ? nullptr : allocate(bytes, EXTERNAL); + } + assert(caps == (MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA | MALLOC_CAP_8BIT)); + ++internal_calls; + assert(dma_free >= 16384 + bytes && dma_largest >= bytes); + return fail_internal ? nullptr : allocate(bytes, INTERNAL); +} + +size_t heap_caps_get_free_size(uint32_t caps) { + assert(caps == (MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA)); + ++free_queries; + return dma_free; +} + +size_t heap_caps_get_largest_free_block(uint32_t caps) { + assert(caps == (MALLOC_CAP_INTERNAL | MALLOC_CAP_DMA)); + ++largest_queries; + return dma_largest; +} + +extern "C" void* esp_transport_get_context_data(esp_transport_handle_t t) { + assert(t && t != &parent); + return t->context; +} + +extern "C" int esp_transport_destroy(esp_transport_handle_t t) { + assert(t && t != &parent); + ++destroy_calls; + TestWs* ws = static_cast(t->context); + assert(ws->parent == &parent); + sdk_free(ws->buffer); + sdk_free(ws->path); + sdk_free(ws->sub_protocol); + sdk_free(ws->user_agent); + sdk_free(ws->headers); + sdk_free(ws); + sdk_free(t); + return 0; +} + +extern "C" esp_transport_handle_t __real_esp_transport_ws_init(esp_transport_handle_t p) { + assert(p == &parent); + if (fail_sdk) return nullptr; + auto* t = static_cast(allocate(sizeof(TestTransport), SDK)); + auto* ws = static_cast(allocate(sizeof(TestWs), SDK)); + t->context = ws; + ws->parent = p; + ws->path = static_cast(allocate(2, SDK)); + ws->sub_protocol = static_cast(allocate(5, SDK)); + ws->user_agent = static_cast(allocate(8, SDK)); + ws->headers = static_cast(allocate(8, SDK)); + ws->buffer = static_cast(allocate(1024, SDK)); + original = ws->buffer; + return t; +} + +int main(int argc, char** argv) { + assert(argc == 2); + const std::string scenario(argv[1]); + bool expect_failure = false; + Kind expected = GENERIC; + if (scenario == "sdk_failure") { + fail_sdk = true; + expect_failure = true; + } else if (scenario == "external_success") { + // External success must not inspect or consume scarce internal memory. + dma_free = 0; + dma_largest = 0; + expected = EXTERNAL; + } else if (scenario == "internal_boundary") { + fail_external = true; + dma_free = 16384 + 1025; + dma_largest = 1025; + expected = INTERNAL; + } else if (scenario == "reserve_denied") { + fail_external = true; + dma_free = 16384 + 1024; + expect_failure = true; + } else if (scenario == "largest_denied") { + fail_external = true; + dma_largest = 1024; + expect_failure = true; + } else if (scenario == "both_fail") { + fail_external = true; + fail_internal = true; + expect_failure = true; + } else if (scenario == "generic_failure") { + fail_generic = true; + expect_failure = true; + } else { + assert(scenario == "generic_success" || scenario == "idf5_passthrough"); + } + + esp_transport_handle_t t = __wrap_esp_transport_ws_init(&parent); + if (expect_failure) { + assert(t == nullptr); + assert(allocation_count == 0); + assert(destroy_calls == (fail_sdk ? 0U : 1U)); + assert(original_frees == (fail_sdk ? 0U : 1U)); + } else { + assert(t && destroy_calls == 0); + auto* ws = static_cast(t->context); + Allocation a = lookup(ws->buffer); + if (scenario == "idf5_passthrough") { + assert(ws->buffer == original && a.size == 1024 && a.kind == SDK); + assert(original_frees == 0); + } else { + assert(ws->buffer != original && a.size == 1025 && a.kind == expected); + assert(original_frees == 1); + // Exercise the exact off-by-one index the workaround must retain. + ws->buffer[1024] = '\0'; + } + esp_transport_destroy(t); + assert(allocation_count == 0 && destroy_calls == 1 && original_frees == 1); + } + if (scenario == "external_success") { + assert(external_calls == 1 && internal_calls == 0 && generic_calls == 0); + assert(free_queries == 0 && largest_queries == 0); + } else if (scenario == "reserve_denied" || scenario == "largest_denied") { + assert(external_calls == 1 && internal_calls == 0 && generic_calls == 0); + assert(free_queries == 1 && largest_queries == 1); + } else if (scenario == "internal_boundary" || scenario == "both_fail") { + assert(external_calls == 1 && internal_calls == 1 && generic_calls == 0); + assert(free_queries == 1 && largest_queries == 1); + } else if (scenario == "generic_success" || scenario == "generic_failure") { + assert(external_calls == 0 && internal_calls == 0 && generic_calls == 1); + } else { + assert(external_calls == 0 && internal_calls == 0 && generic_calls == 0); + } +} +""" + + +class WsTransportPsramTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++") + if not compiler: + raise unittest.SkipTest("C++ compiler unavailable") + cls.temp = tempfile.TemporaryDirectory(prefix="ws-transport-psram-") + cls.addClassCleanup(cls.temp.cleanup) + work = Path(cls.temp.name) + for name, contents in HEADERS.items(): + (work / name).write_text(contents, encoding="ascii") + source = work / "harness.cpp" + source.write_text(HARNESS, encoding="ascii") + cls.binaries = {} + variants = { + "g3_psram": (4, ["STATION_G3_ESP32", "BOARD_HAS_PSRAM"]), + "other_psram": (4, ["BOARD_HAS_PSRAM"]), + "g3_internal": (4, ["STATION_G3_ESP32"]), + "other_internal": (4, []), + "idf5": (5, ["STATION_G3_ESP32", "BOARD_HAS_PSRAM"]), + } + for name, (idf, macros) in variants.items(): + binary = work / name + command = [ + compiler, "-std=c++17", "-O1", "-g", "-Wall", "-Wextra", "-Werror", + "-Wno-unused-function", "-DESP_PLATFORM", f"-DESP_IDF_VERSION_MAJOR={idf}", + *[f"-D{macro}" for macro in macros], "-I", str(work), "-I", str(ROOT), + ] + if os.name != "nt": + command += ["-fsanitize=address,undefined", "-fno-sanitize-recover=all", + "-fno-pie", "-no-pie"] + built = subprocess.run([*command, str(source), "-o", str(binary)], + capture_output=True, text=True) + if built.returncode: + raise RuntimeError(built.stdout + built.stderr) + cls.binaries[name] = binary + + def run_scenario(self, variant, scenario): + result = subprocess.run([str(self.binaries[variant]), scenario], + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_g3_psram_and_guarded_failure_lifecycles(self): + for scenario in ["external_success", "internal_boundary", "reserve_denied", + "largest_denied", "both_fail", "sdk_failure"]: + with self.subTest(scenario=scenario): + self.run_scenario("g3_psram", scenario) + + def test_other_target_allocation_defaults_and_failure_cleanup(self): + for variant in ["other_psram", "g3_internal", "other_internal"]: + for scenario in ["generic_success", "generic_failure", "sdk_failure"]: + with self.subTest(variant=variant, scenario=scenario): + self.run_scenario(variant, scenario) + + def test_idf5_is_unchanged_passthrough(self): + self.run_scenario("idf5", "idf5_passthrough") + self.run_scenario("idf5", "sdk_failure") + + +if __name__ == "__main__": + unittest.main()