diff --git a/boards/waveshare_s3_lr1121_migrator.json b/boards/waveshare_s3_lr1121_migrator.json new file mode 100644 index 000000000..bb7e4dcee --- /dev/null +++ b/boards/waveshare_s3_lr1121_migrator.json @@ -0,0 +1,37 @@ +{ + "build": { + "arduino": {"ldscript": "esp32s3_out.ld"}, + "core": "esp32", + "extra_flags": [ + "-D ARDUINO_USB_CDC_ON_BOOT=0", + "-D ARDUINO_USB_MSC_ON_BOOT=0", + "-D ARDUINO_USB_DFU_ON_BOOT=0", + "-D ARDUINO_USB_MODE=1", + "-D ARDUINO_RUNNING_CORE=1", + "-D ARDUINO_EVENT_RUNNING_CORE=1" + ], + "f_cpu": "240000000L", + "f_flash": "80000000L", + "flash_mode": "dio", + "hwids": [["0x303A", "0x1001"]], + "mcu": "esp32s3", + "variant": "esp32s3" + }, + "connectivity": ["wifi", "bluetooth"], + "debug": { + "default_tool": "esp-builtin", + "onboard_tools": ["esp-builtin"], + "openocd_target": "esp32s3.cfg" + }, + "frameworks": ["arduino", "espidf"], + "name": "Waveshare ESP32-S3 LR1121 partition migrator (USB CDC disabled)", + "upload": { + "flash_size": "4MB", + "maximum_ram_size": 327680, + "maximum_size": 1310720, + "require_upload_port": true, + "speed": 921600 + }, + "url": "https://www.waveshare.com/", + "vendor": "Waveshare" +} diff --git a/docs/esp32_wifi_partition_migration.md b/docs/esp32_wifi_partition_migration.md index 85384c204..9be4431ba 100644 --- a/docs/esp32_wifi_partition_migration.md +++ b/docs/esp32_wifi_partition_migration.md @@ -34,7 +34,7 @@ table-prefix test; the bridge core is otherwise shared. | Flash size | Target table | app0 / app1 size | Typical board | | --- | --- | --- | --- | -| 4 MiB | `variants/dual_ota_full_4MB.csv` | 0x1F0000 / 0x1F0000 | ThinkNode M2/M5, LilyGo T3S3, Nibble, Heltec CT62 | +| 4 MiB | `variants/dual_ota_full_4MB.csv` | 0x1F0000 / 0x1F0000 | ThinkNode M2/M5, LilyGo T3S3, Nibble, Heltec CT62, Waveshare LR1121 observers | | 8 MiB | `default_8MB.csv` | 0x330000 / 0x330000 | Seeed XIAO ESP32-S3, Heltec V3 | | 16 MiB | `default_16MB.csv` | 0x640000 / 0x640000 | Heltec V4 / V4.3, Station G2 | @@ -60,7 +60,7 @@ the bootloader, partition table, and application. Never upload a merged image through an app-only browser or LoRa updater. The new Full defaults outside the migration catalog, including ESP32-C6 -boards and boards introduced with `min_spiffs.csv`, have no reviewed wireless +boards and other boards introduced with `min_spiffs.csv`, have no reviewed wireless expansion package in this recipe. If their Full app fits the unchanged live layout and update identity, an ordinary app update can suffice. A table change requires USB/serial until a matching migration package is reviewed. A Full @@ -69,6 +69,45 @@ through an old updater. Full Companion images with a single app slot likewise need USB/serial for local firmware replacement; their host-backed LoRa sending role does not provide a second local update slot. +## Waveshare ESP32-S3 LR1121 observer migration + +Only the Waveshare **repeater observer** and **room-server observer** select the +larger 4 MiB table. Ordinary repeater, room, Companion and KISS recipes retain +`min_spiffs.csv`. The two exact migration selections are: + +```bash +sh scripts/build_esp32_partition_migration.sh \ + --board waveshare-s3-lr1121-observer-repeater \ + --board waveshare-s3-lr1121-observer-room-server \ + --version v1.18.0-test --radio-preset usa-cascadia --profile cascade --dry-run +``` + +Remove `--dry-run` only to build and verify the packages from a clean commit; +this command does not flash hardware. Each observer role has its own exact-target +Partition Expander. Both its Wi-Fi application and first-stage LoRa package use +that same configuration-preserving bridge, not the older identity-only LoRa +handoff. It accepts a reviewed source layout, stages and verifies identity and +supported configuration before destructive writes, and refuses unknown layouts +or saved data that cannot be preserved safely. The final table has two +1984 KiB application slots and only **64 KiB SPIFFS**. + +The bridge can migrate from either existing OTA slot. After reboot it must +verify the new table, private key and saved configuration before enabling the +LoRa receiver. It uses the saved primary radio profile and fetches only the +exact observer role's Full image. The node name, ACL, radio 1/radio 2 and saved +GPS preferences remain part of the configuration handoff; temporary sessions +are not converted into permanent preferences. Already-expanded boards use the +existing verified-image/no-handoff safety checks rather than migrating again. + +The recipes reuse the real LR1121 board implementation, including its 1.8 V +TCXO and HP-PA/RF-switch wiring. The first-stage image must pass the legacy +1.25 MiB size limit and both final-image and mOTA staging guards. A failed guard +must fail the package, not produce a reduced image. These are build/host-test +contracts, **not physical qualification**: test both A/B placements, restored +settings and interruptions on a sacrificial Waveshare before remote deployment. +The ESP32's single partition-table-sector erase/write is still not atomic; +power loss in that window can require cable recovery. + ## Repeatable ESP32 build recipe From a clean commit, run the shell menu in WSL/Linux: @@ -202,7 +241,7 @@ node name and preferences, radio profiles, ACL and login replay state, region keys, and selected network/OTA, filtering, display, and telemetry settings. It refuses migration if the saved files cannot fit or verify in NVS. The older LoRa handoff route preserves only the private identity, so saved settings may -reset. The two board-specific Partition Expander routes preserve their +reset. The board/role-specific Partition Expander routes preserve their supported configuration and automatically fetch the exact-target Full image; follow the package README for those routes. diff --git a/examples/esp32_partition_migrator/main.cpp b/examples/esp32_partition_migrator/main.cpp index 347fce4eb..7655f7fcc 100644 --- a/examples/esp32_partition_migrator/main.cpp +++ b/examples/esp32_partition_migrator/main.cpp @@ -80,6 +80,10 @@ constexpr ConfigFile kConfigFiles[] = { }; constexpr size_t kConfigFileCount = sizeof(kConfigFiles) / sizeof(kConfigFiles[0]); static_assert(kConfigFileCount <= 32, "config presence mask is 32 bits"); +#if defined(MESHCORE_MIGRATION_STRICT_4MB) && MESHCORE_MIGRATION_STRICT_4MB +static_assert(kConfigFileCount + 1 <= migration::PreservationBudget::kMaximumFiles, + "strict filesystem inventory must cover every staged config file"); +#endif struct ConfigStage { uint32_t magic; uint32_t present; @@ -337,6 +341,11 @@ bool stageLegacyIdentity() { } const bool saved = migration_nvs.putBytes(kMigrationIdentityKey, copy_buffer, kIdentityFileBytes) == kIdentityFileBytes +#if defined(MESHCORE_MIGRATION_STRICT_4MB) && MESHCORE_MIGRATION_STRICT_4MB + && migration_nvs.getBytes(kMigrationIdentityKey, partition_table_verified, + kIdentityFileBytes) == kIdentityFileBytes + && memcmp(copy_buffer, partition_table_verified, kIdentityFileBytes) == 0 +#endif && migration_nvs.putBool(kMigrationIdentityPendingKey, true); migration_nvs.end(); if (!saved) strcpy(status_text, "Could not save private key for migration"); @@ -594,6 +603,90 @@ bool restoreStagedConfig() { return restored; } +#if defined(MESHCORE_MIGRATION_STRICT_4MB) && MESHCORE_MIGRATION_STRICT_4MB +bool inventoryStrictDirectory(const char* path, unsigned depth, + migration::PreservationBudget& budget, + uint64_t& seen) { + if (depth > 1) { + strcpy(status_text, "Refused: unsupported saved-data directory nesting"); + return false; + } + File directory = SPIFFS.open(path, "r"); + if (!directory || !directory.isDirectory()) { + strcpy(status_text, "Refused: could not enumerate saved filesystem"); + return false; + } + bool valid = true; + for (File entry = directory.openNextFile(); valid && entry; + entry = directory.openNextFile()) { + const char* name = entry.path(); + if (!name || name[0] != '/' || strnlen(name, 96) >= 96) { + strcpy(status_text, "Refused: invalid saved-data path"); + valid = false; + } else if (entry.isDirectory()) { + // The preservation set has one known directory. Even an unknown empty + // directory is refused instead of guessing whether its content matters. + if (strcmp(name, "/identity") != 0 || depth != 0) { + snprintf(status_text, sizeof(status_text), "Refused: unsupported saved directory %.80s", name); + valid = false; + } else { + valid = inventoryStrictDirectory(name, depth + 1, budget, seen); + } + } else { + size_t index = 0; + if (strcmp(name, "/identity/_main.id") != 0) { + for (index = 0; index < kConfigFileCount; ++index) + if (strcmp(name, kConfigFiles[index].path) == 0) break; + if (index == kConfigFileCount) { + snprintf(status_text, sizeof(status_text), "Refused: unpreserved saved file %.80s", name); + valid = false; + } + ++index; // bit zero belongs only to the identity + } + if (valid) { + const uint64_t bit = uint64_t(1) << index; + const size_t bytes = entry.size(); + if ((seen & bit) != 0 || (index == 0 && bytes != kIdentityFileBytes)) { + strcpy(status_text, "Refused: ambiguous or unsupported private identity record"); + valid = false; + } else if (bytes > UINT32_MAX || !budget.addFile(static_cast(bytes))) { + snprintf(status_text, sizeof(status_text), "Refused: %.80s exceeds safe 64 KiB preservation budget", name); + valid = false; + } else { + seen |= bit; + } + } + } + entry.close(); + } + directory.close(); + return valid; +} + +bool preflightStrictPreservation(const migration::TargetPlan& plan) { + if (plan.flash_bytes != migration::kSmallRequiredFlashBytes || + plan.layout.spiffs_size != 0x10000 || !SPIFFS.begin(false)) { + strcpy(status_text, "Refused: strict source filesystem is unavailable"); + return false; + } + migration::PreservationBudget budget(plan.layout.spiffs_size); + uint64_t seen = 0; + bool valid = inventoryStrictDirectory("/", 0, budget, seen); + if (valid && (seen & 1U) == 0) { + strcpy(status_text, "Refused: private identity is absent from saved-data inventory"); + valid = false; + } + for (size_t i = 0; valid && i < kConfigFileCount; ++i) { + if (SPIFFS.exists(kConfigFiles[i].path) != ((seen & (uint64_t(1) << (i + 1))) != 0)) { + strcpy(status_text, "Refused: incomplete saved-data inventory"); + valid = false; + } + } + SPIFFS.end(); + return valid; +} +#endif + #if defined(MESHCORE_MIGRATION_RESUME_OTA) || defined(MOTA_MIGRATION_TARGET_ID) uint32_t readLe32(const uint8_t* bytes) { return static_cast(bytes[0]) @@ -905,6 +998,19 @@ void runMigration() { Serial.println(status_text); return; } +#if defined(MESHCORE_MIGRATION_STRICT_4MB) && MESHCORE_MIGRATION_STRICT_4MB + if (!migration::isReviewed4MBSource(flash_bytes, geometry)) { + strcpy(status_text, "Refused: source is not a reviewed 4 MiB partition layout"); + Serial.println(status_text); + return; + } + // Admission happens before identity/config staging, app copying, OTA-select + // writes and the non-atomic partition-table erase/write window. + if (!preflightStrictPreservation(*plan)) { + Serial.println(status_text); + return; + } +#endif const esp_partition_t* running = esp_ota_get_running_partition(); // Partition handles are opaque; their addresses, rather than their pointer diff --git a/scripts/package_esp32_partition_migration.py b/scripts/package_esp32_partition_migration.py index fa734689a..aa262155f 100644 --- a/scripts/package_esp32_partition_migration.py +++ b/scripts/package_esp32_partition_migration.py @@ -270,6 +270,26 @@ add_roles("generic-e22-sx1268", 4, { "repeater": "Generic_E22_sx1268_repeater", }, bridge="esp32_4mb_partition_migrator") +# Only these observer identities use the enlarged Waveshare table. Ordinary +# Waveshare roles retain min_spiffs.csv and must not silently change layout. +add_roles("waveshare-s3-lr1121-observer", 4, { + "repeater": "Waveshare_S3_LR1121_repeater_observer_mqtt", + "room-server": "Waveshare_S3_LR1121_room_server_observer_mqtt", +}) +for role, suffix in (("repeater", "repeater"), ("room-server", "room_server")): + specification = BOARDS[f"waveshare-s3-lr1121-observer-{role}"] + expander = f"Waveshare_S3_LR1121_partition_expander_{suffix}_observer_mqtt" + # Both wireless routes use the same self-hosted receiver, not the older + # identity-only resume bridge which can reset ACL/radio preferences. + specification.update({ + "wifi_bridge": expander, + "lora_bridge": expander, + "expander_bridge": expander, + "source_partition_table": "min_spiffs.csv", + "spiffs_address": 0x3F0000, + "spiffs_bytes": 0x10000, + }) + USB_BRIDGE_MARKER = b"MeshCore USB partition migration bridge v1" USB_NVS_GUARD_MARKER = b"MeshCore USB NVS no-autoerase guard v1" USB_BRIDGE_PROFILES = { @@ -288,7 +308,9 @@ def usb_bridge_profile(spec: dict) -> tuple[int, str]: if wifi.startswith("esp32_c3_"): chip = 5 elif wifi.startswith("esp32_s3_") or wifi in ( - "heltec_v4_partition_migrator", "xiao_s3_partition_migrator"): + "heltec_v4_partition_migrator", "xiao_s3_partition_migrator", + "Waveshare_S3_LR1121_partition_expander_repeater_observer_mqtt", + "Waveshare_S3_LR1121_partition_expander_room_server_observer_mqtt"): chip = 9 elif wifi.startswith("esp32_"): chip = 0 @@ -444,6 +466,8 @@ def check_esp32_stage(package: bytes, image: bytes, slot_bytes: int) -> None: def readme(board: str, spec: dict, version: str, source: str, has_full_mota: bool = True, full_mota_blocks: int = 0) -> str: + self_hosted_lora = bool(spec.get("expander_bridge")) and ( + spec.get("lora_bridge") == spec["expander_bridge"]) usb_return = ( "On native ESP32-S3/C3, it then returns to the chip's ROM download " "mode in software." @@ -473,7 +497,7 @@ failed. `partition-expander.mota` offers the same bridge as a first-stage LoRa update only if the installed old firmware already supports compatible mOTA and its -target ID matches this repeater. A stock image without mOTA must use its +target ID matches this exact hardware/role. A stock image without mOTA must use its existing Wi-Fi updater for stage one. Keep a compatible seeder available before installing the bridge; the bridge does not itself repeat normal traffic. """ @@ -511,7 +535,32 @@ authorize the final app. After verified completion, write `full-application.bin` to the expanded OTA slots with the active slot last. Never write `target-partitions.bin` directly or erase NVS to force success. """ - if spec["lora_bridge"]: + if self_hosted_lora: + lora_instructions = f"""## LoRa route (self-hosted, configuration-preserving) + +The installed old firmware must support compatible mOTA and the exact target +ID in `manifest.json`. `lora-bridge.mota` and `partition-expander.mota` contain +the same role-pinned Partition Expander. It works from either old OTA slot; +it does not reboot into an old receiver with reset settings. + +1. Serve `lora-bridge.mota` from a compatible seeder. On the old node run + `ota ls`, `ota pull flash`, then `ota install` after verification. +2. Keep power stable. The bridge stages and verifies the private key and + supported saved configuration, expands the table, then restores and verifies + them before enabling its LoRa receiver on the saved primary radio profile. +3. Serve this package's `full-application.mota` on that saved profile. The + expander automatically fetches only this exact successor target and installs + it after complete verification. Wi-Fi recovery remains available. + +The final image has {full_mota_blocks} blocks; the seeder needs at least +{full_mota_blocks * 4} bytes of proof scratch. Confirm that a capable seeder +lists it before stage one. The old mOTA receiver's target ID must already +match this observer role; do not use a repeater-only or room-only identity. +This path has not been physically qualified on a Waveshare board. An A/B, +configuration-preservation and interruption test is still required before +remote deployment. The partition-table erase/write is not power-loss atomic. +""" + elif spec["lora_bridge"]: slot_instructions = ( """On this **4 MiB** layout, the bridge must be installed in old slot B while a verified old LoRa receiver remains in old slot A. If the old node is @@ -586,6 +635,23 @@ Some older room-server images also have no LoRa mOTA receiver. Do not upload the Full image through LoRa before the new layout is active. {full_mota_note} `target-partitions.bin` is for verification only, not browser upload. """ + source_note = ( + f"\nThis observer-only recipe expands the reviewed `{spec['source_partition_table']}` " + "layout to `variants/dual_ota_full_4MB.csv`: two 0x1F0000-byte app " + "slots and only 64 KiB SPIFFS. Ordinary Waveshare roles keep their " + "existing table. Oversized or unsupported saved configuration must be " + "rejected before destructive migration.\n" + if spec.get("source_partition_table") else "" + ) + restore_note = ( + "The self-hosted bridge restores and verifies the private identity and\n" + "supported configuration in expanded SPIFFS before enabling LoRa. It\n" + "does not hand off to a legacy receiver." + if self_hosted_lora else + "On 8/16 MiB LoRa and Wi-Fi routes the bridge restores and verifies the\n" + "identity in expanded SPIFFS; on the older 4 MiB LoRa handoff route the\n" + "new Full firmware does so at first boot." + ) return f"""# {board} in-place partition migration Source: `{source}`; firmware version: `{version}`. This package is only for the @@ -598,12 +664,11 @@ preserves only the private identity. The LoRa route, where present, requires the old image's exact target ID. The current partition table must have stable NVS at 0x9000, OTA metadata at 0xE000, two OTA apps, and SPIFFS with `/identity/_main.id`. +{source_note} The bridge checks the live layout and refuses unsupported boards. It stages -the 96-byte public/private identity in unchanged NVS, preserves a bootable old -receiver, and changes the partition table. On 8/16 MiB LoRa and Wi-Fi routes -the bridge restores and verifies the identity in expanded SPIFFS; on the -4 MiB LoRa route the new Full firmware does so at first boot. No full-chip +the 96-byte public/private identity in unchanged NVS, preserves a bootable +recovery image, and changes the partition table. {restore_note} No full-chip erase is used. The Wi-Fi/LoRa routes need no USB connection; the separate USB route below is managed by an identity-verified host flasher. The table sector and inactive app sectors are erased as part of migration. @@ -656,6 +721,9 @@ def package_board(name: str, spec: dict, build_dir: Path, output_dir: Path, or entries.get("app0") != (0x10000, spec["slot_bytes"]) or entries.get("app1") != (spec["slot1_address"], spec["slot_bytes"])): raise ValueError(f"{name}: built partition table differs from the migration plan") + if spec.get("spiffs_bytes") is not None and entries.get("spiffs") != ( + spec["spiffs_address"], spec["spiffs_bytes"]): + raise ValueError(f"{name}: built SPIFFS differs from the reviewed migration plan") _label, slot_bytes = app_partition_size(table) full_image = full_path.read_bytes() full_ident = parse_endf_ident(full_image) @@ -738,7 +806,9 @@ def package_board(name: str, spec: dict, build_dir: Path, output_dir: Path, "application_usb_cdc": False, "completion": "verified-target-table-nvs-restoration-otadata", }, - "lora_bridge_mode": ("slot-b-only-full-identity-recovery" + "lora_bridge_mode": ("self-hosted-either-slot-verified-configuration" + if spec.get("expander_bridge") == spec.get("lora_bridge") + else "slot-b-only-full-identity-recovery" if spec["flash_bytes"] == 4 * 1024 * 1024 else "either-slot-preserve-receiver"), "partition_expander": spec.get("expander_bridge"), diff --git a/src/helpers/ESP32PartitionMigrationPolicy.h b/src/helpers/ESP32PartitionMigrationPolicy.h index 33edaf577..c4b2fecf5 100644 --- a/src/helpers/ESP32PartitionMigrationPolicy.h +++ b/src/helpers/ESP32PartitionMigrationPolicy.h @@ -71,6 +71,17 @@ constexpr PartitionGeometry kExpanded4MBLayout = { 0x3F0000, 0x10000, }; +// Arduino-ESP32 min_spiffs.csv, used by the Waveshare ESP32-S3 LR1121 +// observer builds. The final 64 KiB is a separate coredump partition, NOT +// part of this 128 KiB filesystem. +constexpr PartitionGeometry kMinSpiffs4MBLayout = { + 0x9000, 0x5000, + 0xE000, 0x2000, + 0x10000, 0x1E0000, + 0x1F0000, 0x1E0000, + 0x3D0000, 0x20000, +}; + constexpr bool sameGeometry(const PartitionGeometry& left, const PartitionGeometry& right) { return left.nvs_address == right.nvs_address @@ -93,6 +104,49 @@ constexpr bool isExpandedLayout(const PartitionGeometry& geometry) { return sameGeometry(geometry, kExpandedLayout); } +// New strict 4 MiB packages accept only reviewed maps, not merely partitions +// that happen to be inside flash. The target is included for the no-op path. +constexpr bool isReviewed4MBSource(uint32_t flash_bytes, + const PartitionGeometry& geometry) { + return flash_bytes == kSmallRequiredFlashBytes + && (sameGeometry(geometry, kLegacyLayout) + || sameGeometry(geometry, kMinSpiffs4MBLayout) + || sameGeometry(geometry, kExpanded4MBLayout)); +} + +// Conservative admission budget for a freshly formatted 64 KiB SPIFFS. Charge +// only 192 payload bytes per 256-byte page, two metadata pages per file, four +// whole blocks for lookup/GC/headroom, and one extra largest-file allocation +// for the successor's atomic preference rewrites. This is a refusal guard, +// not an assertion of the physical filesystem's exact free-space accounting. +struct PreservationBudget { + static constexpr uint32_t kMaximumFileBytes = 16384; + static constexpr uint32_t kMaximumFiles = 33; // identity plus 32 config files + uint32_t filesystem_bytes; + uint32_t files = 0; + uint64_t file_bytes = 0; + uint64_t largest_file_bytes = 0; + + explicit PreservationBudget(uint32_t bytes) : filesystem_bytes(bytes) {} + + uint64_t requiredBytes() const { + return 4U * kSectorBytes + file_bytes + largest_file_bytes; + } + + bool addFile(uint32_t bytes) { + if (bytes == 0 || bytes > kMaximumFileBytes || files >= kMaximumFiles) + return false; + const uint64_t cost = ((static_cast(bytes) + 191U) / 192U + 2U) * 256U; + const uint64_t largest = cost > largest_file_bytes ? cost : largest_file_bytes; + if (4U * kSectorBytes + file_bytes + cost + largest > filesystem_bytes) + return false; + ++files; + file_bytes += cost; + largest_file_bytes = largest; + return true; + } +}; + constexpr bool canMigrate(uint32_t flash_bytes, const PartitionGeometry& geometry) { return (flash_bytes == kSmallRequiredFlashBytes diff --git a/test/test_esp32_partition_migration_preservation.py b/test/test_esp32_partition_migration_preservation.py new file mode 100644 index 000000000..0b632e6f9 --- /dev/null +++ b/test/test_esp32_partition_migration_preservation.py @@ -0,0 +1,205 @@ +"""Execute strict 4 MiB source/retention guards without hardware or PlatformIO.""" +from pathlib import Path +import subprocess +import tempfile +import unittest + +from test_replay_reset_integration import extract_braced + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "examples/esp32_partition_migrator/main.cpp" + +HARNESS = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace migration = mesh::esp32_partition_migration; +constexpr size_t kIdentityFileBytes=96; +char status_text[192]={}; +struct Entry {std::string path;bool dir;size_t bytes;std::vector children;}; +struct File { + Entry* item=nullptr;size_t index=0; + explicit operator bool() const{return item!=nullptr;} + bool isDirectory()const{return item&&item->dir;} + const char* path()const{return item?item->path.c_str():nullptr;} + size_t size()const{return item?item->bytes:0;} + File openNextFile(); + void close(){item=nullptr;} +}; +struct Filesystem { + std::map entries; + bool mounted=true;unsigned begins=0,ends=0; + bool begin(bool format){assert(!format);++begins;return mounted;} + void end(){++ends;} + bool exists(const char* path){return entries.count(path)!=0;} + File open(const char* path,const char* mode){ + assert(!strcmp(mode,"r")); + auto it=entries.find(path);return {it==entries.end()?nullptr:&it->second,0}; + } + void add(const char* path,size_t bytes,const char* parent="/") { + entries[path]={path,false,bytes,{}};entries[parent].children.push_back(path); + } + void base() { + *this=Filesystem();entries["/"]={"/",true,0,{}}; + add("/identity/_main.id",96); + } +} SPIFFS; +File File::openNextFile(){ + if(!item||index>=item->children.size())return {}; + return SPIFFS.open(item->children[index++].c_str(),"r"); +} +@CONFIG@ +@METHODS@ +int main(int argc,char**argv) { + assert(argc==2);int scenario=atoi(argv[1]);SPIFFS.base(); + if(scenario==0) { + SPIFFS.add("/com_prefs",512);SPIFFS.add("/s_contacts",2048); + SPIFFS.add("/s_login_replay",256);SPIFFS.add("/radio_profiles",1024); + SPIFFS.add("/mqtt_prefs",2888);SPIFFS.add("/management",512); + assert(preflightStrictPreservation(migration::kTarget4MB)); + assert(SPIFFS.begins==1&&SPIFFS.ends==1); + } else if(scenario==1) { + for(const char* path:{"/custom.ini","/mqtt_prefs.tmp","/identity/other.id","/logs.txt"}) { + SPIFFS.base();SPIFFS.add(path,10); + assert(!preflightStrictPreservation(migration::kTarget4MB)); + assert(strstr(status_text,"unpreserved"));assert(SPIFFS.ends==1); + } + } else if(scenario==2) { + for(size_t bytes:{size_t(0),size_t(16385),size_t(UINT32_MAX)+1}) { + SPIFFS.base();SPIFFS.add("/radio_profiles",bytes); + assert(!preflightStrictPreservation(migration::kTarget4MB)); + } + SPIFFS.base();SPIFFS.add("/s_contacts",12000); + SPIFFS.add("/radio_profiles",12000);SPIFFS.add("/mqtt_prefs",12000); + assert(!preflightStrictPreservation(migration::kTarget4MB)); + assert(strstr(status_text,"budget")); + } else if(scenario==3) { + for(const auto& layout:{migration::kLegacyLayout,migration::kMinSpiffs4MBLayout, + migration::kExpanded4MBLayout}) { + assert(migration::isReviewed4MBSource(4U*1024U*1024U,layout)); + assert(!migration::isReviewed4MBSource(8U*1024U*1024U,layout)); + auto changed=layout;changed.spiffs_address+=4096; + assert(!migration::isReviewed4MBSource(4U*1024U*1024U,changed)); + changed=layout;changed.nvs_address+=4096; + assert(!migration::isReviewed4MBSource(4U*1024U*1024U,changed)); + changed=layout;changed.app1_address=changed.app0_address; + assert(!migration::isReviewed4MBSource(4U*1024U*1024U,changed)); + } + auto target=migration::kTarget4MB;target.layout.spiffs_size=0x30000; + assert(!preflightStrictPreservation(target));assert(SPIFFS.begins==0); + assert(migration::kMinSpiffs4MBLayout.spiffs_address==0x3D0000); + assert(migration::kMinSpiffs4MBLayout.spiffs_size==0x20000); + // min_spiffs has a separate 64 KiB coredump after its 128 KiB filesystem; + // silently treating the coredump as filesystem capacity is not reviewed. + auto includes_coredump=migration::kMinSpiffs4MBLayout; + includes_coredump.spiffs_size=0x30000; + assert(!migration::isReviewed4MBSource(4U*1024U*1024U,includes_coredump)); + assert(migration::kMinSpiffs4MBLayout.app0_address+migration::kMinSpiffs4MBLayout.app0_size + ==migration::kMinSpiffs4MBLayout.app1_address); + // An old-B bridge can copy to A without erasing its currently running bytes. + assert(migration::kExpanded4MBLayout.app0_address+migration::kMinSpiffs4MBLayout.app1_size + <=migration::kMinSpiffs4MBLayout.app1_address); + } else if(scenario==4) { + SPIFFS.entries["/"].children.clear(); + SPIFFS.entries["/"].children.push_back("/identity"); + SPIFFS.entries["/identity"]={"/identity",true,0,{"/identity/_main.id"}}; + assert(preflightStrictPreservation(migration::kTarget4MB)); + SPIFFS.base();SPIFFS.entries["/"].children.push_back("/unknown"); + SPIFFS.entries["/unknown"]={"/unknown",true,0,{}}; + assert(!preflightStrictPreservation(migration::kTarget4MB)); + SPIFFS.base();SPIFFS.entries["/"].children.clear(); + assert(!preflightStrictPreservation(migration::kTarget4MB)); + SPIFFS.base();SPIFFS.entries["/radio_profiles"]={"/radio_profiles",false,10,{}}; + assert(!preflightStrictPreservation(migration::kTarget4MB)); + assert(strstr(status_text,"incomplete")); + SPIFFS.base();SPIFFS.mounted=false; + assert(!preflightStrictPreservation(migration::kTarget4MB)); + } else if(scenario==5) { + SPIFFS.entries["/identity/_main.id"].bytes=128; + assert(!preflightStrictPreservation(migration::kTarget4MB)); + assert(strstr(status_text,"identity")); + SPIFFS.base();SPIFFS.entries["/"].children.push_back("/identity/_main.id"); + assert(!preflightStrictPreservation(migration::kTarget4MB)); + } else if(scenario==6) { + migration::PreservationBudget budget(65536); + for(unsigned i=0;i<33;++i)assert(budget.addFile(96)); + assert(!budget.addFile(96)); + migration::PreservationBudget large(65536); + assert(large.addFile(96)&&large.addFile(16384)); + assert(large.requiredBytes()<=65536); + assert(!large.addFile(16384)); + migration::PreservationBudget invalid(65536); + assert(!invalid.addFile(0)&&!invalid.addFile(UINT32_MAX)); + } else return 2; +} +''' + + +class StrictPartitionPreservationTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.directory = tempfile.TemporaryDirectory(prefix="strict-partition-preservation-") + source = SOURCE.read_text() + start = source.index("struct ConfigFile {") + end = source.index(";", source.index("constexpr size_t kConfigFileCount", start)) + 1 + config = source[start:end] + methods = "\n".join(extract_braced(source, name) for name in ( + "bool inventoryStrictDirectory(", "bool preflightStrictPreservation(")) + path = Path(cls.directory.name) / "test.cpp" + cls.binary = Path(cls.directory.name) / "test" + path.write_text(HARNESS.replace("@CONFIG@", config).replace("@METHODS@", methods)) + result = subprocess.run([ + "g++", "-std=c++17", "-Wall", "-Wextra", "-Werror", + "-fsanitize=address,undefined", "-fno-sanitize-recover=all", "-fno-pie", "-no-pie", + "-I", str(ROOT / "src"), str(path), "-o", str(cls.binary), + ], text=True, capture_output=True) + if result.returncode: + cls.directory.cleanup() + raise AssertionError(result.stdout + result.stderr) + + @classmethod + def tearDownClass(cls): + cls.directory.cleanup() + + def test_saved_node_acl_radios_mqtt_fit(self): + subprocess.run([str(self.binary), "0"], check=True) + + def test_unknown_files_are_not_silently_discarded(self): + subprocess.run([str(self.binary), "1"], check=True) + + def test_empty_oversized_and_over_capacity_config_is_refused(self): + subprocess.run([str(self.binary), "2"], check=True) + + def test_only_reviewed_source_maps_and_target_capacity_are_accepted(self): + subprocess.run([str(self.binary), "3"], check=True) + + def test_nested_flat_missing_and_incomplete_inventory(self): + subprocess.run([str(self.binary), "4"], check=True) + + def test_unpreserved_identity_metadata_and_duplicate_paths_are_refused(self): + subprocess.run([str(self.binary), "5"], check=True) + + def test_budget_reserves_gc_and_an_atomic_largest_file_rewrite(self): + subprocess.run([str(self.binary), "6"], check=True) + + def test_preflight_precedes_every_destructive_migration_step(self): + source = SOURCE.read_text() + body = extract_braced(source, "void runMigration()") + guard = body.index("preflightStrictPreservation(*plan)") + for operation in ("stageLegacyIdentity()", "stageLegacyConfig()", "copyAndVerify(*running", + "esp_ota_set_boot_partition(", "publishExpandedPartitionTable(*plan)"): + self.assertLess(guard, body.index(operation)) + self.assertIn("MESHCORE_MIGRATION_STRICT_4MB", source) + # Existing boards keep their existing migration behavior unless a new, + # reviewed package explicitly opts into this narrower contract. + self.assertIn("#if defined(MESHCORE_MIGRATION_STRICT_4MB) && MESHCORE_MIGRATION_STRICT_4MB", body) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_upstream_observer_recipes.py b/test/test_upstream_observer_recipes.py index 259a7ce5b..d20e1cfc8 100644 --- a/test/test_upstream_observer_recipes.py +++ b/test/test_upstream_observer_recipes.py @@ -5,6 +5,7 @@ Resolve inheritance and interpolation without starting PlatformIO or a build. The existing board recipes remain authoritative for hardware and OTA policy. """ import configparser +import csv import fnmatch from pathlib import Path import re @@ -22,6 +23,7 @@ BOARDS = { "lilygo_teth_elite": ("LilyGo_TETH_Elite_sx1262",), "meshnology_w12": ("meshnology_w12",), "xiao_s3": ("Xiao_S3",), + "waveshare_esp32s3_lr1121": ("Waveshare_S3_LR1121",), } NAMES = tuple( f"{board}_{role}_observer_mqtt" @@ -72,8 +74,8 @@ class UpstreamObserverRecipeTests(unittest.TestCase): def setUpClass(cls): cls.recipes = Recipes() - def test_all_22_named_additions_exist_once(self): - self.assertEqual(len(NAMES), 22) + def test_all_24_named_additions_exist_once(self): + self.assertEqual(len(NAMES), 24) for family, boards in BOARDS.items(): text = (ROOT / "variants" / family / "platformio.ini").read_text() for board in boards: @@ -137,6 +139,26 @@ class UpstreamObserverRecipeTests(unittest.TestCase): section = f"env:{board}_{role}_observer_mqtt" self.assertIn("MQTT_NEIGHBORS_WITHOUT_PSRAM=1", self.recipes.value(section, "build_flags")) + def test_waveshare_observers_have_two_larger_slots_and_64k_filesystem(self): + table = "variants/dual_ota_full_4MB.csv" + for role in ("repeater", "room_server"): + self.assertEqual(self.recipes.value( + f"env:Waveshare_S3_LR1121_{role}_observer_mqtt", "board_build.partitions"), table) + for role in ("repeater", "room_server", "companion_radio_usb", "companion_radio_ble", "kiss_modem"): + self.assertEqual(self.recipes.value( + f"env:Waveshare_S3_LR1121_{role}", "board_build.partitions"), "min_spiffs.csv") + rows = list(csv.reader(line for line in (ROOT / table).read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#"))) + parts = {row[0].strip(): (int(row[3], 0), int(row[4], 0)) for row in rows} + self.assertEqual(parts["app0"], (0x10000, 0x1F0000)) + self.assertEqual(parts["app1"], (0x200000, 0x1F0000)) + self.assertEqual(parts["spiffs"], (0x3F0000, 0x10000)) + end = 0 + for offset, size in sorted(parts.values()): + self.assertGreaterEqual(offset, end) + end = offset + size + self.assertEqual(end, 4 * 1024 * 1024) + def test_rc32_local_startup_platform_and_safety_remain_inherited(self): for board in ("heltec_rc32", "heltec_rc32_without_display"): for role in ("repeater", "room_server"): diff --git a/test/test_waveshare_partition_migration.py b/test/test_waveshare_partition_migration.py new file mode 100644 index 000000000..a4c61fd19 --- /dev/null +++ b/test/test_waveshare_partition_migration.py @@ -0,0 +1,132 @@ +"""No-hardware qualification of exact Waveshare observer migration recipes.""" +import configparser +import json +from pathlib import Path +import re +import struct +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from build_esp32_partition_migration import build_steps +from package_esp32_partition_migration import BOARDS, package_board, readme +from motalib import hardware_id_for_env, target_id_for_env + +KEYS = ("waveshare-s3-lr1121-observer-repeater", + "waveshare-s3-lr1121-observer-room-server") + + +class WaveshareMigrationRecipeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.config = configparser.ConfigParser(interpolation=None, + inline_comment_prefixes=(";",)) + cls.config.read([ROOT / "platformio.ini", *sorted((ROOT / "variants").glob("*/platformio.ini"))], + encoding="utf-8") + + def expanded(self, section, field): + config = self.config[section] + if field not in config and "extends" in config: + return self.expanded(config["extends"], field) + value = config.get(field, "") + return re.sub(r"\$\{([^}]+)\}", + lambda m: self.expanded(*m[1].rsplit(".", 1)), value) + + def test_only_observer_roles_get_exact_self_hosted_routes(self): + waveshare = {key: spec for key, spec in BOARDS.items() + if spec["target"].startswith("Waveshare_")} + self.assertEqual(set(KEYS), set(waveshare)) + for key, role in zip(KEYS, ("repeater", "room-server")): + with self.subTest(key=key): + spec = BOARDS[key] + self.assertEqual(role, spec["role"]) + self.assertEqual(4 * 1024 * 1024, spec["flash_bytes"]) + self.assertEqual(0x1F0000, spec["slot_bytes"]) + self.assertEqual(0x200000, spec["slot1_address"]) + self.assertEqual((0x3F0000, 0x10000), + (spec["spiffs_address"], spec["spiffs_bytes"])) + self.assertEqual("min_spiffs.csv", spec["source_partition_table"]) + self.assertEqual(spec["wifi_bridge"], spec["expander_bridge"]) + self.assertEqual(spec["lora_bridge"], spec["expander_bridge"]) + self.assertEqual("esp32_s3_4mb_partition_migrator_usb", spec["usb_bridge"]) + self.assertEqual(9, spec["usb_chip"]) + self.assertEqual("Waveshare_S3_LR1121", hardware_id_for_env(spec["target"])) + self.assertEqual("variants/dual_ota_full_4MB.csv", + self.expanded("env:" + spec["target"], "board_build.partitions")) + for role in ("repeater", "room_server", "companion_radio_usb", "companion_radio_ble"): + self.assertEqual("min_spiffs.csv", self.expanded( + "env:Waveshare_S3_LR1121_" + role, "board_build.partitions")) + + def test_expanders_reuse_real_lr1121_wiring_without_native_usb_or_legacy_handoff(self): + board = json.loads((ROOT / "boards/waveshare_s3_lr1121_migrator.json").read_text()) + self.assertIn("-D ARDUINO_USB_CDC_ON_BOOT=0", board["build"]["extra_flags"]) + self.assertEqual("dio", board["build"]["flash_mode"]) + self.assertEqual("4MB", board["upload"]["flash_size"]) + for key in KEYS: + spec = BOARDS[key] + section = "env:" + spec["expander_bridge"] + flags = self.expanded(section, "build_flags") + for name, value in (("P_LORA_NSS", 42), ("P_LORA_DIO_1", 38), + ("P_LORA_RESET", 39), ("P_LORA_BUSY", 41), + ("P_LORA_SCLK", 40), ("P_LORA_MISO", 46), ("P_LORA_MOSI", 45)): + self.assertIn(f"-D {name}={value}", flags) + for flag in ("RADIO_CLASS=CustomLR1121", "WRAPPER_CLASS=CustomLR1121Wrapper", + "LR11X0_DIO3_TCXO_VOLTAGE=1.8", "LR1121_FORCE_HP_PA", + "MESHCORE_MIGRATION_STRICT_4MB=1", "OTA_FETCH_PIPELINE=4"): + self.assertIn("-D " + flag, flags) + self.assertIn(f"-D MOTA_MIGRATION_TARGET_ID=0x{target_id_for_env(spec['target']):08x}", flags) + self.assertNotIn("MESHCORE_MIGRATION_RESUME_OTA", flags) + self.assertEqual("waveshare_s3_lr1121_migrator", self.expanded(section, "board")) + self.assertEqual("default.csv", self.expanded(section, "board_build.partitions")) + self.assertEqual("1310720", self.expanded(section, "board_upload.maximum_size")) + self.assertIn("../examples/partition_expander/*.cpp", + self.expanded(section, "build_src_filter")) + + def test_builds_both_exact_full_apps_then_each_wireless_bridge_only_once(self): + steps = build_steps(list(KEYS), "v1.18.0-test", "usa-cascadia", "cascade", 2) + self.assertEqual([True, True, False, False, False], [full for _, full in steps]) + self.assertEqual([BOARDS[key]["target"] for key in KEYS], + [command[3] for command, full in steps if full]) + self.assertEqual({BOARDS[key]["expander_bridge"] for key in KEYS} + | {"esp32_s3_4mb_partition_migrator_usb"}, + {command[3] for command, full in steps if not full}) + for key in KEYS: + guide = readme(key, BOARDS[key], "v1.18.0-test", "01234567", full_mota_blocks=960) + self.assertIn("self-hosted, configuration-preserving", guide) + self.assertIn("either old OTA slot", guide) + self.assertIn("saved primary radio profile", guide) + self.assertIn("has not been physically qualified", guide) + self.assertIn("not power-loss atomic", guide) + self.assertNotIn("bridge must be installed in old slot B", guide) + + def test_packager_rejects_a_wrong_final_filesystem_before_reading_bridges(self): + spec = BOARDS[KEYS[0]] + with tempfile.TemporaryDirectory(prefix="waveshare-migration-plan-") as temporary: + directory = Path(temporary) + stem = spec["target"] + "-full-usa-v1.18.0-test-01234567" + app = b"\xe9test-app" + (directory / (stem + ".bin")).write_bytes(app) + (directory / (stem + ".capabilities.json")).write_text(json.dumps({ + "target": spec["target"], "build_profile": "full", + "platform": "ESP32_PLATFORM", "ota_update_verified": True, + })) + merged = bytearray(b"\xff" * 0x10000 + app) + for index, (name, ptype, subtype, address, size) in enumerate(( + ("nvs", 1, 2, 0x9000, 0x5000), ("otadata", 1, 0, 0xE000, 0x2000), + ("app0", 0, 16, 0x10000, 0x1F0000), ("app1", 0, 17, 0x200000, 0x1F0000), + ("spiffs", 1, 130, 0x3E0000, 0x20000), + )): + struct.pack_into(" +build_flags = ${Waveshare_S3_LR1121.build_flags} + -D MESH_MIN_RUNTIME_HEAP=100352 + -D MESHCORE_MIGRATION_STRICT_4MB=1 + -D OTA_FETCH_PIPELINE=4 +lib_deps = ${Waveshare_S3_LR1121.lib_deps} + ${esp32_ota.lib_deps} + +[env:Waveshare_S3_LR1121_partition_expander_repeater_observer_mqtt] +extends = Waveshare_S3_LR1121_partition_expander +build_flags = ${Waveshare_S3_LR1121_partition_expander.build_flags} + -D MOTA_MIGRATION_TARGET_ID=0xcbe54778 + +[env:Waveshare_S3_LR1121_partition_expander_room_server_observer_mqtt] +extends = Waveshare_S3_LR1121_partition_expander +build_flags = ${Waveshare_S3_LR1121_partition_expander.build_flags} + -D MOTA_MIGRATION_TARGET_ID=0x3de413b1