From e7f6d529a07576d4a105b415cc72c3f249b27e3f Mon Sep 17 00:00:00 2001 From: mikecarper Date: Thu, 24 Sep 2026 10:01:14 -0700 Subject: [PATCH] Expand ESP32 OTA partition migration across infrastructure roles --- build.sh | 24 +- docs/esp32_wifi_partition_migration.md | 72 ++-- examples/esp32_partition_migrator/main.cpp | 4 +- scripts/build_esp32_partition_migration.py | 27 +- scripts/build_esp32_partition_migration.sh | 46 ++- scripts/package_esp32_partition_migration.py | 336 ++++++++++++++++-- src/helpers/ESP32PartitionMigrationPolicy.h | 61 +++- .../test_esp32_partition_migration_policy.cpp | 25 +- test/test_esp32_partition_migration_recipe.py | 93 ++++- test/test_heltec_v4_partition_migrator.py | 10 + .../esp32_partition_migrator/platformio.ini | 54 +++ 11 files changed, 651 insertions(+), 101 deletions(-) create mode 100644 variants/esp32_partition_migrator/platformio.ini diff --git a/build.sh b/build.sh index 0bde3064d..f034ef323 100755 --- a/build.sh +++ b/build.sh @@ -3109,22 +3109,30 @@ is_esp32_full_only_bulk_target() { } # These normal role identities still ship a portable legacy image for installed -# nodes, but also need a canonical FULL artifact for the one-time wired layout +# nodes, but also need a canonical FULL artifact for the one-time layout # migration. They are deliberately separate from the FULL-only list above: -# their deployed 0x140000 slots differ from the expanded table, so LoRa mOTA -# must not try to cross this boundary. The matching merged image is the -# migration package; after it is flashed, later normal-target FULL packages -# use the same logical mOTA identity. +# some deployed 0x140000 slots differ from the expanded table, so LoRa mOTA +# must not try to cross this boundary. An exact-target Wi-Fi bridge and Full +# application are packaged separately; after migration, later normal-target +# FULL packages use the same logical mOTA identity. is_esp32_partition_migration_full_target() { local env_name=${1,,} [ "${PIO_ENV_PLATFORM_BY_NAME[$1]:-}" = "ESP32_PLATFORM" ] || return 1 case "$env_name" in + ebyte_eora-s3_repeater|ebyte_eora-s3_room_server|\ + generic_e22_sx1262_repeater|generic_e22_sx1268_repeater|\ + heltec_ct62_repeater|heltec_ct62_sensor|\ lilygo_t3s3_sx1262_repeater|lilygo_t3s3_sx1262_room_server|\ lilygo_t3s3_sx1276_repeater|lilygo_t3s3_sx1276_room_server|\ - station_g2_repeater|station_g2_room_server|\ - thinknode_m2_repeater|thinknode_m2_room_server) + station_g2_repeater|station_g2_logging_repeater|station_g2_room_server|\ + thinknode_m2_repeater|thinknode_m2_room_server|\ + thinknode_m5_repeater|thinknode_m5_room_server|\ + xiao_c3_repeater|xiao_c3_room_server|\ + tenstar_c3_sx1262_repeater|tenstar_c3_sx1268_repeater|\ + nibble_zero_connect_repeater_|nibble_zero_connect_room_server_|\ + nibble_screen_connect_repeater_|nibble_screen_connect_room_server_) return 0 ;; esac @@ -3202,7 +3210,7 @@ declare_build_capability_contract() { if is_esp32_full_only_bulk_target "$env_name"; then record_build_capability "release.full.same_partition_successor" elif is_esp32_partition_migration_full_target "$env_name"; then - record_build_capability "release.full.partition_migration_usb" + record_build_capability "release.full.partition_migration_ota" fi fi diff --git a/docs/esp32_wifi_partition_migration.md b/docs/esp32_wifi_partition_migration.md index 1b1c821de..b59b5283f 100644 --- a/docs/esp32_wifi_partition_migration.md +++ b/docs/esp32_wifi_partition_migration.md @@ -7,15 +7,17 @@ updater for the full image after migration. The LoRa version retains the old OTA-capable repeater in the other expanded slot and returns to it after restoring identity, so it can fetch the full image over LoRa. -It currently has explicit, byte-for-byte generated target plans for Arduino -ESP32 `default_8MB.csv` and `default_16MB.csv`. It does not guess a table for -other flash sizes. Adding a board later means adding one reviewed target plan -and its generated table-prefix test; the bridge core is otherwise shared. +It has explicit, byte-for-byte generated target plans for the 4 MiB MeshCore +dual-OTA Full table and Arduino ESP32 `default_8MB.csv` and +`default_16MB.csv`. It does not guess a table for other flash sizes. Adding a +flash size later means adding one reviewed target plan and its generated +table-prefix test; the bridge core is otherwise shared. | Flash size | Target table | app0 / app1 size | Typical board | | --- | --- | --- | --- | -| 8 MiB | `default_8MB.csv` | 0x330000 / 0x330000 | Seeed XIAO ESP32-S3 | -| 16 MiB | `default_16MB.csv` | 0x640000 / 0x640000 | Heltec V4 / V4.3 OLED | +| 4 MiB | `variants/dual_ota_full_4MB.csv` | 0x1F0000 / 0x1F0000 | ThinkNode M2/M5, LilyGo T3S3, Nibble, Heltec CT62 | +| 8 MiB | `default_8MB.csv` | 0x330000 / 0x330000 | Seeed XIAO ESP32-S3, Heltec V3 | +| 16 MiB | `default_16MB.csv` | 0x640000 / 0x640000 | Heltec V4 / V4.3, Station G2 | ## Repeatable ESP32 build recipe @@ -25,9 +27,10 @@ From a clean commit, run the shell menu in WSL/Linux: sh scripts/build_esp32_partition_migration.sh ``` -Choose one board, or **all qualified boards**. The script asks for the -firmware version, radio preset, and profile, then builds the Full repeater -images first, both small bridges for each selected board, and verifies each +Choose one exact board/role, or **all listed board/role pairs**. The menu +also offers all repeaters, all room servers, or all sensors separately. It asks +for the firmware version, radio preset, and profile, then builds Full application +images first, the available bridge(s) for each selected board, and verifies each ZIP. Only one PlatformIO process runs at a time; `build.sh` can clean the shared `.pio/build` tree between Full targets. The same menu can be run non-interactively: @@ -39,12 +42,15 @@ sh scripts/build_esp32_partition_migration.sh --board heltec-v4 \ sh scripts/build_esp32_partition_migration.sh --all \ --version v1.17.1.7-halo-keymind-cascade-dev \ --radio-preset usa-cascadia --profile cascade +sh scripts/build_esp32_partition_migration.sh --role sensor \ + --version v1.17.1.7-halo-keymind-cascade-dev \ + --radio-preset usa-cascadia --profile cascade ``` -`--all` creates one verified ZIP per currently qualified board and a release +`--all` creates one verified ZIP per listed board/role pair and a release ZIP containing all of them, their hashes, and a manifest. This is a **local release bundle**, not an upload to GitHub Releases. It is not an archive of -old firmware binaries. `--list-boards` shows the presently qualified choices. +old firmware binaries. `--list-boards` shows the configured choices. The release fails instead of publishing a partial bundle if one board package is absent or fails verification. @@ -56,24 +62,40 @@ python3 -B scripts/build_esp32_partition_migration.py \ --radio-preset usa-cascadia --profile cascade ``` -The default builds all currently qualified boards. Add `--board heltec-v4` +The default builds all configured board/role recipes. Add `--board heltec-v4` or `--board xiao-s3-wio` to build one; repeat `--board` for a selected set. Use `--dry-run` to inspect the command order without building. The resulting ZIPs appear under `.releases/esp32-expanded-/packages/`. The recipe requires a clean checkout so each ZIP identifies the firmware commit it was built from. It creates files only; it does not flash a device or erase flash. -The historical 1.25 MiB release population is wider than these two boards. -In particular, the portable LilyGo T3S3 SX1262/SX1276, Station G2, and -ThinkNode M2 repeater/room-server targets have had 1.25 MiB slots. They are -**not** silently treated as supported by `--all`: the 4 MiB boards require a -different target table and a power-loss-safe LoRa receiver handoff, and every -additional role needs an audited exact target identity. A 4 MiB Full build -offers 1,984 KiB slots, not the 4 MiB-or-more slots possible on larger flash. +The catalog covers canonical ESP32 repeater, room-server, and sensor roles on +4, 8, and 16 MiB chip-family plans, including historical 1.25 MiB candidates +now built with expanded tables. This includes ESP32-S3, original ESP32, and +ESP32-C3 boards with an evidenced old default layout. `--list-boards` is the +authoritative list of exact targets; specialty ESP-NOW/observer aliases are +separate identities and are not packaged under their old IDs. With Wi-Fi OTA +they can migrate to the +same physical board's canonical role image; their feature settings may need +to be recreated. The LoRa route still requires an exact old target ID. The +catalog omits boards with no evidenced legacy default-layout build, such as +boards introduced with `min_spiffs.csv` or another non-1.25 MiB layout. Some +listed boards use larger slots today but had historical default-layout builds. +A listed package is usable only when the installed old image has a working +Wi-Fi updater and the bridge verifies its live source table. Menu inclusion +does **not** override either check. The only LoRa +bridge packages currently available are the exact-target Heltec V4 and XIAO +S3 WIO repeater packages. On 4 MiB flash the expanded app1 overlaps old app1, +so the currently safe LoRa receiver-copy method cannot be offered there. A +4 MiB Full build offers 1,984 KiB slots, not 4 MiB per slot. A Wi-Fi-only ZIP +can omit `full-application.mota` when its application exceeds the current +2 KiB x 1024-block LoRa serving limit; `full-application.bin` remains the +verified Wi-Fi update image. -Other ESP32 boards need a reviewed flash-size/partition plan, exact LoRa OTA -target ID, and board entry before they can be added to this recipe. Matching -flash capacity alone is not enough to claim a package is safe for a board. +Other ESP32 boards need a reviewed source updater, chip/flash-size bridge, +verified Full image, and board entry before they can be added. The LoRa route +also needs an exact old OTA target ID and a non-overlapping receiver handoff. +Matching flash capacity alone is not enough to claim a package is safe. ## What the bridge preserves @@ -100,7 +122,7 @@ The bridge only accepts a source table that has all of the following: - NVS at `0x9000`, size `0x5000`, and OTA metadata at `0xE000`, size `0x2000`. - Two distinct, non-empty OTA application slots and a non-empty SPIFFS partition, all inside the physical flash size. -- A known exact 8 MiB or 16 MiB physical flash capacity and a source table +- A known exact 4, 8, or 16 MiB physical flash capacity and a source table different from that capacity's target table. This means the source app-slot and SPIFFS sizes can vary; the bridge validates @@ -113,6 +135,10 @@ in legacy A or B. Keep power stable while the bridge is replacing the 4 KiB table sector. A loss of power before that point leaves the old table and source intact. After that small critical write, the selected A copy of the bridge is the recovery path. +The stock ESP32 bootloader reads one partition-table sector; rewriting that +sector is **not atomic**. A power loss during its erase/write window can leave +an unreadable table and require cable recovery. This recipe must not be called +power-loss-proof on a device whose USB/serial flash port is inaccessible. ## Migrate a Seeed XIAO ESP32-S3 diff --git a/examples/esp32_partition_migrator/main.cpp b/examples/esp32_partition_migrator/main.cpp index 182fb7b2c..5ed9c5ac5 100644 --- a/examples/esp32_partition_migrator/main.cpp +++ b/examples/esp32_partition_migrator/main.cpp @@ -455,11 +455,11 @@ bool publishExpandedPartitionTable(const migration::TargetPlan& plan) { if (result == ESP_OK) { result = esp_flash_write(chip, partition_table_bytes, migration::kPartitionTableAddress, - sizeof(partition_table_bytes)); + plan.partition_table_prefix_bytes); if (result == ESP_OK) { result = esp_flash_read(chip, partition_table_verified, migration::kPartitionTableAddress, - sizeof(partition_table_verified)); + plan.partition_table_prefix_bytes); ok = result == ESP_OK && memcmp(partition_table_verified, plan.partition_table_prefix, plan.partition_table_prefix_bytes) == 0; if (!ok) strcpy(status_text, "Partition table verification failed"); diff --git a/scripts/build_esp32_partition_migration.py b/scripts/build_esp32_partition_migration.py index 83710daa7..de99b8365 100644 --- a/scripts/build_esp32_partition_migration.py +++ b/scripts/build_esp32_partition_migration.py @@ -45,10 +45,14 @@ def build_steps(boards: list[str], version: str, radio_preset: str, "--require-ota", "--resume", ], True)) # Full builds can clear .pio/build; every bridge must be built afterwards. + built_bridges: set[str] = set() for name in boards: for key in ("wifi_bridge", "lora_bridge"): - steps.append((["pio", "run", "-e", BOARDS[name][key], - "-j", str(jobs)], False)) + bridge = BOARDS[name][key] + if bridge and bridge not in built_bridges: + steps.append((["pio", "run", "-e", bridge, + "-j", str(jobs)], False)) + built_bridges.add(bridge) return steps @@ -82,13 +86,15 @@ def bundle_release(output_root: Path, package_dir: Path, boards: list[str], packages.append((matches[0].name, data)) manifest = { "format": "meshcore-esp32-partition-migration-release-v1", - "scope": "currently-qualified-board-recipes-only", + "scope": "configured-legacy-layout-board-role-recipes", "historical_firmware_binaries_included": False, "source_commit": source, "firmware_version": version, "radio_preset": radio_preset, "profile": profile, "boards": boards, + "board_roles": {name: BOARDS[name]["role"] for name in boards}, + "roles": sorted({BOARDS[name]["role"] for name in boards}), "packages": {name: {"bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()} for name, data in packages}, @@ -125,7 +131,9 @@ def main() -> None: parser.add_argument("--profile", choices=("default", "cascade"), default="default", help="embedded runtime profile") parser.add_argument("--board", action="append", choices=tuple(BOARDS), - help="repeat to select boards (default: all supported boards)") + help="repeat to select board/role recipes (default: all configured recipes)") + parser.add_argument("--role", choices=("repeater", "room-server", "sensor"), + help="build every configured board for this role") parser.add_argument("--jobs", type=int, default=4, help="workers within each serial PlatformIO build") parser.add_argument("--output-root", type=Path, @@ -133,7 +141,7 @@ def main() -> None: parser.add_argument("--dry-run", action="store_true", help="show the build order without writing files") parser.add_argument("--list-boards", action="store_true", - help="print qualified board keys and exit") + help="print configured board/role keys and exit") args = parser.parse_args() if args.list_boards: for name in BOARDS: @@ -145,11 +153,16 @@ def main() -> None: parser.error("--version must be a filename-safe version token") if args.jobs < 1: parser.error("--jobs must be positive") + if args.board and args.role: + parser.error("choose --board or --role, not both") if os.name == "nt" and not args.dry_run: parser.error("run this recipe inside Linux/WSL, where bash and pio are available") source = git_output("rev-parse", "--short=8", "HEAD") - boards = list(dict.fromkeys(args.board or BOARDS)) + if args.role: + boards = [name for name, spec in BOARDS.items() if spec["role"] == args.role] + else: + boards = list(dict.fromkeys(args.board or BOARDS)) requested_root = args.output_root or Path(".releases") / f"esp32-expanded-{source}" output_root = (requested_root if requested_root.is_absolute() else ROOT / requested_root).resolve() @@ -161,7 +174,7 @@ def main() -> None: print(prefix + shlex.join(command), flush=True) print(f"Package {', '.join(boards)} into {package_dir}", flush=True) if set(boards) == set(BOARDS): - print(f"Bundle complete qualified set (not all historical ESP32 targets) in {output_root}", flush=True) + print(f"Bundle all configured board/role recipes (not historical binaries) in {output_root}", flush=True) if args.dry_run: return diff --git a/scripts/build_esp32_partition_migration.sh b/scripts/build_esp32_partition_migration.sh index bba584fa7..7205d842e 100755 --- a/scripts/build_esp32_partition_migration.sh +++ b/scripts/build_esp32_partition_migration.sh @@ -9,24 +9,25 @@ usage() { cat <<'EOF' Usage: scripts/build_esp32_partition_migration.sh [options] -With no arguments, choose one qualified ESP32 board or the complete qualified -set from a menu. The complete set creates one ZIP per board plus a release ZIP. +With no arguments, choose one ESP32 board/role or the complete configured +set from a menu. The complete set creates one ZIP per board/role plus a release ZIP. --board KEY Build one board (repeatable) - --all Build the complete qualified set + --role ROLE Build every repeater, room-server, or sensor board + --all Build all configured repeater, room-server and sensor roles --version VERSION Firmware version (required without the menu) --radio-preset NAME Radio preset (required without the menu) --profile NAME default or cascade (default: cascade) --jobs N PlatformIO workers within each serial build --output-root PATH Release output folder --dry-run Print commands without building - --list-boards Show board keys that have verified migration recipes + --list-boards Show configured board/role keys --help Show this help -Boards without an audited flash/partition/identity plan are not offered. A -name appearing in an old firmware release is not proof that OTA migration is -safe on that hardware. No release ZIP is published unless every selected -package passes the recipe's checks. +Boards with unrelated source layout plans are not offered. A name appearing +in an old firmware release is not proof that OTA migration is safe on that +hardware. The bridge checks the live layout, and no release ZIP is published +unless every selected package passes the recipe's checks. EOF } @@ -41,6 +42,7 @@ profile=cascade jobs= output_root= boards= +role= all=0 dry_run=0 interactive=1 @@ -48,13 +50,14 @@ interactive=1 while [ "$#" -gt 0 ]; do interactive=0 case "$1" in - --board|--version|--radio-preset|--profile|--jobs|--output-root) + --board|--role|--version|--radio-preset|--profile|--jobs|--output-root) if [ "$#" -lt 2 ]; then echo "Missing value for $1" >&2 exit 2 fi case "$1" in --board) boards="${boards}${boards:+ }$2" ;; + --role) role=$2 ;; --version) version=$2 ;; --radio-preset) radio_preset=$2 ;; --profile) profile=$2 ;; @@ -72,24 +75,30 @@ done board_keys=$(python3 -B "$recipe" --list-boards) if [ -z "$board_keys" ]; then - echo "No qualified boards are configured" >&2 + echo "No boards are configured" >&2 exit 1 fi if [ "$interactive" -eq 1 ]; then echo "ESP32 in-place partition migration" - echo "Only audited board/layout recipes are listed." + echo "Repeater, room-server and sensor recipes are listed." index=1 for board in $board_keys; do echo " $index) $board" index=$((index + 1)) done - echo " a) Build all qualified boards and create a release ZIP" + echo " a) Build all configured board/role recipes and create a release ZIP" + echo " r) Build all repeaters" + echo " m) Build all room servers" + echo " s) Build all sensors" echo " q) Quit" printf 'Choose: ' IFS= read -r choice case "$choice" in a|A|all) all=1 ;; + r|R) role=repeater ;; + m|M) role=room-server ;; + s|S) role=sensor ;; q|Q|quit) exit 0 ;; *) index=1 @@ -115,8 +124,9 @@ if [ "$interactive" -eq 1 ]; then profile=${selected_profile:-cascade} fi -if [ "$all" -eq 1 ] && [ -n "$boards" ]; then - echo "Choose either --all or --board, not both" >&2 +if { [ "$all" -eq 1 ] && { [ -n "$boards" ] || [ -n "$role" ]; }; } \ + || { [ -n "$boards" ] && [ -n "$role" ]; }; then + echo "Choose only one of --all, --board, or --role" >&2 exit 2 fi if [ -z "$version" ] || [ -z "$radio_preset" ]; then @@ -127,6 +137,11 @@ if [ "$profile" != default ] && [ "$profile" != cascade ]; then echo "Profile must be default or cascade" >&2 exit 2 fi +if [ -n "$role" ] && [ "$role" != repeater ] \ + && [ "$role" != room-server ] && [ "$role" != sensor ]; then + echo "Role must be repeater, room-server, or sensor" >&2 + exit 2 +fi set -- "$recipe" --version "$version" --radio-preset "$radio_preset" --profile "$profile" if [ -n "$boards" ]; then @@ -136,12 +151,13 @@ if [ -n "$boards" ]; then if [ "$board" = "$known" ]; then found=1; break; fi done if [ "$found" -ne 1 ]; then - echo "Board is not qualified for this recipe: $board" >&2 + echo "Board/role is not configured for this recipe: $board" >&2 exit 2 fi set -- "$@" --board "$board" done fi +if [ -n "$role" ]; then set -- "$@" --role "$role"; fi if [ -n "$jobs" ]; then set -- "$@" --jobs "$jobs"; fi if [ -n "$output_root" ]; then set -- "$@" --output-root "$output_root"; fi if [ "$dry_run" -eq 1 ]; then set -- "$@" --dry-run; fi diff --git a/scripts/package_esp32_partition_migration.py b/scripts/package_esp32_partition_migration.py index b6d570814..9b90a168e 100644 --- a/scripts/package_esp32_partition_migration.py +++ b/scripts/package_esp32_partition_migration.py @@ -39,7 +39,234 @@ BOARDS = { "slot_bytes": 0x330000, "slot1_address": 0x340000, }, + "station-g2-repeater": { + "target": "Station_G2_repeater", + "wifi_bridge": "esp32_s3_16mb_partition_migrator", + "lora_bridge": None, + "flash_bytes": 16 * 1024 * 1024, + "slot_bytes": 0x640000, + "slot1_address": 0x650000, + }, + "station-g2-room-server": { + "target": "Station_G2_room_server", + "wifi_bridge": "esp32_s3_16mb_partition_migrator", + "lora_bridge": None, + "flash_bytes": 16 * 1024 * 1024, + "slot_bytes": 0x640000, + "slot1_address": 0x650000, + }, + "t3s3-sx1262-repeater": { + "target": "LilyGo_T3S3_sx1262_repeater", + "wifi_bridge": "esp32_s3_4mb_partition_migrator", + "lora_bridge": None, + "flash_bytes": 4 * 1024 * 1024, + "slot_bytes": 0x1F0000, + "slot1_address": 0x200000, + }, + "t3s3-sx1262-room-server": { + "target": "LilyGo_T3S3_sx1262_room_server", + "wifi_bridge": "esp32_s3_4mb_partition_migrator", + "lora_bridge": None, + "flash_bytes": 4 * 1024 * 1024, + "slot_bytes": 0x1F0000, + "slot1_address": 0x200000, + }, + "t3s3-sx1276-repeater": { + "target": "LilyGo_T3S3_sx1276_repeater", + "wifi_bridge": "esp32_s3_4mb_partition_migrator", + "lora_bridge": None, + "flash_bytes": 4 * 1024 * 1024, + "slot_bytes": 0x1F0000, + "slot1_address": 0x200000, + }, + "t3s3-sx1276-room-server": { + "target": "LilyGo_T3S3_sx1276_room_server", + "wifi_bridge": "esp32_s3_4mb_partition_migrator", + "lora_bridge": None, + "flash_bytes": 4 * 1024 * 1024, + "slot_bytes": 0x1F0000, + "slot1_address": 0x200000, + }, + "thinknode-m2-repeater": { + "target": "ThinkNode_M2_Repeater", + "wifi_bridge": "esp32_s3_4mb_partition_migrator", + "lora_bridge": None, + "flash_bytes": 4 * 1024 * 1024, + "slot_bytes": 0x1F0000, + "slot1_address": 0x200000, + }, + "thinknode-m2-room-server": { + "target": "ThinkNode_M2_room_server", + "wifi_bridge": "esp32_s3_4mb_partition_migrator", + "lora_bridge": None, + "flash_bytes": 4 * 1024 * 1024, + "slot_bytes": 0x1F0000, + "slot1_address": 0x200000, + }, } + + +def role_for_target(target: str) -> str: + lower = target.lower() + if "_room_server" in lower: + return "room-server" + if "_repeater" in lower: + return "repeater" + if "_sensor" in lower: + return "sensor" + raise ValueError(f"migration target has no infrastructure role: {target}") + + +def add_wifi_only_roles(family: str, flash_mib: int, + targets: dict[str, str], + bridge: str | None = None) -> None: + """Register exact role identities; a family can have one or all roles.""" + if flash_mib not in (4, 8, 16): + raise ValueError(f"unsupported migration flash size: {flash_mib} MiB") + slot_bytes, slot1_address = { + 4: (0x1F0000, 0x200000), + 8: (0x330000, 0x340000), + 16: (0x640000, 0x650000), + }[flash_mib] + if bridge is None: + bridge = f"esp32_s3_{flash_mib}mb_partition_migrator" + for role, target in targets.items(): + if role not in ("repeater", "room-server", "sensor"): + raise ValueError(f"unsupported migration role: {role}") + key = f"{family}-{role}" + if key in BOARDS: + raise ValueError(f"duplicate migration board role: {key}") + BOARDS[key] = { + "target": target, + "role": role, + "wifi_bridge": bridge, + "lora_bridge": None, + "flash_bytes": flash_mib * 1024 * 1024, + "slot_bytes": slot_bytes, + "slot1_address": slot1_address, + } + + +# The first two repeater keys above retain their published LoRa-capable bridge +# names. Other role packages use the chip-family Wi-Fi bridge. These are the +# canonical ESP32 infrastructure identities on 4/8/16 MiB boards which can +# have a deployed Arduino default.csv 1.25 MiB table, including historically +# default.csv boards now using a larger layout. Other boards that were only +# ever built with unrelated layouts are not represented as 1.25 MiB migrants. +for family, flash_mib, roles in ( + ("heltec-v4", 16, {"room-server": "heltec_v4_room_server", + "sensor": "heltec_v4_sensor"}), + ("heltec-v4-expansionkit", 16, { + "repeater": "heltec_v4_expansionkit_repeater"}), + ("xiao-s3-wio", 8, {"room-server": "Xiao_S3_WIO_room_server", + "sensor": "Xiao_S3_WIO_sensor"}), + ("nibble-zero", 4, {"repeater": "nibble_zero_connect_repeater_", + "room-server": "nibble_zero_connect_room_server_"}), + ("nibble-screen", 4, {"repeater": "nibble_screen_connect_repeater_", + "room-server": "nibble_screen_connect_room_server_"}), + ("ebyte-eora-s3", 4, {"repeater": "Ebyte_EoRa-S3_Repeater", + "room-server": "Ebyte_EoRa-S3_room_server"}), + ("thinknode-m5", 4, {"repeater": "ThinkNode_M5_Repeater", + "room-server": "ThinkNode_M5_room_server"}), + ("heltec-wireless-tracker", 8, { + "repeater": "Heltec_Wireless_Tracker_repeater", + "room-server": "Heltec_Wireless_Tracker_room_server"}), + ("heltec-tracker-v2", 8, {"repeater": "heltec_tracker_v2_repeater", + "room-server": "heltec_tracker_v2_room_server", + "sensor": "heltec_tracker_v2_sensor"}), + ("heltec-v3", 8, {"repeater": "Heltec_v3_repeater", + "room-server": "Heltec_v3_room_server", + "sensor": "Heltec_v3_sensor"}), + ("heltec-wsl3", 8, {"repeater": "Heltec_WSL3_repeater", + "room-server": "Heltec_WSL3_room_server", + "sensor": "Heltec_WSL3_sensor"}), + ("heltec-wireless-paper", 8, { + "repeater": "Heltec_Wireless_Paper_repeater", + "room-server": "Heltec_Wireless_Paper_room_server"}), + ("t-beam-s3-supreme-sx1262", 8, { + "repeater": "T_Beam_S3_Supreme_SX1262_repeater", + "room-server": "T_Beam_S3_Supreme_SX1262_room_server"}), + ("thinknode-m7", 8, {"repeater": "ThinkNode_M7_repeater", + "room-server": "ThinkNode_M7_room_server"}), + ("xiao-s3", 8, {"repeater": "Xiao_S3_repeater", + "room-server": "Xiao_S3_room_server", + "sensor": "Xiao_S3_sensor"}), + ("mke-s3", 8, {"repeater": "MKE_s3_repeater", + "room-server": "MKE_s3_room_server", + "sensor": "MKE_s3_sensor"}), + ("rak-3112", 8, {"repeater": "RAK_3112_repeater", + "room-server": "RAK_3112_room_server", + "sensor": "RAK_3112_sensor"}), + ("heltec-e213", 16, {"repeater": "Heltec_E213_repeater", + "room-server": "Heltec_E213_room_server"}), + ("heltec-e290", 16, {"repeater": "Heltec_E290_repeater", + "room-server": "Heltec_E290_room_server"}), + ("heltec-rc32", 16, {"repeater": "heltec_rc32_repeater", + "room-server": "heltec_rc32_room_server", + "sensor": "heltec_rc32_sensor"}), + ("heltec-rc32-no-display", 16, { + "repeater": "heltec_rc32_without_display_repeater", + "room-server": "heltec_rc32_without_display_room_server", + "sensor": "heltec_rc32_without_display_sensor"}), + ("heltec-t190", 16, {"repeater": "Heltec_T190_repeater_", + "room-server": "Heltec_T190_room_server_"}), + ("heltec-v4-tft", 16, {"repeater": "heltec_v4_tft_repeater", + "room-server": "heltec_v4_tft_room_server", + "sensor": "heltec_v4_tft_sensor"}), + ("heltec-v4-r8", 16, {"repeater": "heltec_v4_r8_repeater", + "room-server": "heltec_v4_r8_room_server", + "sensor": "heltec_v4_r8_sensor"}), + ("heltec-v4-r8-tft", 16, {"repeater": "heltec_v4_r8_tft_repeater", + "room-server": "heltec_v4_r8_tft_room_server", + "sensor": "heltec_v4_r8_tft_sensor"}), + ("lilygo-tbeam-1w", 16, {"repeater": "LilyGo_TBeam_1W_repeater", + "room-server": "LilyGo_TBeam_1W_room_server"}), + ("lilygo-tdeck", 16, {"repeater": "LilyGo_TDeck_repeater"}), + ("lilygo-teth-elite-sx1262", 16, { + "repeater": "LilyGo_TETH_Elite_sx1262_repeater", + "room-server": "LilyGo_TETH_Elite_sx1262_room_server"}), + ("meshnology-w12", 16, {"repeater": "meshnology_w12_repeater", + "room-server": "meshnology_w12_room_server", + "sensor": "meshnology_w12_sensor"}), + ("station-g3-esp32", 16, {"repeater": "Station_G3_ESP32_repeater", + "room-server": "Station_G3_ESP32_room_server"}), + ("station-g3-esp32-logging", 16, { + "repeater": "Station_G3_ESP32_logging_repeater"}), + ("station-g2-logging", 16, { + "repeater": "Station_G2_logging_repeater"}), + ("thinknode-m9", 16, {"repeater": "ThinkNode_M9_repeater_", + "room-server": "ThinkNode_M9_room_server_"}), +): + add_wifi_only_roles(family, flash_mib, roles) + +add_wifi_only_roles("heltec-v2", 8, { + "repeater": "Heltec_v2_repeater", + "room-server": "Heltec_v2_room_server", +}, bridge="esp32_8mb_partition_migrator") +add_wifi_only_roles("heltec-ct62", 4, { + "repeater": "Heltec_ct62_repeater", + "sensor": "Heltec_ct62_sensor", +}, bridge="esp32_c3_4mb_partition_migrator") +add_wifi_only_roles("xiao-c3", 4, { + "repeater": "Xiao_C3_repeater", + "room-server": "Xiao_C3_room_server", +}, bridge="esp32_c3_4mb_partition_migrator") +add_wifi_only_roles("tenstar-c3-sx1262", 4, { + "repeater": "Tenstar_C3_sx1262_repeater", +}, bridge="esp32_c3_4mb_partition_migrator") +add_wifi_only_roles("tenstar-c3-sx1268", 4, { + "repeater": "Tenstar_C3_sx1268_repeater", +}, bridge="esp32_c3_4mb_partition_migrator") +add_wifi_only_roles("generic-e22-sx1262", 4, { + "repeater": "Generic_E22_sx1262_repeater", +}, bridge="esp32_4mb_partition_migrator") +add_wifi_only_roles("generic-e22-sx1268", 4, { + "repeater": "Generic_E22_sx1268_repeater", +}, bridge="esp32_4mb_partition_migrator") + +for specification in BOARDS.values(): + specification.setdefault("role", role_for_target(specification["target"])) + LEGACY_SLOT_BYTES = 0x140000 @@ -92,29 +319,19 @@ def check_esp32_stage(package: bytes, image: bytes, slot_bytes: int) -> None: raise ValueError("mOTA image cannot be staged in the inactive ESP32 slot") -def readme(board: str, spec: dict, version: str, source: str) -> str: - return f"""# {board} in-place partition migration +def readme(board: str, spec: dict, version: str, source: str, + has_full_mota: bool = True) -> str: + wifi_instructions = f"""## Wi-Fi route -Source: `{source}`; firmware version: `{version}`. This package is only for the -exact `{spec['target']}` board/role with {spec['flash_bytes'] // 1048576} MiB flash. -The current partition table must have stable NVS at 0x9000, OTA metadata at -0xE000, two OTA apps, and SPIFFS with `/identity/_main.id`. - -The bridge checks the live layout and refuses unsupported boards. It stages -the 96-byte public/private identity in unchanged NVS, copies and verifies the -required application, changes the partition table, then restores and verifies -the identity in expanded SPIFFS. No full-chip erase or USB connection is used. -The table sector and inactive app sectors are erased as part of migration. -SPIFFS may be reformatted; other SPIFFS settings can be recreated. - -## Wi-Fi route - -1. Upload `wifi-bridge.bin` through the existing repeater's Wi-Fi OTA page. +1. On the old node, run `start ota ap` through its authenticated terminal. + Join its `MeshCore-OTA` Wi-Fi and upload `wifi-bridge.bin` at the exact + `Started:` URL returned by the command (the port can vary by old build). 2. Keep power stable. Join `MeshCore-Migrate` (password `meshcore-migrate`), open `http://192.168.4.1/`, and wait for **Expanded layout ready**. 3. At that page's `/update`, upload `full-application.bin`. - -## LoRa route +""" + if spec["lora_bridge"]: + lora_instructions = """## LoRa route The old repeater must already have working MeshCore LoRa mOTA, a valid EndF, and the exact target ID in `manifest.json`. A stock image without that updater @@ -131,8 +348,51 @@ table and refuses if its image or target does not match. A full-image LoRa transfer can take considerable time. `full-application.bin` and `full-application.mota` are the same exact-target -Full repeater image. `target-partitions.bin` is included for verification only; +Full image. `target-partitions.bin` is included for verification only; do not upload it to the browser. Neither bridge is a USB merged image. +""" + else: + full_mota_note = ( + "Do not upload `full-application.mota` until the expanded layout " + "and new Full image are in place." + if has_full_mota else + "This ZIP does not include `full-application.mota`: the Full image " + "exceeds the current LoRa mOTA 2 KiB x 1024-block serving limit. " + "Use `full-application.bin` through Wi-Fi." + ) + lora_instructions = f"""## LoRa limitation + +This package has **no LoRa bridge**. The Wi-Fi updater in the installed old +firmware is required. On 4 MiB flash, the expanded app1 overlaps the old app1, +so the current LoRa receiver-copy method cannot survive a power loss safely. +Some older room-server images also have no LoRa mOTA receiver. Do not upload +the Full image through LoRa before the new layout is active. {full_mota_note} +`target-partitions.bin` is for verification only, not browser upload. +""" + return f"""# {board} in-place partition migration + +Source: `{source}`; firmware version: `{version}`. This package is only for the +physical board/role represented by `{spec['target']}` with +{spec['flash_bytes'] // 1048576} MiB flash. A legacy Wi-Fi-updatable observer or +bridge variant on the same hardware may migrate to this canonical Full target; +its previous feature settings are not preserved. The LoRa route, where present, +requires the old image's exact target ID. +The current partition table must have stable NVS at 0x9000, OTA metadata at +0xE000, two OTA apps, and SPIFFS with `/identity/_main.id`. + +The bridge checks the live layout and refuses unsupported boards. It stages +the 96-byte public/private identity in unchanged NVS, copies and verifies the +required application, changes the partition table, then restores and verifies +the identity in expanded SPIFFS. No full-chip erase or USB connection is used. +The table sector and inactive app sectors are erased as part of migration. +SPIFFS may be reformatted; other SPIFFS settings can be recreated. +The stock bootloader has no atomic backup partition table: power loss during +the table-sector erase/write can still require cable recovery. Do not use this +on an inaccessible node without accepting that risk and testing a sacrificial +example of the same board first. + +{wifi_instructions} +{lora_instructions} """ @@ -141,7 +401,10 @@ def package_board(name: str, spec: dict, build_dir: Path, output_dir: Path, target = spec["target"] target_id = target_id_for_env(target) hw_id = hardware_id_for_env(target) - stem = f"{target}-full-usb-wifi-ota-{version}-{source}" + images = sorted(build_dir.glob(f"{target}-full-*-{version}-{source}.bin")) + if len(images) != 1: + raise ValueError(f"{name}: expected one exact-target Full image, found {len(images)}") + stem = images[0].stem full_path = build_dir / (stem + ".bin") capability_path = build_dir / (stem + ".capabilities.json") validate_package(build_dir / stem) @@ -172,27 +435,41 @@ def package_board(name: str, spec: dict, build_dir: Path, output_dir: Path, raise ValueError(f"{name}: Full image identity or partition fit failed") wifi_bridge = (ROOT / ".pio" / "build" / spec["wifi_bridge"] / "firmware.bin").read_bytes() - lora_bridge_body = (ROOT / ".pio" / "build" / spec["lora_bridge"] / "firmware.bin").read_bytes() bridge_ident = FwIdent(pack_version(version.split("-", 1)[0]), target_id, hw_id) - lora_bridge, _ = ensure_endf(lora_bridge_body, bridge_ident) - if max(len(wifi_bridge), len(lora_bridge)) > LEGACY_SLOT_BYTES: + if len(wifi_bridge) > LEGACY_SLOT_BYTES: raise ValueError(f"{name}: bridge does not fit the 1.25 MiB legacy slot") if full_ident.fw_version != bridge_ident.fw_version: raise ValueError(f"{name}: bridge and Full image versions differ") files = { "wifi-bridge.bin": wifi_bridge, - "lora-bridge.mota": mota_full(lora_bridge, bridge_ident, 1024), "full-application.bin": full_image, - "full-application.mota": mota_full(full_image, full_ident, 2048), "target-partitions.bin": table, "capabilities.json": capability_path.read_bytes(), } - check_esp32_stage(files["lora-bridge.mota"], lora_bridge, LEGACY_SLOT_BYTES) - check_esp32_stage(files["full-application.mota"], full_image, slot_bytes) - files["README.md"] = readme(name, spec, version, source).encode() + if (len(full_image) + 2047) // 2048 <= 1024: + candidate_mota = mota_full(full_image, full_ident, 2048) + try: + check_esp32_stage(candidate_mota, full_image, slot_bytes) + except ValueError: + if spec["lora_bridge"]: + raise + else: + files["full-application.mota"] = candidate_mota + elif spec["lora_bridge"]: + raise ValueError(f"{name}: LoRa Full image exceeds the mOTA block limit") + if spec["lora_bridge"]: + lora_bridge_body = (ROOT / ".pio" / "build" / spec["lora_bridge"] / "firmware.bin").read_bytes() + lora_bridge, _ = ensure_endf(lora_bridge_body, bridge_ident) + if len(lora_bridge) > LEGACY_SLOT_BYTES: + raise ValueError(f"{name}: LoRa bridge does not fit the legacy slot") + files["lora-bridge.mota"] = mota_full(lora_bridge, bridge_ident, 1024) + check_esp32_stage(files["lora-bridge.mota"], lora_bridge, LEGACY_SLOT_BYTES) + files["README.md"] = readme(name, spec, version, source, + "full-application.mota" in files).encode() manifest = { "board": name, + "role": spec["role"], "target": target, "target_id": f"0x{target_id:08x}", "hardware_id": hw_id, @@ -201,6 +478,7 @@ def package_board(name: str, spec: dict, build_dir: Path, output_dir: Path, "flash_bytes": spec["flash_bytes"], "legacy_slot_bytes": LEGACY_SLOT_BYTES, "expanded_slot_bytes": slot_bytes, + "atomic_power_loss_recovery": False, "files": {filename: {"bytes": len(data), "sha256": sha256(data)} for filename, data in files.items()}, } diff --git a/src/helpers/ESP32PartitionMigrationPolicy.h b/src/helpers/ESP32PartitionMigrationPolicy.h index 9b421c6dc..a77747549 100644 --- a/src/helpers/ESP32PartitionMigrationPolicy.h +++ b/src/helpers/ESP32PartitionMigrationPolicy.h @@ -12,6 +12,7 @@ namespace esp32_partition_migration { constexpr uint32_t kRequiredFlashBytes = 16U * 1024U * 1024U; constexpr uint32_t kXiaoRequiredFlashBytes = 8U * 1024U * 1024U; +constexpr uint32_t kSmallRequiredFlashBytes = 4U * 1024U * 1024U; constexpr uint32_t kPartitionTableAddress = 0x8000; constexpr uint32_t kPartitionTableBytes = 0x1000; constexpr uint32_t kSectorBytes = 0x1000; @@ -59,6 +60,17 @@ constexpr PartitionGeometry kExpanded8MBLayout = { 0x670000, 0x180000, }; +// variants/dual_ota_full_4MB.csv. This is the largest practical dual-OTA +// layout on a 4 MiB board. It preserves only a 64 KiB SPIFFS region, so the +// private identity must be staged in stable NVS before the table is changed. +constexpr PartitionGeometry kExpanded4MBLayout = { + 0x9000, 0x5000, + 0xE000, 0x2000, + 0x10000, 0x1F0000, + 0x200000, 0x1F0000, + 0x3F0000, 0x10000, +}; + constexpr bool sameGeometry(const PartitionGeometry& left, const PartitionGeometry& right) { return left.nvs_address == right.nvs_address @@ -83,7 +95,10 @@ constexpr bool isExpandedLayout(const PartitionGeometry& geometry) { constexpr bool canMigrate(uint32_t flash_bytes, const PartitionGeometry& geometry) { - return flash_bytes >= kRequiredFlashBytes && isLegacyLayout(geometry); + return (flash_bytes == kSmallRequiredFlashBytes + || flash_bytes == kXiaoRequiredFlashBytes + || flash_bytes == kRequiredFlashBytes) + && isLegacyLayout(geometry); } // Binary prefix generated from Arduino-ESP32's default_16MB.csv by @@ -160,6 +175,29 @@ constexpr uint8_t kExpanded8MBPartitionTablePrefix[] = { constexpr size_t kExpanded8MBPartitionTablePrefixBytes = sizeof(kExpanded8MBPartitionTablePrefix); +// Generated from variants/dual_ota_full_4MB.csv with gen_esp32part.py. +constexpr uint8_t kExpanded4MBPartitionTablePrefix[] = { + 0xAA, 0x50, 0x01, 0x02, 0x00, 0x90, 0x00, 0x00, 0x00, 0x50, 0x00, 0x00, + 0x6E, 0x76, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAA, 0x50, 0x01, 0x00, + 0x00, 0xE0, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x6F, 0x74, 0x61, 0x64, + 0x61, 0x74, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xAA, 0x50, 0x00, 0x10, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x1F, 0x00, 0x61, 0x70, 0x70, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xAA, 0x50, 0x00, 0x11, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x1F, 0x00, + 0x61, 0x70, 0x70, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xAA, 0x50, 0x01, 0x82, + 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x01, 0x00, 0x73, 0x70, 0x69, 0x66, + 0x66, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xEB, 0xEB, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x80, 0xC5, 0x69, 0x36, + 0x65, 0xF8, 0xAB, 0x77, 0x12, 0x5A, 0xB8, 0x36, 0xBD, 0x27, 0xF1, 0x1F, +}; + +constexpr size_t kExpanded4MBPartitionTablePrefixBytes = + sizeof(kExpanded4MBPartitionTablePrefix); + struct TargetPlan { uint32_t flash_bytes; const char* name; @@ -174,6 +212,12 @@ constexpr TargetPlan kTarget8MB = { kExpanded8MBPartitionTablePrefixBytes, }; +constexpr TargetPlan kTarget4MB = { + kSmallRequiredFlashBytes, "MeshCore dual_ota_full_4MB", + kExpanded4MBLayout, kExpanded4MBPartitionTablePrefix, + kExpanded4MBPartitionTablePrefixBytes, +}; + constexpr TargetPlan kTarget16MB = { kRequiredFlashBytes, "Arduino default_16MB", kExpandedLayout, kExpandedPartitionTablePrefix, @@ -181,7 +225,8 @@ constexpr TargetPlan kTarget16MB = { }; constexpr const TargetPlan* targetForFlash(uint32_t flash_bytes) { - return flash_bytes == kXiaoRequiredFlashBytes ? &kTarget8MB + return flash_bytes == kSmallRequiredFlashBytes ? &kTarget4MB + : flash_bytes == kXiaoRequiredFlashBytes ? &kTarget8MB : flash_bytes == kRequiredFlashBytes ? &kTarget16MB : nullptr; } @@ -273,9 +318,21 @@ static_assert(kExpanded8MBLayout.app0_address + kExpanded8MBLayout.app0_size static_assert(kExpanded8MBLayout.app1_address + kExpanded8MBLayout.app1_size == kExpanded8MBLayout.spiffs_address, "8 MiB expanded SPIFFS must follow both OTA slots"); +static_assert(kExpanded4MBLayout.app0_address + kExpanded4MBLayout.app0_size + == kExpanded4MBLayout.app1_address, + "4 MiB expanded OTA slots must be adjacent"); +static_assert(kExpanded4MBLayout.app1_address + kExpanded4MBLayout.app1_size + == kExpanded4MBLayout.spiffs_address, + "4 MiB expanded SPIFFS must follow both OTA slots"); +static_assert(kExpanded4MBLayout.spiffs_address + kExpanded4MBLayout.spiffs_size + == kSmallRequiredFlashBytes, + "4 MiB target partitions must fit physical flash"); static_assert(kExpandedPartitionTablePrefixBytes == kExpanded8MBPartitionTablePrefixBytes, "target table buffers share one fixed DRAM size"); +static_assert(kExpanded4MBPartitionTablePrefixBytes + <= kExpandedPartitionTablePrefixBytes, + "4 MiB partition table must fit the shared DRAM buffer"); } // namespace esp32_partition_migration } // namespace mesh diff --git a/test/test_esp32_partition_migration_policy/test_esp32_partition_migration_policy.cpp b/test/test_esp32_partition_migration_policy/test_esp32_partition_migration_policy.cpp index f00a8f197..2de7fdd32 100644 --- a/test/test_esp32_partition_migration_policy/test_esp32_partition_migration_policy.cpp +++ b/test/test_esp32_partition_migration_policy/test_esp32_partition_migration_policy.cpp @@ -4,12 +4,16 @@ namespace policy = mesh::esp32_partition_migration; -TEST(Esp32PartitionMigrationPolicy, AcceptsOnlyTheKnownLegacyV4Layout) { +TEST(Esp32PartitionMigrationPolicy, AcceptsOnlyKnownFlashSizesWithLegacyLayout) { EXPECT_TRUE(policy::isLegacyLayout(policy::kLegacyLayout)); + EXPECT_TRUE(policy::canMigrate(policy::kSmallRequiredFlashBytes, + policy::kLegacyLayout)); + EXPECT_TRUE(policy::canMigrate(policy::kXiaoRequiredFlashBytes, + policy::kLegacyLayout)); EXPECT_TRUE(policy::canMigrate(policy::kRequiredFlashBytes, policy::kLegacyLayout)); - EXPECT_TRUE(policy::canMigrate(policy::kRequiredFlashBytes + 1, - policy::kLegacyLayout)); + EXPECT_FALSE(policy::canMigrate(policy::kRequiredFlashBytes + 1, + policy::kLegacyLayout)); EXPECT_FALSE(policy::canMigrate(policy::kRequiredFlashBytes - 1, policy::kLegacyLayout)); @@ -33,14 +37,19 @@ TEST(Esp32PartitionMigrationPolicy, ExpandedLayoutRetainsAndGrowsSpiffs) { policy::kExpandedLayout.spiffs_address); } -TEST(Esp32PartitionMigrationPolicy, GenericPlansAcceptStableEightAndSixteenMiBLayouts) { +TEST(Esp32PartitionMigrationPolicy, GenericPlansAcceptStableFourEightAndSixteenMiBLayouts) { + EXPECT_EQ(&policy::kTarget4MB, policy::targetForFlash(4U * 1024U * 1024U)); EXPECT_EQ(&policy::kTarget8MB, policy::targetForFlash(8U * 1024U * 1024U)); EXPECT_EQ(&policy::kTarget16MB, policy::targetForFlash(16U * 1024U * 1024U)); - EXPECT_EQ(nullptr, policy::targetForFlash(4U * 1024U * 1024U)); + EXPECT_EQ(nullptr, policy::targetForFlash(2U * 1024U * 1024U)); + EXPECT_TRUE(policy::isTargetLayout(4U * 1024U * 1024U, + policy::kExpanded4MBLayout)); EXPECT_TRUE(policy::isTargetLayout(8U * 1024U * 1024U, policy::kExpanded8MBLayout)); EXPECT_TRUE(policy::canMigrateGeneric(8U * 1024U * 1024U, policy::kLegacyLayout)); + EXPECT_TRUE(policy::canMigrateGeneric(4U * 1024U * 1024U, + policy::kLegacyLayout)); EXPECT_TRUE(policy::canMigrateGeneric(16U * 1024U * 1024U, policy::kLegacyLayout)); @@ -77,6 +86,9 @@ TEST(Esp32PartitionMigrationPolicy, LoRaResumePreservesTheOldApplicationInEither policy::kLegacyLayout.app0_address).valid); EXPECT_FALSE(policy::planLoRaResume(policy::kLegacyLayout, policy::kExpandedLayout, 0x300000).valid); + EXPECT_FALSE(policy::planLoRaResume(policy::kLegacyLayout, + policy::kExpanded4MBLayout, + policy::kLegacyLayout.app0_address).valid); } TEST(Esp32PartitionMigrationPolicy, GeneratedTablePrefixContainsAllSixEntriesAndMd5) { @@ -92,6 +104,9 @@ TEST(Esp32PartitionMigrationPolicy, GeneratedTablePrefixContainsAllSixEntriesAnd EXPECT_EQ(0x33, policy::kExpanded8MBPartitionTablePrefix[0x4A]); EXPECT_EQ(0x34, policy::kExpanded8MBPartitionTablePrefix[0x66]); EXPECT_EQ(0x67, policy::kExpanded8MBPartitionTablePrefix[0x86]); + EXPECT_EQ(0xC0U, policy::kExpanded4MBPartitionTablePrefixBytes); + EXPECT_EQ(0xEB, policy::kExpanded4MBPartitionTablePrefix[0xA0]); + EXPECT_EQ(0xEB, policy::kExpanded4MBPartitionTablePrefix[0xA1]); } int main(int argc, char** argv) { diff --git a/test/test_esp32_partition_migration_recipe.py b/test/test_esp32_partition_migration_recipe.py index 483cc3812..83210fe27 100644 --- a/test/test_esp32_partition_migration_recipe.py +++ b/test/test_esp32_partition_migration_recipe.py @@ -4,6 +4,7 @@ import hashlib import json import os from pathlib import Path +import re import subprocess import sys import tempfile @@ -16,9 +17,39 @@ sys.path.insert(0, str(ROOT / "scripts")) from build_esp32_partition_migration import ( # noqa: E402 build_steps, bundle_release, verify_archive, ) +from package_esp32_partition_migration import BOARDS, readme # noqa: E402 class Esp32MigrationRecipeTest(unittest.TestCase): + def test_catalog_covers_all_three_roles_and_exact_flash_plans(self): + self.assertEqual({"repeater", "room-server", "sensor"}, + {spec["role"] for spec in BOARDS.values()}) + self.assertIn("heltec-v4-sensor", BOARDS) + self.assertIn("xiao-s3-wio-sensor", BOARDS) + self.assertIn("heltec-ct62-sensor", BOARDS) + self.assertIn("nibble-zero-room-server", BOARDS) + self.assertEqual("esp32_c3_4mb_partition_migrator", + BOARDS["heltec-ct62-sensor"]["wifi_bridge"]) + self.assertEqual("esp32_4mb_partition_migrator", + BOARDS["generic-e22-sx1262-repeater"]["wifi_bridge"]) + self.assertEqual("esp32_s3_8mb_partition_migrator", + BOARDS["heltec-v3-sensor"]["wifi_bridge"]) + self.assertEqual("esp32_8mb_partition_migrator", + BOARDS["heltec-v2-room-server"]["wifi_bridge"]) + expected_slots = {4 * 1024 * 1024: 0x1F0000, + 8 * 1024 * 1024: 0x330000, + 16 * 1024 * 1024: 0x640000} + env_names = set() + for config in (ROOT / "variants").glob("*/platformio.ini"): + env_names.update(re.findall(r"^\[env:([^]]+)\]", + config.read_text(), re.MULTILINE)) + for board, spec in BOARDS.items(): + with self.subTest(board=board): + self.assertIn(spec["target"], env_names) + self.assertIn(spec["wifi_bridge"], env_names) + self.assertEqual(expected_slots[spec["flash_bytes"]], + spec["slot_bytes"]) + @unittest.skipIf(os.name == "nt", "POSIX shell menu runs in WSL/Linux") def test_shell_menu_and_noninteractive_selection(self): script = str(ROOT / "scripts/build_esp32_partition_migration.sh") @@ -28,11 +59,18 @@ class Esp32MigrationRecipeTest(unittest.TestCase): cwd=ROOT, text=True, capture_output=True, check=True) self.assertIn("build-firmware heltec_v4_repeater", one.stdout) self.assertNotIn("build-firmware Xiao_S3_WIO_repeater", one.stdout) - self.assertNotIn("Bundle complete qualified set", one.stdout) + self.assertNotIn("Bundle all configured board/role recipes", one.stdout) all_boards = subprocess.run(["sh", script, "--all", *common], cwd=ROOT, text=True, capture_output=True, check=True) self.assertIn("build-firmware Xiao_S3_WIO_repeater", all_boards.stdout) - self.assertIn("Bundle complete qualified set", all_boards.stdout) + self.assertIn("build-firmware ThinkNode_M2_Repeater", all_boards.stdout) + self.assertIn("Bundle all configured board/role recipes", all_boards.stdout) + sensors = subprocess.run(["sh", script, "--role", "sensor", *common], + cwd=ROOT, text=True, capture_output=True, + check=True) + self.assertIn("build-firmware Xiao_S3_WIO_sensor", sensors.stdout) + self.assertNotIn("build-firmware Xiao_S3_WIO_repeater", sensors.stdout) + self.assertNotIn("Bundle all configured board/role recipes", sensors.stdout) interactive = subprocess.run(["sh", script], input="q\n", cwd=ROOT, text=True, capture_output=True, check=True) @@ -40,7 +78,7 @@ class Esp32MigrationRecipeTest(unittest.TestCase): invalid = subprocess.run(["sh", script, "--board", "unknown", *common], cwd=ROOT, text=True, capture_output=True) self.assertNotEqual(0, invalid.returncode) - self.assertIn("not qualified", invalid.stderr) + self.assertIn("not configured", invalid.stderr) def test_full_images_precede_every_bridge_and_builds_are_serial(self): steps = build_steps(["heltec-v4", "xiao-s3-wio"], "v1.17.1.7-test", @@ -58,6 +96,31 @@ class Esp32MigrationRecipeTest(unittest.TestCase): "xiao_s3_partition_migrator_lora_repeater", ], [command[3] for command, _ in steps[2:]]) + def test_wifi_only_4mb_roles_share_one_bridge(self): + steps = build_steps(["thinknode-m2-repeater", "thinknode-m2-room-server"], + "v1.17.1.7-test", "usa-cascadia", "cascade", 4) + self.assertEqual([True, True, False], [full for _, full in steps]) + self.assertEqual("ThinkNode_M2_Repeater", steps[0][0][3]) + self.assertEqual("ThinkNode_M2_room_server", steps[1][0][3]) + self.assertEqual("esp32_s3_4mb_partition_migrator", steps[2][0][3]) + guide = readme("thinknode-m2-repeater", BOARDS["thinknode-m2-repeater"], + "v1.17.1.7-test", "01234567") + self.assertIn("no LoRa bridge", guide) + self.assertIn("power loss during", guide) + self.assertIn("exact\n `Started:` URL", guide) + self.assertNotIn("ota pull ", guide) + large_guide = readme("heltec-v3-sensor", BOARDS["heltec-v3-sensor"], + "v1.17.1.7-test", "01234567", has_full_mota=False) + self.assertIn("does not include `full-application.mota`", large_guide) + + def test_sensor_and_room_roles_share_the_8mb_bridge(self): + steps = build_steps(["xiao-s3-wio-sensor", "xiao-s3-wio-room-server"], + "v1.17.1.7-test", "usa-cascadia", "cascade", 4) + self.assertEqual([True, True, False], [full for _, full in steps]) + self.assertEqual("Xiao_S3_WIO_sensor", steps[0][0][3]) + self.assertEqual("Xiao_S3_WIO_room_server", steps[1][0][3]) + self.assertEqual("esp32_s3_8mb_partition_migrator", steps[2][0][3]) + def test_dry_run_has_no_build_side_effects(self): with tempfile.TemporaryDirectory(prefix="esp32-migration-plan-") as temporary: output_root = Path(temporary) / "release" @@ -102,7 +165,9 @@ class Esp32MigrationRecipeTest(unittest.TestCase): package_dir.mkdir() version = "v1.17.1.7-test" source = "01234567" - for board in ("heltec-v4", "xiao-s3-wio"): + sample_boards = ["heltec-v4", "xiao-s3-wio-room-server", + "heltec-ct62-sensor"] + for board in sample_boards: payload = (board + " image").encode() manifest = { "board": board, "firmware_version": version, @@ -118,24 +183,32 @@ class Esp32MigrationRecipeTest(unittest.TestCase): archive.writestr("manifest.json", json.dumps(manifest)) archive.writestr("full-application.bin", payload) release = bundle_release(output_root, package_dir, - ["heltec-v4", "xiao-s3-wio"], + sample_boards, version, source, "usa-cascadia", "cascade") with zipfile.ZipFile(release) as archive: self.assertIsNone(archive.testzip()) release_manifest = json.loads(archive.read("release-manifest.json")) - self.assertEqual(["heltec-v4", "xiao-s3-wio"], + self.assertEqual(sample_boards, release_manifest["boards"]) + self.assertEqual(["repeater", "room-server", "sensor"], + release_manifest["roles"]) + self.assertEqual("repeater", + release_manifest["board_roles"]["heltec-v4"]) + self.assertEqual("room-server", release_manifest["board_roles"][ + "xiao-s3-wio-room-server"]) + self.assertEqual("sensor", release_manifest["board_roles"][ + "heltec-ct62-sensor"]) for name, checks in release_manifest["packages"].items(): data = archive.read(name) self.assertEqual(checks["bytes"], len(data)) self.assertEqual(checks["sha256"], hashlib.sha256(data).hexdigest()) self.assertEqual(release, bundle_release( - output_root, package_dir, ["heltec-v4", "xiao-s3-wio"], + output_root, package_dir, sample_boards, version, source, "usa-cascadia", "cascade")) - (package_dir / f"xiao-s3-wio-{version}-{source}-migration.zip").unlink() - with self.assertRaisesRegex(ValueError, "missing xiao-s3-wio"): + (package_dir / f"heltec-ct62-sensor-{version}-{source}-migration.zip").unlink() + with self.assertRaisesRegex(ValueError, "missing heltec-ct62-sensor"): bundle_release(output_root, package_dir, - ["heltec-v4", "xiao-s3-wio"], + sample_boards, version, source, "usa-cascadia", "cascade") diff --git a/test/test_heltec_v4_partition_migrator.py b/test/test_heltec_v4_partition_migrator.py index 334f14b33..475a84eb5 100644 --- a/test/test_heltec_v4_partition_migrator.py +++ b/test/test_heltec_v4_partition_migrator.py @@ -17,6 +17,7 @@ XIAO_PROFILE = ROOT / "variants/xiao_s3_wio/platformio.ini" XIAO_MIGRATOR_BOARD = ROOT / "boards/seeed_xiao_esp32s3_migrator.json" PARTITIONS = Path.home() / ".platformio/packages/framework-arduinoespressif32/tools/partitions/default_16MB.csv" PARTITIONS_8MB = Path.home() / ".platformio/packages/framework-arduinoespressif32/tools/partitions/default_8MB.csv" +PARTITIONS_4MB = ROOT / "variants/dual_ota_full_4MB.csv" GENERATOR = Path.home() / ".platformio/packages/framework-arduinoespressif32/tools/gen_esp32part.py" @@ -98,6 +99,15 @@ class HeltecV4PartitionMigratorTest(unittest.TestCase): embedded_8mb = bytes(int(value, 16) for value in re.findall(r"0x([0-9A-F]{2})", block_8mb)) self.assertEqual(embedded_8mb, generated_8mb.read_bytes()[:len(embedded_8mb)]) + generated_4mb = Path(directory) / "default4.bin" + subprocess.run([sys.executable, str(GENERATOR), str(PARTITIONS_4MB), + str(generated_4mb)], check=True, capture_output=True, text=True) + start_4mb = text.index("kExpanded4MBPartitionTablePrefix[]") + block_4mb = text[start_4mb:text.index("};", start_4mb)] + embedded_4mb = bytes(int(value, 16) + for value in re.findall(r"0x([0-9A-F]{2})", block_4mb)) + self.assertEqual(0xC0, len(embedded_4mb)) + self.assertEqual(embedded_4mb, generated_4mb.read_bytes()[:len(embedded_4mb)]) if __name__ == "__main__": diff --git a/variants/esp32_partition_migrator/platformio.ini b/variants/esp32_partition_migrator/platformio.ini new file mode 100644 index 000000000..b0a54f643 --- /dev/null +++ b/variants/esp32_partition_migrator/platformio.ini @@ -0,0 +1,54 @@ +; Board-independent ESP32-S3 Wi-Fi bridge for 4 MiB dual-OTA nodes. This +; utility uses Wi-Fi, NVS and SPIFFS only; no board GPIO or PSRAM is needed. +; DIO and disabled USB CDC minimize board-specific boot assumptions. +[env:esp32_s3_4mb_partition_migrator] +platform = platformio/espressif32@6.11.0 +board = esp32-s3-devkitc-1 +framework = arduino +board_build.partitions = default.csv +board_build.flash_mode = dio +board_upload.flash_size = 4MB +board_upload.maximum_size = 4194304 +extra_scripts = post:scripts/check_firmware_ram.py +build_flags = + -D ESP32_PLATFORM + -D ARDUINO_USB_CDC_ON_BOOT=0 + -D MESH_MIN_RUNTIME_HEAP=100352 + -I src +build_src_filter = + -<*> + +<../examples/esp32_partition_migrator> +lib_deps = + ESP32Async/AsyncTCP @ 3.5.0 + ESP32Async/ESPAsyncWebServer @ 3.12.0 + file://arch/esp32/AsyncElegantOTA + +; The same board-independent bridge with a 16 MiB app image header. The +; source updater still sees Arduino default.csv's 1.25 MiB slots. +[env:esp32_s3_16mb_partition_migrator] +extends = env:esp32_s3_4mb_partition_migrator +board_upload.flash_size = 16MB +board_upload.maximum_size = 16777216 + +[env:esp32_s3_8mb_partition_migrator] +extends = env:esp32_s3_4mb_partition_migrator +board_upload.flash_size = 8MB +board_upload.maximum_size = 8388608 + +; Heltec V2 and other first-generation ESP32 (non-S3) 8 MiB boards need a +; matching chip-family image. The migration logic still verifies the live +; partition table and flash size before touching flash. +[env:esp32_8mb_partition_migrator] +extends = env:esp32_s3_8mb_partition_migrator +board = esp32dev + +; The 4 MiB C3 bridge covers historical default.csv images such as the +; Heltec CT62. It never assumes the old app slots are exactly 1.25 MiB: the +; live table must still pass the common geometry and identity checks. +[env:esp32_c3_4mb_partition_migrator] +extends = env:esp32_s3_4mb_partition_migrator +board = esp32-c3-devkitm-1 + +[env:esp32_4mb_partition_migrator] +extends = env:esp32_s3_4mb_partition_migrator +board = esp32dev