diff --git a/build.sh b/build.sh index 0b50f6ca..0bde3064 100755 --- a/build.sh +++ b/build.sh @@ -23,6 +23,7 @@ MQTT_DEBUG_OVERRIDE="${MQTT_DEBUG_OVERRIDE-}" FIRMWARE_FILENAME_INFIX="" ESP32_FULL_BUILD=0 SINGLE_TARGET_FULL_BUILD="${SINGLE_TARGET_FULL_BUILD:-0}" +EXACT_IDENTITY_FULL_BUILD="${EXACT_IDENTITY_FULL_BUILD:-0}" AUTO_PREFER_FULL_BUILD=0 AUTO_COMPLETE_FIRST_PASS=0 AUTO_REDUCED_FALLBACK_TARGET="" @@ -140,6 +141,7 @@ Options: --profile : Override runtime settings embedded in the firmware (not its feature set). --build-profile : Select feature/partition policy. Auto uses the combined Full MQTT/USB/WiFi recipe when it covers the plain infrastructure target; otherwise it first builds complete LoRa-OTA-capable firmware with a measured-size fallback. Internal-flash nRF52 repeaters and room servers also publish the reduced image for delta-staging headroom; XIAO nRF52 repeaters with external-QSPI staging publish one combined image. Standard preserves portable images, including the 1.25 MiB slot; full requires the expanded ESP32 recipe. --auto|--standard|--full: Short forms of --build-profile. + --full-exact: Build one expanded ESP32 release image under the requested target's own LoRa OTA identity. --skip-kiss|--include-kiss: Exclude (default) or include KISS modem targets in bulk builds. --clean|--resume: Clean output or resume existing Option 3/FULL-only artifacts. --require-ota: Require a verified wireless self-update path for infrastructure. @@ -995,6 +997,13 @@ parse_cli_options() { BUILD_PROFILE_EXPLICIT=1 shift ;; + --full-exact) + BUILD_PROFILE_OVERRIDE="full" + BUILD_PROFILE_EXPLICIT=1 + SINGLE_TARGET_FULL_BUILD=1 + EXACT_IDENTITY_FULL_BUILD=1 + shift + ;; --skip-kiss) KISS_MODE_OVERRIDE="skip" shift @@ -5367,7 +5376,15 @@ configure_effective_build_profile() { ;; esac - if [ "$SINGLE_TARGET_FULL_BUILD" = "1" ]; then + if [ "$EXACT_IDENTITY_FULL_BUILD" = "1" ]; then + if [ "$command_name" != "build-firmware" ] \ + || [ "${#RESOLVED_BUILD_TARGETS[@]}" -ne 1 ] \ + || ! is_esp32_canonical_full_release_target "$target"; then + echo "--full-exact requires one canonical ESP32 infrastructure target." + return 1 + fi + BUILD_PROFILE_EFFECTIVE="full" + elif [ "$SINGLE_TARGET_FULL_BUILD" = "1" ]; then BUILD_PROFILE_EFFECTIVE="full" elif is_automatic_profile_command "$command_name"; then if [ "$BUILD_PROFILE_EXPLICIT" = "1" ] \ @@ -6270,6 +6287,11 @@ run_command() { return $? fi + if [ "$EXACT_IDENTITY_FULL_BUILD" = "1" ]; then + run_full_only_esp32_profile "${RESOLVED_BUILD_TARGETS[@]}" + return $? + fi + if [ "$SINGLE_TARGET_FULL_BUILD" = "1" ]; then run_full_esp32_build_targets "all" "${RESOLVED_BUILD_TARGETS[@]}" return $? @@ -6461,7 +6483,7 @@ launch_background_build() { PLATFORMIO_BUILD_UNFLAGS PLATFORMIO_BUILD_SRC_FILTER PLATFORMIO_EXTRA_SCRIPTS DISABLE_DEBUG OPTION3_PIO_JOBS OUTPUT_DIR FIRMWARE_VERSION FIRMWARE_BUILD_NUMBER BUILD_PROFILE_OVERRIDE - BUILD_PROFILE_EXPLICIT SINGLE_TARGET_FULL_BUILD FIRMWARE_PROFILE_OVERRIDE + BUILD_PROFILE_EXPLICIT SINGLE_TARGET_FULL_BUILD EXACT_IDENTITY_FULL_BUILD FIRMWARE_PROFILE_OVERRIDE MESHDEBUG_OVERRIDE PACKET_LOGGING_OVERRIDE MQTT_BRIDGE_OVERRIDE MQTT_DEBUG_OVERRIDE RADIO_SETTING_TITLE RADIO_FREQ_OVERRIDE RADIO_BW_OVERRIDE RADIO_SF_OVERRIDE RADIO_CR_OVERRIDE diff --git a/docs/esp32_wifi_partition_migration.md b/docs/esp32_wifi_partition_migration.md index ec77672f..741e9be4 100644 --- a/docs/esp32_wifi_partition_migration.md +++ b/docs/esp32_wifi_partition_migration.md @@ -1,10 +1,11 @@ -# ESP32 legacy-partition migration over Wi-Fi +# ESP32 legacy-partition migration over Wi-Fi or LoRa The partition-migration bridge lets a supported ESP32 node move from a small -dual-OTA layout to a verified larger dual-OTA layout without cable flashing. -It is designed for inaccessible repeaters: the bridge itself fits a legacy -1.25 MiB slot, replaces the table from Wi-Fi, and then exposes Wi-Fi again for -the full application image. +dual-OTA layout to a verified larger dual-OTA layout without cable flashing or +a full-chip erase. It fits a legacy 1.25 MiB slot. The Wi-Fi version exposes an +updater for the full image after migration. The LoRa version retains the old +OTA-capable repeater in the other expanded slot and returns to it after +restoring identity, so it can fetch the full image over LoRa. It currently has explicit, byte-for-byte generated target plans for Arduino ESP32 `default_8MB.csv` and `default_16MB.csv`. It does not guess a table for @@ -31,6 +32,9 @@ image is deliberately not raw-copied into an expanded partition: SPIFFS is not safe to resize that way. Other SPIFFS-backed settings can be recreated after the full image boots. +Migration does erase the 4 KiB partition-table sector and the destination app +sectors. Expanded SPIFFS can be reformatted. It never issues a full-chip erase. + ## Eligible source layout The bridge only accepts a source table that has all of the following: @@ -76,6 +80,27 @@ bootloader and table offsets for cable flashing, not browser OTA. `xiao_s3_partition_legacy_seed` is a disposable-hardware test image only. It models the legacy table and an identity file; it is not a deployable repeater. +## LoRa-only migration of a repeater + +The exact old repeater must already support MeshCore LoRa mOTA and have a valid +EndF image identity. The LoRa bridge checks the old app's target ID and body +hash before any partition-table write; an unsupported or damaged receiver is +refused. It works from either legacy A or B. It copies legacy B to the future +B address, boots the bridge once to restore the private key in expanded +SPIFFS, then boots the preserved old repeater. That repeater can receive the +full image into its now-expanded inactive slot. + +The board-specific LoRa bridges are `heltec_v4_partition_migrator_lora_repeater` +and `xiao_s3_partition_migrator_lora_repeater`. Build the final repeater with +`bash build.sh build-firmware --full-exact` so its mOTA target ID +still matches the old repeater. `scripts/package_esp32_partition_migration.py` +checks the images, partition tables, hashes and mOTA containers and creates +separate Wi-Fi/LoRa migration ZIPs for those two boards. The ZIP README gives +the exact `ota ls`, `ota pull flash`, `ota install` sequence. + +A stock repeater without LoRa mOTA cannot use the LoRa-only route. The +Wi-Fi bridge remains the route for that device. + ## Heltec V4 / V4.3 Build `heltec_v4_partition_migrator`, upload it through the legacy Wi-Fi diff --git a/docs/heltec_v4_wifi_partition_migration.md b/docs/heltec_v4_wifi_partition_migration.md index 28aa493c..3ce69057 100644 --- a/docs/heltec_v4_wifi_partition_migration.md +++ b/docs/heltec_v4_wifi_partition_migration.md @@ -1,7 +1,7 @@ # Heltec V4 legacy-partition migration over Wi-Fi Heltec V4/V4.3 OLED repeaters with their normal 16 MiB flash use the shared -[ESP32 Wi-Fi partition migration](esp32_wifi_partition_migration.md) bridge. +[ESP32 partition migration](esp32_wifi_partition_migration.md) bridge. It changes Arduino's old `default.csv` layout, with two 1.25 MiB OTA slots, to `default_16MB.csv`, with two 6.25 MiB OTA slots. @@ -20,3 +20,8 @@ Do not use an `-merged.bin` in either browser uploader. Merged images contain bootloader and table offsets for cable flashing. The bridge preserves the private identity; other SPIFFS-backed settings may be recreated after the Full image starts. + +For an existing `heltec_v4_repeater` that already supports LoRa mOTA, use the +separate LoRa migration package described in the shared guide. It preserves +the old repeater across the partition change so the full image can also be +installed over LoRa without reaching the board's Wi-Fi network. diff --git a/examples/esp32_partition_migrator/main.cpp b/examples/esp32_partition_migrator/main.cpp index 5ae89f67..02026ccb 100644 --- a/examples/esp32_partition_migrator/main.cpp +++ b/examples/esp32_partition_migrator/main.cpp @@ -9,6 +9,12 @@ #include #include #include +#if defined(MESHCORE_MIGRATION_RESUME_OTA) +#include +#ifndef MESHCORE_MIGRATION_EXPECTED_TARGET_ID +#error "LoRa migration requires the exact legacy OTA target ID" +#endif +#endif #include #include @@ -33,6 +39,10 @@ constexpr char kMigrationNvsNamespace[] = "mesh-pt-migrate"; constexpr char kMigrationIdentityKey[] = "identity"; // NVS key names are limited to 15 characters. constexpr char kMigrationIdentityPendingKey[] = "id-pending"; +#if defined(MESHCORE_MIGRATION_RESUME_OTA) +constexpr char kMigrationResumeSlotKey[] = "resume-slot"; +constexpr size_t kEndfBytes = 56; +#endif static_assert(sizeof(kMigrationNvsNamespace) - 1 <= 15, "ESP32 NVS namespace names are limited to 15 characters"); @@ -142,10 +152,11 @@ esp_partition_t rawPartition(uint32_t address, uint32_t size, const char* label) return part; } -esp_partition_t rawApp0Partition(uint32_t address, uint32_t size, const char* label) { +esp_partition_t rawAppPartition(uint32_t address, uint32_t size, + esp_partition_subtype_t subtype, const char* label) { esp_partition_t part = rawPartition(address, size, label); part.type = ESP_PARTITION_TYPE_APP; - part.subtype = ESP_PARTITION_SUBTYPE_APP_OTA_0; + part.subtype = subtype; return part; } @@ -322,6 +333,104 @@ bool restoreStagedIdentity() { return true; } +#if defined(MESHCORE_MIGRATION_RESUME_OTA) +uint32_t readLe32(const uint8_t* bytes) { + return static_cast(bytes[0]) + | (static_cast(bytes[1]) << 8) + | (static_cast(bytes[2]) << 16) + | (static_cast(bytes[3]) << 24); +} + +// The preserved app must really be the LoRa OTA receiver for this exact +// package. A valid EndF is emitted only for OTA-enabled MeshCore builds, and +// its body hash prevents a stale marker from authorizing a damaged image. +bool validLegacyOtaReceiver(const esp_partition_t& source) { + uint8_t header = 0; + if (esp_partition_read(&source, 0, &header, 1) != ESP_OK || header != 0xE9) { + strcpy(status_text, "Refused: old LoRa application image is invalid"); + return false; + } + uint8_t trailer[kEndfBytes]; + for (uint32_t base = 0; base < source.size; + base += sizeof(copy_buffer)) { + const uint32_t count = source.size - base < sizeof(copy_buffer) + ? source.size - base : sizeof(copy_buffer); + if (esp_partition_read(&source, base, copy_buffer, count) != ESP_OK) break; + for (uint32_t index = 0; index < count; ++index) { + if (copy_buffer[index] != 'E') continue; + const uint32_t marker = base + index; + if (marker + kEndfBytes > source.size + || esp_partition_read(&source, marker, trailer, sizeof(trailer)) != ESP_OK + || memcmp(trailer, "EndF", 4) != 0 + || readLe32(trailer + 4) != marker + || readLe32(trailer + 20) != MESHCORE_MIGRATION_EXPECTED_TARGET_ID) { + continue; + } + mbedtls_sha256_context hash; + mbedtls_sha256_init(&hash); + bool valid = mbedtls_sha256_starts_ret(&hash, 0) == 0; + for (uint32_t offset = 0; valid && offset < marker;) { + const uint32_t length = marker - offset < sizeof(copy_buffer) + ? marker - offset : sizeof(copy_buffer); + valid = esp_partition_read(&source, offset, copy_buffer, length) == ESP_OK + && mbedtls_sha256_update_ret(&hash, copy_buffer, length) == 0; + offset += length; + } + uint8_t digest[32]; + valid = valid && mbedtls_sha256_finish_ret(&hash, digest) == 0 + && memcmp(digest, trailer + 8, 8) == 0; + mbedtls_sha256_free(&hash); + if (valid) return true; + strcpy(status_text, "Refused: old LoRa firmware EndF hash is invalid"); + return false; + } + } + strcpy(status_text, "Refused: old LoRa firmware has no matching valid EndF"); + return false; +} + +bool stageResumeSlot(uint8_t slot) { + Preferences migration_nvs; + const bool opened = migration_nvs.begin(kMigrationNvsNamespace, false); + const bool saved = opened && migration_nvs.putUChar(kMigrationResumeSlotKey, slot) == 1; + if (opened) migration_nvs.end(); + if (!saved) strcpy(status_text, "Could not stage old LoRa application slot"); + return saved; +} + +bool resumeLegacyOtaReceiver(const migration::TargetPlan& plan) { + Preferences migration_nvs; + if (!migration_nvs.begin(kMigrationNvsNamespace, false)) { + strcpy(status_text, "Could not read old LoRa application slot"); + return false; + } + const uint8_t slot = migration_nvs.getUChar(kMigrationResumeSlotKey, 0xFF); + migration_nvs.end(); + if (slot > 1) { + strcpy(status_text, "Expanded layout ready; no LoRa resume record"); + return false; + } + const esp_partition_t old_app = slot == 0 + ? rawAppPartition(plan.layout.app0_address, plan.layout.app0_size, + ESP_PARTITION_SUBTYPE_APP_OTA_0, "app0") + : rawAppPartition(plan.layout.app1_address, plan.layout.app1_size, + ESP_PARTITION_SUBTYPE_APP_OTA_1, "app1"); + const esp_err_t selected = esp_ota_set_boot_partition(&old_app); + if (selected != ESP_OK) { + snprintf(status_text, sizeof(status_text), "Could not resume old LoRa app: %s", + errName(selected)); + return false; + } + if (migration_nvs.begin(kMigrationNvsNamespace, false)) { + migration_nvs.remove(kMigrationResumeSlotKey); + migration_nvs.end(); + } + strcpy(status_text, "Private key restored; resuming old LoRa firmware"); + reboot_at = millis() + 1000; + return true; +} +#endif + bool publishExpandedPartitionTable(const migration::TargetPlan& plan) { // Preserve ESP-IDF's normal OS flash hooks. In particular, their start/end // hooks suspend the other core and safely disable/re-enable caches. Do not @@ -402,6 +511,22 @@ void runMigration() { return; } +#if defined(MESHCORE_MIGRATION_RESUME_OTA) + const migration::LoRaResumePlan resume = migration::planLoRaResume( + geometry, plan->layout, running->address); + if (!resume.valid) { + strcpy(status_text, "Refused: legacy slots cannot preserve LoRa receiver"); + Serial.println(status_text); + return; + } + const esp_partition_t* old_receiver = resume.resume_slot == 0 + ? refs.app0 : refs.app1; + if (!validLegacyOtaReceiver(*old_receiver)) { + Serial.println(status_text); + return; + } +#endif + Serial.println("Migration: staging private key; do not interrupt power"); if (!stageLegacyIdentity()) { Serial.println(status_text); @@ -409,9 +534,19 @@ void runMigration() { } Serial.println("Migration: private key safely staged in NVS"); - // Always make target app0 contain this migration image. If it was uploaded - // into a different legacy slot, this preserves a Wi-Fi endpoint after the - // table change. Never erase a running source range while copying it. + // The Wi-Fi bridge always runs from target app0. The LoRa bridge instead + // keeps the old application in the other expanded slot, restores identity, + // then boots that old application to receive the final image by LoRa. +#if defined(MESHCORE_MIGRATION_RESUME_OTA) + Serial.println("Migration: preserving legacy app1 in expanded app1"); + if (!copyAndVerify(*refs.app1, plan->layout.app1_address, + refs.app1->size, "future-app1") + || !stageResumeSlot(resume.resume_slot)) { + Serial.println(status_text); + return; + } +#else + // Never erase a running source range while copying it. if (running->address != plan->layout.app0_address) { if (running->size > plan->layout.app0_size) { strcpy(status_text, "Refused: migration slot is larger than future app0"); @@ -432,29 +567,36 @@ void runMigration() { } Serial.println("Migration: bridge image copied to app0"); } +#endif // The OTA-select data records a slot identity. Select a descriptor for the - // target app0 before replacing the table so the next boot runs this bridge - // at the target table's app0 address, never the blank new app1. - const esp_partition_t target_app0 = rawApp0Partition(plan->layout.app0_address, - plan->layout.app0_size, - "app0"); - const esp_err_t select_result = esp_ota_set_boot_partition(&target_app0); + // bridge's expanded slot before replacing the table so the next boot runs + // the bridge at the correct address under the new layout. +#if defined(MESHCORE_MIGRATION_RESUME_OTA) + const bool bridge_in_app0 = resume.bridge_slot == 0; +#else + const bool bridge_in_app0 = true; +#endif + const esp_partition_t target_bridge = bridge_in_app0 + ? rawAppPartition(plan->layout.app0_address, plan->layout.app0_size, + ESP_PARTITION_SUBTYPE_APP_OTA_0, "app0") + : rawAppPartition(plan->layout.app1_address, plan->layout.app1_size, + ESP_PARTITION_SUBTYPE_APP_OTA_1, "app1"); + const esp_err_t select_result = esp_ota_set_boot_partition(&target_bridge); if (select_result != ESP_OK) { - snprintf(status_text, sizeof(status_text), "Could not select app0: %s", + snprintf(status_text, sizeof(status_text), "Could not select bridge slot: %s", errName(select_result)); Serial.println(status_text); return; } - Serial.println("Migration: app0 selected for restart"); + Serial.println("Migration: bridge slot selected for restart"); Serial.println("Migration: publishing expanded partition table"); // This bridge is built without native USB CDC, so a connected USB host cannot // post a flash-backed event while the partition-sector operation disables the // flash cache. Stop UART0 as well: the S3's serial event path is otherwise - // still able to interrupt the raw flash operation. The bridge itself is - // deliberately Wi-Fi-only; the final normal repeater build restores its - // standard USB behavior. + // still able to interrupt the raw flash operation. The bridge has no native + // USB CDC; the final normal repeater build restores its standard USB behavior. Serial.flush(); Serial.end(); const bool table_published = publishExpandedPartitionTable(*plan); @@ -469,7 +611,11 @@ void runMigration() { if (kMigrationRestartDelayMs == 0) { strcpy(status_text, "Partition table verified; waiting for test reboot"); } else { +#if defined(MESHCORE_MIGRATION_RESUME_OTA) + strcpy(status_text, "Migration complete; restarting LoRa bridge"); +#else strcpy(status_text, "Migration complete; restarting Wi-Fi uploader"); +#endif reboot_at = millis() + kMigrationRestartDelayMs; } Serial.println(status_text); @@ -553,6 +699,10 @@ void setup() { && migration::isTargetLayout(flash_bytes, geometry)) { if (restoreStagedIdentity()) { strcpy(status_text, "Expanded layout ready"); +#if defined(MESHCORE_MIGRATION_RESUME_OTA) + const migration::TargetPlan* plan = migration::targetForFlash(flash_bytes); + if (plan) resumeLegacyOtaReceiver(*plan); +#endif } } else if (findPartitions(refs, geometry) && migration::canMigrateGeneric(flash_bytes, geometry)) { diff --git a/scripts/package_esp32_partition_migration.py b/scripts/package_esp32_partition_migration.py new file mode 100644 index 00000000..418193bf --- /dev/null +++ b/scripts/package_esp32_partition_migration.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Package the two-stage Wi-Fi and LoRa ESP32 partition migrations.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import struct +import sys +import zipfile + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools" / "mota")) +from motalib import ( # noqa: E402 + CODEC_FULL, FwIdent, build_container, build_manifest, ensure_endf, + hardware_id_for_env, pack_version, parse_container, parse_endf_ident, + target_id_for_env, verify, +) +from check_esp32_app_size import app_partition_size # noqa: E402 +from firmware_memory_manifest import validate_package # noqa: E402 + + +BOARDS = { + "heltec-v4": { + "target": "heltec_v4_repeater", + "full_env": "heltec_v4_repeater_observer_mqtt", + "wifi_bridge": "heltec_v4_partition_migrator", + "lora_bridge": "heltec_v4_partition_migrator_lora_repeater", + "flash_bytes": 16 * 1024 * 1024, + "slot_bytes": 0x640000, + "slot1_address": 0x650000, + }, + "xiao-s3-wio": { + "target": "Xiao_S3_WIO_repeater", + "full_env": "Xiao_S3_WIO_repeater_observer_mqtt", + "wifi_bridge": "xiao_s3_partition_migrator", + "lora_bridge": "xiao_s3_partition_migrator_lora_repeater", + "flash_bytes": 8 * 1024 * 1024, + "slot_bytes": 0x330000, + "slot1_address": 0x340000, + }, +} +LEGACY_SLOT_BYTES = 0x140000 + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def partition_entries(table: bytes) -> dict[str, tuple[int, int]]: + entries = {} + for offset in range(0, len(table) - 31, 32): + entry = table[offset:offset + 32] + if entry[:2] != b"\xaa\x50": + continue + address, size = struct.unpack_from(" bytes: + manifest = build_manifest( + target_id=identity.target_id, + fw_version=identity.fw_version, + image_size=len(image), + payload=image, + block_size=1024, + image_hash=hashlib.sha256(image).digest(), + codec_id=CODEC_FULL, + is_full=True, + hw_id=identity.hw_id, + ) + package = build_container(manifest, image) + problems = verify(parse_container(package)) + if problems: + raise ValueError("mOTA verification failed: " + "; ".join(problems)) + return package + + +def readme(board: str, spec: dict, version: str, source: str) -> str: + return f"""# {board} in-place partition migration + +Source: `{source}`; firmware version: `{version}`. This package is only for the +exact `{spec['target']}` board/role with {spec['flash_bytes'] // 1048576} MiB flash. +The current partition table must have stable NVS at 0x9000, OTA metadata at +0xE000, two OTA apps, and SPIFFS with `/identity/_main.id`. + +The bridge checks the live layout and refuses unsupported boards. It stages +the 96-byte public/private identity in unchanged NVS, copies and verifies the +required application, changes the partition table, then restores and verifies +the identity in expanded SPIFFS. No full-chip erase or USB connection is used. +The table sector and inactive app sectors are erased as part of migration. +SPIFFS may be reformatted; other SPIFFS settings can be recreated. + +## Wi-Fi route + +1. Upload `wifi-bridge.bin` through the existing repeater's Wi-Fi OTA page. +2. Keep power stable. Join `MeshCore-Migrate` (password `meshcore-migrate`), + open `http://192.168.4.1/`, and wait for **Expanded layout ready**. +3. At that page's `/update`, upload `full-application.bin`. + +## LoRa route + +The old repeater must already have working MeshCore LoRa mOTA, a valid EndF, +and the exact target ID in `manifest.json`. A stock image without that updater +cannot take this route. The bridge checks the old receiver before changing the +table and refuses if its image or target does not match. + +1. Make `lora-bridge.mota` available from a compatible mOTA seeder. On the old + repeater use `ota ls`, `ota pull flash`, then `ota install` after the + complete download. A manual install handles an equal-version bridge. +2. Keep power stable. The bridge expands the table, restores the private key, + and reboots into the preserved old LoRa repeater in the opposite slot. +3. Serve `full-application.mota`; again use `ota ls`, `ota pull flash`, + then `ota install`. The old repeater now sees the expanded inactive slot. + A full-image LoRa transfer can take considerable time. + +`full-application.bin` and `full-application.mota` are the same exact-target +Full repeater image. `target-partitions.bin` is included for verification only; +do not upload it to the browser. Neither bridge is a USB merged image. +""" + + +def package_board(name: str, spec: dict, build_dir: Path, output_dir: Path, + version: str, source: str) -> Path: + target = spec["target"] + target_id = target_id_for_env(target) + hw_id = hardware_id_for_env(target) + stem = f"{target}-full-usb-wifi-ota-{version}-{source}" + full_path = build_dir / (stem + ".bin") + capability_path = build_dir / (stem + ".capabilities.json") + validate_package(build_dir / stem) + capabilities = json.loads(capability_path.read_text()) + if (capabilities.get("target") != target + or capabilities.get("build_profile") != "full" + or capabilities.get("platform") != "ESP32_PLATFORM" + or not capabilities.get("ota_update_verified")): + raise ValueError(f"{name}: Full build has the wrong target or lacks OTA") + + table_path = ROOT / ".pio" / "build" / spec["full_env"] / "partitions.bin" + table = table_path.read_bytes() + entries = partition_entries(table) + if (entries.get("nvs") != (0x9000, 0x5000) + or entries.get("otadata") != (0xE000, 0x2000) + or entries.get("app0") != (0x10000, spec["slot_bytes"]) + or entries.get("app1") != (spec["slot1_address"], spec["slot_bytes"])): + raise ValueError(f"{name}: built partition table differs from the migration plan") + _label, slot_bytes = app_partition_size(table) + full_image = full_path.read_bytes() + full_ident = parse_endf_ident(full_image) + if (full_ident is None or full_ident.target_id != target_id + or full_ident.hw_id != hw_id or len(full_image) >= slot_bytes): + raise ValueError(f"{name}: Full image identity or partition fit failed") + + wifi_bridge = (ROOT / ".pio" / "build" / spec["wifi_bridge"] / "firmware.bin").read_bytes() + lora_bridge_body = (ROOT / ".pio" / "build" / spec["lora_bridge"] / "firmware.bin").read_bytes() + bridge_ident = FwIdent(pack_version(version.split("-", 1)[0]), target_id, hw_id) + lora_bridge, _ = ensure_endf(lora_bridge_body, bridge_ident) + if max(len(wifi_bridge), len(lora_bridge)) > LEGACY_SLOT_BYTES: + raise ValueError(f"{name}: bridge does not fit the 1.25 MiB legacy slot") + if full_ident.fw_version != bridge_ident.fw_version: + raise ValueError(f"{name}: bridge and Full image versions differ") + + files = { + "wifi-bridge.bin": wifi_bridge, + "lora-bridge.mota": mota_full(lora_bridge, bridge_ident), + "full-application.bin": full_image, + "full-application.mota": mota_full(full_image, full_ident), + "target-partitions.bin": table, + "capabilities.json": capability_path.read_bytes(), + } + files["README.md"] = readme(name, spec, version, source).encode() + manifest = { + "board": name, + "target": target, + "target_id": f"0x{target_id:08x}", + "hardware_id": hw_id, + "source_commit": source, + "firmware_version": version, + "flash_bytes": spec["flash_bytes"], + "legacy_slot_bytes": LEGACY_SLOT_BYTES, + "expanded_slot_bytes": slot_bytes, + "files": {filename: {"bytes": len(data), "sha256": sha256(data)} + for filename, data in files.items()}, + } + files["manifest.json"] = (json.dumps(manifest, indent=2) + "\n").encode() + files["SHA256SUMS.txt"] = "".join( + f"{sha256(data)} {filename}\n" for filename, data in sorted(files.items()) + ).encode() + directory = output_dir / name + directory.mkdir(parents=True) + for filename, data in files.items(): + (directory / filename).write_bytes(data) + archive = output_dir / f"{name}-{version}-{source}-migration.zip" + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_STORED) as zip_file: + for filename in sorted(files): + zip_file.write(directory / filename, arcname=filename) + return archive + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--build-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--source", required=True, help="eight-character source commit") + args = parser.parse_args() + if args.output_dir.exists() and any(args.output_dir.iterdir()): + parser.error("output directory must be empty") + args.output_dir.mkdir(parents=True, exist_ok=True) + archives = [package_board(name, spec, args.build_dir, args.output_dir, + args.version, args.source) + for name, spec in BOARDS.items()] + for archive in archives: + print(f"{archive}: {archive.stat().st_size} bytes") + + +if __name__ == "__main__": + main() diff --git a/src/helpers/ESP32PartitionMigrationPolicy.h b/src/helpers/ESP32PartitionMigrationPolicy.h index 419bb01e..9b421c6d 100644 --- a/src/helpers/ESP32PartitionMigrationPolicy.h +++ b/src/helpers/ESP32PartitionMigrationPolicy.h @@ -222,6 +222,43 @@ inline bool canMigrateGeneric(uint32_t flash_bytes, && !sameGeometry(geometry, plan->layout); } +// A LoRa migration keeps the old OTA-capable application alive while the +// bridge restores SPIFFS identity. Legacy app0 stays at its address; legacy +// app1 is copied to the new app1 before the table is published. The bridge +// boots first, then hands off to whichever slot contains the old application. +struct LoRaResumePlan { + bool valid; + uint8_t bridge_slot; + uint8_t resume_slot; +}; + +constexpr bool overlaps(uint32_t a, uint32_t a_size, + uint32_t b, uint32_t b_size) { + return static_cast(a) < static_cast(b) + b_size + && static_cast(b) < static_cast(a) + a_size; +} + +inline LoRaResumePlan planLoRaResume(const PartitionGeometry& source, + const PartitionGeometry& target, + uint32_t running_address) { + const bool in_app0 = running_address == source.app0_address; + const bool in_app1 = running_address == source.app1_address; + return { + (in_app0 || in_app1) + && source.app0_address == target.app0_address + && source.app0_size <= target.app0_size + && source.app1_size <= target.app1_size + && !overlaps(source.app0_address, source.app0_size, + source.app1_address, source.app1_size) + && !overlaps(source.app0_address, source.app0_size, + target.app1_address, source.app1_size) + && !overlaps(source.app1_address, source.app1_size, + target.app1_address, source.app1_size), + static_cast(in_app0 ? 0 : 1), + static_cast(in_app0 ? 1 : 0), + }; +} + static_assert(kExpandedLayout.app0_address + kExpandedLayout.app0_size == kExpandedLayout.app1_address, "expanded OTA slots must be adjacent"); diff --git a/test/test_esp32_partition_migration_policy/test_esp32_partition_migration_policy.cpp b/test/test_esp32_partition_migration_policy/test_esp32_partition_migration_policy.cpp index 0b9fecb8..f00a8f19 100644 --- a/test/test_esp32_partition_migration_policy/test_esp32_partition_migration_policy.cpp +++ b/test/test_esp32_partition_migration_policy/test_esp32_partition_migration_policy.cpp @@ -55,6 +55,30 @@ TEST(Esp32PartitionMigrationPolicy, GenericPlansAcceptStableEightAndSixteenMiBLa out_of_bounds)); } +TEST(Esp32PartitionMigrationPolicy, LoRaResumePreservesTheOldApplicationInEitherSlot) { + for (const auto& target : {policy::kExpanded8MBLayout, + policy::kExpandedLayout}) { + const auto from_a = policy::planLoRaResume(policy::kLegacyLayout, target, + policy::kLegacyLayout.app0_address); + EXPECT_TRUE(from_a.valid); + EXPECT_EQ(0, from_a.bridge_slot); + EXPECT_EQ(1, from_a.resume_slot); + + const auto from_b = policy::planLoRaResume(policy::kLegacyLayout, target, + policy::kLegacyLayout.app1_address); + EXPECT_TRUE(from_b.valid); + EXPECT_EQ(1, from_b.bridge_slot); + EXPECT_EQ(0, from_b.resume_slot); + } + + auto overlapping = policy::kExpanded8MBLayout; + overlapping.app1_address = policy::kLegacyLayout.app1_address; + EXPECT_FALSE(policy::planLoRaResume(policy::kLegacyLayout, overlapping, + policy::kLegacyLayout.app0_address).valid); + EXPECT_FALSE(policy::planLoRaResume(policy::kLegacyLayout, + policy::kExpandedLayout, 0x300000).valid); +} + TEST(Esp32PartitionMigrationPolicy, GeneratedTablePrefixContainsAllSixEntriesAndMd5) { EXPECT_EQ(0xE0U, policy::kExpandedPartitionTablePrefixBytes); EXPECT_EQ(0xAA, policy::kExpandedPartitionTablePrefix[0]); diff --git a/test/test_full_infrastructure_selection.sh b/test/test_full_infrastructure_selection.sh index 38558ec6..5b79889c 100644 --- a/test/test_full_infrastructure_selection.sh +++ b/test/test_full_infrastructure_selection.sh @@ -42,6 +42,24 @@ assert_auto_unified Heltec_T190_repeater_ Heltec_T190_repeater_observer_mqtt assert_auto_unified LilyGo_TLora_V2_1_1_6_room_server LilyGo_TLora_V2_1_1_6_room_server_observer_mqtt_ assert_auto_unified heltec_v4_r8_repeater_observer_mqtt heltec_v4_r8_repeater_observer_mqtt +# A one-board migration package must retain the old repeater's mOTA identity, +# even when its Full sources come from the matching observer environment. +( + BUILD_PROFILE_OVERRIDE=full + BUILD_PROFILE_EXPLICIT=1 + SINGLE_TARGET_FULL_BUILD=1 + EXACT_IDENTITY_FULL_BUILD=1 + RESOLVED_BUILD_TARGETS=(heltec_v4_repeater) + configure_effective_build_profile build-firmware >/dev/null + [ "${RESOLVED_BUILD_TARGETS[*]}" = heltec_v4_repeater ] \ + || fail "exact Full target changed identity" + run_logged_build_targets() { + [ "${*}" = heltec_v4_repeater ] || fail "exact Full built a different target" + [ "$ESP32_FULL_BUILD" = 1 ] || fail "exact Full lost expanded partitions" + } + run_command build-firmware >/dev/null +) + # Only targets whose standard and FULL board recipes retain the exact same # partition table may omit their redundant portable bulk artifact. full_only_count=0 @@ -157,7 +175,7 @@ done observed_filename="" observed_output="" build_artifacts_exist() { - observed_filename=$2 + observed_filename=$3 observed_output="$PACKET_LOGGING_OVERRIDE/$MQTT_BRIDGE_OVERRIDE/$MESHDEBUG_OVERRIDE/$MQTT_DEBUG_OVERRIDE" return 0 } @@ -183,7 +201,7 @@ done observed_filename="" observed_full=0 build_artifacts_exist() { - observed_filename=$2 + observed_filename=$3 observed_full=$ESP32_FULL_BUILD return 0 } @@ -203,7 +221,7 @@ done FIRMWARE_FILENAME_INFIX="" observed_filename="" build_artifacts_exist() { - observed_filename=$2 + observed_filename=$3 return 0 } build_firmware heltec_v4_r8_repeater >/dev/null diff --git a/test/test_heltec_v4_partition_migrator.py b/test/test_heltec_v4_partition_migrator.py index bc6795d8..ac88d71e 100644 --- a/test/test_heltec_v4_partition_migrator.py +++ b/test/test_heltec_v4_partition_migrator.py @@ -54,10 +54,16 @@ class HeltecV4PartitionMigratorTest(unittest.TestCase): migration_body = source[source.index("void runMigration()") :] self.assertLess(migration_body.index("stageLegacyIdentity()"), migration_body.index("copyAndVerify(*running")) - self.assertLess(migration_body.index("esp_ota_set_boot_partition(&target_app0)"), + self.assertLess(migration_body.index("esp_ota_set_boot_partition(&target_bridge)"), migration_body.index("publishExpandedPartitionTable(*plan)")) setup_body = source[source.index("void setup()") :] self.assertIn("restoreStagedIdentity()", setup_body) + self.assertIn("resumeLegacyOtaReceiver(*plan)", setup_body) + self.assertLess(setup_body.index("restoreStagedIdentity()"), + setup_body.index("resumeLegacyOtaReceiver(*plan)")) + self.assertIn("validLegacyOtaReceiver(*old_receiver)", migration_body) + self.assertIn("copyAndVerify(*refs.app1", migration_body) + self.assertIn("stageResumeSlot(resume.resume_slot)", migration_body) self.assertIn("AsyncElegantOTA.begin(&server)", source) self.assertIn("MeshCore-Migrate", source) self.assertIn("DRAM_ATTR esp_partition_t copy_destination", source) diff --git a/variants/heltec_v4/platformio.ini b/variants/heltec_v4/platformio.ini index 31a51cfd..46ba414a 100644 --- a/variants/heltec_v4/platformio.ini +++ b/variants/heltec_v4/platformio.ini @@ -894,6 +894,16 @@ lib_deps = ESP32Async/ESPAsyncWebServer @ 3.12.0 file://arch/esp32/AsyncElegantOTA +; LoRa-delivered first stage for an OTA-enabled heltec_v4_repeater. It keeps +; that exact old application bootable after the table change, restores its +; private identity, and returns to it for the expanded-slot LoRa update. +[env:heltec_v4_partition_migrator_lora_repeater] +extends = env:heltec_v4_partition_migrator +build_flags = + ${env:heltec_v4_partition_migrator.build_flags} + -D MESHCORE_MIGRATION_RESUME_OTA=1 + -D MESHCORE_MIGRATION_EXPECTED_TARGET_ID=0xE792A051 + ; Disposable-hardware harness: leaves a ten-minute Wi-Fi OTA window for ; installing the production bridge into legacy slot B. This is not a release ; profile. diff --git a/variants/xiao_s3_wio/platformio.ini b/variants/xiao_s3_wio/platformio.ini index d29c7838..15491ce3 100644 --- a/variants/xiao_s3_wio/platformio.ini +++ b/variants/xiao_s3_wio/platformio.ini @@ -81,6 +81,14 @@ lib_deps = ESP32Async/ESPAsyncWebServer @ 3.12.0 file://arch/esp32/AsyncElegantOTA +; LoRa-delivered first stage for an OTA-enabled Xiao_S3_WIO_repeater. +[env:xiao_s3_partition_migrator_lora_repeater] +extends = env:xiao_s3_partition_migrator +build_flags = + ${env:xiao_s3_partition_migrator.build_flags} + -D MESHCORE_MIGRATION_RESUME_OTA=1 + -D MESHCORE_MIGRATION_EXPECTED_TARGET_ID=0x9F19BCD1 + ; Disposable-hardware seed. It reproduces a real legacy identity file and ; exposes browser OTA without trying to squeeze a current full repeater into ; its historical 1.25 MiB app slot.