From 3226e208bfdcade61f513fe78d5acf477112f528 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Fri, 23 Oct 2026 17:34:57 -0700 Subject: [PATCH] Automatically check embedded bootloader after 70 seconds with reboot protection --- .github/workflows/run-unit-tests.yml | 1 + docs/nrf52_embedded_bootloader.md | 28 +- examples/simple_repeater/MyMesh.h | 6 + examples/simple_room_server/MyMesh.h | 6 + scripts/qualify_nrf52_embedded_bootloader.py | 1 + src/Mesh.cpp | 5 +- src/helpers/ota/BootloaderMigration.cpp | 88 ++++- src/helpers/ota/BootloaderMigration.h | 1 + ...t_bootloader_migration_role_persistence.py | 196 +++++++++++ test/test_nrf52_embedded_variant.py | 1 + .../mota/test_bootloader_migration_runtime.py | 50 +++ tools/mota/tests/migration_stubs/nrf_soc.h | 2 + tools/mota/tests/migration_stubs/runtime.cpp | 330 +++++++++++++++++- 13 files changed, 682 insertions(+), 33 deletions(-) create mode 100644 test/test_bootloader_migration_role_persistence.py diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index 4b9fe7174..a8d9c24a3 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -280,6 +280,7 @@ jobs: python3 -B tools/mota/test_bootloader_migration.py -v python3 -B tools/mota/test_bootloader_migration_runtime.py -v python3 -B test/test_bootloader_update_cli.py -v + python3 -B test/test_bootloader_migration_role_persistence.py -v - name: Verify old-firmware compatibility in both OTA launchers run: python3 -B -m unittest discover -s tools/lora_ota -p test_lora_ota_bootloader_version.py -v diff --git a/docs/nrf52_embedded_bootloader.md b/docs/nrf52_embedded_bootloader.md index 7e1f67671..a547d78c5 100644 --- a/docs/nrf52_embedded_bootloader.md +++ b/docs/nrf52_embedded_bootloader.md @@ -6,9 +6,11 @@ An opt-in Repeater or Room Server build can carry its application firmware and the exact board's [OTAFIX 2.4.12 bootloader](https://github.com/mikecarper/Adafruit_nRF52_Bootloader_OTAFIX/releases/tag/v0.11.0-OTAFIX2.4.12) -in one application image. Install that application first, then activate the -bootloader replacement with an authenticated admin command. Booting the image -does not flash the bootloader automatically. +in one application image. After installing that application, it checks the +bootloader once at 70 seconds after startup and updates it when needed. The +check waits for idle radio and OTA activity, and skips a valid matching +bootloader at the same or newer version. Ordinary firmware has no automatic +bootloader update. This follows the internal-flash bridge in [MeshEnvy firmware PR #2](https://github.com/MeshEnvy/meshcore-firmware/pull/2). @@ -74,8 +76,10 @@ P1 image, not a recovery image. updater. An internal-only legacy receiver needs a delta from its exact installed application; this image does not give a stock bootloader its first LoRa update capability. -2. After the new application is running, use the local CLI or authenticated - remote admin CLI: +2. Leave the new application running for at least 70 seconds. It checks once + when radio and OTA activity are idle, then waits for the usual 1.5-second + grace and persistence checks before replacement and reboot. To start sooner, + use the local CLI or authenticated remote admin CLI: ```text ota bootloader update @@ -92,6 +96,20 @@ P1 image, not a recovery image. `ota bootloader embedded install HASH16` command remains available with the confirmation shown by `status`. +The automatic check runs once per boot. A manual attempt also consumes that +boot's check, so a cancelled or failed attempt needs another explicit command. +An update failure or a retained attempt guard suppresses another automatic +attempt after a warm reboot, preventing a repeated reboot loop; the CLI can +still retry. The +guard is saved before the automatic version/hash check and retained through +the bootloader copy, so an unexpected reset during either step also stops +automatic retries. The guard also remains after a successful replacement; +the next boot stays up and reports its installed version. A completed no-op +clears its temporary guard. This protects +against repeated update attempts; it cannot repair a bootloader that fails +before the application starts. Status shows `auto=70s after boot` while waiting, +`auto=checked` afterward, or `auto=blocked` when the reboot guard prevents it. + The bridge validates the live application, candidate hash, installed identity, chip/layout, MBR addresses, watchdog and flash protection before copying. A protected region is refused; the bridge does not unlock it. Existing staged diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 0bb4aea4f..4e3e6d3eb 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -1323,6 +1323,12 @@ public: bool handleHostCliSerialReply(const char* command, char* reply); #endif void loop(); +#if defined(OTA_BOOTLOADER_MIGRATION) + bool prepareForOtaReboot() override { + // Let the normal ACL save finish, including its failure backoff. + return dirty_contacts_expiry == 0; + } +#endif bool canRecoverUsbLogging() const { return dirty_contacts_expiry == 0 && !hasOutbound() && !isAnyTempRadioActive() && !hasPendingOtaApply() && !saved_radio_apply_pending diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index 08cbf1721..a2722510b 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -654,6 +654,12 @@ public: return mesh::wireless::WiFi; } void loop(); +#if defined(OTA_BOOTLOADER_MIGRATION) + bool prepareForOtaReboot() override { + // Let the normal ACL save finish, including its failure backoff. + return dirty_contacts_expiry == 0; + } +#endif bool canRecoverUsbLogging() const { return dirty_contacts_expiry == 0 && !hasOutbound() && !isAnyTempRadioActive() && !hasPendingOtaApply() && !saved_radio_apply_pending diff --git a/scripts/qualify_nrf52_embedded_bootloader.py b/scripts/qualify_nrf52_embedded_bootloader.py index 216a74a13..76fcce4a4 100644 --- a/scripts/qualify_nrf52_embedded_bootloader.py +++ b/scripts/qualify_nrf52_embedded_bootloader.py @@ -52,6 +52,7 @@ def qualify(directory, manifest_path): "legacy_sha256": metadata.get("legacy_sha256"), "app_offset": offset, "image_bytes": BOOT_SIZE, "activation": "ota bootloader update", + "automatic_delay_seconds": 70, } manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") return manifest["embedded_bootloader"] diff --git a/src/Mesh.cpp b/src/Mesh.cpp index 284c462e3..235fc61da 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -481,13 +481,14 @@ void __attribute__((noinline)) Mesh::serviceLoopMaintenance() { } #if defined(ENABLE_OTA) #if defined(OTA_BOOTLOADER_MIGRATION) - if (ota::bootloader_migration_pending()) { + if (ota::bootloader_migration_service_due()) { // This bridge owns no OTA workspace. Service it even after a dynamic // context has been released, and never allocate one just to poll it. const ota::OtaContext* context = ota::ota_context_if_active(); const bool ota_idle = !context || context->embeddedBootloaderIdle(); const bool outbound_idle = !hasPendingRadioWork(); - const bool ready = outbound_idle && ota_idle && prepareForOtaReboot(); + const bool ready = outbound_idle && ota_idle && + (!ota::bootloader_migration_pending() || prepareForOtaReboot()); ota::bootloader_migration_tick(ready, ota_idle); } #endif diff --git a/src/helpers/ota/BootloaderMigration.cpp b/src/helpers/ota/BootloaderMigration.cpp index aaaa0ee8f..668652972 100644 --- a/src/helpers/ota/BootloaderMigration.cpp +++ b/src/helpers/ota/BootloaderMigration.cpp @@ -24,6 +24,10 @@ static_assert(migration::ACL_READ_DENIED == ACL_ACL_PERM_READ_Msk, "ACL read per static_assert(migration::ACL_WRITE_DENIED == ACL_ACL_PERM_WRITE_Msk, "ACL write permission bit"); namespace { bool pending = false; +bool automatic_attempted = false; +bool automatic_blocked = false; +constexpr uint32_t AUTOMATIC_DELAY_MS = 70000u; +constexpr uint8_t ATTEMPT_MARKER = 0xD0; uint32_t armed_at = 0; uint8_t last_result = 0; bool result_is_current = false; @@ -77,19 +81,40 @@ bool installed_up_to_date() { migration_softdevice_fwid); } -uint8_t retained_result() { - if (result_is_current) return last_result; +bool read_reboot_marker(uint32_t &result) { uint8_t enabled = 0; - uint32_t result = 0; - if (sd_softdevice_is_enabled(&enabled) != NRF_SUCCESS) return 0; + if (sd_softdevice_is_enabled(&enabled) != NRF_SUCCESS) return false; if (enabled) { - if (sd_power_gpregret_get(0, &result) != NRF_SUCCESS) return 0; + if (sd_power_gpregret_get(0, &result) != NRF_SUCCESS) return false; } else { result = NRF_POWER->GPREGRET; } + return true; +} + +uint8_t retained_result() { + if (result_is_current) return last_result; + uint32_t result = 0; + if (!read_reboot_marker(result)) return 0; return result >= 0xC0 && result <= 0xCF ? result & 15 : last_result; } +bool write_reboot_marker(uint8_t marker) { + uint8_t enabled = 0; + if (sd_softdevice_is_enabled(&enabled) != NRF_SUCCESS) return false; + if (enabled) { + uint32_t current = 0; + if (sd_power_gpregret_get(0, ¤t) != NRF_SUCCESS) return false; + // Arming after the automatic image check must preserve its existing + // guard, rather than creating an unguarded gap between clear and set. + if (current == marker) return true; + if (sd_power_gpregret_clr(0, 0xFF) != NRF_SUCCESS) return false; + return marker == 0 || sd_power_gpregret_set(0, marker) == NRF_SUCCESS; + } + NRF_POWER->GPREGRET = marker; + return NRF_POWER->GPREGRET == marker; +} + // Called only after the operator reply has drained and SoftDevice is stopped. // GPREGRET (not GPREGRET2) survives this legacy loader's diagnostic writes. void fail_and_reset(uint8_t result) { @@ -174,7 +199,10 @@ void install() { if (NRF_WDT->RREN & (1u << i)) NRF_WDT->RR[i] = 0x6E524635u; } } - NRF_POWER->GPREGRET = 0; + // The installed loader preserves our non-DFU marker. If COPY_BL or its + // preflight is interrupted, the next application boot will stay available + // instead of automatically trying the same operation again. + NRF_POWER->GPREGRET = ATTEMPT_MARKER; NRF_POWER->GPREGRET2 = 0; sd_mbr_command_t command = {}; command.command = SD_MBR_COMMAND_COPY_BL; @@ -191,12 +219,17 @@ bool bootloader_migration_pending() { return pending; } +bool bootloader_migration_service_due() { + return pending || (!automatic_attempted && millis() >= AUTOMATIC_DELAY_MS); +} + void bootloader_migration_status(char *reply, size_t capacity) { char hash[17]; hex8(migration_candidate_sha256, hash); - snprintf(reply, capacity, "BL embedded 2.4.12: 40960 B; hash=%s; %s; last=%u", hash, + snprintf(reply, capacity, "BL embedded 2.4.12: 40960 B; hash=%s; %s; last=%u; auto=%s", hash, pending ? "armed" : already_installed ? "already installed" : - installed_up_to_date() ? "up to date" : "idle", retained_result()); + installed_up_to_date() ? "up to date" : "idle", retained_result(), + automatic_blocked ? "blocked" : automatic_attempted ? "checked" : "70s after boot"); } bool bootloader_migration_request(const char *confirmation, char *reply, size_t capacity) { @@ -206,6 +239,10 @@ bool bootloader_migration_request(const char *confirmation, char *reply, size_t snprintf(reply, capacity, "ERR confirm exact image: ota bootloader embedded install %s", hash); return false; } + // A manual attempt also satisfies the startup check, including a cancelled + // or failed attempt. Only another explicit command may retry it this boot. + automatic_attempted = true; + automatic_blocked = false; if (pending) { snprintf(reply, capacity, "OK bootloader update already armed"); return true; @@ -229,6 +266,12 @@ bool bootloader_migration_request(const char *confirmation, char *reply, size_t return true; } already_installed = false; + if (!write_reboot_marker(ATTEMPT_MARKER)) { + last_result = 17; + result_is_current = true; + snprintf(reply, capacity, "ERR bootloader reboot guard unavailable"); + return false; + } pending = true; armed_at = millis(); last_result = 0; @@ -258,7 +301,34 @@ void bootloader_migration_audit(char *reply, size_t capacity) { } void bootloader_migration_tick(bool outbound_idle, bool ota_idle) { - if (!pending) return; + if (!pending) { + if (!bootloader_migration_service_due() || !outbound_idle || !ota_idle) return; + // A failed MBR/preflight reset leaves this marker. Do not create a loop + // that retries and resets every 70 seconds; the CLI can retry explicitly. + uint32_t marker = 0; + if (!read_reboot_marker(marker) || marker == ATTEMPT_MARKER || + (marker > 0xC0 && marker <= 0xCF)) { + automatic_attempted = true; + automatic_blocked = true; + return; + } + automatic_attempted = true; + // Mark before reading either image too, so a reset in the version/hash + // check cannot turn into a recurring 70-second reboot loop. + if (!write_reboot_marker(ATTEMPT_MARKER)) { + last_result = 17; + result_is_current = true; + automatic_blocked = true; + return; + } + char reply[160]; + bootloader_migration_request(nullptr, reply, sizeof(reply)); + if (!pending && last_result == 0 && !write_reboot_marker(0)) { + last_result = 17; + result_is_current = true; + } + return; + } const uint32_t elapsed = millis() - armed_at; // An incoming advert may have started autofetch since the admin command. // Never disable SoftDevice while another OTA session owns the flash store. diff --git a/src/helpers/ota/BootloaderMigration.h b/src/helpers/ota/BootloaderMigration.h index e0d8bfe8b..84453d161 100644 --- a/src/helpers/ota/BootloaderMigration.h +++ b/src/helpers/ota/BootloaderMigration.h @@ -6,6 +6,7 @@ namespace mesh { namespace ota { bool bootloader_migration_pending(); +bool bootloader_migration_service_due(); // nullptr selects the pinned embedded image; a supplied hash must match exactly. bool bootloader_migration_request(const char *confirmation, char *reply, size_t capacity); void bootloader_migration_status(char *reply, size_t capacity); diff --git a/test/test_bootloader_migration_role_persistence.py b/test/test_bootloader_migration_role_persistence.py new file mode 100644 index 000000000..e39ea056c --- /dev/null +++ b/test/test_bootloader_migration_role_persistence.py @@ -0,0 +1,196 @@ +"""Exercise embedded migration gates against the roles' normal ACL saves.""" + +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + +from cpp_source import body + + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = r''' +#include +#include +#include +#include "helpers/LazyPersistence.h" +#define LAZY_CONTACTS_WRITE_DELAY 5000 +#define MESH_DEBUG_PRINTLN(...) ((void)0) +static uint32_t now = 70000; +static bool millisHasNowPassed(unsigned long deadline) { + return int32_t(now - uint32_t(deadline)) > 0; +} +static unsigned long futureMillis(uint32_t delay) { return uint32_t(now + delay); } +struct Base { virtual bool prepareForOtaReboot() { return true; } }; +struct Role : Base { + using MyMesh = Role; + unsigned long dirty_contacts_expiry = 75000; + uint8_t contacts_save_failures = 0; + static bool saveFilter(int*) { return true; } + void* _fs = nullptr; + struct Acl { + bool succeeds = false; + unsigned saves = 0; + bool save(void*, bool (*)(int*) = nullptr) { saves++; return succeeds; } + } acl; + @GATE@ + void servicePersistence() { + @SAVE@ + } +}; +int main() { + Role role; +#ifdef OTA_BOOTLOADER_MIGRATION + static_assert(std::is_same::value, + "embedded role must override the reboot gate"); + assert(!role.prepareForOtaReboot()); + role.servicePersistence(); + assert(role.acl.saves == 0 && !role.prepareForOtaReboot()); + now = 75001; + role.servicePersistence(); + assert(role.acl.saves == 1 && role.contacts_save_failures == 1); + assert(!role.prepareForOtaReboot()); + // Repeated migration polls must not bypass the failed save's backoff. + now = 80000; + for (unsigned poll = 0; poll < 100; poll++) { + assert(!role.prepareForOtaReboot()); + role.servicePersistence(); + } + assert(role.acl.saves == 1); + role.acl.succeeds = true; + now = 80002; + role.servicePersistence(); + assert(role.acl.saves == 2 && role.contacts_save_failures == 0); + assert(role.dirty_contacts_expiry == 0 && role.prepareForOtaReboot()); +#else + // Ordinary firmware retains the base implementation without an override. + assert(role.prepareForOtaReboot()); + static_assert(std::is_same::value, + "ordinary role must retain its base reboot gate"); +#endif +} +''' + +MESH_SOURCE = r''' +#include +namespace ota { +static bool due = false, pending = false; +static unsigned context_polls = 0, ticks = 0; +static bool last_ready = false, last_idle = false; +struct OtaContext { + bool idle = true; + bool embeddedBootloaderIdle() const { return idle; } +}; +static OtaContext* active = nullptr; +static bool bootloader_migration_service_due() { return due; } +static bool bootloader_migration_pending() { return pending; } +static const OtaContext* ota_context_if_active() { context_polls++; return active; } +static void bootloader_migration_tick(bool ready, bool idle) { + ticks++; last_ready = ready; last_idle = idle; +} +} +struct Mesh { + bool radio_busy = false, persistence_ready = false; + unsigned persistence_checks = 0; + bool hasPendingRadioWork() const { return radio_busy; } + bool prepareForOtaReboot() { persistence_checks++; return persistence_ready; } + void service() { @SERVICE@ } +}; +int main() { + Mesh mesh; + mesh.service(); + assert(ota::ticks == 0 && ota::context_polls == 0); + ota::due = true; + mesh.service(); + // Startup checks work with a released workspace and no persistence flush. + assert(ota::ticks == 1 && ota::context_polls == 1); + assert(ota::last_ready && ota::last_idle && mesh.persistence_checks == 0); + ota::pending = true; + mesh.service(); + assert(!ota::last_ready && ota::last_idle && mesh.persistence_checks == 1); + mesh.persistence_ready = true; + mesh.service(); + assert(ota::last_ready && ota::last_idle && mesh.persistence_checks == 2); + mesh.radio_busy = true; + mesh.service(); + assert(!ota::last_ready && ota::last_idle && mesh.persistence_checks == 2); + mesh.radio_busy = false; + ota::OtaContext context; + context.idle = false; + ota::active = &context; + mesh.service(); + assert(!ota::last_ready && !ota::last_idle && mesh.persistence_checks == 2); + ota::pending = false; + context.idle = true; + mesh.service(); + assert(ota::last_ready && ota::last_idle && mesh.persistence_checks == 2); +} +''' + + +class BootloaderMigrationRolePersistenceTest(unittest.TestCase): + def test_mesh_services_startup_checks_without_allocating_an_ota_workspace(self): + source = (ROOT / "src/Mesh.cpp").read_text(encoding="utf-8") + maintenance = body(source, "void __attribute__((noinline)) Mesh::serviceLoopMaintenance()") + service = body(maintenance, "if (ota::bootloader_migration_service_due())") + dynamic_context = maintenance.index("#if OTA_DYNAMIC_CONTEXT") + self.assertLess(maintenance.index(service) + len(service), dynamic_context) + released = body(maintenance[dynamic_context:], "if (!ota::ota_context_if_active())") + self.assertIn("return;", released) + # Allocating ota_ctx() merely to poll would defeat dynamic-context RAM savings. + self.assertNotRegex(service, r"\bota_ctx\s*\(") + compiler = shutil.which("g++") or shutil.which("clang++") + if compiler is None: + self.skipTest("a host C++17 compiler is required") + sanitize = [] if os.name == "nt" else [ + "-fsanitize=address,undefined", "-fno-sanitize-recover=all", "-fno-pie", "-no-pie"] + with tempfile.TemporaryDirectory(prefix="migration-mesh-service-") as directory: + unit = Path(directory) / "service.cpp" + binary = Path(directory) / ("service.exe" if os.name == "nt" else "service") + unit.write_text(MESH_SOURCE.replace("@SERVICE@", service), encoding="ascii") + built = subprocess.run([ + compiler, "-std=c++17", "-O1", "-g", "-Wall", "-Wextra", "-Werror", + *sanitize, str(unit), "-o", str(binary), + ], capture_output=True, text=True, timeout=60) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + tested = subprocess.run([str(binary)], capture_output=True, text=True, timeout=30) + self.assertEqual(tested.returncode, 0, tested.stdout + tested.stderr) + + def test_repeater_and_room_wait_for_the_normal_acl_save(self): + compiler = shutil.which("g++") or shutil.which("clang++") + if compiler is None: + self.skipTest("a host C++17 compiler is required") + sanitize = [] if os.name == "nt" else [ + "-fsanitize=address,undefined", "-fno-sanitize-recover=all", "-fno-pie", "-no-pie"] + with tempfile.TemporaryDirectory(prefix="migration-role-persistence-") as directory: + for role in ("simple_repeater", "simple_room_server"): + header = (ROOT / "examples" / role / "MyMesh.h").read_text(encoding="utf-8") + source = (ROOT / "examples" / role / "MyMesh.cpp").read_text(encoding="utf-8") + gate = re.search( + r"#if defined\(OTA_BOOTLOADER_MIGRATION\)\n" + r"\s*bool prepareForOtaReboot\(\) override \{.*?\n\s*\}\n#endif", + header, re.DOTALL) + self.assertIsNotNone(gate, role) + save = body(source, "if (dirty_contacts_expiry && millisHasNowPassed(dirty_contacts_expiry))") + generated = SOURCE.replace("@GATE@", gate.group()).replace("@SAVE@", save) + for embedded in (False, True): + with self.subTest(role=role, embedded=embedded): + unit = Path(directory) / (role + ".cpp") + binary = Path(directory) / (role + (".exe" if os.name == "nt" else "")) + unit.write_text(generated, encoding="ascii") + built = subprocess.run([ + compiler, "-std=c++17", "-O1", "-g", "-Wall", "-Wextra", "-Werror", + "-Wno-unused-function", *sanitize, + *(["-DOTA_BOOTLOADER_MIGRATION=1"] if embedded else []), + "-I", str(ROOT / "src"), str(unit), "-o", str(binary), + ], capture_output=True, text=True, timeout=60) + self.assertEqual(built.returncode, 0, built.stdout + built.stderr) + tested = subprocess.run([str(binary)], capture_output=True, text=True, timeout=30) + self.assertEqual(tested.returncode, 0, tested.stdout + tested.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_nrf52_embedded_variant.py b/test/test_nrf52_embedded_variant.py index 9451a7aa3..535469ae4 100644 --- a/test/test_nrf52_embedded_variant.py +++ b/test/test_nrf52_embedded_variant.py @@ -180,6 +180,7 @@ class PackagedApplicationTests(unittest.TestCase): self.assertEqual(result["profile"], self.profile["id"]) self.assertEqual(result["candidate_sha256"], hashlib.sha256(self.candidate).hexdigest()) self.assertEqual(result["activation"], "ota bootloader update") + self.assertEqual(result["automatic_delay_seconds"], 70) self.patches["select_profile"].assert_called_once_with( {"profiles": [self.profile]}, TARGET) self.patches["validate_candidate_zip"].assert_called_once_with( diff --git a/tools/mota/test_bootloader_migration_runtime.py b/tools/mota/test_bootloader_migration_runtime.py index df0ebaf89..f911d2a4a 100644 --- a/tools/mota/test_bootloader_migration_runtime.py +++ b/tools/mota/test_bootloader_migration_runtime.py @@ -40,6 +40,56 @@ class MigrationRuntimeTests(unittest.TestCase): def run_scenario(self, scenario): subprocess.run([str(self.binary), scenario], check=True) + def test_automatic_update_waits_70_seconds_then_observes_install_grace(self): + for scenario in ('automatic-delay', 'automatic-arm-enabled'): + with self.subTest(scenario=scenario): + self.run_scenario(scenario) + + def test_automatic_update_defers_until_radio_and_ota_are_idle(self): + self.run_scenario('automatic-busy') + + def test_automatic_update_skips_same_or_newer_version_and_exact_image(self): + for scenario in ('automatic-same-version', 'automatic-same-version-enabled', + 'automatic-newer-version', + 'automatic-exact-image'): + with self.subTest(scenario=scenario): + self.run_scenario(scenario) + + def test_retained_failure_prevents_automatic_retry_but_allows_manual_retry(self): + self.run_scenario('automatic-retained-failure') + + def test_manual_attempt_prevents_a_second_automatic_attempt(self): + for scenario in ('manual-cancel-consumes-automatic', + 'manual-failure-consumes-automatic'): + with self.subTest(scenario=scenario): + self.run_scenario(scenario) + + def test_cancelled_automatic_update_is_not_rearmed(self): + self.run_scenario('automatic-cancel') + + def test_invalid_confirmation_does_not_cancel_the_automatic_update(self): + self.run_scenario('invalid-confirmation-preserves-automatic') + + def test_reboot_guard_is_written_through_softdevice_before_arming(self): + self.run_scenario('guard-softdevice-enabled') + + def test_reboot_guard_read_errors_block_the_automatic_attempt(self): + for scenario in ('automatic-status-read-failure', 'automatic-marker-read-failure'): + with self.subTest(scenario=scenario): + self.run_scenario(scenario) + + def test_reboot_guard_api_failures_refuse_to_arm(self): + for scenario in ('guard-softdevice-status-failure', 'guard-marker-read-failure', + 'guard-clear-failure', + 'guard-set-failure'): + with self.subTest(scenario=scenario): + self.run_scenario(scenario) + + def test_reset_during_check_or_armed_attempt_cannot_create_an_update_loop(self): + for scenario in ('automatic-interrupted-check', 'automatic-interrupted-arm'): + with self.subTest(scenario=scenario): + self.run_scenario(scenario) + def test_wrong_confirmation_cannot_arm(self): self.run_scenario('wrong-confirmation') diff --git a/tools/mota/tests/migration_stubs/nrf_soc.h b/tools/mota/tests/migration_stubs/nrf_soc.h index d484aee6e..18db044f6 100644 --- a/tools/mota/tests/migration_stubs/nrf_soc.h +++ b/tools/mota/tests/migration_stubs/nrf_soc.h @@ -2,3 +2,5 @@ #include uint32_t sd_power_gpregret_get(uint32_t index, uint32_t *result); +uint32_t sd_power_gpregret_clr(uint32_t index, uint32_t mask); +uint32_t sd_power_gpregret_set(uint32_t index, uint32_t mask); diff --git a/tools/mota/tests/migration_stubs/runtime.cpp b/tools/mota/tests/migration_stubs/runtime.cpp index a5eef1dc2..762d91466 100644 --- a/tools/mota/tests/migration_stubs/runtime.cpp +++ b/tools/mota/tests/migration_stubs/runtime.cpp @@ -30,7 +30,16 @@ unsigned softdevice_checks = 0; unsigned disables = 0; unsigned resets = 0; unsigned copies = 0; +unsigned retained_reads = 0; +unsigned retained_clears = 0; +unsigned retained_sets = 0; bool candidate_installed = false; +bool interrupt_hash = false; +bool softdevice_enabled = false; +uint32_t softdevice_status = NRF_SUCCESS; +uint32_t retained_get_status = NRF_SUCCESS; +uint32_t retained_clear_status = NRF_SUCCESS; +uint32_t retained_set_status = NRF_SUCCESS; void mbr_command() { ++copies; @@ -83,8 +92,8 @@ void NVIC_SystemReset() { uint32_t sd_softdevice_is_enabled(uint8_t *enabled) { ++migration_test::softdevice_checks; - *enabled = 0; - return NRF_SUCCESS; + *enabled = migration_test::softdevice_enabled; + return migration_test::softdevice_status; } uint32_t sd_softdevice_disable() { @@ -92,12 +101,34 @@ uint32_t sd_softdevice_disable() { throw std::runtime_error("unexpected SoftDevice disable in state-machine test"); } -uint32_t sd_power_gpregret_get(uint32_t, uint32_t *result) { - *result = 0; - return NRF_SUCCESS; +uint32_t sd_power_gpregret_get(uint32_t index, uint32_t *result) { + assert(index == 0); + ++migration_test::retained_reads; + *result = power_registers.GPREGRET; + return migration_test::retained_get_status; +} + +uint32_t sd_power_gpregret_clr(uint32_t index, uint32_t mask) { + assert(index == 0 && mask == 0xFF); + assert(!mesh::ota::bootloader_migration_pending()); + ++migration_test::retained_clears; + if (migration_test::retained_clear_status == NRF_SUCCESS) + power_registers.GPREGRET &= ~mask; + return migration_test::retained_clear_status; +} + +uint32_t sd_power_gpregret_set(uint32_t index, uint32_t mask) { + assert(index == 0 && mask == mesh::ota::ATTEMPT_MARKER); + assert(!mesh::ota::bootloader_migration_pending()); + ++migration_test::retained_sets; + if (migration_test::retained_set_status == NRF_SUCCESS) + power_registers.GPREGRET |= mask; + return migration_test::retained_set_status; } void SHA256::reset() { + if (migration_test::interrupt_hash) + throw std::runtime_error("interrupted bootloader check"); } void SHA256::update(const void *, size_t) { } @@ -170,11 +201,12 @@ void arm() { assert(bootloader_migration_request(confirmation, reply, sizeof(reply))); assert(bootloader_migration_pending()); assert(std::strstr(reply, "OK migration armed")); + assert(power_registers.GPREGRET == mesh::ota::ATTEMPT_MARKER); } -void no_install() { +void no_install(unsigned softdevice_reads = 0) { assert(migration_test::endf_checks == 0); - assert(migration_test::softdevice_checks == 0); + assert(migration_test::softdevice_checks == softdevice_reads); assert(migration_test::disables == 0); assert(migration_test::resets == 0); assert(migration_test::copies == 0); @@ -186,6 +218,19 @@ void status_result(unsigned result) { std::snprintf(expected, sizeof(expected), "last=%u", result); assert(std::strstr(reply, expected)); } + +void restart_application() { + // Recreate zero-initialized driver state while retaining GPREGRET, as a + // hardware reset would. Test counters deliberately survive for assertions. + mesh::ota::pending = false; + mesh::ota::automatic_attempted = false; + mesh::ota::automatic_blocked = false; + mesh::ota::armed_at = 0; + mesh::ota::last_result = 0; + mesh::ota::result_is_current = false; + mesh::ota::already_installed = false; + migration_test::now = 0; +} } // namespace int main(int argc, char **argv) { @@ -199,7 +244,255 @@ int main(int argc, char **argv) { MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0) != MAP_FAILED); #endif const char *scenario = argv[1]; - if (std::strcmp(scenario, "already-installed") == 0) { + if (std::strcmp(scenario, "automatic-delay") == 0 || + std::strcmp(scenario, "automatic-arm-enabled") == 0) { + softdevice_enabled = std::strcmp(scenario, "automatic-arm-enabled") == 0; + assert(!bootloader_migration_service_due()); + now = 69999; + assert(!bootloader_migration_service_due()); + bootloader_migration_tick(true, true); + assert(!bootloader_migration_pending()); + no_install(); + now = 70000; + assert(bootloader_migration_service_due()); + bootloader_migration_tick(true, true); + assert(bootloader_migration_pending()); + assert(bootloader_migration_service_due()); + no_install(3); + assert(power_registers.GPREGRET == mesh::ota::ATTEMPT_MARKER); + assert(retained_reads == (softdevice_enabled ? 3u : 0u)); + assert(retained_clears == (softdevice_enabled ? 1u : 0u)); + assert(retained_sets == (softdevice_enabled ? 1u : 0u)); + now = 71499; + bootloader_migration_tick(true, true); + assert(bootloader_migration_pending()); + no_install(3); + now = 71500; + bootloader_migration_tick(true, true); + assert(endf_checks == 1); + assert(!bootloader_migration_pending()); + assert(!bootloader_migration_service_due()); + now = 140000; + bootloader_migration_tick(true, true); + assert(endf_checks == 1); // A failed preflight must not be retried automatically. + assert(softdevice_checks == 3); + status_result(1); + } else if (std::strcmp(scenario, "automatic-busy") == 0) { + now = 70000; + bootloader_migration_tick(false, true); + assert(!bootloader_migration_pending()); + assert(bootloader_migration_service_due()); + no_install(); + now = 72000; + bootloader_migration_tick(true, false); + assert(!bootloader_migration_pending()); + assert(bootloader_migration_service_due()); + no_install(); + now = 74000; + bootloader_migration_tick(false, false); + assert(!bootloader_migration_pending()); + assert(bootloader_migration_service_due()); + no_install(); + now = 76000; + bootloader_migration_tick(true, true); + assert(bootloader_migration_pending()); + no_install(3); + now = 77499; + bootloader_migration_tick(true, true); + no_install(3); + now = 77500; + bootloader_migration_tick(true, true); + assert(endf_checks == 1); // Grace starts when the deferred request is armed. + } else if (std::strcmp(scenario, "automatic-same-version") == 0 || + std::strcmp(scenario, "automatic-same-version-enabled") == 0 || + std::strcmp(scenario, "automatic-newer-version") == 0 || + std::strcmp(scenario, "automatic-exact-image") == 0) { + softdevice_enabled = std::strcmp(scenario, "automatic-same-version-enabled") == 0; + if (std::strcmp(scenario, "automatic-exact-image") == 0) { + candidate_installed = true; + } else { + seal_manifest(installed_manifest( + (std::strcmp(scenario, "automatic-same-version") == 0 || softdevice_enabled) + ? migration_boot_version : 0x02040D01)); + } + now = 70000; + bootloader_migration_tick(true, true); + assert(!bootloader_migration_pending()); + assert(!bootloader_migration_service_due()); + no_install(3); + assert(power_registers.GPREGRET == 0); // Clear the provisional check marker. + assert(retained_reads == (softdevice_enabled ? 3u : 0u)); + assert(retained_clears == (softdevice_enabled ? 2u : 0u)); + assert(retained_sets == (softdevice_enabled ? 1u : 0u)); + now = 140000; + bootloader_migration_tick(true, true); + assert(!bootloader_migration_pending()); + assert(!bootloader_migration_service_due()); + no_install(3); + bootloader_migration_status(reply, sizeof(reply)); + assert(std::strstr(reply, candidate_installed ? "already installed" : "up to date")); + } else if (std::strcmp(scenario, "automatic-retained-failure") == 0) { + power_registers.GPREGRET = 0xCC; + now = 70000; + bootloader_migration_tick(true, true); + assert(!bootloader_migration_pending()); + assert(!bootloader_migration_service_due()); + no_install(1); + now = 140000; + bootloader_migration_tick(true, true); + no_install(1); // Retained failures must not create a warm-reset retry loop. + bootloader_migration_status(reply, sizeof(reply)); + assert(std::strstr(reply, "auto=blocked")); + assert(bootloader_migration_request(nullptr, reply, sizeof(reply))); + assert(bootloader_migration_pending()); + assert(std::strstr(reply, "OK migration armed")); + now = 141500; + bootloader_migration_tick(true, true); + assert(endf_checks == 1); // An explicit operator request may still retry. + assert(!bootloader_migration_pending()); + } else if (std::strcmp(scenario, "manual-cancel-consumes-automatic") == 0 || + std::strcmp(scenario, "manual-failure-consumes-automatic") == 0) { + arm(); + now = 1500; + const bool cancel = std::strcmp(scenario, "manual-cancel-consumes-automatic") == 0; + bootloader_migration_tick(true, !cancel); + assert(!bootloader_migration_pending()); + assert(endf_checks == (cancel ? 0u : 1u)); + now = 70000; + assert(!bootloader_migration_service_due()); + bootloader_migration_tick(true, true); + assert(!bootloader_migration_pending()); + assert(endf_checks == (cancel ? 0u : 1u)); + assert(softdevice_checks == 1); + } else if (std::strcmp(scenario, "automatic-cancel") == 0) { + now = 70000; + bootloader_migration_tick(true, true); + assert(bootloader_migration_pending()); + now = 70100; + bootloader_migration_tick(true, false); + assert(!bootloader_migration_pending()); + assert(!bootloader_migration_service_due()); + no_install(3); + now = 140000; + bootloader_migration_tick(true, true); + assert(!bootloader_migration_pending()); + assert(!bootloader_migration_service_due()); + no_install(3); + status_result(16); + } else if (std::strcmp(scenario, "invalid-confirmation-preserves-automatic") == 0) { + now = 1000; + assert(!bootloader_migration_request("0123456789ABCDE0", reply, sizeof(reply))); + assert(!bootloader_migration_pending()); + now = 70000; + assert(bootloader_migration_service_due()); + bootloader_migration_tick(true, true); + assert(bootloader_migration_pending()); + no_install(3); + now = 71500; + bootloader_migration_tick(true, true); + assert(endf_checks == 1); + assert(!bootloader_migration_pending()); + assert(!bootloader_migration_service_due()); + } else if (std::strcmp(scenario, "automatic-status-read-failure") == 0 || + std::strcmp(scenario, "automatic-marker-read-failure") == 0) { + power_registers.GPREGRET = mesh::ota::ATTEMPT_MARKER; + if (std::strcmp(scenario, "automatic-status-read-failure") == 0) { + softdevice_status = 1; + } else { + softdevice_enabled = true; + retained_get_status = 1; + } + now = 70000; + bootloader_migration_tick(true, true); + assert(!bootloader_migration_pending()); + assert(!bootloader_migration_service_due()); + no_install(1); + assert(retained_reads == (softdevice_enabled ? 1u : 0u)); + assert(retained_clears == 0 && retained_sets == 0); + assert(power_registers.GPREGRET == mesh::ota::ATTEMPT_MARKER); + bootloader_migration_status(reply, sizeof(reply)); + assert(std::strstr(reply, "auto=blocked")); + } else if (std::strcmp(scenario, "guard-softdevice-enabled") == 0) { + softdevice_enabled = true; + power_registers.GPREGRET = 0xAA; + arm(); + assert(retained_clears == 1 && retained_sets == 1); + no_install(1); + assert(bootloader_migration_request(nullptr, reply, sizeof(reply))); + assert(retained_clears == 1 && retained_sets == 1); // Duplicates do no work. + now = 1500; + bootloader_migration_tick(true, true); + assert(endf_checks == 1); + assert(power_registers.GPREGRET == mesh::ota::ATTEMPT_MARKER); + } else if (std::strcmp(scenario, "guard-softdevice-status-failure") == 0 || + std::strcmp(scenario, "guard-marker-read-failure") == 0 || + std::strcmp(scenario, "guard-clear-failure") == 0 || + std::strcmp(scenario, "guard-set-failure") == 0) { + power_registers.GPREGRET = 0x37; + if (std::strcmp(scenario, "guard-softdevice-status-failure") == 0) { + softdevice_status = 1; + } else { + softdevice_enabled = true; + if (std::strcmp(scenario, "guard-marker-read-failure") == 0) + retained_get_status = 1; + else if (std::strcmp(scenario, "guard-clear-failure") == 0) + retained_clear_status = 1; + else + retained_set_status = 1; + } + assert(!bootloader_migration_request(nullptr, reply, sizeof(reply))); + assert(std::strstr(reply, "ERR bootloader reboot guard unavailable")); + assert(!bootloader_migration_pending()); + no_install(1); + assert(retained_clears == (softdevice_enabled && !retained_get_status ? 1u : 0u)); + assert(retained_sets == (retained_set_status ? 1u : 0u)); + assert(power_registers.GPREGRET == (retained_set_status ? 0u : 0x37u)); + now = 70000; + assert(!bootloader_migration_service_due()); + bootloader_migration_tick(true, true); + no_install(1); + status_result(17); + } else if (std::strcmp(scenario, "automatic-interrupted-check") == 0 || + std::strcmp(scenario, "automatic-interrupted-arm") == 0) { + const bool during_check = std::strcmp(scenario, "automatic-interrupted-check") == 0; + now = 70000; + interrupt_hash = during_check; + bool interrupted = false; + try { + bootloader_migration_tick(true, true); + } catch (const std::runtime_error &error) { + assert(during_check); + assert(std::strcmp(error.what(), "interrupted bootloader check") == 0); + interrupted = true; + } + assert(interrupted == during_check); + assert(bootloader_migration_pending() == !during_check); + assert(power_registers.GPREGRET == mesh::ota::ATTEMPT_MARKER); + assert(endf_checks == 0); + const unsigned initial_reads = softdevice_checks; + restart_application(); + interrupt_hash = false; + now = 69999; + assert(!bootloader_migration_service_due()); + now = 70000; + assert(bootloader_migration_service_due()); + bootloader_migration_tick(true, true); + assert(!bootloader_migration_pending()); + assert(!bootloader_migration_service_due()); + assert(power_registers.GPREGRET == mesh::ota::ATTEMPT_MARKER); + no_install(initial_reads + 1); + now = 140000; + bootloader_migration_tick(true, true); + no_install(initial_reads + 1); // A second reset-time check must not repeat. + bootloader_migration_status(reply, sizeof(reply)); + assert(std::strstr(reply, "auto=blocked")); + arm(); // An operator can retry after correcting an interruption's cause. + now = 141500; + bootloader_migration_tick(true, true); + assert(endf_checks == 1); + assert(!bootloader_migration_pending()); + } else if (std::strcmp(scenario, "already-installed") == 0) { + power_registers.GPREGRET = 0x37; candidate_installed = true; assert(bootloader_migration_request(confirmation, reply, sizeof(reply))); assert(!bootloader_migration_pending()); @@ -207,10 +500,12 @@ int main(int argc, char **argv) { now = 2000; bootloader_migration_tick(true, true); no_install(); + assert(power_registers.GPREGRET == 0x37); bootloader_migration_status(reply, sizeof(reply)); assert(std::strstr(reply, "already installed")); } else if (std::strcmp(scenario, "same-version") == 0 || std::strcmp(scenario, "newer-version") == 0) { + power_registers.GPREGRET = 0x37; const uint32_t version = std::strcmp(scenario, "same-version") == 0 ? migration_boot_version : 0x02040D01; seal_manifest(installed_manifest(version)); @@ -229,6 +524,7 @@ int main(int argc, char **argv) { now += 2000; bootloader_migration_tick(true, true); no_install(); + assert(power_registers.GPREGRET == 0x37); } } else if (std::strcmp(scenario, "easy-update") == 0 || std::strcmp(scenario, "older-preview") == 0) { @@ -239,7 +535,7 @@ int main(int argc, char **argv) { assert(std::strstr(reply, "OK migration armed")); now = 1499; bootloader_migration_tick(true, true); - no_install(); + no_install(1); ++now; bootloader_migration_tick(true, true); assert(endf_checks == 1); @@ -279,13 +575,13 @@ int main(int argc, char **argv) { arm(); now = 1499; bootloader_migration_tick(true, true); - no_install(); + no_install(1); now = 1500; bootloader_migration_tick(false, true); - no_install(); + no_install(1); now = 29999; bootloader_migration_tick(false, true); - no_install(); + no_install(1); assert(bootloader_migration_pending()); bootloader_migration_tick(true, true); assert(endf_checks == 1); @@ -309,7 +605,7 @@ int main(int argc, char **argv) { now = 30000; bootloader_migration_tick(true, true); assert(!bootloader_migration_pending()); - no_install(); + no_install(1); status_result(15); bootloader_migration_tick(true, true); assert(endf_checks == 0); @@ -318,7 +614,7 @@ int main(int argc, char **argv) { now = 100; bootloader_migration_tick(false, false); assert(!bootloader_migration_pending()); - no_install(); + no_install(1); status_result(16); now = 2000; bootloader_migration_tick(true, true); @@ -335,7 +631,7 @@ int main(int argc, char **argv) { now = 1500; bootloader_migration_tick(true, false); assert(!bootloader_migration_pending()); - no_install(); + no_install(1); status_result(16); } else if (std::strcmp(scenario, "stale-retained-result") == 0) { power_registers.GPREGRET = 0xCC; @@ -352,14 +648,14 @@ int main(int argc, char **argv) { bootloader_migration_tick(true, false); assert(!bootloader_migration_pending()); status_result(16); - assert(power_registers.GPREGRET == 0xCC); + assert(power_registers.GPREGRET == mesh::ota::ATTEMPT_MARKER); assert(endf_checks == 0); } else if (std::strcmp(scenario, "clock-wrap") == 0) { now = UINT32_MAX - 999u; arm(); now += 1499u; bootloader_migration_tick(true, true); - no_install(); + no_install(1); ++now; bootloader_migration_tick(true, true); assert(endf_checks == 1);