diff --git a/tools/lora_ota/lora_ota.py b/tools/lora_ota/lora_ota.py index aaa82b691..43244427a 100755 --- a/tools/lora_ota/lora_ota.py +++ b/tools/lora_ota/lora_ota.py @@ -139,6 +139,7 @@ class TargetInfo: status: str self_status: str current_version: str | None = None + current_version_source: str | None = None @dataclass(frozen=True) @@ -1207,11 +1208,13 @@ def query_target( "could not read the nRF52 bootloader ABI and codec mask from `ota self`" ) current_version = None + current_version_source = None try: stats = controller.remote_command(args.target, "ota stats") version_match = re.search(r"\bfw (v\d+\.\d+\.\d+(?:\.\d+)?)\b", stats) if version_match: current_version = version_match.group(1) + current_version_source = "ota stats" except TransmissionStopped: raise except OtaError as exc: @@ -1224,9 +1227,11 @@ def query_target( f"could not read destination firmware version from `ver`: {version_reply}" ) current_version = format_version(version_value) + current_version_source = "ver" return TargetInfo( args.target, target_id, base_hash, platform, nrf_sd, hw_id, bootloader_abi, bootloader_codecs, status, self_status, current_version, + current_version_source, ) @@ -1594,6 +1599,56 @@ def find_and_start_pull( raise OtaError( f"could not reset failed destination download: {cancel_reply}" ) + elif active_id == getattr(args, "clear_completed_manifest", None): + expected_hash = args.clear_completed_on_body_hash + if "ready to install" not in status.lower(): + raise OtaError( + f"refusing to clear previous mOTA {active_id} because it is " + f"not complete: {status}" + ) + identity = remote_command_with_seeder( + controller, args.target, "ota self", seeder, + "while proving a completed previous manifest", + ) + hash_match = re.search( + r"\bbase_hash=([0-9A-Fa-f]{16})\b", identity + ) + running_hash = hash_match.group(1).upper() if hash_match else None + if running_hash != expected_hash: + raise OtaError( + f"refusing to clear previous mOTA {active_id}: running body " + f"hash is {running_hash or 'unknown'}, expected {expected_hash}" + ) + try: + cancel_reply = controller.remote_command( + args.target, "ota cancel", retry=False + ) + except TransmissionError as cancel_error: + resolved = remote_command_with_seeder( + controller, args.target, "ota status", seeder, + "while resolving a lost completed-manifest cancel reply", + ) + if "no download" not in resolved.lower(): + raise OtaError( + f"completed-manifest cancel outcome is ambiguous: {resolved}" + ) from cancel_error + else: + if not cancel_reply.startswith("OK"): + raise OtaError( + f"could not clear completed previous mOTA: {cancel_reply}" + ) + status = remote_command_with_seeder( + controller, args.target, "ota status", seeder, + "after clearing a completed previous manifest", + ) + if "no download" not in status.lower(): + raise OtaError( + f"completed previous mOTA {active_id} remains: {status}" + ) + print( + f"[download] cleared completed previous session {active_id} " + f"after proving running body {expected_hash}" + ) elif not args.replace_active_download: raise OtaError( f"destination already has mOTA {active_id} staged or downloading; " @@ -1694,6 +1749,48 @@ def require_temp_radio_reply(node: str, reply: str) -> None: raise OtaError(f"{node} did not accept TempRadio: {reply}") +def arm_target_temp_radio( + controller: Controller, + args: argparse.Namespace, + command: str, + temp_radio: RadioSettings, + normal_radio: RadioSettings, +) -> None: + """Resolve a lost TempRadio reply without blindly replaying the command.""" + try: + reply = controller.remote_command(args.target, command, retry=False) + except TransmissionError as command_error: + print( + "[destination] TempRadio reply was lost; probing the declared " + "temporary channel" + ) + controller.set_radio(temp_radio, "probe destination TempRadio state") + try: + identity = controller.remote_command( + args.target, "ota self", retry=False + ) + if re.search(r"\bbase_hash=[0-9A-Fa-f]{16}\b", identity) is None: + raise OtaError( + "destination replied on TempRadio but did not return a valid " + "running EndF identity" + ) + except (OtaError, TransmissionError) as probe_error: + raise OtaError( + "destination TempRadio outcome is ambiguous; the controller was " + "restored to its normal channel and the command was not replayed" + ) from probe_error + finally: + controller.set_radio( + normal_radio, "restore controller after TempRadio probe" + ) + print( + "[destination] resolved lost TempRadio reply from the exact target " + "identity; continuing" + ) + return + require_temp_radio_reply(args.target, reply) + + def temp_radio_command_for_minutes( args: argparse.Namespace, minutes: int, @@ -1895,7 +1992,6 @@ def verify_installed( ) installed_version = None - version_source = "`ota stats`" try: stats_reply = controller.remote_command(args.target, "ota stats") version_match = re.search( @@ -1905,20 +2001,31 @@ def verify_installed( installed_version = parse_version(version_match.group(1)) except OtaError as exc: print(f"[warn] post-reboot `ota stats` version query failed: {exc}") - if installed_version is None: - version_source = "`ver`" + if installed_version is not None: + if installed_version != package.fw_version: + raise OtaError( + f"destination `ota stats` reports " + f"{format_version(installed_version)} after reboot; " + f"expected {package.version}" + ) + else: version_reply = controller.remote_command(args.target, "ver") - installed_version = extract_reply_version(version_reply) - if installed_version is None: + runtime_version = extract_reply_version(version_reply) + if runtime_version is None: raise OtaError( f"post-reboot `ver` returned no firmware version: {version_reply}" ) - if installed_version != package.fw_version: - raise OtaError( - f"destination {version_source} reports " - f"{format_version(installed_version)} after reboot; " - f"expected {package.version}" - ) + if expected_body_hash is None and runtime_version != package.fw_version: + raise OtaError( + f"destination `ver` reports {format_version(runtime_version)} " + f"after reboot; expected {package.version}" + ) + if expected_body_hash is not None and runtime_version != package.fw_version: + print( + f"[verified] `ota stats` did not expose an EndF version; runtime " + f"label {format_version(runtime_version)} differs from package " + f"{package.version}, but the exact expected body hash matches" + ) print( f"[verified] {args.target}: {package.version} " f"body={installed_hash.hex().upper()}" @@ -2011,6 +2118,19 @@ def build_parser() -> argparse.ArgumentParser: "--replace-active-download", action="store_true", help="discard a different update already staged on the destination", ) + parser.add_argument( + "--clear-completed-manifest", + metavar="ID", + help=( + "clear this exact previous ready-to-install manifest only after " + "--clear-completed-on-body-hash proves it is already running" + ), + ) + parser.add_argument( + "--clear-completed-on-body-hash", + metavar="HASH", + help="16-hex running EndF body hash required by --clear-completed-manifest", + ) parser.add_argument("--yes", action="store_true", help="skip the destructive-action confirmation") parser.add_argument( "--require-system-watchdog-off", @@ -2040,6 +2160,24 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) -> args.temp_values = parse_temp_radio(args.temp_radio) except argparse.ArgumentTypeError as exc: parser.error(f"--temp-radio: {exc}") + clear_manifest = args.clear_completed_manifest + clear_body_hash = args.clear_completed_on_body_hash + if bool(clear_manifest) != bool(clear_body_hash): + parser.error( + "--clear-completed-manifest and --clear-completed-on-body-hash " + "must be supplied together" + ) + if clear_manifest: + if not re.fullmatch(r"[0-9A-Fa-f]{8}", clear_manifest): + parser.error("--clear-completed-manifest must be 8 hexadecimal characters") + if not re.fullmatch(r"[0-9A-Fa-f]{16}", clear_body_hash): + parser.error( + "--clear-completed-on-body-hash must be 16 hexadecimal characters" + ) + args.clear_completed_manifest = clear_manifest.upper() + args.clear_completed_on_body_hash = clear_body_hash.upper() + if args.prepare_only: + parser.error("completed-manifest cleanup is only valid during a live run") if args.prepare_only: if not args.platform or not args.target_id: parser.error("--prepare-only requires --platform and --target-id") @@ -2225,10 +2363,18 @@ def main(argv: list[str] | None = None) -> int: freq, bandwidth, sf, cr, _minutes = args.temp_values temp_command = f"tempradio {args.temp_radio}" + temp_radio = RadioSettings( + freq, bandwidth, sf, cr, original_radio.repeat + ) + # Move far nodes first while the controller is still on the normal - # channel. Relays are supplied in farthest-to-nearest order. - temp_reply = controller.remote_command(args.target, temp_command) - require_temp_radio_reply(args.target, temp_reply) + # channel. Relays are supplied in farthest-to-nearest order. A target + # can process TempRadio even when its reply is lost, so resolve that + # ambiguity by probing its exact identity on the temporary channel + # instead of replaying the command from the normal channel. + arm_target_temp_radio( + controller, args, temp_command, temp_radio, original_radio + ) for relay_name, relay_password in args.relay_values: temp_reply = controller.remote_command( relay_name, temp_command, password=relay_password @@ -2238,9 +2384,6 @@ def main(argv: list[str] | None = None) -> int: source_cli_command(args, temp_command) source_temp_owned = True - temp_radio = RadioSettings( - freq, bandwidth, sf, cr, original_radio.repeat - ) controller_changed = True controller.set_radio(temp_radio, "switch controller to TempRadio") if args.source_shares_controller: diff --git a/tools/lora_ota/rak3401_mota_chain.py b/tools/lora_ota/rak3401_mota_chain.py index 8a2aacc85..8f072ab4b 100755 --- a/tools/lora_ota/rak3401_mota_chain.py +++ b/tools/lora_ota/rak3401_mota_chain.py @@ -539,24 +539,21 @@ def find_resume_index( steps: list[ChainStep], final_body_hash: bytes, ) -> int: - current_version = version_number(target.current_version or "") - final_version = version_number(EXPECTED_FINAL_VERSION) - if current_version == final_version: - if target.base_hash != final_body_hash: - raise ota.OtaError( - "target reports the final version but its body hash does not match the release endpoint" - ) + if target.base_hash == final_body_hash: return len(steps) - for index, step in enumerate(steps): - if current_version == version_number(step.from_version): - if target.base_hash != step.base_hash: - raise ota.OtaError( - f"target version matches step {step.number}, but body hash is " - f"{target.base_hash.hex().upper()} instead of {step.base_hash.hex().upper()}" - ) - return index + matching_indexes = [ + index for index, step in enumerate(steps) + if target.base_hash == step.base_hash + ] + if len(matching_indexes) == 1: + return matching_indexes[0] + if len(matching_indexes) > 1: + raise ota.OtaError( + "live body hash occurs at multiple points in the pinned chain" + ) raise ota.OtaError( - f"live version {target.current_version} is not a recognized point in this exact chain" + f"live body hash {target.base_hash.hex().upper()} is not a recognized " + "point in this exact chain" ) @@ -666,6 +663,44 @@ def append_progress( output.write(json.dumps(record, sort_keys=True) + "\n") +def clear_completed_download( + controller: ota.Controller, + target_name: str, + step: ChainStep, +) -> None: + """Clear only a retained record for the exact, proven installed step.""" + status = controller.remote_command(target_name, "ota status") + active_id = ota.download_manifest_id(status) + if active_id is None: + if "no download" not in status.lower(): + raise ota.OtaError( + f"post-install download state is ambiguous after step {step.number}: " + f"{status}" + ) + return + if active_id != step.package.manifest_id: + raise ota.OtaError( + f"post-install target retained mOTA {active_id} after step " + f"{step.number}; expected only {step.package.manifest_id}" + ) + if "ready to install" not in status.lower(): + raise ota.OtaError( + f"post-install target still has an active step-{step.number} session: " + f"{status}" + ) + reply = controller.remote_command(target_name, "ota cancel") + if not reply.startswith("OK"): + raise ota.OtaError( + f"could not clear completed step-{step.number} staging record: {reply}" + ) + status = controller.remote_command(target_name, "ota status") + if "no download" not in status.lower(): + raise ota.OtaError( + f"completed step-{step.number} staging record remains: {status}" + ) + print(f"[chain] cleared retained staging record for step {step.number:02d}") + + def connection_arguments(args: argparse.Namespace) -> list[str]: values: list[str] = [] if args.controller_serial: @@ -694,6 +729,7 @@ def run_step( args: argparse.Namespace, target_name: str, step: ChainStep, + previous_step: ChainStep | None, work_dir: Path, ) -> None: command = [ @@ -714,8 +750,17 @@ def run_step( "--reboot-wait", str(args.reboot_wait), "--work-dir", str(next_attempt_dir(work_dir, step.number)), "--require-system-watchdog-off", + # Some audited bridge binaries retain a later historical runtime + # version string than their pinned EndF chain version. The exact base + # hash, target hash, package ID, and ordered step still gate the move. + "--allow-non-upgrade", "--yes", ] + if previous_step is not None: + command.extend([ + "--clear-completed-manifest", previous_step.package.manifest_id, + "--clear-completed-on-body-hash", step.base_hash.hex().upper(), + ]) for relay in args.relay: command.extend(["--relay", relay]) result = ota.main(command) @@ -981,20 +1026,36 @@ def main(argv: list[str] | None = None) -> int: f"\n[chain] step {step.number:02d}/{len(steps)}: " f"{step.from_version} -> {step.to_version}" ) - run_step(args, target_name, step, work_dir) + previous_step = steps[index - 1] if index > 0 else None + run_step(args, target_name, step, previous_step, work_dir) target = query_live_target(controller, args, target_name) expected_hash = expected_hash_after(steps, final_body_hash, index) - if version_number(target.current_version or "") != version_number(step.to_version): - raise ota.OtaError( - f"step {step.number} returned version {target.current_version}, " - f"expected {step.to_version}" - ) if target.base_hash != expected_hash: raise ota.OtaError( f"step {step.number} returned body hash {target.base_hash.hex().upper()}, " f"expected {expected_hash.hex().upper()}" ) + if ( + target.current_version_source == "ota stats" + and version_number(target.current_version or "") + != version_number(step.to_version) + ): + raise ota.OtaError( + f"step {step.number} EndF metadata reports version " + f"{target.current_version}, expected {step.to_version}" + ) + if ( + target.current_version_source == "ver" + and version_number(target.current_version or "") + != version_number(step.to_version) + ): + print( + f"[chain] step {step.number:02d} exact body hash is verified; " + f"runtime label {target.current_version} is historical and is " + f"not the EndF chain version {step.to_version}" + ) + clear_completed_download(controller, target_name, step) require_watchdog_state(controller, target_name, "off") append_progress(work_dir, step, target.base_hash) print(f"[chain] step {step.number:02d} verified") diff --git a/tools/lora_ota/test_lora_ota.py b/tools/lora_ota/test_lora_ota.py index 1d22746c3..6df57f12a 100644 --- a/tools/lora_ota/test_lora_ota.py +++ b/tools/lora_ota/test_lora_ota.py @@ -406,6 +406,39 @@ class DownloadSessionTests(unittest.TestCase): ota.find_and_start_pull(controller, self.args(), self.package) self.assertEqual(controller.commands, ["ota status"]) + def test_completed_previous_session_requires_exact_running_hash(self) -> None: + args = self.args() + args.clear_completed_manifest = "DEADBEEF" + args.clear_completed_on_body_hash = "0011223344556677" + controller = self.Controller([ + "OTA | download: ready to install 9/9 id=DEADBEEF 2s", + "self body=1 image=2 base_hash=8899AABBCCDDEEFF", + ]) + with self.assertRaisesRegex(ota.OtaError, "running body hash"): + ota.find_and_start_pull(controller, args, self.package) + self.assertEqual(controller.commands, ["ota status", "ota self"]) + + def test_completed_previous_session_is_cleared_after_exact_proof(self) -> None: + args = self.args() + args.clear_completed_manifest = "DEADBEEF" + args.clear_completed_on_body_hash = "0011223344556677" + controller = self.Controller([ + "OTA | download: ready to install 9/9 id=DEADBEEF 2s", + "self body=1 image=2 base_hash=0011223344556677", + "OK dropped session", + "OTA | no download", + "Updates 1/1", + f"OK pulling mid={self.package.manifest_id} -> flash (low priority)", + ]) + ota.find_and_start_pull(controller, args, self.package) + self.assertEqual( + controller.commands, + [ + "ota status", "ota self", "ota cancel", "ota status", "ota ls", + f"ota pull {self.package.manifest_id} flash", + ], + ) + def test_replace_active_session_requires_explicit_flag(self) -> None: controller = self.Controller([ "OTA | download: downloading 3/9 id=DEADBEEF 2s", @@ -488,6 +521,7 @@ class ReliabilityTests(unittest.TestCase): Controller(), argparse.Namespace(target="remote") ) self.assertEqual(result.current_version, "v1.16.9") + self.assertEqual(result.current_version_source, "ver") def test_unattended_prompt_waits_ten_seconds_and_continues(self) -> None: output = io.StringIO() @@ -631,6 +665,73 @@ class ReliabilityTests(unittest.TestCase): with self.assertRaisesRegex(ota.OtaError, "read back"): controller.set_radio(requested, "set radio") + def test_lost_target_temp_reply_is_resolved_on_temporary_channel(self) -> None: + normal = ota.RadioSettings(910.525, 62.5, 7, 5, False) + temporary = ota.RadioSettings(909.95, 250.0, 7, 5, False) + + class Controller: + def __init__(self) -> None: + self.commands: list[str] = [] + self.radios: list[ota.RadioSettings] = [] + + def remote_command( + self, _target: str, command: str, **_kwargs: object + ) -> str: + self.commands.append(command) + if command.startswith("tempradio "): + raise ota.TransmissionError("lost reply") + return "self body=1 image=2 base_hash=0011223344556677" + + def set_radio(self, radio: ota.RadioSettings, _label: str) -> None: + self.radios.append(radio) + + controller = Controller() + ota.arm_target_temp_radio( + controller, + argparse.Namespace(target="remote"), + "tempradio 909.95,250,7,5,120", + temporary, + normal, + ) + self.assertEqual( + controller.commands, + ["tempradio 909.95,250,7,5,120", "ota self"], + ) + self.assertEqual(controller.radios, [temporary, normal]) + + def test_ambiguous_target_temp_probe_is_not_replayed(self) -> None: + normal = ota.RadioSettings(910.525, 62.5, 7, 5, False) + temporary = ota.RadioSettings(909.95, 250.0, 7, 5, False) + + class Controller: + def __init__(self) -> None: + self.commands: list[str] = [] + self.radios: list[ota.RadioSettings] = [] + + def remote_command( + self, _target: str, command: str, **_kwargs: object + ) -> str: + self.commands.append(command) + raise ota.TransmissionError("lost reply") + + def set_radio(self, radio: ota.RadioSettings, _label: str) -> None: + self.radios.append(radio) + + controller = Controller() + with self.assertRaisesRegex(ota.OtaError, "outcome is ambiguous"): + ota.arm_target_temp_radio( + controller, + argparse.Namespace(target="remote"), + "tempradio 909.95,250,7,5,120", + temporary, + normal, + ) + self.assertEqual( + controller.commands, + ["tempradio 909.95,250,7,5,120", "ota self"], + ) + self.assertEqual(controller.radios, [temporary, normal]) + def test_install_retries_only_after_still_ready_is_confirmed(self) -> None: image = firmware(b"install" * 900, VERSION_NEW) package = ota.parse_mota(mota_blob(image)) @@ -780,6 +881,26 @@ class ReliabilityTests(unittest.TestCase): Controller(), argparse.Namespace(target="remote"), package, expected_hash ) + def test_exact_hash_allows_historical_runtime_label_when_stats_unknown(self) -> None: + image = firmware(b"verify-historical" * 700, VERSION_NEW) + package = ota.parse_mota(mota_blob(image)) + expected_hash = ota.parse_endf(image).body_hash + + class Controller: + def __init__(self) -> None: + self.replies = iter([ + f"self body=1 image=2 base_hash={expected_hash.hex()}", + "OTA | fw v? id=?", + "v1.16.9 (Build: historical)", + ]) + + def remote_command(self, *_args: object, **_kwargs: object) -> str: + return next(self.replies) + + ota.verify_installed( + Controller(), argparse.Namespace(target="remote"), package, expected_hash + ) + def test_unknown_new_hash_must_differ_from_pre_install_hash(self) -> None: image = firmware(b"delta-result" * 700, VERSION_NEW) package = ota.parse_mota(mota_blob( @@ -893,6 +1014,51 @@ class ReliabilityTests(unittest.TestCase): class Rak3401KnownUnsafeReleaseTests(unittest.TestCase): + def test_chain_resume_uses_exact_body_hash_not_runtime_label(self) -> None: + steps = [ + mock.Mock(base_hash=bytes.fromhex("0011223344556677")), + mock.Mock(base_hash=bytes.fromhex("8899AABBCCDDEEFF")), + ] + live = target( + platform="nrf52", + base_hash=steps[1].base_hash, + boot_codecs=1 << ota.MOTA_CODEC_IN_PLACE, + current_version="v9.9.9", + ) + self.assertEqual( + rak_chain.find_resume_index(live, steps, b"FINAL123"), 1 + ) + + def test_completed_chain_step_clears_only_its_retained_manifest(self) -> None: + class Controller: + def __init__(self) -> None: + self.commands: list[str] = [] + self.replies = iter([ + "OTA | download: ready to install 9/9 id=1234ABCD 2s", + "OK dropped session", + "OTA | no download | target:2FA509C1", + ]) + + def remote_command(self, _target: str, command: str) -> str: + self.commands.append(command) + return next(self.replies) + + controller = Controller() + step = mock.Mock(number=12, package=mock.Mock(manifest_id="1234ABCD")) + rak_chain.clear_completed_download(controller, "remote", step) + self.assertEqual( + controller.commands, ["ota status", "ota cancel", "ota status"] + ) + + def test_completed_chain_step_refuses_another_retained_manifest(self) -> None: + class Controller: + def remote_command(self, _target: str, _command: str) -> str: + return "OTA | download: ready to install 9/9 id=DEADBEEF 2s" + + step = mock.Mock(number=12, package=mock.Mock(manifest_id="1234ABCD")) + with self.assertRaisesRegex(ota.OtaError, "retained mOTA DEADBEEF"): + rak_chain.clear_completed_download(Controller(), "remote", step) + def test_live_chain_is_blocked_after_failed_physical_step_6(self) -> None: self.assertEqual(rak_chain.KNOWN_UNSAFE_STEP, 6) self.assertEqual(rak_chain.KNOWN_UNSAFE_VERSION, "1.16.8.7")