Harden resumed LoRa OTA chain runs

This commit is contained in:
mikecarper committed 2026-08-13 00:52:22 -07:00
1 parent 8ffafddd6a
commit 451130acbf
3 files changed
+409 -39

No files matched your search

+160 -17
View File
@@ -139,6 +139,7 @@ class TargetInfo:
status: str
self_status: str
current_version: str | None = None
current_version_source: str | None = None
@dataclass(frozen=True)
@@ -1207,11 +1208,13 @@ def query_target(
"could not read the nRF52 bootloader ABI and codec mask from `ota self`"
)
current_version = None
current_version_source = None
try:
stats = controller.remote_command(args.target, "ota stats")
version_match = re.search(r"\bfw (v\d+\.\d+\.\d+(?:\.\d+)?)\b", stats)
if version_match:
current_version = version_match.group(1)
current_version_source = "ota stats"
except TransmissionStopped:
raise
except OtaError as exc:
@@ -1224,9 +1227,11 @@ def query_target(
f"could not read destination firmware version from `ver`: {version_reply}"
)
current_version = format_version(version_value)
current_version_source = "ver"
return TargetInfo(
args.target, target_id, base_hash, platform, nrf_sd, hw_id,
bootloader_abi, bootloader_codecs, status, self_status, current_version,
current_version_source,
)
@@ -1594,6 +1599,56 @@ def find_and_start_pull(
raise OtaError(
f"could not reset failed destination download: {cancel_reply}"
)
elif active_id == getattr(args, "clear_completed_manifest", None):
expected_hash = args.clear_completed_on_body_hash
if "ready to install" not in status.lower():
raise OtaError(
f"refusing to clear previous mOTA {active_id} because it is "
f"not complete: {status}"
)
identity = remote_command_with_seeder(
controller, args.target, "ota self", seeder,
"while proving a completed previous manifest",
)
hash_match = re.search(
r"\bbase_hash=([0-9A-Fa-f]{16})\b", identity
)
running_hash = hash_match.group(1).upper() if hash_match else None
if running_hash != expected_hash:
raise OtaError(
f"refusing to clear previous mOTA {active_id}: running body "
f"hash is {running_hash or 'unknown'}, expected {expected_hash}"
)
try:
cancel_reply = controller.remote_command(
args.target, "ota cancel", retry=False
)
except TransmissionError as cancel_error:
resolved = remote_command_with_seeder(
controller, args.target, "ota status", seeder,
"while resolving a lost completed-manifest cancel reply",
)
if "no download" not in resolved.lower():
raise OtaError(
f"completed-manifest cancel outcome is ambiguous: {resolved}"
) from cancel_error
else:
if not cancel_reply.startswith("OK"):
raise OtaError(
f"could not clear completed previous mOTA: {cancel_reply}"
)
status = remote_command_with_seeder(
controller, args.target, "ota status", seeder,
"after clearing a completed previous manifest",
)
if "no download" not in status.lower():
raise OtaError(
f"completed previous mOTA {active_id} remains: {status}"
)
print(
f"[download] cleared completed previous session {active_id} "
f"after proving running body {expected_hash}"
)
elif not args.replace_active_download:
raise OtaError(
f"destination already has mOTA {active_id} staged or downloading; "
@@ -1694,6 +1749,48 @@ def require_temp_radio_reply(node: str, reply: str) -> None:
raise OtaError(f"{node} did not accept TempRadio: {reply}")
def arm_target_temp_radio(
controller: Controller,
args: argparse.Namespace,
command: str,
temp_radio: RadioSettings,
normal_radio: RadioSettings,
) -> None:
"""Resolve a lost TempRadio reply without blindly replaying the command."""
try:
reply = controller.remote_command(args.target, command, retry=False)
except TransmissionError as command_error:
print(
"[destination] TempRadio reply was lost; probing the declared "
"temporary channel"
)
controller.set_radio(temp_radio, "probe destination TempRadio state")
try:
identity = controller.remote_command(
args.target, "ota self", retry=False
)
if re.search(r"\bbase_hash=[0-9A-Fa-f]{16}\b", identity) is None:
raise OtaError(
"destination replied on TempRadio but did not return a valid "
"running EndF identity"
)
except (OtaError, TransmissionError) as probe_error:
raise OtaError(
"destination TempRadio outcome is ambiguous; the controller was "
"restored to its normal channel and the command was not replayed"
) from probe_error
finally:
controller.set_radio(
normal_radio, "restore controller after TempRadio probe"
)
print(
"[destination] resolved lost TempRadio reply from the exact target "
"identity; continuing"
)
return
require_temp_radio_reply(args.target, reply)
def temp_radio_command_for_minutes(
args: argparse.Namespace,
minutes: int,
@@ -1895,7 +1992,6 @@ def verify_installed(
)
installed_version = None
version_source = "`ota stats`"
try:
stats_reply = controller.remote_command(args.target, "ota stats")
version_match = re.search(
@@ -1905,20 +2001,31 @@ def verify_installed(
installed_version = parse_version(version_match.group(1))
except OtaError as exc:
print(f"[warn] post-reboot `ota stats` version query failed: {exc}")
if installed_version is None:
version_source = "`ver`"
if installed_version is not None:
if installed_version != package.fw_version:
raise OtaError(
f"destination `ota stats` reports "
f"{format_version(installed_version)} after reboot; "
f"expected {package.version}"
)
else:
version_reply = controller.remote_command(args.target, "ver")
installed_version = extract_reply_version(version_reply)
if installed_version is None:
runtime_version = extract_reply_version(version_reply)
if runtime_version is None:
raise OtaError(
f"post-reboot `ver` returned no firmware version: {version_reply}"
)
if installed_version != package.fw_version:
raise OtaError(
f"destination {version_source} reports "
f"{format_version(installed_version)} after reboot; "
f"expected {package.version}"
)
if expected_body_hash is None and runtime_version != package.fw_version:
raise OtaError(
f"destination `ver` reports {format_version(runtime_version)} "
f"after reboot; expected {package.version}"
)
if expected_body_hash is not None and runtime_version != package.fw_version:
print(
f"[verified] `ota stats` did not expose an EndF version; runtime "
f"label {format_version(runtime_version)} differs from package "
f"{package.version}, but the exact expected body hash matches"
)
print(
f"[verified] {args.target}: {package.version} "
f"body={installed_hash.hex().upper()}"
@@ -2011,6 +2118,19 @@ def build_parser() -> argparse.ArgumentParser:
"--replace-active-download", action="store_true",
help="discard a different update already staged on the destination",
)
parser.add_argument(
"--clear-completed-manifest",
metavar="ID",
help=(
"clear this exact previous ready-to-install manifest only after "
"--clear-completed-on-body-hash proves it is already running"
),
)
parser.add_argument(
"--clear-completed-on-body-hash",
metavar="HASH",
help="16-hex running EndF body hash required by --clear-completed-manifest",
)
parser.add_argument("--yes", action="store_true", help="skip the destructive-action confirmation")
parser.add_argument(
"--require-system-watchdog-off",
@@ -2040,6 +2160,24 @@ def validate_args(args: argparse.Namespace, parser: argparse.ArgumentParser) ->
args.temp_values = parse_temp_radio(args.temp_radio)
except argparse.ArgumentTypeError as exc:
parser.error(f"--temp-radio: {exc}")
clear_manifest = args.clear_completed_manifest
clear_body_hash = args.clear_completed_on_body_hash
if bool(clear_manifest) != bool(clear_body_hash):
parser.error(
"--clear-completed-manifest and --clear-completed-on-body-hash "
"must be supplied together"
)
if clear_manifest:
if not re.fullmatch(r"[0-9A-Fa-f]{8}", clear_manifest):
parser.error("--clear-completed-manifest must be 8 hexadecimal characters")
if not re.fullmatch(r"[0-9A-Fa-f]{16}", clear_body_hash):
parser.error(
"--clear-completed-on-body-hash must be 16 hexadecimal characters"
)
args.clear_completed_manifest = clear_manifest.upper()
args.clear_completed_on_body_hash = clear_body_hash.upper()
if args.prepare_only:
parser.error("completed-manifest cleanup is only valid during a live run")
if args.prepare_only:
if not args.platform or not args.target_id:
parser.error("--prepare-only requires --platform and --target-id")
@@ -2225,10 +2363,18 @@ def main(argv: list[str] | None = None) -> int:
freq, bandwidth, sf, cr, _minutes = args.temp_values
temp_command = f"tempradio {args.temp_radio}"
temp_radio = RadioSettings(
freq, bandwidth, sf, cr, original_radio.repeat
)
# Move far nodes first while the controller is still on the normal
# channel. Relays are supplied in farthest-to-nearest order.
temp_reply = controller.remote_command(args.target, temp_command)
require_temp_radio_reply(args.target, temp_reply)
# channel. Relays are supplied in farthest-to-nearest order. A target
# can process TempRadio even when its reply is lost, so resolve that
# ambiguity by probing its exact identity on the temporary channel
# instead of replaying the command from the normal channel.
arm_target_temp_radio(
controller, args, temp_command, temp_radio, original_radio
)
for relay_name, relay_password in args.relay_values:
temp_reply = controller.remote_command(
relay_name, temp_command, password=relay_password
@@ -2238,9 +2384,6 @@ def main(argv: list[str] | None = None) -> int:
source_cli_command(args, temp_command)
source_temp_owned = True
temp_radio = RadioSettings(
freq, bandwidth, sf, cr, original_radio.repeat
)
controller_changed = True
controller.set_radio(temp_radio, "switch controller to TempRadio")
if args.source_shares_controller:
+83 -22
View File
@@ -539,24 +539,21 @@ def find_resume_index(
steps: list[ChainStep],
final_body_hash: bytes,
) -> int:
current_version = version_number(target.current_version or "")
final_version = version_number(EXPECTED_FINAL_VERSION)
if current_version == final_version:
if target.base_hash != final_body_hash:
raise ota.OtaError(
"target reports the final version but its body hash does not match the release endpoint"
)
if target.base_hash == final_body_hash:
return len(steps)
for index, step in enumerate(steps):
if current_version == version_number(step.from_version):
if target.base_hash != step.base_hash:
raise ota.OtaError(
f"target version matches step {step.number}, but body hash is "
f"{target.base_hash.hex().upper()} instead of {step.base_hash.hex().upper()}"
)
return index
matching_indexes = [
index for index, step in enumerate(steps)
if target.base_hash == step.base_hash
]
if len(matching_indexes) == 1:
return matching_indexes[0]
if len(matching_indexes) > 1:
raise ota.OtaError(
"live body hash occurs at multiple points in the pinned chain"
)
raise ota.OtaError(
f"live version {target.current_version} is not a recognized point in this exact chain"
f"live body hash {target.base_hash.hex().upper()} is not a recognized "
"point in this exact chain"
)
@@ -666,6 +663,44 @@ def append_progress(
output.write(json.dumps(record, sort_keys=True) + "\n")
def clear_completed_download(
controller: ota.Controller,
target_name: str,
step: ChainStep,
) -> None:
"""Clear only a retained record for the exact, proven installed step."""
status = controller.remote_command(target_name, "ota status")
active_id = ota.download_manifest_id(status)
if active_id is None:
if "no download" not in status.lower():
raise ota.OtaError(
f"post-install download state is ambiguous after step {step.number}: "
f"{status}"
)
return
if active_id != step.package.manifest_id:
raise ota.OtaError(
f"post-install target retained mOTA {active_id} after step "
f"{step.number}; expected only {step.package.manifest_id}"
)
if "ready to install" not in status.lower():
raise ota.OtaError(
f"post-install target still has an active step-{step.number} session: "
f"{status}"
)
reply = controller.remote_command(target_name, "ota cancel")
if not reply.startswith("OK"):
raise ota.OtaError(
f"could not clear completed step-{step.number} staging record: {reply}"
)
status = controller.remote_command(target_name, "ota status")
if "no download" not in status.lower():
raise ota.OtaError(
f"completed step-{step.number} staging record remains: {status}"
)
print(f"[chain] cleared retained staging record for step {step.number:02d}")
def connection_arguments(args: argparse.Namespace) -> list[str]:
values: list[str] = []
if args.controller_serial:
@@ -694,6 +729,7 @@ def run_step(
args: argparse.Namespace,
target_name: str,
step: ChainStep,
previous_step: ChainStep | None,
work_dir: Path,
) -> None:
command = [
@@ -714,8 +750,17 @@ def run_step(
"--reboot-wait", str(args.reboot_wait),
"--work-dir", str(next_attempt_dir(work_dir, step.number)),
"--require-system-watchdog-off",
# Some audited bridge binaries retain a later historical runtime
# version string than their pinned EndF chain version. The exact base
# hash, target hash, package ID, and ordered step still gate the move.
"--allow-non-upgrade",
"--yes",
]
if previous_step is not None:
command.extend([
"--clear-completed-manifest", previous_step.package.manifest_id,
"--clear-completed-on-body-hash", step.base_hash.hex().upper(),
])
for relay in args.relay:
command.extend(["--relay", relay])
result = ota.main(command)
@@ -981,20 +1026,36 @@ def main(argv: list[str] | None = None) -> int:
f"\n[chain] step {step.number:02d}/{len(steps)}: "
f"{step.from_version} -> {step.to_version}"
)
run_step(args, target_name, step, work_dir)
previous_step = steps[index - 1] if index > 0 else None
run_step(args, target_name, step, previous_step, work_dir)
target = query_live_target(controller, args, target_name)
expected_hash = expected_hash_after(steps, final_body_hash, index)
if version_number(target.current_version or "") != version_number(step.to_version):
raise ota.OtaError(
f"step {step.number} returned version {target.current_version}, "
f"expected {step.to_version}"
)
if target.base_hash != expected_hash:
raise ota.OtaError(
f"step {step.number} returned body hash {target.base_hash.hex().upper()}, "
f"expected {expected_hash.hex().upper()}"
)
if (
target.current_version_source == "ota stats"
and version_number(target.current_version or "")
!= version_number(step.to_version)
):
raise ota.OtaError(
f"step {step.number} EndF metadata reports version "
f"{target.current_version}, expected {step.to_version}"
)
if (
target.current_version_source == "ver"
and version_number(target.current_version or "")
!= version_number(step.to_version)
):
print(
f"[chain] step {step.number:02d} exact body hash is verified; "
f"runtime label {target.current_version} is historical and is "
f"not the EndF chain version {step.to_version}"
)
clear_completed_download(controller, target_name, step)
require_watchdog_state(controller, target_name, "off")
append_progress(work_dir, step, target.base_hash)
print(f"[chain] step {step.number:02d} verified")
+166
View File
@@ -406,6 +406,39 @@ class DownloadSessionTests(unittest.TestCase):
ota.find_and_start_pull(controller, self.args(), self.package)
self.assertEqual(controller.commands, ["ota status"])
def test_completed_previous_session_requires_exact_running_hash(self) -> None:
args = self.args()
args.clear_completed_manifest = "DEADBEEF"
args.clear_completed_on_body_hash = "0011223344556677"
controller = self.Controller([
"OTA | download: ready to install 9/9 id=DEADBEEF 2s",
"self body=1 image=2 base_hash=8899AABBCCDDEEFF",
])
with self.assertRaisesRegex(ota.OtaError, "running body hash"):
ota.find_and_start_pull(controller, args, self.package)
self.assertEqual(controller.commands, ["ota status", "ota self"])
def test_completed_previous_session_is_cleared_after_exact_proof(self) -> None:
args = self.args()
args.clear_completed_manifest = "DEADBEEF"
args.clear_completed_on_body_hash = "0011223344556677"
controller = self.Controller([
"OTA | download: ready to install 9/9 id=DEADBEEF 2s",
"self body=1 image=2 base_hash=0011223344556677",
"OK dropped session",
"OTA | no download",
"Updates 1/1",
f"OK pulling mid={self.package.manifest_id} -> flash (low priority)",
])
ota.find_and_start_pull(controller, args, self.package)
self.assertEqual(
controller.commands,
[
"ota status", "ota self", "ota cancel", "ota status", "ota ls",
f"ota pull {self.package.manifest_id} flash",
],
)
def test_replace_active_session_requires_explicit_flag(self) -> None:
controller = self.Controller([
"OTA | download: downloading 3/9 id=DEADBEEF 2s",
@@ -488,6 +521,7 @@ class ReliabilityTests(unittest.TestCase):
Controller(), argparse.Namespace(target="remote")
)
self.assertEqual(result.current_version, "v1.16.9")
self.assertEqual(result.current_version_source, "ver")
def test_unattended_prompt_waits_ten_seconds_and_continues(self) -> None:
output = io.StringIO()
@@ -631,6 +665,73 @@ class ReliabilityTests(unittest.TestCase):
with self.assertRaisesRegex(ota.OtaError, "read back"):
controller.set_radio(requested, "set radio")
def test_lost_target_temp_reply_is_resolved_on_temporary_channel(self) -> None:
normal = ota.RadioSettings(910.525, 62.5, 7, 5, False)
temporary = ota.RadioSettings(909.95, 250.0, 7, 5, False)
class Controller:
def __init__(self) -> None:
self.commands: list[str] = []
self.radios: list[ota.RadioSettings] = []
def remote_command(
self, _target: str, command: str, **_kwargs: object
) -> str:
self.commands.append(command)
if command.startswith("tempradio "):
raise ota.TransmissionError("lost reply")
return "self body=1 image=2 base_hash=0011223344556677"
def set_radio(self, radio: ota.RadioSettings, _label: str) -> None:
self.radios.append(radio)
controller = Controller()
ota.arm_target_temp_radio(
controller,
argparse.Namespace(target="remote"),
"tempradio 909.95,250,7,5,120",
temporary,
normal,
)
self.assertEqual(
controller.commands,
["tempradio 909.95,250,7,5,120", "ota self"],
)
self.assertEqual(controller.radios, [temporary, normal])
def test_ambiguous_target_temp_probe_is_not_replayed(self) -> None:
normal = ota.RadioSettings(910.525, 62.5, 7, 5, False)
temporary = ota.RadioSettings(909.95, 250.0, 7, 5, False)
class Controller:
def __init__(self) -> None:
self.commands: list[str] = []
self.radios: list[ota.RadioSettings] = []
def remote_command(
self, _target: str, command: str, **_kwargs: object
) -> str:
self.commands.append(command)
raise ota.TransmissionError("lost reply")
def set_radio(self, radio: ota.RadioSettings, _label: str) -> None:
self.radios.append(radio)
controller = Controller()
with self.assertRaisesRegex(ota.OtaError, "outcome is ambiguous"):
ota.arm_target_temp_radio(
controller,
argparse.Namespace(target="remote"),
"tempradio 909.95,250,7,5,120",
temporary,
normal,
)
self.assertEqual(
controller.commands,
["tempradio 909.95,250,7,5,120", "ota self"],
)
self.assertEqual(controller.radios, [temporary, normal])
def test_install_retries_only_after_still_ready_is_confirmed(self) -> None:
image = firmware(b"install" * 900, VERSION_NEW)
package = ota.parse_mota(mota_blob(image))
@@ -780,6 +881,26 @@ class ReliabilityTests(unittest.TestCase):
Controller(), argparse.Namespace(target="remote"), package, expected_hash
)
def test_exact_hash_allows_historical_runtime_label_when_stats_unknown(self) -> None:
image = firmware(b"verify-historical" * 700, VERSION_NEW)
package = ota.parse_mota(mota_blob(image))
expected_hash = ota.parse_endf(image).body_hash
class Controller:
def __init__(self) -> None:
self.replies = iter([
f"self body=1 image=2 base_hash={expected_hash.hex()}",
"OTA | fw v? id=?",
"v1.16.9 (Build: historical)",
])
def remote_command(self, *_args: object, **_kwargs: object) -> str:
return next(self.replies)
ota.verify_installed(
Controller(), argparse.Namespace(target="remote"), package, expected_hash
)
def test_unknown_new_hash_must_differ_from_pre_install_hash(self) -> None:
image = firmware(b"delta-result" * 700, VERSION_NEW)
package = ota.parse_mota(mota_blob(
@@ -893,6 +1014,51 @@ class ReliabilityTests(unittest.TestCase):
class Rak3401KnownUnsafeReleaseTests(unittest.TestCase):
def test_chain_resume_uses_exact_body_hash_not_runtime_label(self) -> None:
steps = [
mock.Mock(base_hash=bytes.fromhex("0011223344556677")),
mock.Mock(base_hash=bytes.fromhex("8899AABBCCDDEEFF")),
]
live = target(
platform="nrf52",
base_hash=steps[1].base_hash,
boot_codecs=1 << ota.MOTA_CODEC_IN_PLACE,
current_version="v9.9.9",
)
self.assertEqual(
rak_chain.find_resume_index(live, steps, b"FINAL123"), 1
)
def test_completed_chain_step_clears_only_its_retained_manifest(self) -> None:
class Controller:
def __init__(self) -> None:
self.commands: list[str] = []
self.replies = iter([
"OTA | download: ready to install 9/9 id=1234ABCD 2s",
"OK dropped session",
"OTA | no download | target:2FA509C1",
])
def remote_command(self, _target: str, command: str) -> str:
self.commands.append(command)
return next(self.replies)
controller = Controller()
step = mock.Mock(number=12, package=mock.Mock(manifest_id="1234ABCD"))
rak_chain.clear_completed_download(controller, "remote", step)
self.assertEqual(
controller.commands, ["ota status", "ota cancel", "ota status"]
)
def test_completed_chain_step_refuses_another_retained_manifest(self) -> None:
class Controller:
def remote_command(self, _target: str, _command: str) -> str:
return "OTA | download: ready to install 9/9 id=DEADBEEF 2s"
step = mock.Mock(number=12, package=mock.Mock(manifest_id="1234ABCD"))
with self.assertRaisesRegex(ota.OtaError, "retained mOTA DEADBEEF"):
rak_chain.clear_completed_download(Controller(), "remote", step)
def test_live_chain_is_blocked_after_failed_physical_step_6(self) -> None:
self.assertEqual(rak_chain.KNOWN_UNSAFE_STEP, 6)
self.assertEqual(rak_chain.KNOWN_UNSAFE_VERSION, "1.16.8.7")