Verify normal radio return for every OTA participant

This commit is contained in:
mikecarper committed 2026-10-21 04:31:07 -07:00
1 parent c37742152d
commit e285214349
7 files changed
+754 -111

No files matched your search

+29
View File
@@ -1067,6 +1067,35 @@ TempRadio window ends, or after putting the controller back on that tuple),
rerunning the same package recognizes its manifest ID and resumes the existing
session instead of clearing it.
The destination and each controlled relay also receive separate protected
`target-tempradio-cleanup.json` or `relay-tempradio-cleanup-<full-key>.json`
records. Each record is flushed before its single one-minute shortening attempt.
A lost acknowledgement remains unconfirmed; a reply on the temporary channel
does not establish that the shortening arrived. Restoring the controller or
reaching the destination through another path does not establish that every
relay returned either.
Default cleanup succeeds only after the controller is back on its normal
radio and every owned participant independently returns its bound full public
key and an inactive `get tempradio` result. Shared Full Companions must also
report their live TempRadio override inactive; Binary radio settings alone
show the saved preference. Unresolved participants retain their own record and
the run reports failure. A deliberately staged `--leave-controller-radio` run
preserves its windows and records. After an install, that option proves all
participants during the normal-radio verification interval before returning
the controller to the requested temporary tuple.
After a process interruption, retain these records and use a new work directory.
Existing records cannot be overwritten to reset a shortening attempt. Never
repeat a relative `tempradio` command whose attempt is recorded. On the normal
route, use the recorded full key and a separately supplied admin password to
read `get public.key` and `get tempradio`. Both must confirm that same participant
and an inactive override; a timeout, changed key, active profile, or malformed
reply leaves its obligation pending. Wait for its original finite lease and
repeat only these read-only checks. Keep the original record and a separate
private restoration receipt binding its file hash and successful proofs. These
records contain no admin passwords and do not authorize reboot or installation.
When a chained run has already proved the newly running body hash, its retained
previous package can briefly report `verifying staged blocks` after TempRadio
reactivates the OTA manager. The runner waits only through the configured
+317 -99
View File
@@ -157,6 +157,9 @@ LEGACY_OTA_CACHE_ROUNDS = 2
LEGACY_RELAY_REBOOT_MIN_REMAINING_SECONDS = 8 * 60
TARGET_RXPS_RECOVERY_FILE = "target-rxps-settings.json"
SOURCE_RXPS_RECOVERY_FILE = "source-rxps-settings.json"
TARGET_TEMP_RECOVERY_FILE = "target-tempradio-cleanup.json"
RELAY_TEMP_RECOVERY_PREFIX = "relay-tempradio-cleanup-"
TARGET_TEMP_NORMAL_PROOF_SECONDS = 90
MIN_MESHCLI_VERSION = (1, 6, 0)
# Tested format-3 inspect/verify/serve support. Keep automatic repairs pinned;
# neither a package nor a moving branch may choose code to install on the host.
@@ -7706,112 +7709,289 @@ def request_install(
def shorten_relay_temp_windows(
controller: Controller,
args: argparse.Namespace,
controller: Controller, args: argparse.Namespace,
relay_values: list[tuple[str, str]] | None = None,
expected_public_keys: dict[str, str] | None = None,
) -> None:
*, cleanup_states: dict[str, RelayTempCleanup] | None = None,
) -> bool:
"""Send each relative setter once; ACKs alone never establish normal return.
Production passes durable per-participant ownership. Standalone callers
receive only the aggregate acknowledgement result, never a return proof.
"""
values = args.relay_values if relay_values is None else relay_values
if not values:
return
command = temp_radio_command_for_minutes(args, TEMP_RADIO_RETURN_MINUTES)
print(
f"[relays] scheduling return to the normal channel in "
f"{TEMP_RADIO_RETURN_MINUTES} minute"
)
errors: list[str] = []
interrupted: KeyboardInterrupt | None = None
for relay_name, relay_password in values:
if cleanup_states is not None:
for key, _ in values:
state = cleanup_states.get(key)
if (state is None or (expected_public_keys is not None
and expected_public_keys.get(key) != state.key)):
raise OtaError("relay cleanup is missing bound durable ownership")
errors = []
interrupted = None
acknowledged = True
for key, password in values:
try:
try:
reply = controller.remote_command(
relay_name,
command,
password=relay_password,
retry=False,
)
except TransmissionError as exc:
# Never replay a relative lease. Prove which authenticated
# relay remained reachable on this tuple; its original window
# is still bounded if the one-minute write did not arrive.
actual_key = read_remote_public_key_bounded(
controller,
relay_name,
password=relay_password,
)
expected_key = (
expected_public_keys.get(relay_name)
if expected_public_keys is not None
else None
)
if expected_key is not None and actual_key != expected_key:
raise OtaError(
f"identity changed after lost cleanup reply: expected "
f"{expected_key}, got {actual_key}"
)
print(
f"[relays] {relay_name} one-minute TempRadio reply was "
f"lost ({exc}); exact identity reconciled the one-shot "
"request, and its original lease remains bounded"
)
if cleanup_states is None:
ack = shorten_remote_temp_window(controller, args, key,
password=password, label="relay")
else:
require_temp_radio_reply(
relay_name, reply, TEMP_RADIO_RETURN_MINUTES
)
state = cleanup_states[key]
state.shorten_once(controller, args)
ack = state.report["shorten_acknowledged"]
acknowledged = acknowledged and ack
except KeyboardInterrupt as exc:
# Continue through the remaining independently owned relays. The
# interrupt is re-raised only after each has received one attempt.
if interrupted is None:
interrupted = exc
errors.append(f"{relay_name}: interrupted")
interrupted = interrupted or exc
errors.append(f"{key}: interrupted")
except (OtaError, OSError) as exc:
errors.append(f"{relay_name}: {exc}")
errors.append(f"{key}: " + redact_text(str(exc), (password,)))
if interrupted is not None:
raise interrupted
if errors:
detail = "; ".join(errors)
if interrupted is not None:
print(
"WARNING: relay cleanup continued after interruption: "
+ detail,
file=sys.stderr,
)
raise interrupted
raise OtaError("could not shorten every relay TempRadio lease: " + detail)
raise OtaError("could not shorten every relay TempRadio lease: " + "; ".join(errors))
return acknowledged
def shorten_target_temp_window(
controller: Controller,
args: argparse.Namespace,
) -> None:
def shorten_remote_temp_window(
controller: Controller, args: argparse.Namespace, key: str,
*, password: str | None = None, label: str = "destination",
) -> bool:
"""Attempt one relative shortening; return whether its ACK was proven.
Reachability on the temporary tuple cannot prove that a lost setter was
delivered. The caller retains a separate normal-return obligation either
way, and must not replay this duration after a missing acknowledgement.
"""
command = temp_radio_command_for_minutes(args, TEMP_RADIO_RETURN_MINUTES)
print(
f"[destination] scheduling return to the normal channel in "
f"[{label}] scheduling return to the normal channel in "
f"{TEMP_RADIO_RETURN_MINUTES} minute"
)
try:
reply = controller.remote_command(
args.target, command, retry=False
)
options = {"password": password} if password is not None else {}
reply = controller.remote_command(key, command, retry=False, **options)
except TransmissionError as exc:
identity = retry_transmission_bounded(
lambda: controller.remote_command(
args.target, "ota self", retry=False
),
f"cleanup identity reconciliation for {args.target}",
)
if re.search(r"\bbase_hash=[0-9A-Fa-f]{16}\b", identity) is None:
raise OtaError(
"destination replied after a lost cleanup request but did not "
"return a valid running EndF identity"
)
print(
"[destination] one-minute TempRadio reply was lost "
f"({exc}); exact identity reconciled the one-shot request, and the "
"original lease remains bounded"
f"WARNING: {label} one-minute TempRadio acknowledgement was "
f"lost ({redact_text(str(exc), (password,) if password else ())}); shortening is unconfirmed and was not replayed. "
"The original long lease may remain active; exact normal-channel "
"identity and inactive TempRadio must still be proven.",
file=sys.stderr,
)
return
require_temp_radio_reply(
args.target, reply, TEMP_RADIO_RETURN_MINUTES
)
return False
require_temp_radio_reply(key, reply, TEMP_RADIO_RETURN_MINUTES)
return True
class RemoteTempCleanup:
"""One bound participant's durable return obligation and bounded read proof."""
def __init__(self, path: Path, args: argparse.Namespace, normal_radio: RadioSettings,
key: str, role: str, password: str | None = None):
if path.exists() or path.is_symlink():
raise OtaError("existing TempRadio recovery obligation must not be overwritten: " + str(path))
self.path = path
self.normal_radio = normal_radio
self.key, self.role, self.password = key, role, password
self.sensitive_values = tuple(value for value in (password, getattr(args, "password", None)) if value)
self.report: dict = {
"schema": 1,
"target": key,
"role": role,
"normal_source_radio": normal_radio.meshcli_value(),
"temporary_radio": args.temp_radio,
"cleanup_required": True,
"shorten_attempted": False,
"shorten_acknowledged": False,
"normal_proof_attempted": False,
"normal_return_verified": False,
"proof_records": [],
}
self.persist()
def redact_error(self, exc: BaseException) -> str:
return redact_text(str(exc), self.sensitive_values)
def _shorten(self, controller: Controller, args: argparse.Namespace) -> bool:
raise NotImplementedError
def persist(self) -> None:
write_private_recovery_file(
self.path, json.dumps(self.report, indent=2, sort_keys=True) + "\n"
)
def require_binding(self, args: argparse.Namespace) -> None:
expected = self.report["target"]
bound = (args.target == expected if self.role == "destination" else
any(key == expected for key, _ in args.relay_values))
if (not re.fullmatch(r"[0-9A-Fa-f]{64}", expected)
or self.key != expected or not bound):
raise OtaError("normal-return proof requires the unchanged full participant key")
def shorten_once(self, controller: Controller, args: argparse.Namespace) -> None:
if self.report["shorten_attempted"]:
return
self.require_binding(args)
self.report["shorten_attempted"] = True
self.persist()
try:
self.report["shorten_acknowledged"] = (
self._shorten(controller, args) is True
)
except BaseException as exc:
self.report["shorten_error"] = self.redact_error(exc)
raise
finally:
self.persist()
def prove_normal(self, controller: Controller, args: argparse.Namespace) -> None:
if not self.report["cleanup_required"]:
return
if self.report["normal_proof_attempted"]:
raise OtaError(f"{self.role} normal return remains unproven; inspect " + str(self.path))
self.require_binding(args)
expected = self.key
self.report["normal_proof_attempted"] = True
self.persist()
deadline = time.monotonic() + TARGET_TEMP_NORMAL_PROOF_SECONDS
def read(command: str) -> str:
remaining = deadline - time.monotonic()
if remaining < 1:
raise TransmissionError(f"{self.role} normal-return proof deadline expired")
record = {"command": command, "started_at": time.monotonic(),
"operation_timeout": min(30.0, remaining)}
self.report["proof_records"].append(record)
self.persist()
try:
reply = controller.remote_command(
expected, command, retry=False,
**({"password": self.password} if self.password is not None else {}),
reply_timeout=min(20, int(remaining)),
operation_timeout=record["operation_timeout"],
)
record["reply"] = redact_text(reply, self.sensitive_values)
return reply
except BaseException as exc:
record["error"] = self.redact_error(exc)
raise
finally:
self.persist()
def proof() -> None:
remaining = deadline - time.monotonic()
if remaining < 1:
raise TransmissionError(f"{self.role} normal-return proof deadline expired")
if getattr(args, "source_shares_controller", False):
# Binary GET_RADIO reports saved preferences on a Full
# Companion, even while its live TempRadio override is active.
record = {"command": "shared source tempradio", "started_at": time.monotonic()}
self.report["proof_records"].append(record)
self.persist()
try:
source_status = read_source_temp_radio_status(args, deadline=deadline)
record["state"] = source_status[0] if source_status is not None else "unsupported"
except BaseException as exc:
record["error"] = self.redact_error(exc)
raise
finally:
self.persist()
if source_status is None:
raise OtaError("shared controller live TempRadio state is unavailable")
if source_status[0] != "inactive":
raise TransmissionError("shared controller live TempRadio remains active; saved normal settings are insufficient")
remaining = deadline - time.monotonic()
if remaining < 1:
raise TransmissionError("shared controller normal-return proof deadline expired")
require_radio_settings(
controller.get_radio(timeout=min(30.0, remaining)),
self.normal_radio, f"{self.role} cleanup normal controller",
)
public = parse_cli_public_key(read("get public.key"), f"{self.role} cleanup")
if public.lower() != expected.lower():
raise OtaError(f"{self.role} normal-return full public key changed")
status = read("get tempradio").strip()
status = status[1:].strip() if status.startswith(">") else status
if status == "off":
return
parts = status.split(",")
try:
active = (len(parts) == 5
and all(math.isfinite(float(x)) for x in parts)
and 150 <= float(parts[0]) <= 2500
and float(parts[1]) > 0
and 5 <= int(parts[2]) <= 12
and 5 <= int(parts[3]) <= 8
and 1 <= int(parts[4]) <= 65535)
except ValueError:
active = False
if not active:
raise OtaError(f"{self.role} returned malformed TempRadio state during cleanup")
raise TransmissionError(f"{self.role} still reports active TempRadio; normal return is unproven")
try:
# A confirmed one-minute lease can remain active during the first
# checks. Poll only replay-safe reads within one absolute bound.
retry_transmission_before_deadline(
proof, f"{self.role} normal-return proof", deadline,
retry_limit=12, deadline_label=f"90-second {self.role} cleanup window",
)
except BaseException as exc:
self.report["normal_proof_error"] = self.redact_error(exc)
raise
else:
self.report.update(cleanup_required=False, normal_return_verified=True)
print(f"[{self.role}] exact full identity and inactive TempRadio verified on the normal channel")
finally:
self.persist()
class TargetTempCleanup(RemoteTempCleanup):
def __init__(self, work_dir: Path, args: argparse.Namespace, normal_radio: RadioSettings,
password: str | None = None):
super().__init__(work_dir / TARGET_TEMP_RECOVERY_FILE, args, normal_radio,
args.target, "destination", password)
def _shorten(self, controller: Controller, args: argparse.Namespace) -> bool:
return shorten_target_temp_window(controller, args, password=self.password)
class RelayTempCleanup(RemoteTempCleanup):
def __init__(self, work_dir: Path, args: argparse.Namespace, normal_radio: RadioSettings,
key: str, password: str, expected_key: str):
if (not re.fullmatch(r"[0-9A-Fa-f]{64}", key)
or key != expected_key or not any(k == key for k, _ in args.relay_values)):
raise OtaError("relay cleanup requires its unchanged bound full public key")
super().__init__(work_dir / f"{RELAY_TEMP_RECOVERY_PREFIX}{key}.json", args,
normal_radio, key, "relay", password)
def _shorten(self, controller: Controller, args: argparse.Namespace) -> bool:
return shorten_remote_temp_window(controller, args, self.key,
password=self.password, label="relay")
def shorten_target_temp_window(controller: Controller, args: argparse.Namespace,
*, password: str | None = None) -> bool:
return shorten_remote_temp_window(controller, args, args.target, password=password)
def prove_owned_temp_cleanups(controller: Controller, args: argparse.Namespace,
target: TargetTempCleanup | None,
relays: dict[str, RelayTempCleanup]) -> None:
"""Attempt each independent normal-return proof even if another fails."""
errors = []
interrupted = None
for state in ([target] if target is not None else []) + list(relays.values()):
try:
state.prove_normal(controller, args)
except KeyboardInterrupt as exc:
interrupted = interrupted or exc
errors.append(f"{state.role}: interrupted; recovery {state.path}")
except (OtaError, OSError) as exc:
errors.append(f"{state.role}: {state.redact_error(exc)}; recovery {state.path}")
if interrupted is not None:
raise interrupted
if errors:
raise OtaError("normal return remains unproven: " + "; ".join(errors))
def shorten_source_temp_window(
@@ -9234,6 +9414,8 @@ def main(
source_rxps_changed = False
source_rxps_recovery_path: Path | None = None
target_temp_owned = False
target_temp_cleanup: TargetTempCleanup | None = None
relay_temp_cleanups: dict[str, RelayTempCleanup] = {}
target_rxps_saved: RxpsSettings | None = None
target_rxps_changed = False
armed_relay_values: list[tuple[str, str]] = []
@@ -9599,6 +9781,14 @@ def main(
print(f"[legacy] relay reboot fallback eligible: "
f"{args.relay_reboot_normal_hops} normal-channel hops")
target_temp_cleanup = TargetTempCleanup(work_dir, args, original_radio, password=password)
for relay_key, relay_password in args.relay_values:
if relay_key in relay_temp_cleanups:
raise OtaError("duplicate bound relay cleanup identity")
relay_temp_cleanups[relay_key] = RelayTempCleanup(
work_dir, args, original_radio, relay_key, relay_password,
relay_public_keys[relay_key],
)
target_temp_armed_at = time.monotonic()
target_temp_owned = True
if args.source_shares_controller:
@@ -9728,11 +9918,11 @@ def main(
"--leave-controller-radio was requested"
)
else:
# Retire one-shot ownership before the write. If its reply is
# lost, reconciliation is read-only and outer cleanup must not
# send the relative duration a second time.
target_temp_owned = False
shorten_target_temp_window(controller, args)
# One-shot write ownership and the outstanding normal-return
# obligation are separate. A missing ACK cannot retire either
# the durable target recovery record or the return proof.
assert target_temp_cleanup is not None
target_temp_cleanup.shorten_once(controller, args)
relays_to_shorten = list(armed_relay_values)
armed_relay_values.clear()
shorten_relay_temp_windows(
@@ -9740,6 +9930,7 @@ def main(
args,
relays_to_shorten,
relay_public_keys,
cleanup_states=relay_temp_cleanups,
)
if target_rxps_changed and target_rxps_saved is not None:
if args.leave_controller_radio:
@@ -9770,6 +9961,10 @@ def main(
controller_changed = False
restore_source_rxps_once("stage-only cleanup")
retire_local_source_rxps_recovery()
if not args.leave_controller_radio:
assert target_temp_cleanup is not None
prove_owned_temp_cleanups(controller, args, target_temp_cleanup, relay_temp_cleanups)
target_temp_owned = False
return 0
install_confirmed = request_install(controller, args, package)
@@ -9796,6 +9991,7 @@ def main(
args,
relays_to_shorten,
relay_public_keys,
cleanup_states=relay_temp_cleanups,
)
# Seeding already stopped before policy restoration and installation.
@@ -9829,6 +10025,9 @@ def main(
target.base_hash,
identity_reply,
)
if args.leave_controller_radio:
assert target_temp_cleanup is not None
prove_owned_temp_cleanups(controller, args, target_temp_cleanup, relay_temp_cleanups)
if target_rxps_changed and target_rxps_saved is not None:
restore_remote_rxps(
controller, args.target, target_rxps_saved
@@ -9842,6 +10041,9 @@ def main(
controller_changed = True
restore_source_rxps_once("successful update cleanup")
retire_local_source_rxps_recovery()
if not args.leave_controller_radio:
assert target_temp_cleanup is not None
prove_owned_temp_cleanups(controller, args, target_temp_cleanup, relay_temp_cleanups)
return 0
except KeyboardInterrupt:
print("\nInterrupted; any partial target download remains resumable.", file=sys.stderr)
@@ -9858,7 +10060,8 @@ def main(
if seeder_attempted and args.source_serial:
source_cli_command(args, "ota folder off", check=False)
needs_remote_cleanup = (
target_temp_owned
(target_temp_owned and target_temp_cleanup is not None
and not target_temp_cleanup.report["shorten_attempted"])
or bool(armed_relay_values)
or bool(relay_timing_settings)
)
@@ -9901,12 +10104,11 @@ def main(
file=sys.stderr,
)
if target_temp_owned and not args.leave_controller_radio:
# This relative cleanup write is one-shot even when its
# acknowledgement is lost; retire ownership first so a
# later finally path cannot extend it again.
target_temp_owned = False
# Persist the one-shot attempt before sending without
# dropping the independent target return obligation.
assert target_temp_cleanup is not None
try:
shorten_target_temp_window(controller, args)
target_temp_cleanup.shorten_once(controller, args)
except (OtaError, OSError) as exc:
print(
"WARNING: could not shorten the destination "
@@ -9925,6 +10127,7 @@ def main(
args,
relays_to_shorten,
relay_public_keys,
cleanup_states=relay_temp_cleanups,
)
except (OtaError, OSError) as exc:
print(
@@ -10022,6 +10225,21 @@ def main(
"after checking the current policy and radio channel",
file=sys.stderr,
)
pending_temp = ([target_temp_cleanup] if target_temp_cleanup is not None else []) + list(relay_temp_cleanups.values())
if any(state.report["cleanup_required"] for state in pending_temp) and not args.leave_controller_radio:
if controller is not None and not controller_changed and not source_temp_owned:
try:
prove_owned_temp_cleanups(controller, args, target_temp_cleanup, relay_temp_cleanups)
target_temp_owned = False
except (OtaError, OSError) as exc:
print(f"CRITICAL: participant normal return is unproven: {exc}", file=sys.stderr)
for state in pending_temp:
if state.report["cleanup_required"]:
print(
f"CRITICAL: restoring the controller does not prove the {state.role} returned. "
"Its original long TempRadio lease may remain active; no relative cleanup setter was replayed. "
f"Retained recovery obligation: {state.path}", file=sys.stderr,
)
if not source_rxps_changed:
try:
retire_local_source_rxps_recovery()
+5 -6
View File
@@ -6543,7 +6543,7 @@ class ReliabilityTests(unittest.TestCase):
[("offline", False), ("online", False)],
)
def test_lost_relay_cleanup_reconciles_identity_without_replay(self) -> None:
def test_lost_relay_cleanup_does_not_claim_return_from_temporary_identity(self) -> None:
key = "AB" * 32
class Controller:
@@ -6565,11 +6565,10 @@ class ReliabilityTests(unittest.TestCase):
relay_values=[("relay", "secret")],
temp_values=(909.95, 250.0, 5, 5, 120),
)
ota.shorten_relay_temp_windows(
controller,
args,
expected_public_keys={"relay": key.lower()},
acknowledged = ota.shorten_relay_temp_windows(
controller, args, expected_public_keys={"relay": key.lower()},
)
self.assertFalse(acknowledged)
relative_writes = [
call for call in controller.calls if call[1].startswith("tempradio ")
]
@@ -6577,7 +6576,7 @@ class ReliabilityTests(unittest.TestCase):
relative_writes,
[("relay", "tempradio 909.95,250,5,5,1", False)],
)
self.assertIn(("relay", "get public.key", False), controller.calls)
self.assertEqual(controller.calls, relative_writes)
def test_managed_relay_timing_is_saved_guarded_and_restored(self) -> None:
class Controller:
+30 -3
View File
@@ -4,6 +4,7 @@ import argparse
import contextlib
from dataclasses import replace
import io
import json
import os
from pathlib import Path
import shutil
@@ -328,10 +329,22 @@ class BootloaderStagingTests(unittest.TestCase):
source_key: {"adv_name": "source", "public_key": source_key},
}]
normal = ota.RadioSettings(910.525, 62.5, 7, 5, False)
controller.get_radio.return_value = normal
current_radio = normal
cleanup_proof_radios = []
def set_radio(settings, _label):
nonlocal current_radio
current_radio = settings
controller.get_radio.side_effect = lambda **_kwargs: current_radio
controller.set_radio.side_effect = set_radio
def remote_command(station, command, **_kwargs):
self.assertEqual(station, destination_key)
if command in ("get public.key", "get tempradio"):
self.assertEqual(current_radio, normal)
cleanup_proof_radios.append(current_radio)
return f"> {destination_key}" if command == "get public.key" else "> off"
replies = {
"ota status": "OTA | no download | target:1234ABCD",
"ota ls": "Updates 1",
@@ -356,7 +369,7 @@ class BootloaderStagingTests(unittest.TestCase):
"require_command", "require_meshcli_version", "require_bootloader_tool_support", "preflight_source_cli",
"ensure_source_clock_gate_safe", "ensure_controller_clock_safe",
"verify_with_motatool", "arm_target_temp_radio",
"switch_controller_to_temp_radio", "source_cli_command",
"source_cli_command",
):
stack.enter_context(mock.patch.object(ota, name))
stack.enter_context(mock.patch.object(ota, "read_source_public_key_bounded", return_value=source_key))
@@ -387,8 +400,22 @@ class BootloaderStagingTests(unittest.TestCase):
rehearsal.assert_called_once()
seeder.start.assert_called_once()
cleanup.assert_called_once()
controller.set_radio.assert_called_once_with(normal, "restore controller radio after staging")
temporary = ota.RadioSettings(*ota.parse_temp_radio(ota.DEFAULT_LAB_TEMP_RADIO)[:4], normal.repeat)
self.assertEqual(controller.set_radio.call_args_list, [
mock.call(temporary, "switch controller to TempRadio"),
mock.call(normal, "restore controller radio after staging"),
])
saved_cleanup = json.loads((root / "work" / ota.TARGET_TEMP_RECOVERY_FILE).read_text())
self.assertEqual(saved_cleanup["target"], destination_key)
self.assertTrue(saved_cleanup["shorten_attempted"])
self.assertTrue(saved_cleanup["normal_return_verified"])
self.assertFalse(saved_cleanup["cleanup_required"])
self.assertEqual([item["command"] for item in saved_cleanup["proof_records"]],
["get public.key", "get tempradio"])
self.assertEqual(cleanup_proof_radios, [normal, normal])
install.assert_not_called()
self.assertFalse(any("install" in call.args[1].split()
for call in controller.remote_command.call_args_list))
self.assertIn(
mock.call(destination_key, f"ota pull {package.manifest_id} flash", retry=False),
controller.remote_command.call_args_list,
+26 -3
View File
@@ -24,8 +24,10 @@ class Stations:
self.overrides = {}
self.seeding = False
self.normal = ota.RadioSettings(910.525, 62.5, 7, 5, False)
self.actual_radio = self.normal
def command(self, name, command, **options):
name = {"a" * 64: "relay0", "b" * 64: "remote"}.get(name, name)
self.calls.append((name, command, options))
if name in self.fail_reads and command == "ota config":
raise ota.TransmissionError("lost config read")
@@ -57,6 +59,7 @@ class Stations:
return self.normal
def set_radio(self, _radio, _label):
self.actual_radio = _radio
self.events.append("controller-restore")
def close(self):
@@ -327,6 +330,8 @@ class OtaHopLifecycleTests(unittest.TestCase):
def phase(name):
def action(*_args, **_kwargs):
stations.events.append(name)
if name == "switch":
stations.actual_radio = ota.RadioSettings(909.5, 500, 5, 5, False)
if failure == name:
if name == "interrupt":
raise KeyboardInterrupt
@@ -348,15 +353,24 @@ class OtaHopLifecycleTests(unittest.TestCase):
raise ota.OtaError("synthetic install failure")
return True
argv = ["release.mota", "remote", "--controller-serial", "/dev/controller",
argv = ["release.mota", "b" * 64, "--controller-serial", "/dev/controller",
"--source-serial", "/dev/source", "--password", "secret",
"--relay", "relay0=relay-secret", "--work-dir", str(Path(directory) / "work"),
"--relay", "a" * 64 + "=relay-secret", "--work-dir", str(Path(directory) / "work"),
"--yes"]
if no_install:
argv.append("--no-install")
if leave:
argv.append("--leave-controller-radio")
with contextlib.ExitStack() as stack:
# This fixture isolates hop lifecycle using bound fixture keys.
# Exact full-key/TempRadio proof has separate transport regressions.
stack.enter_context(mock.patch.object(
ota.RemoteTempCleanup, "prove_normal", autospec=True,
side_effect=lambda cleanup, *_: (
self.assertTrue(stations.normal.matches(stations.actual_radio)),
stations.events.append("normal-proof:" + cleanup.role),
cleanup.report.update(cleanup_required=False, normal_return_verified=True)),
))
for name in ("preflight_inputs", "preflight_source_cli", "bind_contact_selectors",
"verify_shared_source_identity", "ensure_controller_clock_safe",
"restore_relay_timings", "shorten_relay_temp_windows",
@@ -377,7 +391,7 @@ class OtaHopLifecycleTests(unittest.TestCase):
stack.enter_context(mock.patch.object(ota, "arm_relay_temp_radio_once", return_value=True))
stack.enter_context(mock.patch.object(ota, "arm_source_temp_radio_once", return_value=True))
stack.enter_context(mock.patch.object(ota, "switch_controller_to_temp_radio", side_effect=phase("switch")))
stack.enter_context(mock.patch.object(ota, "read_relay_timing", return_value=ota.RelayTimingSettings("relay0", "relay-secret", 0, 0.3)))
stack.enter_context(mock.patch.object(ota, "read_relay_timing", return_value=ota.RelayTimingSettings("a" * 64, "relay-secret", 0, 0.3)))
stack.enter_context(mock.patch.object(ota, "read_remote_public_key_bounded", return_value="a" * 64))
stack.enter_context(mock.patch.object(ota, "shorten_source_temp_window", return_value=True))
stack.enter_context(mock.patch.object(ota, "SeederProcess", return_value=seeder))
@@ -411,6 +425,15 @@ class OtaHopLifecycleTests(unittest.TestCase):
self.assertLess(events.index("set:remote:1"), events.index("arm-target"))
self.assertLess(events.index("seeder-stop"), events.index("set:source:0"))
def test_install_leave_controller_proves_every_participant_while_actually_normal(self):
with tempfile.TemporaryDirectory() as directory:
result,stations=self.run_main(directory,leave=True,no_install=False)
self.assertEqual(result,0)
self.assertIn("normal-proof:destination",stations.events)
self.assertIn("normal-proof:relay",stations.events)
self.assertFalse(stations.normal.matches(stations.actual_radio))
self.assertGreater(stations.events.index("normal-proof:relay"),stations.events.index("install"))
def test_rehearsal_and_confirmation_failures_never_write_hop_settings(self):
for failure in ("confirm", "rehearse"):
with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory:
@@ -0,0 +1,171 @@
"""Offline per-relay lease obligations share the production target proof."""
import argparse
import contextlib
import io
import json
from pathlib import Path
import tempfile
import subprocess
import sys
import types
import unittest
from unittest import mock
import lora_ota as ota
TARGET='ab'*32
RELAYS=['cd'*32,'ef'*32,'12'*32]
NORMAL=ota.RadioSettings(910.525,62.5,7,5,False)
class RelayCleanupTests(unittest.TestCase):
def args(self):
return argparse.Namespace(target=TARGET,temp_radio='909.5,500,7,5,180',
temp_values=(909.5,500,7,5,180),relay_values=[(key,'private-'+str(i)) for i,key in enumerate(RELAYS)])
def state(self,directory,args,key):
return ota.RelayTempCleanup(Path(directory),args,NORMAL,key,dict(args.relay_values)[key],key)
def controller(self,args,modes=None):
modes=modes or {};calls=[]
def remote(key,command,**kwargs):
self.assertIs(kwargs['retry'],False)
if key!=TARGET:self.assertEqual(kwargs['password'],dict(args.relay_values)[key])
calls.append((key,command))
mode=modes.get(key,'off')
if command.startswith('tempradio '):
if mode in ('lost-delivered','lost-undelivered'):raise ota.TransmissionError('lost ACK')
if mode=='error':raise ota.OtaError('failed private-0')
if mode=='interrupt':raise KeyboardInterrupt()
return 'OK - temp params for 1 mins'
if command=='get public.key':return '> '+('34'*32 if mode=='changed-key' else key)
if command=='get tempradio':return '> 909.500,500.000,7,5,32' if mode in ('active','lost-undelivered') else '> off'
raise AssertionError(command)
return types.SimpleNamespace(remote_command=remote,get_radio=lambda **kwargs:NORMAL,calls=calls)
def clock(self):
clock=types.SimpleNamespace(now=100.0)
def sleep(seconds):clock.now+=seconds
return clock,types.SimpleNamespace(monotonic=lambda:clock.now,sleep=sleep)
def test_lost_delivered_relay_ack_resolves_only_from_independent_normal_off(self):
args=self.args();key=RELAYS[0];c=self.controller(args,{key:'lost-delivered'})
with tempfile.TemporaryDirectory() as d,contextlib.redirect_stderr(io.StringIO()):
state=self.state(d,args,key)
self.assertFalse(ota.shorten_relay_temp_windows(c,args,[(key,'private-0')],{key:key},cleanup_states={key:state}))
self.assertTrue(state.report['cleanup_required'])
self.assertFalse(state.report['shorten_acknowledged'])
state.prove_normal(c,args)
self.assertTrue(state.report['normal_return_verified'])
self.assertEqual(c.calls,[(key,'tempradio 909.5,500,7,5,1'),(key,'get public.key'),(key,'get tempradio')])
self.assertNotIn('private-0',state.path.read_text())
def test_lost_undelivered_relay_ack_keeps_its_own_obligation(self):
args=self.args();key=RELAYS[0];c=self.controller(args,{key:'lost-undelivered'});clock,fake=self.clock()
with tempfile.TemporaryDirectory() as d,contextlib.redirect_stderr(io.StringIO()),contextlib.redirect_stdout(io.StringIO()):
state=self.state(d,args,key)
state.shorten_once(c,args)
with mock.patch.object(ota,'time',fake),self.assertRaises(ota.TransmissionError):state.prove_normal(c,args)
saved=json.loads(state.path.read_text())
self.assertTrue(saved['cleanup_required']);self.assertFalse(saved['normal_return_verified'])
self.assertLessEqual(clock.now,190)
state.shorten_once(c,args)
self.assertEqual(sum(cmd.startswith('tempradio ') for _,cmd in c.calls),1)
def test_changed_relay_full_identity_is_terminal_and_never_resolves(self):
args=self.args();key=RELAYS[0];c=self.controller(args,{key:'changed-key'})
with tempfile.TemporaryDirectory() as d:
state=self.state(d,args,key)
with self.assertRaisesRegex(ota.OtaError,'public key changed'):state.prove_normal(c,args)
self.assertEqual(c.calls,[(key,'get public.key')]);self.assertTrue(state.report['cleanup_required'])
def test_target_off_does_not_prove_a_relay_off_and_other_proofs_continue(self):
args=self.args();c=self.controller(args,{RELAYS[0]:'active'});clock,fake=self.clock()
with tempfile.TemporaryDirectory() as d,contextlib.redirect_stdout(io.StringIO()):
target=ota.TargetTempCleanup(Path(d),args,NORMAL)
states={key:self.state(d,args,key) for key in RELAYS[:2]}
with mock.patch.object(ota,'time',fake),self.assertRaisesRegex(ota.OtaError,'normal return remains unproven'):
ota.prove_owned_temp_cleanups(c,args,target,states)
self.assertFalse(target.report['cleanup_required'])
self.assertTrue(states[RELAYS[0]].report['cleanup_required'])
self.assertFalse(states[RELAYS[1]].report['cleanup_required'])
before=list(c.calls)
with self.assertRaises(ota.OtaError):ota.prove_owned_temp_cleanups(c,args,target,states)
self.assertEqual(c.calls,before) # No repeated failed proof nor relative setter.
def test_each_independent_shortening_attempt_survives_errors_and_interrupts_once(self):
args=self.args();modes={RELAYS[0]:'error',RELAYS[1]:'interrupt',RELAYS[2]:'lost-delivered'}
c=self.controller(args,modes)
with tempfile.TemporaryDirectory() as d,contextlib.redirect_stderr(io.StringIO()):
states={key:self.state(d,args,key) for key in RELAYS}
original=c.remote_command
def before_send(key,command,**kwargs):
self.assertTrue(json.loads(states[key].path.read_text())['shorten_attempted'])
return original(key,command,**kwargs)
c.remote_command=before_send
with self.assertRaises(KeyboardInterrupt):ota.shorten_relay_temp_windows(c,args,cleanup_states=states)
self.assertEqual([key for key,_ in c.calls],RELAYS)
self.assertFalse(ota.shorten_relay_temp_windows(c,args,cleanup_states=states))
self.assertEqual(len(c.calls),3)
for key,state in states.items():
self.assertTrue(state.report['shorten_attempted']);self.assertTrue(state.report['cleanup_required'])
self.assertNotIn(dict(args.relay_values)[key],state.path.read_text())
self.assertEqual(state.path.stat().st_mode&0o777,0o600)
def test_missing_or_rebound_ownership_stops_before_any_write(self):
args=self.args();c=self.controller(args)
with tempfile.TemporaryDirectory() as d:
states={RELAYS[0]:self.state(d,args,RELAYS[0])}
with self.assertRaises(ota.OtaError):ota.shorten_relay_temp_windows(c,args,cleanup_states=states)
self.assertEqual(c.calls,[])
with self.assertRaises(ota.OtaError):ota.RelayTempCleanup(Path(d),args,NORMAL,RELAYS[1],'private-1',RELAYS[2])
args.relay_values=[]
with self.assertRaises(ota.OtaError):states[RELAYS[0]].prove_normal(c,args)
self.assertEqual(c.calls,[])
def test_fresh_process_keeps_attempt_and_constructor_cannot_reset_it(self):
args=self.args();key=RELAYS[0];c=self.controller(args,{key:'lost-delivered'})
with tempfile.TemporaryDirectory() as d,contextlib.redirect_stderr(io.StringIO()):
state=self.state(d,args,key);state.shorten_once(c,args)
before=state.path.read_bytes()
probe=subprocess.run([sys.executable,'-B','-c',
'import json,sys;d=json.load(open(sys.argv[1]));assert d["shorten_attempted"] and d["cleanup_required"];assert not d["shorten_acknowledged"]',str(state.path)],
capture_output=True,text=True,timeout=10)
self.assertEqual(probe.returncode,0,probe.stderr)
with self.assertRaisesRegex(ota.OtaError,'must not be overwritten'):self.state(d,args,key)
self.assertEqual(state.path.read_bytes(),before)
self.assertEqual(sum(cmd.startswith('tempradio ') for _,cmd in c.calls),1)
def test_saved_normal_radio_does_not_hide_a_shared_live_temporary_override(self):
args=self.args();args.source_shares_controller=True
c=self.controller(args);clock,fake=self.clock()
with tempfile.TemporaryDirectory() as d,contextlib.redirect_stdout(io.StringIO()):
state=self.state(d,args,RELAYS[0])
with mock.patch.object(ota,'time',fake),mock.patch.object(ota,'read_source_temp_radio_status',return_value=('active',ota.RadioSettings(909.5,500,7,5,False))),self.assertRaises(ota.TransmissionError):
state.prove_normal(c,args)
self.assertTrue(state.report['cleanup_required'])
self.assertEqual(c.calls,[])
self.assertTrue(all(r.get('state')=='active' for r in state.report['proof_records']))
def test_authenticated_reply_capture_never_persists_private_password_text(self):
args=self.args();key=RELAYS[0];c=self.controller(args);original=c.remote_command
def reply(key,command,**kwargs):
value=original(key,command,**kwargs)
return value+'\nprivate-0' if command=='get public.key' else value
c.remote_command=reply
with tempfile.TemporaryDirectory() as d:
state=self.state(d,args,key);state.prove_normal(c,args)
self.assertFalse(state.report['cleanup_required'])
self.assertNotIn('private-0',state.path.read_text())
def test_target_env_or_getpass_secret_is_redacted_without_args_password(self):
for fail in (False,True):
with self.subTest(fail=fail),tempfile.TemporaryDirectory() as d:
args=self.args();args.password=None;secret='resolved-getpass-fixture'
def remote(key,command,**kwargs):
self.assertEqual(kwargs['password'],secret)
if command.startswith('tempradio '):raise ota.TransmissionError('lost '+secret)
if command=='get public.key':return '> '+TARGET+'\n'+secret
if fail:raise ota.OtaError('proof failed '+secret)
return '> off'
c=types.SimpleNamespace(remote_command=remote,get_radio=lambda **kwargs:NORMAL)
state=ota.TargetTempCleanup(Path(d),args,NORMAL,password=secret)
warning=io.StringIO()
with contextlib.redirect_stderr(warning):state.shorten_once(c,args)
self.assertNotIn(secret,warning.getvalue())
if fail:
with self.assertRaises(ota.OtaError):state.prove_normal(c,args)
self.assertTrue(state.report['cleanup_required'])
else:state.prove_normal(c,args)
self.assertNotIn(secret,state.path.read_text())
if __name__=='__main__':unittest.main()
@@ -0,0 +1,176 @@
"""Offline regression tests for uncertain target TempRadio cleanup."""
import argparse
import contextlib
import io
import json
from pathlib import Path
import tempfile
import types
import unittest
from unittest import mock
import lora_ota as ota
import test_lora_ota as fixtures
KEY='ab'*32
NORMAL=ota.RadioSettings(910.525,62.5,7,5,False)
class CleanupTests(unittest.TestCase):
def args(self):
return argparse.Namespace(target=KEY,temp_radio='909.5,500,7,5,180',temp_values=(909.5,500,7,5,180))
def controller(self,mode='off'):
calls=[]
def remote(target,command,**kwargs):
self.assertEqual(target,KEY);self.assertIs(kwargs['retry'],False);calls.append(command)
if command.startswith('tempradio '):
if mode in ('lost','long'):raise ota.TransmissionError('lost cleanup ACK')
return 'OK - temp params for 1 mins'
if command=='get public.key':
return '> '+('cd'*32 if mode=='wrong-key' else KEY)
if command=='get tempradio':
if mode=='long':raise ota.TransmissionError('still only reachable on old temporary tuple')
if mode=='malformed':return '> unexpected'
return '> off'
raise AssertionError(command)
return types.SimpleNamespace(remote_command=remote,get_radio=lambda **kwargs:NORMAL,calls=calls)
def state(self,directory):return ota.TargetTempCleanup(Path(directory),self.args(),NORMAL)
def test_lost_ack_does_not_accept_temporary_identity_as_return(self):
controller=self.controller('lost')
with contextlib.redirect_stderr(io.StringIO()) as warning:
acknowledged=ota.shorten_target_temp_window(controller,self.args())
self.assertFalse(acknowledged)
self.assertEqual(controller.calls,['tempradio 909.5,500,7,5,1'])
self.assertIn('original long lease may remain active',warning.getvalue())
def test_attempt_is_persisted_before_send_and_never_replayed(self):
with tempfile.TemporaryDirectory() as directory:
state=self.state(directory);controller=self.controller('lost');remote=controller.remote_command
def checked(*args,**kwargs):
self.assertTrue(json.loads(state.path.read_text())['shorten_attempted'])
return remote(*args,**kwargs)
controller.remote_command=checked
with contextlib.redirect_stderr(io.StringIO()):state.shorten_once(controller,self.args())
state.shorten_once(controller,self.args())
self.assertEqual(len(controller.calls),1)
self.assertTrue(state.report['cleanup_required'])
self.assertFalse(state.report['shorten_acknowledged'])
self.assertEqual(state.path.stat().st_mode&0o777,0o600)
def test_positive_ack_still_requires_independent_normal_return(self):
with tempfile.TemporaryDirectory() as directory:
state=self.state(directory);controller=self.controller()
state.shorten_once(controller,self.args())
self.assertTrue(state.report['shorten_acknowledged'])
self.assertTrue(state.report['cleanup_required'])
state.prove_normal(controller,self.args())
self.assertFalse(state.report['cleanup_required'])
self.assertTrue(state.report['normal_return_verified'])
self.assertEqual(controller.calls,['tempradio 909.5,500,7,5,1','get public.key','get tempradio'])
def test_lost_ack_can_resolve_only_from_full_key_and_inactive_temp_on_normal(self):
with tempfile.TemporaryDirectory() as directory:
state=self.state(directory);controller=self.controller('lost')
with contextlib.redirect_stderr(io.StringIO()):state.shorten_once(controller,self.args())
state.prove_normal(controller,self.args())
self.assertTrue(state.report['normal_return_verified'])
self.assertFalse(state.report['shorten_acknowledged'])
self.assertEqual(len([x for x in controller.calls if x.startswith('tempradio ')]),1)
def test_wrong_key_and_malformed_status_are_not_retried(self):
for mode,expected_count in [('wrong-key',1),('malformed',2)]:
with self.subTest(mode=mode),tempfile.TemporaryDirectory() as directory:
state=self.state(directory);controller=self.controller(mode)
with self.assertRaises(ota.OtaError):state.prove_normal(controller,self.args())
self.assertEqual(len(controller.calls),expected_count)
self.assertTrue(json.loads(state.path.read_text())['cleanup_required'])
def test_active_profile_is_not_mistaken_for_normal_return(self):
with tempfile.TemporaryDirectory() as directory:
state=self.state(directory);controller=self.controller();remote=controller.remote_command
clock=types.SimpleNamespace(now=100.0)
def sleep(seconds):clock.now+=seconds
fake_time=types.SimpleNamespace(monotonic=lambda:clock.now,sleep=sleep)
def active_until_return(target,command,**kwargs):
if command=='get tempradio' and clock.now<175:
controller.calls.append(command);return '> 909.500,500.000,7,5,32'
return remote(target,command,**kwargs)
controller.remote_command=active_until_return
with mock.patch.object(ota,'time',fake_time),contextlib.redirect_stdout(io.StringIO()):
state.prove_normal(controller,self.args())
self.assertGreaterEqual(clock.now,175)
self.assertTrue(state.report['normal_return_verified'])
self.assertGreater(len(state.report['proof_records']),2)
def test_unresolved_long_lease_keeps_obligation_and_bounded_reads(self):
with tempfile.TemporaryDirectory() as directory:
state=self.state(directory);controller=self.controller('long');clock=types.SimpleNamespace(now=100.0)
def sleep(seconds):clock.now+=seconds
fake_time=types.SimpleNamespace(monotonic=lambda:clock.now,sleep=sleep)
remote=controller.remote_command
def bounded_loss(target,command,**kwargs):
if command=='get tempradio':clock.now+=min(20,kwargs['operation_timeout'])
return remote(target,command,**kwargs)
controller.remote_command=bounded_loss
with mock.patch.object(ota,'time',fake_time),contextlib.redirect_stdout(io.StringIO()),self.assertRaises(ota.TransmissionError):
state.prove_normal(controller,self.args())
saved=json.loads(state.path.read_text())
self.assertTrue(saved['cleanup_required']);self.assertFalse(saved['normal_return_verified'])
self.assertLessEqual(clock.now,190)
self.assertTrue(all(r['operation_timeout']<=190-r['started_at'] for r in saved['proof_records']))
self.assertFalse(any(x.startswith('tempradio ') for x in controller.calls))
def test_wrong_controller_channel_stops_before_target_read(self):
with tempfile.TemporaryDirectory() as directory:
state=self.state(directory);controller=self.controller()
controller.get_radio=lambda **kwargs:ota.RadioSettings(909.5,500,7,5,False)
with self.assertRaises(ota.OtaError):state.prove_normal(controller,self.args())
self.assertEqual(controller.calls,[])
self.assertTrue(state.report['cleanup_required'])
def test_failed_main_keeps_obligation_after_lost_cleanup_ack(self):
controller=mock.Mock()
controller.get_radio.return_value=NORMAL
calls=[];clock=types.SimpleNamespace(now=100.0)
def remote(target,command,**kwargs):
self.assertEqual(target,KEY);self.assertIs(kwargs['retry'],False)
calls.append(command)
if command.startswith('tempradio '):
raise ota.TransmissionError('lost one-minute cleanup ACK')
self.assertEqual(command,'get public.key')
clock.now+=min(20,kwargs['operation_timeout'])
raise ota.TransmissionError('target remains on the original long lease')
controller.remote_command.side_effect=remote
def sleep(seconds):clock.now+=seconds
fake_time=types.SimpleNamespace(monotonic=lambda:clock.now,sleep=sleep,time=lambda:1800000000)
image=fixtures.firmware(b'cleanup main regression'*300,fixtures.VERSION_NEW)
package=ota.parse_mota(fixtures.mota_blob(image));seeder=mock.Mock()
with tempfile.TemporaryDirectory() as directory:
work=Path(directory)/'work'
argv=['release.mota',KEY,'--controller-serial','/dev/controller',
'--source-serial','/dev/source','--password','test-only',
'--work-dir',str(work),'--yes']
error=io.StringIO()
with contextlib.ExitStack() as stack:
for name in ('preflight_inputs','preflight_source_cli','bind_contact_selectors',
'ensure_source_clock_gate_safe','ensure_controller_clock_safe',
'verify_shared_source_identity','confirm_update','run_temp_radio_preflight',
'arm_target_temp_radio','switch_controller_to_temp_radio',
'find_and_start_pull'):
stack.enter_context(mock.patch.object(ota,name))
stack.enter_context(mock.patch.object(ota,'time',fake_time))
stack.enter_context(mock.patch.object(ota,'read_source_rxps',return_value=ota.RxpsSettings(False,0,0)))
stack.enter_context(mock.patch.object(ota,'read_remote_rxps',return_value=ota.RxpsSettings(False,0,0)))
stack.enter_context(mock.patch.object(ota,'disable_source_rxps',return_value=False))
stack.enter_context(mock.patch.object(ota,'read_lora_ota_participant_versions',return_value={}))
stack.enter_context(mock.patch.object(ota,'prepare_ota_hop_settings',return_value=[]))
stack.enter_context(mock.patch.object(ota,'query_target',return_value=fixtures.target()))
stack.enter_context(mock.patch.object(ota,'prepare_package',return_value=(Path('release.mota'),package,None)))
stack.enter_context(mock.patch.object(ota,'arm_source_temp_radio_once',return_value=True))
stack.enter_context(mock.patch.object(ota,'shorten_source_temp_window',return_value=True))
stack.enter_context(mock.patch.object(ota,'source_cli_command',return_value='OK'))
stack.enter_context(mock.patch.object(ota,'SeederProcess',return_value=seeder))
stack.enter_context(mock.patch.object(ota,'monitor_download',side_effect=ota.OtaError('synthetic transfer failure')))
stack.enter_context(contextlib.redirect_stdout(io.StringIO()))
stack.enter_context(contextlib.redirect_stderr(error))
result=ota.main(argv,controller_override=controller)
saved=json.loads((work/ota.TARGET_TEMP_RECOVERY_FILE).read_text())
self.assertEqual(result,2)
self.assertEqual([c for c in calls if c.startswith('tempradio ')],['tempradio 909.5,500,5,5,1'])
self.assertTrue(saved['cleanup_required']);self.assertTrue(saved['shorten_attempted'])
self.assertFalse(saved['shorten_acknowledged']);self.assertFalse(saved['normal_return_verified'])
self.assertNotIn('ota self',calls)
self.assertIn('restoring the controller does not prove the destination returned',error.getvalue())
if __name__=='__main__':unittest.main()