1767 lines
71 KiB
Python
1767 lines
71 KiB
Python
#!/usr/bin/env python3
|
|
"""Behavioral tests for the T1000-E ExtraFS host stress harness."""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import importlib.util
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
import sys
|
|
import unittest
|
|
from unittest.mock import AsyncMock
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
HARNESS_PATH = ROOT / "tools" / "hil" / "t1000e_extrafs_stress.py"
|
|
SPEC = importlib.util.spec_from_file_location("t1000e_extrafs_stress", HARNESS_PATH)
|
|
assert SPEC is not None and SPEC.loader is not None
|
|
HIL = importlib.util.module_from_spec(SPEC)
|
|
sys.modules[SPEC.name] = HIL
|
|
SPEC.loader.exec_module(HIL)
|
|
|
|
|
|
def key(byte: int) -> str:
|
|
return (bytes([byte]) * 32).hex()
|
|
|
|
|
|
def event(name: str, payload=None):
|
|
return SimpleNamespace(
|
|
type=SimpleNamespace(name=name), payload={} if payload is None else payload
|
|
)
|
|
|
|
|
|
class FakeCommands:
|
|
def __init__(self, owner, responses):
|
|
self.owner = owner
|
|
self.responses = list(responses)
|
|
self.calls = []
|
|
|
|
async def get_contacts(self, *, lastmod, timeout):
|
|
self.calls.append((lastmod, timeout))
|
|
contacts, advertised = self.responses.pop(0)
|
|
self.owner._reader.contacts = dict(contacts)
|
|
if advertised is not None:
|
|
self.owner._reader.contact_nb = advertised
|
|
return SimpleNamespace(
|
|
type=SimpleNamespace(name="CONTACTS"), payload=dict(contacts)
|
|
)
|
|
|
|
|
|
class FakeMeshCore:
|
|
def __init__(self, responses):
|
|
self._reader = SimpleNamespace(contact_nb=999, contacts={"stale": {}})
|
|
self.commands = FakeCommands(self, responses)
|
|
|
|
|
|
def runner(responses, *, passes=None, digest=None):
|
|
config = SimpleNamespace(
|
|
contact_enumeration_passes=passes,
|
|
expected_contact_keyset_sha256=digest,
|
|
verbose=False,
|
|
)
|
|
identity = SimpleNamespace(serial_number=HIL.EXPECTED_USB_SERIAL)
|
|
result = HIL.StressRunner(config, identity)
|
|
result.mc = FakeMeshCore(responses)
|
|
return result
|
|
|
|
|
|
class ContactEnumerationHarnessTests(unittest.IsolatedAsyncioTestCase):
|
|
async def test_safe_snapshot_retries_a_short_stream(self):
|
|
expected = {key(1): {}, key(2): {}}
|
|
subject = runner([({key(1): {}}, 2), (expected, 2)])
|
|
|
|
actual = await subject.contacts()
|
|
|
|
self.assertEqual(set(actual), set(expected))
|
|
self.assertEqual(subject.contact_enumeration_transactions, 2)
|
|
self.assertEqual(subject.contact_enumeration_short_attempts, 1)
|
|
self.assertEqual(subject.contact_enumeration_retries, 1)
|
|
self.assertEqual(subject.mc.commands.calls, [(0, 20), (0, 20)])
|
|
|
|
async def test_safe_snapshot_never_returns_three_short_streams(self):
|
|
subject = runner([({key(1): {}}, 2)] * 3)
|
|
|
|
with self.assertRaisesRegex(
|
|
HIL.HilFailure, r"incomplete contact enumeration; attempt 1:"
|
|
):
|
|
await subject.contacts()
|
|
|
|
self.assertEqual(subject.contact_enumeration_transactions, 3)
|
|
self.assertEqual(subject.contact_enumeration_short_attempts, 3)
|
|
self.assertEqual(subject.contact_enumeration_retries, 2)
|
|
|
|
async def test_missing_start_count_cannot_reuse_stale_reader_state(self):
|
|
subject = runner([({key(1): {}}, None)])
|
|
|
|
with self.assertRaisesRegex(HIL.HilFailure, "valid CONTACTS_START count"):
|
|
await subject.contacts()
|
|
|
|
self.assertIsNone(subject.mc._reader.contact_nb)
|
|
|
|
async def test_repeated_stress_accepts_only_the_same_complete_keyset(self):
|
|
expected = {key(1), key(2), key(3)}
|
|
payload = {item: {} for item in expected}
|
|
subject = runner([(payload, 3)] * 4, passes=4)
|
|
|
|
actual = await subject.stress_contact_enumerations(
|
|
expected, "unit-test table"
|
|
)
|
|
|
|
self.assertEqual(set(actual), expected)
|
|
self.assertEqual(len(subject.mc.commands.calls), 4)
|
|
step = subject.steps[-1]
|
|
self.assertEqual(step["step"], "contact_enumeration_stress")
|
|
self.assertTrue(step["ok"])
|
|
self.assertEqual(step["completed_passes"], 4)
|
|
self.assertEqual(step["transport_transactions"], 4)
|
|
self.assertEqual(step["short_attempts"], 0)
|
|
self.assertEqual(step["retry_attempts"], 0)
|
|
self.assertEqual(step["advertised_counts"], [3, 3, 3, 3])
|
|
self.assertEqual(step["received_counts"], [3, 3, 3, 3])
|
|
|
|
async def test_stress_reports_retried_short_stream_and_fails(self):
|
|
expected = {key(1), key(2)}
|
|
complete = {item: {} for item in expected}
|
|
subject = runner(
|
|
[({key(1): {}}, 2), (complete, 2), (complete, 2)], passes=2
|
|
)
|
|
|
|
with self.assertRaisesRegex(
|
|
HIL.HilFailure,
|
|
r"observed 1 incomplete contact stream\(s\).*1 retry attempt",
|
|
):
|
|
await subject.stress_contact_enumerations(expected, "unit-test table")
|
|
|
|
step = subject.steps[-1]
|
|
self.assertFalse(step["ok"])
|
|
self.assertEqual(step["completed_passes"], 2)
|
|
self.assertEqual(step["transport_transactions"], 3)
|
|
self.assertEqual(step["short_attempts"], 1)
|
|
self.assertEqual(step["retry_attempts"], 1)
|
|
self.assertEqual(step["advertised_counts"], [2, 2])
|
|
self.assertEqual(step["received_counts"], [2, 2])
|
|
|
|
async def test_same_count_wrong_keyset_fails_on_exact_pass(self):
|
|
expected = {key(1), key(2)}
|
|
wrong = {key(1): {}, key(3): {}}
|
|
subject = runner([(wrong, 2)], passes=1)
|
|
|
|
with self.assertRaisesRegex(
|
|
HIL.HilFailure, r"pass 1/1: contact key set mismatch"
|
|
):
|
|
await subject.stress_contact_enumerations(expected, "unit-test table")
|
|
|
|
self.assertEqual(subject.steps[-1]["completed_passes"], 0)
|
|
self.assertEqual(len(subject.mc.commands.calls), 1)
|
|
|
|
async def test_expected_keyset_digest_anchors_read_only_stress(self):
|
|
expected = {key(1), key(2)}
|
|
payload = {item: {} for item in expected}
|
|
subject = runner([(payload, 2)], passes=1, digest="0" * 64)
|
|
|
|
with self.assertRaisesRegex(HIL.HilFailure, "keyset digest is"):
|
|
await subject.stress_contact_enumerations(None, "unit-test table")
|
|
|
|
self.assertFalse(subject.steps[-1]["ok"])
|
|
|
|
|
|
class ContactEnumerationArgumentTests(unittest.TestCase):
|
|
def test_new_corruption_option_is_backward_compatible_and_off_by_default(self):
|
|
legacy_positional = HIL.RunConfig(
|
|
"COM23",
|
|
"inspect",
|
|
False,
|
|
None,
|
|
None,
|
|
350,
|
|
131072,
|
|
39,
|
|
7,
|
|
35,
|
|
False,
|
|
False,
|
|
None,
|
|
False,
|
|
None,
|
|
None,
|
|
False,
|
|
)
|
|
self.assertIsNone(legacy_positional.corrupt_occupied_page)
|
|
self.assertFalse(legacy_positional.advert_remove_rollback)
|
|
self.assertIsNone(legacy_positional.fail_read_page)
|
|
self.assertIsNone(legacy_positional.fail_stat_page)
|
|
|
|
args = HIL.build_argument_parser().parse_args(["--port", "COM23"])
|
|
del args.corrupt_occupied_page
|
|
del args.advert_remove_rollback
|
|
del args.fail_read_page
|
|
del args.fail_stat_page
|
|
config = HIL.config_from_args(args)
|
|
self.assertIsNone(config.corrupt_occupied_page)
|
|
self.assertFalse(config.advert_remove_rollback)
|
|
self.assertIsNone(config.fail_read_page)
|
|
self.assertIsNone(config.fail_stat_page)
|
|
|
|
def test_parser_accepts_read_only_stress_without_destructive_consent(self):
|
|
args = HIL.build_argument_parser().parse_args(
|
|
[
|
|
"--port",
|
|
"COM23",
|
|
"--contact-enumeration-passes",
|
|
"25",
|
|
"--expected-contact-keyset-sha256",
|
|
"AB" * 32,
|
|
]
|
|
)
|
|
|
|
config = HIL.config_from_args(args)
|
|
|
|
self.assertEqual(config.scenario, "inspect")
|
|
self.assertEqual(config.contact_enumeration_passes, 25)
|
|
self.assertEqual(config.expected_contact_keyset_sha256, "ab" * 32)
|
|
HIL.require_destructive_consent(config)
|
|
|
|
def test_digest_requires_enumeration_option(self):
|
|
args = HIL.build_argument_parser().parse_args(
|
|
[
|
|
"--port",
|
|
"COM23",
|
|
"--expected-contact-keyset-sha256",
|
|
"ab" * 32,
|
|
]
|
|
)
|
|
|
|
with self.assertRaisesRegex(
|
|
HIL.HilFailure, "requires --contact-enumeration-passes"
|
|
):
|
|
HIL.config_from_args(args)
|
|
|
|
def test_runtime_port_gate_still_rejects_wrong_physical_identity(self):
|
|
wrong = SimpleNamespace(
|
|
device="COM23",
|
|
serial_number="NOT-THE-SPARE",
|
|
vid=HIL.EXPECTED_USB_VID,
|
|
pid=HIL.EXPECTED_USB_PID,
|
|
description="T1000-E",
|
|
product="T1000-E",
|
|
interface="MI_00",
|
|
location="1-8:x.0",
|
|
hwid="USB MI_00",
|
|
)
|
|
|
|
with self.assertRaisesRegex(HIL.HilFailure, "expected the locked spare"):
|
|
HIL.resolve_initial_port("COM23", port_source=lambda: [wrong])
|
|
|
|
def test_occupied_mask_corruption_is_comprehensive_only_and_consent_gated(self):
|
|
parser = HIL.build_argument_parser()
|
|
inspect_args = parser.parse_args(
|
|
["--port", "COM23", "--corrupt-occupied-page", "13"]
|
|
)
|
|
with self.assertRaisesRegex(HIL.HilFailure, "require --scenario"):
|
|
HIL.config_from_args(inspect_args)
|
|
|
|
comprehensive_args = parser.parse_args(
|
|
[
|
|
"--port",
|
|
"COM23",
|
|
"--scenario",
|
|
"comprehensive",
|
|
"--corrupt-occupied-page",
|
|
"13",
|
|
"--allow-existing-contacts",
|
|
]
|
|
)
|
|
config = HIL.config_from_args(comprehensive_args)
|
|
self.assertEqual(config.corrupt_occupied_page, 13)
|
|
with self.assertRaisesRegex(HIL.HilFailure, "--allow-destructive"):
|
|
HIL.require_destructive_consent(config)
|
|
|
|
def test_restorative_page_corruption_accepts_reviewed_existing_inventory(self):
|
|
parser = HIL.build_argument_parser()
|
|
args = parser.parse_args(
|
|
[
|
|
"--port",
|
|
"COM23",
|
|
"--scenario",
|
|
"comprehensive",
|
|
"--corrupt-page",
|
|
"7",
|
|
"--allow-existing-contacts",
|
|
"--allow-destructive",
|
|
"--confirm-usb-serial",
|
|
HIL.EXPECTED_USB_SERIAL,
|
|
]
|
|
)
|
|
|
|
config = HIL.config_from_args(args)
|
|
|
|
self.assertEqual(config.corrupt_page, 7)
|
|
self.assertTrue(config.allow_existing_contacts)
|
|
HIL.require_destructive_consent(config)
|
|
|
|
def test_contact_page_mask_parser_is_strict(self):
|
|
self.assertEqual(
|
|
HIL.parse_contact_page_mask(
|
|
"HIL contact page 13 occupied=81Ff00a5"
|
|
),
|
|
HIL.ContactPageMask(13, 0x81FF00A5),
|
|
)
|
|
for malformed in (
|
|
"HIL contact page 14 occupied=01ffffff",
|
|
"HIL contact page 0 occupied=1ffffff",
|
|
"HIL contact page 0 mask=01ffffff",
|
|
):
|
|
with self.subTest(malformed=malformed):
|
|
with self.assertRaises(HIL.HilFailure):
|
|
HIL.parse_contact_page_mask(malformed)
|
|
|
|
def test_advert_remove_rollback_is_opt_in_comprehensive_and_gated(self):
|
|
parser = HIL.build_argument_parser()
|
|
inspect_args = parser.parse_args(
|
|
["--port", "COM23", "--advert-remove-rollback"]
|
|
)
|
|
with self.assertRaisesRegex(HIL.HilFailure, "require --scenario"):
|
|
HIL.config_from_args(inspect_args)
|
|
|
|
args = parser.parse_args(
|
|
[
|
|
"--port",
|
|
"COM23",
|
|
"--scenario",
|
|
"comprehensive",
|
|
"--advert-remove-rollback",
|
|
]
|
|
)
|
|
config = HIL.config_from_args(args)
|
|
self.assertTrue(config.advert_remove_rollback)
|
|
with self.assertRaisesRegex(HIL.HilFailure, "--allow-destructive"):
|
|
HIL.require_destructive_consent(config)
|
|
|
|
def test_contact_slot_and_advert_seed_parsers_are_strict(self):
|
|
self.assertEqual(HIL.parse_contact_slot("HIL contact slot=349"), 349)
|
|
self.assertEqual(HIL.parse_advert_seed_slot("HIL advert seeded slot=0"), 0)
|
|
for parser, malformed in (
|
|
(HIL.parse_contact_slot, "HIL contact slot=350"),
|
|
(HIL.parse_contact_slot, "HIL contact slot=-1"),
|
|
(HIL.parse_advert_seed_slot, "HIL advert seed failed"),
|
|
(HIL.parse_advert_seed_slot, "HIL advert seeded slot=999"),
|
|
):
|
|
with self.subTest(malformed=malformed):
|
|
with self.assertRaises(HIL.HilFailure):
|
|
parser(malformed)
|
|
|
|
def test_fail_read_page_is_bounded_comprehensive_only_and_gated(self):
|
|
parser = HIL.build_argument_parser()
|
|
inspect_args = parser.parse_args(
|
|
["--port", "COM23", "--fail-read-page", "13"]
|
|
)
|
|
with self.assertRaisesRegex(HIL.HilFailure, "require --scenario"):
|
|
HIL.config_from_args(inspect_args)
|
|
|
|
with self.assertRaisesRegex(argparse.ArgumentTypeError, "in 0..13"):
|
|
HIL._nonnegative_page("14")
|
|
|
|
args = parser.parse_args(
|
|
[
|
|
"--port",
|
|
"COM23",
|
|
"--scenario",
|
|
"comprehensive",
|
|
"--fail-read-page",
|
|
"0",
|
|
]
|
|
)
|
|
config = HIL.config_from_args(args)
|
|
self.assertEqual(config.fail_read_page, 0)
|
|
with self.assertRaisesRegex(HIL.HilFailure, "--allow-destructive"):
|
|
HIL.require_destructive_consent(config)
|
|
|
|
def test_fail_stat_page_is_bounded_exclusive_and_gated(self):
|
|
parser = HIL.build_argument_parser()
|
|
inspect_args = parser.parse_args(
|
|
["--port", "COM23", "--fail-stat-page", "13"]
|
|
)
|
|
with self.assertRaisesRegex(HIL.HilFailure, "require --scenario"):
|
|
HIL.config_from_args(inspect_args)
|
|
|
|
args = parser.parse_args(
|
|
[
|
|
"--port",
|
|
"COM23",
|
|
"--scenario",
|
|
"comprehensive",
|
|
"--fail-stat-page",
|
|
"0",
|
|
]
|
|
)
|
|
config = HIL.config_from_args(args)
|
|
self.assertEqual(config.fail_stat_page, 0)
|
|
with self.assertRaisesRegex(HIL.HilFailure, "--allow-destructive"):
|
|
HIL.require_destructive_consent(config)
|
|
|
|
both = parser.parse_args(
|
|
[
|
|
"--port",
|
|
"COM23",
|
|
"--scenario",
|
|
"comprehensive",
|
|
"--fail-read-page",
|
|
"0",
|
|
"--fail-stat-page",
|
|
"1",
|
|
]
|
|
)
|
|
with self.assertRaisesRegex(HIL.HilFailure, "mutually exclusive"):
|
|
HIL.config_from_args(both)
|
|
|
|
|
|
class ExactContactFrameTests(unittest.TestCase):
|
|
def test_exact_frame_preserves_lastmod_and_all_contact_fields(self):
|
|
contact = HIL.make_contact(7)
|
|
contact.update(
|
|
{
|
|
"flags": 5,
|
|
"out_path_hash_mode": 1,
|
|
"out_path_len": 2,
|
|
"out_path": "00112233",
|
|
"adv_name": "HIL-EXACT",
|
|
"adv_lat": 12.345678,
|
|
"adv_lon": -87.654321,
|
|
"lastmod": 1_800_000_007,
|
|
}
|
|
)
|
|
|
|
frame = HIL.encode_exact_contact_frame(contact)
|
|
|
|
self.assertEqual(len(frame), 148)
|
|
self.assertEqual(frame[0], 0x09)
|
|
self.assertEqual(frame[1:33], bytes.fromhex(contact["public_key"]))
|
|
self.assertEqual(frame[33:36], bytes((1, 5, 0x42)))
|
|
self.assertEqual(frame[36:40], bytes.fromhex("00112233"))
|
|
self.assertEqual(frame[40:100], bytes(60))
|
|
self.assertEqual(frame[100:109], b"HIL-EXACT")
|
|
self.assertEqual(frame[109:132], bytes(23))
|
|
self.assertEqual(
|
|
int.from_bytes(frame[132:136], "little"), contact["last_advert"]
|
|
)
|
|
self.assertEqual(
|
|
int.from_bytes(frame[136:140], "little", signed=True), 12_345_678
|
|
)
|
|
self.assertEqual(
|
|
int.from_bytes(frame[140:144], "little", signed=True), -87_654_321
|
|
)
|
|
self.assertEqual(int.from_bytes(frame[144:148], "little"), 1_800_000_007)
|
|
|
|
def test_exact_frame_requires_original_lastmod(self):
|
|
with self.assertRaisesRegex(HIL.HilFailure, "lastmod"):
|
|
HIL.encode_exact_contact_frame(HIL.make_contact(7))
|
|
|
|
|
|
class DestructiveScenarioTests(unittest.IsolatedAsyncioTestCase):
|
|
def make_runner(self):
|
|
config = SimpleNamespace(
|
|
contact_count=350,
|
|
contact_enumeration_passes=None,
|
|
expected_contact_keyset_sha256=None,
|
|
settle_seconds=0.001,
|
|
corrupt_occupied_page=0,
|
|
allow_existing_contacts=True,
|
|
verbose=False,
|
|
)
|
|
identity = SimpleNamespace(serial_number=HIL.EXPECTED_USB_SERIAL)
|
|
return HIL.StressRunner(config, identity)
|
|
|
|
@staticmethod
|
|
def full_contacts():
|
|
contacts = [HIL.make_contact(index) for index in range(350)]
|
|
for index, contact in enumerate(contacts):
|
|
contact["lastmod"] = 1_800_000_000 + index
|
|
return {contact["public_key"]: dict(contact) for contact in contacts}
|
|
|
|
async def test_occupied_mask_scenario_repairs_populated_empty_and_high_bits(self):
|
|
subject = self.make_runner()
|
|
contacts = self.full_contacts()
|
|
target_key = HIL.make_contact(0)["public_key"]
|
|
without_target = {
|
|
key_value: value
|
|
for key_value, value in contacts.items()
|
|
if key_value != target_key
|
|
}
|
|
subject.contacts = AsyncMock(
|
|
side_effect=[
|
|
contacts,
|
|
contacts,
|
|
contacts,
|
|
contacts,
|
|
without_target,
|
|
without_target,
|
|
without_target,
|
|
without_target,
|
|
contacts,
|
|
]
|
|
)
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
subject.cli = AsyncMock(
|
|
side_effect=[
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
"HIL contact page 0 occupancy bit 0 corrupted",
|
|
"HIL contact page 0 occupied=01fffffe",
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
"HIL contact page 0 occupancy bit 31 corrupted",
|
|
"HIL contact page 0 occupied=81ffffff",
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
"HIL contact page 0 occupied=01fffffe",
|
|
"HIL contact page 0 occupancy bit 0 corrupted",
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
"HIL contact page 0 occupied=01fffffe",
|
|
"HIL contact page 0 occupied=01fffffe",
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
]
|
|
)
|
|
commands = SimpleNamespace(
|
|
remove_contact=AsyncMock(return_value=event("OK")),
|
|
add_contact=AsyncMock(return_value=event("OK")),
|
|
update_contact=AsyncMock(return_value=event("OK")),
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
await subject.test_contact_occupied_mask_recovery(0, contacts)
|
|
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 8)
|
|
self.assertEqual(subject.contacts.await_count, 9)
|
|
commands.remove_contact.assert_awaited_once_with(target_key)
|
|
commands.add_contact.assert_awaited_once_with(contacts[target_key])
|
|
commands.update_contact.assert_not_awaited()
|
|
self.assertEqual(
|
|
[call.args[0] for call in subject.cli.await_args_list],
|
|
[
|
|
"hil extrafs page-mask 0",
|
|
"hil extrafs corrupt-occupied 0 0 CONFIRM",
|
|
"hil extrafs page-mask 0",
|
|
"hil extrafs page-mask 0",
|
|
"hil extrafs page-mask 0",
|
|
"hil extrafs corrupt-occupied 0 31 CONFIRM",
|
|
"hil extrafs page-mask 0",
|
|
"hil extrafs page-mask 0",
|
|
"hil extrafs page-mask 0",
|
|
"hil extrafs page-mask 0",
|
|
"hil extrafs corrupt-occupied 0 0 CONFIRM",
|
|
"hil extrafs page-mask 0",
|
|
"hil extrafs page-mask 0",
|
|
"hil extrafs page-mask 0",
|
|
"hil extrafs page-mask 0",
|
|
],
|
|
)
|
|
step = subject.steps[-1]
|
|
self.assertEqual(step["step"], "contact_occupied_mask_recovery")
|
|
self.assertEqual(step["contacts_preserved"], 350)
|
|
self.assertEqual(
|
|
[case["corrupted_mask"] for case in step["cases"]],
|
|
["01fffffe", "81ffffff", "01ffffff"],
|
|
)
|
|
self.assertEqual(
|
|
[case["durable_mask"] for case in step["cases"]],
|
|
["01ffffff", "01ffffff", "01fffffe"],
|
|
)
|
|
empty_case = step["cases"][-1]
|
|
self.assertEqual(empty_case["kind"], "valid empty")
|
|
self.assertEqual(empty_case["contact_count"], 349)
|
|
self.assertEqual(empty_case["restored_contact_count"], 350)
|
|
self.assertEqual(empty_case["restored_mask"], "01ffffff")
|
|
self.assertTrue(step["original_contact_restored"])
|
|
|
|
async def test_crc_restore_adds_missing_page_in_original_slot_order(self):
|
|
subject = self.make_runner()
|
|
baseline = self.full_contacts()
|
|
baseline_order = list(baseline)
|
|
expected_slots = {
|
|
public_key: slot for slot, public_key in enumerate(baseline_order)
|
|
}
|
|
page = 7
|
|
first = page * HIL.EXPECTED_CONTACTS_PER_PAGE
|
|
lost = set(baseline_order[first:first + HIL.EXPECTED_CONTACTS_PER_PAGE])
|
|
surviving = {
|
|
public_key: value
|
|
for public_key, value in baseline.items()
|
|
if public_key not in lost
|
|
}
|
|
subject.contacts = AsyncMock(side_effect=[baseline, baseline])
|
|
subject.assert_contact_slot_map = AsyncMock()
|
|
subject.set_contact_exact = AsyncMock(return_value=event("OK"))
|
|
subject.contact_slot = AsyncMock(
|
|
side_effect=list(
|
|
range(first, first + HIL.EXPECTED_CONTACTS_PER_PAGE)
|
|
)
|
|
)
|
|
subject.contact_page_mask = AsyncMock(
|
|
return_value=HIL.EXPECTED_FULL_PAGE_MASK
|
|
)
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
|
|
restored, digests = await subject.restore_contact_inventory_exact(
|
|
baseline,
|
|
baseline_order,
|
|
expected_slots,
|
|
page,
|
|
current=surviving,
|
|
context="unit CRC restore",
|
|
)
|
|
|
|
self.assertEqual(restored, baseline)
|
|
self.assertEqual(digests, subject.exact_inventory_digests(baseline))
|
|
self.assertEqual(
|
|
[
|
|
call.args[0]["public_key"]
|
|
for call in subject.set_contact_exact.await_args_list
|
|
],
|
|
baseline_order[first:first + HIL.EXPECTED_CONTACTS_PER_PAGE],
|
|
)
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 2)
|
|
self.assertEqual(subject.contacts.await_count, 2)
|
|
self.assertEqual(subject.assert_contact_slot_map.await_count, 2)
|
|
self.assertEqual(subject.contact_page_mask.await_count, 3)
|
|
|
|
async def test_crc_page_corruption_restores_exact_inventory_and_slots(self):
|
|
subject = self.make_runner()
|
|
subject.config.expected_contact_keyset_sha256 = None
|
|
baseline = self.full_contacts()
|
|
baseline_order = list(baseline)
|
|
expected_slots = {
|
|
public_key: slot for slot, public_key in enumerate(baseline_order)
|
|
}
|
|
page = 7
|
|
first = page * HIL.EXPECTED_CONTACTS_PER_PAGE
|
|
lost_order = baseline_order[
|
|
first:first + HIL.EXPECTED_CONTACTS_PER_PAGE
|
|
]
|
|
surviving = {
|
|
public_key: value
|
|
for public_key, value in baseline.items()
|
|
if public_key not in set(lost_order)
|
|
}
|
|
status = HIL.ExtraFsStatus(True, 60, 100, 0)
|
|
digests = subject.exact_inventory_digests(baseline)
|
|
subject.clear_filler = AsyncMock()
|
|
subject.status = AsyncMock(side_effect=[status, status])
|
|
subject.contact_page_mask = AsyncMock(
|
|
return_value=HIL.EXPECTED_FULL_PAGE_MASK
|
|
)
|
|
subject.capture_contact_slot_map = AsyncMock(
|
|
return_value=expected_slots
|
|
)
|
|
subject.cli = AsyncMock(
|
|
return_value=f"HIL contact page {page} CRC corrupted"
|
|
)
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
subject.contacts = AsyncMock(return_value=surviving)
|
|
subject.expect_contact_not_found = AsyncMock()
|
|
subject.restore_contact_inventory_exact = AsyncMock(
|
|
return_value=(baseline, digests)
|
|
)
|
|
|
|
result = await subject.corrupt_contact_page(page, baseline)
|
|
|
|
self.assertEqual(result, set(baseline))
|
|
subject.cli.assert_awaited_once_with(
|
|
f"hil extrafs corrupt-page {page} CONFIRM"
|
|
)
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 1)
|
|
self.assertEqual(subject.expect_contact_not_found.await_count, 25)
|
|
self.assertEqual(
|
|
[call.args[0] for call in subject.expect_contact_not_found.await_args_list],
|
|
lost_order,
|
|
)
|
|
restore_call = subject.restore_contact_inventory_exact.await_args
|
|
self.assertEqual(restore_call.args[0], baseline)
|
|
self.assertEqual(restore_call.args[1], baseline_order)
|
|
self.assertEqual(restore_call.args[2], expected_slots)
|
|
self.assertEqual(restore_call.args[3], page)
|
|
self.assertEqual(restore_call.kwargs["current"], surviving)
|
|
step = subject.steps[-1]
|
|
self.assertEqual(step["step"], "contact_page_crc_loss_and_restore")
|
|
self.assertEqual(step["isolated_after"], 325)
|
|
self.assertEqual(step["lost_slots"], list(range(175, 200)))
|
|
self.assertTrue(step["slots_restored"])
|
|
self.assertTrue(step["stream_order_restored"])
|
|
self.assertEqual(step["semantic_sha256"], digests["semantic_sha256"])
|
|
self.assertEqual(step["wire_sha256"], digests["wire_sha256"])
|
|
|
|
async def test_crc_page_failure_runs_emergency_exact_restore(self):
|
|
subject = self.make_runner()
|
|
subject.config.expected_contact_keyset_sha256 = None
|
|
baseline = self.full_contacts()
|
|
baseline_order = list(baseline)
|
|
expected_slots = {
|
|
public_key: slot for slot, public_key in enumerate(baseline_order)
|
|
}
|
|
page = 7
|
|
lost = set(baseline_order[175:200])
|
|
# An extra missing record proves the isolation assertion fails after
|
|
# corruption, while the emergency path still restores the baseline.
|
|
lost.add(baseline_order[200])
|
|
wrong_survivors = {
|
|
public_key: value
|
|
for public_key, value in baseline.items()
|
|
if public_key not in lost
|
|
}
|
|
status = HIL.ExtraFsStatus(True, 60, 100, 0)
|
|
digests = subject.exact_inventory_digests(baseline)
|
|
subject.clear_filler = AsyncMock()
|
|
subject.status = AsyncMock(side_effect=[status, status])
|
|
subject.contact_page_mask = AsyncMock(
|
|
return_value=HIL.EXPECTED_FULL_PAGE_MASK
|
|
)
|
|
subject.capture_contact_slot_map = AsyncMock(
|
|
return_value=expected_slots
|
|
)
|
|
subject.cli = AsyncMock(
|
|
return_value=f"HIL contact page {page} CRC corrupted"
|
|
)
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
subject.contacts = AsyncMock(return_value=wrong_survivors)
|
|
subject.restore_contact_inventory_exact = AsyncMock(
|
|
return_value=(baseline, digests)
|
|
)
|
|
|
|
with self.assertRaisesRegex(HIL.HilFailure, "contact key set mismatch"):
|
|
await subject.corrupt_contact_page(page, baseline)
|
|
|
|
self.assertEqual(subject.clear_filler.await_count, 2)
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 2)
|
|
emergency_call = subject.restore_contact_inventory_exact.await_args
|
|
self.assertNotIn("current", emergency_call.kwargs)
|
|
self.assertEqual(
|
|
emergency_call.kwargs["context"],
|
|
"emergency contact-page CRC restoration",
|
|
)
|
|
step = subject.steps[-1]
|
|
self.assertEqual(
|
|
step["step"], "contact_page_corruption_emergency_restore"
|
|
)
|
|
self.assertTrue(step["recovered_exactly"])
|
|
self.assertEqual(step["wire_sha256"], digests["wire_sha256"])
|
|
|
|
async def test_empty_mask_failure_emergency_restores_exact_contact(self):
|
|
subject = self.make_runner()
|
|
contacts = self.full_contacts()
|
|
target_key = HIL.make_contact(0)["public_key"]
|
|
without_target = {
|
|
key_value: value
|
|
for key_value, value in contacts.items()
|
|
if key_value != target_key
|
|
}
|
|
subject.contacts = AsyncMock(
|
|
side_effect=[
|
|
contacts,
|
|
contacts,
|
|
contacts,
|
|
contacts,
|
|
without_target,
|
|
without_target,
|
|
contacts,
|
|
]
|
|
)
|
|
subject.reboot_and_reconnect = AsyncMock(
|
|
side_effect=[None, None, None, None, None, HIL.HilFailure("injected"), None]
|
|
)
|
|
subject.cli = AsyncMock(
|
|
side_effect=[
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
"HIL contact page 0 occupancy bit 0 corrupted",
|
|
"HIL contact page 0 occupied=01fffffe",
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
"HIL contact page 0 occupancy bit 31 corrupted",
|
|
"HIL contact page 0 occupied=81ffffff",
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
"HIL contact page 0 occupied=01fffffe",
|
|
"HIL contact page 0 occupancy bit 0 corrupted",
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
"HIL contact page 0 occupied=01ffffff",
|
|
]
|
|
)
|
|
commands = SimpleNamespace(
|
|
remove_contact=AsyncMock(return_value=event("OK")),
|
|
add_contact=AsyncMock(return_value=event("OK")),
|
|
update_contact=AsyncMock(return_value=event("OK")),
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
with self.assertRaisesRegex(HIL.HilFailure, "injected"):
|
|
await subject.test_contact_occupied_mask_recovery(0, contacts)
|
|
|
|
commands.remove_contact.assert_awaited_once_with(target_key)
|
|
commands.add_contact.assert_awaited_once_with(contacts[target_key])
|
|
commands.update_contact.assert_not_awaited()
|
|
self.assertEqual(subject.contacts.await_count, 7)
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 7)
|
|
self.assertEqual(
|
|
subject.steps[-1]["step"],
|
|
"contact_occupied_mask_emergency_restore",
|
|
)
|
|
self.assertTrue(subject.steps[-1]["original_restored"])
|
|
|
|
async def test_occupied_mask_preflight_rejects_non_harness_inventory(self):
|
|
subject = self.make_runner()
|
|
subject.contacts = AsyncMock(
|
|
return_value={key(0xAA): {"public_key": key(0xAA)}}
|
|
)
|
|
|
|
with self.assertRaisesRegex(
|
|
HIL.HilFailure, "inventory made only from this harness"
|
|
):
|
|
await subject.add_contacts_to_capacity()
|
|
|
|
async def test_storage_fill_helper_is_bounded_verified_and_scoped(self):
|
|
subject = self.make_runner()
|
|
subject.config.fill_bytes = 131072
|
|
subject.clear_filler = AsyncMock()
|
|
subject.cli = AsyncMock(
|
|
return_value=(
|
|
"HIL ExtraFS fill requested=131072 written=36864 "
|
|
"used=99KiB total=100KiB"
|
|
)
|
|
)
|
|
|
|
fill = await subject.fill_storage_to_enospc()
|
|
|
|
subject.clear_filler.assert_awaited_once()
|
|
subject.cli.assert_awaited_once_with(
|
|
"hil extrafs fill 131072",
|
|
timeout=HIL.EXTRAFS_FILL_TIMEOUT_SECONDS,
|
|
)
|
|
self.assertEqual(fill, HIL.ExtraFsFill(131072, 36864, 99, 100))
|
|
|
|
async def test_full_storage_contact_ack_refusal_recovery_and_restore(self):
|
|
subject = self.make_runner()
|
|
original = self.full_contacts()
|
|
target_key = HIL.make_contact(349)["public_key"]
|
|
changed = {key_value: dict(value) for key_value, value in original.items()}
|
|
changed[target_key]["adv_name"] = "HIL-ENOSPC"
|
|
changed[target_key]["flags"] = 1
|
|
subject.contacts = AsyncMock(
|
|
side_effect=[original, changed, changed, changed, original]
|
|
)
|
|
subject.generated_keys = list(original)
|
|
subject.core_uptime = AsyncMock(side_effect=[20, 21])
|
|
subject.clear_filler = AsyncMock()
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
commands = SimpleNamespace(
|
|
update_contact=AsyncMock(side_effect=[event("OK"), event("OK")]),
|
|
send=AsyncMock(return_value=event("ERROR", {"error_code": 5})),
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
await subject.test_contact_update_recovery_when_full(set(original))
|
|
|
|
self.assertEqual(commands.update_contact.await_count, 2)
|
|
first_update = commands.update_contact.await_args_list[0].args[0]
|
|
second_update = commands.update_contact.await_args_list[1].args[0]
|
|
self.assertEqual(first_update["adv_name"], "HIL-ENOSPC")
|
|
self.assertEqual(first_update["flags"], 1)
|
|
self.assertEqual(second_update["adv_name"], "HIL-349")
|
|
self.assertEqual(second_update["flags"], 0)
|
|
reboot_call = commands.send.await_args
|
|
self.assertEqual(reboot_call.args[0], b"\x13reboot")
|
|
self.assertEqual(
|
|
reboot_call.kwargs["timeout"],
|
|
HIL.ENOSPC_REBOOT_REFUSAL_TIMEOUT_SECONDS,
|
|
)
|
|
self.assertEqual(subject.clear_filler.await_count, 2)
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 2)
|
|
step = subject.steps[-1]
|
|
self.assertEqual(step["step"], "full_storage_contact_recovery")
|
|
self.assertEqual(step["uptime_before_refusal"], 20)
|
|
self.assertEqual(step["uptime_after_refusal"], 21)
|
|
self.assertTrue(step["persisted_after_filler_clear"])
|
|
self.assertTrue(step["original_restored"])
|
|
|
|
async def test_full_storage_failure_restores_only_harness_owned_contact(self):
|
|
subject = self.make_runner()
|
|
synthetic = [HIL.make_contact(index) for index in range(349)]
|
|
baseline_key = key(0xFF)
|
|
mixed = {
|
|
**{contact["public_key"]: dict(contact) for contact in synthetic},
|
|
baseline_key: {
|
|
**HIL.make_contact(999),
|
|
"public_key": baseline_key,
|
|
"adv_name": "USER-CONTACT",
|
|
},
|
|
}
|
|
subject.generated_keys = [contact["public_key"] for contact in synthetic]
|
|
subject.contacts = AsyncMock(
|
|
side_effect=[mixed, HIL.HilFailure("injected after ACK"), mixed]
|
|
)
|
|
subject.clear_filler = AsyncMock()
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
commands = SimpleNamespace(
|
|
update_contact=AsyncMock(side_effect=[event("OK"), event("OK")]),
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
with self.assertRaisesRegex(HIL.HilFailure, "injected after ACK"):
|
|
await subject.test_contact_update_recovery_when_full(set(mixed))
|
|
|
|
first_target = commands.update_contact.await_args_list[0].args[0]
|
|
restored_target = commands.update_contact.await_args_list[1].args[0]
|
|
self.assertNotEqual(first_target["public_key"], baseline_key)
|
|
self.assertEqual(restored_target["public_key"], first_target["public_key"])
|
|
self.assertEqual(restored_target["adv_name"], "HIL-348")
|
|
subject.clear_filler.assert_awaited_once()
|
|
subject.reboot_and_reconnect.assert_awaited_once()
|
|
self.assertEqual(
|
|
subject.steps[-1]["step"], "full_storage_contact_emergency_restore"
|
|
)
|
|
|
|
async def test_full_storage_add_ack_refusal_persistence_and_exact_restore(self):
|
|
subject = self.make_runner()
|
|
original = self.full_contacts()
|
|
original_key = HIL.make_contact(349)["public_key"]
|
|
replacement = HIL.make_contact(2351)
|
|
replacement_key = replacement["public_key"]
|
|
without_original = {
|
|
key_value: dict(value)
|
|
for key_value, value in original.items()
|
|
if key_value != original_key
|
|
}
|
|
substituted = {
|
|
**without_original,
|
|
replacement_key: dict(replacement),
|
|
}
|
|
subject.generated_keys = list(original)
|
|
subject.contacts = AsyncMock(
|
|
side_effect=[
|
|
original,
|
|
without_original,
|
|
substituted,
|
|
substituted,
|
|
substituted,
|
|
substituted,
|
|
original,
|
|
]
|
|
)
|
|
fill = HIL.ExtraFsFill(131072, 36864, 99, 100)
|
|
subject.fill_storage_to_enospc = AsyncMock(return_value=fill)
|
|
subject.core_uptime = AsyncMock(side_effect=[20, 21])
|
|
subject.clear_filler = AsyncMock()
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
subject.set_contact_exact = AsyncMock(return_value=event("OK"))
|
|
commands = SimpleNamespace(
|
|
remove_contact=AsyncMock(side_effect=[event("OK"), event("OK")]),
|
|
add_contact=AsyncMock(side_effect=[event("OK"), event("OK")]),
|
|
update_contact=AsyncMock(return_value=event("OK")),
|
|
send=AsyncMock(return_value=event("ERROR", {"error_code": 5})),
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
await subject.test_contact_add_recovery_when_full(set(original))
|
|
|
|
self.assertEqual(
|
|
[entry.args[0] for entry in commands.remove_contact.await_args_list],
|
|
[original_key, replacement_key],
|
|
)
|
|
self.assertEqual(
|
|
[
|
|
entry.args[0]["public_key"]
|
|
for entry in commands.add_contact.await_args_list
|
|
],
|
|
[replacement_key],
|
|
)
|
|
commands.update_contact.assert_not_awaited()
|
|
self.assertEqual(
|
|
subject.set_contact_exact.await_args.args[0]["public_key"],
|
|
original_key,
|
|
)
|
|
self.assertEqual(subject.fill_storage_to_enospc.await_count, 1)
|
|
self.assertEqual(subject.clear_filler.await_count, 2)
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 3)
|
|
self.assertEqual(subject.contacts.await_count, 7)
|
|
step = subject.steps[-1]
|
|
self.assertEqual(step["step"], "full_storage_contact_add_recovery")
|
|
self.assertEqual(step["uptime_before_refusal"], 20)
|
|
self.assertEqual(step["uptime_after_refusal"], 21)
|
|
self.assertTrue(step["add_acknowledged"])
|
|
self.assertTrue(step["persisted_after_filler_clear"])
|
|
self.assertTrue(step["original_restored"])
|
|
self.assertEqual(step["keyset_sha256"], subject.keyset_digest(original))
|
|
|
|
async def test_full_storage_add_failure_emergency_restores_substitution(self):
|
|
subject = self.make_runner()
|
|
original = self.full_contacts()
|
|
original_key = HIL.make_contact(349)["public_key"]
|
|
replacement = HIL.make_contact(2351)
|
|
replacement_key = replacement["public_key"]
|
|
without_original = {
|
|
key_value: dict(value)
|
|
for key_value, value in original.items()
|
|
if key_value != original_key
|
|
}
|
|
substituted = {
|
|
**without_original,
|
|
replacement_key: dict(replacement),
|
|
}
|
|
subject.generated_keys = list(original)
|
|
subject.contacts = AsyncMock(
|
|
side_effect=[
|
|
original,
|
|
without_original,
|
|
HIL.HilFailure("injected after ADD ACK"),
|
|
substituted,
|
|
original,
|
|
]
|
|
)
|
|
subject.fill_storage_to_enospc = AsyncMock(
|
|
return_value=HIL.ExtraFsFill(131072, 36864, 99, 100)
|
|
)
|
|
subject.clear_filler = AsyncMock()
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
subject.set_contact_exact = AsyncMock(return_value=event("OK"))
|
|
commands = SimpleNamespace(
|
|
remove_contact=AsyncMock(side_effect=[event("OK"), event("OK")]),
|
|
add_contact=AsyncMock(side_effect=[event("OK"), event("OK")]),
|
|
update_contact=AsyncMock(return_value=event("OK")),
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
with self.assertRaisesRegex(HIL.HilFailure, "injected after ADD ACK"):
|
|
await subject.test_contact_add_recovery_when_full(set(original))
|
|
|
|
self.assertEqual(
|
|
[entry.args[0] for entry in commands.remove_contact.await_args_list],
|
|
[original_key, replacement_key],
|
|
)
|
|
self.assertEqual(
|
|
[
|
|
entry.args[0]["public_key"]
|
|
for entry in commands.add_contact.await_args_list
|
|
],
|
|
[replacement_key],
|
|
)
|
|
commands.update_contact.assert_not_awaited()
|
|
self.assertEqual(
|
|
subject.set_contact_exact.await_args.args[0]["public_key"],
|
|
original_key,
|
|
)
|
|
self.assertEqual(subject.clear_filler.await_count, 1)
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 2)
|
|
self.assertEqual(
|
|
subject.steps[-1]["step"],
|
|
"full_storage_contact_add_emergency_restore",
|
|
)
|
|
self.assertEqual(
|
|
subject.steps[-1]["keyset_sha256"], subject.keyset_digest(original)
|
|
)
|
|
|
|
async def test_full_storage_remove_ack_refusal_persistence_and_exact_restore(self):
|
|
subject = self.make_runner()
|
|
original = self.full_contacts()
|
|
target_key = HIL.make_contact(348)["public_key"]
|
|
without_target = {
|
|
key_value: dict(value)
|
|
for key_value, value in original.items()
|
|
if key_value != target_key
|
|
}
|
|
subject.generated_keys = list(original)
|
|
subject.contacts = AsyncMock(
|
|
side_effect=[
|
|
original,
|
|
without_target,
|
|
without_target,
|
|
without_target,
|
|
without_target,
|
|
original,
|
|
]
|
|
)
|
|
fill = HIL.ExtraFsFill(131072, 36864, 99, 100)
|
|
subject.fill_storage_to_enospc = AsyncMock(return_value=fill)
|
|
subject.clear_cached_advert = AsyncMock()
|
|
subject.core_uptime = AsyncMock(side_effect=[20, 21])
|
|
subject.clear_filler = AsyncMock()
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
subject.set_contact_exact = AsyncMock(return_value=event("OK"))
|
|
commands = SimpleNamespace(
|
|
remove_contact=AsyncMock(return_value=event("OK")),
|
|
add_contact=AsyncMock(return_value=event("OK")),
|
|
update_contact=AsyncMock(return_value=event("OK")),
|
|
send=AsyncMock(return_value=event("ERROR", {"error_code": 5})),
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
await subject.test_contact_remove_recovery_when_full(set(original))
|
|
|
|
commands.remove_contact.assert_awaited_once_with(target_key)
|
|
subject.clear_cached_advert.assert_awaited_once_with(target_key)
|
|
commands.add_contact.assert_not_awaited()
|
|
commands.update_contact.assert_not_awaited()
|
|
self.assertEqual(
|
|
subject.set_contact_exact.await_args.args[0]["public_key"], target_key
|
|
)
|
|
self.assertEqual(subject.fill_storage_to_enospc.await_count, 1)
|
|
self.assertEqual(subject.clear_filler.await_count, 2)
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 2)
|
|
self.assertEqual(subject.contacts.await_count, 6)
|
|
step = subject.steps[-1]
|
|
self.assertEqual(step["step"], "full_storage_contact_remove_recovery")
|
|
self.assertEqual(step["uptime_before_refusal"], 20)
|
|
self.assertEqual(step["uptime_after_refusal"], 21)
|
|
self.assertTrue(step["remove_acknowledged"])
|
|
self.assertTrue(step["persisted_after_filler_clear"])
|
|
self.assertTrue(step["original_restored"])
|
|
self.assertEqual(step["keyset_sha256"], subject.keyset_digest(original))
|
|
|
|
async def test_full_storage_remove_failure_emergency_restores_contact(self):
|
|
subject = self.make_runner()
|
|
original = self.full_contacts()
|
|
target_key = HIL.make_contact(348)["public_key"]
|
|
without_target = {
|
|
key_value: dict(value)
|
|
for key_value, value in original.items()
|
|
if key_value != target_key
|
|
}
|
|
subject.generated_keys = list(original)
|
|
subject.contacts = AsyncMock(
|
|
side_effect=[
|
|
original,
|
|
HIL.HilFailure("injected after REMOVE ACK"),
|
|
without_target,
|
|
original,
|
|
]
|
|
)
|
|
subject.fill_storage_to_enospc = AsyncMock(
|
|
return_value=HIL.ExtraFsFill(131072, 36864, 99, 100)
|
|
)
|
|
subject.clear_cached_advert = AsyncMock()
|
|
subject.clear_filler = AsyncMock()
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
subject.set_contact_exact = AsyncMock(return_value=event("OK"))
|
|
commands = SimpleNamespace(
|
|
remove_contact=AsyncMock(return_value=event("OK")),
|
|
add_contact=AsyncMock(return_value=event("OK")),
|
|
update_contact=AsyncMock(return_value=event("OK")),
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
with self.assertRaisesRegex(HIL.HilFailure, "injected after REMOVE ACK"):
|
|
await subject.test_contact_remove_recovery_when_full(set(original))
|
|
|
|
commands.remove_contact.assert_awaited_once_with(target_key)
|
|
subject.clear_cached_advert.assert_awaited_once_with(target_key)
|
|
commands.add_contact.assert_not_awaited()
|
|
commands.update_contact.assert_not_awaited()
|
|
self.assertEqual(
|
|
subject.set_contact_exact.await_args.args[0]["public_key"], target_key
|
|
)
|
|
self.assertEqual(subject.clear_filler.await_count, 1)
|
|
subject.reboot_and_reconnect.assert_awaited_once()
|
|
self.assertEqual(
|
|
subject.steps[-1]["step"],
|
|
"full_storage_contact_remove_emergency_restore",
|
|
)
|
|
self.assertEqual(
|
|
subject.steps[-1]["keyset_sha256"], subject.keyset_digest(original)
|
|
)
|
|
|
|
async def test_full_storage_remove_error_reconciles_uncertain_mutation(self):
|
|
subject = self.make_runner()
|
|
original = self.full_contacts()
|
|
target_key = HIL.make_contact(348)["public_key"]
|
|
subject.generated_keys = list(original)
|
|
subject.contacts = AsyncMock(return_value=original)
|
|
subject.fill_storage_to_enospc = AsyncMock(
|
|
return_value=HIL.ExtraFsFill(131072, 36864, 99, 100)
|
|
)
|
|
subject.clear_cached_advert = AsyncMock()
|
|
subject.clear_filler = AsyncMock()
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
subject.set_contact_exact = AsyncMock(return_value=event("OK"))
|
|
commands = SimpleNamespace(
|
|
remove_contact=AsyncMock(
|
|
return_value=event("ERROR", {"error_code": 5})
|
|
),
|
|
add_contact=AsyncMock(return_value=event("OK")),
|
|
update_contact=AsyncMock(return_value=event("OK")),
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
with self.assertRaisesRegex(HIL.HilFailure, "device returned error"):
|
|
await subject.test_contact_remove_recovery_when_full(set(original))
|
|
|
|
commands.remove_contact.assert_awaited_once_with(target_key)
|
|
subject.clear_cached_advert.assert_awaited_once_with(target_key)
|
|
commands.update_contact.assert_not_awaited()
|
|
commands.add_contact.assert_not_awaited()
|
|
subject.set_contact_exact.assert_awaited_once()
|
|
subject.clear_filler.assert_awaited_once()
|
|
subject.reboot_and_reconnect.assert_awaited_once()
|
|
self.assertEqual(subject.contacts.await_count, 3)
|
|
self.assertEqual(
|
|
subject.steps[-1]["step"],
|
|
"full_storage_contact_remove_emergency_restore",
|
|
)
|
|
|
|
async def test_full_storage_remove_fill_failure_still_clears_partial_filler(self):
|
|
subject = self.make_runner()
|
|
original = self.full_contacts()
|
|
target_key = HIL.make_contact(348)["public_key"]
|
|
subject.generated_keys = list(original)
|
|
subject.contacts = AsyncMock(return_value=original)
|
|
subject.fill_storage_to_enospc = AsyncMock(
|
|
side_effect=HIL.HilFailure("injected partial fill failure")
|
|
)
|
|
subject.clear_cached_advert = AsyncMock()
|
|
subject.clear_filler = AsyncMock()
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
commands = SimpleNamespace(remove_contact=AsyncMock())
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
with self.assertRaisesRegex(HIL.HilFailure, "partial fill failure"):
|
|
await subject.test_contact_remove_recovery_when_full(set(original))
|
|
|
|
commands.remove_contact.assert_not_awaited()
|
|
subject.clear_cached_advert.assert_awaited_once_with(target_key)
|
|
subject.clear_filler.assert_awaited_once()
|
|
subject.reboot_and_reconnect.assert_not_awaited()
|
|
|
|
async def test_advert_hil_helpers_use_exact_direct_only_commands(self):
|
|
subject = self.make_runner()
|
|
public_key = HIL.make_contact(12)["public_key"]
|
|
token = public_key[:14]
|
|
subject.cli = AsyncMock(
|
|
side_effect=[
|
|
"HIL contact slot=27",
|
|
"HIL advert seeded slot=27",
|
|
"HIL advert cleared",
|
|
"HIL contact page 13 read failure armed",
|
|
"HIL contact page 12 stat failure armed",
|
|
]
|
|
)
|
|
|
|
self.assertEqual(await subject.contact_slot(public_key), 27)
|
|
self.assertEqual(await subject.seed_cached_advert(public_key), 27)
|
|
await subject.clear_cached_advert(public_key)
|
|
await subject.arm_contact_read_failure(13)
|
|
await subject.arm_contact_stat_failure(12)
|
|
|
|
self.assertEqual(
|
|
[entry.args[0] for entry in subject.cli.await_args_list],
|
|
[
|
|
f"hil extrafs contact-slot {token}",
|
|
f"hil extrafs seed-advert {token} CONFIRM",
|
|
f"hil extrafs clear-advert {token} CONFIRM",
|
|
"hil extrafs fail-read-page 13 CONFIRM",
|
|
"hil extrafs fail-stat-page 12 CONFIRM",
|
|
],
|
|
)
|
|
|
|
async def test_unread_page_get_contacts_requires_file_io_without_partial_state(self):
|
|
subject = self.make_runner()
|
|
reader = SimpleNamespace(contact_nb=350, contacts={"stale": {}})
|
|
commands = SimpleNamespace(
|
|
send=AsyncMock(
|
|
return_value=event("ERROR", {"error_code": 5})
|
|
)
|
|
)
|
|
subject.mc = SimpleNamespace(_reader=reader, commands=commands)
|
|
|
|
await subject.expect_contacts_file_io("unit unread page")
|
|
|
|
send_call = commands.send.await_args
|
|
self.assertEqual(send_call.args[0], b"\x04")
|
|
self.assertEqual(send_call.kwargs["timeout"], 20)
|
|
self.assertIsNone(reader.contact_nb)
|
|
self.assertEqual(reader.contacts, {})
|
|
|
|
async def test_unread_page_get_contacts_rejects_partial_frames_before_error(self):
|
|
subject = self.make_runner()
|
|
reader = SimpleNamespace(contact_nb=None, contacts={})
|
|
|
|
async def partial_then_error(*_args, **_kwargs):
|
|
reader.contact_nb = 350
|
|
reader.contacts = {HIL.make_contact(0)["public_key"]: {}}
|
|
return event("ERROR", {"error_code": 5})
|
|
|
|
commands = SimpleNamespace(send=AsyncMock(side_effect=partial_then_error))
|
|
subject.mc = SimpleNamespace(_reader=reader, commands=commands)
|
|
|
|
with self.assertRaisesRegex(HIL.HilFailure, "partial contact stream"):
|
|
await subject.expect_contacts_file_io("unit unread page")
|
|
|
|
async def test_advert_backed_remove_rolls_back_live_slot_and_exact_inventory(self):
|
|
subject = self.make_runner()
|
|
baseline = self.full_contacts()
|
|
first_key = HIL.make_contact(0)["public_key"]
|
|
target_key = HIL.make_contact(349)["public_key"]
|
|
without_first = {
|
|
key_value: dict(value)
|
|
for key_value, value in baseline.items()
|
|
if key_value != first_key
|
|
}
|
|
subject.contacts = AsyncMock(
|
|
side_effect=[
|
|
baseline,
|
|
without_first,
|
|
without_first,
|
|
without_first,
|
|
without_first,
|
|
baseline,
|
|
]
|
|
)
|
|
subject.contact_slot = AsyncMock(
|
|
side_effect=[10, 300, 300, 300, 300, 10, 300]
|
|
)
|
|
subject.seed_cached_advert = AsyncMock(return_value=300)
|
|
subject.fill_storage_to_enospc = AsyncMock(
|
|
return_value=HIL.ExtraFsFill(131072, 36864, 99, 100)
|
|
)
|
|
subject.graceful_reboot_or_enospc_refusal = AsyncMock(
|
|
return_value=("refused", 20, 21)
|
|
)
|
|
subject.clear_filler = AsyncMock()
|
|
subject.clear_cached_advert = AsyncMock()
|
|
subject.set_contact_exact = AsyncMock(return_value=event("OK"))
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
commands = SimpleNamespace(
|
|
remove_contact=AsyncMock(
|
|
side_effect=[
|
|
event("OK"),
|
|
event("ERROR", {"error_code": 5}),
|
|
]
|
|
)
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
await subject.test_advert_backed_remove_rollback(set(baseline))
|
|
|
|
self.assertEqual(
|
|
[entry.args[0] for entry in commands.remove_contact.await_args_list],
|
|
[first_key, target_key],
|
|
)
|
|
subject.seed_cached_advert.assert_awaited_once_with(target_key)
|
|
subject.clear_cached_advert.assert_awaited_once_with(target_key)
|
|
self.assertEqual(
|
|
[
|
|
entry.args[0]["public_key"]
|
|
for entry in subject.set_contact_exact.await_args_list
|
|
],
|
|
[first_key, target_key],
|
|
)
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 2)
|
|
self.assertEqual(subject.contacts.await_count, 6)
|
|
step = subject.steps[-1]
|
|
self.assertEqual(step["step"], "advert_remove_rollback")
|
|
self.assertEqual(step["earlier_slot"], 10)
|
|
self.assertEqual(step["target_slot"], 300)
|
|
self.assertEqual(step["reboot_outcome"], "refused")
|
|
self.assertTrue(step["live_target_and_slot_rolled_back"])
|
|
self.assertEqual(
|
|
step["semantic_sha256"], subject.semantic_contact_digest(baseline)
|
|
)
|
|
self.assertEqual(step["wire_sha256"], subject.wire_contact_digest(baseline))
|
|
self.assertEqual(step["keyset_sha256"], subject.keyset_digest(baseline))
|
|
|
|
async def test_advert_rollback_unexpected_remove_ok_emergency_restores_slots(self):
|
|
subject = self.make_runner()
|
|
baseline = self.full_contacts()
|
|
first_key = HIL.make_contact(0)["public_key"]
|
|
target_key = HIL.make_contact(349)["public_key"]
|
|
without_first = {
|
|
key_value: dict(value)
|
|
for key_value, value in baseline.items()
|
|
if key_value != first_key
|
|
}
|
|
without_both = {
|
|
key_value: dict(value)
|
|
for key_value, value in without_first.items()
|
|
if key_value != target_key
|
|
}
|
|
subject.contacts = AsyncMock(
|
|
side_effect=[baseline, without_first, without_both, baseline]
|
|
)
|
|
subject.contact_slot = AsyncMock(
|
|
side_effect=[10, 300, 300, 10, 300]
|
|
)
|
|
subject.seed_cached_advert = AsyncMock(return_value=300)
|
|
subject.fill_storage_to_enospc = AsyncMock(
|
|
return_value=HIL.ExtraFsFill(131072, 36864, 99, 100)
|
|
)
|
|
subject.graceful_reboot_or_enospc_refusal = AsyncMock()
|
|
subject.clear_filler = AsyncMock()
|
|
subject.clear_cached_advert = AsyncMock()
|
|
subject.set_contact_exact = AsyncMock(return_value=event("OK"))
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
commands = SimpleNamespace(
|
|
remove_contact=AsyncMock(
|
|
side_effect=[event("OK"), event("OK")]
|
|
)
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
with self.assertRaisesRegex(HIL.HilFailure, "expected error code 5"):
|
|
await subject.test_advert_backed_remove_rollback(set(baseline))
|
|
|
|
subject.graceful_reboot_or_enospc_refusal.assert_not_awaited()
|
|
subject.clear_filler.assert_awaited_once()
|
|
subject.clear_cached_advert.assert_awaited_once_with(target_key)
|
|
self.assertEqual(
|
|
[
|
|
entry.args[0]["public_key"]
|
|
for entry in subject.set_contact_exact.await_args_list
|
|
],
|
|
[first_key, target_key],
|
|
)
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 2)
|
|
self.assertEqual(
|
|
subject.steps[-1]["step"],
|
|
"advert_remove_rollback_emergency_restore",
|
|
)
|
|
self.assertEqual(
|
|
subject.steps[-1]["wire_sha256"],
|
|
subject.wire_contact_digest(baseline),
|
|
)
|
|
|
|
async def test_read_failure_selector_finds_page_and_live_targets_by_slot(self):
|
|
subject = self.make_runner()
|
|
subject.contact_slot = AsyncMock(side_effect=[325, 326, 0, 1])
|
|
|
|
failed_key, update_key, remove_key, slots = (
|
|
await subject.select_read_failure_contacts(13)
|
|
)
|
|
|
|
self.assertEqual(failed_key, HIL.make_contact(325)["public_key"])
|
|
self.assertEqual(update_key, HIL.make_contact(0)["public_key"])
|
|
self.assertEqual(remove_key, HIL.make_contact(1)["public_key"])
|
|
self.assertEqual(
|
|
slots,
|
|
{
|
|
failed_key: 325,
|
|
update_key: 0,
|
|
remove_key: 1,
|
|
},
|
|
)
|
|
|
|
async def test_contact_read_failure_fails_closed_then_recovers_exactly(self):
|
|
subject = self.make_runner()
|
|
baseline = self.full_contacts()
|
|
failed_key = HIL.make_contact(0)["public_key"]
|
|
update_key = HIL.make_contact(25)["public_key"]
|
|
remove_key = HIL.make_contact(26)["public_key"]
|
|
expected_slots = {failed_key: 0, update_key: 25, remove_key: 26}
|
|
status = HIL.ExtraFsStatus(True, 60, 100, 0)
|
|
subject.clear_filler = AsyncMock()
|
|
subject.status = AsyncMock(side_effect=[status, status, status])
|
|
subject.contacts = AsyncMock(side_effect=[baseline, baseline])
|
|
subject.contact_page_mask = AsyncMock(
|
|
return_value=HIL.EXPECTED_FULL_PAGE_MASK
|
|
)
|
|
subject.select_read_failure_contacts = AsyncMock(
|
|
return_value=(failed_key, update_key, remove_key, expected_slots)
|
|
)
|
|
subject.arm_contact_read_failure = AsyncMock()
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
subject.expect_contacts_file_io = AsyncMock()
|
|
subject.expect_contact_not_found = AsyncMock()
|
|
subject.contact_by_key = AsyncMock(
|
|
side_effect=[
|
|
baseline[update_key],
|
|
baseline[remove_key],
|
|
baseline[update_key],
|
|
baseline[remove_key],
|
|
baseline[update_key],
|
|
]
|
|
)
|
|
subject.set_contact_exact = AsyncMock(
|
|
side_effect=[
|
|
event("ERROR", {"error_code": 5}),
|
|
event("ERROR", {"error_code": 5}),
|
|
]
|
|
)
|
|
subject.contact_slot = AsyncMock(side_effect=[25, 26, 0, 25, 26])
|
|
commands = SimpleNamespace(
|
|
remove_contact=AsyncMock(
|
|
return_value=event("ERROR", {"error_code": 5})
|
|
)
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
await subject.test_contact_page_read_failure(0)
|
|
|
|
subject.arm_contact_read_failure.assert_awaited_once_with(0)
|
|
self.assertEqual(subject.expect_contacts_file_io.await_count, 2)
|
|
self.assertEqual(subject.expect_contact_not_found.await_count, 3)
|
|
self.assertEqual(subject.set_contact_exact.await_count, 2)
|
|
commands.remove_contact.assert_awaited_once_with(remove_key)
|
|
self.assertEqual(
|
|
[entry.args[0] for entry in subject.contact_slot.await_args_list],
|
|
[update_key, remove_key, failed_key, update_key, remove_key],
|
|
)
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 2)
|
|
self.assertEqual(subject.status.await_count, 3)
|
|
step = subject.steps[-1]
|
|
self.assertEqual(step["step"], "contact_read_failure_recovery")
|
|
self.assertEqual(step["page"], 0)
|
|
self.assertTrue(step["partial_inventory_rejected"])
|
|
self.assertTrue(step["live_mutations_rolled_back"])
|
|
self.assertTrue(step["live_slots_rolled_back"])
|
|
self.assertEqual(step["baseline_used_kib"], 60)
|
|
self.assertEqual(step["final_used_kib"], 60)
|
|
self.assertEqual(step["wire_sha256"], subject.wire_contact_digest(baseline))
|
|
self.assertEqual(
|
|
step["semantic_sha256"], subject.semantic_contact_digest(baseline)
|
|
)
|
|
|
|
async def test_contact_read_failure_rejects_final_storage_usage_change(self):
|
|
subject = self.make_runner()
|
|
baseline = self.full_contacts()
|
|
failed_key = HIL.make_contact(0)["public_key"]
|
|
update_key = HIL.make_contact(25)["public_key"]
|
|
remove_key = HIL.make_contact(26)["public_key"]
|
|
expected_slots = {failed_key: 0, update_key: 25, remove_key: 26}
|
|
clean_status = HIL.ExtraFsStatus(True, 60, 100, 0)
|
|
leaked_status = HIL.ExtraFsStatus(True, 61, 100, 0)
|
|
subject.clear_filler = AsyncMock()
|
|
subject.status = AsyncMock(
|
|
side_effect=[
|
|
clean_status,
|
|
clean_status,
|
|
leaked_status,
|
|
clean_status,
|
|
]
|
|
)
|
|
subject.contacts = AsyncMock(
|
|
side_effect=[baseline, baseline, baseline]
|
|
)
|
|
subject.contact_page_mask = AsyncMock(
|
|
return_value=HIL.EXPECTED_FULL_PAGE_MASK
|
|
)
|
|
subject.select_read_failure_contacts = AsyncMock(
|
|
return_value=(failed_key, update_key, remove_key, expected_slots)
|
|
)
|
|
subject.arm_contact_read_failure = AsyncMock()
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
subject.expect_contacts_file_io = AsyncMock()
|
|
subject.expect_contact_not_found = AsyncMock()
|
|
subject.contact_by_key = AsyncMock(
|
|
side_effect=[
|
|
baseline[update_key],
|
|
baseline[remove_key],
|
|
baseline[update_key],
|
|
baseline[remove_key],
|
|
baseline[update_key],
|
|
]
|
|
)
|
|
subject.set_contact_exact = AsyncMock(
|
|
side_effect=[
|
|
event("ERROR", {"error_code": 5}),
|
|
event("ERROR", {"error_code": 5}),
|
|
]
|
|
)
|
|
subject.contact_slot = AsyncMock(
|
|
side_effect=[25, 26, 0, 25, 26, 0, 25, 26]
|
|
)
|
|
commands = SimpleNamespace(
|
|
remove_contact=AsyncMock(
|
|
return_value=event("ERROR", {"error_code": 5})
|
|
)
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
with self.assertRaisesRegex(
|
|
HIL.HilFailure,
|
|
"changed ExtraFS usage from 60 KiB to 61 KiB",
|
|
):
|
|
await subject.test_contact_page_read_failure(0)
|
|
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 4)
|
|
self.assertEqual(subject.status.await_count, 4)
|
|
self.assertEqual(
|
|
subject.steps[-1]["step"],
|
|
"contact_read_failure_emergency_recovery",
|
|
)
|
|
self.assertTrue(subject.steps[-1]["recovered_exactly"])
|
|
|
|
async def test_contact_read_failure_unexpected_add_ok_emergency_recovers(self):
|
|
subject = self.make_runner()
|
|
baseline = self.full_contacts()
|
|
failed_key = HIL.make_contact(0)["public_key"]
|
|
update_key = HIL.make_contact(25)["public_key"]
|
|
remove_key = HIL.make_contact(26)["public_key"]
|
|
expected_slots = {failed_key: 0, update_key: 25, remove_key: 26}
|
|
status = HIL.ExtraFsStatus(True, 60, 100, 0)
|
|
subject.clear_filler = AsyncMock()
|
|
subject.status = AsyncMock(side_effect=[status, status])
|
|
subject.contacts = AsyncMock(side_effect=[baseline, baseline])
|
|
subject.contact_page_mask = AsyncMock(
|
|
return_value=HIL.EXPECTED_FULL_PAGE_MASK
|
|
)
|
|
subject.select_read_failure_contacts = AsyncMock(
|
|
return_value=(failed_key, update_key, remove_key, expected_slots)
|
|
)
|
|
subject.arm_contact_read_failure = AsyncMock()
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
subject.expect_contacts_file_io = AsyncMock()
|
|
subject.expect_contact_not_found = AsyncMock()
|
|
subject.contact_by_key = AsyncMock(
|
|
side_effect=[baseline[update_key], baseline[remove_key]]
|
|
)
|
|
subject.set_contact_exact = AsyncMock(return_value=event("OK"))
|
|
subject.contact_slot = AsyncMock(side_effect=[0, 25, 26])
|
|
commands = SimpleNamespace(remove_contact=AsyncMock())
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
with self.assertRaisesRegex(HIL.HilFailure, "expected error code 5"):
|
|
await subject.test_contact_page_read_failure(0)
|
|
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 3)
|
|
commands.remove_contact.assert_not_awaited()
|
|
self.assertEqual(
|
|
subject.steps[-1]["step"],
|
|
"contact_read_failure_emergency_recovery",
|
|
)
|
|
self.assertTrue(subject.steps[-1]["recovered_exactly"])
|
|
self.assertEqual(
|
|
subject.steps[-1]["keyset_sha256"], subject.keyset_digest(baseline)
|
|
)
|
|
|
|
async def test_contact_stat_failure_blocks_whole_store_then_recovers_exactly(self):
|
|
subject = self.make_runner()
|
|
baseline = self.full_contacts()
|
|
failed_key = HIL.make_contact(0)["public_key"]
|
|
update_key = HIL.make_contact(25)["public_key"]
|
|
remove_key = HIL.make_contact(26)["public_key"]
|
|
expected_slots = {failed_key: 0, update_key: 25, remove_key: 26}
|
|
status = HIL.ExtraFsStatus(True, 60, 100, 0)
|
|
subject.clear_filler = AsyncMock()
|
|
subject.status = AsyncMock(side_effect=[status, status, status])
|
|
subject.contacts = AsyncMock(side_effect=[baseline, baseline])
|
|
subject.contact_page_mask = AsyncMock(
|
|
return_value=HIL.EXPECTED_FULL_PAGE_MASK
|
|
)
|
|
subject.select_read_failure_contacts = AsyncMock(
|
|
return_value=(failed_key, update_key, remove_key, expected_slots)
|
|
)
|
|
subject.arm_contact_stat_failure = AsyncMock()
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
subject.expect_contacts_file_io = AsyncMock()
|
|
subject.set_contact_exact = AsyncMock(
|
|
side_effect=[
|
|
event("ERROR", {"error_code": 5}),
|
|
event("ERROR", {"error_code": 5}),
|
|
]
|
|
)
|
|
subject.contact_slot = AsyncMock(side_effect=[0, 25, 26])
|
|
commands = SimpleNamespace(
|
|
remove_contact=AsyncMock(
|
|
return_value=event("ERROR", {"error_code": 5})
|
|
)
|
|
)
|
|
subject.mc = SimpleNamespace(commands=commands)
|
|
|
|
await subject.test_contact_page_stat_failure(0)
|
|
|
|
subject.arm_contact_stat_failure.assert_awaited_once_with(0)
|
|
self.assertEqual(subject.expect_contacts_file_io.await_count, 2)
|
|
self.assertEqual(subject.set_contact_exact.await_count, 2)
|
|
commands.remove_contact.assert_awaited_once_with(remove_key)
|
|
self.assertEqual(subject.reboot_and_reconnect.await_count, 2)
|
|
self.assertEqual(subject.status.await_count, 3)
|
|
self.assertEqual(
|
|
[entry.args[0] for entry in subject.contact_slot.await_args_list],
|
|
[failed_key, update_key, remove_key],
|
|
)
|
|
step = subject.steps[-1]
|
|
self.assertEqual(step["step"], "contact_stat_failure_recovery")
|
|
self.assertEqual(step["page"], 0)
|
|
self.assertTrue(step["whole_source_discovery_rejected"])
|
|
self.assertTrue(step["marker_storage_reclaimed"])
|
|
self.assertEqual(step["baseline_used_kib"], 60)
|
|
self.assertEqual(step["final_used_kib"], 60)
|
|
self.assertEqual(step["wire_sha256"], subject.wire_contact_digest(baseline))
|
|
self.assertEqual(
|
|
step["semantic_sha256"], subject.semantic_contact_digest(baseline)
|
|
)
|
|
|
|
|
|
class ReconnectRaceTests(unittest.IsolatedAsyncioTestCase):
|
|
def make_runner(self):
|
|
config = SimpleNamespace(verbose=False)
|
|
identity = SimpleNamespace(device="COM23")
|
|
return HIL.StressRunner(config, identity)
|
|
|
|
async def test_retries_transient_windows_access_denied(self):
|
|
subject = self.make_runner()
|
|
subject.connect = AsyncMock(side_effect=[PermissionError(13, "Access denied"), None])
|
|
subject.disconnect = AsyncMock()
|
|
|
|
await subject.connect_when_openable(1.0)
|
|
|
|
self.assertEqual(subject.connect.await_count, 2)
|
|
subject.disconnect.assert_awaited_once()
|
|
|
|
async def test_reenumeration_surfaces_typed_reboot_refusal(self):
|
|
refusal = event("ERROR", {"error_code": 5})
|
|
error_future = asyncio.get_running_loop().create_future()
|
|
error_future.set_result(refusal)
|
|
previous = HIL.PortIdentity(
|
|
"COM23",
|
|
HIL.EXPECTED_USB_SERIAL,
|
|
HIL.EXPECTED_USB_VID,
|
|
HIL.EXPECTED_USB_PID,
|
|
"T1000-E",
|
|
"T1000-E",
|
|
"MI_00",
|
|
"1-8:x.0",
|
|
"USB MI_00",
|
|
)
|
|
|
|
with self.assertRaises(HIL.RebootRefused) as raised:
|
|
await HIL.wait_for_reenumeration(
|
|
previous,
|
|
0.1,
|
|
port_source=lambda: [],
|
|
error_future=error_future,
|
|
)
|
|
|
|
self.assertIs(raised.exception.event, refusal)
|
|
|
|
async def test_identity_mismatch_is_never_retried(self):
|
|
subject = self.make_runner()
|
|
subject.connect = AsyncMock(
|
|
side_effect=HIL.IdentityGateFailure("wrong node public key")
|
|
)
|
|
subject.disconnect = AsyncMock()
|
|
|
|
with self.assertRaisesRegex(HIL.IdentityGateFailure, "wrong node"):
|
|
await subject.connect_when_openable(1.0)
|
|
|
|
subject.connect.assert_awaited_once()
|
|
subject.disconnect.assert_awaited_once()
|
|
|
|
async def test_graceful_reboot_accepts_verified_file_io_refusal(self):
|
|
subject = self.make_runner()
|
|
refusal = event("ERROR", {"error_code": 5})
|
|
subject.core_uptime = AsyncMock(side_effect=[20, 21])
|
|
subject.reboot_and_reconnect = AsyncMock(
|
|
side_effect=HIL.RebootRefused(refusal)
|
|
)
|
|
|
|
outcome = await subject.graceful_reboot_or_enospc_refusal()
|
|
|
|
self.assertEqual(outcome, ("refused", 20, 21))
|
|
self.assertEqual(subject.steps[-1]["step"], "advert_rollback_reboot_refused")
|
|
|
|
async def test_graceful_reboot_relies_on_inner_reset_proof_after_slow_reconnect(self):
|
|
subject = self.make_runner()
|
|
subject.core_uptime = AsyncMock(side_effect=[6, 9])
|
|
subject.reboot_and_reconnect = AsyncMock()
|
|
|
|
outcome = await subject.graceful_reboot_or_enospc_refusal()
|
|
|
|
self.assertEqual(outcome, ("rebooted", 6, 9))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|