Support 2 KiB DEFLATE OTA transport

This commit is contained in:
mikecarper committed 2026-09-03 01:07:06 -07:00
1 parent 066f1abca6
commit e2da2701da
32 files changed
+3877 -357

No files matched your search

+4 -2
View File
@@ -57,6 +57,7 @@ STATUS_OK = 0
STATUS_ERR = 1
MOTA_READ_MAX = 192
MOTA_DEFLATE_CHUNK_MAX = 190
MOTA_DEFLATE_BLOCK_MAX = 2048
MOTA_DESC_WIRE = 38
MOTA_SOURCE_CAP_DEFLATE_BLOCK = 0x01
MOTA_HEADER_LEN = 8
@@ -88,14 +89,15 @@ class MotaFile:
return None
raw_offset = block_index * block_size
raw_length = min(block_size, payload_size - raw_offset)
if raw_length <= 0 or raw_length > 1024 or self.path.stat().st_size != self.size:
if (raw_length <= 0 or raw_length > MOTA_DEFLATE_BLOCK_MAX
or self.path.stat().st_size != self.size):
return None
with self.path.open("rb") as stream:
stream.seek(payload_offset + raw_offset)
raw = stream.read(raw_length)
if len(raw) != raw_length:
return None
compressor = zlib.compressobj(level=9, method=zlib.DEFLATED, wbits=-10)
compressor = zlib.compressobj(level=9, method=zlib.DEFLATED, wbits=-11)
encoded = compressor.compress(raw) + compressor.flush()
return encoded if 0 < len(encoded) < len(raw) else None
+18 -6
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import hashlib
import struct
import tempfile
import unittest
@@ -24,16 +25,21 @@ class TransportDeflateTests(unittest.TestCase):
def setUp(self):
self.directory = tempfile.TemporaryDirectory()
self.path = Path(self.directory.name) / "test.mota"
# This distribution makes zlib level 9 choose a dynamic-Huffman block, proving the
# host is not silently constrained to the bootloader's separate fixed-only profile.
self.raw = bytes(i % 31 if i % 7 else 0 for i in range(1024))
# The repeated 512-byte prefix is 1536 bytes behind its second copy. A 2 KiB
# application window can encode that match; the legacy 1 KiB window cannot.
noise = b"".join(
hashlib.sha256(index.to_bytes(4, "little")).digest()
for index in range(48)
)
self.raw = noise[:512] + noise[512:1536] + noise[:512]
self.assertEqual(len(self.raw), seeder.MOTA_DEFLATE_BLOCK_MAX)
payload_offset = 256
self.path.write_bytes(bytes(payload_offset) + self.raw)
descriptor = bytearray(seeder.MOTA_DESC_WIRE)
struct.pack_into("<I", descriptor, 22, 1) # block_count
struct.pack_into("<I", descriptor, 26, payload_offset)
struct.pack_into("<I", descriptor, 30, len(self.raw))
descriptor[34] = 10
descriptor[34] = 11
mota = seeder.MotaFile(self.path, self.path.stat().st_size, bytes(descriptor))
self.catalog = seeder.Catalog([mota], False)
@@ -49,7 +55,7 @@ class TransportDeflateTests(unittest.TestCase):
self.assertIsNotNone(response)
return response_payload(response, seeder.OP_DEFLATE_BLOCK)
def test_serves_independent_raw_deflate_in_bounded_chunks(self):
def test_serves_2k_raw_deflate_with_long_distance_match(self):
status, payload = self.exchange(0, 0, 0)
self.assertEqual(status, seeder.STATUS_OK)
total = struct.unpack("<H", payload)[0]
@@ -63,7 +69,13 @@ class TransportDeflateTests(unittest.TestCase):
self.assertEqual(struct.unpack_from("<H", payload)[0], total)
encoded.extend(payload[2:])
self.assertEqual((encoded[0] >> 1) & 0x03, 2) # dynamic Huffman (BTYPE=2)
self.assertEqual(zlib.decompress(bytes(encoded), wbits=-10), self.raw)
self.assertEqual(zlib.decompress(bytes(encoded), wbits=-11), self.raw)
compressor_1k = zlib.compressobj(
level=9, method=zlib.DEFLATED, wbits=-10
)
encoded_1k = compressor_1k.compress(self.raw) + compressor_1k.flush()
self.assertLess(len(encoded) + 400, len(encoded_1k))
def test_rejects_bad_block_range_and_oversized_chunk(self):
self.assertEqual(self.exchange(1, 0, 0)[0], seeder.STATUS_ERR)
File diff suppressed because it is too large. Load diff
+83 -22
View File
@@ -29,7 +29,7 @@ import motalib # noqa: E402
ENV_NAME = "RAK_3401_repeater_lora_ota_no_external_sensors"
VERSION_SUFFIX = "halo-keymind-cascade-mota-inflate"
VERSION_SUFFIX = "halo-keymind-cascade-mota-inflate2k"
EXPECTED_TARGET_ID = 0x2FA509C1
EXPECTED_HARDWARE = "RAK_3401"
COMMON_PATCH = SCRIPT_DIR / "rak3401_terminal_ota_backport.patch"
@@ -39,16 +39,18 @@ VERSION_PATCH = SCRIPT_DIR / "rak3401_four_component_endf_backport.patch"
WORKSPACE_PATCH = SCRIPT_DIR / "rak3401_dynamic_workspace_backport.patch"
SELECTIVE_OS_HOOK = SCRIPT_DIR / "rak3401_selective_os.py"
INFLATE_ASSET_ROOT = SCRIPT_DIR / "rak3401_inflate_assets"
INFLATE_SNAPSHOT_COMMIT = "add51bf00c46c15ef54318ca766a6daf08a147ee"
# Provenance of the receive shim these pinned assets extend. The exact amended
# sources are authenticated independently by INFLATE_ASSETS below.
INFLATE_BASE_SNAPSHOT_COMMIT = "add51bf00c46c15ef54318ca766a6daf08a147ee"
INFLATE_ASSETS = {
"src/helpers/ota/OtaDeflate.cpp": (
"OtaDeflate.cpp",
"7cc464fc3c304cc65f52973593068c8d85a783853da121476c4bc2e196798e8e",
"524328f798c931ce483916cf2c517f04df933d6d9d50d868abc907dc0544bbae",
),
"src/helpers/ota/OtaDeflate.h": (
"OtaDeflate.h",
"f6cd5b0fe8b2164178881d93664df37722d6434f8ef61c37d08255bce12f842c",
"39c3e5ca4d90b56f1631c75bb1dd4d8b9589aaa8705b936f46bfa2f890e0c154",
),
"src/helpers/ota/OtaTinf.c": (
"OtaTinf.c",
@@ -244,10 +246,13 @@ def validate_inflate_source(source: Path) -> None:
"void manager receive ABI": "void on_message(const uint8_t* msg, uint16_t len)",
"requested block hook declaration": "uint16_t requestedBlockLength(",
"requested block hook implementation": "uint16_t OtaManager::requestedBlockLength(",
"nominal block geometry hook": "uint16_t* nominal_size = nullptr",
"inflater context member": "OtaTransportInflateReceiver transport_inflate;",
"inflater context begin": "transport_inflate.begin(manager, target_id, send, ctx);",
"v2 request marker": "OTA_REQ_V2_MARK = 0x8000u",
"2 KiB logical block maximum": "#define OTA_MAX_BLOCK 2048",
"v2 fragment size": "#define OTA_FRAG_DATA_V2 171",
"v2 extended length flag": "OTA_REQ_V2_EXTENDED_LENGTH = 0x2000u",
"tinf build filter": "+<helpers/ota/OtaTinf.c>",
}
haystacks = {
@@ -255,10 +260,13 @@ def validate_inflate_source(source: Path) -> None:
"void manager receive ABI": manager_h,
"requested block hook declaration": manager_h,
"requested block hook implementation": manager_cpp,
"nominal block geometry hook": manager_h,
"inflater context member": context_h,
"inflater context begin": context_h,
"v2 request marker": protocol_h,
"2 KiB logical block maximum": manager_h,
"v2 fragment size": manager_h,
"v2 extended length flag": protocol_h,
"tinf build filter": platformio,
}
for label, needle in required.items():
@@ -277,6 +285,16 @@ def apply_inflate_transforms(source: Path) -> None:
"""Apply the add51bf0 receive shim through exact, fail-closed anchors."""
manager_h = source / "src/helpers/ota/OtaManager.h"
replace_exact(
manager_h,
b"#ifndef OTA_MAX_BLOCK\n"
b"#define OTA_MAX_BLOCK 1024 // largest logical block (merkle leaf unit) = reassembly buffer size\n"
b"#endif\n",
b"#ifndef OTA_MAX_BLOCK\n"
b"#define OTA_MAX_BLOCK 2048 // receive bridge accepts extended 2 KiB logical blocks\n"
b"#endif\n",
"OTA logical block maximum",
)
replace_exact(
manager_h,
b"#ifndef OTA_FRAG_DATA\n"
@@ -297,7 +315,8 @@ def apply_inflate_transforms(source: Path) -> None:
b" bool terminallyConsumes(const uint8_t* msg, uint16_t len);\n"
b" // Exact logical length of a block this receiver currently requested, or zero for unsolicited DATA.\n"
b" // Used by the historical transport shim without exposing or changing the manager's reassembly buffers.\n"
b" uint16_t requestedBlockLength(const uint8_t* manifest_id, uint16_t block) const;\n"
b" uint16_t requestedBlockLength(const uint8_t* manifest_id, uint16_t block,\n"
b" uint16_t* nominal_size = nullptr) const;\n"
b" void on_message(const uint8_t* msg, uint16_t len); // feed one received OTA message\n",
"OtaManager receive shim declaration",
)
@@ -306,10 +325,13 @@ def apply_inflate_transforms(source: Path) -> None:
replace_exact(
manager_cpp,
b"bool OtaManager::terminallyConsumes(const uint8_t* msg, uint16_t len) {\n",
b"uint16_t OtaManager::requestedBlockLength(const uint8_t* manifest_id, uint16_t block) const {\n"
b"uint16_t OtaManager::requestedBlockLength(const uint8_t* manifest_id, uint16_t block,\n"
b" uint16_t* nominal_size) const {\n"
b" if (nominal_size) *nominal_size = 0;\n"
b" if (!manifest_id || !_fetch || _fstate != FETCHING || block >= _fbc ||\n"
b" memcmp(manifest_id, _fid, sizeof(_fid)) != 0 || findReassemblySlot(block) < 0) return 0;\n"
b" const uint32_t len = blockLen(block);\n"
b" if (nominal_size && _fbs <= OTA_MAX_BLOCK) *nominal_size = (uint16_t)_fbs;\n"
b" return len <= OTA_MAX_BLOCK ? (uint16_t)len : 0;\n"
b"}\n\n"
b"bool OtaManager::terminallyConsumes(const uint8_t* msg, uint16_t len) {\n",
@@ -326,37 +348,54 @@ def apply_inflate_transforms(source: Path) -> None:
b"};\n"
)
v2_protocol = data_message + (
b"\n// OTA_REQ want_mask extension. These bits sit outside every valid fragment bit for <=1 KiB blocks. A first\n"
b"// v2 request deliberately includes all seven legacy fragment bits so an old source can answer it completely.\n"
b"\n// OTA_REQ want_mask extension. These bits sit outside every valid fragment bit: a 2 KiB block needs at\n"
b"// most 13 legacy or 12 v2 fragments. A first request includes all legacy fragment bits so an old source\n"
b"// can answer a 1 KiB block completely.\n"
b"static const uint16_t OTA_REQ_V2_MARK = 0x8000u;\n"
b"static const uint16_t OTA_REQ_V2_ALLOW_DEFLATE = 0x4000u;\n"
b"static const uint16_t OTA_REQ_V2_RESERVED = 0x2000u;\n"
b"static const uint16_t OTA_REQ_V2_EXTENDED_LENGTH = 0x2000u;\n"
b"static const uint16_t OTA_REQ_V2_FRAGMENT_MASK = 0x1FFFu;\n"
b"\n// OTA_DATA v2 frag_off packing:\n"
b"// bit 15 v2 marker\n"
b"// bit 14 data[] is one raw-RFC1951-DEFLATE stream fragment (clear = raw block bytes)\n"
b"// bits 13..10 fragment index (0..15; byte offset = index * OTA_FRAG_DATA_V2)\n"
b"// bits 9..0 total encoded block length minus one (1..1024 bytes)\n"
b"static const uint16_t OTA_REQ_V2_EXT_FRAGMENT_MASK = 0x0FFFu;\n"
b"static const uint16_t OTA_REQ_V2_EXT_RESERVED = 0x1000u;\n"
b"\n// The 1 KiB OTA_DATA layout remains byte-for-byte compatible. An extended request selects a 2 KiB layout\n"
b"// with fragment index in bits 14..11 and encoded length minus one in bits 10..0. In that layout an\n"
b"// encoded length shorter than the manifest-derived raw length unambiguously identifies DEFLATE.\n"
b"static const uint16_t OTA_DATA_V2_MARK = 0x8000u;\n"
b"static const uint16_t OTA_DATA_V2_DEFLATED = 0x4000u;\n"
b"static const uint16_t OTA_DATA_V2_FRAGMENT_BITS = 0x3C00u;\n"
b"static const uint16_t OTA_DATA_V2_LENGTH_BITS = 0x03FFu;\n"
b"static const uint8_t OTA_DATA_V2_FRAGMENT_SHIFT = 10;\n"
b"static const uint16_t OTA_DATA_V2_MAX_ENCODED = 1024;\n"
b"static const uint16_t OTA_DATA_V2_LEGACY_MAX_ENCODED = 1024;\n"
b"static const uint16_t OTA_DATA_V2_EXT_FRAGMENT_BITS = 0x7800u;\n"
b"static const uint16_t OTA_DATA_V2_EXT_LENGTH_BITS = 0x07FFu;\n"
b"static const uint8_t OTA_DATA_V2_EXT_FRAGMENT_SHIFT = 11;\n"
b"static const uint16_t OTA_DATA_V2_MAX_ENCODED = 2048;\n"
b"static const uint8_t OTA_DATA_V2_STREAM_ID_BYTES = 4;\n"
b"\ninline bool ota_req_is_v2(uint16_t want_mask) {\n"
b" return (want_mask & OTA_REQ_V2_MARK) != 0 && (want_mask & OTA_REQ_V2_RESERVED) == 0;\n"
b" if ((want_mask & OTA_REQ_V2_MARK) == 0) return false;\n"
b" return (want_mask & OTA_REQ_V2_EXTENDED_LENGTH) == 0 ||\n"
b" (want_mask & OTA_REQ_V2_EXT_RESERVED) == 0;\n"
b"}\n"
b"\ninline uint16_t ota_req_v2_fragments(uint16_t want_mask) {\n"
b" return (uint16_t)(want_mask & OTA_REQ_V2_FRAGMENT_MASK);\n"
b" const uint16_t mask = (want_mask & OTA_REQ_V2_EXTENDED_LENGTH)\n"
b" ? OTA_REQ_V2_EXT_FRAGMENT_MASK : OTA_REQ_V2_FRAGMENT_MASK;\n"
b" return (uint16_t)(want_mask & mask);\n"
b"}\n"
b"\ninline uint16_t ota_req_make_v2(uint16_t fragments, bool allow_deflate) {\n"
b" return (uint16_t)((fragments & OTA_REQ_V2_FRAGMENT_MASK) | OTA_REQ_V2_MARK |\n"
b" (allow_deflate ? OTA_REQ_V2_ALLOW_DEFLATE : 0));\n"
b"\ninline bool ota_req_v2_extended_length(uint16_t want_mask) {\n"
b" return (want_mask & OTA_REQ_V2_EXTENDED_LENGTH) != 0;\n"
b"}\n"
b"\ninline uint16_t ota_req_make_v2(uint16_t fragments, bool allow_deflate,\n"
b" bool extended_length = false) {\n"
b" const uint16_t fragment_mask = extended_length\n"
b" ? OTA_REQ_V2_EXT_FRAGMENT_MASK : OTA_REQ_V2_FRAGMENT_MASK;\n"
b" return (uint16_t)((fragments & fragment_mask) | OTA_REQ_V2_MARK |\n"
b" (allow_deflate ? OTA_REQ_V2_ALLOW_DEFLATE : 0) |\n"
b" (extended_length ? OTA_REQ_V2_EXTENDED_LENGTH : 0));\n"
b"}\n"
b"\ninline bool ota_data_v2_pack(uint8_t fragment, uint16_t encoded_len, bool deflated,\n"
b" uint16_t& packed) {\n"
b" if (fragment >= 16 || encoded_len == 0 || encoded_len > OTA_DATA_V2_MAX_ENCODED) return false;\n"
b" if (fragment >= 16 || encoded_len == 0 ||\n"
b" encoded_len > OTA_DATA_V2_LEGACY_MAX_ENCODED) return false;\n"
b" packed = (uint16_t)(OTA_DATA_V2_MARK |\n"
b" (deflated ? OTA_DATA_V2_DEFLATED : 0) |\n"
b" ((uint16_t)fragment << OTA_DATA_V2_FRAGMENT_SHIFT) |\n"
@@ -371,6 +410,26 @@ def apply_inflate_transforms(source: Path) -> None:
b" deflated = (packed & OTA_DATA_V2_DEFLATED) != 0;\n"
b" return true;\n"
b"}\n"
b"\ninline bool ota_data_v2_pack_extended(uint8_t fragment, uint16_t encoded_len,\n"
b" uint16_t& packed) {\n"
b" if (fragment >= 16 || encoded_len == 0 || encoded_len > OTA_DATA_V2_MAX_ENCODED) return false;\n"
b" packed = (uint16_t)(OTA_DATA_V2_MARK |\n"
b" ((uint16_t)fragment << OTA_DATA_V2_EXT_FRAGMENT_SHIFT) |\n"
b" (encoded_len - 1u));\n"
b" return true;\n"
b"}\n"
b"\ninline bool ota_data_v2_unpack_extended(uint16_t packed, uint16_t raw_len,\n"
b" uint8_t& fragment, uint16_t& encoded_len,\n"
b" bool& deflated) {\n"
b" if ((packed & OTA_DATA_V2_MARK) == 0 || raw_len == 0 ||\n"
b" raw_len > OTA_DATA_V2_MAX_ENCODED) return false;\n"
b" fragment = (uint8_t)((packed & OTA_DATA_V2_EXT_FRAGMENT_BITS) >>\n"
b" OTA_DATA_V2_EXT_FRAGMENT_SHIFT);\n"
b" encoded_len = (uint16_t)((packed & OTA_DATA_V2_EXT_LENGTH_BITS) + 1u);\n"
b" if (encoded_len > raw_len) return false;\n"
b" deflated = encoded_len < raw_len;\n"
b" return true;\n"
b"}\n"
)
replace_exact(protocol_h, data_message, v2_protocol, "OTA v2 protocol helpers")
@@ -679,7 +738,7 @@ def main() -> int:
"target_id": f"{EXPECTED_TARGET_ID:08X}",
"hardware": EXPECTED_HARDWARE,
"version_suffix": VERSION_SUFFIX,
"inflate_snapshot_commit": INFLATE_SNAPSHOT_COMMIT,
"inflate_base_snapshot_commit": INFLATE_BASE_SNAPSHOT_COMMIT,
"inflate_assets": asset_metadata,
"builder_script": Path(__file__).name,
"builder_script_sha256": sha256_file(Path(__file__)),
@@ -696,6 +755,8 @@ def main() -> int:
"selective_os_hook": SELECTIVE_OS_HOOK.name,
"selective_os_hook_sha256": sha256_file(SELECTIVE_OS_HOOK),
"transport_fragment_bytes": 171,
"transport_max_block_bytes": 2048,
"transport_descriptor_layouts": ["v2-1k", "v2-2k-extended"],
"transport_codec": "raw-deflate-receive-only",
"fetch_pipeline": 1,
"targets": records,
+44 -2
View File
@@ -57,7 +57,8 @@ ENDF_MAGIC = b"EndF"
ENDF_SIZE = 56
MAX_ARCHIVE_MEMBER_SIZE = 64 * 1024 * 1024
MAX_FIRMWARE_IMAGE_SIZE = 64 * 1024 * 1024
MOTA_MAX_BLOCK_SIZE = 1024
LEGACY_TARGET_MAX_BLOCK_SIZE = 1024
MOTA_MAX_BLOCK_SIZE = 2048
TRANSMISSION_RETRY_LIMIT = 3
TRANSMISSION_RETRY_WINDOW_SECONDS = 90
TRANSMISSION_RETRY_DELAY_SECONDS = 2
@@ -230,6 +231,8 @@ class TargetInfo:
current_version: str | None = None
current_version_source: str | None = None
nrf_qspi: bool = False
# Firmware predating the live maxblk marker had 1 KiB receive buffers.
max_block_size: int = LEGACY_TARGET_MAX_BLOCK_SIZE
@property
def nrf_external(self) -> bool:
@@ -832,7 +835,7 @@ def parse_mota(blob: bytes, path: Path | None = None) -> MotaInfo:
target_id, fw_version, image_size, payload_size = struct.unpack_from("<IIII", blob, 11)
block_size_log2 = blob[27]
if block_size_log2 == 0 or block_size_log2 > 10:
if block_size_log2 == 0 or block_size_log2 > 11:
raise OtaError(
f"invalid mOTA block size; firmware supports at most "
f"{MOTA_MAX_BLOCK_SIZE} bytes"
@@ -1002,6 +1005,11 @@ def compatible_mota(info: MotaInfo, target: TargetInfo) -> tuple[bool, str]:
return False, f"target {info.target_id:08X}, need {target.target_id:08X}"
if info.hw_id and target.hw_id and info.hw_id != target.hw_id:
return False, f"hardware {info.hw_id!r}, destination is {target.hw_id!r}"
if info.block_size > target.max_block_size:
return False, (
f"block size {info.block_size} bytes exceeds destination "
f"maxblk:{target.max_block_size}"
)
if info.is_full:
if target.platform == "nrf52" and not target.nrf_external:
return False, "internal-flash nRF52 accepts only an in-place delta"
@@ -1382,6 +1390,8 @@ def prepare_package(
"build",
"--fw",
str(new_image),
"--block-size",
str(target.max_block_size),
"--out",
str(output),
]
@@ -2359,6 +2369,33 @@ def split_host_port(value: str, default_port: int) -> tuple[str, int]:
return host, port
def parse_target_max_block_size(status: str, self_status: str) -> int:
"""Read the live application receive limit, with a safe deployed fallback."""
markers = re.findall(
r"\bmaxblk:(\d+)\b", f"{status} {self_status}", re.IGNORECASE
)
if not markers:
return LEGACY_TARGET_MAX_BLOCK_SIZE
values = {int(marker) for marker in markers}
if len(values) != 1:
rendered = ", ".join(str(value) for value in sorted(values))
raise OtaError(
f"destination reports conflicting maxblk capabilities: {rendered}"
)
value = values.pop()
if (
value <= 0
or value > MOTA_MAX_BLOCK_SIZE
or value & (value - 1)
):
raise OtaError(
f"destination reports unsupported maxblk:{value}; this runner "
f"supports power-of-two application blocks through "
f"{MOTA_MAX_BLOCK_SIZE} bytes"
)
return value
def query_target(
controller: Controller,
args: argparse.Namespace,
@@ -2381,6 +2418,7 @@ def query_target(
if not hash_match:
raise OtaError("could not read destination base hash from `ota self`")
base_hash = bytes.fromhex(hash_match.group(1))
max_block_size = parse_target_max_block_size(status, self_status)
combined = f"{status} {self_status}"
if reported_platform is None:
platform = (
@@ -2479,6 +2517,7 @@ def query_target(
current_version=current_version,
current_version_source=current_version_source,
nrf_qspi=nrf_qspi,
max_block_size=max_block_size,
)
@@ -3894,6 +3933,7 @@ def confirm_update(
print("\nValidated update plan:")
print(f" destination : {target.name} ({target.target_id:08X}, {target.platform})")
print(f" running base: {target.base_hash.hex().upper()}")
print(f" max block : {target.max_block_size} bytes")
if target.platform == "nrf52":
version = target.bootloader_version or "unknown"
print(
@@ -7023,6 +7063,8 @@ def offline_target(args: argparse.Namespace) -> TargetInfo:
status="offline",
self_status="offline",
nrf_qspi=args.nrf_qspi,
# There is no live query in prepare-only mode; retain the current application default.
max_block_size=MOTA_MAX_BLOCK_SIZE,
)
@@ -115,7 +115,12 @@ void OtaTransportInflateReceiver::send(const uint8_t* msg, uint16_t len, bool fl
}
if (!_legacy_session) {
request.want_mask = ota_req_make_v2(fragments, _allow_deflate);
uint16_t nominal_block_size = 0;
(void)_manager->requestedBlockLength(request.manifest_id, request.block_idx,
&nominal_block_size);
request.want_mask = ota_req_make_v2(
fragments, _allow_deflate,
nominal_block_size > OTA_DATA_V2_LEGACY_MAX_ENCODED);
uint8_t wire[16];
const uint16_t wire_len = encode_req(wire, sizeof(wire), request);
if (wire_len != 0) {
@@ -162,13 +167,20 @@ bool OtaTransportInflateReceiver::handle_v2_data(const uint8_t* msg, uint16_t le
if (!decode_data(msg, len, data) || !is_active(data.manifest_id, data.block_idx) ||
data.data_len <= OTA_DATA_V2_STREAM_ID_BYTES || _legacy_session) return false;
const uint16_t block_len = _manager->requestedBlockLength(data.manifest_id, data.block_idx);
uint16_t nominal_block_size = 0;
const uint16_t block_len = _manager->requestedBlockLength(
data.manifest_id, data.block_idx, &nominal_block_size);
if (block_len == 0) return false;
uint8_t fragment = 0;
uint16_t encoded_len = 0;
bool deflated = false;
if (!ota_data_v2_unpack(data.frag_off, fragment, encoded_len, deflated)) return false;
const bool extended_length = nominal_block_size > OTA_DATA_V2_LEGACY_MAX_ENCODED;
const bool unpacked = extended_length
? ota_data_v2_unpack_extended(data.frag_off, block_len,
fragment, encoded_len, deflated)
: ota_data_v2_unpack(data.frag_off, fragment, encoded_len, deflated);
if (!unpacked) return false;
if (encoded_len > block_len || (deflated ? encoded_len >= block_len : encoded_len != block_len) ||
(deflated && !_allow_deflate)) return false;
@@ -15,10 +15,11 @@ bool ota_transport_inflate(void* context, const uint8_t* src, uint16_t src_len,
uint8_t* dst, uint16_t dst_cap, uint16_t* dst_len);
// Receive-only compatibility shim for historical OTA managers. The wrapped manager continues to reassemble
// canonical 160-byte DATA, verify the Merkle proof, and stage the original `.mota`. This shim only negotiates
// the v2 wire profile, locks and reassembles one encoded representation, exact-inflates it, and injects the
// resulting logical block back into that unchanged path. Bridge builds use OTA_FETCH_PIPELINE=1, so one fixed
// encoded block and one fixed decoded block cover every in-flight request without heap allocation.
// canonical 160-byte DATA, verify the Merkle proof, and stage the original `.mota`. This shim negotiates both
// the original 1 KiB and extended 2 KiB v2 wire descriptors, locks and reassembles one encoded representation,
// exact-inflates it, and injects the resulting logical block back into that unchanged path. Bridge builds use
// OTA_FETCH_PIPELINE=1, so one fixed encoded block and one fixed decoded block cover every in-flight request
// without heap allocation.
class OtaTransportInflateReceiver {
public:
void begin(OtaManager& manager, uint32_t target_id, OtaSend send, void* send_ctx);
+107 -35
View File
@@ -29,24 +29,42 @@ except ImportError:
import lora_ota as ota
# The published b40d2e6c prerelease keeps the first nine package bytes from the
# The published 3f6eddd5 prerelease keeps the first nine package bytes from the
# physically passed fd98bc90 chain and replaces its endpoint package. The new
# step 10 and this hardened host runner have exhaustive offline/simulator
# evidence, but not a clean physical transition run, so live use remains gated.
# step 10 and this updated host runner have exhaustive offline/exact-bootloader
# simulator evidence, but not a clean physical transition run, so live use
# remains gated.
RELEASE_URL = (
"https://github.com/mikecarper/MeshCore/releases/tag/"
"rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.5-b40d2e6c"
"rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.6-3f6eddd5"
)
ASSET_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5.zip"
ASSET_SHA256 = "135783dd8777490db2422a7543f7d80e3cf03f621d5ac362da930abbc1850dc4"
CHECKSUM_LIST_SHA256 = "93835ed497c579be0a2322292e7f15ab095e7c8ab7892833fe7df3130dbdaba4"
BUNDLE_ROOT_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5"
# Preserve the exact b40d2e6c prerelease that was primary before 3f6eddd5. It
# remains a separately recognized, hidden-lab-ack-gated superseded artifact;
# sharing its first nine targets with the current release must never make the
# archives interchangeable.
SUPERSEDED_B40_ASSET_NAME = (
"RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c.zip"
)
SUPERSEDED_B40_ASSET_SHA256 = (
"8e7f0c2565f37d4372fa51dc3ba72b591e7f38b74be09303e48641623f99bce4"
)
SUPERSEDED_B40_CHECKSUM_LIST_SHA256 = (
"29a02a6c4c83eafa80168a80562b0ded5d8432e6cc02cfaca289cb2ed426681a"
)
SUPERSEDED_B40_ROOT_NAME = (
"RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c"
)
ASSET_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c.zip"
ASSET_SHA256 = "8e7f0c2565f37d4372fa51dc3ba72b591e7f38b74be09303e48641623f99bce4"
CHECKSUM_LIST_SHA256 = "29a02a6c4c83eafa80168a80562b0ded5d8432e6cc02cfaca289cb2ed426681a"
BUNDLE_ROOT_NAME = "RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c"
# Preserve the exact fd98bc90 candidate whose ten package transitions passed
# directly on the physical RAK3401 and whose endpoint passed independent SWD
# readback. It remains a separately identified, hidden-lab-ack-gated legacy
# artifact; sharing its version and first nine targets with b40d2e6c must never
# make the two archives interchangeable.
# artifact; sharing its version and first nine targets with later candidates
# must never make the archives interchangeable.
PHYSICALLY_PASSED_FD98_ASSET_NAME = (
"RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-fd98bc90.zip"
)
@@ -160,6 +178,7 @@ MAX_CHECKSUM_LIST_BYTES = 1024 * 1024
MAX_PROGRESS_BYTES = 1024 * 1024
PINNED_ARCHIVE_CHECKSUMS = {
ASSET_SHA256: CHECKSUM_LIST_SHA256,
SUPERSEDED_B40_ASSET_SHA256: SUPERSEDED_B40_CHECKSUM_LIST_SHA256,
PHYSICALLY_PASSED_FD98_ASSET_SHA256: (
PHYSICALLY_PASSED_FD98_CHECKSUM_LIST_SHA256
),
@@ -185,9 +204,12 @@ EXPECTED_HARDWARE = "RAK_3401"
EXPECTED_STEP_COUNT = 10
PHYSICALLY_QUALIFIED_9_STEP_COUNT = 9
EXPECTED_START_VERSION = "1.16.7.0"
EXPECTED_FINAL_VERSION = "1.17.1.5"
EXPECTED_FINAL_VERSION = "1.17.1.6"
SUPERSEDED_B40_FINAL_VERSION = "1.17.1.5"
PHYSICALLY_PASSED_FD98_FINAL_VERSION = "1.17.1.5"
PHYSICALLY_QUALIFIED_9_FINAL_VERSION = "1.17.1.02"
SUPERSEDED_FINAL_VERSION = "1.17.1.0"
EXPECTED_BOOTLOADER_VERSION = "0.9.2-OTAFIX2.4"
WATCHDOG_RESET_WAIT_SECONDS = 90
WATCHDOG_STABILITY_WAIT_SECONDS = 90
TRANSFER_SETTINGS_FILE = "target-transfer-settings.json"
@@ -265,8 +287,8 @@ COMPACT_RELEASE_ANCHORS = (
)
# Every target in the exact legacy fd98bc90 sequence whose package transitions
# passed directly on the physical RAK3401. The b40d2e6c prerelease reuses the
# first nine package bytes exactly, but has its own endpoint pin below.
# passed directly on the physical RAK3401. Later prereleases reuse the first
# nine package bytes exactly, but each has its own endpoint pin below.
PHYSICALLY_PASSED_FD98_ANCHORS = (
(1, "8364257a2b3a219905e870fad6fbb2040a96ca4b4bb7201b2867534cc2b45530"),
(2, "ac5f50e5028378ccfe6ea08bbf32f227f50fdcf5285a7deb866e309fbdd0a88f"),
@@ -280,11 +302,16 @@ PHYSICALLY_PASSED_FD98_ANCHORS = (
(10, "31c182c888ceb1135e5afb2376610d93cee2e807b556c838e07fd4486c79d095"),
)
CURRENT_10_CANDIDATE_ANCHORS = (
SUPERSEDED_B40_ANCHORS = (
*PHYSICALLY_PASSED_FD98_ANCHORS[:9],
(10, "a7761da8af1f2447d7f01d72e2b1456b99c0e874a165d6483f48ab1f58e8d905"),
)
CURRENT_10_CANDIDATE_ANCHORS = (
*PHYSICALLY_PASSED_FD98_ANCHORS[:9],
(10, "cd1e9b819f9e918f9f2ceaf37b8a74bf242e2be78f319af2cf02a42d7729d0f4"),
)
# Exact anchors for every image in the accelerated 30-step release. The outer
# asset and inner checksum-list pins cover every byte as well; the complete
# anchor table keeps structural validation fail-closed if those layers are
@@ -363,13 +390,21 @@ SUPERSEDED_30_MESSAGE = (
"the deployed bootloader. Use --verify-only for the older bundle."
)
CURRENT_10_CANDIDATE_MESSAGE = (
"live installation of the exact published b40d2e6c ten-step prerelease is "
"live installation of the exact published 3f6eddd5 ten-step prerelease is "
"disabled: steps 1-9 are byte-identical to package transitions that "
"completed directly on the physical RAK3401, but the new b40d2e6c step 10 "
"and this hardened host runner have passed offline reconstruction, motatool "
"verification, and both bootloader simulators only. The exact final "
"transition has not had a clean physical run. Use --verify-only, or the "
"hidden controlled-lab override, until that qualification is complete."
"completed directly on the physical RAK3401, but the new 3f6eddd5 step 10 "
"and this updated host runner have passed offline reconstruction, motatool "
"verification, and the exact OTAFIX2.4 bootloader simulator only. The exact "
"final transition has not had a clean physical run. Use --verify-only, or "
"the hidden controlled-lab override, until that qualification is complete."
)
SUPERSEDED_B40_MESSAGE = (
"live installation of the exact superseded b40d2e6c ten-step prerelease is "
"disabled: steps 1-9 are byte-identical to package transitions that "
"completed directly on the physical RAK3401, but its step 10 and this "
"updated host runner have not had a clean physical run. Use --verify-only "
"with the explicit superseded bundle path, or the hidden controlled-lab "
"override."
)
PHYSICALLY_PASSED_FD98_MESSAGE = (
"live installation of the exact legacy fd98bc90 ten-step candidate is "
@@ -398,7 +433,9 @@ def matches_exact_anchors(
def ten_step_candidate_kind(steps: list[ChainStep]) -> str | None:
if matches_exact_anchors(steps, CURRENT_10_CANDIDATE_ANCHORS):
return "current-b40d2e6c"
return "current-3f6eddd5"
if matches_exact_anchors(steps, SUPERSEDED_B40_ANCHORS):
return "superseded-b40d2e6c"
if matches_exact_anchors(steps, PHYSICALLY_PASSED_FD98_ANCHORS):
return "legacy-fd98bc90"
return None
@@ -406,15 +443,25 @@ def ten_step_candidate_kind(steps: list[ChainStep]) -> str | None:
def ten_step_verification_message(steps: list[ChainStep]) -> str:
candidate_kind = ten_step_candidate_kind(steps)
if candidate_kind == "current-b40d2e6c":
if candidate_kind == "current-3f6eddd5":
return (
"Current b40d2e6c ten-step prerelease verified offline: exact "
"Current 3f6eddd5 ten-step prerelease verified offline: exact "
"archive/checksum pins, all target anchors, zero-filled and "
"erased-workspace reconstruction, independent motatool verification, "
"and both bootloader simulations passed. Steps 1-9 are byte-identical "
"to packages that passed directly on the RAK3401. The new step 10 and "
"this hardened host runner have not had a clean physical run; live "
"use remains gated."
"and the exact OTAFIX2.4 bootloader simulation passed. Steps 1-9 are "
"byte-identical to packages that passed directly on the RAK3401. The "
"new step 10 and this updated host runner have not had a clean "
"physical run; live use remains gated."
)
if candidate_kind == "superseded-b40d2e6c":
return (
"Superseded b40d2e6c ten-step prerelease verified offline: exact "
"archive/checksum pins, all target anchors, zero-filled and "
"erased-workspace reconstruction, independent motatool verification, "
"and its pinned bootloader simulations passed. Steps 1-9 are "
"byte-identical to packages that passed directly on the RAK3401. Its "
"step 10 and this updated host runner have not had a clean physical "
"run; live use remains gated."
)
if candidate_kind == "legacy-fd98bc90":
return (
@@ -435,10 +482,14 @@ def require_live_release_safe(
) -> None:
if len(steps) == EXPECTED_STEP_COUNT:
candidate_kind = ten_step_candidate_kind(steps)
if candidate_kind == "current-b40d2e6c":
if candidate_kind == "current-3f6eddd5":
if args.accept_test_candidate:
return
raise KnownUnsafeReleaseError(CURRENT_10_CANDIDATE_MESSAGE)
if candidate_kind == "superseded-b40d2e6c":
if args.accept_test_candidate:
return
raise KnownUnsafeReleaseError(SUPERSEDED_B40_MESSAGE)
if candidate_kind == "legacy-fd98bc90":
if args.accept_test_candidate:
return
@@ -1008,6 +1059,7 @@ def locate_bundle(args: argparse.Namespace, work_dir: Path) -> Path:
)
if actual not in {
ASSET_SHA256,
SUPERSEDED_B40_ASSET_SHA256,
PHYSICALLY_PASSED_FD98_ASSET_SHA256,
PHYSICALLY_QUALIFIED_9_ASSET_SHA256,
SUPERSEDED_30_ASSET_SHA256,
@@ -1033,6 +1085,7 @@ def read_checksum_entries(bundle_root: Path) -> dict[PurePosixPath, str]:
checksum_digest = hashlib.sha256(checksum_bytes).hexdigest()
expected_lists = {
CHECKSUM_LIST_SHA256,
SUPERSEDED_B40_CHECKSUM_LIST_SHA256,
PHYSICALLY_PASSED_FD98_CHECKSUM_LIST_SHA256,
PHYSICALLY_QUALIFIED_9_CHECKSUM_LIST_SHA256,
SUPERSEDED_30_CHECKSUM_LIST_SHA256,
@@ -1292,6 +1345,10 @@ def parse_chain(bundle_root: Path) -> tuple[list[ChainStep], bytes]:
relative = safe_relative_path(row["mota_file"], "mOTA path")
path = bundle_root.joinpath(*relative.parts)
size = int(row["mota_size"])
declared_block_size = row.get("mota_block_size", "")
block_size = (
int(declared_block_size) if declared_block_size else None
)
target_image_size = int(row["target_image_size"])
base_hash = bytes.fromhex(row["base_body_hash"])
target_sha256 = row["target_sha256"].lower()
@@ -1306,6 +1363,10 @@ def parse_chain(bundle_root: Path) -> tuple[list[ChainStep], bytes]:
if not path.is_file() or path.stat().st_size != size:
raise ota.OtaError(f"chain package size/path mismatch at step {number}: {path}")
package = ota.parse_mota(path.read_bytes(), path)
if block_size is not None and package.block_size != block_size:
raise ota.OtaError(
f"step {number} block size does not match CHAIN.csv"
)
expected_version = ota.parse_version(to_version)
if expected_version is None or package.fw_version != expected_version:
raise ota.OtaError(f"step {number} version does not match its mOTA manifest")
@@ -1333,10 +1394,15 @@ def parse_chain(bundle_root: Path) -> tuple[list[ChainStep], bytes]:
candidate_kind = ten_step_candidate_kind(steps)
if candidate_kind is None:
raise ota.OtaError(
"ten-step candidate does not match the exact b40d2e6c or "
"legacy fd98bc90 audited image pins"
"ten-step candidate does not match the exact 3f6eddd5, "
"superseded b40d2e6c, or legacy fd98bc90 audited image pins"
)
expected_final_version = EXPECTED_FINAL_VERSION
if candidate_kind == "current-3f6eddd5":
expected_final_version = EXPECTED_FINAL_VERSION
elif candidate_kind == "superseded-b40d2e6c":
expected_final_version = SUPERSEDED_B40_FINAL_VERSION
else:
expected_final_version = PHYSICALLY_PASSED_FD98_FINAL_VERSION
elif len(steps) == PHYSICALLY_QUALIFIED_9_STEP_COUNT:
for number, expected_sha256 in COMPACT_RELEASE_ANCHORS:
if steps[number - 1].target_sha256 != expected_sha256:
@@ -1537,6 +1603,12 @@ def query_live_target(
)
if target.platform != "nrf52" or target.nrf_external:
raise ota.OtaError("live target is not the expected internal-flash nRF52 node")
if target.bootloader_version != EXPECTED_BOOTLOADER_VERSION:
raise ota.OtaError(
"live target bootloader is "
f"{target.bootloader_version or 'unknown'!r}, expected exact deployed "
f"version {EXPECTED_BOOTLOADER_VERSION!r}"
)
if target.bootloader_abi is None or target.bootloader_abi < 2:
raise ota.OtaError("live target does not report OTAFIX mOTA ABI 2")
if target.bootloader_codecs is None or not target.bootloader_codecs & (1 << ota.MOTA_CODEC_IN_PLACE):
@@ -2327,10 +2399,10 @@ def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description=(
"Verify the exact published ten-step RAK3401 prerelease from "
"c1caa5ad to b40d2e6c. Its first nine package bytes passed directly "
"on hardware; the new final transition and this hardened host runner "
"have offline/simulator qualification only, so live use remains "
"disabled without the controlled-lab override."
"c1caa5ad to 3f6eddd5. Its first nine package bytes passed directly "
"on hardware; the new final transition and this updated host runner "
"have offline/exact-OTAFIX2.4-simulator qualification only, so live "
"use remains disabled without the controlled-lab override."
)
)
parser.add_argument(
+277 -56
View File
@@ -7,9 +7,12 @@ be migrated safely when new candidate images are inserted.
The default secondary objective remains container size. ``--objective transport``
instead measures the exact live motatool encoder and minimizes clean serialized
MeshCore bytes. Inventory boolean ``ota_transport_deflate`` and integer
``ota_fetch_pipeline`` belong to the currently running/source image; the pinned
first bootstrap is always legacy raw.
MeshCore bytes. Inventory fields ``ota_transport_deflate``,
``ota_fetch_pipeline``, and ``transport_max_block_bytes`` belong to the
currently running/source image; the pinned first bootstrap is always legacy
raw. The explicit maximum controls signed-container block geometry independently
of whether DEFLATE is allowed; either DEFLATE or 2 KiB geometry negotiates the
171-byte v2 wire profile.
"""
from __future__ import annotations
@@ -43,9 +46,21 @@ EXPECTED_TARGET_ID = 0x2FA509C1
EXPECTED_HARDWARE = "RAK_3401"
TRANSPORT_CAPABILITY = "ota_transport_deflate"
TRANSPORT_PIPELINE_FIELD = "ota_fetch_pipeline"
TRANSPORT_MAX_BLOCK_FIELD = "transport_max_block_bytes"
TRANSPORT_CAPABILITY_ALIAS = "transport_inflate"
TRANSPORT_PIPELINE_ALIAS = "fetch_pipeline"
BLOCK_SIZE = 1024
LEGACY_BLOCK_SIZE = 1024
DEFLATE_BLOCK_SIZE = 2048
TRANSPORT_PROFILE_MATRIX = {
"1024,deflate=false": "legacy-160-raw",
"1024,deflate=true": "v2-171-deflate",
"2048,deflate=false": "v2-171-raw",
"2048,deflate=true": "v2-171-deflate",
}
# Retain the historical name for callers that are specifically dealing with
# the immutable first package. New route geometry must choose a block size from
# the currently running/source image via ``source_block_size``.
BLOCK_SIZE = LEGACY_BLOCK_SIZE
LEGACY_DATA_BYTES = 160
V2_DATA_BYTES = 171
MANIFEST_BYTES = 197
@@ -58,7 +73,7 @@ TRANSPORT_FIELDS = [
]
FIELDS = [
"source", "target", "source_sha256", "target_sha256", "memory",
"payload", "container", "stage_start", "margin", "feasible", "error",
"payload", "block_size", "container", "stage_start", "margin", "feasible", "error",
] + TRANSPORT_FIELDS
@@ -74,27 +89,46 @@ def align_down(value: int, unit: int = PAGE) -> int:
return value // unit * unit
def container_size(payload_size: int) -> int:
return payload_size + 210 + 4 * ((payload_size + 1023) // 1024)
def _validate_block_size(block_size: int) -> int:
if block_size not in (LEGACY_BLOCK_SIZE, DEFLATE_BLOCK_SIZE):
raise RouteSearchError(
f"logical block size must be {LEGACY_BLOCK_SIZE} or {DEFLATE_BLOCK_SIZE}"
)
return block_size
def payload_size_from_container(total: int) -> int:
def container_size(
payload_size: int, block_size: int = LEGACY_BLOCK_SIZE
) -> int:
block_size = _validate_block_size(block_size)
if payload_size <= 0:
raise RouteSearchError("container payload must be positive")
return payload_size + 210 + 4 * (
(payload_size + block_size - 1) // block_size
)
def payload_size_from_container(
total: int, block_size: int = LEGACY_BLOCK_SIZE
) -> int:
"""Invert the fixed mOTA geometry used by the pinned bootstrap package."""
block_size = _validate_block_size(block_size)
if total <= 210:
raise RouteSearchError("container is too small to contain a payload")
max_blocks = (total - 210 + BLOCK_SIZE - 1) // BLOCK_SIZE
max_blocks = (total - 210 + block_size - 1) // block_size
for blocks in range(1, max_blocks + 1):
payload = total - 210 - 4 * blocks
if payload > 0 and (payload + BLOCK_SIZE - 1) // BLOCK_SIZE == blocks:
if payload > 0 and (payload + block_size - 1) // block_size == blocks:
return payload
raise RouteSearchError(f"container size {total} has no valid payload geometry")
def _block_lengths(payload_size: int) -> list[int]:
def _block_lengths(payload_size: int, block_size: int) -> list[int]:
if payload_size <= 0:
raise RouteSearchError("transport payload must be positive")
full, tail = divmod(payload_size, BLOCK_SIZE)
result = [BLOCK_SIZE] * full
block_size = _validate_block_size(block_size)
full, tail = divmod(payload_size, block_size)
result = [block_size] * full
if tail:
result.append(tail)
return result
@@ -136,7 +170,7 @@ def _common_transport_cost(block_count: int) -> tuple[int, int, int]:
def legacy_transport_cost(payload_size: int) -> dict[str, int | str]:
"""Ideal clean legacy transfer, including requests, DATA, and proofs."""
blocks = _block_lengths(payload_size)
blocks = _block_lengths(payload_size, LEGACY_BLOCK_SIZE)
block_count = len(blocks)
data_packets = sum(
(length + LEGACY_DATA_BYTES - 1) // LEGACY_DATA_BYTES
@@ -151,6 +185,7 @@ def legacy_transport_cost(payload_size: int) -> dict[str, int | str]:
manifest_bytes = common_bytes - proof_bytes
return {
"profile": "legacy-160-raw",
"block_size": LEGACY_BLOCK_SIZE,
"payload_bytes": payload_size,
"wire_bytes": payload_size,
"deflate_bytes": 0,
@@ -197,9 +232,10 @@ def v2_transport_cost(
deflate_blocks: int,
data_packets: int,
pipeline_capacity: int = RAK3401_PIPELINE_MAX,
block_size: int = DEFLATE_BLOCK_SIZE,
) -> dict[str, int | str]:
"""Ideal clean negotiated v2 transfer using measured seeder output."""
blocks = _block_lengths(payload_size)
blocks = _block_lengths(payload_size, block_size)
block_count = len(blocks)
raw_data_packets = sum(
(length + V2_DATA_BYTES - 1) // V2_DATA_BYTES for length in blocks
@@ -226,6 +262,7 @@ def v2_transport_cost(
manifest_bytes = common_bytes - proof_bytes
return {
"profile": "v2-171-deflate",
"block_size": block_size,
"payload_bytes": payload_size,
"wire_bytes": wire_bytes,
"deflate_bytes": deflate_bytes,
@@ -247,6 +284,29 @@ def v2_transport_cost(
}
def v2_raw_transport_cost(
payload_size: int,
pipeline_capacity: int = RAK3401_PIPELINE_MAX,
block_size: int = LEGACY_BLOCK_SIZE,
) -> dict[str, int | str]:
"""Ideal negotiated v2 transfer without compression for a comparison base."""
blocks = _block_lengths(payload_size, block_size)
data_packets = sum(
(length + V2_DATA_BYTES - 1) // V2_DATA_BYTES for length in blocks
)
result = v2_transport_cost(
payload_size,
payload_size,
0,
0,
data_packets,
pipeline_capacity,
block_size,
)
result["profile"] = "v2-171-raw"
return result
def linear_path_bytes(origin_mesh_bytes: int, packets: int, relay_hops: int) -> int:
"""Serialized bytes for an ideal single path; real flood fan-out is external."""
if origin_mesh_bytes < 0 or packets < 0:
@@ -302,6 +362,13 @@ def load_inventory(path: Path) -> list[dict[str, object]]:
isinstance(record, dict) and TRANSPORT_CAPABILITY in record
for record in images
)
strict_block_contract = strict_transport_contract or (
isinstance(capability_contract, dict)
and TRANSPORT_MAX_BLOCK_FIELD in capability_contract
) or any(
isinstance(record, dict) and TRANSPORT_MAX_BLOCK_FIELD in record
for record in images
)
edge_policy = document.get("edge_policy", {})
strict_version_ranks = (
isinstance(edge_policy, dict)
@@ -401,6 +468,25 @@ def load_inventory(path: Path) -> list[dict[str, object]]:
f"{TRANSPORT_CAPABILITY}"
)
record[TRANSPORT_CAPABILITY] = capability
raw_max_block = record.get(TRANSPORT_MAX_BLOCK_FIELD)
if raw_max_block is None:
if strict_block_contract:
raise RouteSearchError(
f"image {index} {TRANSPORT_MAX_BLOCK_FIELD} is required by "
"the inventory capability contract"
)
max_block = LEGACY_BLOCK_SIZE
else:
max_block = _integer(
raw_max_block, f"image {index} {TRANSPORT_MAX_BLOCK_FIELD}"
)
if max_block not in (LEGACY_BLOCK_SIZE, DEFLATE_BLOCK_SIZE):
raise RouteSearchError(
f"image {index} {TRANSPORT_MAX_BLOCK_FIELD} must be "
f"{LEGACY_BLOCK_SIZE} or {DEFLATE_BLOCK_SIZE}"
)
record[TRANSPORT_MAX_BLOCK_FIELD] = max_block
uses_v2 = capability or max_block > LEGACY_BLOCK_SIZE
raw_pipeline = record.get(TRANSPORT_PIPELINE_FIELD)
if (
raw_pipeline is None
@@ -408,12 +494,12 @@ def load_inventory(path: Path) -> list[dict[str, object]]:
and TRANSPORT_PIPELINE_ALIAS in record
):
raw_pipeline = record[TRANSPORT_PIPELINE_ALIAS]
if capability and raw_pipeline is None:
if uses_v2 and raw_pipeline is None:
raise RouteSearchError(
f"image {index} {TRANSPORT_PIPELINE_FIELD} is required when "
f"{TRANSPORT_CAPABILITY} is true"
f"image {index} {TRANSPORT_PIPELINE_FIELD} is required for "
"v2 transport"
)
if capability:
if uses_v2:
pipeline = _integer(
raw_pipeline, f"image {index} {TRANSPORT_PIPELINE_FIELD}"
)
@@ -426,7 +512,7 @@ def load_inventory(path: Path) -> list[dict[str, object]]:
elif raw_pipeline is not None and strict_transport_contract:
raise RouteSearchError(
f"image {index} {TRANSPORT_PIPELINE_FIELD} must be null when "
f"{TRANSPORT_CAPABILITY} is false"
"neither DEFLATE nor 2 KiB blocks are enabled"
)
else:
record[TRANSPORT_PIPELINE_FIELD] = None
@@ -532,7 +618,8 @@ def transport_stats_complete(row: dict[str, object]) -> bool:
def normalize_transport_stats(
row: dict[str, object], payload_size: int, label: str
row: dict[str, object], payload_size: int, label: str,
block_size: int = DEFLATE_BLOCK_SIZE,
) -> dict[str, object]:
present = [row.get(field) not in (None, "") for field in TRANSPORT_FIELDS]
if any(present) and not all(present):
@@ -554,16 +641,22 @@ def normalize_transport_stats(
int(normalized["v2_deflate_bytes"]),
int(normalized["v2_deflate_blocks"]),
int(normalized["v2_data_packets"]),
block_size=block_size,
)
return normalized
def measure_transport_size(tool: Path, payload: Path) -> dict[str, object]:
def measure_transport_size(
tool: Path,
payload: Path,
block_size: int = DEFLATE_BLOCK_SIZE,
) -> dict[str, object]:
"""Ask motatool's live-seeder encoder for exact per-block byte counts."""
block_size = _validate_block_size(block_size)
try:
completed = subprocess.run(
[str(tool), "transport-size", "--payload", str(payload),
"--block-size", str(BLOCK_SIZE)],
"--block-size", str(block_size)],
check=False, capture_output=True, text=True, encoding="utf-8",
timeout=60,
)
@@ -585,9 +678,9 @@ def measure_transport_size(tool: Path, payload: Path) -> dict[str, object]:
expected_payload = payload.stat().st_size
if measured.get("payload_bytes") != expected_payload:
raise RouteSearchError("transport-size tool measured the wrong payload")
if measured.get("block_size") != BLOCK_SIZE:
if measured.get("block_size") != block_size:
raise RouteSearchError("transport-size tool used the wrong block size")
expected_blocks = (expected_payload + BLOCK_SIZE - 1) // BLOCK_SIZE
expected_blocks = (expected_payload + block_size - 1) // block_size
if measured.get("block_count") != expected_blocks:
raise RouteSearchError("transport-size tool returned the wrong block count")
row = {
@@ -598,7 +691,9 @@ def measure_transport_size(tool: Path, payload: Path) -> dict[str, object]:
"v2_deflate_blocks": measured.get("deflate_blocks"),
"v2_data_packets": measured.get("data_packets"),
}
return normalize_transport_stats(row, expected_payload, str(payload))
return normalize_transport_stats(
row, expected_payload, str(payload), block_size
)
def migrate_csv(
@@ -664,6 +759,7 @@ def migrate_csv(
memory = _integer(row["memory"], f"memory at {csv_path}:{line}")
try:
payload = int(row.get("payload", ""))
recorded_block_size = row.get("block_size", "")
total = int(row.get("container", ""))
stage_start = int(row.get("stage_start", ""))
margin = int(row.get("margin", ""))
@@ -681,11 +777,24 @@ def migrate_csv(
f"successful cached geometry carries an error at "
f"{csv_path}:{line}; regenerate it"
)
expected_total = container_size(payload)
expected_stage = align_down(STAGE_CEILING - expected_total)
expected_margin = expected_stage - (APP_BASE + memory)
expected_block_size = source_block_size(images, source_node)
# Geometry caches predating per-source block sizing have no column
# and therefore contain 1 KiB container arithmetic. The detools
# payload itself does not depend on the Merkle block size, so first
# authenticate that historical arithmetic and then migrate only
# the derived container/staging fields. Transport counters do
# depend on block boundaries and are deliberately invalidated.
if recorded_block_size in (None, ""):
recorded_block_size_int = LEGACY_BLOCK_SIZE
else:
recorded_block_size_int = _integer(
recorded_block_size, f"block_size at {csv_path}:{line}"
)
recorded_total = container_size(payload, recorded_block_size_int)
recorded_stage = align_down(STAGE_CEILING - recorded_total)
recorded_margin = recorded_stage - (APP_BASE + memory)
if (total, stage_start, margin) != (
expected_total, expected_stage, expected_margin
recorded_total, recorded_stage, recorded_margin
):
raise RouteSearchError(
f"inconsistent cached geometry at {csv_path}:{line}"
@@ -694,16 +803,30 @@ def migrate_csv(
raise RouteSearchError(
f"cached feasibility is inconsistent at {csv_path}:{line}"
)
block_size = expected_block_size
total = container_size(payload, block_size)
stage_start = align_down(STAGE_CEILING - total)
margin = stage_start - (APP_BASE + memory)
feasible = payload >= 0 and margin >= 0
key = cache_key(source_sha, target_sha, memory)
normalized = {field: row.get(field, "") for field in FIELDS}
normalized.update(source=source_node, target=target_node,
source_sha256=source_sha, target_sha256=target_sha,
memory=memory)
normalized.update(normalize_transport_stats(
row, payload, f"{csv_path}:{line}"
))
memory=memory, payload=payload,
block_size=block_size, container=total,
stage_start=stage_start, margin=margin,
feasible=feasible, error="")
if recorded_block_size_int == block_size:
normalized.update(normalize_transport_stats(
row, payload, f"{csv_path}:{line}", block_size
))
else:
normalized.update({field: "" for field in TRANSPORT_FIELDS})
previous = migrated.get(key)
comparable = ("payload", "container", "stage_start", "margin", "feasible", "error")
comparable = (
"payload", "block_size", "container", "stage_start", "margin",
"feasible", "error",
)
if previous is not None and any(
str(previous.get(item, "")) != str(normalized.get(item, ""))
for item in comparable
@@ -723,7 +846,7 @@ def migrate_csv(
return migrated
Job = tuple[int, int, int, str, str, str, str]
Job = tuple[int, int, int, str, str, str, str, int]
def is_forward_progress(
@@ -745,7 +868,8 @@ def all_jobs(images: list[dict[str, object]]) -> list[Job]:
continue
jobs.append((1, target, FIXED_MEMORY, str(images[1]["path"]),
str(images[target]["path"]), str(images[1]["sha256"]),
str(images[target]["sha256"])))
str(images[target]["sha256"]),
source_block_size(images, 1)))
for source in range(2, endpoint):
source_size = int(images[source]["size"])
for target in range(source + 1, endpoint + 1):
@@ -756,7 +880,8 @@ def all_jobs(images: list[dict[str, object]]) -> list[Job]:
for memory_page in range(first_page, AVAILABLE_PAGES):
jobs.append((source, target, memory_page * PAGE,
str(images[source]["path"]), str(images[target]["path"]),
str(images[source]["sha256"]), str(images[target]["sha256"])))
str(images[source]["sha256"]), str(images[target]["sha256"]),
source_block_size(images, source)))
return jobs
@@ -826,7 +951,7 @@ def job_needs_transport_stats(
) -> bool:
return (
objective == "transport"
and source_uses_v2(images, job[0])
and source_uses_deflate(images, job[0])
and (relevant_pairs is None or (job[0], job[1]) in relevant_pairs)
)
@@ -835,8 +960,12 @@ def cache_satisfies_job(
row: dict[str, object] | None,
needs_transport: bool,
expected_encoder_sha256: str | None = None,
expected_block_size: int | None = None,
) -> bool:
return row is not None and (
expected_block_size is None
or int(row.get("block_size") or LEGACY_BLOCK_SIZE) == expected_block_size
) and (
not needs_transport
or not truth(row.get("feasible", False))
or (
@@ -857,13 +986,17 @@ def geometry_job(args: tuple[object, ...]) -> dict[str, object]:
"detools is required to generate geometry; run this tool in the "
"detools pipx environment"
) from exc
if len(args) not in (8, 9):
if len(args) not in (9, 10):
raise RouteSearchError("invalid geometry job")
source, target, memory, from_raw, to_raw, source_sha, target_sha, patches_raw = args[:8]
transport_tool = str(args[8]) if len(args) == 9 and args[8] else ""
(
source, target, memory, from_raw, to_raw, source_sha, target_sha,
block_size, patches_raw,
) = args[:9]
transport_tool = str(args[9]) if len(args) == 10 and args[9] else ""
source = int(source)
target = int(target)
memory = int(memory)
block_size = _validate_block_size(int(block_size))
from_raw = str(from_raw)
to_raw = str(to_raw)
source_sha = str(source_sha)
@@ -877,12 +1010,12 @@ def geometry_job(args: tuple[object, ...]) -> dict[str, object]:
memory_size=memory, segment_size=SEGMENT, use_mmap=True,
)
payload = patch.stat().st_size
total = container_size(payload)
total = container_size(payload, block_size)
stage_start = align_down(STAGE_CEILING - total)
margin = stage_start - (APP_BASE + memory)
error = ""
transport = (
measure_transport_size(Path(transport_tool), patch)
measure_transport_size(Path(transport_tool), patch, block_size)
if transport_tool and margin >= 0
else {field: "" for field in TRANSPORT_FIELDS}
)
@@ -895,7 +1028,8 @@ def geometry_job(args: tuple[object, ...]) -> dict[str, object]:
patch.unlink(missing_ok=True)
return {"source": source, "target": target, "source_sha256": source_sha,
"target_sha256": target_sha, "memory": memory, "payload": payload,
"container": total, "stage_start": stage_start, "margin": margin,
"block_size": block_size, "container": total,
"stage_start": stage_start, "margin": margin,
"feasible": payload >= 0 and margin >= 0, "error": error,
**transport}
@@ -915,6 +1049,12 @@ def project_cache(cache: dict[tuple[str, str, int], dict[str, object]],
)
row.update(source=index[key[0]], target=index[key[1]],
source_sha256=key[0], target_sha256=key[1], memory=key[2])
expected_block_size = source_block_size(images, int(row["source"]))
if int(row.get("block_size") or LEGACY_BLOCK_SIZE) != expected_block_size:
raise RouteSearchError(
f"required cache geometry has the wrong block size: {key}"
)
row["block_size"] = expected_block_size
rows.append(row)
return rows
@@ -980,11 +1120,50 @@ def write_csv(path: Path, rows: list[dict[str, object]]) -> None:
raise
def source_uses_v2(images: list[dict[str, object]], source: int) -> bool:
"""Capability is the currently running/source image; bootstrap is raw."""
def source_block_size(images: list[dict[str, object]], source: int) -> int:
"""Logical package/Merkle block size accepted by the running receiver."""
if source == 0:
return LEGACY_BLOCK_SIZE
block_size = int(images[source].get(TRANSPORT_MAX_BLOCK_FIELD, 0))
if block_size not in (LEGACY_BLOCK_SIZE, DEFLATE_BLOCK_SIZE):
raise RouteSearchError(
f"source image {source} has no explicit supported application "
"block size"
)
return block_size
def source_uses_deflate(images: list[dict[str, object]], source: int) -> bool:
"""Whether the running receiver permits compressed v2 DATA blocks."""
return source != 0 and bool(images[source][TRANSPORT_CAPABILITY])
def source_uses_v2(images: list[dict[str, object]], source: int) -> bool:
"""Whether either independent capability requires the negotiated v2 wire."""
return source != 0 and (
source_uses_deflate(images, source)
or source_block_size(images, source) > LEGACY_BLOCK_SIZE
)
def source_transport_profile(
images: list[dict[str, object]], source: int
) -> str:
"""Select the profile from the independent compression/block-size axes."""
if source == 0:
return "legacy-160-raw"
key = (
f"{source_block_size(images, source)},deflate="
f"{str(source_uses_deflate(images, source)).lower()}"
)
try:
return TRANSPORT_PROFILE_MATRIX[key]
except KeyError as exc:
raise RouteSearchError(
f"source image {source} has an unsupported transport capability matrix"
) from exc
def source_transport_pipeline(images: list[dict[str, object]], source: int) -> int:
"""Return the request-flight capacity compiled into the source image."""
pipeline = images[source][TRANSPORT_PIPELINE_FIELD]
@@ -1002,7 +1181,7 @@ def transport_encoder_for_rows(
str(row["transport_encoder_sha256"])
for row in rows
if truth(row.get("feasible", False))
and source_uses_v2(images, int(row["source"]))
and source_uses_deflate(images, int(row["source"]))
and (
relevant_pairs is None
or (int(row["source"]), int(row["target"])) in relevant_pairs
@@ -1020,13 +1199,26 @@ def edge_transport_cost(
row: dict[str, object], images: list[dict[str, object]], relay_hops: int
) -> dict[str, int | str]:
source = int(row["source"])
block_size = source_block_size(images, source)
recorded_block_size = int(row.get("block_size") or block_size)
if recorded_block_size != block_size:
raise RouteSearchError(
f"geometry edge {source}->{int(row['target'])} uses "
f"{recorded_block_size}-byte blocks, expected {block_size}"
)
payload = int(row.get("payload", 0))
if payload <= 0:
payload = payload_size_from_container(int(row["container"]))
if source_uses_v2(images, source):
payload = payload_size_from_container(int(row["container"]), block_size)
elif int(row["container"]) != container_size(payload, block_size):
raise RouteSearchError(
f"geometry edge {source}->{int(row['target'])} container does not "
"match its logical block size"
)
profile = source_transport_profile(images, source)
if profile == "v2-171-deflate":
if not transport_stats_complete(row):
raise RouteSearchError(
f"missing transport statistics for capable source edge "
f"missing transport statistics for compressed source edge "
f"{source}->{int(row['target'])} memory=0x{int(row['memory']):X}"
)
cost = v2_transport_cost(
@@ -1036,6 +1228,13 @@ def edge_transport_cost(
int(row["v2_deflate_blocks"]),
int(row["v2_data_packets"]),
source_transport_pipeline(images, source),
block_size,
)
elif profile == "v2-171-raw":
cost = v2_raw_transport_cost(
payload,
source_transport_pipeline(images, source),
block_size,
)
else:
cost = legacy_transport_cost(payload)
@@ -1064,6 +1263,7 @@ def select_route(rows: list[dict[str, object]], images: list[dict[str, object]],
raise RouteSearchError("pinned first package does not fit")
baseline_row: dict[str, object] = {
"source": 0, "target": 1, "memory": FIXED_MEMORY,
"block_size": LEGACY_BLOCK_SIZE,
"container": baseline_size, "stage_start": baseline_stage,
"margin": baseline_margin, "feasible": True,
}
@@ -1160,8 +1360,13 @@ def select_route(rows: list[dict[str, object]], images: list[dict[str, object]],
)
common["transport_accounting"] = {
"relay_hops": relay_hops,
"legacy_block_size": LEGACY_BLOCK_SIZE,
"supported_block_sizes": [LEGACY_BLOCK_SIZE, DEFLATE_BLOCK_SIZE],
"comparison_baseline_block_size": LEGACY_BLOCK_SIZE,
"source_capability_field": TRANSPORT_CAPABILITY,
"source_pipeline_field": TRANSPORT_PIPELINE_FIELD,
"source_max_block_field": TRANSPORT_MAX_BLOCK_FIELD,
"source_profile_matrix": dict(TRANSPORT_PROFILE_MATRIX),
"first_bootstrap_profile": "legacy-160-raw",
"included": (
"OTA messages, repeated 4-byte v2 stream IDs, 171-byte DATA slicing, "
@@ -1186,6 +1391,7 @@ def select_route(rows: list[dict[str, object]], images: list[dict[str, object]],
for source, target in zip(nodes, nodes[1:]):
row = edges[source, target]
step = {"source_node": source, "target_node": target,
"block_size": source_block_size(images, source),
"inplace_memory": f"0x{int(row['memory']):X}",
"reuse_baseline_package": source == 0 and target == 1,
"expected_container_size": int(row["container"]),
@@ -1197,14 +1403,14 @@ def select_route(rows: list[dict[str, object]], images: list[dict[str, object]],
step["transport"] = {
key: cost[key] for key in (
"profile", "payload_bytes", "wire_bytes", "deflate_bytes",
"deflate_blocks", "data_packets", "request_packets",
"deflate_blocks", "block_size", "data_packets", "request_packets",
"manifest_packets", "request_pipeline", "proof_request_packets",
"proof_packets", "packets", "manifest_bytes", "data_bytes",
"block_request_bytes", "proof_request_bytes",
"proof_response_bytes", "origin_mesh_bytes", "linear_path_bytes",
)
}
if source_uses_v2(images, source):
if source_uses_deflate(images, source):
step["transport"]["payload_sha256"] = row["payload_sha256"]
step["transport"]["encoder_sha256"] = row[
"transport_encoder_sha256"
@@ -1284,7 +1490,8 @@ def main(argv: list[str] | None = None) -> int:
)
cache: dict[tuple[str, str, int], dict[str, object]] = {}
comparable = (
"payload", "container", "stage_start", "margin", "feasible", "error"
"payload", "block_size", "container", "stage_start", "margin",
"feasible", "error",
)
for reuse_number, value in enumerate(args.reuse, 1):
left, separator, right = value.partition("=")
@@ -1324,7 +1531,11 @@ def main(argv: list[str] | None = None) -> int:
required = {cache_key(job[5], job[6], job[2]) for job in jobs}
geometry_remaining = [
job for job in jobs
if cache_key(job[5], job[6], job[2]) not in cache
if not cache_satisfies_job(
cache.get(cache_key(job[5], job[6], job[2])),
False,
expected_block_size=job[7],
)
]
print(
f"nodes={len(images)} geometries={len(jobs)} "
@@ -1359,7 +1570,15 @@ def main(argv: list[str] | None = None) -> int:
f"geometry_measured={done}/{len(geometry_remaining)}",
flush=True,
)
geometry_completed = required & cache.keys()
geometry_completed = {
cache_key(job[5], job[6], job[2])
for job in jobs
if cache_satisfies_job(
cache.get(cache_key(job[5], job[6], job[2])),
False,
expected_block_size=job[7],
)
}
rows = project_cache(cache, images, geometry_completed)
write_csv(args.work_dir / "geometry.csv", rows)
geometry_complete = geometry_completed == required
@@ -1380,6 +1599,7 @@ def main(argv: list[str] | None = None) -> int:
job, images, args.objective, relevant_pairs
),
frozen_transport_tool_sha256,
job[7],
)
]
print(
@@ -1437,6 +1657,7 @@ def main(argv: list[str] | None = None) -> int:
job, images, args.objective, relevant_pairs
),
frozen_transport_tool_sha256,
job[7],
)
for job in jobs
)
@@ -98,6 +98,109 @@ class CompactRouteInputTests(unittest.TestCase):
],
}
def transport_schema2_route(self) -> dict[str, object]:
specifications = [
("legacy-160-raw", 1024, None),
("v2-171-deflate", 1024, 1),
("v2-171-raw", 2048, 2),
("v2-171-deflate", 2048, 4),
]
steps = []
for number, (profile, block_size, pipeline) in enumerate(
specifications, 1
):
payload = 3000
if profile == "legacy-160-raw":
cost = compact.route_search.legacy_transport_cost(payload)
elif profile == "v2-171-raw":
cost = compact.route_search.v2_raw_transport_cost(
payload, int(pipeline), block_size
)
else:
cost = compact.route_search.v2_transport_cost(
payload,
1500,
1500,
(payload + block_size - 1) // block_size,
9,
int(pipeline),
block_size,
)
cost["linear_path_bytes"] = compact.route_search.linear_path_bytes(
int(cost["origin_mesh_bytes"]), int(cost["packets"]), 0
)
transport = {
field: cost[field] for field in compact.TRANSPORT_STEP_FIELDS
}
if profile == "v2-171-deflate":
transport.update({
"payload_sha256": f"{number:064x}",
"encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256,
})
steps.append({
"source_node": number - 1,
"target_node": number,
"block_size": block_size,
"inplace_memory": "0x98000" if number <= 2 else "0x80000",
"reuse_baseline_package": number == 1,
"expected_container_size": compact.route_search.container_size(
payload, block_size
),
"expected_staging_margin": 1,
"expected_target_sha256": f"{number:064x}",
"expected_target_version": f"1.0.0.{number}",
"transport": transport,
})
return {
"schema": 2,
"status": "reachable",
"search_complete": True,
"app_base": "0x26000",
"stage_ceiling": "0xD4000",
"node_count": 5,
"objective": compact.TRANSPORT_ROUTE_OBJECTIVE,
"candidate_geometries": 4,
"feasible_edges": 4,
"shortest_package_count": 4,
"shortest_route_count": 1,
"selected_total_bytes": sum(
int(step["expected_container_size"]) for step in steps
),
"selected_total_transport_bytes": sum(
int(step["transport"]["linear_path_bytes"]) for step in steps
),
"transport_accounting": {
"relay_hops": 0,
"legacy_block_size": 1024,
"supported_block_sizes": [1024, 2048],
"comparison_baseline_block_size": 1024,
"source_capability_field": (
compact.route_search.TRANSPORT_CAPABILITY
),
"source_pipeline_field": (
compact.route_search.TRANSPORT_PIPELINE_FIELD
),
"source_max_block_field": (
compact.route_search.TRANSPORT_MAX_BLOCK_FIELD
),
"source_profile_matrix": (
compact.route_search.TRANSPORT_PROFILE_MATRIX
),
"first_bootstrap_profile": "legacy-160-raw",
"included": (
"OTA messages, repeated 4-byte v2 stream IDs, 171-byte DATA "
"slicing, adaptive requests, manifest, exact per-block proofs, "
"and MeshCore framing"
),
"excluded": (
"discovery, retries, flood fan-out, and radio-dependent LoRa "
"PHY coding/preamble"
),
"encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256,
},
"steps": steps,
}
def test_existing_schema1_route_remains_accepted(self) -> None:
route = Path(__file__).with_name("rak3401_compact_route.json")
steps, document = compact.read_route(route)
@@ -151,6 +254,91 @@ class CompactRouteInputTests(unittest.TestCase):
with self.assertRaisesRegex(compact.CompactBuildError, message):
compact.read_route(path)
def test_transport_route_accepts_full_capability_matrix_and_fails_closed(
self,
) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
route = self.transport_schema2_route()
path = root / "route.json"
path.write_text(json.dumps(route), encoding="ascii")
steps, _document = compact.read_route(path)
self.assertEqual(
[step["transport"]["profile"] for step in steps],
[
"legacy-160-raw",
"v2-171-deflate",
"v2-171-raw",
"v2-171-deflate",
],
)
raw_only = self.transport_schema2_route()
for index in (1, 3):
block_size = int(raw_only["steps"][index]["block_size"])
if block_size == 1024:
cost = compact.route_search.legacy_transport_cost(3000)
else:
cost = compact.route_search.v2_raw_transport_cost(
3000,
int(raw_only["steps"][index]["transport"][
"request_pipeline"
]),
block_size,
)
cost["linear_path_bytes"] = compact.route_search.linear_path_bytes(
int(cost["origin_mesh_bytes"]), int(cost["packets"]), 0
)
raw_only["steps"][index]["transport"] = {
field: cost[field]
for field in compact.TRANSPORT_STEP_FIELDS
}
del raw_only["transport_accounting"]["encoder_sha256"]
raw_only["selected_total_transport_bytes"] = sum(
int(step["transport"]["linear_path_bytes"])
for step in raw_only["steps"]
)
raw_path = root / "raw-only.json"
raw_path.write_text(json.dumps(raw_only), encoding="ascii")
raw_steps, raw_document = compact.read_route(raw_path)
self.assertNotIn(
"encoder_sha256", raw_document["transport_accounting"]
)
self.assertEqual(
[step["transport"]["profile"] for step in raw_steps],
[
"legacy-160-raw",
"legacy-160-raw",
"v2-171-raw",
"v2-171-raw",
],
)
mutations = []
bad = self.transport_schema2_route()
del bad["transport_accounting"]["source_profile_matrix"]
mutations.append((bad, "capability contract"))
bad = self.transport_schema2_route()
bad["steps"][2]["transport"].update({
"payload_sha256": "a" * 64,
"encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256,
})
mutations.append((bad, "hashes do not match"))
bad = self.transport_schema2_route()
bad["steps"][2]["block_size"] = 1024
mutations.append((bad, "block_size disagrees"))
bad = self.transport_schema2_route()
del bad["transport_accounting"]["encoder_sha256"]
mutations.append((bad, "encoder presence"))
for number, (document, message) in enumerate(mutations):
candidate = root / f"bad-{number}.json"
candidate.write_text(json.dumps(document), encoding="ascii")
with self.subTest(message=message), self.assertRaisesRegex(
compact.CompactBuildError, message
):
compact.read_route(candidate)
class CompactImageInventoryTests(unittest.TestCase):
def test_schema2_snapshot_freezes_manifest_and_referenced_images(self) -> None:
@@ -322,25 +510,20 @@ class CompactImageInventoryTests(unittest.TestCase):
class CompactSimulatorTests(unittest.TestCase):
def test_exact_required_simulators_are_accepted(self) -> None:
with tempfile.TemporaryDirectory() as directory:
preview5 = Path(directory) / "preview5"
current = Path(directory) / "current"
preview5.write_bytes(b"p5")
current.write_bytes(b"current")
simulator = Path(directory) / "otafix2.4"
simulator.write_bytes(b"exact deployed bootloader")
hashes = {
preview5: compact.REQUIRED_SIMULATORS["preview5"]["sha256"],
current: compact.REQUIRED_SIMULATORS["current"]["sha256"],
simulator: compact.REQUIRED_SIMULATORS["otafix2.4"]["sha256"],
}
with mock.patch.object(compact.common, "sha256_file", side_effect=hashes.get):
result = compact.parse_simulators([
f"preview5={preview5}", f"current={current}"
])
self.assertEqual([label for label, _path in result], ["preview5", "current"])
result = compact.parse_simulators([f"otafix2.4={simulator}"])
self.assertEqual([label for label, _path in result], ["otafix2.4"])
def test_missing_or_wrong_simulator_is_rejected(self) -> None:
with tempfile.TemporaryDirectory() as directory:
simulator = Path(directory) / "sim"
simulator.write_bytes(b"wrong")
for values, digest in (([], None), ([f"preview5={simulator}"], "0" * 64)):
for values, digest in (([], None), ([f"otafix2.4={simulator}"], "0" * 64)):
with self.subTest(values=values), mock.patch.object(
compact.common, "sha256_file", return_value=digest
):
@@ -446,14 +629,17 @@ class CompactGeometryEvidenceTests(unittest.TestCase):
inventory = compact.route_search.load_inventory(inventory_path)
rows = []
for job in compact.route_search.all_jobs(inventory):
source, target, memory, _from, _to, source_sha, target_sha = job
(
source, target, memory, _from, _to, source_sha, target_sha,
block_size,
) = job
if source == 1 and target == 3:
payload = 10
else:
# A successfully generated patch may be too large for staging;
# that is valid infeasible evidence. A tool exception is not.
payload = 700_000
container = compact.route_search.container_size(payload)
container = compact.route_search.container_size(payload, block_size)
stage_start = compact.route_search.align_down(
compact.STAGE_CEILING - container
)
@@ -462,6 +648,7 @@ class CompactGeometryEvidenceTests(unittest.TestCase):
rows.append({
"source": source, "target": target, "source_sha256": source_sha,
"target_sha256": target_sha, "memory": memory, "payload": payload,
"block_size": block_size,
"container": container, "stage_start": stage_start, "margin": margin,
"feasible": feasible, "error": error,
})
@@ -496,6 +683,7 @@ class CompactGeometryEvidenceTests(unittest.TestCase):
"target_sha256": inventory[2]["sha256"],
"memory": compact.FIXED_WORKSPACE,
"payload": payload,
"block_size": compact.LEGACY_BLOCK_SIZE,
"container": container,
"stage_start": stage_start,
"margin": margin,
@@ -523,6 +711,228 @@ class CompactGeometryEvidenceTests(unittest.TestCase):
with self.assertRaisesRegex(compact.CompactBuildError, "inconsistent"):
compact.validate_geometry_results(root / "geometry.csv", inventory, route)
def test_selected_deflate_step_uses_2k_and_reports_1k_raw_baseline(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
payload_path = root / "payload.patch"
payload_path.write_bytes(b"A" * 3000)
payload_sha = hashlib.sha256(payload_path.read_bytes()).hexdigest()
inventory = [
{
compact.route_search.TRANSPORT_CAPABILITY: False,
compact.route_search.TRANSPORT_PIPELINE_FIELD: None,
compact.route_search.TRANSPORT_MAX_BLOCK_FIELD:
compact.LEGACY_BLOCK_SIZE,
},
{
compact.route_search.TRANSPORT_CAPABILITY: True,
compact.route_search.TRANSPORT_PIPELINE_FIELD: 1,
compact.route_search.TRANSPORT_MAX_BLOCK_FIELD:
compact.DEFLATE_BLOCK_SIZE,
},
]
selected = compact.route_search.v2_transport_cost(
3000, 1500, 1500, 2, 10, 1,
compact.DEFLATE_BLOCK_SIZE,
)
selected["linear_path_bytes"] = compact.route_search.linear_path_bytes(
int(selected["origin_mesh_bytes"]), int(selected["packets"]), 0
)
plan_step = {
"source_node": 1,
"block_size": compact.DEFLATE_BLOCK_SIZE,
"transport": {
**{
field: selected[field]
for field in compact.TRANSPORT_STEP_FIELDS
},
"payload_sha256": payload_sha,
"encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256,
},
}
measured = {
"payload_sha256": payload_sha,
"transport_encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256,
"v2_wire_bytes": 1500,
"v2_deflate_bytes": 1500,
"v2_deflate_blocks": 2,
"v2_data_packets": 10,
}
with mock.patch.object(
compact.route_search,
"measure_transport_size",
return_value=measured,
) as measure:
result = compact.validate_selected_transport(
Path("motatool"), payload_path, plan_step, inventory, 0, 3
)
with mock.patch.object(
compact.route_search,
"measure_transport_size",
return_value={
**measured,
"transport_encoder_sha256": "0" * 64,
},
), self.assertRaisesRegex(
compact.CompactBuildError, "different encoder binary"
):
compact.validate_selected_transport(
Path("motatool"), payload_path, plan_step, inventory, 0, 3
)
measure.assert_called_once_with(
Path("motatool"), payload_path, compact.DEFLATE_BLOCK_SIZE
)
baseline = result["one_kib_no_compression"]
self.assertEqual(result["block_size"], compact.DEFLATE_BLOCK_SIZE)
self.assertEqual(baseline["profile"], "v2-171-raw")
self.assertEqual(baseline["block_size"], compact.LEGACY_BLOCK_SIZE)
self.assertEqual(baseline["proof_packets"], 3)
self.assertGreater(baseline["packets"], result["packets"])
def test_selected_transport_validates_independent_capability_matrix(
self,
) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
payload_path = root / "payload.patch"
payload_path.write_bytes(b"A" * 3000)
payload_sha = hashlib.sha256(payload_path.read_bytes()).hexdigest()
inventory = [
{
compact.route_search.TRANSPORT_CAPABILITY: False,
compact.route_search.TRANSPORT_MAX_BLOCK_FIELD: 1024,
compact.route_search.TRANSPORT_PIPELINE_FIELD: None,
},
{
compact.route_search.TRANSPORT_CAPABILITY: False,
compact.route_search.TRANSPORT_MAX_BLOCK_FIELD: 1024,
compact.route_search.TRANSPORT_PIPELINE_FIELD: None,
},
{
compact.route_search.TRANSPORT_CAPABILITY: True,
compact.route_search.TRANSPORT_MAX_BLOCK_FIELD: 1024,
compact.route_search.TRANSPORT_PIPELINE_FIELD: 1,
},
{
compact.route_search.TRANSPORT_CAPABILITY: False,
compact.route_search.TRANSPORT_MAX_BLOCK_FIELD: 2048,
compact.route_search.TRANSPORT_PIPELINE_FIELD: 2,
},
{
compact.route_search.TRANSPORT_CAPABILITY: True,
compact.route_search.TRANSPORT_MAX_BLOCK_FIELD: 2048,
compact.route_search.TRANSPORT_PIPELINE_FIELD: 4,
},
]
def plan(source: int) -> dict[str, object]:
block_size = compact.route_search.source_block_size(
inventory, source
)
profile = compact.route_search.source_transport_profile(
inventory, source
)
if profile == "v2-171-deflate":
cost = compact.route_search.v2_transport_cost(
3000,
1500,
1500,
(3000 + block_size - 1) // block_size,
9,
compact.route_search.source_transport_pipeline(
inventory, source
),
block_size,
)
elif profile == "v2-171-raw":
cost = compact.route_search.v2_raw_transport_cost(
3000,
compact.route_search.source_transport_pipeline(
inventory, source
),
block_size,
)
else:
cost = compact.route_search.legacy_transport_cost(3000)
cost["linear_path_bytes"] = compact.route_search.linear_path_bytes(
int(cost["origin_mesh_bytes"]), int(cost["packets"]), 0
)
transport = {
field: cost[field] for field in compact.TRANSPORT_STEP_FIELDS
}
if profile == "v2-171-deflate":
transport.update({
"payload_sha256": payload_sha,
"encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256,
})
return {
"source_node": source,
"block_size": block_size,
"transport": transport,
}
def measured(
_tool: Path, _payload: Path, block_size: int
) -> dict[str, object]:
return {
"payload_sha256": payload_sha,
"transport_encoder_sha256": (
compact.TRANSPORT_ROUTE_ENCODER_SHA256
),
"v2_wire_bytes": 1500,
"v2_deflate_bytes": 1500,
"v2_deflate_blocks": (3000 + block_size - 1) // block_size,
"v2_data_packets": 9,
}
with mock.patch.object(
compact.route_search,
"measure_transport_size",
side_effect=measured,
) as measure:
results = [
compact.validate_selected_transport(
Path("motatool"), payload_path, plan(source), inventory, 0,
source,
)
for source in range(1, 5)
]
self.assertEqual(
[result["profile"] for result in results],
[
"legacy-160-raw",
"v2-171-deflate",
"v2-171-raw",
"v2-171-deflate",
],
)
self.assertEqual(
[call.args[2] for call in measure.call_args_list], [1024, 2048]
)
raw_with_hashes = plan(3)
raw_with_hashes["transport"].update({
"payload_sha256": payload_sha,
"encoder_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256,
})
with self.assertRaisesRegex(
compact.CompactBuildError, "raw transport unexpectedly has encoder pins"
):
compact.validate_selected_transport(
Path("motatool"), payload_path, raw_with_hashes, inventory, 0, 3
)
wrong_profile = plan(3)
wrong_profile["transport"]["profile"] = "legacy-160-raw"
with self.assertRaisesRegex(
compact.CompactBuildError, "profile disagrees"
):
compact.validate_selected_transport(
Path("motatool"), payload_path, wrong_profile, inventory, 0, 3
)
def test_missing_and_extra_geometry_are_rejected(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
@@ -574,6 +984,132 @@ class CompactGeometryEvidenceTests(unittest.TestCase):
compact.validate_geometry_results(root / "geometry.csv", inventory, route)
class CompactGeneratedReportTests(unittest.TestCase):
def test_chain_csv_pins_block_and_transport_component_details(self) -> None:
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "CHAIN.csv"
compact.write_chain(path, [{
"step": 1,
"mota_block_size": compact.DEFLATE_BLOCK_SIZE,
"transport_relay_hops": 0,
"transport_request_pipeline": 1,
"transport_payload_sha256": "a" * 64,
"transport_encoder_sha256": "b" * 64,
"transport_request_packets": 7,
"transport_proof_packets": 9,
"transport_data_bytes": 1234,
"baseline_1k_raw_packets": 99,
}])
lines = path.read_text(encoding="ascii").splitlines()
header = lines[0].split(",")
values = dict(zip(header, lines[1].split(",")))
self.assertEqual(values["mota_block_size"], "2048")
self.assertEqual(values["transport_request_pipeline"], "1")
self.assertEqual(values["transport_payload_sha256"], "a" * 64)
self.assertEqual(values["transport_encoder_sha256"], "b" * 64)
self.assertEqual(values["transport_proof_packets"], "9")
self.assertEqual(values["transport_data_bytes"], "1234")
self.assertEqual(values["baseline_1k_raw_packets"], "99")
def test_report_contains_derived_2k_vs_1k_raw_comparison(self) -> None:
legacy_payloads = [89_282, 64_756]
v2_measurements = [
(104_543, 60_379, 60_284, 51, 381),
(148_997, 92_549, 92_549, 73, 577),
(112_123, 59_704, 59_704, 55, 375),
(121_934, 63_977, 63_977, 60, 405),
(135_068, 87_425, 87_425, 66, 547),
(176_140, 130_264, 130_252, 86, 807),
(271_013, 210_192, 210_192, 133, 1297),
]
selected = [
compact.route_search.legacy_transport_cost(payload)
for payload in legacy_payloads
] + [
compact.route_search.v2_transport_cost(
*measurement, 1, compact.DEFLATE_BLOCK_SIZE
)
for measurement in v2_measurements
] + [compact.route_search.legacy_transport_cost(196_843)]
baseline = [
compact.route_search.legacy_transport_cost(payload)
for payload in legacy_payloads
] + [
compact.route_search.v2_raw_transport_cost(
measurement[0], 1, compact.LEGACY_BLOCK_SIZE
)
for measurement in v2_measurements
] + [compact.route_search.legacy_transport_cost(196_843)]
rows = []
for number, (actual, raw) in enumerate(zip(selected, baseline), 1):
rows.append({
"step": number,
"mota_size": compact.route_search.container_size(
int(actual["payload_bytes"]), int(actual["block_size"])
),
"staging_margin": 4096,
"transport_profile": actual["profile"],
"transport_relay_hops": 0,
"transport_payload_bytes": actual["payload_bytes"],
"transport_wire_bytes": actual["wire_bytes"],
"transport_deflate_bytes": actual["deflate_bytes"],
"transport_deflate_blocks": actual["deflate_blocks"],
"transport_data_packets": actual["data_packets"],
"transport_packets": actual["packets"],
"transport_linear_path_bytes": actual["origin_mesh_bytes"],
"baseline_1k_raw_packets": raw["packets"],
"baseline_1k_raw_linear_path_bytes": raw["origin_mesh_bytes"],
})
tools = {
"motatool": {
"version": compact.MOTATOOL_VERSION,
"asserted_source_commit": compact.MOTATOOL_COMMIT,
"executable_sha256": compact.TRANSPORT_ROUTE_ENCODER_SHA256,
},
"detools": {
"version": compact.DETOOLS_VERSION,
"launcher_sha256": "d" * 64,
},
}
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
compact.write_docs(
root,
"a" * 40,
"1.17.1.6",
firmware(b"endpoint", "1.17.1.6"),
"1" * 64,
"2" * 64,
"3" * 64,
"4" * 64,
"5" * 64,
"6" * 64,
tools,
rows,
False,
)
readme = (root / "README.md").read_text(encoding="ascii")
provenance = (root / "PROVENANCE.md").read_text(encoding="ascii")
self.assertIn(
"| No DEFLATE baseline; packet profiles preserved | 1 KiB all "
"steps | 11,835 | 1,618,318 |",
readme,
)
self.assertIn(
"| Selected capability-aware route | Per-source 1/2 KiB signed "
"geometry | 8,906 | 1,195,442 |",
readme,
)
self.assertIn("2,929 packets\n (24.75%)", readme)
self.assertIn("422,876 bytes\n (26.13%)", readme)
self.assertIn("preserves each step's packet profile", readme)
self.assertIn("independently selected per running source", provenance)
self.assertIn("`transport_max_block_bytes` capability", provenance)
self.assertIn("DEFLATE permission is a separate capability", provenance)
class CompactPhysicalValidationTests(unittest.TestCase):
def evidence(self) -> tuple[dict[str, object], dict[str, object]]:
expected = {
@@ -4,6 +4,9 @@
from __future__ import annotations
from pathlib import Path
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
@@ -12,6 +15,277 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
import build_rak3401_inflate_bridges as inflate
REPO_ROOT = Path(__file__).resolve().parents[2]
MOCK_MANAGER_HEADER = r'''#pragma once
#include <stdint.h>
#include <string.h>
#include <vector>
#define OTA_MAX_BLOCK 2048
#define OTA_FRAG_DATA 160
#define OTA_FRAG_DATA_V2 171
#define OTA_FETCH_PIPELINE 1
#include "OtaProtocol.h"
namespace mesh {
namespace ota {
typedef void (*OtaSend)(void*, const uint8_t*, uint16_t, bool);
class OtaManager {
public:
void begin(uint32_t target_id, OtaSend send, void* send_ctx) {
(void)target_id;
_send = send;
_send_ctx = send_ctx;
}
void configure(const uint8_t mid[4], uint16_t block, uint16_t block_len,
uint16_t nominal_size) {
memcpy(_mid, mid, sizeof(_mid));
_block = block;
_block_len = block_len;
_nominal_size = nominal_size;
logical.assign(block_len, 0);
offsets.clear();
invalid = false;
}
bool request(uint16_t want_mask) {
if (!_send) return false;
ReqMsg request;
memcpy(request.manifest_id, _mid, sizeof(request.manifest_id));
request.block_idx = _block;
request.want_mask = want_mask;
uint8_t wire[9];
const uint16_t wire_len = encode_req(wire, sizeof(wire), request);
if (wire_len == 0) return false;
_send(_send_ctx, wire, wire_len, false);
return true;
}
uint16_t requestedBlockLength(const uint8_t* manifest_id, uint16_t block,
uint16_t* nominal_size = nullptr) const {
if (nominal_size) *nominal_size = 0;
if (!manifest_id || block != _block ||
memcmp(manifest_id, _mid, sizeof(_mid)) != 0) return 0;
if (nominal_size) *nominal_size = _nominal_size;
return _block_len;
}
void on_message(const uint8_t* msg, uint16_t len) {
DataMsg data;
if (!decode_data(msg, len, data) ||
memcmp(data.manifest_id, _mid, sizeof(_mid)) != 0 ||
data.block_idx != _block || (data.frag_off & OTA_DATA_V2_MARK) != 0 ||
data.frag_off >= _block_len) {
invalid = true;
return;
}
uint16_t expected = (uint16_t)(_block_len - data.frag_off);
if (expected > OTA_FRAG_DATA) expected = OTA_FRAG_DATA;
if (data.frag_off % OTA_FRAG_DATA != 0 || data.data_len != expected) {
invalid = true;
return;
}
memcpy(logical.data() + data.frag_off, data.data, data.data_len);
offsets.push_back(data.frag_off);
}
std::vector<uint8_t> logical;
std::vector<uint16_t> offsets;
bool invalid = false;
private:
OtaSend _send = nullptr;
void* _send_ctx = nullptr;
uint8_t _mid[4] = {0};
uint16_t _block = 0;
uint16_t _block_len = 0;
uint16_t _nominal_size = 0;
};
} // namespace ota
} // namespace mesh
'''
MOCK_MULTIHASH_HEADER = r'''#pragma once
#include <stddef.h>
#include <stdint.h>
namespace mesh {
namespace ota {
inline void mh4(uint8_t out[4], const uint8_t* data, size_t len) {
uint32_t hash = 2166136261u;
for (size_t i = 0; i < len; ++i) {
hash ^= data[i];
hash *= 16777619u;
}
out[0] = (uint8_t)hash;
out[1] = (uint8_t)(hash >> 8);
out[2] = (uint8_t)(hash >> 16);
out[3] = (uint8_t)(hash >> 24);
}
} // namespace ota
} // namespace mesh
'''
BRIDGE_BEHAVIOR_HARNESS = r'''#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <vector>
#include "Multihash.h"
#include "OtaDeflate.h"
using namespace mesh::ota;
namespace {
struct Capture {
std::vector<std::vector<uint8_t> > packets;
};
void capture_send(void* context, const uint8_t* msg, uint16_t len, bool flood) {
(void)flood;
Capture* capture = static_cast<Capture*>(context);
capture->packets.push_back(std::vector<uint8_t>(msg, msg + len));
}
std::vector<uint8_t> make_payload(uint16_t total, uint16_t random_prefix) {
std::vector<uint8_t> payload(total, 'A');
uint32_t state = 0xc001d00du ^ total;
for (uint16_t i = 0; i < random_prefix; ++i) {
state ^= state << 13;
state ^= state >> 17;
state ^= state << 5;
payload[i] = (uint8_t)state;
}
return payload;
}
std::vector<uint8_t> make_deflate(const std::vector<uint8_t>& payload,
uint16_t random_prefix) {
// Non-final stored block for the pseudo-random prefix, followed by a tiny
// final fixed-Huffman block for the repeated 'A' suffix. This deliberately
// keeps the 2 KiB representation long enough to exercise fragment index 9.
static const uint8_t tail_304[] = {0x73, 0x74, 0x1c, 0x05, 0xa4, 0x00, 0x00};
static const uint8_t tail_512[] = {0x73, 0x74, 0x1c, 0x05, 0x23, 0x19, 0x00, 0x00};
const uint8_t* tail = payload.size() == 1024 ? tail_304 : tail_512;
const size_t tail_len = payload.size() == 1024 ? sizeof(tail_304) : sizeof(tail_512);
std::vector<uint8_t> encoded;
encoded.reserve((size_t)random_prefix + 5u + tail_len);
encoded.push_back(0x00); // BFINAL=0, stored block, then align to the next byte.
encoded.push_back((uint8_t)random_prefix);
encoded.push_back((uint8_t)(random_prefix >> 8));
const uint16_t complement = (uint16_t)~random_prefix;
encoded.push_back((uint8_t)complement);
encoded.push_back((uint8_t)(complement >> 8));
encoded.insert(encoded.end(), payload.begin(), payload.begin() + random_prefix);
encoded.insert(encoded.end(), tail, tail + tail_len);
return encoded;
}
int run_case(uint16_t block_len, uint16_t random_prefix, bool extended) {
const uint8_t mid[4] = {0x21, 0x43, 0x65, (uint8_t)(extended ? 0x87 : 0x76)};
const uint16_t block = extended ? 9 : 4;
const uint16_t canonical_fragments =
(uint16_t)((block_len + OTA_FRAG_DATA - 1u) / OTA_FRAG_DATA);
const uint16_t original_want =
(uint16_t)((1u << canonical_fragments) - 1u);
OtaManager manager;
manager.configure(mid, block, block_len, block_len);
Capture capture;
OtaTransportInflateReceiver receiver;
receiver.begin(manager, 0x2fa509c1u, capture_send, &capture);
if (!manager.request(original_want) || capture.packets.size() != 1) return 10;
ReqMsg request;
if (!decode_req(capture.packets[0].data(),
(uint16_t)capture.packets[0].size(), request)) return 11;
const uint16_t expected_want = ota_req_make_v2(original_want, true, extended);
if (request.want_mask != expected_want || !ota_req_is_v2(request.want_mask) ||
ota_req_v2_extended_length(request.want_mask) != extended) return 12;
if ((!extended && request.want_mask != 0xc07fu) ||
(extended && request.want_mask != 0xefffu)) return 13;
const std::vector<uint8_t> payload = make_payload(block_len, random_prefix);
const std::vector<uint8_t> encoded = make_deflate(payload, random_prefix);
if (encoded.size() >= payload.size()) return 14;
const uint16_t wire_fragments =
(uint16_t)((encoded.size() + OTA_FRAG_DATA_V2 - 1u) / OTA_FRAG_DATA_V2);
if ((!extended && wire_fragments != 5) || (extended && wire_fragments != 10)) return 15;
uint8_t stream_id[4];
mh4(stream_id, encoded.data(), encoded.size());
// Reverse order proves that completion is driven by the descriptor bitmap,
// rather than accidentally relying on sequential arrival.
for (int fragment = (int)wire_fragments - 1; fragment >= 0; --fragment) {
const uint16_t offset = (uint16_t)(fragment * OTA_FRAG_DATA_V2);
uint16_t slice = (uint16_t)(encoded.size() - offset);
if (slice > OTA_FRAG_DATA_V2) slice = OTA_FRAG_DATA_V2;
uint16_t descriptor = 0;
const bool packed = extended
? ota_data_v2_pack_extended((uint8_t)fragment,
(uint16_t)encoded.size(), descriptor)
: ota_data_v2_pack((uint8_t)fragment, (uint16_t)encoded.size(),
true, descriptor);
if (!packed) return 16;
std::vector<uint8_t> body(OTA_DATA_V2_STREAM_ID_BYTES + slice);
memcpy(body.data(), stream_id, sizeof(stream_id));
memcpy(body.data() + OTA_DATA_V2_STREAM_ID_BYTES,
encoded.data() + offset, slice);
DataMsg data;
memcpy(data.manifest_id, mid, sizeof(data.manifest_id));
data.block_idx = block;
data.frag_off = descriptor;
data.data = body.data();
data.data_len = (uint16_t)body.size();
std::vector<uint8_t> wire(9u + body.size());
const uint16_t wire_len = encode_data(wire.data(), (uint16_t)wire.size(), data);
if (wire_len != wire.size()) return 17;
receiver.on_message(wire.data(), wire_len);
}
if (manager.invalid || manager.logical != payload ||
manager.offsets.size() != canonical_fragments) return 18;
for (uint16_t fragment = 0; fragment < canonical_fragments; ++fragment) {
if (manager.offsets[fragment] != fragment * OTA_FRAG_DATA) return 19;
}
return 0;
}
} // namespace
int main() {
int result = run_case(1024, 720, false);
if (result != 0) {
fprintf(stderr, "1 KiB bridge case failed: %d\n", result);
return result;
}
result = run_case(2048, 1536, true);
if (result != 0) {
fprintf(stderr, "2 KiB bridge case failed: %d\n", result);
return 100 + result;
}
return 0;
}
'''
class InflateAssetTests(unittest.TestCase):
def test_vendored_snapshot_assets_match_their_pins(self) -> None:
metadata = inflate.inflate_asset_metadata()
@@ -34,6 +308,88 @@ class InflateAssetTests(unittest.TestCase):
for destination in inflate.INFLATE_ASSETS:
self.assertFalse((source / destination).exists())
def test_receiver_negotiates_both_v2_descriptor_layouts(self) -> None:
source = (inflate.INFLATE_ASSET_ROOT / "OtaDeflate.cpp").read_text(
encoding="utf-8"
)
self.assertIn("OTA_DATA_V2_LEGACY_MAX_ENCODED", source)
self.assertIn("ota_data_v2_unpack_extended", source)
self.assertIn("&nominal_block_size", source)
self.assertIn(
"nominal_block_size > OTA_DATA_V2_LEGACY_MAX_ENCODED", source
)
def test_installed_receiver_reassembles_deflated_1k_and_2k_blocks(self) -> None:
cxx = shutil.which("c++") or shutil.which("g++")
cc = shutil.which("cc") or shutil.which("gcc")
if not cxx or not cc:
self.skipTest("a host C and C++ compiler are required")
with tempfile.TemporaryDirectory() as directory:
source = Path(directory)
inflate.install_inflate_assets(source)
ota_dir = source / "src" / "helpers" / "ota"
for filename in ("OtaFormat.h", "OtaProtocol.h", "OtaProtocol.cpp"):
shutil.copy2(REPO_ROOT / "src" / "helpers" / "ota" / filename,
ota_dir / filename)
(ota_dir / "OtaManager.h").write_text(
MOCK_MANAGER_HEADER, encoding="ascii"
)
(ota_dir / "Multihash.h").write_text(
MOCK_MULTIHASH_HEADER, encoding="ascii"
)
harness = source / "bridge_behavior.cpp"
harness.write_text(BRIDGE_BEHAVIOR_HARNESS, encoding="ascii")
tinf_object = source / "ota_tinf.o"
compile_c = subprocess.run(
[
cc,
"-std=c99",
"-DOTA_TRANSPORT_DEFLATE_TEST",
f"-I{ota_dir}",
"-c",
str(ota_dir / "OtaTinf.c"),
"-o",
str(tinf_object),
],
capture_output=True,
text=True,
check=False,
)
self.assertEqual(compile_c.returncode, 0, compile_c.stderr)
executable = source / (
"bridge_behavior.exe" if os.name == "nt" else "bridge_behavior"
)
compile_cpp = subprocess.run(
[
cxx,
"-std=c++11",
"-Wall",
"-Wextra",
"-Werror",
"-DOTA_TRANSPORT_DEFLATE_TEST",
f"-I{ota_dir}",
str(ota_dir / "OtaDeflate.cpp"),
str(ota_dir / "OtaProtocol.cpp"),
str(harness),
str(tinf_object),
"-o",
str(executable),
],
capture_output=True,
text=True,
check=False,
)
self.assertEqual(compile_cpp.returncode, 0, compile_cpp.stderr)
run_result = subprocess.run(
[str(executable)], capture_output=True, text=True, check=False
)
self.assertEqual(run_result.returncode, 0, run_result.stderr)
class ExactTransformTests(unittest.TestCase):
def test_replace_exact_replaces_one_anchor(self) -> None:
@@ -57,10 +413,10 @@ class ExactTransformTests(unittest.TestCase):
class BuilderConfigurationTests(unittest.TestCase):
def test_inflate_builder_has_distinct_reproducibility_identity(self) -> None:
self.assertEqual(
inflate.VERSION_SUFFIX, "halo-keymind-cascade-mota-inflate"
inflate.VERSION_SUFFIX, "halo-keymind-cascade-mota-inflate2k"
)
self.assertEqual(
inflate.INFLATE_SNAPSHOT_COMMIT,
inflate.INFLATE_BASE_SNAPSHOT_COMMIT,
"add51bf00c46c15ef54318ca766a6daf08a147ee",
)
self.assertNotIn("6f03eae8", {target.source_commit[:8] for target in inflate.TARGETS})
+268 -7
View File
@@ -90,6 +90,7 @@ def target(
boot_codecs: int | None = None,
boot_version: str | None = None,
current_version: str | None = None,
max_block_size: int = ota.MOTA_MAX_BLOCK_SIZE,
) -> ota.TargetInfo:
return ota.TargetInfo(
name="remote",
@@ -105,6 +106,7 @@ def target(
self_status="self",
current_version=current_version,
nrf_qspi=nrf_qspi,
max_block_size=max_block_size,
)
@@ -150,10 +152,15 @@ class FormatTests(unittest.TestCase):
with self.assertRaisesRegex(ota.OtaError, "invalid flags in v2"):
ota.parse_mota(bytes(blob))
def test_2k_application_block_is_accepted(self) -> None:
image = firmware(b"C" * 5000, VERSION_NEW)
parsed = ota.parse_mota(mota_blob(image, block_size_log2=11))
self.assertEqual(parsed.block_size, 2048)
def test_block_larger_than_firmware_buffer_is_rejected(self) -> None:
image = firmware(b"C" * 5000, VERSION_NEW)
with self.assertRaisesRegex(ota.OtaError, "at most 1024 bytes"):
ota.parse_mota(mota_blob(image, block_size_log2=11))
with self.assertRaisesRegex(ota.OtaError, "at most 2048 bytes"):
ota.parse_mota(mota_blob(image, block_size_log2=12))
def test_manifest_endf_identity_mismatch_is_rejected(self) -> None:
wrong_image = firmware(b"B" * 5000, VERSION_NEW, target=TARGET + 1)
@@ -2097,6 +2104,38 @@ class CompatibilityTests(unittest.TestCase):
nrf = target(platform="nrf52", nrf_qspi=True, boot_codecs=1)
self.assertTrue(ota.compatible_mota(full, nrf)[0])
def test_2k_mota_requires_explicit_live_target_capability(self) -> None:
package = ota.parse_mota(
mota_blob(self.new_image, block_size_log2=11)
)
legacy = target(max_block_size=ota.LEGACY_TARGET_MAX_BLOCK_SIZE)
good, reason = ota.compatible_mota(package, legacy)
self.assertFalse(good)
self.assertIn("block size 2048", reason)
self.assertIn("maxblk:1024", reason)
self.assertTrue(ota.compatible_mota(package, target())[0])
def test_direct_2k_mota_is_rejected_before_any_tool_is_run(self) -> None:
package = mota_blob(self.new_image, block_size_log2=11)
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
source = root / "update.mota"
source.write_bytes(package)
work = root / "work"
work.mkdir()
with (
mock.patch.object(ota, "run_checked") as run,
self.assertRaisesRegex(
ota.OtaError, r"block size 2048.*maxblk:1024"
),
):
ota.prepare_package(
prepare_args(source, "motatool"),
target(max_block_size=ota.LEGACY_TARGET_MAX_BLOCK_SIZE),
work,
)
run.assert_not_called()
def test_zip_prefers_equal_version_delta(self) -> None:
full = mota_blob(self.new_image)
delta = mota_blob(
@@ -2151,12 +2190,55 @@ class CompatibilityTests(unittest.TestCase):
if call.kwargs["label"] == "build mOTA"
)
self.assertEqual(build_call.kwargs["timeout"], 4321)
build_command = build_call.args[0]
self.assertEqual(
build_command[build_command.index("--block-size") + 1], "2048"
)
verify_call = next(
call for call in run.call_args_list
if call.kwargs["label"].startswith("verify ")
)
self.assertEqual(verify_call.kwargs["timeout"], 120)
def test_raw_package_build_uses_legacy_live_block_limit(self) -> None:
image = firmware(b"legacy live block limit" * 300, VERSION_NEW)
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
archive_path = root / "release.zip"
with zipfile.ZipFile(archive_path, "w") as archive:
archive.writestr("firmware.bin", image)
work = root / "work"
work.mkdir()
def run_tool(
command: list[str], *, label: str, **_kwargs: object,
) -> subprocess.CompletedProcess[str]:
if label == "build mOTA":
output = Path(command[command.index("--out") + 1])
output.write_bytes(mota_blob(image, block_size_log2=10))
return subprocess.CompletedProcess(
args=command, returncode=0, stdout="OK", stderr=""
)
with mock.patch.object(
ota, "run_checked", side_effect=run_tool
) as run:
_path, package, _expected = ota.prepare_package(
prepare_args(archive_path, "motatool"),
target(max_block_size=ota.LEGACY_TARGET_MAX_BLOCK_SIZE),
work,
)
self.assertEqual(package.block_size, 1024)
build_call = next(
call for call in run.call_args_list
if call.kwargs["label"] == "build mOTA"
)
build_command = build_call.args[0]
self.assertEqual(
build_command[build_command.index("--block-size") + 1], "1024"
)
def test_base_zip_selects_running_hash_not_newest_file(self) -> None:
other = firmware(b"other" * 1300, VERSION_NEW)
with tempfile.TemporaryDirectory() as directory:
@@ -4587,6 +4669,9 @@ class ReliabilityTests(unittest.TestCase):
)
self.assertEqual(result.platform, "esp32")
self.assertIsNone(result.bootloader_version)
self.assertEqual(
result.max_block_size, ota.LEGACY_TARGET_MAX_BLOCK_SIZE
)
self.assertEqual(result.current_version, "v1.16.9")
self.assertEqual(result.current_version_source, "ver")
@@ -4625,9 +4710,11 @@ class ReliabilityTests(unittest.TestCase):
def __init__(self) -> None:
self.commands: list[str] = []
self.replies = iter([
"OTA | no download | target:1234ABCD hw=RAK_3401",
"OTA | no download | target:1234ABCD | maxblk:2048 "
"| hw=RAK_3401",
"> 0.9.2-OTAFIX2.4",
"self body=1 image=2 base_hash=0011223344556677 | "
"self body=1 image=2 base_hash=0011223344556677 "
"| maxblk:2048 | "
"bootloader: apply OK (abi=2 codecs=0x4)",
"OTA | fw v1.17.0 id=00112233",
])
@@ -4646,11 +4733,23 @@ class ReliabilityTests(unittest.TestCase):
self.assertEqual(result.bootloader_version, "0.9.2-OTAFIX2.4")
self.assertEqual(result.bootloader_abi, 2)
self.assertEqual(result.bootloader_codecs, 0x4)
self.assertEqual(result.max_block_size, 2048)
self.assertEqual(
controller.commands,
["ota status", "get bootloader.ver", "ota self", "ota stats"],
)
def test_target_rejects_conflicting_or_unsupported_maxblk_markers(self) -> None:
with self.assertRaisesRegex(ota.OtaError, "conflicting maxblk"):
ota.parse_target_max_block_size(
"OTA | target:1234ABCD | maxblk:2048",
"self base_hash=0011223344556677 | maxblk:1024",
)
with self.assertRaisesRegex(ota.OtaError, "unsupported maxblk:4096"):
ota.parse_target_max_block_size(
"OTA | target:1234ABCD | maxblk:4096", "self"
)
def test_optional_ota_stats_loss_is_bounded_then_falls_back_to_ver(
self,
) -> None:
@@ -6779,6 +6878,58 @@ class Rak3401ExtractionCacheTests(unittest.TestCase):
class Rak3401KnownUnsafeReleaseTests(unittest.TestCase):
@staticmethod
def live_chain_args() -> argparse.Namespace:
return argparse.Namespace(
meshcli="meshcli",
controller_serial="/dev/controller",
controller_tcp=None,
controller_ble=None,
controller_baud=115200,
reply_timeout=45,
)
@staticmethod
def live_chain_target(bootloader_version: str | None) -> ota.TargetInfo:
return ota.TargetInfo(
name="remote",
target_id=rak_chain.EXPECTED_TARGET_ID,
base_hash=b"12345678",
platform="nrf52",
nrf_sd=False,
hw_id=rak_chain.EXPECTED_HARDWARE,
bootloader_version=bootloader_version,
bootloader_abi=2,
bootloader_codecs=1 << ota.MOTA_CODEC_IN_PLACE,
status="status",
self_status="self",
nrf_qspi=False,
)
def test_live_chain_accepts_exact_deployed_bootloader_version(self) -> None:
expected = self.live_chain_target(rak_chain.EXPECTED_BOOTLOADER_VERSION)
with mock.patch.object(
rak_chain.ota, "query_target", return_value=expected
):
actual = rak_chain.query_live_target(
mock.Mock(), self.live_chain_args(), "remote"
)
self.assertIs(actual, expected)
def test_live_chain_rejects_any_other_bootloader_version(self) -> None:
for version in (None, "0.9.2-OTAFIX2.3", "0.9.2-OTAFIX2.5"):
with self.subTest(version=version), mock.patch.object(
rak_chain.ota,
"query_target",
return_value=self.live_chain_target(version),
), self.assertRaisesRegex(
ota.OtaError,
"expected exact deployed version '0.9.2-OTAFIX2.4'",
):
rak_chain.query_live_target(
mock.Mock(), self.live_chain_args(), "remote"
)
def test_chain_resume_uses_exact_body_hash_not_runtime_label(self) -> None:
steps = [
mock.Mock(base_hash=bytes.fromhex("0011223344556677")),
@@ -7026,7 +7177,22 @@ class Rak3401KnownUnsafeReleaseTests(unittest.TestCase):
steps[number - 1].target_sha256 = image_sha256
with self.assertRaisesRegex(
rak_chain.KnownUnsafeReleaseError,
"new b40d2e6c step 10.*bootloader simulators only",
"new 3f6eddd5 step 10.*exact OTAFIX2.4 bootloader simulator only",
):
rak_chain.require_live_release_safe(
argparse.Namespace(accept_test_candidate=False), steps
)
rak_chain.require_live_release_safe(
argparse.Namespace(accept_test_candidate=True), steps
)
def test_exact_superseded_b40_ten_step_candidate_remains_gated(self) -> None:
steps = [mock.Mock(target_sha256="") for _ in range(10)]
for number, image_sha256 in rak_chain.SUPERSEDED_B40_ANCHORS:
steps[number - 1].target_sha256 = image_sha256
with self.assertRaisesRegex(
rak_chain.KnownUnsafeReleaseError,
"exact superseded b40d2e6c ten-step prerelease.*step 10",
):
rak_chain.require_live_release_safe(
argparse.Namespace(accept_test_candidate=False), steps
@@ -7050,31 +7216,46 @@ class Rak3401KnownUnsafeReleaseTests(unittest.TestCase):
argparse.Namespace(accept_test_candidate=True), steps
)
def test_current_and_legacy_ten_step_reports_do_not_mix_evidence(self) -> None:
def test_ten_step_reports_do_not_mix_current_and_legacy_evidence(self) -> None:
current = [mock.Mock(target_sha256="") for _ in range(10)]
superseded = [mock.Mock(target_sha256="") for _ in range(10)]
legacy = [mock.Mock(target_sha256="") for _ in range(10)]
for number, image_sha256 in rak_chain.CURRENT_10_CANDIDATE_ANCHORS:
current[number - 1].target_sha256 = image_sha256
for number, image_sha256 in rak_chain.SUPERSEDED_B40_ANCHORS:
superseded[number - 1].target_sha256 = image_sha256
for number, image_sha256 in rak_chain.PHYSICALLY_PASSED_FD98_ANCHORS:
legacy[number - 1].target_sha256 = image_sha256
current_message = rak_chain.ten_step_verification_message(current)
superseded_message = rak_chain.ten_step_verification_message(superseded)
legacy_message = rak_chain.ten_step_verification_message(legacy)
self.assertIn("new step 10", current_message)
self.assertIn("exact OTAFIX2.4 bootloader simulation", current_message)
self.assertIn("not had a clean physical run", current_message)
self.assertNotIn("endpoint passed independent SWD readback", current_message)
self.assertIn("Superseded b40d2e6c", superseded_message)
self.assertNotIn("exact OTAFIX2.4 bootloader simulation", superseded_message)
self.assertIn("exact ten package transitions", legacy_message)
self.assertIn("endpoint passed independent SWD readback", legacy_message)
def test_ten_step_candidates_share_only_the_pinned_physical_prefix(self) -> None:
def test_ten_step_candidates_share_the_pinned_physical_prefix(self) -> None:
self.assertEqual(
rak_chain.CURRENT_10_CANDIDATE_ANCHORS[:9],
rak_chain.PHYSICALLY_PASSED_FD98_ANCHORS[:9],
)
self.assertEqual(
rak_chain.SUPERSEDED_B40_ANCHORS[:9],
rak_chain.PHYSICALLY_PASSED_FD98_ANCHORS[:9],
)
self.assertNotEqual(
rak_chain.CURRENT_10_CANDIDATE_ANCHORS[-1],
rak_chain.PHYSICALLY_PASSED_FD98_ANCHORS[-1],
)
self.assertNotEqual(
rak_chain.CURRENT_10_CANDIDATE_ANCHORS[-1],
rak_chain.SUPERSEDED_B40_ANCHORS[-1],
)
steps = [mock.Mock(target_sha256="") for _ in range(10)]
for number, image_sha256 in rak_chain.CURRENT_10_CANDIDATE_ANCHORS:
steps[number - 1].target_sha256 = image_sha256
@@ -7102,6 +7283,62 @@ class Rak3401KnownUnsafeReleaseTests(unittest.TestCase):
):
rak_chain.download_release_asset(destination)
def test_primary_release_and_endpoint_are_pinned(self) -> None:
self.assertEqual(
rak_chain.RELEASE_URL,
"https://github.com/mikecarper/MeshCore/releases/tag/"
"rak3401-mota-v1.16.07-c1caa5ad-to-v1.17.1.6-3f6eddd5",
)
self.assertEqual(
rak_chain.ASSET_NAME,
"RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5.zip",
)
self.assertEqual(
rak_chain.BUNDLE_ROOT_NAME,
"RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.6-3f6eddd5",
)
self.assertEqual(
rak_chain.ASSET_SHA256,
"135783dd8777490db2422a7543f7d80e3cf03f621d5ac362da930abbc1850dc4",
)
self.assertEqual(
rak_chain.CHECKSUM_LIST_SHA256,
"93835ed497c579be0a2322292e7f15ab095e7c8ab7892833fe7df3130dbdaba4",
)
self.assertEqual(rak_chain.EXPECTED_FINAL_VERSION, "1.17.1.6")
self.assertEqual(
rak_chain.CURRENT_10_CANDIDATE_ANCHORS[-1],
(
10,
"cd1e9b819f9e918f9f2ceaf37b8a74bf242e2be78f319af2cf02a42d7729d0f4",
),
)
self.assertEqual(
rak_chain.PINNED_ARCHIVE_CHECKSUMS[rak_chain.ASSET_SHA256],
rak_chain.CHECKSUM_LIST_SHA256,
)
def test_superseded_b40_archive_and_inner_checksum_remain_pinned(self) -> None:
self.assertEqual(
rak_chain.SUPERSEDED_B40_ASSET_SHA256,
"8e7f0c2565f37d4372fa51dc3ba72b591e7f38b74be09303e48641623f99bce4",
)
self.assertEqual(
rak_chain.SUPERSEDED_B40_CHECKSUM_LIST_SHA256,
"29a02a6c4c83eafa80168a80562b0ded5d8432e6cc02cfaca289cb2ed426681a",
)
self.assertEqual(
rak_chain.SUPERSEDED_B40_ROOT_NAME,
"RAK3401-update-chain-v1.16.7-c1caa5ad-to-v1.17.1.5-b40d2e6c",
)
self.assertEqual(rak_chain.SUPERSEDED_B40_FINAL_VERSION, "1.17.1.5")
self.assertEqual(
rak_chain.PINNED_ARCHIVE_CHECKSUMS[
rak_chain.SUPERSEDED_B40_ASSET_SHA256
],
rak_chain.SUPERSEDED_B40_CHECKSUM_LIST_SHA256,
)
def test_fd98_archive_and_inner_checksum_remain_pinned(self) -> None:
self.assertEqual(
rak_chain.PINNED_ARCHIVE_CHECKSUMS[
@@ -7134,6 +7371,30 @@ class Rak3401KnownUnsafeReleaseTests(unittest.TestCase):
rak_chain.PHYSICALLY_PASSED_FD98_ASSET_SHA256,
)
def test_locate_bundle_accepts_exact_superseded_b40_archive(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
archive = root / rak_chain.SUPERSEDED_B40_ASSET_NAME
archive.write_bytes(b"fixture is identified by the mocked digest")
extracted = root / rak_chain.SUPERSEDED_B40_ROOT_NAME
args = argparse.Namespace(bundle=archive)
with (
mock.patch.object(
rak_chain,
"sha256_file_limited",
return_value=rak_chain.SUPERSEDED_B40_ASSET_SHA256,
),
mock.patch.object(
rak_chain, "extract_bundle", return_value=extracted
) as extract_bundle,
):
self.assertEqual(rak_chain.locate_bundle(args, root / "work"), extracted)
extract_bundle.assert_called_once_with(
archive.resolve(),
root / "work" / "bundle",
rak_chain.SUPERSEDED_B40_ASSET_SHA256,
)
def test_compact_9_step_release_rejects_changed_anchor(self) -> None:
steps = [mock.Mock(target_sha256="") for _ in range(9)]
for number, image_sha256 in rak_chain.COMPACT_RELEASE_ANCHORS:
+331 -5
View File
@@ -33,6 +33,7 @@ class RouteSearchTests(unittest.TestCase):
self, root: Path, payloads: list[bytes], *, baseline_size: int = 100,
versions: list[str] | None = None,
transport_capabilities: list[bool] | None = None,
transport_block_sizes: list[int] | None = None,
transport_pipelines: list[int | None] | None = None,
version_ranks: list[int] | None = None,
) -> tuple[Path, list[dict[str, object]]]:
@@ -60,6 +61,14 @@ class RouteSearchTests(unittest.TestCase):
if transport_capabilities is not None else False
)
record[search.TRANSPORT_CAPABILITY] = capability
record[search.TRANSPORT_MAX_BLOCK_FIELD] = (
transport_block_sizes[node]
if transport_block_sizes is not None
else (
search.DEFLATE_BLOCK_SIZE
if capability else search.LEGACY_BLOCK_SIZE
)
)
record[search.TRANSPORT_PIPELINE_FIELD] = (
transport_pipelines[node]
if transport_pipelines is not None else None
@@ -69,6 +78,10 @@ class RouteSearchTests(unittest.TestCase):
manifest.write_text(json.dumps({
"capability_contract": {
search.TRANSPORT_CAPABILITY: "required boolean on every record",
search.TRANSPORT_MAX_BLOCK_FIELD: (
"required signed-container block size on every record; "
"independent of ota_transport_deflate"
),
},
"edge_policy": {
"require_target_version_rank_gt_source_version_rank": True,
@@ -120,6 +133,49 @@ class RouteSearchTests(unittest.TestCase):
)
self.assertEqual((projected[0]["source"], projected[0]["target"]), (2, 3))
def test_cache_migration_reuses_payload_but_recomputes_2k_geometry(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
manifest, records = self.write_inventory(
root,
[b"start", b"capable", b"endpoint"],
transport_capabilities=[False, True, False],
transport_pipelines=[None, 1, None],
)
payload = 3000
old_total = search.container_size(payload, search.LEGACY_BLOCK_SIZE)
old_stage = search.align_down(search.STAGE_CEILING - old_total)
cache = root / "old-1k.csv"
with cache.open("w", newline="", encoding="ascii") as output:
writer = csv.DictWriter(output, fieldnames=search.FIELDS)
writer.writeheader()
writer.writerow({
"source": 1,
"target": 2,
"source_sha256": records[1]["sha256"],
"target_sha256": records[2]["sha256"],
"memory": search.FIXED_MEMORY,
"payload": payload,
# Blank means a pre-per-step cache with 1 KiB geometry.
"block_size": "",
"container": old_total,
"stage_start": old_stage,
"margin": old_stage - (search.APP_BASE + search.FIXED_MEMORY),
"feasible": True,
"error": "",
})
migrated = search.migrate_csv(manifest, cache)
row = next(iter(migrated.values()))
self.assertEqual(row["payload"], payload)
self.assertEqual(row["block_size"], search.DEFLATE_BLOCK_SIZE)
self.assertEqual(
row["container"],
search.container_size(payload, search.DEFLATE_BLOCK_SIZE),
)
self.assertFalse(search.transport_stats_complete(row))
def test_all_jobs_covers_every_valid_page_workspace(self) -> None:
with tempfile.TemporaryDirectory() as directory:
manifest, _ = self.write_inventory(
@@ -216,7 +272,7 @@ class RouteSearchTests(unittest.TestCase):
1, 2, search.FIXED_MEMORY,
str(Path(directory) / "from.bin"),
str(Path(directory) / "to.bin"),
"1" * 64, "2" * 64, directory,
"1" * 64, "2" * 64, search.LEGACY_BLOCK_SIZE, directory,
))
def test_snapshot_inventory_is_immune_to_later_source_mutation(self) -> None:
@@ -272,7 +328,7 @@ class RouteSearchTests(unittest.TestCase):
1, 2, search.FIXED_MEMORY,
str(inventory_root / str(records[1]["path"])),
str(inventory_root / str(records[2]["path"])),
source_sha, target_sha,
source_sha, target_sha, search.LEGACY_BLOCK_SIZE,
)
def measured(args: tuple[object, ...]) -> dict[str, object]:
@@ -285,6 +341,7 @@ class RouteSearchTests(unittest.TestCase):
"source_sha256": source_sha,
"target_sha256": target_sha,
"memory": memory, "payload": payload,
"block_size": search.LEGACY_BLOCK_SIZE,
"container": total, "stage_start": stage,
"margin": stage - (search.APP_BASE + int(memory)),
"feasible": True, "error": "",
@@ -400,8 +457,14 @@ class RouteSearchTests(unittest.TestCase):
# The receive-inflate compatibility bridges are compiled with a single
# block in flight. Account for their five individual request frames,
# rather than the full RAK3401 receiver's clean [1, 2, 2] flights.
full = search.v2_transport_cost(5 * 1024, 5 * 1024, 0, 0, 30, 4)
bridge = search.v2_transport_cost(5 * 1024, 5 * 1024, 0, 0, 30, 1)
full = search.v2_transport_cost(
5 * 1024, 5 * 1024, 0, 0, 30, 4,
search.LEGACY_BLOCK_SIZE,
)
bridge = search.v2_transport_cost(
5 * 1024, 5 * 1024, 0, 0, 30, 1,
search.LEGACY_BLOCK_SIZE,
)
self.assertEqual(full["request_packets"], 3)
self.assertEqual(full["request_bytes"], 41)
self.assertEqual(bridge["request_packets"], 5)
@@ -410,6 +473,27 @@ class RouteSearchTests(unittest.TestCase):
int(bridge["origin_mesh_bytes"]) - int(full["origin_mesh_bytes"]), 14
)
def test_deflate_geometry_uses_2k_blocks_and_raw_baseline_uses_1k(self) -> None:
payload = 4096
self.assertEqual(
search.container_size(payload, search.LEGACY_BLOCK_SIZE),
payload + 210 + 4 * 4,
)
self.assertEqual(
search.container_size(payload, search.DEFLATE_BLOCK_SIZE),
payload + 210 + 4 * 2,
)
selected = search.v2_transport_cost(
payload, 2000, 2000, 2, 12, 1
)
baseline = search.v2_raw_transport_cost(payload, 1)
self.assertEqual(selected["block_size"], search.DEFLATE_BLOCK_SIZE)
self.assertEqual(selected["proof_packets"], 2)
self.assertEqual(baseline["profile"], "v2-171-raw")
self.assertEqual(baseline["block_size"], search.LEGACY_BLOCK_SIZE)
self.assertEqual(baseline["proof_packets"], 4)
self.assertGreater(baseline["packets"], selected["packets"])
def test_published_legacy_chain_and_new_bootstrap_cost_regressions(self) -> None:
old_payloads = [
89282, 45974, 105407, 149133, 110666,
@@ -429,6 +513,59 @@ class RouteSearchTests(unittest.TestCase):
with self.assertRaisesRegex(search.RouteSearchError, "between 0 and 8"):
search.linear_path_bytes(100, 1, 9)
def test_current_route_2k_and_1k_raw_projection_regressions(self) -> None:
# Exact current-route detools payloads and pinned motatool 2 KiB
# measurements. Steps 1, 2, and 10 remain legacy; steps 3-9 negotiate
# v2 and use a one-block request pipeline.
legacy_payloads = [89_282, 64_756, 196_843]
v2_measurements = [
(104_543, 60_379, 60_284, 51, 381),
(148_997, 92_549, 92_549, 73, 577),
(112_123, 59_704, 59_704, 55, 375),
(121_934, 63_977, 63_977, 60, 405),
(135_068, 87_425, 87_425, 66, 547),
(176_140, 130_264, 130_252, 86, 807),
(271_013, 210_192, 210_192, 133, 1297),
]
selected = [
search.legacy_transport_cost(payload) for payload in legacy_payloads
] + [
search.v2_transport_cost(*measurement, 1, search.DEFLATE_BLOCK_SIZE)
for measurement in v2_measurements
]
baseline = [
search.legacy_transport_cost(payload) for payload in legacy_payloads
] + [
search.v2_raw_transport_cost(
measurement[0], 1, search.LEGACY_BLOCK_SIZE
)
for measurement in v2_measurements
]
self.assertEqual(
(
sum(int(cost["origin_mesh_bytes"]) for cost in selected),
sum(int(cost["packets"]) for cost in selected),
),
(1_195_442, 8_906),
)
self.assertEqual(
(
sum(int(cost["origin_mesh_bytes"]) for cost in baseline),
sum(int(cost["packets"]) for cost in baseline),
),
(1_618_318, 11_835),
)
self.assertEqual(
(
1_618_318 - 1_195_442,
11_835 - 8_906,
),
(422_876, 2_929),
)
self.assertEqual(round(100 * 422_876 / 1_618_318, 2), 26.13)
self.assertEqual(round(100 * 2_929 / 11_835, 2), 24.75)
def test_proof_sizing_matches_promote_odd_merkle_implementation(self) -> None:
for count in range(1, 33):
leaves = [bytes([index]) * 4 for index in range(count)]
@@ -491,6 +628,22 @@ class RouteSearchTests(unittest.TestCase):
self.assertEqual(
transport["steps"][2]["transport"]["request_pipeline"], 1
)
self.assertEqual(
[step["block_size"] for step in transport["steps"]],
[
search.LEGACY_BLOCK_SIZE,
search.LEGACY_BLOCK_SIZE,
search.DEFLATE_BLOCK_SIZE,
],
)
self.assertEqual(
transport["transport_accounting"]["source_max_block_field"],
search.TRANSPORT_MAX_BLOCK_FIELD,
)
self.assertEqual(
transport["transport_accounting"]["source_profile_matrix"],
search.TRANSPORT_PROFILE_MATRIX,
)
self.assertEqual(
transport["selected_total_bytes"],
sum(step["expected_container_size"] for step in transport["steps"]),
@@ -509,6 +662,162 @@ class RouteSearchTests(unittest.TestCase):
)
self.assertEqual(container["nodes"], [0, 1, 3, 4])
def test_block_size_and_deflate_capabilities_form_independent_matrix(
self,
) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
manifest, _ = self.write_inventory(
root, [b"bootstrap", b"legacy", b"deflate-1k", b"raw-2k", b"deflate-2k"],
transport_capabilities=[False, False, True, False, True],
transport_block_sizes=[
search.LEGACY_BLOCK_SIZE,
search.LEGACY_BLOCK_SIZE,
search.LEGACY_BLOCK_SIZE,
search.DEFLATE_BLOCK_SIZE,
search.DEFLATE_BLOCK_SIZE,
],
transport_pipelines=[None, None, 1, 2, 4],
)
images = search.load_inventory(manifest)
self.assertEqual(
[search.source_block_size(images, node) for node in range(1, 5)],
[1024, 1024, 2048, 2048],
)
self.assertEqual(
[
search.source_transport_profile(images, node)
for node in range(1, 5)
],
[
"legacy-160-raw",
"v2-171-deflate",
"v2-171-raw",
"v2-171-deflate",
],
)
jobs = [
(
node, 9, search.FIXED_MEMORY, "from", "to",
"1" * 64, "2" * 64,
search.source_block_size(images, node),
)
for node in range(1, 5)
]
self.assertEqual(
[
search.job_needs_transport_stats(job, images, "transport")
for job in jobs
],
[False, True, False, True],
)
def edge(source: int, include_measurement: bool) -> dict[str, object]:
block_size = search.source_block_size(images, source)
payload = 3000
row: dict[str, object] = {
"source": source,
"target": 4,
"memory": 0x1000,
"payload": payload,
"block_size": block_size,
"container": search.container_size(payload, block_size),
"stage_start": 0xC0000,
"margin": 100,
"feasible": True,
}
if include_measurement:
row.update({
"payload_sha256": "a" * 64,
"transport_encoder_sha256": "b" * 64,
"v2_wire_bytes": 1500,
"v2_deflate_bytes": 1500,
"v2_deflate_blocks": (
payload + block_size - 1
) // block_size,
"v2_data_packets": 9,
})
return row
self.assertEqual(
search.edge_transport_cost(edge(1, False), images, 0)["profile"],
"legacy-160-raw",
)
with self.assertRaisesRegex(search.RouteSearchError, "missing transport"):
search.edge_transport_cost(edge(2, False), images, 0)
self.assertEqual(
search.edge_transport_cost(edge(2, True), images, 0)["profile"],
"v2-171-deflate",
)
# Raw 2 KiB v2 accounting is derived and needs no encoder evidence.
raw_2k = search.edge_transport_cost(edge(3, False), images, 0)
self.assertEqual(raw_2k["profile"], "v2-171-raw")
self.assertEqual(raw_2k["block_size"], search.DEFLATE_BLOCK_SIZE)
self.assertEqual(
search.edge_transport_cost(edge(4, True), images, 0)["profile"],
"v2-171-deflate",
)
def test_raw_2k_route_requires_no_transport_encoder_evidence(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
baseline = search.container_size(64)
manifest, _ = self.write_inventory(
root, [b"bootstrap", b"raw-2k", b"endpoint"],
baseline_size=baseline,
transport_capabilities=[False, False, False],
transport_block_sizes=[1024, 2048, 1024],
transport_pipelines=[None, 2, None],
)
images = search.load_inventory(manifest)
payload = 3000
row = {
"source": 1,
"target": 2,
"memory": search.FIXED_MEMORY,
"payload": payload,
"block_size": 2048,
"container": search.container_size(payload, 2048),
"stage_start": 0xC0000,
"margin": 100,
"feasible": True,
}
result = search.select_route(
[row], images, baseline, root / "route.json", True,
objective="transport",
)
self.assertEqual(
result["steps"][1]["transport"]["profile"], "v2-171-raw"
)
self.assertNotIn("payload_sha256", result["steps"][1]["transport"])
self.assertNotIn("encoder_sha256", result["steps"][1]["transport"])
self.assertNotIn("encoder_sha256", result["transport_accounting"])
def test_transport_image_without_explicit_block_capability_fails_closed(
self,
) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
manifest, _ = self.write_inventory(
root, [b"start", b"capable", b"endpoint"],
transport_capabilities=[False, True, False],
transport_pipelines=[None, 1, None],
)
document = json.loads(manifest.read_text(encoding="ascii"))
del document["capability_contract"][search.TRANSPORT_MAX_BLOCK_FIELD]
for image in document["images"]:
del image[search.TRANSPORT_MAX_BLOCK_FIELD]
manifest.write_text(json.dumps(document), encoding="ascii")
with self.assertRaisesRegex(
search.RouteSearchError,
"transport_max_block_bytes is required by the inventory",
):
search.load_inventory(manifest)
def test_transport_stats_are_required_only_on_the_shortest_hop_dag(self) -> None:
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
@@ -609,7 +918,7 @@ class RouteSearchTests(unittest.TestCase):
tool = root / "motatool"
tool.write_bytes(b"exact encoder binary")
measured = {
"schema": 1, "payload_bytes": 300, "block_size": 1024,
"schema": 1, "payload_bytes": 300, "block_size": 2048,
"block_count": 1, "wire_bytes": 200, "deflate_bytes": 200,
"deflate_blocks": 1, "data_packets": 2,
}
@@ -720,16 +1029,33 @@ class RouteSearchTests(unittest.TestCase):
del bad["images"][1][search.TRANSPORT_CAPABILITY]
cases.append((bad, "required by the inventory capability contract"))
bad = json.loads(manifest.read_text())
del bad["images"][1][search.TRANSPORT_MAX_BLOCK_FIELD]
cases.append((bad, "required by the inventory capability contract"))
bad = json.loads(manifest.read_text())
bad["images"][1][search.TRANSPORT_MAX_BLOCK_FIELD] = 4096
cases.append((bad, "must be 1024 or 2048"))
bad = json.loads(manifest.read_text())
bad["images"][1][search.TRANSPORT_CAPABILITY_ALIAS] = True
cases.append((bad, "disagrees"))
bad = json.loads(manifest.read_text())
bad["images"][1][search.TRANSPORT_CAPABILITY] = True
bad["images"][1][search.TRANSPORT_MAX_BLOCK_FIELD] = (
search.DEFLATE_BLOCK_SIZE
)
cases.append((bad, "is required"))
bad = json.loads(manifest.read_text())
bad["images"][1][search.TRANSPORT_CAPABILITY] = True
bad["images"][1][search.TRANSPORT_MAX_BLOCK_FIELD] = (
search.DEFLATE_BLOCK_SIZE
)
bad["images"][1][search.TRANSPORT_PIPELINE_FIELD] = 0
cases.append((bad, "must be between"))
bad = json.loads(manifest.read_text())
bad["images"][1][search.TRANSPORT_MAX_BLOCK_FIELD] = (
search.DEFLATE_BLOCK_SIZE
)
cases.append((bad, "required for v2 transport"))
bad = json.loads(manifest.read_text())
del bad["images"][1]["version_rank"]
cases.append((bad, "required by the inventory edge policy"))
bad = json.loads(manifest.read_text())
+2
View File
@@ -58,6 +58,8 @@ family derived from the environment name, and `fw_version` = parsed from `FIRMWA
## Privileged nRF52 bootloader reference builder
Ordinary calls to `build_manifest()` continue to emit v2 application packages.
New application packages default to 2048-byte blocks; pass 1024 explicitly when
targeting a deployed receiver that predates the extended transport descriptor.
Passing `bootloader=True` is a deliberately narrow reference-only path: it
requires a nonzero version, a signed exact 40 KiB OTAFIX region, 1024-byte
blocks, full codec, zero base hash, a sane vector table, an exact embedded
+38
View File
@@ -11,6 +11,7 @@ Run: ./meshcore/bin/python tools/mota/gen_vectors.py
from __future__ import annotations
import random
import zlib
from pathlib import Path
import motalib as ml
@@ -23,6 +24,11 @@ def _carr(name, data: bytes) -> str:
return f"static const uint8_t {name}[{len(data)}] = {{{body}}};\n"
def _u16arr(name: str, values: list[int]) -> str:
body = ",".join(str(value) for value in values)
return f"static const uint16_t {name}[{len(values)}] = {{{body}}};\n"
def build_full():
random.seed(42)
fw = bytes(random.getrandbits(8) for _ in range(5 * 1024 + 137)) # 6 blocks, last short
@@ -132,6 +138,38 @@ def main():
lines.append(f"static const uint32_t SIM_MOTA_1K_LEN = {len(sim1k_blob)};")
lines.append(f"static const uint32_t SIM_MOTA_1K_BLOCKS = {sm1k.block_count};")
# 2 KiB-block signed .mota: one full block plus a short final block. The payload is deliberately
# compressible so its independently generated raw-RFC1951 streams exercise both extended-length
# DEFLATE descriptors, including the rule that a short tail still uses the manifest's 2 KiB profile.
sim2k_fw_len = 2 * 1024 + 137
sim2k_phrase = b"MeshCore 2 KiB transport DEFLATE vector. "
sim2k_fw = (sim2k_phrase * ((sim2k_fw_len + len(sim2k_phrase) - 1) // len(sim2k_phrase)))[:sim2k_fw_len]
sim2k_image, _ = ml.ensure_endf(sim2k_fw)
sm2k = ml.build_manifest(target_id=0xCAFEBABE, fw_version=ml.pack_version("3.1.0"),
image_size=len(sim2k_image), payload=sim2k_image, block_size=2048,
image_hash=ml.mh32(sim2k_image), codec_id=ml.CODEC_FULL,
is_full=True, sign_priv=sim_priv)
sim2k_blob = ml.build_container(sm2k, sim2k_image)
compressed = bytearray()
compressed_offsets = []
compressed_lengths = []
for offset in range(0, len(sim2k_image), 2048):
block = sim2k_image[offset:offset + 2048]
encoder = zlib.compressobj(level=9, wbits=-15)
encoded = encoder.compress(block) + encoder.flush()
assert 0 < len(encoded) < len(block), (len(block), len(encoded))
compressed_offsets.append(len(compressed))
compressed_lengths.append(len(encoded))
compressed.extend(encoded)
assert len(compressed_offsets) == sm2k.block_count
lines.append("// 2 KiB-block signed .mota: one full block plus a short tail")
lines.append(_carr("SIM_MOTA_2K", sim2k_blob))
lines.append(f"static const uint32_t SIM_MOTA_2K_LEN = {len(sim2k_blob)};")
lines.append(f"static const uint32_t SIM_MOTA_2K_BLOCKS = {sm2k.block_count};")
lines.append(_carr("SIM_MOTA_2K_DEFLATED", bytes(compressed)))
lines.append(_u16arr("SIM_MOTA_2K_DEFLATED_OFFSETS", compressed_offsets))
lines.append(_u16arr("SIM_MOTA_2K_DEFLATED_LENGTHS", compressed_lengths))
# detools sequential+crle delta vector: base image, the real detools 0.53.0 patch, and the
# expected target image. The native test applies DT_PATCH to DT_BASE with the *vendored detools
# C decoder* (src/helpers/ota/detools) and must reproduce DT_TARGET byte-for-byte -- proving the
+14 -3
View File
@@ -56,7 +56,9 @@ CODEC_NAMES = {CODEC_FULL: "full", CODEC_DETOOLS_SEQUENTIAL: "detools-sequential
APPROVAL_NOT = b"\xff\xff\xff\xff" # erased = not approved
APPROVAL_YES = b"APRV" # 41 50 52 56 = approved
DEFAULT_BLOCK_SIZE = 1024
DEFAULT_BLOCK_SIZE = 2048
MAX_APP_BLOCK_SIZE = 2048
BOOTLOADER_BLOCK_SIZE = 1024
# Conservative fallback for firmware built before the layout record below existed. Keep in sync with
# OtaFlashLayout_nrf52.h and motatool's format.rs.
@@ -503,7 +505,7 @@ class Manifest:
fw_version: int = 0
image_size: int = 0
payload_size: int = 0
block_size_log2: int = 10
block_size_log2: int = 11
merkle_root: bytes = b"\0\0\0\0"
image_hash: bytes = b"\0" * 32
codec_id: int = CODEC_FULL
@@ -859,6 +861,9 @@ def build_manifest(*, target_id: int, fw_version: int, image_size: int, payload:
base_hash: Optional[bytes] = None, sign_priv=None, hw_id=None,
bootloader: bool = False) -> Manifest:
assert (block_size & (block_size - 1)) == 0, "block_size must be a power of two"
if not 1 < block_size <= MAX_APP_BLOCK_SIZE:
raise ValueError(
f"application block size must be at most {MAX_APP_BLOCK_SIZE} bytes")
leaves = leaf_hashes(payload, block_size)
if bootloader:
identity = validate_bootloader_image(payload, target_id)
@@ -868,7 +873,7 @@ def build_manifest(*, target_id: int, fw_version: int, image_size: int, payload:
if (not bootloader_version_valid(fw_version) or
fw_version != identity.boot_version or not is_full or codec_id != CODEC_FULL or
image_size != XIAO_BOOT_IMAGE_SIZE or
block_size != DEFAULT_BLOCK_SIZE or base_hash not in (None, b"\0" * 8)):
block_size != BOOTLOADER_BLOCK_SIZE or base_hash not in (None, b"\0" * 8)):
raise ValueError("bootloader package version must equal BLM2 metadata and use a 40 KiB CODEC_FULL image with 1 KiB blocks")
if sign_priv is None:
raise ValueError("bootloader package must be signed")
@@ -950,6 +955,12 @@ def parse_container(blob: bytes) -> Parsed:
m.hash_algo = take(1)[0]
m.target_id, m.fw_version, m.image_size, m.payload_size = struct.unpack("<IIII", take(16))
m.block_size_log2 = take(1)[0]
if m.format_ver == APP_FORMAT_VER:
if not 1 <= m.block_size_log2 <= MAX_APP_BLOCK_SIZE.bit_length() - 1:
raise ValueError(
f"v2 application block size must be at most {MAX_APP_BLOCK_SIZE} bytes")
elif m.block_size_log2 != BOOTLOADER_BLOCK_SIZE.bit_length() - 1:
raise ValueError("v3 bootloader manifest requires 1 KiB blocks")
m.merkle_root = take(4)
m.image_hash = take(32)
m.codec_id = take(1)[0]
+26
View File
@@ -1544,6 +1544,7 @@ def test_bootloader_builder_rejects_wrong_identity_geometry_and_continuity():
for change in (
{"sign_priv": None},
{"block_size": 512},
{"block_size": ml.DEFAULT_BLOCK_SIZE},
{"fw_version": 0},
{"target_id": ml.XIAO_BOOT_BOARD_ID_SENSE},
{"hw_id": "wrong"},
@@ -1662,6 +1663,31 @@ def test_full_build_parse_verify():
assert ml.verify(parsed) == []
def test_application_default_and_maximum_block_size_are_2k():
assert ml.DEFAULT_BLOCK_SIZE == ml.MAX_APP_BLOCK_SIZE == 2048
assert ml.Manifest().block_size == 2048
fw = _fw(15, 5 * 1024 + 7)
image, _ = ml.ensure_endf(fw)
manifest = ml.build_manifest(
target_id=0xDEADBEEF, fw_version=ml.pack_version("1.17.2"),
image_size=len(image), payload=image, block_size=ml.DEFAULT_BLOCK_SIZE,
image_hash=ml.mh32(image), codec_id=ml.CODEC_FULL, is_full=True)
parsed = ml.parse_container(ml.build_container(manifest, image))
assert parsed.manifest.block_size_log2 == 11
assert parsed.manifest.block_size == 2048
assert ml.verify(parsed) == []
try:
ml.build_manifest(
target_id=0xDEADBEEF, fw_version=ml.pack_version("1.17.2"),
image_size=len(image), payload=image, block_size=4096,
image_hash=ml.mh32(image), codec_id=ml.CODEC_FULL, is_full=True)
assert False, "v2 application block larger than 2 KiB accepted"
except ValueError:
pass
def test_hw_id_roundtrip_and_signed():
# the v2 hw_id is a 32-byte NUL-padded ASCII tag in the SIGNED head; it must round-trip + be covered
# by the signature (tampering it breaks verification).