302 lines
17 KiB
Python
302 lines
17 KiB
Python
"""All OTA names survive front coding; original Companion API stays unchanged."""
|
|
import re
|
|
import os
|
|
import configparser
|
|
import importlib.util
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
import zlib
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path[:0] = [str(ROOT / "tools" / "mota"), str(ROOT / "scripts")]
|
|
import gen_targets
|
|
from test_serial_wifi_sessions import SANITIZER_FLAGS
|
|
|
|
HEADER = ROOT / "src" / "helpers" / "ota" / "OtaTargets.h"
|
|
HARNESS = ROOT / "test" / "fixtures" / "ota_target_names" / "test.cpp"
|
|
|
|
|
|
class OtaTargetNameTest(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
cls.text = HEADER.read_text(encoding="utf-8")
|
|
cls.rows = [(int(target, 16), name) for target, name in re.findall(
|
|
r'\{ 0x([0-9a-fA-F]{8}), "([^"]+)" \},', cls.text)]
|
|
|
|
def compile_and_lookup(self, flags, ids, header=None):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
executable = Path(temp_dir) / "target_names"
|
|
# Keep production SHA-256 and tinf implementations. Instrument
|
|
# only their call boundaries to assert candidate filtering,
|
|
# one decode per chunk, one bounded allocation and cleanup.
|
|
header = self.text if header is None else header
|
|
rows = [(int(target, 16), name) for target, name in re.findall(
|
|
r'\{ 0x([0-9a-fA-F]{8}), "([^"]+)" \},', header)]
|
|
fingerprints = Counter(target & 255 for target, _ in rows)
|
|
instrumented = header.replace("mesh::Utils::sha256(", "target_name_sha256(")
|
|
instrumented = instrumented.replace("tinf_uncompress_exact(", "target_name_uncompress_exact(")
|
|
instrumented = instrumented.replace("malloc(DECODE_CHUNK_BYTES)", "target_name_malloc(DECODE_CHUNK_BYTES)")
|
|
instrumented = instrumented.replace("free(bytes)", "target_name_free(bytes)")
|
|
alternate = Path(temp_dir) / "helpers/ota"
|
|
alternate.mkdir(parents=True)
|
|
(alternate / "OtaTargets.h").write_text(instrumented, encoding="ascii")
|
|
(alternate / "tinf").mkdir()
|
|
(alternate / "tinf/tinf.h").write_text(
|
|
(ROOT / "src/helpers/ota/tinf/tinf.h").read_text(), encoding="utf-8")
|
|
candidates = [Path(os.environ["MESHCORE_CRYPTO_DIR"])] if os.environ.get("MESHCORE_CRYPTO_DIR") else []
|
|
candidates += sorted((ROOT / ".pio/libdeps").glob("*/Crypto"))
|
|
crypto = next((path for path in candidates if (path / "SHA256.cpp").is_file()), None)
|
|
self.assertIsNotNone(crypto, "Install rweather/Crypto or set MESHCORE_CRYPTO_DIR (no hash mock fallback)")
|
|
decoder = Path(temp_dir) / "tinf.o"
|
|
built = subprocess.run([
|
|
os.environ.get("CC") or shutil.which("cc") or "gcc",
|
|
"-Os", *SANITIZER_FLAGS, "-DMESHCORE_TINF_IMPLEMENTATION=1", "-c",
|
|
str(ROOT / "src/helpers/ota/tinf/tinflate.c"), "-o", str(decoder),
|
|
], capture_output=True, text=True, timeout=60)
|
|
self.assertEqual(built.returncode, 0, built.stdout + built.stderr)
|
|
built = subprocess.run([
|
|
os.environ.get("CXX") or shutil.which("g++") or "c++",
|
|
"-std=c++17", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-parameter", "-DHOST_BUILD",
|
|
"-ffunction-sections", "-fdata-sections", "-Wl,--gc-sections", *SANITIZER_FLAGS,
|
|
*flags, f"-DOTA_TEST_TARGET_COUNT={len(rows)}", f"-I{temp_dir}", f"-I{ROOT / 'src'}", f"-I{crypto}",
|
|
"-isystem", str(ROOT / "test/mocks"),
|
|
str(HARNESS), str(ROOT / "src/Utils.cpp"),
|
|
*[str(crypto / name) for name in ("SHA256.cpp", "Hash.cpp", "Crypto.cpp",
|
|
"AES128.cpp", "AESCommon.cpp", "BlockCipher.cpp")],
|
|
str(decoder), "-o", str(executable),
|
|
], capture_output=True, text=True, timeout=60)
|
|
self.assertEqual(built.returncode, 0, built.stdout + built.stderr)
|
|
result = subprocess.run(
|
|
[str(executable)], input="\n".join(f"{target:08x} {fingerprints[target & 255]}" for target in ids),
|
|
text=True, capture_output=True, timeout=30)
|
|
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
|
return result.stdout.splitlines()
|
|
|
|
def test_generator_reproduces_checked_in_header_and_all_ids(self):
|
|
self.assertGreater(len(self.rows), 600)
|
|
self.assertEqual(self.rows, gen_targets.target_rows(name for _, name in self.rows))
|
|
self.assertEqual(self.text, gen_targets.generate_header(self.rows))
|
|
|
|
def test_zopfli_chunks_retain_every_name_and_have_bounded_output(self):
|
|
data, offsets, raw_lengths, firsts = gen_targets.compressed_name_chunks(self.rows)
|
|
actual = []
|
|
for group, raw_length in enumerate(raw_lengths):
|
|
raw = zlib.decompress(data[offsets[group]:offsets[group + 1]], -15)
|
|
self.assertEqual(len(raw), raw_length)
|
|
self.assertLessEqual(raw_length, 512)
|
|
position = 0
|
|
previous = b""
|
|
for row in range(firsts[group], firsts[group + 1]):
|
|
prefix = raw[position]
|
|
self.assertLessEqual(prefix, len(previous))
|
|
end = raw.index(0, position + 1)
|
|
previous = previous[:prefix] + raw[position + 1:end]
|
|
position = end + 1
|
|
actual.append(previous.decode("utf-8"))
|
|
self.assertEqual(position, len(raw))
|
|
self.assertEqual(actual, [name for _, name in self.rows])
|
|
original_size = sum(8 + len(name.encode("utf-8")) + 1 for _, name in self.rows)
|
|
compressed_size = 4 * len(self.rows) + 2 * (len(offsets) + len(raw_lengths) + len(firsts)) + len(data)
|
|
self.assertGreater(original_size - compressed_size, 19000)
|
|
|
|
def test_generator_requires_google_zopfli_1000(self):
|
|
from unittest.mock import patch
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
import zopfli_compress
|
|
gen_targets._compressed_name_chunks.cache_clear()
|
|
original = zopfli_compress.raw_deflate_compress
|
|
with patch.object(zopfli_compress, "raw_deflate_compress", wraps=original) as compress:
|
|
gen_targets.compressed_name_chunks([(1, "unique_test_target")])
|
|
self.assertEqual(compress.call_count, 1)
|
|
self.assertEqual(compress.call_args.kwargs, {"numiterations": 1000})
|
|
|
|
def test_cpp_standard_and_compact_match_all_names(self):
|
|
ids = [target for target, _ in self.rows] + [0, 0xffffffff]
|
|
expected = [name for _, name in self.rows] + ["?", "?"]
|
|
for flags in ([], ["-DOTA_TARGET_NAME_FRONT_CODED=1"],
|
|
["-DOTA_TARGET_NAME_FRONT_CODED=1", "-DOTA_RAK15002_SD_STORE=1", "-DCOMPANION_RADIO_FULL=1"]):
|
|
with self.subTest(flags=flags):
|
|
self.assertEqual(self.compile_and_lookup(flags, ids), expected)
|
|
|
|
def test_rak_full_lookup_fails_closed_without_heap_and_keeps_every_name(self):
|
|
flags = ["-DOTA_TARGET_NAME_FRONT_CODED=1", "-DOTA_RAK15002_SD_STORE=1", "-DCOMPANION_RADIO_FULL=1"]
|
|
data, offsets, lengths, firsts = gen_targets.grouped_name_chunks(
|
|
self.rows, gen_targets.RAK_FULL_NAME_DECODE_CHUNK_SIZE)
|
|
ordinary = gen_targets.compressed_name_chunks(self.rows)
|
|
self.assertGreater(len(ordinary[0]) - len(data), 1500)
|
|
self.assertLessEqual(max(lengths), 4096)
|
|
self.assertEqual(firsts[-1], len(self.rows))
|
|
self.assertEqual(offsets[-1], len(data))
|
|
self.assertEqual(self.compile_and_lookup(flags + ["-DOTA_TEST_MALLOC_FAIL=1"],
|
|
[self.rows[0][0], 0]), ["?", "?"])
|
|
|
|
def grouped_roundtrip(self, rows, chunk_size=4096):
|
|
data, offsets, lengths, firsts = gen_targets.grouped_name_chunks(rows, chunk_size)
|
|
names = []
|
|
for group, length in enumerate(lengths):
|
|
raw = zlib.decompress(data[offsets[group]:offsets[group + 1]], -15)
|
|
self.assertEqual(len(raw), length)
|
|
self.assertLessEqual(length, chunk_size)
|
|
count, dictionary_bytes = raw[0], int.from_bytes(raw[1:3], "little")
|
|
dictionary_end = 3 + dictionary_bytes
|
|
dictionary = raw[3:dictionary_end].split(b"\0")
|
|
self.assertEqual(dictionary[-1], b"")
|
|
self.assertEqual(len(dictionary) - 1, count)
|
|
position, previous, row = dictionary_end, b"", firsts[group]
|
|
while row < firsts[group + 1]:
|
|
prefix = raw[position]
|
|
self.assertLessEqual(prefix, len(previous))
|
|
end = raw.index(0, position + 1)
|
|
previous = previous[:prefix] + raw[position + 1:end]
|
|
aliases = raw[end + 1]
|
|
self.assertGreater(aliases, 0)
|
|
position = end + 2
|
|
for index in raw[position:position + aliases]:
|
|
self.assertLess(index, count)
|
|
names.append((previous + dictionary[index]).decode("utf-8"))
|
|
position += aliases; row += aliases
|
|
self.assertEqual(row, firsts[group + 1])
|
|
self.assertEqual(position, len(raw))
|
|
self.assertEqual(names, [name for _, name in rows])
|
|
return data, lengths
|
|
|
|
def test_rak_full_grouped_roles_keep_all_names_in_same_scratch_and_save_flash(self):
|
|
data, lengths = self.grouped_roundtrip(self.rows)
|
|
previous = gen_targets.compressed_name_chunks(self.rows, 4096)
|
|
self.assertGreater(len(previous[0]) - len(data), 1000)
|
|
self.assertLessEqual(max(lengths), 4096)
|
|
# Unusual/legacy names stay complete; larger future inventories restart
|
|
# board prefixes and carry a complete suffix dictionary in each chunk.
|
|
future = gen_targets.target_rows(
|
|
[f"board_{i:04d}_{role}" for i in range(700)
|
|
for role in ("repeater", "companion_radio_usb", "room_server")]
|
|
+ ["legacy_without_role", "odd_BOARD_repeatr", "case_Companion_ble"])
|
|
_, chunks = self.grouped_roundtrip(future)
|
|
self.assertGreater(len(chunks), 1)
|
|
|
|
def test_grouped_decoder_rejects_bad_dictionary_prefix_count_and_role_index(self):
|
|
from zopfli_compress import raw_deflate_compress
|
|
rows = gen_targets.target_rows(["hello"])
|
|
header = gen_targets.generate_header(rows)
|
|
data, offsets, lengths, firsts = gen_targets.grouped_name_chunks(rows)
|
|
raw = zlib.decompress(data, -15)
|
|
invalid = []
|
|
for offset, value in ((0, 0), (1, 255), (3, 88), (4, 255), (-2, 0), (-1, 255)):
|
|
changed = bytearray(raw); changed[offset] = value; invalid.append(bytes(changed))
|
|
invalid.append(raw[:-1])
|
|
for changed in invalid:
|
|
stream = raw_deflate_compress(changed, numiterations=1000)
|
|
altered = re.sub(
|
|
r"(static const uint8_t NAMES\[\] = \{).*?(\n \};)",
|
|
lambda match: match[1] + "\n " + ", ".join(f"0x{byte:02x}" for byte in stream) + "," + match[2],
|
|
header, count=1, flags=re.S)
|
|
altered = re.sub(
|
|
r"(static const uint16_t OFFSETS\[\] = \{).*?(\n \};)",
|
|
lambda match: match[1] + f"\n 0, {len(stream)}," + match[2], altered, count=1, flags=re.S)
|
|
altered = re.sub(
|
|
r"(static const uint16_t RAW_LENGTHS\[\] = \{).*?(\n \};)",
|
|
lambda match: match[1] + f"\n {len(changed)}," + match[2], altered, count=1, flags=re.S)
|
|
with self.subTest(raw=changed.hex()):
|
|
flags = ["-DOTA_TARGET_NAME_FRONT_CODED=1", "-DOTA_RAK15002_SD_STORE=1", "-DCOMPANION_RADIO_FULL=1"]
|
|
self.assertEqual(self.compile_and_lookup(flags, [rows[0][0]], altered), ["?"])
|
|
|
|
def test_rak_full_fingerprint_collisions_verify_the_entire_target_id(self):
|
|
flags = ["-DOTA_TARGET_NAME_FRONT_CODED=1", "-DOTA_RAK15002_SD_STORE=1", "-DCOMPANION_RADIO_FULL=1"]
|
|
known = {target for target, _ in self.rows}
|
|
buckets = Counter(target & 255 for target in known)
|
|
self.assertGreater(max(buckets.values()), 1)
|
|
# Preserve every real target's low-byte fingerprint while changing
|
|
# another ID byte. No filtered candidate may be accepted by itself.
|
|
unknown = {target ^ 0x100 for target in known} - known
|
|
# Also cover every bucket, including buckets containing no targets.
|
|
unknown.update(0xa5a50000 | fingerprint for fingerprint in range(256))
|
|
unknown.difference_update(known)
|
|
self.assertGreater(len(unknown), len(self.rows))
|
|
self.assertEqual(self.compile_and_lookup(flags, sorted(unknown)), ["?"] * len(unknown))
|
|
|
|
def test_rak_full_index_keeps_one_byte_per_target_without_changing_names(self):
|
|
encoded = re.search(r"static const uint8_t FINGERPRINTS\[\] = \{(.*?)\n \};", self.text, re.S)
|
|
self.assertIsNotNone(encoded)
|
|
fingerprints = [int(value, 16) for value in re.findall(r"0x([0-9a-f]{2})", encoded[1])]
|
|
self.assertEqual(fingerprints, [target & 255 for target, _ in self.rows])
|
|
self.assertGreater(4 * len(self.rows) - len(fingerprints), 2000)
|
|
|
|
def test_table_disabled_preserves_optional_local_name(self):
|
|
local_id, local_name = self.rows[0]
|
|
ids = [local_id, self.rows[1][0], 0]
|
|
for compressed in (0, 1):
|
|
flags = ["-DOTA_TARGET_NAME_TABLE=0", f"-DOTA_TARGET_NAME_FRONT_CODED={compressed}"]
|
|
with self.subTest(compressed=compressed):
|
|
self.assertEqual(self.compile_and_lookup(flags, ids), ["?", "?", "?"])
|
|
self.assertEqual(self.compile_and_lookup(flags + [
|
|
f"-DOTA_LOCAL_TARGET_ID=0x{local_id:08x}",
|
|
f'-DOTA_LOCAL_TARGET_NAME="{local_name}"',
|
|
], ids), [local_name, "?", "?"])
|
|
|
|
def test_reduced_rak_policy_compiles_with_exact_local_names_and_ids(self):
|
|
spec = importlib.util.spec_from_file_location("ota_name_trim", ROOT / "scripts/nrf52_flash_trim.py")
|
|
trim = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(trim)
|
|
for board, suffix in (("3401", ""), ("4631", ""),
|
|
("4631", "_bridge_rs232_serial1"), ("4631", "_bridge_rs232_serial2")):
|
|
target = f"RAK_{board}_repeater{suffix}_lora_ota_no_external_sensors"
|
|
config = configparser.ConfigParser(interpolation=None, inline_comment_prefixes=(";",))
|
|
config.read(ROOT / f"variants/rak{board}/platformio.ini")
|
|
flags = config["env:" + target]["build_flags"]
|
|
# Interpolated hardware flags do not influence the name policy.
|
|
flags = re.sub(r"\$\{[^}]+\}", "", flags)
|
|
target_id = gen_targets.ml.target_id_for_env(target)
|
|
definitions = trim.flag_defines(flags)
|
|
definitions.update(trim.trim_policy("nordicnrf52", "nrf52840", target,
|
|
"+<../examples/simple_repeater>", flags))
|
|
name_flags = [f"-D{name}={value}" for name, value in definitions.items()
|
|
if name in ("OTA_TARGET_NAME_TABLE", "OTA_TARGET_NAME_FRONT_CODED",
|
|
"OTA_LOCAL_TARGET_ID", "OTA_LOCAL_TARGET_NAME")]
|
|
with self.subTest(target=target):
|
|
self.assertEqual(self.compile_and_lookup(name_flags,
|
|
[target_id, self.rows[0][0], 0]), [target, "?", "?"])
|
|
|
|
def test_cpp_decoder_fails_closed_on_invalid_records_and_trailing_bytes(self):
|
|
from zopfli_compress import raw_deflate_compress
|
|
rows = gen_targets.target_rows(["hello"])
|
|
target = rows[0][0]
|
|
header = gen_targets.generate_header(rows)
|
|
original, _, _, _ = gen_targets.compressed_name_chunks(rows)
|
|
invalid = [
|
|
raw_deflate_compress(b"\xffhello\0", numiterations=1000), # invalid restart prefix
|
|
raw_deflate_compress(b"\0hello!", numiterations=1000), # missing terminator
|
|
original + b"\0", # trailing whole byte
|
|
original[:-1], # truncated input
|
|
]
|
|
for stream in invalid:
|
|
altered = re.sub(
|
|
r"(static const uint8_t NAMES\[\] = \{).*?(\n \};)",
|
|
lambda match: match[1] + "\n " + ", ".join(f"0x{byte:02x}" for byte in stream) + "," + match[2],
|
|
header, flags=re.S)
|
|
altered = re.sub(
|
|
r"(static const uint16_t OFFSETS\[\] = \{).*?(\n \};)",
|
|
lambda match: match[1] + f"\n 0, {len(stream)}," + match[2],
|
|
altered, flags=re.S)
|
|
with self.subTest(stream=stream.hex()):
|
|
for flags in (["-DOTA_TARGET_NAME_FRONT_CODED=1"],
|
|
["-DOTA_TARGET_NAME_FRONT_CODED=1", "-DOTA_RAK15002_SD_STORE=1", "-DCOMPANION_RADIO_FULL=1"]):
|
|
self.assertEqual(self.compile_and_lookup(flags, [target], altered), ["?"])
|
|
|
|
def test_generator_rejects_unencodable_names(self):
|
|
for name in ("x\0y", "x" * 256):
|
|
with self.subTest(name=name):
|
|
with self.assertRaises(ValueError):
|
|
gen_targets.compressed_name_chunks([(1, name)])
|
|
with self.assertRaises(ValueError):
|
|
gen_targets.grouped_name_chunks([(1, name)])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|