"""All OTA names survive front coding; original Companion API stays unchanged.""" import re import os import configparser import importlib.util import shutil import subprocess import sys import tempfile import unittest import zlib from collections import Counter from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path[:0] = [str(ROOT / "tools" / "mota"), str(ROOT / "scripts")] import gen_targets from test_serial_wifi_sessions import SANITIZER_FLAGS HEADER = ROOT / "src" / "helpers" / "ota" / "OtaTargets.h" HARNESS = ROOT / "test" / "fixtures" / "ota_target_names" / "test.cpp" class OtaTargetNameTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.text = HEADER.read_text(encoding="utf-8") cls.rows = [(int(target, 16), name) for target, name in re.findall( r'\{ 0x([0-9a-fA-F]{8}), "([^"]+)" \},', cls.text)] def compile_and_lookup(self, flags, ids, header=None): with tempfile.TemporaryDirectory() as temp_dir: executable = Path(temp_dir) / "target_names" # Keep production SHA-256 and tinf implementations. Instrument # only their call boundaries to assert candidate filtering, # one decode per chunk, one bounded allocation and cleanup. header = self.text if header is None else header rows = [(int(target, 16), name) for target, name in re.findall( r'\{ 0x([0-9a-fA-F]{8}), "([^"]+)" \},', header)] fingerprints = Counter(target & 255 for target, _ in rows) instrumented = header.replace("mesh::Utils::sha256(", "target_name_sha256(") instrumented = instrumented.replace("tinf_uncompress_exact(", "target_name_uncompress_exact(") instrumented = instrumented.replace("malloc(DECODE_CHUNK_BYTES)", "target_name_malloc(DECODE_CHUNK_BYTES)") instrumented = instrumented.replace("free(bytes)", "target_name_free(bytes)") alternate = Path(temp_dir) / "helpers/ota" alternate.mkdir(parents=True) (alternate / "OtaTargets.h").write_text(instrumented, encoding="ascii") (alternate / "tinf").mkdir() (alternate / "tinf/tinf.h").write_text( (ROOT / "src/helpers/ota/tinf/tinf.h").read_text(), encoding="utf-8") candidates = [Path(os.environ["MESHCORE_CRYPTO_DIR"])] if os.environ.get("MESHCORE_CRYPTO_DIR") else [] candidates += sorted((ROOT / ".pio/libdeps").glob("*/Crypto")) crypto = next((path for path in candidates if (path / "SHA256.cpp").is_file()), None) self.assertIsNotNone(crypto, "Install rweather/Crypto or set MESHCORE_CRYPTO_DIR (no hash mock fallback)") decoder = Path(temp_dir) / "tinf.o" built = subprocess.run([ os.environ.get("CC") or shutil.which("cc") or "gcc", "-Os", *SANITIZER_FLAGS, "-DMESHCORE_TINF_IMPLEMENTATION=1", "-c", str(ROOT / "src/helpers/ota/tinf/tinflate.c"), "-o", str(decoder), ], capture_output=True, text=True, timeout=60) self.assertEqual(built.returncode, 0, built.stdout + built.stderr) built = subprocess.run([ os.environ.get("CXX") or shutil.which("g++") or "c++", "-std=c++17", "-O1", "-Wall", "-Wextra", "-Werror", "-Wno-unused-parameter", "-DHOST_BUILD", "-ffunction-sections", "-fdata-sections", "-Wl,--gc-sections", *SANITIZER_FLAGS, *flags, f"-DOTA_TEST_TARGET_COUNT={len(rows)}", f"-I{temp_dir}", f"-I{ROOT / 'src'}", f"-I{crypto}", "-isystem", str(ROOT / "test/mocks"), str(HARNESS), str(ROOT / "src/Utils.cpp"), *[str(crypto / name) for name in ("SHA256.cpp", "Hash.cpp", "Crypto.cpp", "AES128.cpp", "AESCommon.cpp", "BlockCipher.cpp")], str(decoder), "-o", str(executable), ], capture_output=True, text=True, timeout=60) self.assertEqual(built.returncode, 0, built.stdout + built.stderr) result = subprocess.run( [str(executable)], input="\n".join(f"{target:08x} {fingerprints[target & 255]}" for target in ids), text=True, capture_output=True, timeout=30) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) return result.stdout.splitlines() def test_generator_reproduces_checked_in_header_and_all_ids(self): self.assertGreater(len(self.rows), 600) self.assertEqual(self.rows, gen_targets.target_rows(name for _, name in self.rows)) self.assertEqual(self.text, gen_targets.generate_header(self.rows)) def test_zopfli_chunks_retain_every_name_and_have_bounded_output(self): data, offsets, raw_lengths, firsts = gen_targets.compressed_name_chunks(self.rows) actual = [] for group, raw_length in enumerate(raw_lengths): raw = zlib.decompress(data[offsets[group]:offsets[group + 1]], -15) self.assertEqual(len(raw), raw_length) self.assertLessEqual(raw_length, 512) position = 0 previous = b"" for row in range(firsts[group], firsts[group + 1]): prefix = raw[position] self.assertLessEqual(prefix, len(previous)) end = raw.index(0, position + 1) previous = previous[:prefix] + raw[position + 1:end] position = end + 1 actual.append(previous.decode("utf-8")) self.assertEqual(position, len(raw)) self.assertEqual(actual, [name for _, name in self.rows]) original_size = sum(8 + len(name.encode("utf-8")) + 1 for _, name in self.rows) compressed_size = 4 * len(self.rows) + 2 * (len(offsets) + len(raw_lengths) + len(firsts)) + len(data) self.assertGreater(original_size - compressed_size, 19000) def test_generator_requires_google_zopfli_1000(self): from unittest.mock import patch sys.path.insert(0, str(ROOT / "scripts")) import zopfli_compress gen_targets._compressed_name_chunks.cache_clear() original = zopfli_compress.raw_deflate_compress with patch.object(zopfli_compress, "raw_deflate_compress", wraps=original) as compress: gen_targets.compressed_name_chunks([(1, "unique_test_target")]) self.assertEqual(compress.call_count, 1) self.assertEqual(compress.call_args.kwargs, {"numiterations": 1000}) def test_cpp_standard_and_compact_match_all_names(self): ids = [target for target, _ in self.rows] + [0, 0xffffffff] expected = [name for _, name in self.rows] + ["?", "?"] for flags in ([], ["-DOTA_TARGET_NAME_FRONT_CODED=1"], ["-DOTA_TARGET_NAME_FRONT_CODED=1", "-DOTA_RAK15002_SD_STORE=1", "-DCOMPANION_RADIO_FULL=1"]): with self.subTest(flags=flags): self.assertEqual(self.compile_and_lookup(flags, ids), expected) def test_rak_full_lookup_fails_closed_without_heap_and_keeps_every_name(self): flags = ["-DOTA_TARGET_NAME_FRONT_CODED=1", "-DOTA_RAK15002_SD_STORE=1", "-DCOMPANION_RADIO_FULL=1"] data, offsets, lengths, firsts = gen_targets.grouped_name_chunks( self.rows, gen_targets.RAK_FULL_NAME_DECODE_CHUNK_SIZE) ordinary = gen_targets.compressed_name_chunks(self.rows) self.assertGreater(len(ordinary[0]) - len(data), 1500) self.assertLessEqual(max(lengths), 4096) self.assertEqual(firsts[-1], len(self.rows)) self.assertEqual(offsets[-1], len(data)) self.assertEqual(self.compile_and_lookup(flags + ["-DOTA_TEST_MALLOC_FAIL=1"], [self.rows[0][0], 0]), ["?", "?"]) def grouped_roundtrip(self, rows, chunk_size=4096): data, offsets, lengths, firsts = gen_targets.grouped_name_chunks(rows, chunk_size) names = [] for group, length in enumerate(lengths): raw = zlib.decompress(data[offsets[group]:offsets[group + 1]], -15) self.assertEqual(len(raw), length) self.assertLessEqual(length, chunk_size) count, dictionary_bytes = raw[0], int.from_bytes(raw[1:3], "little") dictionary_end = 3 + dictionary_bytes dictionary = raw[3:dictionary_end].split(b"\0") self.assertEqual(dictionary[-1], b"") self.assertEqual(len(dictionary) - 1, count) position, previous, row = dictionary_end, b"", firsts[group] while row < firsts[group + 1]: prefix = raw[position] self.assertLessEqual(prefix, len(previous)) end = raw.index(0, position + 1) previous = previous[:prefix] + raw[position + 1:end] aliases = raw[end + 1] self.assertGreater(aliases, 0) position = end + 2 for index in raw[position:position + aliases]: self.assertLess(index, count) names.append((previous + dictionary[index]).decode("utf-8")) position += aliases; row += aliases self.assertEqual(row, firsts[group + 1]) self.assertEqual(position, len(raw)) self.assertEqual(names, [name for _, name in rows]) return data, lengths def test_rak_full_grouped_roles_keep_all_names_in_same_scratch_and_save_flash(self): data, lengths = self.grouped_roundtrip(self.rows) previous = gen_targets.compressed_name_chunks(self.rows, 4096) self.assertGreater(len(previous[0]) - len(data), 1000) self.assertLessEqual(max(lengths), 4096) # Unusual/legacy names stay complete; larger future inventories restart # board prefixes and carry a complete suffix dictionary in each chunk. future = gen_targets.target_rows( [f"board_{i:04d}_{role}" for i in range(700) for role in ("repeater", "companion_radio_usb", "room_server")] + ["legacy_without_role", "odd_BOARD_repeatr", "case_Companion_ble"]) _, chunks = self.grouped_roundtrip(future) self.assertGreater(len(chunks), 1) def test_grouped_decoder_rejects_bad_dictionary_prefix_count_and_role_index(self): from zopfli_compress import raw_deflate_compress rows = gen_targets.target_rows(["hello"]) header = gen_targets.generate_header(rows) data, offsets, lengths, firsts = gen_targets.grouped_name_chunks(rows) raw = zlib.decompress(data, -15) invalid = [] for offset, value in ((0, 0), (1, 255), (3, 88), (4, 255), (-2, 0), (-1, 255)): changed = bytearray(raw); changed[offset] = value; invalid.append(bytes(changed)) invalid.append(raw[:-1]) for changed in invalid: stream = raw_deflate_compress(changed, numiterations=1000) altered = re.sub( r"(static const uint8_t NAMES\[\] = \{).*?(\n \};)", lambda match: match[1] + "\n " + ", ".join(f"0x{byte:02x}" for byte in stream) + "," + match[2], header, count=1, flags=re.S) altered = re.sub( r"(static const uint16_t OFFSETS\[\] = \{).*?(\n \};)", lambda match: match[1] + f"\n 0, {len(stream)}," + match[2], altered, count=1, flags=re.S) altered = re.sub( r"(static const uint16_t RAW_LENGTHS\[\] = \{).*?(\n \};)", lambda match: match[1] + f"\n {len(changed)}," + match[2], altered, count=1, flags=re.S) with self.subTest(raw=changed.hex()): flags = ["-DOTA_TARGET_NAME_FRONT_CODED=1", "-DOTA_RAK15002_SD_STORE=1", "-DCOMPANION_RADIO_FULL=1"] self.assertEqual(self.compile_and_lookup(flags, [rows[0][0]], altered), ["?"]) def test_rak_full_fingerprint_collisions_verify_the_entire_target_id(self): flags = ["-DOTA_TARGET_NAME_FRONT_CODED=1", "-DOTA_RAK15002_SD_STORE=1", "-DCOMPANION_RADIO_FULL=1"] known = {target for target, _ in self.rows} buckets = Counter(target & 255 for target in known) self.assertGreater(max(buckets.values()), 1) # Preserve every real target's low-byte fingerprint while changing # another ID byte. No filtered candidate may be accepted by itself. unknown = {target ^ 0x100 for target in known} - known # Also cover every bucket, including buckets containing no targets. unknown.update(0xa5a50000 | fingerprint for fingerprint in range(256)) unknown.difference_update(known) self.assertGreater(len(unknown), len(self.rows)) self.assertEqual(self.compile_and_lookup(flags, sorted(unknown)), ["?"] * len(unknown)) def test_rak_full_index_keeps_one_byte_per_target_without_changing_names(self): encoded = re.search(r"static const uint8_t FINGERPRINTS\[\] = \{(.*?)\n \};", self.text, re.S) self.assertIsNotNone(encoded) fingerprints = [int(value, 16) for value in re.findall(r"0x([0-9a-f]{2})", encoded[1])] self.assertEqual(fingerprints, [target & 255 for target, _ in self.rows]) self.assertGreater(4 * len(self.rows) - len(fingerprints), 2000) def test_table_disabled_preserves_optional_local_name(self): local_id, local_name = self.rows[0] ids = [local_id, self.rows[1][0], 0] for compressed in (0, 1): flags = ["-DOTA_TARGET_NAME_TABLE=0", f"-DOTA_TARGET_NAME_FRONT_CODED={compressed}"] with self.subTest(compressed=compressed): self.assertEqual(self.compile_and_lookup(flags, ids), ["?", "?", "?"]) self.assertEqual(self.compile_and_lookup(flags + [ f"-DOTA_LOCAL_TARGET_ID=0x{local_id:08x}", f'-DOTA_LOCAL_TARGET_NAME="{local_name}"', ], ids), [local_name, "?", "?"]) def test_reduced_rak_policy_compiles_with_exact_local_names_and_ids(self): spec = importlib.util.spec_from_file_location("ota_name_trim", ROOT / "scripts/nrf52_flash_trim.py") trim = importlib.util.module_from_spec(spec) spec.loader.exec_module(trim) for board, suffix in (("3401", ""), ("4631", ""), ("4631", "_bridge_rs232_serial1"), ("4631", "_bridge_rs232_serial2")): target = f"RAK_{board}_repeater{suffix}_lora_ota_no_external_sensors" config = configparser.ConfigParser(interpolation=None, inline_comment_prefixes=(";",)) config.read(ROOT / f"variants/rak{board}/platformio.ini") flags = config["env:" + target]["build_flags"] # Interpolated hardware flags do not influence the name policy. flags = re.sub(r"\$\{[^}]+\}", "", flags) target_id = gen_targets.ml.target_id_for_env(target) definitions = trim.flag_defines(flags) definitions.update(trim.trim_policy("nordicnrf52", "nrf52840", target, "+<../examples/simple_repeater>", flags)) name_flags = [f"-D{name}={value}" for name, value in definitions.items() if name in ("OTA_TARGET_NAME_TABLE", "OTA_TARGET_NAME_FRONT_CODED", "OTA_LOCAL_TARGET_ID", "OTA_LOCAL_TARGET_NAME")] with self.subTest(target=target): self.assertEqual(self.compile_and_lookup(name_flags, [target_id, self.rows[0][0], 0]), [target, "?", "?"]) def test_cpp_decoder_fails_closed_on_invalid_records_and_trailing_bytes(self): from zopfli_compress import raw_deflate_compress rows = gen_targets.target_rows(["hello"]) target = rows[0][0] header = gen_targets.generate_header(rows) original, _, _, _ = gen_targets.compressed_name_chunks(rows) invalid = [ raw_deflate_compress(b"\xffhello\0", numiterations=1000), # invalid restart prefix raw_deflate_compress(b"\0hello!", numiterations=1000), # missing terminator original + b"\0", # trailing whole byte original[:-1], # truncated input ] for stream in invalid: altered = re.sub( r"(static const uint8_t NAMES\[\] = \{).*?(\n \};)", lambda match: match[1] + "\n " + ", ".join(f"0x{byte:02x}" for byte in stream) + "," + match[2], header, flags=re.S) altered = re.sub( r"(static const uint16_t OFFSETS\[\] = \{).*?(\n \};)", lambda match: match[1] + f"\n 0, {len(stream)}," + match[2], altered, flags=re.S) with self.subTest(stream=stream.hex()): for flags in (["-DOTA_TARGET_NAME_FRONT_CODED=1"], ["-DOTA_TARGET_NAME_FRONT_CODED=1", "-DOTA_RAK15002_SD_STORE=1", "-DCOMPANION_RADIO_FULL=1"]): self.assertEqual(self.compile_and_lookup(flags, [target], altered), ["?"]) def test_generator_rejects_unencodable_names(self): for name in ("x\0y", "x" * 256): with self.subTest(name=name): with self.assertRaises(ValueError): gen_targets.compressed_name_chunks([(1, name)]) with self.assertRaises(ValueError): gen_targets.grouped_name_chunks([(1, name)]) if __name__ == "__main__": unittest.main()