fix: keep feature-complete release builds within flash limits

This commit is contained in:
mikecarper committed 2026-10-20 11:19:27 -07:00
1 parent bc1176c7f4
commit 805dd2cbdf
17 files changed
+735 -300

No files matched your search

+119 -20
View File
@@ -15,6 +15,9 @@ from __future__ import annotations
import json
import os
import re
from collections import Counter
from itertools import groupby
import subprocess
import sys
from functools import lru_cache
@@ -126,11 +129,76 @@ def _compressed_name_chunks(rows, max_chunk_size):
return bytes(data), tuple(offsets), tuple(len(raw) for raw in raw_chunks), tuple(firsts)
def grouped_name_chunks(rows, max_chunk_size=RAK_FULL_NAME_DECODE_CHUNK_SIZE):
"""Share complete role suffixes and contiguous board prefixes, retaining order.
Each independent chunk contains its own bounded suffix dictionary followed
by board groups. One record reconstructs a board base, then lists every role
index for that contiguous run. IDs/names remain byte-for-byte unchanged.
"""
return _grouped_name_chunks(tuple(rows), max_chunk_size)
@lru_cache(maxsize=8)
def _grouped_name_chunks(rows, max_chunk_size):
sys.path.insert(0, str(ROOT / "scripts"))
from zopfli_compress import MAXIMUM_ITERATIONS, raw_deflate_compress
if not rows or max_chunk_size < 1024 or max_chunk_size > 65535:
raise ValueError("OTA grouped-name chunk size or inventory is invalid")
parts = []
for _, name in rows:
encoded = name.encode("utf-8")
if b"\0" in encoded or len(encoded) > 255:
raise ValueError(f"OTA target name is not encodable: {name!r}")
role = re.search(r"_(?:companion(?:_radio)?|comp_radio|repeater|repeatr|room_server|room_svr|sensor|kiss_modem|terminal_chat|term_chat|partition_expander)(?:_|$)", name, re.I)
parts.append((name[:role.start()], name[role.start():]) if role else (name, ""))
frequency = Counter(suffix for _, suffix in parts if suffix)
suffixes = sorted(frequency)
# Leave at least half the same scratch buffer for records. Rare/future
# suffixes can stay in the board base without losing or renaming a target.
while len(suffixes) > 254 or 1 + sum(len(s.encode("utf-8")) + 1 for s in suffixes) > max_chunk_size // 2:
suffixes.remove(min(suffixes, key=lambda suffix: (frequency[suffix], -len(suffix), suffix)))
suffixes.insert(0, "")
indices = {suffix: index for index, suffix in enumerate(suffixes)}
dictionary = b"".join(suffix.encode("utf-8") + b"\0" for suffix in suffixes)
header = bytes([len(suffixes)]) + len(dictionary).to_bytes(2, "little") + dictionary
chosen = [(base, suffix) if suffix in indices else (name, "")
for (_, name), (base, suffix) in zip(rows, parts)]
records = []
for base, run in groupby(chosen, key=lambda part: part[0]):
roles = [indices[suffix] for _, suffix in run]
for start in range(0, len(roles), 255):
records.append((base.encode("utf-8"), bytes(roles[start:start + 255])))
raw_chunks, firsts, row = [], [0], 0
current, previous = bytearray(header), b""
for base, roles in records:
prefix = 0
while prefix < min(len(previous), len(base)) and previous[prefix] == base[prefix]:
prefix += 1
record = bytes([prefix]) + base[prefix:] + b"\0" + bytes([len(roles)]) + roles
if len(current) + len(record) > max_chunk_size:
raw_chunks.append(bytes(current)); firsts.append(row)
current, previous = bytearray(header), b""
record = b"\0" + base + b"\0" + bytes([len(roles)]) + roles
if len(header) + len(record) > max_chunk_size:
raise ValueError("OTA board-name group exceeds its bounded chunk")
current.extend(record); previous = base; row += len(roles)
raw_chunks.append(bytes(current)); firsts.append(len(rows))
data, offsets = bytearray(), [0]
for raw in raw_chunks:
data.extend(raw_deflate_compress(raw, numiterations=MAXIMUM_ITERATIONS)); offsets.append(len(data))
if len(data) > 65535 or len(rows) > 65535:
raise ValueError("OTA grouped-name data exceeds uint16_t restart offsets")
return bytes(data), tuple(offsets), tuple(map(len, raw_chunks)), tuple(firsts)
def generate_header(rows):
capacity = max((len(name.encode("utf-8")) + 1 for _, name in rows), default=1)
def chunk_tables(chunk_size):
data, offsets, raw_lengths, firsts = compressed_name_chunks(rows, chunk_size)
def chunk_tables(chunk_size, grouped=False):
codec = grouped_name_chunks if grouped else compressed_name_chunks
data, offsets, raw_lengths, firsts = codec(rows, chunk_size)
lines = [f" enum {{ DECODE_CHUNK_BYTES = {chunk_size} }};"]
for name, values in (("OFFSETS", offsets), ("RAW_LENGTHS", raw_lengths), ("FIRSTS", firsts)):
lines += [f" static const uint16_t {name}[] = {{"]
@@ -188,7 +256,7 @@ def generate_header(rows):
]
out += [" " + ", ".join(f"0x{tid & 0xff:02x}" for tid, _ in rows[i:i + 16]) + ","
for i in range(0, len(rows), 16)]
out += [" };"] + chunk_tables(RAK_FULL_NAME_DECODE_CHUNK_SIZE)
out += [" };"] + chunk_tables(RAK_FULL_NAME_DECODE_CHUNK_SIZE, grouped=True)
out += [
" // Rare display lookup: one heap buffer, each candidate chunk decoded once.",
" // Keep complete names private until both ID and caller capacity are validated.",
@@ -206,31 +274,62 @@ def generate_header(rows):
" if (tinf_uncompress_exact(raw, &produced, NAMES + OFFSETS[group],",
" OFFSETS[group + 1] - OFFSETS[group]) != TINF_OK ||",
" produced != RAW_LENGTHS[group]) return nullptr;",
" const uint8_t* record = raw;",
" if (produced < 3) return nullptr;",
" const unsigned suffix_count = raw[0];",
" const size_t suffix_bytes = (size_t)raw[1] | (size_t)raw[2] << 8;",
" if (!suffix_count || suffix_bytes > produced - 3) return nullptr;",
" const uint8_t* suffixes = raw + 3;",
" const uint8_t* record = suffixes + suffix_bytes;",
" const uint8_t* end = raw + produced;",
" const uint8_t* checked_suffix = suffixes;",
" for (unsigned n = 0; n < suffix_count; n++) {",
" while (checked_suffix < record && *checked_suffix) checked_suffix++;",
" if (checked_suffix == record) return nullptr;",
" checked_suffix++;",
" }",
" if (checked_suffix != record) return nullptr;",
" size_t previous_length = 0;",
" for (unsigned row = FIRSTS[group]; row < FIRSTS[group + 1]; row++) {",
" unsigned row = FIRSTS[group];",
" while (row < FIRSTS[group + 1]) {",
" if (record == end) return nullptr;",
" size_t length = *record++;",
" if (length > previous_length || length >= sizeof(name)) return nullptr;",
" size_t base_length = *record++;",
" if (base_length > previous_length || base_length >= sizeof(name)) return nullptr;",
" while (record < end && *record) {",
" if (length + 1 >= sizeof(name)) return nullptr;",
" name[length++] = (char)*record++;",
" if (base_length + 1 >= sizeof(name)) return nullptr;",
" name[base_length++] = (char)*record++;",
" }",
" if (record == end) return nullptr;",
" record++;",
" name[length] = 0;",
" previous_length = length;",
" if (FINGERPRINTS[row] != fingerprint) continue;",
" uint8_t hash[4];",
" mesh::Utils::sha256(hash, sizeof(hash), (const uint8_t*)name, length);",
" const uint32_t id = (uint32_t)hash[0] | (uint32_t)hash[1] << 8 |",
" (uint32_t)hash[2] << 16 | (uint32_t)hash[3] << 24;",
" if (id != target_id) continue;",
" if (length >= capacity) return nullptr;",
" memcpy(buffer, name, length + 1);",
" return buffer;",
" previous_length = base_length;",
" if (record == end) return nullptr;",
" const unsigned aliases = *record++;",
" if (!aliases || aliases > FIRSTS[group + 1] - row || aliases > (size_t)(end - record)) return nullptr;",
" for (unsigned alias = 0; alias < aliases; alias++, row++) {",
" const unsigned suffix_index = *record++;",
" if (suffix_index >= suffix_count) return nullptr;",
" if (FINGERPRINTS[row] != fingerprint) continue;",
" const uint8_t* suffix = suffixes;",
" for (unsigned n = 0; n < suffix_index; n++) {",
" while (*suffix) suffix++;",
" suffix++;",
" }",
" size_t length = base_length;",
" while (*suffix) {",
" if (length + 1 >= sizeof(name)) return nullptr;",
" name[length++] = (char)*suffix++;",
" }",
" name[length] = 0;",
" uint8_t hash[4];",
" mesh::Utils::sha256(hash, sizeof(hash), (const uint8_t*)name, length);",
" const uint32_t id = (uint32_t)hash[0] | (uint32_t)hash[1] << 8 |",
" (uint32_t)hash[2] << 16 | (uint32_t)hash[3] << 24;",
" if (id != target_id) continue;",
" if (length >= capacity) return nullptr;",
" memcpy(buffer, name, length + 1);",
" return buffer;",
" }",
" }",
" if (record != end) return nullptr;",
" }",
" return nullptr;",
"#else",