Reduce nRF52 infrastructure flash usage without trimming sensors
Keep Companion and Terminal Chat profiles unchanged; retain full-sensor and reduced-sensor OTA builds. Add compact crypto and OTA names, qualified display asset compression, font license packaging, and scope regression tests.
This commit is contained in:
mikecarper committed
1 parent
010374d7db
commit
8a69e73226
44 files changed
+11150
-14
No files matched your search
@@ -0,0 +1,329 @@
|
||||
"""Generate opt-in, lossless Arial font storage experiments.
|
||||
|
||||
The checked-in source font remains authoritative. Every one of its 224 slots,
|
||||
including blank slots and their advance widths, survives these experiments.
|
||||
Mode 2 uses per-glyph byte RLE; mode 3 independently inflates <=512-byte glyph
|
||||
groups. Whole-font compression is measured only, not instantiated on the device.
|
||||
Google Zopfli is used at 1,000 iterations; no compressor runs on the device.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from zopfli_compress import MAXIMUM_ITERATIONS, raw_deflate_compress
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SOURCE = ROOT / "src/helpers/ui/OLEDDisplayFonts.cpp"
|
||||
OUTPUT = ROOT / "src/helpers/ui/CompactArialFonts.h"
|
||||
CHUNK_LIMIT = 512
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Glyph:
|
||||
width: int
|
||||
bitmap: bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Font:
|
||||
name: str
|
||||
original: bytes
|
||||
glyphs: tuple[Glyph, ...]
|
||||
|
||||
@property
|
||||
def header(self) -> bytes:
|
||||
return self.original[:4]
|
||||
|
||||
|
||||
def read_fonts(path: Path = SOURCE) -> tuple[Font, ...]:
|
||||
source = path.read_text(encoding="utf-8")
|
||||
fonts = []
|
||||
for name, body in re.findall(
|
||||
r"const uint8_t (ArialMT_Plain_\d+)\[\] PROGMEM = \{(.*?)\};", source, re.S
|
||||
):
|
||||
body = re.sub(r"//[^\n]*", "", body)
|
||||
data = bytes(int(value, 16) for value in re.findall(r"0x([0-9a-fA-F]+)", body))
|
||||
if len(data) < 4:
|
||||
raise ValueError(f"{name}: incomplete header")
|
||||
base = 4 + 4 * data[3]
|
||||
if base > len(data) or data[2] + data[3] > 256:
|
||||
raise ValueError(f"{name}: invalid character table")
|
||||
glyphs = []
|
||||
consumed = 0
|
||||
for index in range(data[3]):
|
||||
entry = data[4 + 4 * index:8 + 4 * index]
|
||||
offset = int.from_bytes(entry[:2], "big")
|
||||
length, width = entry[2:]
|
||||
if offset == 0xffff:
|
||||
if length:
|
||||
raise ValueError(f"{name}: blank slot contains bitmap bytes")
|
||||
bitmap = b""
|
||||
else:
|
||||
if not length or offset != consumed or base + offset + length > len(data):
|
||||
raise ValueError(f"{name}: invalid or non-contiguous bitmap")
|
||||
bitmap = data[base + offset:base + offset + length]
|
||||
consumed += length
|
||||
glyphs.append(Glyph(width, bitmap))
|
||||
if base + consumed != len(data):
|
||||
raise ValueError(f"{name}: trailing bitmap bytes")
|
||||
fonts.append(Font(name, data, tuple(glyphs)))
|
||||
if [font.name for font in fonts] != [f"ArialMT_Plain_{size}" for size in (10, 16, 24)]:
|
||||
raise ValueError("Expected the original Arial 10, 16 and 24 fonts")
|
||||
return tuple(fonts)
|
||||
|
||||
|
||||
def rle_encode(data: bytes) -> bytes:
|
||||
"""PackBits-style literals (1..128) and repeated bytes (3..130)."""
|
||||
output = bytearray()
|
||||
position = 0
|
||||
while position < len(data):
|
||||
run = 1
|
||||
while position + run < len(data) and data[position + run] == data[position] and run < 130:
|
||||
run += 1
|
||||
if run >= 3:
|
||||
output.extend((0x80 | (run - 3), data[position]))
|
||||
position += run
|
||||
continue
|
||||
first = position
|
||||
position += run
|
||||
while position < len(data) and position - first < 128:
|
||||
run = 1
|
||||
while position + run < len(data) and data[position + run] == data[position] and run < 3:
|
||||
run += 1
|
||||
if run >= 3:
|
||||
break
|
||||
position += min(run, 128 - (position - first))
|
||||
output.append(position - first - 1)
|
||||
output.extend(data[first:position])
|
||||
return bytes(output)
|
||||
|
||||
|
||||
def rle_font(font: Font) -> tuple[bytes, tuple[tuple[int, int, int], ...]]:
|
||||
output = bytearray()
|
||||
records = []
|
||||
for glyph in font.glyphs:
|
||||
if not glyph.bitmap:
|
||||
records.append((0xffff, 0, glyph.width))
|
||||
continue
|
||||
compressed = rle_encode(glyph.bitmap)
|
||||
raw = len(compressed) >= len(glyph.bitmap)
|
||||
location = len(output) | (0x8000 if raw else 0)
|
||||
output.extend(glyph.bitmap if raw else compressed)
|
||||
records.append((location, len(glyph.bitmap), glyph.width))
|
||||
if len(output) >= 0x7fff:
|
||||
raise ValueError("RLE offsets no longer fit the 15-bit experiment format")
|
||||
return bytes(output), tuple(records)
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def chunk_font(font: Font) -> tuple[bytes, tuple[tuple[int, int, int], ...], tuple[int, ...], tuple[int, ...]]:
|
||||
chunks = []
|
||||
current = bytearray()
|
||||
records = []
|
||||
for glyph in font.glyphs:
|
||||
if not glyph.bitmap:
|
||||
records.append((0xffff, 0, glyph.width))
|
||||
continue
|
||||
if len(glyph.bitmap) > CHUNK_LIMIT:
|
||||
raise ValueError("A glyph exceeds the bounded font scratch buffer")
|
||||
if len(current) + len(glyph.bitmap) > CHUNK_LIMIT:
|
||||
chunks.append(bytes(current))
|
||||
current.clear()
|
||||
records.append(((len(chunks) << 9) | len(current), len(glyph.bitmap), glyph.width))
|
||||
current.extend(glyph.bitmap)
|
||||
if current:
|
||||
chunks.append(bytes(current))
|
||||
if len(chunks) >= 128:
|
||||
raise ValueError("Chunk identifiers exceed the packed 7-bit format")
|
||||
output = bytearray()
|
||||
offsets = [0]
|
||||
for chunk in chunks:
|
||||
output.extend(raw_deflate_compress(chunk, numiterations=MAXIMUM_ITERATIONS))
|
||||
offsets.append(len(output))
|
||||
if len(output) > 0xffff:
|
||||
raise ValueError("Compressed font offsets exceed 16 bits")
|
||||
return bytes(output), tuple(records), tuple(offsets), tuple(map(len, chunks))
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def whole_font(font: Font) -> bytes:
|
||||
return raw_deflate_compress(font.original, numiterations=MAXIMUM_ITERATIONS)
|
||||
|
||||
|
||||
def array(name: str, data, ctype: str = "uint8_t", columns: int = 16) -> str:
|
||||
values = [f"0x{value:02x}" if ctype == "uint8_t" else str(value) for value in data]
|
||||
lines = [f"static const {ctype} {name}[] = {{"]
|
||||
for first in range(0, len(values), columns):
|
||||
lines.append(" " + ", ".join(values[first:first + columns]) + ",")
|
||||
lines.append("};")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def descriptor(font: Font, rows) -> bytes:
|
||||
output = bytearray(font.header)
|
||||
for location, length, width in rows:
|
||||
output.extend(location.to_bytes(2, "big"))
|
||||
output.extend((length, width))
|
||||
return bytes(output)
|
||||
|
||||
|
||||
def generate_header(fonts: tuple[Font, ...]) -> str:
|
||||
output = [HEADER_START]
|
||||
for mode in (2, 3):
|
||||
output.append(f"#if MESH_ARIAL_EXPERIMENT == {mode}")
|
||||
for index, font in enumerate(fonts[1:], 1):
|
||||
prefix = f"FONT_{index}"
|
||||
# Inline function-local statics have one address across translation
|
||||
# units. Namespace-static arrays would silently break identify().
|
||||
output.append(f"inline const Font& fontStorage{index}() {{")
|
||||
if mode == 2:
|
||||
data, rows = rle_font(font)
|
||||
offsets, lengths = (), ()
|
||||
elif mode == 3:
|
||||
data, rows, offsets, lengths = chunk_font(font)
|
||||
output.append(array(prefix + "_DATA", data))
|
||||
output.append(array(prefix + "_DESCRIPTOR", descriptor(font, rows)))
|
||||
if offsets:
|
||||
output.append(array(prefix + "_OFFSETS", offsets, "uint16_t"))
|
||||
output.append(array(prefix + "_LENGTHS", lengths, "uint16_t"))
|
||||
output.append(
|
||||
f"static const Font {prefix} = {{ " + prefix + "_DESCRIPTOR, " + prefix + "_DATA, "
|
||||
+ (prefix + "_OFFSETS, " + prefix + "_LENGTHS" if offsets else "nullptr, nullptr")
|
||||
+ f", {len(lengths)}, {len(data)} }};"
|
||||
)
|
||||
output.append(f"return {prefix};\n}}")
|
||||
output.append("#endif")
|
||||
output.append(HEADER_END)
|
||||
return "\n\n".join(output) + "\n"
|
||||
|
||||
|
||||
HEADER_START = r'''// Generated by scripts/generate_compact_arial.py; do not edit.
|
||||
// Experiments are LOSSLESS: same Arial pixels, all 224 slots and advance widths.
|
||||
// 2 = glyph RLE (<=91 scratch bytes); 3 = Zopfli 1000 chunks (<=512 scratch).
|
||||
// Only Arial16 and Arial24 are instantiated; the unused initial Arial10 is not.
|
||||
#pragma once
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#if MESH_ARIAL_EXPERIMENT == 3
|
||||
#include "../ota/tinf/tinf.h"
|
||||
#endif
|
||||
#if MESH_ARIAL_EXPERIMENT != 2 && MESH_ARIAL_EXPERIMENT != 3
|
||||
#error "CompactArialFonts.h is an opt-in experiment for modes 2 and 3 only"
|
||||
#endif
|
||||
namespace mesh { namespace ui { namespace arial {
|
||||
static constexpr size_t kScratchSize = MESH_ARIAL_EXPERIMENT == 2 ? 91 : 512;
|
||||
struct Font {
|
||||
const uint8_t* descriptor;
|
||||
const uint8_t* data;
|
||||
const uint16_t* offsets;
|
||||
const uint16_t* rawLengths;
|
||||
uint16_t chunkCount;
|
||||
uint16_t dataLength;
|
||||
};
|
||||
struct Glyph {
|
||||
const uint8_t* data;
|
||||
uint8_t width;
|
||||
uint8_t length;
|
||||
bool drawable;
|
||||
bool valid;
|
||||
};'''
|
||||
|
||||
|
||||
HEADER_END = r'''inline const uint8_t* font(unsigned id) {
|
||||
return id == 1 ? fontStorage1().descriptor : id == 2 ? fontStorage2().descriptor : nullptr;
|
||||
}
|
||||
inline int identify(const uint8_t* descriptor) {
|
||||
return descriptor == fontStorage1().descriptor ? 1 : descriptor == fontStorage2().descriptor ? 2 : -1;
|
||||
}
|
||||
inline const Font* storage(const uint8_t* descriptor) {
|
||||
const int id = identify(descriptor);
|
||||
return id == 1 ? &fontStorage1() : id == 2 ? &fontStorage2() : nullptr;
|
||||
}
|
||||
// Writes only to caller scratch. No global cache, heap allocation, or mutable state.
|
||||
inline bool decodeRle(const uint8_t* input, size_t inputLength,
|
||||
uint8_t* output, size_t outputLength) {
|
||||
if ((!input && inputLength) || (!output && outputLength)) return false;
|
||||
size_t source = 0, dest = 0;
|
||||
while (source < inputLength) {
|
||||
const uint8_t token = input[source++];
|
||||
const size_t count = token < 128 ? size_t(token) + 1 : size_t(token & 127) + 3;
|
||||
if (count > outputLength - dest) return false;
|
||||
if (token < 128) {
|
||||
if (count > inputLength - source) return false;
|
||||
for (size_t j = 0; j < count; ++j) output[dest++] = input[source++];
|
||||
} else {
|
||||
if (source == inputLength) return false;
|
||||
const uint8_t value = input[source++];
|
||||
for (size_t j = 0; j < count; ++j) output[dest++] = value;
|
||||
}
|
||||
}
|
||||
return dest == outputLength;
|
||||
}
|
||||
inline Glyph glyph(const uint8_t* descriptor, uint8_t code, uint8_t* scratch, size_t capacity) {
|
||||
const Font* f = storage(descriptor);
|
||||
if (!f || code < descriptor[2] || unsigned(code - descriptor[2]) >= descriptor[3])
|
||||
return { nullptr, 0, 0, false, false };
|
||||
const unsigned slot = code - descriptor[2];
|
||||
const uint8_t* record = descriptor + 4 + 4 * slot;
|
||||
const unsigned location = (unsigned(record[0]) << 8) | record[1];
|
||||
const uint8_t length = record[2], width = record[3];
|
||||
if (!length) return { nullptr, width, 0, false, location == 0xffff };
|
||||
#if MESH_ARIAL_EXPERIMENT == 2
|
||||
const unsigned first = location & 0x7fff;
|
||||
unsigned end = f->dataLength;
|
||||
for (unsigned next = slot + 1; next < descriptor[3]; ++next) {
|
||||
const uint8_t* nextRecord = descriptor + 4 + 4 * next;
|
||||
if (nextRecord[2]) { end = ((unsigned(nextRecord[0]) << 8) | nextRecord[1]) & 0x7fff; break; }
|
||||
}
|
||||
if (first > end || end > f->dataLength) return { nullptr, width, 0, false, false };
|
||||
if (location & 0x8000) {
|
||||
if (end - first != length) return { nullptr, width, 0, false, false };
|
||||
return { f->data + first, width, length, true, true };
|
||||
}
|
||||
if (!scratch || capacity < length || !decodeRle(f->data + first, end - first, scratch, length))
|
||||
return { nullptr, width, 0, false, false };
|
||||
return { scratch, width, length, true, true };
|
||||
#elif MESH_ARIAL_EXPERIMENT == 3
|
||||
const unsigned group = location >> 9;
|
||||
const unsigned offset = location & 511;
|
||||
if (group >= f->chunkCount) return { nullptr, width, 0, false, false };
|
||||
const unsigned first = f->offsets[group], end = f->offsets[group + 1];
|
||||
const unsigned expected = f->rawLengths[group];
|
||||
if (!scratch || expected > kScratchSize || capacity < expected || offset + length > expected || first > end || end > f->dataLength)
|
||||
return { nullptr, width, 0, false, false };
|
||||
unsigned int produced = expected;
|
||||
if (tinf_uncompress_exact(scratch, &produced, f->data + first, end - first) != TINF_OK || produced != expected)
|
||||
return { nullptr, width, 0, false, false };
|
||||
return { scratch + offset, width, length, true, true };
|
||||
#endif
|
||||
}
|
||||
}}} // namespace mesh::ui::arial'''
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--check", action="store_true", help="Check the generated header without writing")
|
||||
args = parser.parse_args()
|
||||
fonts = read_fonts()
|
||||
header = generate_header(fonts)
|
||||
if args.check:
|
||||
if not OUTPUT.exists() or OUTPUT.read_text(encoding="utf-8") != header:
|
||||
raise SystemExit("CompactArialFonts.h is stale; regenerate the font experiments")
|
||||
else:
|
||||
OUTPUT.write_text(header, encoding="utf-8", newline="\n")
|
||||
print("font original RLE+table chunks+table wholefont chunks")
|
||||
for font in fonts:
|
||||
rle, _ = rle_font(font)
|
||||
chunk, _, offsets, lengths = chunk_font(font)
|
||||
print(f"{font.name:20} {len(font.original):5} {len(rle) + 4 * len(font.glyphs) + 4:9} "
|
||||
f"{len(chunk) + 4 * len(font.glyphs) + 2 * (len(offsets) + len(lengths)) + 4:12} "
|
||||
f"{len(whole_font(font)):9} {len(lengths):6}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user