From a7e22955cdfb0567f9919b0786b11a82d4ad4094 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Fri, 18 Sep 2026 12:37:46 -0700 Subject: [PATCH] Use Brotli and Zopfli for compressed assets --- .../setup-build-environment/action.yml | 2 +- README.md | 1 + requirements-build.txt | 3 + scripts/brotli_compress.py | 31 + scripts/generate_webconfig_html.py | 80 +- scripts/zopfli_compress.py | 64 + src/helpers/bible/ReaderData.generated.h | 4375 ++++++++--------- src/helpers/esp32/WebConfigHtml.h | 4067 +++++++-------- src/helpers/esp32/WebConfigServer.cpp | 8 +- test/test_companion_reader.py | 2 +- test/test_ota/mota_vectors.h | 6 +- test/test_webconfig_secret_masking.py | 8 +- test/test_webconfig_ui.py | 17 +- test/test_webconfig_ui_runtime.py | 8 +- tools/bible/README.md | 20 +- tools/bible/pack_reader.py | 9 +- tools/ble_mota/ble_mota_seeder.py | 7 +- tools/ble_mota/test_ble_mota_seeder.py | 3 +- tools/mota/gen_vectors.py | 10 +- 19 files changed, 4249 insertions(+), 4472 deletions(-) create mode 100644 requirements-build.txt create mode 100644 scripts/brotli_compress.py create mode 100644 scripts/zopfli_compress.py diff --git a/.github/actions/setup-build-environment/action.yml b/.github/actions/setup-build-environment/action.yml index 026fb3df5..9d7ea5877 100644 --- a/.github/actions/setup-build-environment/action.yml +++ b/.github/actions/setup-build-environment/action.yml @@ -19,7 +19,7 @@ runs: - name: Install PlatformIO shell: bash run: | - pip install --upgrade platformio + pip install --upgrade platformio -r requirements-build.txt # Keep the complete version suffix. For example: # - room-server-v1.2.3 -> v1.2.3 diff --git a/README.md b/README.md index 984450cde..d44ba3e9c 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ For developers: - Install [PlatformIO](https://docs.platformio.org) in [Visual Studio Code](https://code.visualstudio.com). - Clone and open the MeshCore repository in Visual Studio Code. +- Install the pinned build-time compressors with `python -m pip install -r requirements-build.txt`. - See the example applications you can modify and run: - [Companion Radio](./examples/companion_radio) - For use with an external chat app, over BLE, USB or Wi-Fi. - [KISS Modem](./examples/kiss_modem) - Serial KISS protocol bridge for host applications. ([protocol docs](./docs/kiss_modem_protocol.md)) diff --git a/requirements-build.txt b/requirements-build.txt new file mode 100644 index 000000000..8dd1371f6 --- /dev/null +++ b/requirements-build.txt @@ -0,0 +1,3 @@ +# Reproducible compressors for firmware-resident and OTA assets. +brotli==1.2.0 +zopfli==0.4.3 diff --git a/scripts/brotli_compress.py b/scripts/brotli_compress.py new file mode 100644 index 000000000..43830a750 --- /dev/null +++ b/scripts/brotli_compress.py @@ -0,0 +1,31 @@ +"""Pinned Google Brotli helper for the browser-decoded WebConfig asset.""" + +from __future__ import annotations + +BROTLI_VERSION = "1.2.0" +QUALITY = 11 +# Maximum RFC 7932/browser-compatible Brotli window (16 MiB). The page is +# smaller than the default 4 MiB window, but pin the requested maximum anyway. +LGWIN = 24 + +try: + import brotli +except ImportError as error: + raise RuntimeError( + "Google Brotli 1.2.0 is required to generate the MeshCore WebConfig " + "asset. Install it with 'python -m pip install -r " + "requirements-build.txt'." + ) from error + +if brotli.__version__ != BROTLI_VERSION: + raise RuntimeError( + "MeshCore requires Google Brotli %s, but %s is installed. Run " + "'python -m pip install -r requirements-build.txt'." + % (BROTLI_VERSION, brotli.__version__) + ) + + +def compress_html(data: bytes) -> bytes: + """Return a deterministic, maximum-quality RFC 7932 Brotli text stream.""" + return brotli.compress(data, mode=brotli.MODE_TEXT, quality=QUALITY, + lgwin=LGWIN) diff --git a/scripts/generate_webconfig_html.py b/scripts/generate_webconfig_html.py index 20f857068..271b25f39 100644 --- a/scripts/generate_webconfig_html.py +++ b/scripts/generate_webconfig_html.py @@ -1,10 +1,10 @@ #!/usr/bin/env python # -# Pre-build script: gzip webui/index.html into a PROGMEM C header so the +# Pre-build script: Brotli-compress webui/index.html into a PROGMEM C header so the # webconfig portal can serve the page straight from flash with -# Content-Encoding: gzip. The generated header is tracked so ESP32 targets +# Content-Encoding: br. The generated header is tracked so ESP32 targets # that replace extra_scripts still compile from a clean checkout; this script -# refreshes it whenever the source page (or this script) is newer. +# rewrites it only when the emitted firmware bytes actually change. # # Comments and indentation are stripped before compressing (see strip_source). # The source page is heavily commented by house style and none of it is worth @@ -12,14 +12,13 @@ # # Output: src/helpers/esp32/WebConfigHtml.h # WEBCONFIG_HTML_LOADER[] - small bootstrap page (PROGMEM) -# WEBCONFIG_HTML_GZ[] - complete gzipped page (PROGMEM) +# WEBCONFIG_HTML_BR[] - complete Brotli-compressed page (PROGMEM) # WEBCONFIG_HTML_ETAG - quoted strong ETag for the loader page # # Runnable outside SCons to inspect exactly what gets shipped: # python3 scripts/generate_webconfig_html.py --emit /tmp/shipped.html # or served directly by the mock backend with its --minify flag. -import gzip import hashlib import os import sys @@ -32,43 +31,26 @@ except NameError: SOURCE = os.path.join("webui", "index.html") OUTPUT = os.path.join("src", "helpers", "esp32", "WebConfigHtml.h") # __file__ is not defined inside PIO/SCons-executed extra_scripts -SCRIPT = os.path.join("scripts", "generate_webconfig_html.py") -MINIFIER = os.path.join("scripts", "webconfig_minify.py") -HASH_MARKER = "// build-inputs-sha256: " +HASH_MARKER = "// shipped-page-sha256: " sys.path.insert(0, os.path.join(os.getcwd(), "scripts")) from webconfig_minify import check_stripped, strip_source # noqa: E402 +from brotli_compress import BROTLI_VERSION, LGWIN, QUALITY, compress_html # noqa: E402 def status(msg): sys.stderr.write("WebConfig HTML: %s\n" % msg) -def content_hash(): - # Hash the source page, this generator and the minifier so any change to - # any of them forces a regenerate, independent of file timestamps. - h = hashlib.sha256() - with open(SOURCE, "rb") as f: - h.update(f.read()) - for path in (SCRIPT, MINIFIER): - if os.path.isfile(path): - h.update(b"\0") - with open(path, "rb") as f: - h.update(f.read()) - return h.hexdigest() +def content_hash(raw): + """Identify the bytes that the device actually serves. - -def stored_hash(): - try: - with open(OUTPUT, "r") as f: - for line in f: - if line.startswith(HASH_MARKER): - return line[len(HASH_MARKER):].strip() - if line.startswith("#"): # reached the C preprocessor lines - break - except OSError: - return None - return None + Do not hash this generator or the minifier source: a comment-only tooling + edit must not dirty the checked-in firmware header. Generator changes are + still safe because main() renders the candidate header and compares its + full byte sequence before deciding whether to write. + """ + return hashlib.sha256(raw).hexdigest() def shipped_page(): @@ -104,16 +86,12 @@ def main(): status("%s -> %s (%d -> %d bytes)" % (SOURCE, dest, len(src), len(stripped))) return - src_hash = content_hash() - if os.path.isfile(OUTPUT) and stored_hash() == src_hash: - return - src, stripped = shipped_page() raw = stripped.encode("utf-8") + src_hash = content_hash(raw) - # mtime=0 keeps the gzip output and its immutable URL deterministic. - gz = gzip.compress(raw, compresslevel=9, mtime=0) - version = hashlib.sha256(gz).hexdigest()[:16] + br = compress_html(raw) + version = hashlib.sha256(br).hexdigest()[:16] loader = ("MeshCore " "WebConfigLoading MeshCore WebConfig…