Use Brotli and Zopfli for compressed assets
This commit is contained in:
1 parent
a37c345dbe
commit
8d8c7e5885
19 files changed
+4249
-4472
No files matched your search
@@ -19,7 +19,7 @@ runs:
|
||||
- name: Install PlatformIO
|
||||
shell: bash
|
||||
run: |
|
||||
pip install --upgrade platformio
|
||||
pip install --upgrade platformio -r requirements-build.txt
|
||||
|
||||
# Keep the complete version suffix. For example:
|
||||
# - room-server-v1.2.3 -> v1.2.3
|
||||
|
||||
@@ -50,6 +50,7 @@ For developers:
|
||||
|
||||
- Install [PlatformIO](https://docs.platformio.org) in [Visual Studio Code](https://code.visualstudio.com).
|
||||
- Clone and open the MeshCore repository in Visual Studio Code.
|
||||
- Install the pinned build-time compressors with `python -m pip install -r requirements-build.txt`.
|
||||
- See the example applications you can modify and run:
|
||||
- [Companion Radio](./examples/companion_radio) - For use with an external chat app, over BLE, USB or Wi-Fi.
|
||||
- [KISS Modem](./examples/kiss_modem) - Serial KISS protocol bridge for host applications. ([protocol docs](./docs/kiss_modem_protocol.md))
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
# Reproducible compressors for firmware-resident and OTA assets.
|
||||
brotli==1.2.0
|
||||
zopfli==0.4.3
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Pinned Google Brotli helper for the browser-decoded WebConfig asset."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
BROTLI_VERSION = "1.2.0"
|
||||
QUALITY = 11
|
||||
# Maximum RFC 7932/browser-compatible Brotli window (16 MiB). The page is
|
||||
# smaller than the default 4 MiB window, but pin the requested maximum anyway.
|
||||
LGWIN = 24
|
||||
|
||||
try:
|
||||
import brotli
|
||||
except ImportError as error:
|
||||
raise RuntimeError(
|
||||
"Google Brotli 1.2.0 is required to generate the MeshCore WebConfig "
|
||||
"asset. Install it with 'python -m pip install -r "
|
||||
"requirements-build.txt'."
|
||||
) from error
|
||||
|
||||
if brotli.__version__ != BROTLI_VERSION:
|
||||
raise RuntimeError(
|
||||
"MeshCore requires Google Brotli %s, but %s is installed. Run "
|
||||
"'python -m pip install -r requirements-build.txt'."
|
||||
% (BROTLI_VERSION, brotli.__version__)
|
||||
)
|
||||
|
||||
|
||||
def compress_html(data: bytes) -> bytes:
|
||||
"""Return a deterministic, maximum-quality RFC 7932 Brotli text stream."""
|
||||
return brotli.compress(data, mode=brotli.MODE_TEXT, quality=QUALITY,
|
||||
lgwin=LGWIN)
|
||||
@@ -1,10 +1,10 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Pre-build script: gzip webui/index.html into a PROGMEM C header so the
|
||||
# Pre-build script: Brotli-compress webui/index.html into a PROGMEM C header so the
|
||||
# webconfig portal can serve the page straight from flash with
|
||||
# Content-Encoding: gzip. The generated header is tracked so ESP32 targets
|
||||
# Content-Encoding: br. The generated header is tracked so ESP32 targets
|
||||
# that replace extra_scripts still compile from a clean checkout; this script
|
||||
# refreshes it whenever the source page (or this script) is newer.
|
||||
# rewrites it only when the emitted firmware bytes actually change.
|
||||
#
|
||||
# Comments and indentation are stripped before compressing (see strip_source).
|
||||
# The source page is heavily commented by house style and none of it is worth
|
||||
@@ -12,14 +12,13 @@
|
||||
#
|
||||
# Output: src/helpers/esp32/WebConfigHtml.h
|
||||
# WEBCONFIG_HTML_LOADER[] - small bootstrap page (PROGMEM)
|
||||
# WEBCONFIG_HTML_GZ[] - complete gzipped page (PROGMEM)
|
||||
# WEBCONFIG_HTML_BR[] - complete Brotli-compressed page (PROGMEM)
|
||||
# WEBCONFIG_HTML_ETAG - quoted strong ETag for the loader page
|
||||
#
|
||||
# Runnable outside SCons to inspect exactly what gets shipped:
|
||||
# python3 scripts/generate_webconfig_html.py --emit /tmp/shipped.html
|
||||
# or served directly by the mock backend with its --minify flag.
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
@@ -32,43 +31,26 @@ except NameError:
|
||||
SOURCE = os.path.join("webui", "index.html")
|
||||
OUTPUT = os.path.join("src", "helpers", "esp32", "WebConfigHtml.h")
|
||||
# __file__ is not defined inside PIO/SCons-executed extra_scripts
|
||||
SCRIPT = os.path.join("scripts", "generate_webconfig_html.py")
|
||||
MINIFIER = os.path.join("scripts", "webconfig_minify.py")
|
||||
HASH_MARKER = "// build-inputs-sha256: "
|
||||
HASH_MARKER = "// shipped-page-sha256: "
|
||||
|
||||
sys.path.insert(0, os.path.join(os.getcwd(), "scripts"))
|
||||
from webconfig_minify import check_stripped, strip_source # noqa: E402
|
||||
from brotli_compress import BROTLI_VERSION, LGWIN, QUALITY, compress_html # noqa: E402
|
||||
|
||||
|
||||
def status(msg):
|
||||
sys.stderr.write("WebConfig HTML: %s\n" % msg)
|
||||
|
||||
|
||||
def content_hash():
|
||||
# Hash the source page, this generator and the minifier so any change to
|
||||
# any of them forces a regenerate, independent of file timestamps.
|
||||
h = hashlib.sha256()
|
||||
with open(SOURCE, "rb") as f:
|
||||
h.update(f.read())
|
||||
for path in (SCRIPT, MINIFIER):
|
||||
if os.path.isfile(path):
|
||||
h.update(b"\0")
|
||||
with open(path, "rb") as f:
|
||||
h.update(f.read())
|
||||
return h.hexdigest()
|
||||
def content_hash(raw):
|
||||
"""Identify the bytes that the device actually serves.
|
||||
|
||||
|
||||
def stored_hash():
|
||||
try:
|
||||
with open(OUTPUT, "r") as f:
|
||||
for line in f:
|
||||
if line.startswith(HASH_MARKER):
|
||||
return line[len(HASH_MARKER):].strip()
|
||||
if line.startswith("#"): # reached the C preprocessor lines
|
||||
break
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
Do not hash this generator or the minifier source: a comment-only tooling
|
||||
edit must not dirty the checked-in firmware header. Generator changes are
|
||||
still safe because main() renders the candidate header and compares its
|
||||
full byte sequence before deciding whether to write.
|
||||
"""
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def shipped_page():
|
||||
@@ -104,16 +86,12 @@ def main():
|
||||
status("%s -> %s (%d -> %d bytes)" % (SOURCE, dest, len(src), len(stripped)))
|
||||
return
|
||||
|
||||
src_hash = content_hash()
|
||||
if os.path.isfile(OUTPUT) and stored_hash() == src_hash:
|
||||
return
|
||||
|
||||
src, stripped = shipped_page()
|
||||
raw = stripped.encode("utf-8")
|
||||
src_hash = content_hash(raw)
|
||||
|
||||
# mtime=0 keeps the gzip output and its immutable URL deterministic.
|
||||
gz = gzip.compress(raw, compresslevel=9, mtime=0)
|
||||
version = hashlib.sha256(gz).hexdigest()[:16]
|
||||
br = compress_html(raw)
|
||||
version = hashlib.sha256(br).hexdigest()[:16]
|
||||
loader = ("<!doctype html><meta charset=utf-8><meta name=viewport "
|
||||
"content=\"width=device-width,initial-scale=1\"><title>MeshCore "
|
||||
"WebConfig</title><body>Loading MeshCore WebConfig…<script>"
|
||||
@@ -133,27 +111,37 @@ def main():
|
||||
lines = []
|
||||
lines.append("// Auto-generated by scripts/generate_webconfig_html.py from %s" % SOURCE.replace(os.sep, "/"))
|
||||
lines.append("// DO NOT EDIT - edit webui/index.html instead.")
|
||||
lines.append("// Compressed with Google Brotli %s (quality %d, lgwin %d)." %
|
||||
(BROTLI_VERSION, QUALITY, LGWIN))
|
||||
lines.append("%s%s" % (HASH_MARKER, src_hash))
|
||||
lines.append("#pragma once")
|
||||
lines.append("#include <stdint.h>")
|
||||
lines.append("#include <pgmspace.h>")
|
||||
lines.append("")
|
||||
lines.append("const uint32_t WEBCONFIG_HTML_RAW_LEN = %d;" % len(raw))
|
||||
lines.append("const uint32_t WEBCONFIG_HTML_GZ_LEN = %d;" % len(gz))
|
||||
lines.append("const uint32_t WEBCONFIG_HTML_BR_LEN = %d;" % len(br))
|
||||
lines.append('const char WEBCONFIG_HTML_ETAG[] = "\\"%s\\"";' % etag)
|
||||
lines.append('const char WEBCONFIG_HTML_VERSION[] = "%s";' % version)
|
||||
lines.append("const uint16_t WEBCONFIG_HTML_LOADER_LEN = %d;" % len(loader_bytes))
|
||||
append_byte_array(lines, "WEBCONFIG_HTML_LOADER", loader_bytes)
|
||||
lines.append("")
|
||||
append_byte_array(lines, "WEBCONFIG_HTML_GZ", gz)
|
||||
append_byte_array(lines, "WEBCONFIG_HTML_BR", br)
|
||||
lines.append("")
|
||||
|
||||
os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
|
||||
with open(OUTPUT, "w") as f:
|
||||
f.write("\n".join(lines))
|
||||
rendered = "\n".join(lines)
|
||||
try:
|
||||
with open(OUTPUT, "r", encoding="utf-8", newline="") as f:
|
||||
if f.read() == rendered:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
status("%s -> %s (%d bytes source, %d stripped, %d gzip)"
|
||||
% (SOURCE, OUTPUT, len(src.encode("utf-8")), len(raw), len(gz)))
|
||||
os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
|
||||
with open(OUTPUT, "w", encoding="utf-8", newline="") as f:
|
||||
f.write(rendered)
|
||||
|
||||
status("%s -> %s (%d bytes source, %d stripped, %d Brotli)"
|
||||
% (SOURCE, OUTPUT, len(src.encode("utf-8")), len(raw), len(br)))
|
||||
|
||||
|
||||
# This script is attached only to esp32_base. Generate unconditionally here:
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Pinned Google Zopfli helpers for static and host-generated DEFLATE data.
|
||||
|
||||
Zopfli writes standards-compliant DEFLATE streams, so the existing device
|
||||
decoders need no additional firmware code or RAM. It is deliberately used
|
||||
only where compression happens on a development machine or host controller:
|
||||
firmware assets, test vectors, and BLE mOTA transport blocks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
ZOPFLI_VERSION = "0.4.3"
|
||||
|
||||
try:
|
||||
import zopfli
|
||||
from zopfli import gzip as _gzip
|
||||
from zopfli import zlib as _zlib
|
||||
except ImportError as error:
|
||||
raise RuntimeError(
|
||||
"Google Zopfli 0.4.3 is required to generate MeshCore compressed "
|
||||
"assets. Install it with 'python -m pip install -r "
|
||||
"requirements-build.txt'."
|
||||
) from error
|
||||
|
||||
if zopfli.__version__ != ZOPFLI_VERSION:
|
||||
raise RuntimeError(
|
||||
"MeshCore requires Google Zopfli %s, but %s is installed. Run "
|
||||
"'python -m pip install -r requirements-build.txt'."
|
||||
% (ZOPFLI_VERSION, zopfli.__version__)
|
||||
)
|
||||
|
||||
# Above zlib's maximum level 9: Zopfli refines the LZ77 model repeatedly.
|
||||
# Fifteen iterations is the upstream binding's practical default for assets
|
||||
# of this size and keeps CI/build times reasonable.
|
||||
ITERATIONS = 15
|
||||
|
||||
|
||||
def _options() -> dict[str, int | bool]:
|
||||
return {
|
||||
"numiterations": ITERATIONS,
|
||||
"blocksplitting": True,
|
||||
"blocksplittinglast": False,
|
||||
"blocksplittingmax": 15,
|
||||
}
|
||||
|
||||
|
||||
def gzip_compress(data: bytes) -> bytes:
|
||||
"""Compress *data* as deterministic RFC 1952 gzip using Google Zopfli."""
|
||||
encoded = bytearray(_gzip.compress(data, **_options()))
|
||||
# The gzip OS byte is informational. Normalize it to RFC 1952's unknown
|
||||
# value so a checked-in asset is identical across operating systems.
|
||||
if len(encoded) < 10 or encoded[:3] != b"\x1f\x8b\x08":
|
||||
raise RuntimeError("Zopfli returned an invalid gzip stream")
|
||||
encoded[9] = 0xFF
|
||||
return bytes(encoded)
|
||||
|
||||
|
||||
def raw_deflate_compress(data: bytes) -> bytes:
|
||||
"""Compress *data* to an RFC 1951 raw DEFLATE stream using Google Zopfli."""
|
||||
encoded = _zlib.compress(data, **_options())
|
||||
# A zlib wrapper is exactly a two-byte header and four-byte Adler-32
|
||||
# trailer. Removing it preserves the RFC 1951 body expected by tinf.
|
||||
if len(encoded) < 6 or (encoded[0] & 0x0F) != 8:
|
||||
raise RuntimeError("Zopfli returned an invalid zlib stream")
|
||||
return encoded[2:-4]
|
||||
+2160
-2215
File diff suppressed because it is too large.
Load diff
+1894
-2173
File diff suppressed because it is too large.
Load diff
@@ -1642,9 +1642,11 @@ void WebConfigServer::handleUi(AsyncWebServerRequest* req) {
|
||||
}
|
||||
AsyncWebServerResponse* res =
|
||||
new WebConfigPacedProgmemResponse("text/html; charset=utf-8",
|
||||
WEBCONFIG_HTML_GZ,
|
||||
WEBCONFIG_HTML_GZ_LEN);
|
||||
res->addHeader("Content-Encoding", "gzip");
|
||||
WEBCONFIG_HTML_BR,
|
||||
WEBCONFIG_HTML_BR_LEN);
|
||||
// Keep exactly one whole-page asset in flash. Brotli is decoded by the
|
||||
// browser; the ESP32 only streams these precompressed bytes.
|
||||
res->addHeader("Content-Encoding", "br");
|
||||
res->addHeader("Cache-Control", "public, max-age=31536000, immutable");
|
||||
req->send(res);
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ class CompanionReaderTest(unittest.TestCase):
|
||||
packed, blocks = packer.pack(self.document)
|
||||
self.assertEqual(sum(block[2] for block in blocks), 96568)
|
||||
self.assertEqual(len(blocks), 49)
|
||||
self.assertEqual(len(packed) + 12 * len(blocks), 43891)
|
||||
self.assertEqual(len(packed) + 12 * len(blocks), 42775)
|
||||
|
||||
def test_invalid_sources_are_rejected(self):
|
||||
for bad_text in ("", "leading\ncommand", "bad\x1b[2J", "bad\0text",
|
||||
|
||||
@@ -104,11 +104,11 @@ static const uint8_t SIM_MOTA_2K[2459] = {109,79,84,65,155,9,0,0,2,3,18,190,186,
|
||||
|
||||
static const uint32_t SIM_MOTA_2K_LEN = 2459;
|
||||
static const uint32_t SIM_MOTA_2K_BLOCKS = 2;
|
||||
static const uint8_t SIM_MOTA_2K_DEFLATED[127] = {243,77,45,206,112,206,47,74,85,48,82,240,206,116,82,40,41,74,204,43,46,200,47,42,81,112,113,117,243,113,12,113,85,40,75,77,46,201,47,210,83,240,29,85,56,170,112,84,225,168,194,81,133,163,10,71,21,14,23,133,0,211,83,240,77,45,206,112,206,47,74,85,48,82,240,206,116,82,40,41,74,204,43,46,200,47,42,81,112,113,117,243,113,12,113,85,40,75,77,46,201,47,210,163,177,66,215,188,20,183,78,14,6,134,31,155,53,95,239,44,136,125,205,64,36,0,0};
|
||||
static const uint8_t SIM_MOTA_2K_DEFLATED[125] = {243,77,45,206,112,206,47,74,85,48,82,240,206,116,82,40,41,74,204,43,46,200,47,42,81,112,113,117,243,113,12,113,85,40,75,77,46,201,47,210,83,24,54,10,71,21,142,42,28,85,56,170,112,84,225,168,194,81,133,0,211,83,240,77,45,206,112,206,47,74,85,48,82,240,206,116,82,40,41,74,204,43,46,200,47,42,81,112,113,117,243,113,12,113,85,40,75,77,46,201,47,162,181,66,215,188,20,183,78,14,6,134,31,155,53,95,239,44,136,125,205,64,36,0,0};
|
||||
|
||||
static const uint16_t SIM_MOTA_2K_DEFLATED_OFFSETS[2] = {0,61};
|
||||
static const uint16_t SIM_MOTA_2K_DEFLATED_OFFSETS[2] = {0,60};
|
||||
|
||||
static const uint16_t SIM_MOTA_2K_DEFLATED_LENGTHS[2] = {61,66};
|
||||
static const uint16_t SIM_MOTA_2K_DEFLATED_LENGTHS[2] = {60,65};
|
||||
|
||||
// detools sequential+crle delta: apply DT_PATCH to DT_BASE -> DT_TARGET
|
||||
static const uint8_t DT_BASE[3056] = {120,46,186,148,77,51,227,185,104,193,183,194,67,136,62,162,208,188,127,90,106,134,186,157,246,55,79,139,180,84,132,19,187,198,255,221,52,176,192,186,119,236,181,212,223,167,37,136,54,222,105,250,14,197,89,160,106,119,31,185,190,35,195,83,99,84,88,203,51,83,109,106,81,145,54,231,222,104,58,52,10,191,57,195,4,248,221,66,216,129,81,197,245,145,205,180,107,157,28,84,217,167,155,199,59,60,254,118,93,34,51,94,126,152,214,160,36,67,99,159,86,85,240,181,255,182,119,220,43,175,178,196,220,33,84,236,52,148,175,16,25,240,215,44,1,230,38,112,180,60,89,58,20,50,205,72,61,177,118,154,67,123,134,225,111,169,248,106,51,215,18,77,77,71,34,144,169,187,64,134,25,126,55,228,50,200,99,45,131,217,57,81,90,192,179,228,204,11,148,230,238,173,139,96,239,189,184,243,162,18,30,58,14,132,32,241,212,53,232,162,157,236,22,242,129,44,60,124,149,204,187,42,41,22,32,158,26,207,241,152,143,207,254,154,161,7,152,27,143,46,139,178,80,0,31,71,7,46,15,26,162,219,159,172,158,187,53,148,53,165,48,118,47,121,80,69,187,116,162,112,213,183,206,210,55,102,150,221,114,221,107,152,179,34,225,53,41,79,101,50,192,45,91,116,175,3,30,85,172,0,197,57,192,129,107,168,249,9,54,155,118,141,127,140,206,12,110,85,2,130,87,141,249,230,240,224,65,170,187,40,57,154,248,27,211,203,216,110,17,245,225,34,44,6,220,86,78,242,234,64,126,41,92,213,119,237,111,241,125,157,83,40,9,197,31,30,93,108,162,244,46,129,180,24,13,168,111,228,6,207,233,227,240,69,59,30,81,139,222,145,35,54,145,149,27,110,26,241,153,180,178,68,111,143,40,195,59,240,0,131,31,50,96,137,146,104,114,146,201,44,212,165,236,63,141,235,195,74,230,208,70,151,91,115,163,28,103,101,194,72,81,128,133,58,60,210,199,206,91,58,107,201,119,146,79,73,245,172,175,236,49,119,165,138,13,64,97,211,166,53,67,105,132,34,167,80,72,176,137,206,241,34,195,23,129,56,118,155,71,75,63,165,132,99,189,72,244,47,246,228,233,247,122,206,93,247,7,152,165,96,177,16,193,185,231,34,25,108,156,82,48,255,196,244,244,19,194,148,65,8,206,163,198,66,171,217,133,48,241,218,204,111,42,49,182,120,211,68,17,118,31,25,151,68,33,191,98,200,250,150,212,165,25,57,193,149,60,42,75,166,229,35,209,177,238,206,98,175,27,249,33,86,105,84,161,179,85,140,210,192,60,5,154,71,97,35,145,68,43,129,204,230,65,55,225,230,140,33,179,109,189,138,48,32,208,33,172,177,59,64,6,163,157,173,28,251,108,135,107,8,121,39,70,182,92,118,88,74,123,227,173,148,114,232,141,165,8,174,233,72,47,98,166,229,126,163,92,128,124,93,240,40,16,129,188,248,249,141,68,50,46,112,119,83,159,1,36,188,29,108,12,72,193,168,191,20,181,224,21,171,122,118,241,53,134,128,172,191,216,59,171,163,169,80,167,131,162,77,124,75,148,14,173,54,172,186,138,121,182,62,79,166,255,80,64,84,10,35,159,137,202,138,66,88,88,38,13,89,3,158,80,192,80,50,47,239,72,206,116,135,12,251,40,25,201,123,163,67,100,13,105,51,174,34,34,93,3,115,92,192,106,231,63,31,87,218,156,115,83,1,188,172,152,212,206,133,105,183,160,198,113,51,32,213,29,130,254,173,239,245,104,126,255,101,127,115,186,56,236,11,217,178,248,114,215,168,163,134,209,19,244,68,229,206,194,29,94,160,221,137,217,92,17,83,227,202,123,155,151,121,220,191,199,202,241,205,234,70,164,104,185,211,96,128,163,65,230,28,4,63,219,134,230,237,183,181,229,109,41,192,38,48,3,205,3,69,15,27,85,41,249,58,60,229,132,70,199,19,39,247,213,215,10,64,234,230,209,250,91,21,134,94,143,43,27,92,64,200,3,206,207,147,130,194,137,110,114,158,192,62,200,9,173,197,36,195,222,90,68,139,0,117,232,231,62,93,112,14,37,53,86,17,150,199,31,117,20,200,185,21,58,0,193,20,229,210,41,161,119,53,225,179,50,59,120,212,104,243,159,148,20,197,167,106,194,135,192,98,133,68,210,210,227,227,161,183,180,36,62,92,189,50,39,122,5,177,246,255,185,32,70,105,173,233,213,61,174,111,102,212,174,40,98,96,113,237,146,163,30,186,69,139,67,157,37,83,30,161,170,92,20,88,126,31,148,49,32,133,35,206,117,26,155,154,102,119,184,196,144,146,104,188,112,247,229,124,82,237,146,81,215,157,134,60,215,44,249,56,157,202,243,236,105,6,130,117,190,131,93,11,78,190,251,41,176,233,229,147,70,59,56,145,165,174,147,111,242,160,241,201,203,118,88,132,232,178,22,117,168,133,225,76,86,240,4,246,158,51,191,121,245,52,164,72,112,109,131,246,21,37,123,166,143,107,76,69,128,184,87,200,23,39,48,21,138,248,161,140,8,151,174,127,172,55,217,205,164,191,41,40,21,65,102,38,231,245,193,137,82,79,200,219,185,83,168,181,87,125,247,47,112,61,195,99,115,20,131,186,193,11,9,106,72,43,175,52,216,11,212,247,27,151,35,162,53,237,20,208,201,47,218,30,26,86,4,185,26,142,255,250,59,219,212,222,152,155,131,117,49,122,51,2,78,187,225,215,90,99,196,255,71,7,19,218,205,190,132,213,178,193,114,94,81,104,244,138,51,254,116,38,23,73,190,214,169,31,220,41,172,172,205,57,159,93,237,199,135,219,63,101,42,222,90,192,55,33,98,171,182,18,47,63,214,206,6,76,251,28,178,150,174,154,63,127,87,170,172,107,230,55,12,253,165,15,70,55,191,80,114,158,203,222,55,160,212,2,147,59,21,136,61,126,64,193,123,132,221,255,44,162,133,94,13,143,204,9,115,165,105,81,111,100,16,20,44,80,204,209,171,181,144,67,195,225,253,33,209,171,193,155,183,25,142,1Line truncated
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import gzip
|
||||
import brotli
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unittest
|
||||
@@ -19,14 +19,14 @@ def function_body(source: str, start: str, end: str) -> str:
|
||||
|
||||
def embedded_ui() -> str:
|
||||
header = EMBEDDED_UI.read_text(encoding="utf-8")
|
||||
length = int(re.search(r"WEBCONFIG_HTML_GZ_LEN = (\d+);", header).group(1))
|
||||
length = int(re.search(r"WEBCONFIG_HTML_BR_LEN = (\d+);", header).group(1))
|
||||
array = header.split(
|
||||
"const uint8_t WEBCONFIG_HTML_GZ[] PROGMEM = {", 1
|
||||
"const uint8_t WEBCONFIG_HTML_BR[] PROGMEM = {", 1
|
||||
)[1]
|
||||
blob = bytes(
|
||||
int(value, 16) for value in re.findall(r"0x([0-9a-f]{2})", array)
|
||||
)[:length]
|
||||
return gzip.decompress(blob).decode("utf-8")
|
||||
return brotli.decompress(blob).decode("utf-8")
|
||||
|
||||
|
||||
class WebConfigSecretMaskingTest(unittest.TestCase):
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import gzip
|
||||
import brotli
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unittest
|
||||
@@ -226,14 +226,15 @@ class WebConfigUiTest(unittest.TestCase):
|
||||
|
||||
def test_generated_asset_contains_show_controls(self):
|
||||
header = HEADER.read_text(encoding="utf-8")
|
||||
length = int(re.search(r"WEBCONFIG_HTML_GZ_LEN = (\d+);", header).group(1))
|
||||
self.assertIn("Compressed with Google Brotli", header)
|
||||
length = int(re.search(r"WEBCONFIG_HTML_BR_LEN = (\d+);", header).group(1))
|
||||
array = header.split(
|
||||
"const uint8_t WEBCONFIG_HTML_GZ[] PROGMEM = {", 1
|
||||
"const uint8_t WEBCONFIG_HTML_BR[] PROGMEM = {", 1
|
||||
)[1]
|
||||
blob = bytes(
|
||||
int(value, 16) for value in re.findall(r"0x([0-9a-f]{2})", array)
|
||||
)[:length]
|
||||
page = gzip.decompress(blob).decode("utf-8")
|
||||
page = brotli.decompress(blob).decode("utf-8")
|
||||
self.assertEqual(page.count('onclick="toggleWiFiPassword(this)"'), 2)
|
||||
self.assertIn("function toggleWiFiPassword(btn)", page)
|
||||
self.assertIn("function wzSkipRadio()", page)
|
||||
@@ -248,6 +249,14 @@ class WebConfigUiTest(unittest.TestCase):
|
||||
self.assertIn('case"bluetooth.mac":return r.bluetooth_mac||"";', page)
|
||||
self.assertIn("random-every-boot", page)
|
||||
|
||||
def test_webconfig_serves_the_single_brotli_asset(self):
|
||||
server = (ROOT / "src/helpers/esp32/WebConfigServer.cpp").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertIn("WEBCONFIG_HTML_BR", server)
|
||||
self.assertIn('res->addHeader("Content-Encoding", "br")', server)
|
||||
self.assertNotIn("WEBCONFIG_HTML_GZ", server)
|
||||
|
||||
def test_skipped_mqtt_does_not_pin_display_in_setup_mode(self):
|
||||
main = (ROOT / "examples" / "companion_radio" / "main.cpp").read_text(
|
||||
encoding="utf-8"
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
"""Optional real-browser checks for WebConfig's asynchronous setup races."""
|
||||
|
||||
import gzip
|
||||
import brotli
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
@@ -38,12 +38,12 @@ BROWSER = chromium_path()
|
||||
|
||||
def embedded_page():
|
||||
header = HEADER.read_text(encoding="utf-8")
|
||||
length = int(re.search(r"WEBCONFIG_HTML_GZ_LEN = (\d+);", header).group(1))
|
||||
array = header.split("const uint8_t WEBCONFIG_HTML_GZ[] PROGMEM = {", 1)[1]
|
||||
length = int(re.search(r"WEBCONFIG_HTML_BR_LEN = (\d+);", header).group(1))
|
||||
array = header.split("const uint8_t WEBCONFIG_HTML_BR[] PROGMEM = {", 1)[1]
|
||||
blob = bytes(
|
||||
int(value, 16) for value in re.findall(r"0x([0-9a-f]{2})", array)
|
||||
)[:length]
|
||||
return gzip.decompress(blob).decode("utf-8")
|
||||
return brotli.decompress(blob).decode("utf-8")
|
||||
|
||||
|
||||
class WebConfigUiRuntimeTest(unittest.TestCase):
|
||||
|
||||
+12
-8
@@ -101,6 +101,7 @@ moving upstream text. See the [official download inventory](https://ebible.org/f
|
||||
Regenerate or verify the firmware header:
|
||||
|
||||
```sh
|
||||
python -m pip install -r requirements-build.txt
|
||||
python tools/bible/pack_reader.py
|
||||
python tools/bible/pack_reader.py --check
|
||||
python tools/bible/pack_reader.py --compare
|
||||
@@ -119,16 +120,19 @@ terminators, versus 98,911 bytes for the pinned UTF-8 source):
|
||||
|
||||
| Block limit | Blocks | DEFLATE bytes | Index bytes | Total flash data | Savings |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| 1 KiB | 101 | 48,175 | 1,212 | 49,387 | 48.86% |
|
||||
| **2 KiB** | **49** | **43,303** | **588** | **43,891** | **54.55%** |
|
||||
| 4 KiB | 24 | 39,549 | 288 | 39,837 | 58.75% |
|
||||
| 1 KiB | 101 | 47,133 | 1,212 | 48,345 | 49.94% |
|
||||
| **2 KiB** | **49** | **42,187** | **588** | **42,775** | **55.70%** |
|
||||
| 4 KiB | 24 | 38,366 | 288 | 38,654 | 59.97% |
|
||||
|
||||
Savings in the table are relative to uncompressed ASCII. At the chosen 2 KiB
|
||||
limit, ASCII typography saves **981 data bytes (2.19%)** compared with the
|
||||
previous 44,872-byte UTF-8 DEFLATE data and index. No 6-bit packing is used.
|
||||
Savings in the table are relative to uncompressed ASCII. Google Zopfli 0.4.3
|
||||
uses 15 optimization iterations to write standards-compliant raw DEFLATE, so
|
||||
the existing firmware decoder and RAM use do not change. At the chosen 2 KiB
|
||||
limit, the Zopfli asset is **2,097 bytes (4.67%)** smaller than the prior
|
||||
44,872-byte UTF-8 level-9 DEFLATE data and index; that result includes the
|
||||
existing ASCII typography conversion. No 6-bit packing is used.
|
||||
|
||||
The chosen 2 KiB limit saves 5,496 flash bytes over 1 KiB for another 1 KiB
|
||||
of temporary decode storage. Going to 4 KiB saves only another 4,054 flash
|
||||
The chosen 2 KiB limit saves 5,570 flash bytes over 1 KiB for another 1 KiB
|
||||
of temporary decode storage. Going to 4 KiB saves only another 4,121 flash
|
||||
bytes and doubles that buffer. The 2 KiB choice leaves more stack headroom
|
||||
for nRF52's existing BLE/USB/radio calls. These figures include block-index
|
||||
padding but exclude the small command code, attribution strings and decoder.
|
||||
|
||||
@@ -4,10 +4,13 @@ import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import unicodedata
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from zopfli_compress import ZOPFLI_VERSION, raw_deflate_compress # noqa: E402
|
||||
|
||||
SOURCE = Path(__file__).with_name("reader-web.json")
|
||||
HEADER = ROOT / "src/helpers/bible/ReaderData.generated.h"
|
||||
BLOCK_SIZE = 2048
|
||||
@@ -71,8 +74,7 @@ def pack(document, block_size=BLOCK_SIZE):
|
||||
raise ValueError("source contains no verse text")
|
||||
packed, descriptors = bytearray(), []
|
||||
for first_verse, block in zip(starts, blocks):
|
||||
compressor = zlib.compressobj(level=9, wbits=-15)
|
||||
compressed = compressor.compress(block) + compressor.flush()
|
||||
compressed = raw_deflate_compress(block)
|
||||
descriptors.append((len(packed), len(compressed), len(block), first_verse))
|
||||
packed.extend(compressed)
|
||||
return bytes(packed), descriptors
|
||||
@@ -83,6 +85,7 @@ def render(document):
|
||||
digest = hashlib.sha256(json.dumps(document, ensure_ascii=False, sort_keys=True,
|
||||
separators=(",", ":")).encode()).hexdigest()
|
||||
lines = ["// Generated by tools/bible/pack_reader.py; do not edit.",
|
||||
f"// Compressed with Google Zopfli {ZOPFLI_VERSION} (15 iterations).",
|
||||
"// Offline Reader corpus. Public domain; see tools/bible/README.md.",
|
||||
"// Firmware typography: ASCII quotes, apostrophes, dashes and spaces; wording unchanged.",
|
||||
f"// Canonical source SHA-256: {digest}",
|
||||
|
||||
@@ -19,6 +19,10 @@ import sys
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from zopfli_compress import raw_deflate_compress # noqa: E402
|
||||
|
||||
BLEAK_IMPORT_ERROR: ImportError | None = None
|
||||
try:
|
||||
from bleak import BleakClient, BleakScanner
|
||||
@@ -97,8 +101,7 @@ class MotaFile:
|
||||
raw = stream.read(raw_length)
|
||||
if len(raw) != raw_length:
|
||||
return None
|
||||
compressor = zlib.compressobj(level=9, method=zlib.DEFLATED, wbits=-11)
|
||||
encoded = compressor.compress(raw) + compressor.flush()
|
||||
encoded = raw_deflate_compress(raw)
|
||||
return encoded if 0 < len(encoded) < len(raw) else None
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -68,7 +68,8 @@ class TransportDeflateTests(unittest.TestCase):
|
||||
self.assertEqual(status, seeder.STATUS_OK)
|
||||
self.assertEqual(struct.unpack_from("<H", payload)[0], total)
|
||||
encoded.extend(payload[2:])
|
||||
self.assertEqual((encoded[0] >> 1) & 0x03, 2) # dynamic Huffman (BTYPE=2)
|
||||
# Zopfli may start with a stored, fixed, or dynamic sub-block; the
|
||||
# protocol promises a valid raw stream, not a particular block type.
|
||||
self.assertEqual(zlib.decompress(bytes(encoded), wbits=-11), self.raw)
|
||||
|
||||
compressor_1k = zlib.compressobj(
|
||||
|
||||
@@ -11,12 +11,15 @@ Run: ./meshcore/bin/python tools/mota/gen_vectors.py
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import zlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import motalib as ml
|
||||
|
||||
OUT = Path(__file__).resolve().parents[2] / "test" / "test_ota" / "mota_vectors.h"
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
OUT = ROOT / "test" / "test_ota" / "mota_vectors.h"
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from zopfli_compress import raw_deflate_compress # noqa: E402
|
||||
|
||||
|
||||
def _carr(name, data: bytes) -> str:
|
||||
@@ -155,8 +158,7 @@ def main():
|
||||
compressed_lengths = []
|
||||
for offset in range(0, len(sim2k_image), 2048):
|
||||
block = sim2k_image[offset:offset + 2048]
|
||||
encoder = zlib.compressobj(level=9, wbits=-15)
|
||||
encoded = encoder.compress(block) + encoder.flush()
|
||||
encoded = raw_deflate_compress(block)
|
||||
assert 0 < len(encoded) < len(block), (len(block), len(encoded))
|
||||
compressed_offsets.append(len(compressed))
|
||||
compressed_lengths.append(len(encoded))
|
||||
|
||||
Reference in new issue
Block a user