Use Brotli and Zopfli for compressed assets
This commit is contained in:
1 parent
9d91b039bf
commit
a7e22955cd
19 files changed
+4249
-4472
No files matched your search
@@ -0,0 +1,31 @@
|
||||
"""Pinned Google Brotli helper for the browser-decoded WebConfig asset."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
BROTLI_VERSION = "1.2.0"
|
||||
QUALITY = 11
|
||||
# Maximum RFC 7932/browser-compatible Brotli window (16 MiB). The page is
|
||||
# smaller than the default 4 MiB window, but pin the requested maximum anyway.
|
||||
LGWIN = 24
|
||||
|
||||
try:
|
||||
import brotli
|
||||
except ImportError as error:
|
||||
raise RuntimeError(
|
||||
"Google Brotli 1.2.0 is required to generate the MeshCore WebConfig "
|
||||
"asset. Install it with 'python -m pip install -r "
|
||||
"requirements-build.txt'."
|
||||
) from error
|
||||
|
||||
if brotli.__version__ != BROTLI_VERSION:
|
||||
raise RuntimeError(
|
||||
"MeshCore requires Google Brotli %s, but %s is installed. Run "
|
||||
"'python -m pip install -r requirements-build.txt'."
|
||||
% (BROTLI_VERSION, brotli.__version__)
|
||||
)
|
||||
|
||||
|
||||
def compress_html(data: bytes) -> bytes:
|
||||
"""Return a deterministic, maximum-quality RFC 7932 Brotli text stream."""
|
||||
return brotli.compress(data, mode=brotli.MODE_TEXT, quality=QUALITY,
|
||||
lgwin=LGWIN)
|
||||
@@ -1,10 +1,10 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Pre-build script: gzip webui/index.html into a PROGMEM C header so the
|
||||
# Pre-build script: Brotli-compress webui/index.html into a PROGMEM C header so the
|
||||
# webconfig portal can serve the page straight from flash with
|
||||
# Content-Encoding: gzip. The generated header is tracked so ESP32 targets
|
||||
# Content-Encoding: br. The generated header is tracked so ESP32 targets
|
||||
# that replace extra_scripts still compile from a clean checkout; this script
|
||||
# refreshes it whenever the source page (or this script) is newer.
|
||||
# rewrites it only when the emitted firmware bytes actually change.
|
||||
#
|
||||
# Comments and indentation are stripped before compressing (see strip_source).
|
||||
# The source page is heavily commented by house style and none of it is worth
|
||||
@@ -12,14 +12,13 @@
|
||||
#
|
||||
# Output: src/helpers/esp32/WebConfigHtml.h
|
||||
# WEBCONFIG_HTML_LOADER[] - small bootstrap page (PROGMEM)
|
||||
# WEBCONFIG_HTML_GZ[] - complete gzipped page (PROGMEM)
|
||||
# WEBCONFIG_HTML_BR[] - complete Brotli-compressed page (PROGMEM)
|
||||
# WEBCONFIG_HTML_ETAG - quoted strong ETag for the loader page
|
||||
#
|
||||
# Runnable outside SCons to inspect exactly what gets shipped:
|
||||
# python3 scripts/generate_webconfig_html.py --emit /tmp/shipped.html
|
||||
# or served directly by the mock backend with its --minify flag.
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
@@ -32,43 +31,26 @@ except NameError:
|
||||
SOURCE = os.path.join("webui", "index.html")
|
||||
OUTPUT = os.path.join("src", "helpers", "esp32", "WebConfigHtml.h")
|
||||
# __file__ is not defined inside PIO/SCons-executed extra_scripts
|
||||
SCRIPT = os.path.join("scripts", "generate_webconfig_html.py")
|
||||
MINIFIER = os.path.join("scripts", "webconfig_minify.py")
|
||||
HASH_MARKER = "// build-inputs-sha256: "
|
||||
HASH_MARKER = "// shipped-page-sha256: "
|
||||
|
||||
sys.path.insert(0, os.path.join(os.getcwd(), "scripts"))
|
||||
from webconfig_minify import check_stripped, strip_source # noqa: E402
|
||||
from brotli_compress import BROTLI_VERSION, LGWIN, QUALITY, compress_html # noqa: E402
|
||||
|
||||
|
||||
def status(msg):
|
||||
sys.stderr.write("WebConfig HTML: %s\n" % msg)
|
||||
|
||||
|
||||
def content_hash():
|
||||
# Hash the source page, this generator and the minifier so any change to
|
||||
# any of them forces a regenerate, independent of file timestamps.
|
||||
h = hashlib.sha256()
|
||||
with open(SOURCE, "rb") as f:
|
||||
h.update(f.read())
|
||||
for path in (SCRIPT, MINIFIER):
|
||||
if os.path.isfile(path):
|
||||
h.update(b"\0")
|
||||
with open(path, "rb") as f:
|
||||
h.update(f.read())
|
||||
return h.hexdigest()
|
||||
def content_hash(raw):
|
||||
"""Identify the bytes that the device actually serves.
|
||||
|
||||
|
||||
def stored_hash():
|
||||
try:
|
||||
with open(OUTPUT, "r") as f:
|
||||
for line in f:
|
||||
if line.startswith(HASH_MARKER):
|
||||
return line[len(HASH_MARKER):].strip()
|
||||
if line.startswith("#"): # reached the C preprocessor lines
|
||||
break
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
Do not hash this generator or the minifier source: a comment-only tooling
|
||||
edit must not dirty the checked-in firmware header. Generator changes are
|
||||
still safe because main() renders the candidate header and compares its
|
||||
full byte sequence before deciding whether to write.
|
||||
"""
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def shipped_page():
|
||||
@@ -104,16 +86,12 @@ def main():
|
||||
status("%s -> %s (%d -> %d bytes)" % (SOURCE, dest, len(src), len(stripped)))
|
||||
return
|
||||
|
||||
src_hash = content_hash()
|
||||
if os.path.isfile(OUTPUT) and stored_hash() == src_hash:
|
||||
return
|
||||
|
||||
src, stripped = shipped_page()
|
||||
raw = stripped.encode("utf-8")
|
||||
src_hash = content_hash(raw)
|
||||
|
||||
# mtime=0 keeps the gzip output and its immutable URL deterministic.
|
||||
gz = gzip.compress(raw, compresslevel=9, mtime=0)
|
||||
version = hashlib.sha256(gz).hexdigest()[:16]
|
||||
br = compress_html(raw)
|
||||
version = hashlib.sha256(br).hexdigest()[:16]
|
||||
loader = ("<!doctype html><meta charset=utf-8><meta name=viewport "
|
||||
"content=\"width=device-width,initial-scale=1\"><title>MeshCore "
|
||||
"WebConfig</title><body>Loading MeshCore WebConfig…<script>"
|
||||
@@ -133,27 +111,37 @@ def main():
|
||||
lines = []
|
||||
lines.append("// Auto-generated by scripts/generate_webconfig_html.py from %s" % SOURCE.replace(os.sep, "/"))
|
||||
lines.append("// DO NOT EDIT - edit webui/index.html instead.")
|
||||
lines.append("// Compressed with Google Brotli %s (quality %d, lgwin %d)." %
|
||||
(BROTLI_VERSION, QUALITY, LGWIN))
|
||||
lines.append("%s%s" % (HASH_MARKER, src_hash))
|
||||
lines.append("#pragma once")
|
||||
lines.append("#include <stdint.h>")
|
||||
lines.append("#include <pgmspace.h>")
|
||||
lines.append("")
|
||||
lines.append("const uint32_t WEBCONFIG_HTML_RAW_LEN = %d;" % len(raw))
|
||||
lines.append("const uint32_t WEBCONFIG_HTML_GZ_LEN = %d;" % len(gz))
|
||||
lines.append("const uint32_t WEBCONFIG_HTML_BR_LEN = %d;" % len(br))
|
||||
lines.append('const char WEBCONFIG_HTML_ETAG[] = "\\"%s\\"";' % etag)
|
||||
lines.append('const char WEBCONFIG_HTML_VERSION[] = "%s";' % version)
|
||||
lines.append("const uint16_t WEBCONFIG_HTML_LOADER_LEN = %d;" % len(loader_bytes))
|
||||
append_byte_array(lines, "WEBCONFIG_HTML_LOADER", loader_bytes)
|
||||
lines.append("")
|
||||
append_byte_array(lines, "WEBCONFIG_HTML_GZ", gz)
|
||||
append_byte_array(lines, "WEBCONFIG_HTML_BR", br)
|
||||
lines.append("")
|
||||
|
||||
os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
|
||||
with open(OUTPUT, "w") as f:
|
||||
f.write("\n".join(lines))
|
||||
rendered = "\n".join(lines)
|
||||
try:
|
||||
with open(OUTPUT, "r", encoding="utf-8", newline="") as f:
|
||||
if f.read() == rendered:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
status("%s -> %s (%d bytes source, %d stripped, %d gzip)"
|
||||
% (SOURCE, OUTPUT, len(src.encode("utf-8")), len(raw), len(gz)))
|
||||
os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
|
||||
with open(OUTPUT, "w", encoding="utf-8", newline="") as f:
|
||||
f.write(rendered)
|
||||
|
||||
status("%s -> %s (%d bytes source, %d stripped, %d Brotli)"
|
||||
% (SOURCE, OUTPUT, len(src.encode("utf-8")), len(raw), len(br)))
|
||||
|
||||
|
||||
# This script is attached only to esp32_base. Generate unconditionally here:
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Pinned Google Zopfli helpers for static and host-generated DEFLATE data.
|
||||
|
||||
Zopfli writes standards-compliant DEFLATE streams, so the existing device
|
||||
decoders need no additional firmware code or RAM. It is deliberately used
|
||||
only where compression happens on a development machine or host controller:
|
||||
firmware assets, test vectors, and BLE mOTA transport blocks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
ZOPFLI_VERSION = "0.4.3"
|
||||
|
||||
try:
|
||||
import zopfli
|
||||
from zopfli import gzip as _gzip
|
||||
from zopfli import zlib as _zlib
|
||||
except ImportError as error:
|
||||
raise RuntimeError(
|
||||
"Google Zopfli 0.4.3 is required to generate MeshCore compressed "
|
||||
"assets. Install it with 'python -m pip install -r "
|
||||
"requirements-build.txt'."
|
||||
) from error
|
||||
|
||||
if zopfli.__version__ != ZOPFLI_VERSION:
|
||||
raise RuntimeError(
|
||||
"MeshCore requires Google Zopfli %s, but %s is installed. Run "
|
||||
"'python -m pip install -r requirements-build.txt'."
|
||||
% (ZOPFLI_VERSION, zopfli.__version__)
|
||||
)
|
||||
|
||||
# Above zlib's maximum level 9: Zopfli refines the LZ77 model repeatedly.
|
||||
# Fifteen iterations is the upstream binding's practical default for assets
|
||||
# of this size and keeps CI/build times reasonable.
|
||||
ITERATIONS = 15
|
||||
|
||||
|
||||
def _options() -> dict[str, int | bool]:
|
||||
return {
|
||||
"numiterations": ITERATIONS,
|
||||
"blocksplitting": True,
|
||||
"blocksplittinglast": False,
|
||||
"blocksplittingmax": 15,
|
||||
}
|
||||
|
||||
|
||||
def gzip_compress(data: bytes) -> bytes:
|
||||
"""Compress *data* as deterministic RFC 1952 gzip using Google Zopfli."""
|
||||
encoded = bytearray(_gzip.compress(data, **_options()))
|
||||
# The gzip OS byte is informational. Normalize it to RFC 1952's unknown
|
||||
# value so a checked-in asset is identical across operating systems.
|
||||
if len(encoded) < 10 or encoded[:3] != b"\x1f\x8b\x08":
|
||||
raise RuntimeError("Zopfli returned an invalid gzip stream")
|
||||
encoded[9] = 0xFF
|
||||
return bytes(encoded)
|
||||
|
||||
|
||||
def raw_deflate_compress(data: bytes) -> bytes:
|
||||
"""Compress *data* to an RFC 1951 raw DEFLATE stream using Google Zopfli."""
|
||||
encoded = _zlib.compress(data, **_options())
|
||||
# A zlib wrapper is exactly a two-byte header and four-byte Adler-32
|
||||
# trailer. Removing it preserves the RFC 1951 body expected by tinf.
|
||||
if len(encoded) < 6 or (encoded[0] & 0x0F) != 8:
|
||||
raise RuntimeError("Zopfli returned an invalid zlib stream")
|
||||
return encoded[2:-4]
|
||||
Reference in new issue
Block a user