From 7a24cf950c3f869089134dfa2edbd909554c1f1c Mon Sep 17 00:00:00 2001 From: agessaman Date: Fri, 7 Aug 2026 22:35:00 -0700 Subject: [PATCH] build(webconfig): strip comments before embedding the portal page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generator gzipped webui/index.html verbatim, so the page's comments — and this file is commented heavily by house style — were paying flash rent. A line-based pass now drops comments, indentation and blank lines before compressing. The source stays as readable as it was. Conservative on purpose: only a comment that starts its own line is removed, so a `//` inside a URL or a `/*` inside a regex can never be mistaken for one. Line breaks survive, which leaves JS statement boundaries (and the space a newline contributes between HTML inline elements) exactly as written. This ships to thousands of devices, so it is not taken on trust: - check_stripped() fails the build if the page's structure changed or the output shrank implausibly - the pass lives in its own module, shared with the mock backend's new --minify flag, so the bytes exercised in a browser are the bytes that get embedded rather than a second implementation that could drift - webconfig_minify.py joins the generator in the freshness hash, so editing the stripper forces a regenerate Today's page: 22,678 -> 17,671 bytes gzipped. --- scripts/generate_webconfig_html.py | 57 +++++++++++++++---- scripts/webconfig_minify.py | 88 ++++++++++++++++++++++++++++++ scripts/webconfig_mock_server.py | 22 +++++++- 3 files changed, 154 insertions(+), 13 deletions(-) create mode 100644 scripts/webconfig_minify.py diff --git a/scripts/generate_webconfig_html.py b/scripts/generate_webconfig_html.py index 143687d32..18afaab3d 100644 --- a/scripts/generate_webconfig_html.py +++ b/scripts/generate_webconfig_html.py @@ -14,39 +14,55 @@ # hashes two small files and returns, so running it from esp32_base on every # ESP32 build is negligible even for targets that don't compile the portal. # +# Comments and indentation are stripped before compressing (see strip_source). +# The source page is heavily commented by house style and none of it is worth +# flash, so the page ships smaller than it reads. +# # Output: src/helpers/esp32/WebConfigHtml.h # WEBCONFIG_HTML_GZ[] - gzipped page (PROGMEM) # WEBCONFIG_HTML_GZ_LEN - byte length # WEBCONFIG_HTML_ETAG - quoted strong ETag (sha256 prefix of the gz body) +# +# Runnable outside SCons to inspect exactly what gets shipped: +# python3 scripts/generate_webconfig_html.py --emit /tmp/shipped.html +# or served directly by the mock backend with its --minify flag. import gzip import hashlib import os import sys -Import("env") # noqa: F821 +try: + Import("env") # noqa: F821 +except NameError: + pass # running standalone (--emit), not as a PIO extra_script SOURCE = os.path.join("webui", "index.html") OUTPUT = os.path.join("src", "helpers", "esp32", "WebConfigHtml.h") # __file__ is not defined inside PIO/SCons-executed extra_scripts SCRIPT = os.path.join("scripts", "generate_webconfig_html.py") +MINIFIER = os.path.join("scripts", "webconfig_minify.py") HASH_MARKER = "// build-inputs-sha256: " +sys.path.insert(0, os.path.join(os.getcwd(), "scripts")) +from webconfig_minify import check_stripped, strip_source # noqa: E402 + def status(msg): sys.stderr.write("WebConfig HTML: %s\n" % msg) def content_hash(): - # Hash the source page and this generator so any change to either forces a - # regenerate, independent of file timestamps. + # Hash the source page, this generator and the minifier so any change to + # any of them forces a regenerate, independent of file timestamps. h = hashlib.sha256() with open(SOURCE, "rb") as f: h.update(f.read()) - if os.path.isfile(SCRIPT): - h.update(b"\0") - with open(SCRIPT, "rb") as f: - h.update(f.read()) + for path in (SCRIPT, MINIFIER): + if os.path.isfile(path): + h.update(b"\0") + with open(path, "rb") as f: + h.update(f.read()) return h.hexdigest() @@ -63,17 +79,37 @@ def stored_hash(): return None +def shipped_page(): + """The exact bytes the device serves: the source page, stripped.""" + with open(SOURCE, "r", encoding="utf-8") as f: + raw = f.read() + stripped = strip_source(raw) + problem = check_stripped(raw, stripped) + if problem: + status("ERROR: comment stripping corrupted the page (%s)" % problem) + sys.exit(2) + return raw, stripped + + def main(): if not os.path.isfile(SOURCE): status("ERROR: %s not found" % SOURCE) sys.exit(2) + if "--emit" in sys.argv: + dest = sys.argv[sys.argv.index("--emit") + 1] + src, stripped = shipped_page() + with open(dest, "w", encoding="utf-8") as f: + f.write(stripped) + status("%s -> %s (%d -> %d bytes)" % (SOURCE, dest, len(src), len(stripped))) + return + src_hash = content_hash() if os.path.isfile(OUTPUT) and stored_hash() == src_hash: return - with open(SOURCE, "rb") as f: - raw = f.read() + src, stripped = shipped_page() + raw = stripped.encode("utf-8") # mtime=0 keeps the gzip output (and therefore the ETag) deterministic gz = gzip.compress(raw, compresslevel=9, mtime=0) @@ -100,7 +136,8 @@ def main(): with open(OUTPUT, "w") as f: f.write("\n".join(lines)) - status("%s -> %s (%d bytes raw, %d bytes gzipped)" % (SOURCE, OUTPUT, len(raw), len(gz))) + status("%s -> %s (%d bytes source, %d stripped, %d gzipped)" + % (SOURCE, OUTPUT, len(src.encode("utf-8")), len(raw), len(gz))) main() diff --git a/scripts/webconfig_minify.py b/scripts/webconfig_minify.py new file mode 100644 index 000000000..0f7e17464 --- /dev/null +++ b/scripts/webconfig_minify.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Comment/indentation stripping for webui/index.html. + +Shared by the build-time generator (which embeds the stripped page in flash) +and the mock backend's --minify flag (which serves it), so what you test in a +browser is byte-for-byte what the device ships. A second implementation would +only drift. + +Stdlib only; imported from a PIO extra_script, so it must stay side-effect free. +""" + +# Comment openers/closers per region of the page. The region is tracked so a +# `")], + "css": [("/*", "*/")], + "js": [("/*", "*/")], +} + + +def strip_source(text): + """Drop comments, indentation and blank lines from the page. + + Deliberately line-based and conservative. Only a comment that *starts* its + own line is removed, so a `//` inside a URL or a `/*` inside a regex is + never mistaken for one; trailing comments survive, which costs a little + flash and removes the entire class of "the minifier ate a string" bug. + + Line breaks are preserved. That keeps JS statement boundaries exactly as + written (no ASI surprises) and keeps the single collapsed space a newline + contributes between HTML inline elements. + """ + out, mode, closer = [], "html", None + for line in text.split("\n"): + s = line.strip() + + if closer is not None: # inside a multi-line comment + at = s.find(closer) + if at < 0: + continue + s = s[at + len(closer):].strip() # code may follow the close + closer = None + + if not s: + continue + + for opener, close in _COMMENTS[mode]: + if not s.startswith(opener): + continue + if s.endswith(close) and len(s) > len(opener) + len(close) - 1: + s = "" # the whole line is a comment + elif close not in s[len(opener):]: + closer = close # ... and it continues below + s = "" + break # else code follows the close + if not s: # on this line: leave it be + continue + + if mode == "js" and s.startswith("//"): + continue + + out.append(s) + + low = s.lower() + if mode == "html" and "" in low or "" in low): + mode = "html" + + return "\n".join(out) + "\n" + + +def check_stripped(raw, stripped): + """Guard against a stripper bug silently shipping a broken portal to the + fleet. Returns a reason string when the output looks wrong, else None.""" + for tag in ("", "", ""): + if raw.count(tag) != stripped.count(tag): + return "%s count changed" % tag + if len(stripped) < len(raw) * 0.5: + return "output shrank by more than half (%d -> %d)" % (len(raw), len(stripped)) + # Only comments and whitespace may go, so no structural token may appear + # that the source did not already have. + for token in ("{", "}", "(", ")", " raw.count(token): + return "gained a %s" % token + return None diff --git a/scripts/webconfig_mock_server.py b/scripts/webconfig_mock_server.py index cd63cf93b..e1b73d2e7 100644 --- a/scripts/webconfig_mock_server.py +++ b/scripts/webconfig_mock_server.py @@ -33,6 +33,7 @@ import json import os import re import secrets +import sys import threading import time from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -41,6 +42,13 @@ from urllib.parse import parse_qs, urlsplit HERE = os.path.dirname(os.path.abspath(__file__)) INDEX_HTML = os.path.join(HERE, "..", "webui", "index.html") +sys.path.insert(0, HERE) +# The build-time comment stripper, shared so --minify serves byte-for-byte what +# the generator embeds rather than a second implementation that could drift. +from webconfig_minify import strip_source # noqa: E402 + +MINIFY = False + SENTINEL = "********" ADMIN_PASSWORD = "password" # matches the default ADMIN_PASSWORD build flag BATCH_PENDING_SECS = 0.8 # how long POST->done takes, to exercise polling @@ -582,6 +590,12 @@ class Handler(BaseHTTPRequestHandler): except OSError: self.send_error(500, "webui/index.html not found") return + if MINIFY: + # Serve what the device actually serves. The generator strips + # comments and indentation before compressing, so --minify is how + # you exercise those bytes in a browser rather than trusting that + # stripping a 100 KB page never changes its behaviour. + html = strip_source(html.decode("utf-8")).encode("utf-8") self.send_response(200) self.send_header("Content-Type", "text/html; charset=utf-8") self.send_header("Content-Length", str(len(html))) @@ -790,17 +804,19 @@ class Handler(BaseHTTPRequestHandler): def main(): - global ST, PORT + global ST, PORT, MINIFY ap = argparse.ArgumentParser(description="Mock WebConfig portal backend") ap.add_argument("--port", type=int, default=8080) ap.add_argument("--setup", action="store_true", help="first-boot setup wizard mode") ap.add_argument("--active-slots", type=int, default=5, help="server slots to expose (2 or 5)") + ap.add_argument("--minify", action="store_true", + help="serve the comment-stripped page the device ships, not the source") args = ap.parse_args() - ST, PORT = State(args), args.port + ST, PORT, MINIFY = State(args), args.port, args.minify srv = ThreadingHTTPServer(("127.0.0.1", args.port), Handler) mode = "SETUP (wizard)" if args.setup else "LAN (login: %s)" % ADMIN_PASSWORD - print("WebConfig mock backend — %s" % mode) + print("WebConfig mock backend — %s%s" % (mode, " [minified]" if MINIFY else "")) print(" open http://localhost:%d/ (Ctrl-C to stop)" % args.port) try: srv.serve_forever()