build(webconfig): strip comments before embedding the portal page

The generator gzipped webui/index.html verbatim, so the page's comments — and
this file is commented heavily by house style — were paying flash rent. A
line-based pass now drops comments, indentation and blank lines before
compressing. The source stays as readable as it was.

Conservative on purpose: only a comment that starts its own line is removed, so
a `//` inside a URL or a `/*` inside a regex can never be mistaken for one.
Line breaks survive, which leaves JS statement boundaries (and the space a
newline contributes between HTML inline elements) exactly as written.

This ships to thousands of devices, so it is not taken on trust:
  - check_stripped() fails the build if the page's structure changed or the
    output shrank implausibly
  - the pass lives in its own module, shared with the mock backend's new
    --minify flag, so the bytes exercised in a browser are the bytes that get
    embedded rather than a second implementation that could drift
  - webconfig_minify.py joins the generator in the freshness hash, so editing
    the stripper forces a regenerate

Today's page: 22,678 -> 17,671 bytes gzipped.

scripts/generate_webconfig_html.py

@@ -14,39 +14,55 @@
 # hashes two small files and returns, so running it from esp32_base on every
 # ESP32 build is negligible even for targets that don't compile the portal.
 #
+# Comments and indentation are stripped before compressing (see strip_source).
+# The source page is heavily commented by house style and none of it is worth
+# flash, so the page ships smaller than it reads.
+#
 # Output: src/helpers/esp32/WebConfigHtml.h
 #   WEBCONFIG_HTML_GZ[]   - gzipped page (PROGMEM)
 #   WEBCONFIG_HTML_GZ_LEN - byte length
 #   WEBCONFIG_HTML_ETAG   - quoted strong ETag (sha256 prefix of the gz body)
+#
+# Runnable outside SCons to inspect exactly what gets shipped:
+#   python3 scripts/generate_webconfig_html.py --emit /tmp/shipped.html
+# or served directly by the mock backend with its --minify flag.
 
 import gzip
 import hashlib
 import os
 import sys
 
-Import("env")  # noqa: F821
+try:
+    Import("env")  # noqa: F821
+except NameError:
+    pass           # running standalone (--emit), not as a PIO extra_script
 
 SOURCE = os.path.join("webui", "index.html")
 OUTPUT = os.path.join("src", "helpers", "esp32", "WebConfigHtml.h")
 # __file__ is not defined inside PIO/SCons-executed extra_scripts
 SCRIPT = os.path.join("scripts", "generate_webconfig_html.py")
+MINIFIER = os.path.join("scripts", "webconfig_minify.py")
 HASH_MARKER = "// build-inputs-sha256: "
 
+sys.path.insert(0, os.path.join(os.getcwd(), "scripts"))
+from webconfig_minify import check_stripped, strip_source  # noqa: E402
+
 
 def status(msg):
     sys.stderr.write("WebConfig HTML: %s\n" % msg)
 
 
 def content_hash():
-    # Hash the source page and this generator so any change to either forces a
-    # regenerate, independent of file timestamps.
+    # Hash the source page, this generator and the minifier so any change to
+    # any of them forces a regenerate, independent of file timestamps.
     h = hashlib.sha256()
     with open(SOURCE, "rb") as f:
         h.update(f.read())
-    if os.path.isfile(SCRIPT):
-        h.update(b"\0")
-        with open(SCRIPT, "rb") as f:
-            h.update(f.read())
+    for path in (SCRIPT, MINIFIER):
+        if os.path.isfile(path):
+            h.update(b"\0")
+            with open(path, "rb") as f:
+                h.update(f.read())
     return h.hexdigest()
 
 
@@ -63,17 +79,37 @@ def stored_hash():
     return None
 
 
+def shipped_page():
+    """The exact bytes the device serves: the source page, stripped."""
+    with open(SOURCE, "r", encoding="utf-8") as f:
+        raw = f.read()
+    stripped = strip_source(raw)
+    problem = check_stripped(raw, stripped)
+    if problem:
+        status("ERROR: comment stripping corrupted the page (%s)" % problem)
+        sys.exit(2)
+    return raw, stripped
+
+
 def main():
     if not os.path.isfile(SOURCE):
         status("ERROR: %s not found" % SOURCE)
         sys.exit(2)
 
+    if "--emit" in sys.argv:
+        dest = sys.argv[sys.argv.index("--emit") + 1]
+        src, stripped = shipped_page()
+        with open(dest, "w", encoding="utf-8") as f:
+            f.write(stripped)
+        status("%s -> %s (%d -> %d bytes)" % (SOURCE, dest, len(src), len(stripped)))
+        return
+
     src_hash = content_hash()
     if os.path.isfile(OUTPUT) and stored_hash() == src_hash:
         return
 
-    with open(SOURCE, "rb") as f:
-        raw = f.read()
+    src, stripped = shipped_page()
+    raw = stripped.encode("utf-8")
 
     # mtime=0 keeps the gzip output (and therefore the ETag) deterministic
     gz = gzip.compress(raw, compresslevel=9, mtime=0)
@@ -100,7 +136,8 @@ def main():
     with open(OUTPUT, "w") as f:
         f.write("\n".join(lines))
 
-    status("%s -> %s (%d bytes raw, %d bytes gzipped)" % (SOURCE, OUTPUT, len(raw), len(gz)))
+    status("%s -> %s (%d bytes source, %d stripped, %d gzipped)"
+           % (SOURCE, OUTPUT, len(src.encode("utf-8")), len(raw), len(gz)))
 
 
 main()

scripts/webconfig_minify.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""Comment/indentation stripping for webui/index.html.
+
+Shared by the build-time generator (which embeds the stripped page in flash)
+and the mock backend's --minify flag (which serves it), so what you test in a
+browser is byte-for-byte what the device ships. A second implementation would
+only drift.
+
+Stdlib only; imported from a PIO extra_script, so it must stay side-effect free.
+"""
+
+# Comment openers/closers per region of the page. The region is tracked so a
+# `<!--` inside <script> is never treated as a comment, and vice versa.
+_COMMENTS = {
+    "html": [("<!--", "-->")],
+    "css": [("/*", "*/")],
+    "js": [("/*", "*/")],
+}
+
+
+def strip_source(text):
+    """Drop comments, indentation and blank lines from the page.
+
+    Deliberately line-based and conservative. Only a comment that *starts* its
+    own line is removed, so a `//` inside a URL or a `/*` inside a regex is
+    never mistaken for one; trailing comments survive, which costs a little
+    flash and removes the entire class of "the minifier ate a string" bug.
+
+    Line breaks are preserved. That keeps JS statement boundaries exactly as
+    written (no ASI surprises) and keeps the single collapsed space a newline
+    contributes between HTML inline elements.
+    """
+    out, mode, closer = [], "html", None
+    for line in text.split("\n"):
+        s = line.strip()
+
+        if closer is not None:                    # inside a multi-line comment
+            at = s.find(closer)
+            if at < 0:
+                continue
+            s = s[at + len(closer):].strip()      # code may follow the close
+            closer = None
+
+        if not s:
+            continue
+
+        for opener, close in _COMMENTS[mode]:
+            if not s.startswith(opener):
+                continue
+            if s.endswith(close) and len(s) > len(opener) + len(close) - 1:
+                s = ""                            # the whole line is a comment
+            elif close not in s[len(opener):]:
+                closer = close                    # ... and it continues below
+                s = ""
+            break                                 # else code follows the close
+        if not s:                                 # on this line: leave it be
+            continue
+
+        if mode == "js" and s.startswith("//"):
+            continue
+
+        out.append(s)
+
+        low = s.lower()
+        if mode == "html" and "<style" in low:
+            mode = "css"
+        elif mode == "html" and "<script" in low:
+            mode = "js"
+        elif mode != "html" and ("</style>" in low or "</script>" in low):
+            mode = "html"
+
+    return "\n".join(out) + "\n"
+
+
+def check_stripped(raw, stripped):
+    """Guard against a stripper bug silently shipping a broken portal to the
+    fleet. Returns a reason string when the output looks wrong, else None."""
+    for tag in ("<script>", "</script>", "<style>", "</style>", "</body>"):
+        if raw.count(tag) != stripped.count(tag):
+            return "%s count changed" % tag
+    if len(stripped) < len(raw) * 0.5:
+        return "output shrank by more than half (%d -> %d)" % (len(raw), len(stripped))
+    # Only comments and whitespace may go, so no structural token may appear
+    # that the source did not already have.
+    for token in ("{", "}", "(", ")", "<script", "<style"):
+        if stripped.count(token) > raw.count(token):
+            return "gained a %s" % token
+    return None

scripts/webconfig_mock_server.py

@@ -33,6 +33,7 @@ import json
 import os
 import re
 import secrets
+import sys
 import threading
 import time
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
@@ -41,6 +42,13 @@ from urllib.parse import parse_qs, urlsplit
 HERE = os.path.dirname(os.path.abspath(__file__))
 INDEX_HTML = os.path.join(HERE, "..", "webui", "index.html")
 
+sys.path.insert(0, HERE)
+# The build-time comment stripper, shared so --minify serves byte-for-byte what
+# the generator embeds rather than a second implementation that could drift.
+from webconfig_minify import strip_source  # noqa: E402
+
+MINIFY = False
+
 SENTINEL = "********"
 ADMIN_PASSWORD = "password"          # matches the default ADMIN_PASSWORD build flag
 BATCH_PENDING_SECS = 0.8             # how long POST->done takes, to exercise polling
@@ -582,6 +590,12 @@ class Handler(BaseHTTPRequestHandler):
         except OSError:
             self.send_error(500, "webui/index.html not found")
             return
+        if MINIFY:
+            # Serve what the device actually serves. The generator strips
+            # comments and indentation before compressing, so --minify is how
+            # you exercise those bytes in a browser rather than trusting that
+            # stripping a 100 KB page never changes its behaviour.
+            html = strip_source(html.decode("utf-8")).encode("utf-8")
         self.send_response(200)
         self.send_header("Content-Type", "text/html; charset=utf-8")
         self.send_header("Content-Length", str(len(html)))
@@ -790,17 +804,19 @@ class Handler(BaseHTTPRequestHandler):
 
 
 def main():
-    global ST, PORT
+    global ST, PORT, MINIFY
     ap = argparse.ArgumentParser(description="Mock WebConfig portal backend")
     ap.add_argument("--port", type=int, default=8080)
     ap.add_argument("--setup", action="store_true", help="first-boot setup wizard mode")
     ap.add_argument("--active-slots", type=int, default=5, help="server slots to expose (2 or 5)")
+    ap.add_argument("--minify", action="store_true",
+                    help="serve the comment-stripped page the device ships, not the source")
     args = ap.parse_args()
-    ST, PORT = State(args), args.port
+    ST, PORT, MINIFY = State(args), args.port, args.minify
 
     srv = ThreadingHTTPServer(("127.0.0.1", args.port), Handler)
     mode = "SETUP (wizard)" if args.setup else "LAN (login: %s)" % ADMIN_PASSWORD
-    print("WebConfig mock backend — %s" % mode)
+    print("WebConfig mock backend — %s%s" % (mode, "  [minified]" if MINIFY else ""))
     print("  open http://localhost:%d/   (Ctrl-C to stop)" % args.port)
     try:
         srv.serve_forever()