Merge observer firmware branch into keymindCascade

Integrate compatible observer board, preset, WebConfig, display, CI, and policy updates. Preserve the checkout's newer MQTT binary preferences and WiFi bridge runtime where the forked implementations conflict. Keep ThinkNode observer networking WiFi-based until the Ethernet transport is wired into that runtime.

scripts/check_mqtt_preset_parity.py

@@ -1,13 +1,25 @@
 #!/usr/bin/env python3
-"""Compare MQTT built-in preset *names* between two MQTTPresets.h files.
+"""Compare the MQTT broker preset table between two MQTTPresets.h files.
 
-Only the first string field of each ``MQTT_PRESETS`` entry is compared (set
-equality, case-sensitive). URL, auth, CA, keepalive, and credentials are
-ignored so channel branches may diverge on config details without failing CI.
+With ``--exact`` (what CI uses) the two files must be byte-identical. The table
+is fleet state, not just a build detail: a slot's preset is stored in
+/mqtt.json by NAME, and firmware that does not know a name does not merely
+ignore it — MQTTPrefsSerializer repairs it to "none" and the repaired file is
+written back to flash. A node that rolls back from one channel to the other
+therefore loses that slot permanently. Byte equality also catches URL, CA and
+credential drift, which a name-only check passes silently even though the two
+channels are meant to dial the same brokers.
+
+Channel-specific behaviour belongs in MQTTPresetPolicy.h, which is not compared,
+so this file can stay identical while the channels differ elsewhere.
+
+Without ``--exact`` only the first string field of each ``MQTT_PRESETS`` entry
+is compared (set equality, case-sensitive). That is the older, weaker check,
+kept for ad-hoc use.
 
 Usage::
 
-    python3 scripts/check_mqtt_preset_parity.py FILE_A FILE_B \\
+    python3 scripts/check_mqtt_preset_parity.py FILE_A FILE_B --exact \\
         --label-a observer-firmware --label-b observer-firmware-dev
 
     python3 scripts/check_mqtt_preset_parity.py --self-test
@@ -138,6 +150,49 @@ def compare(
     ]
 
 
+def compare_exact(
+    path_a: Path,
+    path_b: Path,
+    *,
+    label_a: str,
+    label_b: str,
+) -> list[str]:
+    """Return error lines if the two files differ byte for byte."""
+
+    data_a = path_a.read_bytes()
+    data_b = path_b.read_bytes()
+    if data_a == data_b:
+        return []
+
+    import difflib
+
+    errors = [f"{label_a} and {label_b} do not match byte for byte."]
+    try:
+        diff = list(
+            difflib.unified_diff(
+                data_a.decode("utf-8").splitlines(),
+                data_b.decode("utf-8").splitlines(),
+                fromfile=label_a,
+                tofile=label_b,
+                lineterm="",
+                n=1,
+            )
+        )
+    except UnicodeDecodeError:
+        errors.append("(binary difference; cannot render a text diff)")
+        return errors
+
+    # Enough to identify the drift without pasting the whole table into a log.
+    errors.extend(diff[:60])
+    if len(diff) > 60:
+        errors.append(f"... {len(diff) - 60} more diff line(s)")
+    errors.append(
+        "Preset rows must be identical on both channels. Behaviour that differs "
+        "per channel belongs in MQTTPresetPolicy.h."
+    )
+    return errors
+
+
 def load_names(path: Path) -> set[str]:
     text = path.read_text(encoding="utf-8")
     names, _ = parse_presets(text, source=str(path))
@@ -238,6 +293,26 @@ static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = {{
             print("self-test failed: URL scheme was treated as a comment.", file=sys.stderr)
             return 1
 
+        # --exact: identical bytes pass, any drift fails — including a change
+        # that a name-only comparison waves through.
+        if compare_exact(equal_a, equal_a, label_a="a", label_b="a-copy"):
+            print("self-test failed: a file was not equal to itself.", file=sys.stderr)
+            return 1
+        if compare(load_names(equal_a), load_names(equal_b), label_a="a", label_b="b"):
+            print("self-test failed: fixture assumption broken.", file=sys.stderr)
+            return 1
+        exact_errs = compare_exact(equal_a, equal_b, label_a="a", label_b="b")
+        if not exact_errs or "byte for byte" not in exact_errs[0]:
+            print(
+                "self-test failed: --exact accepted files that differ only in "
+                f"config: {exact_errs!r}",
+                file=sys.stderr,
+            )
+            return 1
+        if not any("MQTTPresetPolicy.h" in line for line in exact_errs):
+            print("self-test failed: --exact failure omitted the remedy.", file=sys.stderr)
+            return 1
+
         # Silence unused path in fixture layout.
         dupes.write_text("// unused\n", encoding="utf-8")
 
@@ -251,6 +326,11 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("file_b", type=Path, nargs="?", help="Second MQTTPresets.h (e.g. dev)")
     parser.add_argument("--label-a", default="file_a", help="Label for file_a in reports")
     parser.add_argument("--label-b", default="file_b", help="Label for file_b in reports")
+    parser.add_argument(
+        "--exact",
+        action="store_true",
+        help="Require the two files to be byte-identical (what CI enforces)",
+    )
     parser.add_argument("--self-test", action="store_true")
     args = parser.parse_args(argv)
 
@@ -260,6 +340,23 @@ def main(argv: list[str] | None = None) -> int:
     if args.file_a is None or args.file_b is None:
         parser.error("FILE_A and FILE_B are required unless --self-test is set")
 
+    if args.exact:
+        try:
+            errors = compare_exact(
+                args.file_a, args.file_b, label_a=args.label_a, label_b=args.label_b
+            )
+        except OSError as error:
+            print(f"MQTT preset parity check could not run: {error}", file=sys.stderr)
+            return 2
+        if errors:
+            print("MQTT preset parity check failed:", *errors, sep="\n  ", file=sys.stderr)
+            return 1
+        print(
+            f"MQTT preset parity check passed: {args.label_a} and {args.label_b} "
+            "have identical preset tables."
+        )
+        return 0
+
     try:
         names_a = load_names(args.file_a)
         names_b = load_names(args.file_b)

scripts/fetch_mbedtls_4k.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Fetch the reduced-TLS mbedTLS archives into .mbedtls-4k/<arch>/.
+#
+# These archives are built from the shipped sdkconfig plus three lines
+# (CONFIG_MBEDTLS_ASYMMETRIC_CONTENT_LEN=y, IN_CONTENT_LEN 16384,
+# OUT_CONTENT_LEN 4096) and save ~12 KiB of internal DRAM per TLS connection.
+# See docs/mbedtls-tls-footprint.md for the rationale and the build recipe.
+#
+# They are not committed: ~6 MB per architecture, and they must be rebuilt for
+# every platform bump, so they are published as a release asset keyed on the
+# espressif32 platform version instead.
+#
+#   scripts/fetch_mbedtls_4k.sh [arch]        # default: esp32s3
+#
+# Set MBEDTLS_4K_LOCAL to skip the download and copy from a local build tree:
+#   MBEDTLS_4K_LOCAL=~/mbedtls-4k-esp32s3/staged scripts/fetch_mbedtls_4k.sh
+set -euo pipefail
+
+ARCH="${1:-esp32s3}"
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+DEST="$REPO_ROOT/.mbedtls-4k/$ARCH"
+MANIFEST="$REPO_ROOT/scripts/mbedtls_4k_manifest.txt"
+BASE_URL="${MBEDTLS_4K_BASE_URL:-https://github.com/agessaman/MeshCore/releases/download/mbedtls-4k}"
+
+if [ ! -f "$MANIFEST" ]; then
+  echo "error: missing $MANIFEST" >&2
+  exit 1
+fi
+
+# Manifest lines: <arch> <sha256> <filename>. Blank lines and # comments ignored.
+# The "platform" and "stock:<arch>" lines bind the archives to a framework version;
+# they are the build check's business, not ours, and are not architectures.
+expected="$(awk -v a="$ARCH" '$1 == a && $0 !~ /^#/ {print $2"  "$3}' "$MANIFEST")"
+if [ -z "$expected" ]; then
+  echo "error: no manifest entries for arch '$ARCH'" >&2
+  echo "known arches: $(awk '$0 !~ /^#/ && NF && $1 != "platform" && $1 !~ /^stock:/ {print $1}' \
+    "$MANIFEST" | sort -u | tr '\n' ' ')" >&2
+  exit 1
+fi
+
+mkdir -p "$DEST"
+
+if [ -n "${MBEDTLS_4K_LOCAL:-}" ]; then
+  echo "copying from $MBEDTLS_4K_LOCAL"
+  while read -r _sha name; do
+    cp "$MBEDTLS_4K_LOCAL/$name" "$DEST/$name"
+  done <<< "$expected"
+else
+  TARBALL="mbedtls-4k-$ARCH.tar.gz"
+  echo "downloading $BASE_URL/$TARBALL"
+  tmp="$(mktemp -d)"
+  trap 'rm -rf "$tmp"' EXIT
+  curl -fsSL "$BASE_URL/$TARBALL" -o "$tmp/$TARBALL"
+  tar -xzf "$tmp/$TARBALL" -C "$tmp"
+  while read -r _sha name; do
+    # Accept the archive whether or not the tarball has a leading directory.
+    found="$(find "$tmp" -name "$name" -type f | head -1)"
+    if [ -z "$found" ]; then
+      echo "error: $name missing from $TARBALL" >&2
+      exit 1
+    fi
+    cp "$found" "$DEST/$name"
+  done <<< "$expected"
+fi
+
+# Verify every archive against the manifest. A wrong or truncated archive would
+# otherwise link silently and produce a firmware without the reduced buffers.
+cd "$DEST"
+if command -v shasum >/dev/null 2>&1; then
+  echo "$expected" | shasum -a 256 -c -
+else
+  echo "$expected" | sha256sum -c -
+fi
+
+echo "ok: $ARCH archives verified in $DEST"

scripts/mbedtls_4k.py

@@ -0,0 +1,244 @@
+"""Link the reduced-TLS mbedTLS archives, and prove they were actually linked.
+
+Opt in per build with MESHCORE_REDUCED_TLS=1. Off by default, so an ordinary build
+needs no 6 MB artifact and behaves exactly as before.
+
+    MESHCORE_REDUCED_TLS=1 pio run -e Heltec_v3_repeater_observer_mqtt
+
+The archives lower the mbedTLS outbound record buffer from 16 KiB to 4 KiB, saving
+~12 KiB of internal DRAM per TLS connection on non-PSRAM observers. The inbound
+buffer stays at 16 KiB, so the contiguous allocation a handshake needs is unchanged
+— this buys headroom, it does not move that floor. See docs/mbedtls-tls-footprint.md.
+
+Three failure modes this guards against, each of which produces a firmware that looks
+fine and is silently wrong:
+
+  - a -L pointing at a missing or partial directory. The linker ignores an
+    unusable search path and quietly resolves mbedTLS from the framework instead.
+  - archives that do not match the manifest, e.g. left over from an earlier
+    platform version.
+  - a manifest and archives that agree with each other but not with the installed
+    framework, which is what a platform bump without a rebuild leaves behind. That
+    one links cleanly and drifts on struct layout at runtime.
+
+So the opt-in path verifies every archive by sha256 before the build, checks the
+framework's own mbedTLS archives still fingerprint as the ones these were built
+against, and after the link re-reads firmware.map to confirm every libmbed*.a came
+from our directory.
+"""
+Import("env")
+
+import hashlib
+import os
+import sys
+
+REQUIRED = ("libmbedcrypto.a", "libmbedtls_2.a", "libmbedtls.a", "libmbedx509.a")
+
+
+def _fail(msg):
+    print("\n*** reduced-TLS build failed ***", file=sys.stderr)
+    print(msg, file=sys.stderr)
+    print(
+        "\nFetch the archives with:  scripts/fetch_mbedtls_4k.sh <arch>"
+        "\nOr build without them by unsetting MESHCORE_REDUCED_TLS.",
+        file=sys.stderr,
+    )
+    env.Exit(1)
+
+
+def _sha256(path):
+    h = hashlib.sha256()
+    with open(path, "rb") as fh:
+        for chunk in iter(lambda: fh.read(1 << 20), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _manifest(project_dir, arch):
+    path = os.path.join(project_dir, "scripts", "mbedtls_4k_manifest.txt")
+    if not os.path.isfile(path):
+        _fail("missing scripts/mbedtls_4k_manifest.txt")
+    wanted = {}
+    stock = {}
+    platform_id = ""
+    with open(path) as fh:
+        for line in fh:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            parts = line.split()
+            if len(parts) == 2 and parts[0] == "platform":
+                platform_id = parts[1]
+            elif len(parts) == 3 and parts[0] == "stock:" + arch:
+                stock[parts[2]] = parts[1]
+            elif len(parts) == 3 and parts[0] == arch:
+                wanted[parts[2]] = parts[1]
+    return wanted, stock, platform_id
+
+
+# Where each espressif32 generation keeps the archives we displace. First directory
+# holding all four wins, so this resolves without knowing which platform is in play.
+FRAMEWORK_LIB_DIRS = (
+    ("framework-arduinoespressif32", "tools/sdk/%s/lib"),
+    ("framework-arduinoespressif32-libs", "%s/lib"),
+    ("framework-arduinoespressif32", "tools/esp32-arduino-libs/%s/lib"),
+)
+
+
+def _framework_lib_dir(platform, arch):
+    for package, layout in FRAMEWORK_LIB_DIRS:
+        try:
+            base = platform.get_package_dir(package)
+        except Exception:
+            base = None
+        if not base:
+            continue
+        path = os.path.join(base, *(layout % arch).split("/"))
+        if all(os.path.isfile(os.path.join(path, name)) for name in REQUIRED):
+            return path
+    return None
+
+
+if os.environ.get("MESHCORE_REDUCED_TLS", "") not in ("1", "true", "yes"):
+    Return()
+
+project_dir = env.subst("$PROJECT_DIR")
+arch = env.BoardConfig().get("build.mcu", "")
+if not arch:
+    _fail("could not determine board MCU, so cannot pick an archive set")
+
+staged = os.path.join(project_dir, ".mbedtls-4k", arch)
+if not os.path.isdir(staged):
+    _fail("no archives for %s at %s" % (arch, staged))
+
+wanted, stock, manifest_platform = _manifest(project_dir, arch)
+if not wanted:
+    _fail("manifest has no entries for arch '%s'" % arch)
+
+for name in REQUIRED:
+    archive = os.path.join(staged, name)
+    if not os.path.isfile(archive):
+        _fail("missing %s" % archive)
+    if name not in wanted:
+        _fail("%s is not in the manifest for %s" % (name, arch))
+    actual = _sha256(archive)
+    if actual != wanted[name]:
+        _fail(
+            "%s does not match the manifest\n  expected %s\n  actual   %s\n"
+            "Rebuild it for this platform version, or re-run the fetch script."
+            % (archive, wanted[name], actual)
+        )
+
+# Bind the staged archives to the framework they were built against. The hashes above
+# only prove the staged files are the ones the manifest names; they say nothing about
+# whether the manifest is still current. Bump the platform without rebuilding, and
+# every check above still passes while the link takes mbedTLS built against a
+# different IDF — a struct-layout drift that corrupts silently at runtime. So
+# fingerprint the framework's own archives, the ones being displaced: if those moved,
+# the staged pair is stale by construction.
+platform = env.PioPlatform()
+platform_id = "%s@%s" % (platform.name, platform.version)
+stock_dir = _framework_lib_dir(platform, arch)
+if stock_dir is None:
+    _fail("cannot locate the framework's own mbedTLS archives for %s, so the staged "
+          "ones cannot be tied to a framework version" % arch)
+if not stock:
+    _fail("manifest has no stock:%s fingerprints — it predates the framework binding.\n"
+          "Add these lines for the framework now installed (%s):\n%s"
+          % (arch, platform_id,
+             "\n".join("stock:%s %s %s" % (arch, _sha256(os.path.join(stock_dir, n)), n)
+                       for n in REQUIRED)))
+
+for name in REQUIRED:
+    actual = _sha256(os.path.join(stock_dir, name))
+    if name not in stock:
+        _fail("manifest has no stock:%s entry for %s" % (arch, name))
+    if actual != stock[name]:
+        _fail(
+            "the framework's mbedTLS archives are not the ones these were built against.\n"
+            "  %s\n  manifest %s\n  installed %s\n"
+            "Manifest records %s; installed is %s.\n"
+            "Rebuild the reduced-TLS archives against this framework "
+            "(docs/mbedtls-tls-footprint.md), then update every hash in the manifest."
+            % (os.path.join(stock_dir, name), stock[name], actual,
+               manifest_platform or "no platform", platform_id)
+        )
+
+if manifest_platform and manifest_platform != platform_id:
+    # Hashes are the check; the version string is orientation. Identical archives
+    # under a renamed platform are not a compatibility problem.
+    print("reduced-TLS: manifest says %s, installed is %s — archives match, so this is "
+          "only a stale label" % (manifest_platform, platform_id))
+
+# Prepend so these satisfy mbedTLS symbols ahead of the framework's own copies:
+# the linker takes each archive member from the first archive that resolves it.
+env.Prepend(LIBPATH=[staged])
+print("reduced-TLS: linking mbedTLS from %s (verified)" % staged)
+
+
+def _verify_map(source, target, env):
+    """Confirm every mbedTLS archive in the link came from our directory.
+
+    Fails closed. Anything that stops this from *proving* the link — no map, an
+    unparsable map, a short archive list — is a failure, not a warning. A warning
+    here would leave exactly the hole the check exists to close: an opt-in build
+    that succeeds while silently linking the framework's 16 KiB buffers.
+    """
+    # Derive the map name from PROGNAME rather than hardcoding firmware.map, so a
+    # renamed program cannot leave us inspecting a stale or absent file.
+    map_path = os.path.join(env.subst("$BUILD_DIR"),
+                            env.subst("${PROGNAME}") + ".map")
+    if not os.path.isfile(map_path):
+        legacy = os.path.join(env.subst("$BUILD_DIR"), "firmware.map")
+        map_path = legacy if os.path.isfile(legacy) else map_path
+    if not os.path.isfile(map_path):
+        print("\n*** reduced-TLS: no linker map at %s ***" % map_path,
+              file=sys.stderr)
+        print("Cannot prove the reduced-TLS archives were linked. Ensure the env "
+              "emits a map (-Wl,-Map), or unset MESHCORE_REDUCED_TLS.",
+              file=sys.stderr)
+        env.Exit(1)
+        return
+    # The map records whatever the linker was given, which for a -L hit is a path
+    # relative to the linker's cwd (the project dir). Resolve before comparing, or
+    # every one of our own archives reads as stray.
+    staged_real = os.path.realpath(staged)
+    stray = set()
+    seen = set()
+    with open(map_path, errors="replace") as fh:
+        for line in fh:
+            for token in line.split():
+                if "libmbed" not in token or ".a" not in token:
+                    continue
+                path = token.split("(")[0]
+                base = os.path.basename(path)
+                if not base.startswith("libmbed") or not base.endswith(".a"):
+                    continue
+                seen.add(base)
+                resolved = os.path.realpath(os.path.join(project_dir, path))
+                if os.path.dirname(resolved) != staged_real:
+                    stray.add(path)
+    if stray:
+        print("\n*** reduced-TLS: archives linked from the WRONG place ***",
+              file=sys.stderr)
+        for path in sorted(stray):
+            print("  " + path, file=sys.stderr)
+        env.Exit(1)
+        return
+    # Every required archive must appear. Seeing only some of them means the rest
+    # resolved somewhere this parse did not recognise, which is not proof of anything.
+    missing = [name for name in REQUIRED if name not in seen]
+    if missing:
+        print("\n*** reduced-TLS: %s names only %d of %d archives ***"
+              % (os.path.basename(map_path), len(seen), len(REQUIRED)),
+              file=sys.stderr)
+        print("  missing: " + ", ".join(missing), file=sys.stderr)
+        print("Either the map format changed or mbedTLS was resolved elsewhere; "
+              "the reduced buffers cannot be assumed.", file=sys.stderr)
+        env.Exit(1)
+        return
+    print("reduced-TLS: confirmed all %d archives linked from %s"
+          % (len(REQUIRED), staged))
+
+
+env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", _verify_map)

scripts/mbedtls_4k_manifest.txt

@@ -0,0 +1,23 @@
+# Reduced-TLS mbedTLS archives (OUT_CONTENT_LEN 4096, IN_CONTENT_LEN 16384).
+# Format: <arch> <sha256> <filename>
+#
+# Rebuild these for every espressif32 platform bump — the archives must match the
+# rest of the framework they link against. Built as of platformio/espressif32@6.11.0
+# from the recipe in docs/mbedtls-tls-footprint.md.
+#
+# Fetch with: scripts/fetch_mbedtls_4k.sh esp32s3
+esp32s3 01629f635b33ffa2c1fdfcd8ac52327cd3a92e8d4e7c9a32d92974e0cfdfe398 libmbedcrypto.a
+esp32s3 07e7a09847589fefc35bc8d2f739d556535d535acd7185a9824b2af8edd5b05a libmbedtls_2.a
+esp32s3 e12bcc8d76a368e819987f266e73c265178d6b6675f05d0e29014bb543c402af libmbedtls.a
+esp32s3 c1e10324e19f6d7763737f72d9032cec832bdfce60c7a7d381a5fcfc7dcad573 libmbedx509.a
+
+# Framework binding, read by scripts/mbedtls_4k.py and ignored by the fetch script.
+# "stock:" lines fingerprint the framework's own archives — the ones the entries above
+# displace. They are what makes "rebuild on every platform bump" enforceable instead of
+# advisory: if the framework's copies move, the entries above are stale by construction
+# and the build stops. Regenerate both sets together, never one alone.
+platform espressif32@6.11.0
+stock:esp32s3 abdaf759ee17aa697468427b55a86c6c0082ac4cdeb643a63d8b3ac2df324633 libmbedcrypto.a
+stock:esp32s3 9a580d2ff7c885e1bf59479a14422e3d10a485ae9083acb5db943df714c0ac35 libmbedtls_2.a
+stock:esp32s3 a0f331245feb5cf4fe8d9f99b03d34d64f10e38d8a8d680e35b34ec82aa3cf9e libmbedtls.a
+stock:esp32s3 3d56277224b066118b6c48a973ae18fdf8078448bf54b482ac5557699e52f223 libmbedx509.a

scripts/webconfig_cli_audit.py

@@ -12,9 +12,17 @@ that come back an error, so the two stay honest about each other.
     python3 scripts/webconfig_mock_server.py --port 8137 &
     python3 scripts/webconfig_cli_audit.py
 
+Board-specific commands (Board::handleCommand) are offered only when the node
+says it answers them, so one run exercises one shape of board. The mock claims
+none by default, like a Heltec V3; drive the other shapes with --board-cmds:
+
+    python3 scripts/webconfig_mock_server.py --port 8137 \
+        --board-cmds radio.fem.rxgain,radio.fem.txgain,fan &
+
 Exits non-zero if anything fails that is not in EXPECTED_FAILURES. Stdlib only.
 """
 
+import glob
 import json
 import os
 import re
@@ -30,12 +38,7 @@ INDEX_HTML = os.path.join(HERE, "..", "webui", "index.html")
 
 # Errors that are the correct answer, not a gap.
 EXPECTED_FAILURES = {
-    # Runtime-gated on the real device by Board::canControlLoRaFemLna(); the
-    # command exists in every build and the board answers for itself. The mock
-    # board is a Heltec V3, which has no front-end module.
-    "get radio.fem.rxgain": "unsupported",
-    "set radio.fem.rxgain on": "unsupported",
-    # Guarded by the firmware the same way when no alert PSK is configured.
+    # Guarded by the firmware when no alert PSK is configured.
     "alert test": "not configured",
 }
 
@@ -45,8 +48,11 @@ SKIP = {"reboot", "clkreboot", "poweroff", "shutdown", "erase", "start ota",
         "log", "get acl"}  # Listings use /api/terminal, not this bounded API audit.
 
 
-def table():
-    """The commands autocomplete offers, read straight out of the page."""
+def table(board_cmds=()):
+    """The commands autocomplete offers, read straight out of the page.
+
+    `board_cmds` is what /api/status said this board answers; the CLI_BOARD_KEYS
+    entries gated on anything else are not offered and so not driven."""
     html = open(INDEX_HTML, encoding="utf-8").read()
 
     def section(start, end):
@@ -54,15 +60,25 @@ def table():
 
     verbs = re.findall(r'\["([^"]+)","', section("var CLI_VERBS=", "var CLI_KEYS="))
     keys = re.findall(r'\["([^"]+)","(?:[^"\\]|\\.)*",(\d)',
-                      section("var CLI_KEYS=", "var CLI_SLOT="))
+                      section("var CLI_KEYS=", "var CLI_BOARD_KEYS="))
     fields = re.findall(r'\["(\w+)","', section("var CLI_SLOT=", "var CLI_TYPES="))
 
     gets = ["get " + k for k, mode in keys if mode != "2"]
     gets += ["get mqtt%d.%s" % (n, f) for n in (1, 3) for f in fields]
     # Verbs taking an argument need a value the node will accept; those are
     # covered by the round-trip probes below rather than guessed at here.
+    gets += ["get " + k for gate, k, mode in board_table()
+             if mode != "2" and gate in board_cmds]
     plain = [v for v in verbs if not v.endswith(" ") and v not in SKIP]
-    return gets + plain
+    return [command for command in gets if command not in SKIP] + plain
+
+
+def board_table():
+    """[(gate, key, mode)] from CLI_BOARD_KEYS — the keys the page offers only
+    when /api/status says this board answers for their gate."""
+    html = open(INDEX_HTML, encoding="utf-8").read()
+    section = html[html.index("var CLI_BOARD_KEYS="):html.index("// Per-slot keys")]
+    return re.findall(r'\["([^"]+)","([^"]+)","(?:[^"\\]|\\.)*",(\d)', section)
 
 
 # Where top-level commands are implemented. MyMesh handles a few before
@@ -74,15 +90,74 @@ COMMAND_SOURCES = [
     "examples/simple_repeater/MyMesh.cpp",
 ]
 
-# Firmware commands the table deliberately does not offer. Everything below
-# except tls.bundletest is also rejected by /api/cli (wcCliUnavailable), so the
-# portal never pretends to run something it cannot.
+# Board::handleCommand() is dispatched BEFORE CommonCLI's own table, and the FEM
+# commands now live there rather than in CommonCLI. A board file contributes
+# whole commands ("get radio.fem.rxgain"), not the bare verbs the sources above
+# yield, so the two are collected separately and merged.
+BOARD_SOURCES = "variants/*/*Board.cpp"
+
+# Firmware commands the table deliberately does not offer.
 NOT_OFFERED = {
     "tls.bundletest",  # TLS debugging, not an operator command
     "start ota",       # binds port 80, which the portal is already using
+    "stop ota",        # nothing to stop: `start ota` cannot run from here
     "clock sync",      # takes its time from the caller; a web request has none
+    "get acl",         # requires the streaming terminal endpoint
 }
 
+# Of those, the ones /api/cli does NOT reject at POST (wcCliUnavailable). They
+# are left out of the table rather than blocked, because running them is
+# harmless — `stop ota` just reports that no OTA server is running, which is
+# always true here. Everything else in NOT_OFFERED must come back a 400 with a
+# reason, so the portal never pretends to run something it cannot.
+NOT_REFUSED = {"tls.bundletest", "stop ota"}
+
+# The local firmware has additional management and diagnostics commands. They
+# remain available for manual entry in the terminal; its compact autocomplete
+# table intentionally omits them. Keep the board-command parity check focused
+# on controls the page actually offers.
+ADVANCED_UNLISTED = {
+    "clear recent.repeater", "clock.sync.mesh now",
+    "get battery.alert", "get battery.alert.critical",
+    "get battery.alert.low", "get battery.alert.region", "get clock.sync",
+    "get clock.sync.drift", "get clock.sync.internet", "get clock.sync.mesh",
+    "get clock.sync.mesh.edge", "get clock.sync.samples", "get flood.channel.data",
+    "get flood.channel.data.hops", "get host", "get outpath path",
+    "get recent.repeater", "get recent.repeaters", "get rx.watchdog",
+    "get tempradio", "powerlog", "send text.flood", "set battery.alert",
+    "set battery.alert.critical", "set battery.alert.low", "set clock.sync.drift",
+    "set clock.sync.internet", "set clock.sync.mesh", "set clock.sync.mesh.edge",
+    "set clock.sync.samples", "set outpath path", "set rx.watchdog",
+    "set tempradio",
+}
+
+
+def webconfig_variants():
+    """Variant directories whose build serves the portal (ESP32 + MQTT bridge).
+
+    Only these boards can put a command in front of this page; a board command
+    on an nRF52 variant is real but unreachable from here, so it is not a gap.
+    """
+    out = set()
+    for ini in glob.glob(os.path.join(HERE, "..", "variants", "*", "platformio.ini")):
+        if "WITH_MQTT_BRIDGE" in open(ini, encoding="utf-8").read():
+            out.add(os.path.basename(os.path.dirname(ini)))
+    return out
+
+
+def board_commands():
+    """Whole commands Board::handleCommand() answers, across portal variants."""
+    variants = webconfig_variants()
+    found = set()
+    for path in glob.glob(os.path.join(HERE, "..", *BOARD_SOURCES.split("/"))):
+        if os.path.basename(os.path.dirname(path)) not in variants:
+            continue
+        src = open(path, encoding="utf-8").read()
+        body = src[src.find("::handleCommand"):]
+        for lit in re.findall(r'(?:mem|str)n?cmp\(\s*command\s*,\s*"([^"]+)"', body):
+            found.add(lit.strip())
+    return found
+
 
 def firmware_commands():
     """Top-level command literals the firmware dispatches on."""
@@ -95,7 +170,7 @@ def firmware_commands():
             continue
         for lit in re.findall(r'(?:mem|str)n?cmp\(\s*command\s*,\s*"([^"]+)"', src):
             found.add(lit.strip())
-    return found - NOT_OFFERED
+    return (found | board_commands()) - NOT_OFFERED - ADVANCED_UNLISTED
 ROUND_TRIPS = [
     ("set mqtt1.port 0", "get mqtt1.port", "0"),
     ("set usb.logging on", "get usb.logging", "on"),
@@ -126,6 +201,29 @@ ROUND_TRIPS = [
 ]
 
 
+# Gated on /api/status's board_cmds, keyed by the gate the page probes for.
+BOARD_ROUND_TRIPS = {
+    "radio.fem.rxgain": [("set radio.fem.rxgain off", "get radio.fem.rxgain", "off")],
+    "radio.fem.txgain": [("set radio.fem.txgain on", "get radio.fem.txgain", "on")],
+    "fan": [("set fan on", "get fan", "on 41.0C fan=on cd=0s")],
+}
+
+# Board commands with no getter to read back, so a round-trip cannot reach them.
+# Run in order, and drive the rejection path too: a set-only key that silently
+# accepted anything would look identical to one that works.
+# [(command, should_succeed, expected substring of the reply)]
+BOARD_SET_PROBES = {
+    "fan": [
+        ("set fan.lo 40", True, "OK"),
+        ("set fan.hi 70", True, "OK"),
+        ("set fan.lo 200", False, "0..100"),          # out of range
+        ("set fan.lo 90", False, "< fan.hi"),         # crosses the upper bound
+        ("set fan.hi 10", False, "> fan.lo"),         # crosses the lower bound
+        ("set fan.hi 130", False, "<= 120"),          # out of range
+    ],
+}
+
+
 class Client:
     def __init__(self, base):
         self.base = base
@@ -133,7 +231,19 @@ class Client:
         self.cookie = r.headers["Set-Cookie"].split(";")[0]
         # The node caps a sequence at MAX_BATCH and reports it; chunk to match
         # rather than hardcoding a number that drifts when the slot is resized.
-        self.max_cmds = json.load(self._open("/api/status")).get("max_cmds", 24)
+        status = json.load(self._open("/api/status"))
+        self.max_cmds = status.get("max_cmds", 24)
+        # Board::handleCommand() commands this node probed for at startup. The
+        # field is absent until the probe has run, which is NOT the same as an
+        # empty list, so wait for it rather than audit a board half-known.
+        for _ in range(20):
+            if "board_cmds" in status:
+                break
+            time.sleep(0.25)
+            status = json.load(self._open("/api/status"))
+        else:
+            sys.exit("node never reported board_cmds; it has not probed its board")
+        self.board_cmds = [c for c in status["board_cmds"].split(",") if c]
 
     def _open(self, path, data=None):
         headers = {"Content-Type": "application/json"}
@@ -186,7 +296,7 @@ def main():
 
     failures = []
 
-    cmds = table()
+    cmds = table(cli.board_cmds)
     unexpected = []
     for cmd, res in cli.run(cmds):
         if res["ok"]:
@@ -196,6 +306,7 @@ def main():
             continue
         unexpected.append((cmd, res["reply"]))
     print("commands offered by autocomplete : %d" % len(cmds))
+    print("board commands this node answers : %s" % (", ".join(cli.board_cmds) or "none"))
     print("answered                         : %d" % (len(cmds) - len(unexpected)))
     print("sequence cap reported by the node: %d" % cli.max_cmds)
     for cmd, reply in unexpected:
@@ -205,17 +316,39 @@ def main():
     # The reverse direction: a command the firmware implements but the table
     # never offers is invisible to the check above, because the check only ever
     # drives what the table already knows about.
-    offered = " ".join(cmds) + " " + " ".join(
-        re.findall(r'\["([^"]+)","', open(INDEX_HTML, encoding="utf-8").read()))
+    # A board command is a whole `get`/`set` line, so the get-only list the audit
+    # drives cannot decide it is offered: `set fan.lo` has no `get` counterpart in
+    # the table at all, and a board key this mock does not claim is absent from
+    # `cmds` while still being offerable. Both are expanded from the page here.
+    html = open(INDEX_HTML, encoding="utf-8").read()
+    keys = re.findall(r'\["([^"]+)","(?:[^"\\]|\\.)*",(\d)',
+                      html[html.index("var CLI_KEYS="):html.index("var CLI_BOARD_KEYS=")])
+    offered = " ".join(cmds) + " " \
+        + " ".join("set " + k for k, mode in keys if mode != "1") + " " \
+        + " ".join("get %s set %s" % (k, k) for _, k, _ in board_table()) + " " \
+        + " ".join(re.findall(r'\["([^"]+)","', html))
     missing = sorted(c for c in firmware_commands() if c not in offered)
     print("\nfirmware commands not in the table: %d" % len(missing))
     for c in missing:
         print("   MISSING  %s" % c)
     failures += [(c, "not offered by autocomplete") for c in missing]
 
-    results = cli.run([c for probe in ROUND_TRIPS for c in probe[:2]])
-    print("\nround-trips                      : %d" % len(ROUND_TRIPS))
-    for i, (setc, getc, want) in enumerate(ROUND_TRIPS):
+    # The page asks the node for each gate by name (`get <gate>`), so a gate that
+    # no board getter answers would hide its keys on every board, silently.
+    fw = firmware_commands()
+    gates = sorted({gate for gate, _, _ in board_table()})
+    bad_gates = [g for g in gates if "get " + g not in fw]
+    print("\nboard-command gates                : %d" % len(gates))
+    for g in bad_gates:
+        print("   FAIL  %-30s no board answers `get %s`" % (g, g))
+    failures += [(g, "gate has no getter") for g in bad_gates]
+
+    probes = list(ROUND_TRIPS)
+    for gate in cli.board_cmds:
+        probes += BOARD_ROUND_TRIPS.get(gate, [])
+    results = cli.run([c for probe in probes for c in probe[:2]])
+    print("\nround-trips                      : %d" % len(probes))
+    for i, (setc, getc, want) in enumerate(probes):
         setr, getr = results[i * 2][1], results[i * 2 + 1][1]
         # `get` answers "> value"; compare the value, as the terminal displays it
         got = re.sub(r"^>\s?", "", getr["reply"])
@@ -225,10 +358,20 @@ def main():
               % (getc, got, want, setr["reply"]))
         failures.append((getc, got))
 
+    # Set-only board commands, which no round-trip can reach.
+    probes = [p for gate in cli.board_cmds for p in BOARD_SET_PROBES.get(gate, [])]
+    if probes:
+        print("\nset-only board commands          : %d" % len(probes))
+        for (cmd, want_ok, want), (_, res) in zip(probes, cli.run([p[0] for p in probes])):
+            if res["ok"] == want_ok and want in res["reply"]:
+                continue
+            print("   FAIL  %-30s %s" % (cmd, res["reply"]))
+            failures.append((cmd, res["reply"]))
+
     # Commands the portal refuses must be refused clearly, not run and fudged.
     print("\nrefused with a reason              : ", end="")
     refused = []
-    for cmd in sorted(NOT_OFFERED - {"tls.bundletest"}):
+    for cmd in sorted(NOT_OFFERED - NOT_REFUSED):
         try:
             cli._sequence([cmd])
             refused.append((cmd, "was accepted, expected a 400"))
@@ -236,7 +379,8 @@ def main():
             body = json.load(e) if e.code == 400 else {}
             if e.code != 400 or not body.get("error"):
                 refused.append((cmd, "HTTP %d, expected 400 with a reason" % e.code))
-    print("%d/%d" % (len(NOT_OFFERED) - 1 - len(refused), len(NOT_OFFERED) - 1))
+    checked = len(NOT_OFFERED) - len(NOT_REFUSED)
+    print("%d/%d" % (checked - len(refused), checked))
     for cmd, why in refused:
         print("   FAIL  %-30s %s" % (cmd, why))
     failures += refused

scripts/webconfig_mock_server.py

@@ -103,6 +103,7 @@ def default_config(setup_mode):
         "radio": {
             "freq": 910.525, "bw": 62.5, "sf": 7, "cr": 5, "tx": 22, "af": 1.0,
             "rxdelay": 0.0, "txdelay": 0.5, "cad": False, "rxgain": True,
+            "fem_rxgain": False, "fem_txgain": False,
             "rxps_enabled": True, "rxps_level": 8, "rxps_preamble": 16,
             "rxps_rx_us": 18205, "rxps_sleep_us": 20423,
             "powersaving": True,
@@ -157,6 +158,10 @@ class State:
         self.lock = threading.Lock()
         self.setup_mode = args.setup
         self.active_slots = args.active_slots
+        # Board::handleCommand() commands this "board" answers. Empty by default:
+        # the mock is a Heltec V3, which implements no such hook at all.
+        self.board_cmds = [c for c in args.board_cmds.split(",") if c]
+        self.probe_delay = args.probe_delay
         self.cfg = default_config(args.setup)
         # latched at AP start, like WebConfigServer::_initial_setup
         self.initial_setup = args.setup and self.cfg["wifi"]["ssid"] == ""
@@ -188,7 +193,7 @@ class State:
         return c
 
     def status_json(self, authed):
-        return {
+        out = {
             "mode": "setup" if self.setup_mode else "lan",
             "auth": authed,
             "needs_setup": self.cfg["wifi"]["ssid"] == "",
@@ -207,6 +212,12 @@ class State:
             "capabilities": 0x37FF,
             "max_cmds": CLI_MAX_CMDS,
         }
+        # ABSENT, not empty, until the board has been probed: the node's routes
+        # go live a tick before probeBoardCommands() runs, and the page has to
+        # retry rather than latch "this board has no board commands".
+        if time.time() - self.start >= self.probe_delay:
+            out["board_cmds"] = ",".join(self.board_cmds)
+        return out
 
 
 # ---------------------------------------------------------------------------
@@ -282,7 +293,7 @@ def apply_set(cfg, key, val):
         ADMIN_PASSWORD = val
         return True, "OK"
 
-    if key == "radio.fem.rxgain":
+    if key == "radio.fem.rxgain" and key not in ST.board_cmds:
         return False, "Error: unsupported"   # no FEM on the mock board, see GETTERS
 
     if key == "radio.rxps":
@@ -406,13 +417,74 @@ def apply_set(cfg, key, val):
         sec, f = STR_KEYS[key]
         cfg[sec][f] = val
         return True, "OK"
+    gate = BOARD_CMD_GATE.get(key)
+    if gate:
+        if gate not in ST.board_cmds:
+            return False, "unknown config: %s" % key   # no such command on this board
+        return apply_board_set(cfg, key, val)
+
     # Strict fallthrough: this function is the single authority on what can be
     # set, for the batch and the CLI alike. Accepting unknown keys here once hid
     # the fact that the CLI could not reach `dutycycle` or `radio.fem.rxgain`.
     # Verbatim shape from CommonCLI::handleSetCmd's fallthrough.
+    #
+    # This is also where the board-specific commands land. `radio.fem.*` and
+    # `fan*` come from Board::handleCommand(), which is dispatched ahead of
+    # CommonCLI; the mock board is a Heltec V3 and implements no such hook, so
+    # they reach this fallthrough exactly as they do on the real thing.
     return False, "unknown config: %s" % key
 
 
+# Commands that reach Board::handleCommand() rather than CommonCLI. Which ones a
+# node answers is a property of the board, so the portal probes for them at
+# startup and reports the answers in /api/status; --board-cmds picks which ones
+# this mock claims. `fan.lo` / `fan.hi` have no getter and ride on `fan`.
+BOARD_CMD_GATE = {
+    "radio.fem.rxgain": "radio.fem.rxgain",
+    "radio.fem.txgain": "radio.fem.txgain",
+    "fan": "fan",
+    "fan.lo": "fan",
+    "fan.hi": "fan",
+}
+BOARD_STATE = {"fan": "auto", "fan.lo": 45, "fan.hi": 60}
+FEM_FIELD = {"radio.fem.rxgain": "fem_rxgain", "radio.fem.txgain": "fem_txgain"}
+
+
+def apply_board_set(cfg, key, val):
+    if key in FEM_FIELD:
+        if val not in ("on", "off"):
+            return False, "Error: state must be on or off"
+        cfg["radio"][FEM_FIELD[key]] = val == "on"
+        return True, "OK - LoRa FEM %s gain %s" % ("RX" if "rx" in key else "TX", val)
+    if key == "fan":
+        if val not in ("on", "off", "auto"):
+            return False, "Error: fan must be on, off, or auto"
+        BOARD_STATE["fan"] = val
+        return True, "OK - fan %s" % val
+    try:
+        n = int(val)
+    except ValueError:
+        return False, "Error: expected a number"
+    if key == "fan.lo" and not (0 <= n <= 100 and n < BOARD_STATE["fan.hi"]):
+        return False, "Error: fan.lo must be 0..100 and < fan.hi"
+    if key == "fan.hi" and not (BOARD_STATE["fan.lo"] < n <= 120):
+        return False, "Error: fan.hi must be > fan.lo and <= 120"
+    BOARD_STATE[key] = n
+    return True, "OK - %s %d" % (key, n)
+
+
+def board_get(cfg, key):
+    """Getter reply, or None when this board does not answer the command."""
+    if BOARD_CMD_GATE.get(key) not in ST.board_cmds:
+        return None
+    if key == "fan":
+        return "%s 41.0C fan=%s cd=0s" % (
+            BOARD_STATE["fan"], "on" if BOARD_STATE["fan"] == "on" else "off")
+    if key in FEM_FIELD:
+        return "on" if cfg["radio"][FEM_FIELD[key]] else "off"
+    return None           # fan.lo / fan.hi are set-only on the board too
+
+
 # Payload-type names accepted alongside the decimal form. Mirrors
 # namedPacketTypes() in src/helpers/MQTTPacketFilter.h; 12-14 are reserved
 # upstream and stay reachable by number only.
@@ -552,6 +624,12 @@ GETTERS = {
     "wifi.status": lambda c: (
         "SSID: %s\nIP: 192.168.1.42\nRSSI: -58 dBm\nUptime: %dm"
         % (c["wifi"]["ssid"] or "(not set)", int(time.time() - ST.start) // 60)),
+    "link.status": lambda c: (
+        "wifi: connected, IP: 192.168.1.42, RSSI: -58 dBm, Uptime: %dm"
+        % (int(time.time() - ST.start) // 60)),
+    "link.diag": lambda c: (
+        "why:ethernet-not-enabled selected:wifi\n"
+        "wifi:state:connected ip:192.168.1.42"),
     "mqtt.status": lambda c: cli_mqtt_status(c),
     "mqtt.presets": lambda c: "\n".join(
         "%2d. %s%s" % (i + 1, n, "" if nd == "none" else "  (needs %s)" % nd)
@@ -599,6 +677,11 @@ def cli_get(cfg, key):
 
 
 def _cli_get_value(cfg, key):
+    if key in BOARD_CMD_GATE:
+        val = board_get(cfg, key)
+        # Not answered: the board has no hook for it, so CommonCLI's own getter
+        # fallthrough is what replies — exactly as on the real thing.
+        return (True, val) if val is not None else (False, "??: %s" % key)
     if key in GETTERS:
         val = GETTERS[key](cfg)
         return (True, val) if val is not None else (False, "Error: unsupported")
@@ -633,6 +716,10 @@ def cli_read_key(cfg, key):
             "on" if r["rxps_enabled"] else "off", r["rxps_rx_us"], r["rxps_sleep_us"]),
         "bw": r["bw"], "sf": r["sf"], "cr": r["cr"],
         "mqtt.iata": cfg["mqtt"]["iata"], "mqtt.owner": cfg["mqtt"]["owner"],
+        "display.mode": "button", "display.timeout": "15",
+        "display.usb.mode": "button", "display.usb.timeout": "15",
+        "display.inbox": "off", "bluetooth.mac": "",
+        "bluetooth.stealth": "off",
     }.get(key)
 
 
@@ -1110,6 +1197,12 @@ def main():
     ap.add_argument("--port", type=int, default=8080)
     ap.add_argument("--setup", action="store_true", help="first-boot setup wizard mode")
     ap.add_argument("--active-slots", type=int, default=5, help="server slots to expose (2 or 5)")
+    ap.add_argument("--probe-delay", type=float, default=0.0,
+                    help="seconds to withhold board_cmds from /api/status, simulating "
+                         "the gap before the node has probed its board")
+    ap.add_argument("--board-cmds", default="",
+                    help="comma list of Board::handleCommand() commands to answer, e.g. "
+                         "radio.fem.rxgain,radio.fem.txgain,fan (default: none, like a Heltec V3)")
     ap.add_argument("--fw-version", default=FW_VERSION,
                     help="version string to report, shaped like build.sh's embedded one")
     ap.add_argument("--minify", action="store_true",