From cc4bcb9cad59930de3e7ebc61a13f831df381c11 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Mon, 5 Oct 2026 01:14:02 -0700 Subject: [PATCH] Qualify 1.17.1.9 memory limits and exact release asset routing --- .../workflows/build-repeater-firmwares.yml | 2 + .github/workflows/run-unit-tests.yml | 1 + RELEASE.md | 44 +++- build_legacy.sh | 11 +- docs/releases/1.17.1.9.md | 71 ++++++ mkdocs.yml | 1 + scripts/build_local_release.py | 3 +- scripts/generate_mesh_america_release.py | 206 ++++++++++++++++++ scripts/package_cascade_release.py | 141 ++++++++++-- test/test_build_size_detection.sh | 2 +- test/test_release_publication.py | 174 +++++++++++++++ 11 files changed, 627 insertions(+), 29 deletions(-) create mode 100644 docs/releases/1.17.1.9.md create mode 100644 scripts/generate_mesh_america_release.py create mode 100644 test/test_release_publication.py diff --git a/.github/workflows/build-repeater-firmwares.yml b/.github/workflows/build-repeater-firmwares.yml index 46b9076c..8a29d461 100644 --- a/.github/workflows/build-repeater-firmwares.yml +++ b/.github/workflows/build-repeater-firmwares.yml @@ -8,6 +8,8 @@ on: push: tags: - 'repeater-*' + # Cascade pages contain already-qualified local matrix artifacts. + - '!repeater-room-*' jobs: build-repeater-firmwares: diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index a19173ca..eb798919 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -368,6 +368,7 @@ jobs: python3 -B test/test_neighbor_scope_storage.py python3 -B test/test_ed25519_compact.py python3 -B test/test_cascade_release_package.py + python3 -B test/test_release_publication.py python3 -B test/test_local_release_full_selection.py python3 -B test/test_t096_full_memory.py python3 -B test/test_nrf52_ble_startup.py diff --git a/RELEASE.md b/RELEASE.md index abd33a5c..3cf10bda 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,5 +1,44 @@ # Releasing Firmware +For 1.17.1.9 and later, build the complete qualified local release from a +clean, committed checkout. The builder runs one PlatformIO process at a time: + +```bash +python3 scripts/build_local_release.py \ + --firmware-version v1.17.1.9-halo-keymind-cascade-dev --pio-jobs 4 --resume +``` + +Package its completed directory for stable publication: + +```bash +python3 scripts/package_cascade_release.py \ + --input .releases/v1.17.1.9-halo-keymind-cascade-dev-SOURCE8 \ + --output /path/to/publication-stage --local-release --stable \ + --version 1.17.1.9 --commit FULL_SOURCE_COMMIT +``` + +The packager rechecks the local checksum inventory, qualified firmware, +full/reduced nRF52 pairs, and every exact ESP32 migration package. It keeps +migration-only application identities inside their migration ZIPs. It does +not upload anything. Review `release-plan.json`, generate the picker controls +as described in [the picker guide](docs/firmware_picker.md), and generate +both Mesh America catalogs from the same staged asset manifests: + +```bash +python3 scripts/generate_mesh_america_release.py \ + --stage /path/to/publication-stage \ + --catalog mesh-america/keymind-cascade-v1.16.0-provider.json \ + --output /path/to/new-provider.json +``` + +Both active provider URLs use the same binaries. Preserve their stable +filenames when updating them. Create supporting releases first and publish +the Companion family root last, marking only that root as GitHub Latest. +Upload each group's complete asset inventory and verify the public asset +names, sizes and checksums before switching the catalogs. Regenerate the +downloadable picker from the same family and update its checksum on every +page. Documentation-only follow-up commits do not move firmware tags. + For the USA Cascade 1.17.1.5 matrix, use [the option 3 release instructions](docs/old-releases/1.17.1.5.md). Package the qualified outputs with `scripts/package_cascade_release.py`. @@ -16,12 +55,13 @@ and `room-server-*` tags (for example `repeater-v1.0.0`). They invoke the shared `firmware-builder.yml`, build their own target set, and request a draft release. They are separate from the qualified option 3 matrix and local release staging. -**The broad `repeater-*` trigger also matches `repeater-room-*` Cascade tags.** -For 1.17.1.5 this workflow added its own artifacts and changed the title of the +The older broad `repeater-*` trigger also matched `repeater-room-*` Cascade tags. +For 1.17.1.5 that workflow added its own artifacts and changed the title of the already-published Repeater/Room Server page. Do not assume a published Cascade page is untouched while that workflow is running. Check workflow completion, release title/body, and the final asset inventory after publication. Preserve existing assets when making a documentation-only release edit. +The current trigger explicitly excludes `repeater-room-*`. Use role-specific introductions for Companion, Repeater/Room Server, Sensor, LoRa OTA Repeater, and expanded Full infrastructure pages. Link the exact diff --git a/build_legacy.sh b/build_legacy.sh index 6fd56c14..6c6f1fd5 100755 --- a/build_legacy.sh +++ b/build_legacy.sh @@ -3757,16 +3757,17 @@ apply_nrf52_size_profile() { export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -Os" # These color-display repeaters reserve a 64 KiB reset-retained OTA arena - # plus a 25 KiB framebuffer. Keep their board-declared 50 neighbours and - # the complete flood-rule engine with a smaller table so runtime startup - # allocations also fit; do not lower the heap qualification requirement. + # plus a 25 KiB framebuffer. With the 8 KiB loop stack, the complete sensor + # profile is 1.5-1.9 KiB short of the runtime heap requirement at 16 rules. + # Keep their 50 neighbours, sensors, display, OTA and stack; bound the + # optional flood-rule table instead of lowering the qualification margin. case "${env_name,,}" in heltec_t096_repeater_lora_ota_no_external_sensors|\ heltec_t1_repeater_lora_ota_no_external_sensors) append_platformio_build_unflags "-DFLOOD_PACKET_FILTER_SLOTS=63" - export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DFLOOD_PACKET_FILTER_SLOTS=16" + export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DFLOOD_PACKET_FILTER_SLOTS=4" record_build_reduction \ - "mesh.flood_rules limited to 16 by measured internal RAM; complete rule engine, color display, GPS, and OTA retained" + "mesh.flood_rules limited to 4 by measured internal RAM; complete rule engine, 8 KiB loop stack, 50 neighbors, color display, GPS, sensors, and OTA retained" ;; esac } diff --git a/docs/releases/1.17.1.9.md b/docs/releases/1.17.1.9.md new file mode 100644 index 00000000..7cd81891 --- /dev/null +++ b/docs/releases/1.17.1.9.md @@ -0,0 +1,71 @@ +# MeshCore 1.17.1.9 USA Cascade + +USA Cascade defaults are 910.525 MHz, 62.5 kHz bandwidth, SF7 and CR5. Saved +radio settings take precedence. Match the exact board, role and storage +layout before installing an image. + +## Changes since 1.17.1.8 + +- ESP32 native USB recipes now use hardware CDC where the chip supports it. + Classic ESP32 and explicit UART transports keep their UART path. nRF52, + RP2040 and STM32 USB backends are unchanged by this migration. +- ESP32 USB output is bounded so a stopped reader cannot indefinitely hold + the radio loop. The G2 preserves complete serial packet-log lines under + congestion. This supports unmodified Cisien meshcoretomqtt; congested + diagnostic output can still be dropped. +- The pinned ESP32-S3 framework releases retained TinyUSB PHY ownership when + starting hardware CDC after a software restart. KISS clears incomplete + host frames on a USB session reset while preserving active radio work. +- Companion packet streaming supports stock serial MQTT consumers. Text CLI + and Binary Companion handoff and reply deadlines are covered against the + official app's protocol behavior. +- Access Control replies fit the encrypted LoRa route budget and the + Companion frame envelope. Large ACL listings use bounded pages. ESP32 ACL + persistence avoids repeated metadata opens while retaining recovery checks. +- nRF52 keeps its 8 KiB loop stack. Crypto, contact CRC and background storage + work reduce peak stack use and long pauses. OTA metadata is cached after + verification instead of repeatedly reopening storage for status displays. +- T096 and T1 repeater profiles reserve four flood-rule entries to fit the + complete sensor profile with the existing 8 KiB loop stack, 50 neighbors, + color framebuffer and OTA arena. Their capability manifests record this + capacity limit; heap qualification margins are unchanged. + +## Installation + +An ESP32 merged `.bin` installs bootloader, partitions and application over +USB. An app-only `.bin` requires a compatible existing layout. Partition +expansion requires the exact board and role migration ZIP from this release +family's utility page, or the matching merged image over USB. Follow the ZIP's +README and use its supplied Full application. `get storage.layout` reports +the live layout on firmware that supports it. + +Native hardware CDC can change the USB product name and Linux `/dev/serial/by-id` +path. Update serial consumers to the new device path after the first upgrade. +Do not identify a radio solely by a changing `/dev/ttyACM` number. + +nRF52 application UF2 and DFU ZIP files do not install a bootloader. Use the +exact board/storage OTAFIX mapping in the [firmware picker](../firmware_picker.md). +See the [role controls](../role_feature_switches.md), +[Full Companion guide](../full_companion_features.md), and +[USB web console](https://flasher.meshcore.io/console). + +## Verification scope + +The ESP32 migration passed the native unit suite, SDK and protocol regression +tests, sanitizer fixtures and representative builds across all affected board +families. The G2 passed physical USB command/reconnect tests with its original +identity and settings preserved. A separate RAK4631 recovery admin was verified +by passwordless LoRa login, remote status and a remote version reply. + +The attempted old-TinyUSB Wi-Fi OTA transition did not reach an upload: its +WebConfig/Wi-Fi handoff and a subsequent old-firmware USB timeout prevented +that path. The G2 was restored over USB and its hardware CDC runtime passed +444 strict-deadline queries and a 150-second closed-port idle/reconnect test. +The retained-PHY transition is covered synthetically; this attempt does not +establish a successful physical TinyUSB-to-hardware-CDC OTA upgrade. + +Build qualification does not establish physical compatibility on every board. +Hardware CDC has finite buffers, and closing a Linux TTY does not necessarily +produce a hardware session reset. Saturated diagnostic output is not lossless. +See [USB backpressure test results](../test-results/usb_serial_backpressure.md) +for measured scope and the upstream issues reviewed. diff --git a/mkdocs.yml b/mkdocs.yml index 75df1443..ed408c15 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -96,6 +96,7 @@ nav: - Research: research/index.md - Releases: - Current release: + - 1.17.1.9 USA Cascade: releases/1.17.1.9.md - 1.17.1.8 USA Cascade: releases/1.17.1.8.md - Test Results: test-results/index.md - Old Releases: old-releases/index.md diff --git a/scripts/build_local_release.py b/scripts/build_local_release.py index a738d5cf..5a305eaa 100644 --- a/scripts/build_local_release.py +++ b/scripts/build_local_release.py @@ -21,7 +21,7 @@ from package_cascade_release import category, collect_artifacts, nrf52_sensor_pr ROOT = Path(__file__).resolve().parents[1] -DEFAULT_VERSION = "v1.17.1.8-halo-keymind-cascade-dev" +DEFAULT_VERSION = "v1.17.1.9-halo-keymind-cascade-dev" RADIO = {"frequency_mhz": 910.525, "bandwidth_khz": 62.5, "spreading_factor": 7, "coding_rate": 5} @@ -268,6 +268,7 @@ def main() -> None: if args.dry_run: return environment = os.environ.copy() + environment["OPTION3_PIO_JOBS"] = str(args.pio_jobs) try: utility_build_dir = resolve_pio_build_dir(environment=environment, project_dir=ROOT) except ValueError as error: diff --git a/scripts/generate_mesh_america_release.py b/scripts/generate_mesh_america_release.py new file mode 100644 index 00000000..91f2d6c6 --- /dev/null +++ b/scripts/generate_mesh_america_release.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Generate Mesh America catalogs from exact, staged release assets.""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +from pathlib import Path +import re +import subprocess +from urllib.parse import quote + + +ROOT = Path(__file__).resolve().parents[1] +SUFFIXES = {".bin", ".zip", ".uf2", ".hex"} +PLATFORMS = {"ESP32_PLATFORM": "esp32", "NRF52_PLATFORM": "nrf52", + "RP2040_PLATFORM": "noflash", "STM32_PLATFORM": "noflash"} + + +def identity(name): + stem = Path(name).stem.removesuffix("-merged") + match = re.fullmatch(r"(.+?)-v\d+(?:\.\d+){2,3}-.+", stem) + if not match: + raise ValueError(f"firmware filename has no release identity: {name}") + return match[1] + + +def target_profiles(targets): + # Use the same exact hardware/role parser as the online firmware picker. + program = """const fs = require('fs'); +const picker = require(process.argv[1]); +const targets = JSON.parse(fs.readFileSync(0, 'utf8')); +process.stdout.write(JSON.stringify(Object.fromEntries( + targets.map(t => [t, picker.parseTargetProfile(t)])))); +""" + return json.loads(subprocess.check_output( + ["node", "-e", program, str(ROOT / "docs/_javascript/firmware_picker.js")], + input=json.dumps(sorted(targets)), text=True)) + + +def file_entry(path, platform, tag, repo): + if path.name.endswith("-merged.bin"): + kind, title = "flash-wipe", "USB full install (bootloader + partitions + app)" + elif path.suffix == ".bin" and platform == "esp32": + kind, title = "flash-update", "App update (matching partition layout only)" + elif path.suffix == ".zip" and platform == "nrf52": + kind, title = "flash", "Serial DFU package" + else: + kind, title = "download", path.suffix[1:].upper() + " download" + return {"type": kind, "name": path.name, "title": title, + "url": f"https://github.com/{repo}/releases/download/{quote(tag, safe='')}/{quote(path.name, safe='')}"} + + +def read_stage(stage): + plan = json.loads((stage / "release-plan.json").read_text()) + source = plan["source"] + if not re.fullmatch(r"[a-f0-9]{40}", source): + raise ValueError("release source must be a full commit") + family = f"v{plan['version']}-halo-keymind-cascade-dev-{source[:8]}" + records = [] + seen = set() + for group in plan["groups"]: + expected_tag = family if group["key"] == "companion" else group["key"] + "-" + family + if group["tag"] != expected_tag: + raise ValueError("release group belongs to a different family") + directory = stage / group["key"] + checksums = {} + for line in (directory / "SHA256SUMS.txt").read_text(encoding="ascii").splitlines(): + digest, name = line.split(" ", 1) + if name in checksums or Path(name).name != name: + raise ValueError("duplicate or unsafe release asset") + checksums[name] = digest + manifest_path = directory / "TARGET-MANIFEST.json" + if hashlib.sha256(manifest_path.read_bytes()).hexdigest() != checksums.get(manifest_path.name): + raise ValueError("release target manifest checksum failed") + manifests = json.loads(manifest_path.read_text()) + if len(manifests) != group["target_count"]: + raise ValueError("release target count differs from manifest") + for row in manifests: + key = row["artifact_target"] + if key in seen or row.get("verified") is not True: + raise ValueError("duplicate or unqualified release profile") + seen.add(key) + files = [] + for name in row["files"]: + if Path(name).name != name or name not in checksums: + raise ValueError("manifest asset is outside the release checksum inventory") + path = directory / name + if hashlib.sha256(path.read_bytes()).hexdigest() != checksums[name]: + raise ValueError(f"release asset checksum failed: {name}") + if path.suffix in SUFFIXES: + if identity(name) != key or family not in name: + raise ValueError("firmware filename differs from qualified profile") + files.append(path) + if not files: + raise ValueError("qualified profile contains no firmware") + records.append((row, group["tag"], files)) + return plan, family, records + + +def generate(catalog, plan, family, records, profiles, repo): + result = copy.deepcopy(catalog) + by_identity, by_hardware, existing_roles = {}, {}, {} + for index, device in enumerate(catalog["device"]): + for firmware in device["firmware"]: + for version in firmware["version"].values(): + for file in version["files"]: + key = identity(file["name"]) + by_identity.setdefault(key, set()).add(index) + existing_roles.setdefault(key, set()).add((firmware["role"], firmware.get("title", firmware["role"]))) + hardware = profiles[key]["hardware"] + by_hardware.setdefault((device["type"], hardware), set()).add(index) + result["device"][index]["firmware"] = [] + for row, tag, files in records: + target = row["artifact_target"] + parsed = profiles[target] + platform = PLATFORMS[row["platform"]] + indexes = by_identity.get(target) or by_identity.get(row["target"]) + if not indexes: + indexes = by_hardware.get((platform, parsed["hardware"]), set()) + if len(indexes) != 1: + raise ValueError(f"new profile needs an exact hardware card mapping: {target}") + if any(catalog["device"][i]["type"] != platform for i in indexes): + raise ValueError("firmware platform differs from its hardware card") + role, title = {"repeater": ("repeater", "Repeater"), + "room": ("roomServer", "Room Server"), + "sensor": ("sensor", "Sensor"), + "terminal": ("terminalChat", "Terminal Chat")}.get(parsed["role"], (None, None)) + if parsed["role"] == "companion": + role, title = {"full": ("companionFull", "Full Companion"), + "ble": ("companionBle", "Companion Bluetooth"), + "usb": ("companionUsb", "Companion USB"), + "wifi": ("companionWifi", "Companion WiFi"), + "ethernet": ("companionEthernet", "Companion Ethernet"), + "usb-ble": ("companionBle", "Companion USB + Bluetooth")}.get(parsed["mode"], (None, None)) + if role is None and len(existing_roles.get(target, set())) == 1: + # Some older exact recipes (PicoW_companion_radio) omit a + # transport suffix. Preserve their curated role without guessing. + role, title = next(iter(existing_roles[target])) + if role is None: + raise ValueError(f"unsupported firmware role or transport: {target}") + sensor = row.get("sensor_profile", "") + profile = ("Full supported sensors + LoRa OTA" if sensor == "full" else + "Reduced optional sensors + LoRa OTA" if sensor == "reduced" else + "Full profile" if row["build_profile"] == "full" or parsed["mode"] == "full" else + "Standard profile") + url = f"https://github.com/{repo}/releases/download/{tag}/" + cap = next(name for name in row["files"] if name.endswith(".capabilities.json")) + memory = next(name for name in row["files"] if name.endswith(".memory.json")) + methods = ", ".join(row.get("ota_update_methods", [])) or "USB or board-specific wired programming" + notes = [f"Keymind Cascade MeshCore {plan['version']}. USA Cascade: 910.525 MHz / BW62.5 / SF7 / CR5. Saved radio settings take precedence.", + f"EXACT TARGET - {row['target']}. Match the exact board, radio, screen, and storage hardware.", + f"PROFILE - {profile}. See the qualified capabilities and memory reports for this image.", + f"SELF-UPDATE - Verified methods: {methods}."] + if platform == "esp32": + notes.append("INSTALL - The merged .bin installs the bootloader, partitions and application over USB. An app-only .bin requires a compatible existing partition layout. Use the exact board and role migration ZIP from the utility release when expanding partitions over Wi-Fi or LoRa; follow its README and install its supplied Full application.") + elif platform == "nrf52": + notes.append("INSTALL - UF2 and Serial DFU ZIP files contain application firmware. LoRa updates require the matching board and storage OTAFIX bootloader. Use the firmware picker's exact bootloader mapping.") + if row.get("reductions"): + notes.append("CAPACITY - " + "; ".join(row["reductions"])) + if "-full-usb-wifi" in target: + notes.append("OUTPUT - Select saved output with set logging.output off|usb|wifi|both. USB-connected MQTT uses the serial packet log. Direct Wi-Fi MQTT uses the saved broker configuration.") + elif parsed["mode"] == "full" and parsed["role"] == "companion": + notes.append("TRANSPORT - Full Companion starts its primary USB interface in the ASCII terminal; a complete framed app command selects Binary Companion. USB logging is off by default. Use the picker directions for this image's supported transports and runtime logging controls.") + notes += [f"SETUP - https://mikecarper.github.io/MeshCore/firmware_picker/?hardware={quote(parsed['hardware'])}", + "CAPABILITIES - " + url + quote(cap), "MEMORY - " + url + quote(memory), + f"SOURCE - https://github.com/{repo}/tree/{row.get('source_commit', plan['source'])}", + f"RELEASE - https://github.com/{repo}/releases/tag/{tag}"] + entry = {"role": role, "title": title, "subTitle": target + " - " + profile, + "version": {family: {"notes": "\n\n".join(notes), + "files": [file_entry(path, platform, tag, repo) for path in sorted(files, + key=lambda p: (not p.name.endswith('-merged.bin'), p.suffix != '.bin', p.name))]}}} + for index in sorted(indexes): + result["device"][index]["firmware"].append(copy.deepcopy(entry)) + result["device"] = [device for device in result["device"] if device["firmware"]] + for device in result["device"]: + device["firmware"].sort(key=lambda entry: (entry["role"], entry["subTitle"])) + result["description"] = (f"Keymind Cascade MeshCore {plan['version']} USA Cascade. " + f"910.525 MHz / BW62.5 / SF7 / CR5. {len(records)} qualified firmware profiles " + f"across {len(plan['groups'])} GitHub release pages. See each exact image's capability and memory reports.") + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--stage", type=Path, required=True) + parser.add_argument("--catalog", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--repo", default="mikecarper/MeshCore") + args = parser.parse_args() + catalog = json.loads(args.catalog.read_text()) + plan, family, records = read_stage(args.stage) + targets = {row["artifact_target"] for row, _, _ in records} + targets.update(identity(file["name"]) for device in catalog["device"] + for firmware in device["firmware"] for version in firmware["version"].values() + for file in version["files"]) + result = generate(catalog, plan, family, records, target_profiles(targets), args.repo) + args.output.write_text(json.dumps(result, ensure_ascii=True, indent=2) + "\n", encoding="ascii") + print(json.dumps({"profiles": len(records), "hardware_cards": len(result["device"]), + "firmware_entries": sum(len(d["firmware"]) for d in result["device"])})) + + +if __name__ == "__main__": + main() diff --git a/scripts/package_cascade_release.py b/scripts/package_cascade_release.py index 7bfe5692..159323c3 100644 --- a/scripts/package_cascade_release.py +++ b/scripts/package_cascade_release.py @@ -250,33 +250,123 @@ def portable_profile_exclusions(status): for name in names] +def collect_local_release(directory, version, source): + """Recheck a completed local release without adding migration-only images.""" + from build_esp32_partition_migration import verify_archive + from package_esp32_partition_migration import BOARDS + + manifest = json.loads((directory / "manifest.json").read_text()) + if (manifest.get("format") != "meshcore-local-release-v1" + or manifest.get("source_commit") != source + or manifest.get("firmware_version") != version + or manifest.get("profile") != "cascade" + or manifest.get("publication") != "local-only"): + raise ValueError("local release belongs to another source, version or profile") + expected_radio = {"frequency_mhz": 910.525, "bandwidth_khz": 62.5, + "spreading_factor": 7, "coding_rate": 5} + if manifest.get("radio") != expected_radio: + raise ValueError("local release did not use USA Cascade radio defaults") + checked = set() + for line in (directory / "SHA256SUMS.txt").read_text(encoding="ascii").splitlines(): + match = re.fullmatch(r"([0-9a-f]{64}) (.+)", line) + if not match: + raise ValueError("invalid local release checksum line") + name = match[2] + path = directory / name + if (name in checked or path.is_symlink() or not path.is_file() + or not path.resolve().is_relative_to(directory.resolve()) + or sha256(path) != match[1]): + raise ValueError(f"local release checksum failed: {name}") + checked.add(name) + actual = {str(p.relative_to(directory)) for p in directory.rglob("*") + if p.is_file() and p != directory / "SHA256SUMS.txt"} + if checked != actual: + raise ValueError("local release checksum inventory is incomplete") + records = [] + for group in sorted((directory / "firmware").iterdir()): + if not group.is_dir(): + raise ValueError("unexpected file outside firmware groups") + group_records = collect_artifacts(group, f"{version}-{source[:8]}") + if any(category(record) != group.name for record in group_records): + raise ValueError("local release firmware group disagrees with qualification") + records.extend(group_records) + indexed = {record["manifest"]["artifact_target"]: record for record in records} + entries = manifest["firmware"] + if (len(indexed) != len(records) or len(entries) != len(records) + or len(records) != manifest["firmware_target_count"] + or len({entry["artifact_target"] for entry in entries}) != len(entries)): + raise ValueError("local release target inventory disagrees with qualification") + for entry in entries: + record = indexed.get(entry["artifact_target"]) + if record is None: + raise ValueError("local release contains an unqualified target") + qualified = record["manifest"] + if (qualified["target"] != entry["target"] + or qualified.get("source_commit", source) != source + or qualified["platform"] != entry["platform"] + or qualified["build_profile"] != entry["profile"] + or set(entry["files"]) != { + str(p.relative_to(directory)) for p in record["files"]}): + raise ValueError("local release manifest disagrees with qualified files") + extras = [] + for board in BOARDS: + path = directory / "esp32-partition-migration" / f"{board}-{version}-{source[:8]}-migration.zip" + verify_archive(path, board, version, source[:8]) + extras.append(path) + if manifest["migration_board_role_count"] != len(extras): + raise ValueError("local release migration inventory is incomplete") + bundles = list(directory.glob(f"esp32-partition-migration-{version}-{source[:8]}-release.zip")) + if len(bundles) != 1: + raise ValueError("local release needs exactly one migration bundle") + with zipfile.ZipFile(bundles[0]) as archive: + if set(archive.namelist()) != {p.name for p in extras} | {"release-manifest.json"}: + raise ValueError("migration bundle inventory is incomplete") + for path in extras: + if hashlib.sha256(archive.read(path.name)).hexdigest() != sha256(path): + raise ValueError("migration bundle differs from individual packages") + extras.extend(bundles) + return records, extras, {"frequency": "910.525", "bandwidth": "62.5", "sf": "7", "cr": "5"} + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) - parser.add_argument("--build-status", required=True, type=Path) + provenance = parser.add_mutually_exclusive_group(required=True) + provenance.add_argument("--build-status", type=Path) + provenance.add_argument("--local-release", action="store_true", + help="Package the verified output of build_local_release.py") parser.add_argument("--version", default="1.17.1.5") parser.add_argument("--commit", required=True) parser.add_argument("--repo", default="mikecarper/MeshCore") + parser.add_argument("--stable", action="store_true", help="Stage a stable release family") parser.add_argument("--allow-partial", action="store_true", help="Publish qualified outputs from a finished matrix with failures, listing every failed attempt") args = parser.parse_args() - status = dict(line.split("=", 1) for line in args.build_status.read_text().splitlines() if "=" in line) - failures = completed_matrix_failures(status, args.allow_partial) - portable_exclusions = portable_profile_exclusions(status) - output_directory = Path(status["working_directory"]) / status["output_directory"] - if output_directory.resolve() != args.input.resolve(): - raise ValueError("build status belongs to another output directory") firmware_label = f"v{args.version}-halo-keymind-cascade-dev" - if status.get("source_commit") != args.commit or status.get("firmware_version") != firmware_label: - raise ValueError("build status belongs to another source revision or version") - if status.get("firmware_profile") != "cascade": - raise ValueError("matrix did not use Cascade runtime defaults") - radio = {key: status["radio_" + key] for key in ("frequency", "bandwidth", "sf", "cr")} + version = f"{firmware_label}-{args.commit[:8]}" + extras = [] + if args.local_release: + if args.allow_partial: + parser.error("a completed local release cannot be partially published") + records, extras, radio = collect_local_release(args.input, firmware_label, args.commit) + status = {"state": "completed", "exit_code": "0"} + failures, portable_exclusions = [], [] + else: + status = dict(line.split("=", 1) for line in args.build_status.read_text().splitlines() if "=" in line) + failures = completed_matrix_failures(status, args.allow_partial) + portable_exclusions = portable_profile_exclusions(status) + output_directory = Path(status["working_directory"]) / status["output_directory"] + if output_directory.resolve() != args.input.resolve(): + raise ValueError("build status belongs to another output directory") + if status.get("source_commit") != args.commit or status.get("firmware_version") != firmware_label: + raise ValueError("build status belongs to another source revision or version") + if status.get("firmware_profile") != "cascade": + raise ValueError("matrix did not use Cascade runtime defaults") + radio = {key: status["radio_" + key] for key in ("frequency", "bandwidth", "sf", "cr")} + records = collect_artifacts(args.input, version) if args.output.exists() and any(args.output.iterdir()): raise ValueError("staging directory must be empty; existing releases are never overwritten") - version = f"{firmware_label}-{args.commit[:8]}" - records = collect_artifacts(args.input, version) base_tag = version groups = [] titles = {"companion": "Companion builds", "repeater-room": "Repeater and Room Server builds", @@ -297,10 +387,14 @@ def main(): for index, chunk in enumerate(chunks): key = name + (f"-{index + 1}" if index else "") tag = base_tag if key == "companion" else f"{key}-{base_tag}" - title = f"MeshCore {args.version} Dev - {titles[name]}" + title = f"MeshCore {args.version}{'' if args.stable else ' Dev'} - {titles[name]}" if index: title += f" (part {index + 1})" - groups.append({"key": key, "tag": tag, "title": title, "prerelease": True, "records": chunk}) + groups.append({"key": key, "tag": tag, "title": title, "prerelease": not args.stable, "records": chunk}) + if extras and not any(g['key'] == 'utility' for g in groups): + groups.append({'key': 'utility', 'tag': 'utility-' + base_tag, + 'title': f'MeshCore {args.version} - Migration utilities', + 'prerelease': not args.stable, 'records': []}) links = "\n".join(f"- [{g['key']}](https://github.com/{args.repo}/releases/tag/{g['tag']})" for g in groups) source_url = f"https://github.com/{args.repo}/blob/{args.commit}" rows = [] @@ -320,7 +414,12 @@ def main(): methods = ", ".join(manifest.get("ota_update_methods", [])) or "USB" summaries.append({**manifest, "sensor_profile": nrf52_sensor_profile(manifest), "files": [path.name for path in record["files"]]}) - rows.append(f"{html.escape(manifest['artifact_target'])}{html.escape(release_profile_label(manifest))}{html.escape(methods)}{' · '.join(file_links)}") + rows.append(f"{html.escape(manifest['artifact_target'])}{html.escape(release_profile_label(manifest))}{html.escape(methods)}{' | '.join(file_links)}") + if group['key'] == 'utility': + for path in extras: + if (destination / path.name).exists(): + raise ValueError(f'duplicate utility asset: {path.name}') + shutil.copy2(path, destination / path.name) (destination / "TARGET-MANIFEST.json").write_text(json.dumps(summaries, indent=2) + "\n") columns = ("artifact_target", "target", "platform", "build_profile", "sensor_profile", "ota_update_methods", "files") @@ -366,11 +465,12 @@ def main(): report_url = f"https://github.com/{args.repo}/releases/download/{group['tag']}/BUILD-FAILURES.md" partial_note = (f"**Partial matrix:** {len(failures)} build attempt(s) failed and await repair. " f"Only individually qualified firmware is published. [Missing builds]({report_url}).\n\n") - body = (f"> **Development prerelease.** Firmware identifier: **{base_tag}**.\n\n" - f"# MeshCore {args.version} Dev — USA Cascade\n\n" + channel = 'Stable release' if args.stable else 'Development prerelease' + body = (f"> **{channel}.** Firmware identifier: **{base_tag}**.\n\n" + f"# MeshCore {args.version} - USA Cascade\n\n" f"Source: `{args.commit}`. USA/Canada: **{radio['frequency']} MHz, BW{radio['bandwidth']}, SF{radio['sf']}, CR{radio['cr']}**; Cascade defaults.\n\n" f"This page contains **{len(summaries)} qualified firmware profiles**. Full Companions send MOTA and normally update over USB. Infrastructure has a verified wireless update path; see each capability manifest.\n\n" - f"[Feature on/off and update directions]({source_url}/docs/full_companion_features.md) · " + f"[Feature on/off and update directions]({source_url}/docs/full_companion_features.md) | " f"[Release details]({source_url}/docs/releases/{args.version}.md)\n\n" "Match the exact board, radio, display, and storage variant. ESP32 WiFi updates use the application `.bin`; the merged image installs boot/partition data over USB. nRF52 `.zip` files are native application DFU packages. LoRa MOTA installation requires an exact destination package; nRF52 also requires its matching OTAFIX bootloader. Full Companions do not LoRa-install onto themselves.\n\n" "Firmware and capability checks passed in the build matrix. Hardware update testing across every board was not performed.\n\n" @@ -404,6 +504,7 @@ def main(): group["asset_count"] = len(files) + 1 group["target_count"] = len(group.pop("records")) (args.output / "release-plan.json").write_text(json.dumps({"source": args.commit, "version": args.version, "radio": radio, "groups": groups, + "provenance": "local-release" if args.local_release else "matrix-status", "matrix": {"state": status["state"], "exit_code": int(status["exit_code"]), "failures": failures, "portable_profile_exclusions": portable_exclusions}}, indent=2) + "\n") print(json.dumps({"targets": len(records), "groups": groups}, indent=2)) diff --git a/test/test_build_size_detection.sh b/test/test_build_size_detection.sh index 3ddcaecb..d8f38b1c 100644 --- a/test/test_build_size_detection.sh +++ b/test/test_build_size_detection.sh @@ -110,7 +110,7 @@ for target in Heltec_t096_repeater_lora_ota_no_external_sensors Heltec_t1_repeat apply_repeater_neighbor_capacity "$target" apply_nrf52_size_profile "$target" [[ "$PLATFORMIO_BUILD_FLAGS" == *'-DMAX_NEIGHBOURS=50'* ]] || fail "$target omitted RAM-safe neighbours" - [[ "$PLATFORMIO_BUILD_FLAGS" == *'-DFLOOD_PACKET_FILTER_SLOTS=16'* ]] || fail "$target omitted RAM-safe rule table" + [[ "$PLATFORMIO_BUILD_FLAGS" == *'-DFLOOD_PACKET_FILTER_SLOTS=4'* ]] || fail "$target omitted RAM-safe rule table" [[ "$PLATFORMIO_BUILD_FLAGS" != *'MESH_MIN_RUNTIME_HEAP'* ]] || fail "runtime RAM guard was overridden" done for target in Heltec_t096_companion_radio_full_femon RAK_4631_repeater; do diff --git a/test/test_release_publication.py b/test/test_release_publication.py new file mode 100644 index 00000000..1a2678cc --- /dev/null +++ b/test/test_release_publication.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Release integration checks for exact asset routing and stable staging.""" +import hashlib +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch +import zipfile + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +import generate_mesh_america_release as catalogs +import package_cascade_release as package + + +SOURCE = "a" * 40 +VERSION = "v1.17.1.9-halo-keymind-cascade-dev" +FAMILY = VERSION + "-" + SOURCE[:8] + + +class ReleasePublicationTest(unittest.TestCase): + def record(self, directory, target, platform="ESP32_PLATFORM", sensor=""): + directory.mkdir(parents=True, exist_ok=True) + stem = target + "-" + FAMILY + names = [stem + suffix for suffix in ( + [".uf2", ".zip"] if platform == "NRF52_PLATFORM" else [".bin", "-merged.bin"])] + names += [stem + ".capabilities.json", stem + ".memory.json"] + for name in names: + (directory / name).write_bytes((name + "\n").encode("ascii")) + row = {"artifact_target": target, "target": target.removesuffix("-" + sensor + "-ota") if sensor else target, + "platform": platform, "build_profile": "auto", "verified": True, + "sensor_profile": sensor, "files": names, "reductions": []} + return row, [directory / name for name in names] + + def stage(self, directory): + definitions = [("companion", "Station_G2_companion_radio_full", "ESP32_PLATFORM", ""), + ("repeater-room", "Station_G2_repeater", "ESP32_PLATFORM", ""), + ("lora-ota", "RAK_4631_repeater-full-ota", "NRF52_PLATFORM", "full"), + ("lora-ota", "RAK_4631_repeater-reduced-ota", "NRF52_PLATFORM", "reduced")] + groups = {} + for group, target, platform, sensor in definitions: + row, files = self.record(directory / group, target, platform, sensor) + groups.setdefault(group, []).append(row) + plan = {"source": SOURCE, "version": "1.17.1.9", "groups": []} + for group, rows in groups.items(): + path = directory / group + (path / "TARGET-MANIFEST.json").write_text(json.dumps(rows)) + (path / "SHA256SUMS.txt").write_text("".join( + hashlib.sha256(p.read_bytes()).hexdigest() + " " + p.name + "\n" + for p in sorted(path.iterdir()))) + plan["groups"].append({"key": group, "tag": FAMILY if group == "companion" else group + "-" + FAMILY, + "target_count": len(rows)}) + (directory / "release-plan.json").write_text(json.dumps(plan)) + return plan + + def template(self): + devices = [] + for hardware, platform, targets in ( + ("Station G2", "esp32", ["Station_G2_companion_radio_full", "Station_G2_repeater"]), + ("RAK4631", "nrf52", ["RAK_4631_repeater"])): + devices.append({"name": hardware, "type": platform, "maker": "test", "firmware": [ + {"role": "repeater", "version": {"old": {"notes": "stale old URLs", + "files": [{"name": target + "-v1.17.1.8-dev-12345678.bin"}]}}} + for target in targets]}) + return {"maker": {"test": {"name": "Test"}}, "device": devices} + + def test_catalog_routes_each_asset_to_its_manifest_group_and_keeps_sensor_pairs(self): + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + self.stage(directory) + plan, family, records = catalogs.read_stage(directory) + template = self.template() + targets = {row["artifact_target"] for row, _, _ in records} | { + "Station_G2_companion_radio_full", "Station_G2_repeater", "RAK_4631_repeater"} + result = catalogs.generate(template, plan, family, records, + catalogs.target_profiles(targets), "mikecarper/MeshCore") + self.assertEqual([len(d["firmware"]) for d in result["device"]], [2, 2]) + for device in result["device"]: + for firmware in device["firmware"]: + version = firmware["version"][FAMILY] + self.assertNotIn("stale", version["notes"]) + for file in version["files"]: + target = catalogs.identity(file["name"]) + expected = next(tag for row, tag, _ in records if row["artifact_target"] == target) + self.assertIn("/download/" + expected + "/", file["url"]) + self.assertEqual(template, self.template(), "input catalog was mutated") + + def test_catalog_rejects_corrupt_assets_and_cross_family_tags(self): + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + plan = self.stage(directory) + binary = next((directory / "companion").glob("*.bin")) + original = binary.read_bytes() + binary.write_bytes(b"corrupt") + with self.assertRaisesRegex(ValueError, "checksum"): + catalogs.read_stage(directory) + binary.write_bytes(original) + plan["groups"][1]["tag"] = FAMILY + (directory / "release-plan.json").write_text(json.dumps(plan)) + with self.assertRaisesRegex(ValueError, "different family"): + catalogs.read_stage(directory) + + def test_catalog_does_not_guess_an_unknown_hardware_card(self): + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + row, files = self.record(directory, "Unknown_repeater") + targets = {"Unknown_repeater", "Station_G2_companion_radio_full", + "Station_G2_repeater", "RAK_4631_repeater"} + with self.assertRaisesRegex(ValueError, "exact hardware card"): + catalogs.generate(self.template(), {"version": "1.17.1.9", "source": SOURCE, "groups": []}, + FAMILY, [(row, FAMILY, files)], catalogs.target_profiles(targets), "mikecarper/MeshCore") + + def test_stable_publication_notes_and_assets_are_ascii(self): + with tempfile.TemporaryDirectory() as temp: + base = Path(temp) + row, files = self.record(base / "input", "Station_G2_companion_radio_usb") + output = base / "stage" + argv = ["package", "--input", str(base / "input"), "--output", str(output), + "--local-release", "--stable", "--version", "1.17.1.9", "--commit", SOURCE] + with patch.object(sys, "argv", argv), patch.object(package, "collect_local_release", + return_value=([{"manifest": row, "files": files}], [], + {"frequency": "910.525", "bandwidth": "62.5", "sf": "7", "cr": "5"})): + package.main() + plan = json.loads((output / "release-plan.json").read_text()) + self.assertFalse(plan["groups"][0]["prerelease"]) + notes = (output / "companion-notes.md").read_text(encoding="ascii") + self.assertIn("Stable release", notes) + self.assertNotIn("Development prerelease", notes) + for path in output.rglob("*"): + if path.is_file(): + path.read_text(encoding="ascii") + + def test_local_release_preserves_selected_profiles_and_verifies_complete_inventory(self): + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + row, files = self.record(directory / "firmware/companion", "Station_G2_companion_radio_usb") + manifest = {"format": "meshcore-local-release-v1", "source_commit": SOURCE, + "firmware_version": VERSION, "profile": "cascade", "publication": "local-only", + "radio": {"frequency_mhz": 910.525, "bandwidth_khz": 62.5, + "spreading_factor": 7, "coding_rate": 5}, + "firmware_target_count": 1, "migration_board_role_count": 1, + "firmware": [{"target": row["target"], "artifact_target": row["artifact_target"], + "profile": "auto", "platform": "ESP32_PLATFORM", + "files": [str(p.relative_to(directory)) for p in files]}]} + (directory / "manifest.json").write_text(json.dumps(manifest)) + migrations = directory / "esp32-partition-migration" + migrations.mkdir() + archive = migrations / ("example-" + FAMILY + "-migration.zip") + archive.write_bytes(b"qualified migration fixture") + bundle = directory / ("esp32-partition-migration-" + FAMILY + "-release.zip") + with zipfile.ZipFile(bundle, "w") as z: + z.writestr("release-manifest.json", "{}") + z.writestr(archive.name, archive.read_bytes()) + checksum = directory / "SHA256SUMS.txt" + checksum.write_text("".join(hashlib.sha256(p.read_bytes()).hexdigest() + " " + + str(p.relative_to(directory)) + "\n" + for p in sorted(directory.rglob("*")) if p.is_file())) + with patch.object(package, "collect_artifacts", return_value=[{"manifest": row, "files": files}]), \ + patch("package_esp32_partition_migration.BOARDS", {"example": {}}), \ + patch("build_esp32_partition_migration.verify_archive"): + records, extras, _ = package.collect_local_release(directory, VERSION, SOURCE) + self.assertEqual(len(records), 1) + self.assertEqual(extras, [archive, bundle]) + with self.assertRaisesRegex(ValueError, "another source"): + package.collect_local_release(directory, VERSION, "b" * 40) + (directory / "unqualified.bin").write_bytes(b"unexpected") + with self.assertRaisesRegex(ValueError, "inventory is incomplete"): + package.collect_local_release(directory, VERSION, SOURCE) + + +if __name__ == "__main__": + unittest.main()