From 3e0ce84486a1fe0d07a73fd657df3b0cd2e6de31 Mon Sep 17 00:00:00 2001 From: mikecarper Date: Mon, 12 Oct 2026 21:47:08 -0700 Subject: [PATCH] Publish immutable short firmware download links Mirror pinned GitHub release bytes at shorter version-specific Pages URLs. Verify size, SHA-256 and firmware format before publishing, and preserve deployed alias identities across later site builds. Add RAK4631 Full Companion 1.17.1.10 preview2 UF2 and native DFU ZIP aliases, a short-download directory, documentation and CI regressions. Existing flashing tools use the ZIP URL without losing board-family checks. Validation: 16 short-link and Pages guard regressions, hosted-doc routing tests, firmware picker tests, full MkDocs build and exact UF2/ZIP byte checks. --- .github/workflows/github-pages.yml | 8 + .github/workflows/run-unit-tests.yml | 5 + .gitignore | 1 + docs/_data/firmware_shortlinks.json | 24 ++ docs/d.md | 35 +++ docs/firmware_picker.md | 4 + mkdocs.yml | 2 + .../check_published_firmware_shortlinks.py | 54 +++++ scripts/docs_firmware_shortlinks.py | 50 ++++ scripts/firmware_shortlinks.py | 223 +++++++++++++++++ test/test_docs_firmware_shortlinks.py | 121 ++++++++++ test/test_firmware_shortlinks.py | 228 ++++++++++++++++++ 12 files changed, 755 insertions(+) create mode 100644 docs/_data/firmware_shortlinks.json create mode 100644 docs/d.md create mode 100644 scripts/check_published_firmware_shortlinks.py create mode 100644 scripts/docs_firmware_shortlinks.py create mode 100644 scripts/firmware_shortlinks.py create mode 100644 test/test_docs_firmware_shortlinks.py create mode 100644 test/test_firmware_shortlinks.py diff --git a/.github/workflows/github-pages.yml b/.github/workflows/github-pages.yml index 7511497c..e9556c07 100644 --- a/.github/workflows/github-pages.yml +++ b/.github/workflows/github-pages.yml @@ -23,6 +23,12 @@ jobs: with: python-version: '3.13' + - name: Preserve published firmware short-link identities + run: | + python scripts/check_published_firmware_shortlinks.py \ + --manifest docs/_data/firmware_shortlinks.json \ + --previous "$RUNNER_TEMP/previous-firmware-shortlinks.json" + - name: Build env: GH_TOKEN: ${{ github.token }} @@ -32,6 +38,8 @@ jobs: MKDOCS_EDIT_URI: edit/${{ github.ref_name }}/docs/ run: | pip install mkdocs-material + python -B test/test_firmware_shortlinks.py -v + python -B test/test_docs_firmware_shortlinks.py -v node test/test_firmware_picker.js node test/test_partition_catalog_viewer.js python scripts/sync_bootloader_manifest.py diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml index c06b9c16..c9945661 100644 --- a/.github/workflows/run-unit-tests.yml +++ b/.github/workflows/run-unit-tests.yml @@ -191,6 +191,11 @@ jobs: - name: Verify build output cleanup safety run: python3 -B test/test_build_output_safety.py -v + - name: Verify immutable short firmware downloads and Pages publishing + run: | + python3 -B test/test_firmware_shortlinks.py -v + python3 -B test/test_docs_firmware_shortlinks.py -v + - name: Verify serial mOTA response deadlines and RX fairness run: | python3 -B test/test_mota_serial_deadline.py -v diff --git a/.gitignore b/.gitignore index ea6f96bb..88c834c2 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ .vscode/ipch out/ site/ +.cache/firmware-shortlinks/ # Local firmware archives and build logs .releases/ out.*/ diff --git a/docs/_data/firmware_shortlinks.json b/docs/_data/firmware_shortlinks.json new file mode 100644 index 00000000..aabcf96b --- /dev/null +++ b/docs/_data/firmware_shortlinks.json @@ -0,0 +1,24 @@ +{ + "schemaVersion": 1, + "repository": "mikecarper/MeshCore", + "links": [ + { + "filename": "rak4631-1.17.1.10-p2.uf2", + "version": "1.17.1.10", + "releaseTag": "rak4631-full-companion-v1.17.1.10-preview2", + "asset": "RAK_4631_companion_radio_full-v1.17.1.10-halo-keymind-cascade-preview2-015a3e8e.uf2", + "sha256": "b74243f9741133b0f0ae858fab02de9aaa999f84c474e0a659efce85fe35ad8c", + "size": 1408000, + "label": "RAK4631 Full Companion 1.17.1.10 preview2: UF2" + }, + { + "filename": "rak4631-1.17.1.10-p2.zip", + "version": "1.17.1.10", + "releaseTag": "rak4631-full-companion-v1.17.1.10-preview2", + "asset": "RAK_4631_companion_radio_full-v1.17.1.10-halo-keymind-cascade-preview2-015a3e8e.zip", + "sha256": "11783d291f80c8084bb16f46ad2e35f0b1631da10606723488e2061320a5f6db", + "size": 704759, + "label": "RAK4631 Full Companion 1.17.1.10 preview2: Serial/Bluetooth DFU ZIP" + } + ] +} diff --git a/docs/d.md b/docs/d.md new file mode 100644 index 00000000..bf60439c --- /dev/null +++ b/docs/d.md @@ -0,0 +1,35 @@ + +

View this page on MeshCore Docs.

+ + +# Short firmware links + +These links download the same firmware bytes as the original GitHub release, +with shorter filenames. Each link is pinned to one version and verified against +its SHA-256 checksum when the site is built. Existing links keep their original +firmware; a new release gets a new link. + + + +For **RAK4631 Full Companion**, use the **UF2** for a bootloader-drive +drag-and-drop update. Use the **DFU ZIP** when pasting a custom download URL +into the Windows or Linux MeshCore firmware tool. The ZIP is the native +application update package, not the release's download bundle. These files do +not install a bootloader or SoftDevice. + +The 1.17.1.10 preview2 files are for MeshTalk testing with Keymind Cascade and +USA Cascade defaults. The original release includes instructions, qualification +reports and checksums. See the [firmware picker](firmware_picker.md) for other +board and release choices. + +## Adding links for another release + +Add an entry to `docs/_data/firmware_shortlinks.json` with a unique, +version-specific filename, exact release tag and asset name, byte size, label +and SHA-256 checksum from the validated release. Keep the original firmware +extension. Never change or remove a published filename's version, source or +checksum. Labels can be corrected without changing the download. + +The documentation build downloads and verifies the pinned files before +publishing them under `d/`. A missing asset, incorrect checksum or changed +published link stops deployment instead of publishing a different image. diff --git a/docs/firmware_picker.md b/docs/firmware_picker.md index 7838f67d..06cb0cf8 100644 --- a/docs/firmware_picker.md +++ b/docs/firmware_picker.md @@ -4,6 +4,10 @@ # Firmware picker +For shorter, version-pinned firmware download URLs, use +[Short firmware links](d.md). Those links serve the original verified firmware +files and include the RAK4631 Full Companion 1.17.1.10 preview2 for MeshTalk testing. + Each result now includes **Restore your settings after flashing**. Its commands follow the selected logging mode and the exact image's verified role/hardware controls. Click On, Off, or Check to view and copy the commands; Companion diff --git a/mkdocs.yml b/mkdocs.yml index 234602cf..b9cf0776 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,6 +8,7 @@ edit_uri: !ENV [MKDOCS_EDIT_URI, 'edit/main/docs/'] hooks: - scripts/docs_partition_catalog.py + - scripts/docs_firmware_shortlinks.py # Keep historical URLs for bookmarks while showing only the organized copies. not_in_nav: | @@ -45,6 +46,7 @@ nav: - Home: index.md - Firmware and tools: - firmware_picker.md + - Short firmware links: d.md - notifications.md - preset_test.md - filter_tool.md diff --git a/scripts/check_published_firmware_shortlinks.py b/scripts/check_published_firmware_shortlinks.py new file mode 100644 index 00000000..f9b003a8 --- /dev/null +++ b/scripts/check_published_firmware_shortlinks.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Keep short download identities unchanged from the deployed Pages registry.""" + +import argparse +import os +from pathlib import Path +import subprocess + +from firmware_shortlinks import load_manifest, validate_immutable + + +PUBLISHED_PATH = "_data/firmware_shortlinks.json" + + +def check_published(manifest, previous): + current = load_manifest(manifest) + previous = Path(previous) + previous.unlink(missing_ok=True) + found = subprocess.run( + ["git", "ls-remote", "--exit-code", "--heads", "origin", "gh-pages"], + capture_output=True, text=True, check=False) + if found.returncode == 2: + return False # First Pages deployment: no published branch exists. + if found.returncode: + raise RuntimeError("Cannot check published Pages branch: " + found.stderr.strip()) + subprocess.run(["git", "fetch", "--depth=1", "--no-tags", "origin", "gh-pages"], check=True) + names = subprocess.check_output( + ["git", "ls-tree", "--name-only", "FETCH_HEAD", "--", PUBLISHED_PATH], text=True) + if not names.strip(): + return False # Existing site predates short firmware links. + data = subprocess.check_output(["git", "show", "FETCH_HEAD:" + PUBLISHED_PATH]) + previous.parent.mkdir(parents=True, exist_ok=True) + previous.write_bytes(data) + validate_immutable(load_manifest(previous), current) + return True + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--previous", type=Path, required=True) + args = parser.parse_args() + try: + existing = check_published(args.manifest, args.previous) + except (OSError, ValueError, RuntimeError, subprocess.CalledProcessError) as error: + parser.exit(1, "Firmware short-link deployment rejected: " + str(error) + "\n") + if existing and os.environ.get("GITHUB_ENV"): + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as environment: + environment.write("MESHCORE_SHORTLINKS_PREVIOUS=" + str(args.previous.resolve()) + "\n") + print("Published firmware links preserved." if existing else "First firmware short-link deployment.") + + +if __name__ == "__main__": + main() diff --git a/scripts/docs_firmware_shortlinks.py b/scripts/docs_firmware_shortlinks.py new file mode 100644 index 00000000..34ee0644 --- /dev/null +++ b/scripts/docs_firmware_shortlinks.py @@ -0,0 +1,50 @@ +"""Publish version-pinned firmware downloads at short, real binary URLs.""" + +import html +import logging +import os +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from firmware_shortlinks import download_url, load_manifest, mirror, validate_immutable + + +MARKER = "" + + +def registry(config): + return load_manifest(Path(config["docs_dir"]) / "_data/firmware_shortlinks.json") + + +def on_page_markdown(markdown, page, config, **kwargs): + if MARKER not in markdown: + return markdown + data = registry(config) + base = config["site_url"].rstrip("/") + "/d/" + rows = ["" + ""] + for link in data["links"]: + label = html.escape(link["label"], quote=True) + alias = base + link["filename"] + original = download_url(data, link).split("/releases/download/", 1)[0] + original += "/releases/tag/" + link["releaseTag"] + rows.append( + "") + rows.append("
FirmwareShort download linkOriginal release
" + label + "" + html.escape(link["filename"]) + "Release
") + return markdown.replace(MARKER, "\n".join(rows)) + + +def on_post_build(config, **kwargs): + data = registry(config) + previous = os.environ.get("MESHCORE_SHORTLINKS_PREVIOUS") + if previous: + validate_immutable(load_manifest(Path(previous)), data) + cache = Path(os.environ.get( + "MESHCORE_SHORTLINKS_CACHE", + str(Path(config.config_file_path).resolve().parent / ".cache/firmware-shortlinks"))) + outputs = mirror(data, Path(config["site_dir"]) / "d", cache) + logging.getLogger("mkdocs.plugins.firmware_shortlinks").info( + "Published %d checksum-verified short firmware downloads", len(outputs)) diff --git a/scripts/firmware_shortlinks.py b/scripts/firmware_shortlinks.py new file mode 100644 index 00000000..ba512369 --- /dev/null +++ b/scripts/firmware_shortlinks.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Mirror pinned release firmware as immutable, version-specific short URLs.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import struct +import tempfile +from urllib.parse import quote, urlsplit +from urllib.request import build_opener, HTTPRedirectHandler + + +REPOSITORY = "mikecarper/MeshCore" +EXTENSIONS = {".bin", ".uf2", ".zip", ".hex", ".mota"} +MAX_ASSET_SIZE = 32 * 1024 * 1024 +IMMUTABLE_FIELDS = ("version", "releaseTag", "asset", "sha256", "size") +DELIVERY_HOSTS = {"github.com", "release-assets.githubusercontent.com", "objects.githubusercontent.com"} + + +def _version_in(value, version): + return re.search(r"(? 128: + raise ValueError("links must contain 1..128 firmware aliases") + names = set() + for link in links: + if not isinstance(link, dict): + raise ValueError("firmware link must be an object") + filename, version = link.get("filename"), link.get("version") + if (not isinstance(filename, str) + or re.fullmatch(r"[a-z0-9][a-z0-9._-]{0,159}", filename) is None + or "latest" in filename): + raise ValueError("alias must be a safe flat filename without latest") + if (not isinstance(version, str) + or re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:\.[0-9]+)?", version) is None + or not _version_in(filename, version)): + raise ValueError(filename + ": alias must contain its exact numeric version") + asset, tag = link.get("asset"), link.get("releaseTag") + for field, value in (("asset", asset), ("releaseTag", tag)): + if (not isinstance(value, str) + or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,239}", value) is None + or not _version_in(value, version) or "latest" in value.lower()): + raise ValueError(filename + ": invalid version-specific " + field) + extension = Path(filename).suffix + if extension not in EXTENSIONS or Path(asset).suffix != extension: + raise ValueError(filename + ": alias and source need matching firmware extensions") + if (not isinstance(link.get("sha256"), str) + or re.fullmatch(r"[0-9a-f]{64}", link["sha256"]) is None): + raise ValueError(filename + ": invalid SHA-256") + if type(link.get("size")) is not int or not 1 <= link["size"] <= MAX_ASSET_SIZE: + raise ValueError(filename + ": size must be 1..32 MiB") + label = link.get("label") + if (not isinstance(label, str) or not label.strip() or len(label) > 160 + or any(ord(char) < 32 or ord(char) > 126 for char in label)): + raise ValueError(filename + ": label must be printable ASCII") + if filename in names: + raise ValueError("duplicate firmware alias: " + filename) + names.add(filename) + return manifest + + +def load_manifest(path): + path = Path(path) + if path.stat().st_size > 1024 * 1024: + raise ValueError("firmware registry exceeds 1 MiB") + return _validate_manifest(json.loads(path.read_text(encoding="utf-8"))) + + +def validate_immutable(previous, current): + """Keep published aliases present and pinned; display labels may change.""" + _validate_manifest(current) + if previous is None: + return + _validate_manifest(previous) + by_name = {link["filename"]: link for link in current["links"]} + for old in previous["links"]: + new = by_name.get(old["filename"]) + if new is None or any(new[field] != old[field] for field in IMMUTABLE_FIELDS): + raise ValueError("published firmware alias removed or retargeted: " + old["filename"]) + + +def download_url(manifest, link): + _validate_manifest(manifest) + if link not in manifest["links"]: + raise ValueError("link is not in the firmware registry") + return ("https://github.com/" + REPOSITORY + "/releases/download/" + + quote(link["releaseTag"], safe="") + "/" + quote(link["asset"], safe="")) + + +def _delivery_url_allowed(url): + parsed = urlsplit(url) + return (parsed.scheme == "https" and parsed.hostname in DELIVERY_HOSTS + and parsed.port in (None, 443) and not parsed.username and not parsed.password) + + +class _ReleaseRedirect(HTTPRedirectHandler): + def redirect_request(self, request, response, code, message, headers, url): + if not _delivery_url_allowed(url): + raise ValueError("release download redirected outside GitHub HTTPS delivery") + return super().redirect_request(request, response, code, message, headers, url) + + +def _open_release(url): + return build_opener(_ReleaseRedirect()).open(url, timeout=30) + + +def _check_content(head, extension): + text = head.lstrip(b"\xef\xbb\xbf \t\r\n").lower() + if text.startswith((b" link["size"]: + raise ValueError("release download exceeds its pinned size") + output.write(chunk) + output.flush() + os.fsync(output.fileno()) + _verify_file(destination, link) + + +def mirror(manifest, output_dir, cache_dir, opener=None): + """Validate cache and existing aliases before publishing real firmware files.""" + _validate_manifest(manifest) + output_dir, cache_dir = Path(output_dir), Path(cache_dir) + output_dir.mkdir(parents=True, exist_ok=True) + cache_dir.mkdir(parents=True, exist_ok=True) + for link in manifest["links"]: + output = output_dir / link["filename"] + if output.exists() or output.is_symlink(): + _verify_file(output, link) # An alias is immutable even across builds. + outputs = [] + for link in manifest["links"]: + output = output_dir / link["filename"] + cached = cache_dir / (link["sha256"] + Path(link["filename"]).suffix) + try: + _verify_file(cached, link) + except ValueError: + with tempfile.TemporaryDirectory(prefix=".firmware-", dir=cache_dir) as temporary: + candidate = Path(temporary) / "download" + _download(download_url(manifest, link), link, candidate, opener or _open_release) + os.replace(candidate, cached) + if not output.exists(): + with tempfile.TemporaryDirectory(prefix=".firmware-", dir=output_dir) as temporary: + candidate = Path(temporary) / "publish" + shutil.copyfile(cached, candidate) + _verify_file(candidate, link) + try: + os.link(candidate, output) # Atomic publication without overwriting a racing alias. + except FileExistsError: + _verify_file(output, link) + _verify_file(output, link) + outputs.append(output) + return outputs + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument("--previous", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--cache", type=Path) + parser.add_argument("--check-only", action="store_true") + args = parser.parse_args() + try: + manifest = load_manifest(args.manifest) + previous = load_manifest(args.previous) if args.previous is not None else None + validate_immutable(previous, manifest) + if not args.check_only: + if args.output is None or args.cache is None: + parser.error("--output and --cache are required unless --check-only") + mirror(manifest, args.output, args.cache) + print("Firmware shortlinks validated: " + str(len(manifest["links"]))) + except (OSError, ValueError, KeyError) as error: + parser.exit(1, "Firmware shortlink validation failed: " + str(error) + "\n") + + +if __name__ == "__main__": + main() diff --git a/test/test_docs_firmware_shortlinks.py b/test/test_docs_firmware_shortlinks.py new file mode 100644 index 00000000..b8038098 --- /dev/null +++ b/test/test_docs_firmware_shortlinks.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Verify rendered download URLs and the deployed-link identity guard.""" + +import copy +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from firmware_shortlinks import load_manifest + + +def module(name, filename): + spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / filename) + loaded = importlib.util.module_from_spec(spec) + spec.loader.exec_module(loaded) + return loaded + + +HOOK = module("docs_shortlinks_hook", "docs_firmware_shortlinks.py") +GUARD = module("published_shortlinks_guard", "check_published_firmware_shortlinks.py") +MANIFEST = ROOT / "docs/_data/firmware_shortlinks.json" + + +class DocsFirmwareShortlinksTests(unittest.TestCase): + def test_table_uses_site_base_and_versioned_real_file_urls(self): + config = {"docs_dir": str(ROOT / "docs"), "site_url": "https://example.test/fork/"} + data = load_manifest(MANIFEST) + rendered = HOOK.on_page_markdown("Before\n\n" + HOOK.MARKER + "\n\nAfter", None, config) + self.assertNotIn(HOOK.MARKER, rendered) + for entry in data["links"]: + self.assertIn('href="https://example.test/fork/d/' + entry["filename"] + '"', rendered) + self.assertIn("/releases/tag/" + entry["releaseTag"], rendered) + self.assertTrue(rendered.startswith("Before")) + self.assertTrue(rendered.endswith("After")) + self.assertEqual(HOOK.on_page_markdown("Unrelated page", None, config), "Unrelated page") + + def test_registry_labels_are_plain_text_not_html_or_markdown(self): + data = load_manifest(MANIFEST) + data["links"][0]["label"] = '|[bad](javascript:alert(1))' + with mock.patch.object(HOOK, "registry", return_value=data): + result = HOOK.on_page_markdown(HOOK.MARKER, None, {"site_url": "https://example.test/"}) + self.assertNotIn("