Publish immutable short firmware download links
Mirror pinned GitHub release bytes at shorter version-specific Pages URLs. Verify size, SHA-256 and firmware format before publishing, and preserve deployed alias identities across later site builds. Add RAK4631 Full Companion 1.17.1.10 preview2 UF2 and native DFU ZIP aliases, a short-download directory, documentation and CI regressions. Existing flashing tools use the ZIP URL without losing board-family checks. Validation: 16 short-link and Pages guard regressions, hosted-doc routing tests, firmware picker tests, full MkDocs build and exact UF2/ZIP byte checks.
This commit is contained in:
mikecarper committed
1 parent
42e5c0c07b
commit
5bd2c1fe4c
12 files changed
+755
No files matched your search
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify rendered download URLs and the deployed-link identity guard."""
|
||||
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
from firmware_shortlinks import load_manifest
|
||||
|
||||
|
||||
def module(name, filename):
|
||||
spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / filename)
|
||||
loaded = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(loaded)
|
||||
return loaded
|
||||
|
||||
|
||||
HOOK = module("docs_shortlinks_hook", "docs_firmware_shortlinks.py")
|
||||
GUARD = module("published_shortlinks_guard", "check_published_firmware_shortlinks.py")
|
||||
MANIFEST = ROOT / "docs/_data/firmware_shortlinks.json"
|
||||
|
||||
|
||||
class DocsFirmwareShortlinksTests(unittest.TestCase):
|
||||
def test_table_uses_site_base_and_versioned_real_file_urls(self):
|
||||
config = {"docs_dir": str(ROOT / "docs"), "site_url": "https://example.test/fork/"}
|
||||
data = load_manifest(MANIFEST)
|
||||
rendered = HOOK.on_page_markdown("Before\n\n" + HOOK.MARKER + "\n\nAfter", None, config)
|
||||
self.assertNotIn(HOOK.MARKER, rendered)
|
||||
for entry in data["links"]:
|
||||
self.assertIn('href="https://example.test/fork/d/' + entry["filename"] + '"', rendered)
|
||||
self.assertIn("/releases/tag/" + entry["releaseTag"], rendered)
|
||||
self.assertTrue(rendered.startswith("Before"))
|
||||
self.assertTrue(rendered.endswith("After"))
|
||||
self.assertEqual(HOOK.on_page_markdown("Unrelated page", None, config), "Unrelated page")
|
||||
|
||||
def test_registry_labels_are_plain_text_not_html_or_markdown(self):
|
||||
data = load_manifest(MANIFEST)
|
||||
data["links"][0]["label"] = '<script>alert("bad")</script>|[bad](javascript:alert(1))'
|
||||
with mock.patch.object(HOOK, "registry", return_value=data):
|
||||
result = HOOK.on_page_markdown(HOOK.MARKER, None, {"site_url": "https://example.test/"})
|
||||
self.assertNotIn("<script>", result)
|
||||
self.assertNotIn('href="javascript:', result)
|
||||
self.assertIn("<script>", result)
|
||||
|
||||
def test_pages_hook_mirrors_real_files_and_checks_previous_before_publication(self):
|
||||
data = load_manifest(MANIFEST)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
previous = Path(directory) / "previous.json"
|
||||
previous.write_text(json.dumps(data), encoding="ascii")
|
||||
class Config(dict):
|
||||
config_file_path = ROOT / "mkdocs.yml"
|
||||
config = Config(docs_dir=ROOT / "docs", site_dir=Path(directory) / "site")
|
||||
environment = {"MESHCORE_SHORTLINKS_PREVIOUS": str(previous),
|
||||
"MESHCORE_SHORTLINKS_CACHE": str(Path(directory) / "cache")}
|
||||
with mock.patch.dict(HOOK.os.environ, environment), mock.patch.object(HOOK, "mirror", return_value=[]) as mirror:
|
||||
HOOK.on_post_build(config)
|
||||
self.assertEqual(mirror.call_args.args,
|
||||
(data, Path(directory) / "site/d", Path(directory) / "cache"))
|
||||
changed = copy.deepcopy(data)
|
||||
changed["links"][0]["sha256"] = "0" * 64
|
||||
previous.write_text(json.dumps(changed), encoding="ascii")
|
||||
with mock.patch.dict(HOOK.os.environ, environment), mock.patch.object(HOOK, "mirror") as mirror:
|
||||
with self.assertRaises(ValueError):
|
||||
HOOK.on_post_build(config)
|
||||
mirror.assert_not_called()
|
||||
|
||||
def test_first_pages_deployment_does_not_reuse_an_old_previous_file(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
previous = Path(directory) / "previous.json"
|
||||
previous.write_text("stale", encoding="ascii")
|
||||
result = subprocess.CompletedProcess([], 2, "", "")
|
||||
with mock.patch.object(GUARD.subprocess, "run", return_value=result):
|
||||
self.assertFalse(GUARD.check_published(MANIFEST, previous))
|
||||
self.assertFalse(previous.exists())
|
||||
|
||||
def test_existing_site_without_registry_is_a_first_shortlink_deployment(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
previous = Path(directory) / "previous.json"
|
||||
with mock.patch.object(GUARD.subprocess, "run", return_value=subprocess.CompletedProcess([], 0, "sha", "")) as run, \
|
||||
mock.patch.object(GUARD.subprocess, "check_output", return_value=""):
|
||||
self.assertFalse(GUARD.check_published(MANIFEST, previous))
|
||||
self.assertEqual(run.call_args_list[1].args[0],
|
||||
["git", "fetch", "--depth=1", "--no-tags", "origin", "gh-pages"])
|
||||
self.assertFalse(previous.exists())
|
||||
|
||||
def test_published_alias_retarget_and_removal_fail_the_deployment_guard(self):
|
||||
original = load_manifest(MANIFEST)
|
||||
for changed in (False, True):
|
||||
with self.subTest(retarget=changed), tempfile.TemporaryDirectory() as directory:
|
||||
current = copy.deepcopy(original)
|
||||
if changed:
|
||||
current["links"][0]["sha256"] = "0" * 64
|
||||
else:
|
||||
current["links"].pop()
|
||||
manifest = Path(directory) / "new.json"
|
||||
manifest.write_text(json.dumps(current), encoding="ascii")
|
||||
previous = Path(directory) / "previous.json"
|
||||
replies = [GUARD.PUBLISHED_PATH + "\n", json.dumps(original).encode("ascii")]
|
||||
with mock.patch.object(GUARD.subprocess, "run", return_value=subprocess.CompletedProcess([], 0, "sha", "")), \
|
||||
mock.patch.object(GUARD.subprocess, "check_output", side_effect=replies):
|
||||
with self.assertRaises(ValueError):
|
||||
GUARD.check_published(manifest, previous)
|
||||
|
||||
def test_network_failure_is_not_treated_as_an_empty_published_registry(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
result = subprocess.CompletedProcess([], 128, "", "Network unavailable")
|
||||
with mock.patch.object(GUARD.subprocess, "run", return_value=result):
|
||||
with self.assertRaisesRegex(RuntimeError, "Cannot check"):
|
||||
GUARD.check_published(MANIFEST, Path(directory) / "previous.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,228 @@
|
||||
"""Keep short firmware URLs pinned, binary, bounded and safe to mirror."""
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from urllib.request import Request
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPT = ROOT / "scripts/firmware_shortlinks.py"
|
||||
SPEC = importlib.util.spec_from_file_location("firmware_shortlinks", SCRIPT)
|
||||
SHORTLINKS = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(SHORTLINKS)
|
||||
|
||||
UF2 = struct.pack("<II", 0x0A324655, 0x9E5D5157) + bytes(504)
|
||||
ZIP = b"PK\x03\x04" + b"application DFU fixture" * 7
|
||||
TAG = "rak4631-full-companion-v1.17.1.10-preview2"
|
||||
|
||||
|
||||
def registry(data=UF2, extension="uf2"):
|
||||
return {"schemaVersion": 1, "repository": "mikecarper/MeshCore", "links": [{
|
||||
"filename": "rak4631-1.17.1.10-p2." + extension,
|
||||
"version": "1.17.1.10", "releaseTag": TAG,
|
||||
"asset": "RAK_4631_companion_radio_full-v1.17.1.10-halo-keymind-cascade-preview2-015a3e8e." + extension,
|
||||
"sha256": hashlib.sha256(data).hexdigest(), "size": len(data),
|
||||
"label": "RAK4631 Full Companion preview2"}]}
|
||||
|
||||
|
||||
class Response(io.BytesIO):
|
||||
def __init__(self, data, content_type="application/octet-stream", url=None):
|
||||
super().__init__(data)
|
||||
self.headers = {"Content-Type": content_type}
|
||||
self.url = url or "https://release-assets.githubusercontent.com/example"
|
||||
|
||||
def read(self, size=-1):
|
||||
return super().read(min(size, 17) if size >= 0 else 17)
|
||||
|
||||
def geturl(self):
|
||||
return self.url
|
||||
|
||||
|
||||
class FirmwareShortlinkTests(unittest.TestCase):
|
||||
def load(self, manifest):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "registry.json"
|
||||
path.write_text(json.dumps(manifest), encoding="ascii")
|
||||
return SHORTLINKS.load_manifest(path)
|
||||
|
||||
def test_real_preview_url_is_derived_from_fixed_repository(self):
|
||||
manifest = self.load(registry())
|
||||
self.assertEqual(SHORTLINKS.download_url(manifest, manifest["links"][0]),
|
||||
"https://github.com/mikecarper/MeshCore/releases/download/" + TAG + "/"
|
||||
"RAK_4631_companion_radio_full-v1.17.1.10-halo-keymind-cascade-preview2-015a3e8e.uf2")
|
||||
foreign = dict(manifest["links"][0], asset="other-v1.17.1.10.uf2")
|
||||
with self.assertRaisesRegex(ValueError, "not in"):
|
||||
SHORTLINKS.download_url(manifest, foreign)
|
||||
|
||||
def test_registry_rejects_paths_mutable_names_wrong_sources_and_bad_types(self):
|
||||
changes = [
|
||||
("filename", "../rak4631-1.17.1.10.uf2"),
|
||||
("filename", "directory/rak4631-1.17.1.10.uf2"),
|
||||
("filename", "https://example.com/1.17.1.10.uf2"),
|
||||
("filename", "rak4631-latest-1.17.1.10.uf2"),
|
||||
("filename", "rak4631-1.17.1.100-p2.uf2"),
|
||||
("filename", "rak4631-1.17.1.10-p2.html"),
|
||||
("asset", "other-v1.17.1.10.zip"),
|
||||
("asset", "https://example.com/v1.17.1.10.uf2"),
|
||||
("asset", "firmware-v1.17.1.10.uf2?download=1"),
|
||||
("releaseTag", "latest-v1.17.1.10"),
|
||||
("releaseTag", "v1.17.1.9-preview2"),
|
||||
("version", "latest"), ("version", "v1.17.1.10"),
|
||||
("sha256", "a" * 63), ("sha256", "z" * 64),
|
||||
("sha256", int("1" * 64)),
|
||||
("size", 0), ("size", True), ("size", 32 * 1024 * 1024 + 1),
|
||||
("label", "bad\nlabel"), ("label", "")]
|
||||
for field, value in changes:
|
||||
with self.subTest(field=field, value=value):
|
||||
manifest = registry()
|
||||
manifest["links"][0][field] = value
|
||||
with self.assertRaises(ValueError):
|
||||
self.load(manifest)
|
||||
for field, value in (("repository", "attacker/MeshCore"), ("schemaVersion", True),
|
||||
("schemaVersion", 2), ("links", [])):
|
||||
manifest = registry()
|
||||
manifest[field] = value
|
||||
with self.subTest(field=field), self.assertRaises(ValueError):
|
||||
self.load(manifest)
|
||||
manifest = registry()
|
||||
manifest["links"].append(dict(manifest["links"][0]))
|
||||
with self.assertRaisesRegex(ValueError, "duplicate"):
|
||||
self.load(manifest)
|
||||
|
||||
def test_published_aliases_cannot_disappear_or_change_but_labels_can(self):
|
||||
previous = registry()
|
||||
current = copy.deepcopy(previous)
|
||||
current["links"][0]["label"] = "Improved description"
|
||||
current["links"].extend(registry(ZIP, "zip")["links"])
|
||||
SHORTLINKS.validate_immutable(previous, current)
|
||||
SHORTLINKS.validate_immutable(None, current)
|
||||
for field, value in (("releaseTag", TAG + "-changed"),
|
||||
("asset", "different-v1.17.1.10.uf2"),
|
||||
("sha256", "a" * 64), ("size", 1024)):
|
||||
modified = copy.deepcopy(current)
|
||||
modified["links"][0][field] = value
|
||||
with self.subTest(field=field), self.assertRaisesRegex(ValueError, "retargeted"):
|
||||
SHORTLINKS.validate_immutable(previous, modified)
|
||||
removed = registry(ZIP, "zip")
|
||||
with self.assertRaisesRegex(ValueError, "removed"):
|
||||
SHORTLINKS.validate_immutable(previous, removed)
|
||||
|
||||
def test_mirror_streams_real_bytes_and_reuses_only_validated_cache(self):
|
||||
manifest = registry()
|
||||
manifest["links"].extend(registry(ZIP, "zip")["links"])
|
||||
calls = []
|
||||
def opener(url):
|
||||
calls.append(url)
|
||||
return Response(UF2 if url.endswith(".uf2") else ZIP)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output, cache = Path(directory) / "out", Path(directory) / "cache"
|
||||
published = SHORTLINKS.mirror(manifest, output, cache, opener)
|
||||
self.assertEqual([path.read_bytes() for path in published], [UF2, ZIP])
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertTrue(all(url.startswith("https://github.com/mikecarper/MeshCore/releases/download/")
|
||||
for url in calls))
|
||||
SHORTLINKS.mirror(manifest, output, cache, opener)
|
||||
self.assertEqual(len(calls), 2)
|
||||
# Corruption with the same size must not bypass the cache check.
|
||||
cached = cache / (manifest["links"][0]["sha256"] + ".uf2")
|
||||
cached.write_bytes(UF2[:20] + b"!" + UF2[21:])
|
||||
published[0].unlink()
|
||||
SHORTLINKS.mirror(manifest, output, cache, opener)
|
||||
self.assertEqual(len(calls), 3)
|
||||
self.assertEqual(published[0].read_bytes(), UF2)
|
||||
self.assertEqual(cached.read_bytes(), UF2)
|
||||
self.assertFalse(list(output.glob(".firmware-*")))
|
||||
self.assertFalse(list(cache.glob(".firmware-*")))
|
||||
|
||||
def test_existing_alias_conflicts_fail_before_any_download(self):
|
||||
manifest = registry()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output, cache = Path(directory) / "out", Path(directory) / "cache"
|
||||
output.mkdir()
|
||||
alias = output / manifest["links"][0]["filename"]
|
||||
conflict = UF2[:20] + b"!" + UF2[21:]
|
||||
alias.write_bytes(conflict)
|
||||
def opener(_url):
|
||||
self.fail("existing alias conflicts must fail before network work")
|
||||
with self.assertRaisesRegex(ValueError, "SHA-256"):
|
||||
SHORTLINKS.mirror(manifest, output, cache, opener)
|
||||
self.assertEqual(alias.read_bytes(), conflict)
|
||||
|
||||
def test_alias_created_during_download_is_not_overwritten(self):
|
||||
manifest = registry()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output, cache = Path(directory) / "out", Path(directory) / "cache"
|
||||
alias = output / manifest["links"][0]["filename"]
|
||||
def opener(_url):
|
||||
alias.write_bytes(b"existing unrelated file")
|
||||
return Response(UF2)
|
||||
with self.assertRaises(ValueError):
|
||||
SHORTLINKS.mirror(manifest, output, cache, opener)
|
||||
self.assertEqual(alias.read_bytes(), b"existing unrelated file")
|
||||
|
||||
def test_bad_network_data_never_publishes_or_poison_cache(self):
|
||||
cases = ((registry(), UF2[:-1], "application/octet-stream", None),
|
||||
(registry(), UF2 + b"excess", "application/octet-stream", None),
|
||||
(registry(), UF2[:20] + b"!" + UF2[21:], "application/octet-stream", None),
|
||||
(registry(), UF2, "text/html", None),
|
||||
(registry(), UF2, "application/javascript", None),
|
||||
(registry(), UF2, "application/octet-stream", "http://release-assets.githubusercontent.com/example"),
|
||||
(registry(), UF2, "application/octet-stream", "https://attacker.example/download"),
|
||||
(registry(b"<!DOCTYPE html><html>redirect</html>", "bin"),
|
||||
b"<!DOCTYPE html><html>redirect</html>", "application/octet-stream", None),
|
||||
(registry(b"window.location = 'https://example.com';", "bin"),
|
||||
b"window.location = 'https://example.com';", "application/octet-stream", None),
|
||||
(registry(b"not firmware", "uf2"), b"not firmware", "application/octet-stream", None))
|
||||
for manifest, data, content_type, url in cases:
|
||||
with self.subTest(content_type=content_type, url=url, data=data[:20]):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output, cache = Path(directory) / "out", Path(directory) / "cache"
|
||||
with self.assertRaises(ValueError):
|
||||
SHORTLINKS.mirror(manifest, output, cache,
|
||||
lambda _url: Response(data, content_type, url))
|
||||
self.assertFalse(list(output.iterdir()))
|
||||
self.assertFalse(list(cache.iterdir()))
|
||||
|
||||
def test_redirects_allow_only_github_https_asset_delivery(self):
|
||||
handler = SHORTLINKS._ReleaseRedirect()
|
||||
request = Request("https://github.com/mikecarper/MeshCore/releases/download/" + TAG + "/app.uf2")
|
||||
for url in ("http://github.com/app.uf2", "https://github.com.attacker.example/app.uf2",
|
||||
"https://attacker.example/app.uf2", "https://user:secret@github.com/app.uf2"):
|
||||
with self.subTest(url=url), self.assertRaises(ValueError):
|
||||
handler.redirect_request(request, None, 302, "Found", {}, url)
|
||||
allowed = handler.redirect_request(request, None, 302, "Found", {},
|
||||
"https://release-assets.githubusercontent.com/example?token=signed")
|
||||
self.assertEqual(allowed.full_url,
|
||||
"https://release-assets.githubusercontent.com/example?token=signed")
|
||||
|
||||
def test_cli_check_only_runs_without_network_and_enforces_previous(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
current, previous = Path(directory) / "current.json", Path(directory) / "previous.json"
|
||||
manifest = registry()
|
||||
current.write_text(json.dumps(manifest), encoding="ascii")
|
||||
previous.write_text(json.dumps(manifest), encoding="ascii")
|
||||
command = [sys.executable, str(SCRIPT), "--manifest", str(current), "--check-only"]
|
||||
result = subprocess.run(command, capture_output=True, text=True, timeout=10)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
result = subprocess.run(command + ["--previous", str(previous)],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
manifest["links"][0]["sha256"] = "a" * 64
|
||||
current.write_text(json.dumps(manifest), encoding="ascii")
|
||||
result = subprocess.run(command + ["--previous", str(previous)],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("retargeted", result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user