Add retained-RAM staging for nRF52 mOTA

tools/mota/motalib.py

@@ -73,6 +73,7 @@ NRF52_APP_END = 0x000ED000
 NRF52_BOOT_CONTAINER_SIZE = 41330
 NRF52_SHARED_BOOT_STAGE_START = 0x000E2000
 NRF52_FLASH_PAGE = 4096
+NRF52_HYBRID_RAM_SIZE = 65536
 
 # A validated nRF52 firmware carries this record immediately before EndF. It lets an offline packager
 # derive the actual app base and staging ceiling from the built artifact, without a board-name allowlist.
@@ -84,6 +85,7 @@ NRF52_LAYOUT_FLAG_SD = 0x01
 NRF52_LAYOUT_FLAG_INTERNAL_EXTRAFS = 0x02
 NRF52_LAYOUT_FLAG_QSPI = 0x04
 NRF52_LAYOUT_FLAG_BOOTLOADER_SCRATCH = 0x08
+NRF52_LAYOUT_FLAG_HYBRID_RAM = 0x10
 
 XIAO_BOOT_BOARD_ID_BASE = 0x28860044
 XIAO_BOOT_BOARD_ID_SENSE = 0x28860045
@@ -302,6 +304,10 @@ class Nrf52Layout:
     def bootloader_scratch(self) -> bool:
         return bool(self.flags & NRF52_LAYOUT_FLAG_BOOTLOADER_SCRATCH)
 
+    @property
+    def hybrid_ram(self) -> bool:
+        return bool(self.flags & NRF52_LAYOUT_FLAG_HYBRID_RAM)
+
 def nrf52_stage_ceiling_for_layout(linked_app_end: int, uses_internal_extrafs: bool) -> int:
     """Select a safe staging ceiling from linker geometry and actual secondary-storage type."""
     if uses_internal_extrafs:
@@ -311,6 +317,33 @@ def nrf52_stage_ceiling_for_layout(linked_app_end: int, uses_internal_extrafs: b
     return NRF52_EXTRAFS_START
 
 
+def nrf52_hybrid_stage_plan(total_size: int, app_base: int, app_end: int,
+                            stage_ceiling: int) -> Optional[Tuple[int, int, int]]:
+    """Return the frozen (flash_start, flash_len, ram_len) split, or None.
+
+    Keep this byte-for-byte equivalent to
+    mota_nrf52_hybrid_stage_plan() in OtaFlashLayout_nrf52.h. Hybrid is an
+    expanded ED000-only profile; at least one complete flash page contains the
+    header, manifest, leaves, and APRV word.
+    """
+    if (stage_ceiling != NRF52_APP_END or
+            app_base not in (NRF52_APP_BASE_S140_V6,
+                             NRF52_APP_BASE_S140_V7) or
+            not (app_base <= app_end <= stage_ceiling) or
+            total_size < 8 + 197 + 5 or total_size > 0xFFFFFFFF):
+        return None
+    flash_needed = max(NRF52_FLASH_PAGE,
+                       max(0, total_size - NRF52_HYBRID_RAM_SIZE))
+    flash_len = ((flash_needed + NRF52_FLASH_PAGE - 1) //
+                 NRF52_FLASH_PAGE) * NRF52_FLASH_PAGE
+    if flash_len >= total_size or flash_len > stage_ceiling - app_end:
+        return None
+    ram_len = total_size - flash_len
+    if not (0 < ram_len <= NRF52_HYBRID_RAM_SIZE):
+        return None
+    return stage_ceiling - flash_len, flash_len, ram_len
+
+
 def build_nrf52_layout(layout: Nrf52Layout) -> bytes:
     if layout.app_base not in (NRF52_APP_BASE_S140_V6, NRF52_APP_BASE_S140_V7):
         raise ValueError(f"unsupported nRF52 app base 0x{layout.app_base:X}")
@@ -322,13 +355,22 @@ def build_nrf52_layout(layout: Nrf52Layout) -> bytes:
     if not (layout.app_base < layout.linked_app_end <= NRF52_APP_END):
         raise ValueError("invalid nRF52 app region")
     known_flags = (NRF52_LAYOUT_FLAG_SD | NRF52_LAYOUT_FLAG_INTERNAL_EXTRAFS |
-                   NRF52_LAYOUT_FLAG_QSPI | NRF52_LAYOUT_FLAG_BOOTLOADER_SCRATCH)
+                   NRF52_LAYOUT_FLAG_QSPI | NRF52_LAYOUT_FLAG_BOOTLOADER_SCRATCH |
+                   NRF52_LAYOUT_FLAG_HYBRID_RAM)
     if layout.flags & ~known_flags:
         raise ValueError(f"unsupported nRF52 layout flags 0x{layout.flags:X}")
     if layout.sd_backed and layout.qspi_backed:
         raise ValueError("nRF52 layout cannot use both SD and QSPI staging")
     if layout.external_backed and layout.flags & NRF52_LAYOUT_FLAG_INTERNAL_EXTRAFS:
         raise ValueError("nRF52 external staging cannot also reserve internal ExtraFS")
+    if layout.hybrid_ram:
+        if (layout.external_backed or
+                layout.flags & NRF52_LAYOUT_FLAG_INTERNAL_EXTRAFS):
+            raise ValueError("nRF52 hybrid RAM staging requires internal flash without ExtraFS")
+        if (layout.linked_app_end != NRF52_APP_END or
+                layout.stage_ceiling != NRF52_APP_END):
+            raise ValueError(
+                "nRF52 hybrid RAM staging requires the exact 0xED000 internal-only profile")
     if layout.bootloader_scratch:
         qspi_shape = (layout.qspi_backed and not layout.sd_backed and
                       layout.linked_app_end == NRF52_BOOT_SCRATCH_START)

tools/mota/pio_endf.py

@@ -91,6 +91,57 @@ def _cppdef(name):                                # value of a -D<name>=<value>
     return None
 
 
+def _normalized_linker_path(value, project_dir):
+    """Resolve one SCons linker-path value without guessing among candidates."""
+    if isinstance(value, (list, tuple)):
+        if len(value) != 1:
+            return ""
+        value = value[0]
+    if hasattr(value, "get_abspath"):
+        value = value.get_abspath()
+    raw = str(value or "").strip().strip('"').strip("'")
+    if not raw:
+        return ""
+    if not os.path.isabs(raw):
+        raw = os.path.join(str(project_dir), raw)
+    return os.path.normcase(os.path.abspath(raw))
+
+
+def _validate_nrf52_hybrid_profile(app_start, linked_app_end,
+                                    hybrid_ram, internal_bootloader_update,
+                                    active_linker, configured_linker,
+                                    project_dir):
+    """Fail closed unless both macros and the dedicated SRAM linker agree."""
+    if hybrid_ram != internal_bootloader_update:
+        raise RuntimeError(
+            "nRF52 hybrid RAM staging and internal bootloader update must be enabled together")
+    if not hybrid_ram:
+        return
+
+    suffix_by_base = {
+        ml.NRF52_APP_BASE_S140_V6: "v6",
+        ml.NRF52_APP_BASE_S140_V7: "v7",
+    }
+    suffix = suffix_by_base.get(app_start)
+    if suffix is None or linked_app_end != ml.NRF52_APP_END:
+        raise RuntimeError(
+            "nRF52 hybrid RAM staging requires the exact S140 v6/v7 0xED000 profile")
+
+    expected = os.path.join(
+        str(project_dir), "boards", f"nrf52840_s140_{suffix}_mota64.ld")
+    expected = _normalized_linker_path(expected, project_dir)
+    if not expected or not os.path.isfile(expected):
+        raise RuntimeError("nRF52 hybrid RAM linker is missing")
+    for label, value in (("active", active_linker),
+                         ("configured", configured_linker)):
+        actual = _normalized_linker_path(value, project_dir)
+        if actual != expected:
+            shown = str(value or "none")
+            raise RuntimeError(
+                f"nRF52 hybrid RAM staging requires {expected}; "
+                f"{label} linker is {shown}")
+
+
 def _board_maximum_size(build_env):
     """Return PlatformIO's resolved application-size limit, including per-env overrides."""
     try:
@@ -215,8 +266,14 @@ def _append_endf_hex(source, target, env):        # Intel-HEX path (nRF52: app f
     sd_backed = _cppdef("OTA_SD_STORE") is not None
     qspi_backed = _cppdef("OTA_QSPI_STORE") is not None
     qspi_bootloader_update = _cppdef("OTA_QSPI_BOOTLOADER_UPDATE") is not None
+    hybrid_ram = _cppdef("OTA_HYBRID_RAM_STORE") is not None
     internal_bootloader_update = _cppdef("OTA_INTERNAL_BOOTLOADER_UPDATE") is not None
     sd_bootloader_update = _cppdef("OTA_SD_BOOTLOADER_UPDATE") is not None
+    board = env.BoardConfig()
+    _validate_nrf52_hybrid_profile(
+        app_start, linked_app_end, hybrid_ram, internal_bootloader_update,
+        env.get("LDSCRIPT_PATH", ""), board.get("build.ldscript", ""),
+        env["PROJECT_DIR"])
     if sd_backed and qspi_backed:
         raise RuntimeError("nRF52 build cannot enable both SD and QSPI OTA stores")
     if qspi_backed and _cppdef("QSPIFLASH") is not None:
@@ -242,7 +299,8 @@ def _append_endf_hex(source, target, env):        # Intel-HEX path (nRF52: app f
     layout_flags = ((ml.NRF52_LAYOUT_FLAG_SD if sd_backed else 0) |
                     (ml.NRF52_LAYOUT_FLAG_QSPI if qspi_backed else 0) |
                     (ml.NRF52_LAYOUT_FLAG_INTERNAL_EXTRAFS if internal_extrafs else 0) |
-                    (ml.NRF52_LAYOUT_FLAG_BOOTLOADER_SCRATCH if qspi_bootloader_update else 0))
+                    (ml.NRF52_LAYOUT_FLAG_BOOTLOADER_SCRATCH if qspi_bootloader_update else 0) |
+                    (ml.NRF52_LAYOUT_FLAG_HYBRID_RAM if hybrid_ram else 0))
     layout = ml.Nrf52Layout(app_start, linked_app_end, stage_ceiling, layout_flags)
     body = ml.ensure_nrf52_layout(raw_body, layout)
     ident = _firmware_ident()

tools/mota/test_mota.py

@@ -60,6 +60,13 @@ def test_internal_bootloader_target_wiring_is_central_and_not_duplicated():
     }
     assert gat_targets <= set(inventory)
     assert not any("GAT562_Mesh_Watch13" in target for target in inventory)
+    assert not any("xiao" in target.lower() for target in inventory)
+    assert not any("sdcard" in target.lower() for target in inventory)
+    assert {
+        "Heltec_t096_repeater_lora_ota_no_external_sensors",
+        "Heltec_t114_repeater_lora_ota_no_external_sensors",
+        "RAK_3401_repeater_lora_ota_no_external_sensors",
+    } <= set(inventory)
 
     target_header = (root / "src/helpers/ota/OtaTargets.h").read_text(encoding="utf-8")
     for target in inventory:
@@ -75,6 +82,83 @@ def test_internal_bootloader_target_wiring_is_central_and_not_duplicated():
     assert "scripts/nrf52_internal_bootloader_link.py" not in recipe
     assert "export MESHCORE_NRF52_INTERNAL_BOOTLOADER_UPDATE=1" in recipe
 
+    selector = (root / "scripts/nrf52_internal_bootloader_link.py").read_text(
+        encoding="utf-8")
+    assert '"OTA_HYBRID_RAM_STORE"' in selector
+    assert "nrf52840_s140_%s_mota64.ld" in selector
+    assert "env[\"PIOENV\"] in direct_targets" in selector
+
+    for version, app_base in (("v6", "0x26000"), ("v7", "0x27000")):
+        linker = (root / f"boards/nrf52840_s140_{version}_mota64.ld").read_text(
+            encoding="utf-8")
+        assert f"ORIGIN = {app_base}" in linker
+        assert "PERSISTENT_RAM (rwx) : ORIGIN = 0x20006000, LENGTH = 80" in linker
+        assert "MOTA_RAM (rwx) : ORIGIN = 0x20030000, LENGTH = 0x10000" in linker
+        assert ".mota_ram (NOLOAD)" in linker
+        ordinary = (root / f"boards/nrf52840_s140_{version}.ld").read_text(
+            encoding="utf-8")
+        assert "MOTA_RAM" not in ordinary
+
+    # Explicit external-storage and ExtraFS profiles remain on their existing
+    # linkers and are absent from the exact activation inventory.
+    tower = (root / "variants/heltec_tower_v2/platformio.ini").read_text(
+        encoding="utf-8")
+    assert "[env:Heltec_tower_v2_sdcard_repeater_lora_ota_no_external_sensors]" in tower
+    assert "board_build.ldscript = boards/nrf52840_s140_v6_sd_ota.ld" in tower
+    assert "board_build.ldscript = boards/nrf52840_s140_v6_extrafs.ld" in tower
+    xiao = (root / "variants/xiao_nrf52/platformio.ini").read_text(
+        encoding="utf-8")
+    assert "board_build.ldscript = boards/nrf52840_s140_v7.ld" in xiao
+
+
+def test_nrf52_hybrid_endf_profile_fails_closed():
+    root = Path(__file__).resolve().parents[2]
+    script = root / "tools/mota/pio_endf.py"
+    spec = importlib.util.spec_from_file_location("pio_endf_policy_test", script)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    module.Import = lambda _name: None
+    module.env = {"PROJECT_DIR": str(root), "CPPDEFINES": []}
+    spec.loader.exec_module(module)
+
+    validate = module._validate_nrf52_hybrid_profile
+    correct = root / "boards/nrf52840_s140_v6_mota64.ld"
+    validate(ml.NRF52_APP_BASE_S140_V6, ml.NRF52_APP_END,
+             True, True, correct, correct, root)
+    correct_v7 = root / "boards/nrf52840_s140_v7_mota64.ld"
+    validate(ml.NRF52_APP_BASE_S140_V7, ml.NRF52_APP_END,
+             True, True, [correct_v7], correct_v7, root)
+    # Ordinary profiles do not need a linker proof.
+    validate(ml.NRF52_APP_BASE_S140_V6, ml.NRF52_EXTRAFS_START,
+             False, False, "", "", root)
+
+    def rejected(*args):
+        try:
+            validate(*args)
+            assert False, "invalid nRF52 hybrid build profile accepted"
+        except RuntimeError as exc:
+            return str(exc)
+
+    assert "enabled together" in rejected(
+        ml.NRF52_APP_BASE_S140_V6, ml.NRF52_APP_END,
+        True, False, correct, correct, root)
+    assert "enabled together" in rejected(
+        ml.NRF52_APP_BASE_S140_V6, ml.NRF52_APP_END,
+        False, True, correct, correct, root)
+    assert "exact S140" in rejected(
+        ml.NRF52_APP_BASE_S140_V6, ml.NRF52_EXTRAFS_START,
+        True, True, correct, correct, root)
+    assert "exact S140" in rejected(
+        0x00028000, ml.NRF52_APP_END,
+        True, True, correct, correct, root)
+    ordinary = root / "boards/nrf52840_s140_v6.ld"
+    assert "active linker" in rejected(
+        ml.NRF52_APP_BASE_S140_V6, ml.NRF52_APP_END,
+        True, True, ordinary, correct, root)
+    assert "configured linker" in rejected(
+        ml.NRF52_APP_BASE_S140_V6, ml.NRF52_APP_END,
+        True, True, correct, ordinary, root)
+
 
 def test_full_esp32_profile_unifies_usb_logging_and_wifi_mqtt():
     root = Path(__file__).resolve().parents[2]
@@ -437,10 +521,9 @@ def test_measured_full_companion_promotions_are_exact_and_bounded():
     assert "+<helpers/esp32/SerialWifiInterface.cpp>" in profile
     assert "meshadventurer_sx1262_companion_radio_full" in profile
     assert "meshadventurer_sx1268_companion_radio_full" in profile
-    assert "-DMAX_CONTACTS=160" in profile
-    assert "-DMAX_GROUP_CHANNELS=30" in profile
-    assert "-DOFFLINE_QUEUE_SIZE=64" in profile
-    assert "160 contacts, 30 channels, and 64 queued frames" in profile
+    assert ("-DMAX_CONTACTS=100 -DMAX_GROUP_CHANNELS=30 "
+            "-DOFFLINE_QUEUE_SIZE=16") in profile
+    assert "100 contacts, 30 channels, and 16 queued frames" in profile
 
 
 def test_full_companion_wireless_startup_and_psram_contacts_are_resilient():
@@ -1083,6 +1166,19 @@ def test_nrf52_layout_record_roundtrip_and_policy():
         ml.ensure_nrf52_layout(_fw(9, 2048), shared_internal))
     assert ml.parse_nrf52_layout(shared_internal_image) == shared_internal
     assert not shared_internal.bootloader_scratch
+    hybrid = ml.Nrf52Layout(
+        ml.NRF52_APP_BASE_S140_V6, ml.NRF52_APP_END,
+        ml.NRF52_APP_END, ml.NRF52_LAYOUT_FLAG_HYBRID_RAM)
+    hybrid_image, _ = ml.ensure_endf(
+        ml.ensure_nrf52_layout(_fw(10, 2048), hybrid))
+    assert ml.parse_nrf52_layout(hybrid_image) == hybrid
+    assert hybrid.hybrid_ram and not hybrid.external_backed
+    assert ml.build_nrf52_layout(hybrid) == bytes.fromhex(
+        "6d4f54414c617931011018000060020000d00e0000d00e00")
+    hybrid_v7 = ml.Nrf52Layout(
+        ml.NRF52_APP_BASE_S140_V7, ml.NRF52_APP_END,
+        ml.NRF52_APP_END, ml.NRF52_LAYOUT_FLAG_HYBRID_RAM)
+    assert ml.build_nrf52_layout(hybrid_v7)
     assert ((ml.NRF52_APP_END - ml.NRF52_BOOT_CONTAINER_SIZE) &
             ~(ml.NRF52_FLASH_PAGE - 1)) == ml.NRF52_SHARED_BOOT_STAGE_START
     try:
@@ -1095,7 +1191,10 @@ def test_nrf52_layout_record_roundtrip_and_policy():
     for flags in (
         ml.NRF52_LAYOUT_FLAG_SD | ml.NRF52_LAYOUT_FLAG_QSPI,
         ml.NRF52_LAYOUT_FLAG_QSPI | ml.NRF52_LAYOUT_FLAG_INTERNAL_EXTRAFS,
-        0x10,
+        ml.NRF52_LAYOUT_FLAG_HYBRID_RAM | ml.NRF52_LAYOUT_FLAG_SD,
+        ml.NRF52_LAYOUT_FLAG_HYBRID_RAM | ml.NRF52_LAYOUT_FLAG_QSPI,
+        ml.NRF52_LAYOUT_FLAG_HYBRID_RAM |
+        ml.NRF52_LAYOUT_FLAG_INTERNAL_EXTRAFS,
     ):
         try:
             ml.build_nrf52_layout(ml.Nrf52Layout(
@@ -1105,6 +1204,50 @@ def test_nrf52_layout_record_roundtrip_and_policy():
         except ValueError:
             pass
 
+    # The flag is also a promise that the application used the dedicated
+    # mota64 RAM linker. A shorter flash linker with no ExtraFS mounted may
+    # ordinarily reclaim ED000 for staging, but it cannot reserve SRAM.
+    for linked_end, ceiling in (
+        (ml.NRF52_EXTRAFS_START, ml.NRF52_APP_END),
+        (ml.NRF52_APP_END, ml.NRF52_EXTRAFS_START),
+    ):
+        try:
+            ml.build_nrf52_layout(ml.Nrf52Layout(
+                ml.NRF52_APP_BASE_S140_V6, linked_end, ceiling,
+                ml.NRF52_LAYOUT_FLAG_HYBRID_RAM))
+            assert False, "non-mota64 hybrid layout accepted"
+        except ValueError:
+            pass
+
+
+def test_nrf52_hybrid_stage_plan_matches_frozen_geometry():
+    app = ml.NRF52_APP_BASE_S140_V6
+    app_end = app + 512 * 1024
+    page = ml.NRF52_FLASH_PAGE
+    ceiling = ml.NRF52_APP_END
+
+    assert ml.nrf52_hybrid_stage_plan(8 + 197 + 5, app, app_end, ceiling) is None
+    assert ml.nrf52_hybrid_stage_plan(page, app, app_end, ceiling) is None
+    assert ml.nrf52_hybrid_stage_plan(page + 1, app, app_end, ceiling) == (
+        ceiling - page, page, 1)
+    assert ml.nrf52_hybrid_stage_plan(17 * page, app, app_end, ceiling) == (
+        ceiling - page, page, 16 * page)
+    assert ml.nrf52_hybrid_stage_plan(17 * page + 1, app, app_end, ceiling) == (
+        ceiling - 2 * page, 2 * page, 15 * page + 1)
+
+    # Hybrid is never admitted against the D4000/ExtraFS ceiling, and exact
+    # runtime app headroom must fit the charged flash prefix.
+    assert ml.nrf52_hybrid_stage_plan(
+        17 * page, app, app_end, ml.NRF52_EXTRAFS_START) is None
+    assert ml.nrf52_hybrid_stage_plan(
+        17 * page + 1, app, ceiling - page, ceiling) is None
+    assert ml.nrf52_hybrid_stage_plan(
+        17 * page, 0x00028000, app_end, ceiling) is None
+    v7 = ml.NRF52_APP_BASE_S140_V7
+    assert ml.nrf52_hybrid_stage_plan(
+        17 * page, v7, v7 + 512 * 1024, ceiling) == (
+            ceiling - page, page, 16 * page)
+
 
 def _write_boot_continuity(image, offset, *, version=0x0117010D,
                            family=ml.BOOT_CONTINUITY_FAMILY_S140,