Fit direct OTA administration into RAK4631 Full

test/fixtures/ota_folder_resume.cpp

@@ -319,7 +319,36 @@ static void established_capture_still_refuses_replacement_file() {
   assert(f.host.begins == 0 && f.flight.manifests == 0);
 }
 
+static void pull_parser_accepts_only_complete_destination_words() {
+  for (const char* text : {"pull 1 folder", "pull 1   folder   ",
+      "pull 1 \tfolder\v\fvalidate\r\n", "pull 1 folder validate"}) {
+    Fixture f;
+    char reply[160] = {};
+    assert(pull_command(text, reply, f.context));
+    assert(strncmp(reply, "OK ", 3) == 0);
+  }
+  // Reaching the seeder policy proves both flash syntaxes parsed successfully.
+  for (const char* text : {"pull 1 flash", "pull 1 \tflash\v\frescue\r\n"}) {
+    Fixture f;
+    char reply[160] = {};
+    assert(pull_command(text, reply, f.context));
+    assert(strcmp(reply, "ERR seeder-only build cannot stage or install firmware; use `folder`") == 0);
+  }
+  for (const char* text : {"pull 1 \t\r\n", "pull 1 flashgarbage",
+      "pull 1 folders", "pull 1 folderss", "pull 1 folder validates",
+      "pull 1 folder validatess", "pull 1 folder validate extra",
+      "pull 1 flash rescue extra", "pull 1 folder rescue", "pull 1 flash validate",
+      "pull 1 foldervalidate"}) {
+    Fixture f;
+    char reply[160] = {};
+    assert(pull_command(text, reply, f.context));
+    assert(strncmp(reply, "ERR usage:", 10) == 0);
+    assert(f.host.begins == 0 && f.flight.manifests == 0);
+  }
+}
+
 int main() {
+  pull_parser_accepts_only_complete_destination_words();
   for (auto failure : {HostStream::Timeout, HostStream::Truncated, HostStream::BadChecksum,
                        HostStream::Error, HostStream::ShortWrite, HostStream::WrongOp}) {
     for (bool retry_by_pull : {false, true}) {

test/test_bootloader_update_cli.py

@@ -157,6 +157,28 @@ int main() {
   expect("bootloader install 12345678 0123456789ABCDEF extra",
          "ERR usage: ota bootloader install <MID8> <HASH16>", c);
   assert(c.staged_installs == 1);
+  for (const char* text : {
+      "bootloader install   12345678   0123456789ABCDEF   ",
+      "bootloader install \t12345678\v\f0123456789ABCDEF\r\n"}) {
+    expect(text, "OK | staged install", c);
+  }
+  assert(c.staged_installs == 3);
+  for (const char* text : {
+      "bootloader install \t\n", "bootloader install 12345678",
+      "bootloader install 123456789 0123456789ABCDEF",
+      "bootloader install 12345678 0123456789ABCDEFF",
+      "bootloader install 123456780123456789ABCDEF",
+      "bootloader install 12345678 0123456789ABCDEF \t extra"}) {
+    expect(text, "ERR usage: ota bootloader install <MID8> <HASH16>", c);
+  }
+  for (const char* text : {
+      "bootloader install 1234567 0123456789ABCDEF",
+      "bootloader install 12345678 0123456789ABCDE",
+      "bootloader install 1234567Z 0123456789ABCDEF",
+      "bootloader install 12345678 0123456789ABCDEZ"}) {
+    expect(text, "ERR MID must be 8 hex and HASH must be 16 hex characters", c);
+  }
+  assert(c.staged_installs == 3);
 #else
   expect("bootloader install 12345678 0123456789ABCDEF",
          "ERR this build cannot update its bootloader over LoRa", c);
@@ -171,7 +193,8 @@ def harness():
     context = (ROOT / "src/helpers/ota/OtaContext.h").read_text(encoding="utf-8")
     return (SOURCE.replace("@IDLE@", body(context, "bool embeddedBootloaderIdle() const"))
             .replace("@HELPERS@", "\n".join(body(cli, signature) for signature in (
-                "static bool is_cmd(", "static bool parse_hex_exact(")))
+                "static bool is_cmd(", "static bool parse_hex_exact(",
+                "static __attribute__((noinline)) int parse_two_words(")))
             .replace("@COMMAND@", body(cli, 'if (is_cmd(a, "bootloader|blupdate", &rest))')))
 
 

test/test_companion_direct_admin_guard.py

@@ -25,19 +25,23 @@ HARNESS = r'''
 #define TXT_TYPE_CLI_DATA 1
 struct ContactInfo {
   mesh::Identity id;
-  uint8_t type=0, out_path_len=0, tx_radio=0;
+  uint8_t type=0, out_path_len=0, tx_radio=0, flags=0;
+  char name[32]{};
   uint32_t sync_since=0;
-  uint8_t out_path[64]{},secret[32]{};
-  const uint8_t* getSharedSecret(const mesh::LocalIdentity&) {memset(secret,0x44,32);return secret;}
+  uint8_t out_path[64]{};
+  mutable uint8_t secret[32]{};
+  const uint8_t* getSharedSecret(const mesh::LocalIdentity&) const {memset(secret,0x44,32);return secret;}
   bool setPath(const uint8_t*,uint8_t length){out_path_len=length;memset(out_path,0,sizeof(out_path));return true;}
 };
 struct BaseSerialInterface {BaseSerialInterface* route=this;BaseSerialInterface* captureReplyRoute(){return route;}};
 struct Timer { uint32_t now=100;uint32_t getMillis()const{return now;} };
 struct BaseChatMesh:mesh::Mesh {
   ContactInfo contact,anonymous[MAX_ANON_CONTACTS];
-  int data_calls=0,path_calls=0,login_calls=0,clear_calls=0;
-  BaseChatMesh(){memset(contact.id.pub_key,0x11,32);contact.type=2;contact.out_path_len=2;contact.out_path[0]=0xAA;}
+  int data_calls=0,path_calls=0,login_calls=0,clear_calls=0,lookup_length=0;
+  BaseChatMesh(){memset(contact.id.pub_key,0x11,32);contact.type=2;contact.out_path_len=2;contact.out_path[0]=0xAA;
+    contact.flags=31;contact.sync_since=987;contact.tx_radio=2;strcpy(contact.name,"Saved contact");}
   ContactInfo* lookupContactByPubKey(const uint8_t* key,int n){
+    lookup_length=n;
     for(auto& p:anonymous) if(!memcmp(key,p.id.pub_key,n)) return &p;
     return !memcmp(key,contact.id.pub_key,n)?&contact:nullptr;
   }
@@ -45,7 +49,13 @@ struct BaseChatMesh:mesh::Mesh {
   bool isTransientContact(const ContactInfo& p){for(auto& a:anonymous)if(&a==&p)return true;return false;}
   void resetContactValue(ContactInfo& c){c=ContactInfo();}
   bool clearTransientContact(ContactInfo& c){assert(isTransientContact(c));resetContactValue(c);++clear_calls;return true;}
-  uint8_t getContactTxRadio(const mesh::Identity&)const{return 2;}
+  virtual uint8_t getContactTxRadio(const mesh::Identity&)const{return 2;}
+  int sendLogin(const ContactInfo&,const char*,uint32_t&);
+  int sendContactPacket(const ContactInfo& peer,mesh::Packet* packet,uint32_t& timeout){
+    assert(peer.out_path_len==0);timeout=0;
+    packet->tx_radio=getContactTxRadio(peer.id);
+    return sendDirect(packet,nullptr,peer.out_path_len)?1:0;
+  }
   void onPeerDataRecv(mesh::Packet*,uint8_t,int,const uint8_t*,uint8_t*,size_t){++data_calls;}
   bool onPeerPathRecv(mesh::Packet*,int,const uint8_t*,uint8_t*,uint8_t,uint8_t,uint8_t*,uint8_t){++path_calls;return true;}
   void onContactResponse(const ContactInfo&,const uint8_t*,uint8_t){++login_calls;}
@@ -71,9 +81,9 @@ struct MyMesh:BaseChatMesh {
   bool handleDirectOtaFrame(size_t len);
   bool handleDirectOtaCommand(const char*,uint32_t,char*,size_t);
   void closeDirectOtaAdmin();
-  void mode(int value,const mesh::Identity* peer=nullptr){
+  void mode(int value,const mesh::Identity* peer=nullptr,uint8_t type=ADV_TYPE_REPEATER){
     cmd_frame[0]=52;cmd_frame[1]=value;memcpy(cmd_frame+2,(peer?peer->pub_key:contact.id.pub_key),32);
-    cmd_frame[34]=2;dispatch(value==1?35:34);
+    cmd_frame[34]=type;dispatch(value==1?35:34);
   }
   bool command(const char* command,uint32_t stamp,char* reply){
 #if FULL
@@ -84,7 +94,11 @@ struct MyMesh:BaseChatMesh {
   }
   void expect(const char* prefix){cmd_frame[0]=52;cmd_frame[1]=4;memcpy(cmd_frame+2,contact.id.pub_key,32);memcpy(cmd_frame+34,prefix,3);dispatch(37);assert(response==0);}
   void onPeerDataRecv(mesh::Packet*,uint8_t,int,const uint8_t*,uint8_t*,size_t);
+#if FULL
   uint8_t getContactTxRadio(const mesh::Identity&)const;
+#else
+  uint8_t getContactTxRadio(const mesh::Identity& peer)const override{return BaseChatMesh::getContactTxRadio(peer);}
+#endif
   bool onPeerPathRecv(mesh::Packet*,int,const uint8_t*,uint8_t*,uint8_t,uint8_t,uint8_t*,uint8_t);
   void rawRoute(mesh::Packet* pkt){
 #if FULL
@@ -96,6 +110,7 @@ struct MyMesh:BaseChatMesh {
 };
 @CALLBACKS@
 @BINARY@
+@SEND_LOGIN@
 static void repeaterPrefix(char*& command,char*& reply){@REPEATER_PREFIX@}
 static void roomPrefix(char*& command,char*& reply){@ROOM_PREFIX@}
 [[maybe_unused]] static void body(uint8_t* data,uint32_t stamp,const char* prefix,const char* text){
@@ -119,7 +134,12 @@ int main(){
 #if FULL
   assert(m.response==0&&m.calls==0); // capability negotiation has no RF
   assert(m.contact.out_path_len==2&&m.contact.out_path[0]==0xAA);
+  assert(m.lookup_length==PUB_KEY_SIZE&&m._ota_direct_contact.id.matches(m.contact.id));
   assert(m._ota_direct_contact.out_path_len==0&&m._ota_direct_contact.tx_radio==1);
+  assert(m._ota_direct_contact.type==ADV_TYPE_REPEATER&&m._ota_direct_contact.sync_since==0);
+  assert(m._ota_direct_contact.flags==0&&m._ota_direct_contact.name[0]==0);
+  assert(m.contact.flags==31&&m.contact.sync_since==987&&m.contact.tx_radio==2);
+  assert(!strcmp(m.contact.name,"Saved contact"));
   assert(m.getContactTxRadio(m.contact.id)==mesh::RADIO_TX_PRIMARY);
   mesh::Identity another;assert(m.getContactTxRadio(another)==2);
   mesh::Packet ota;ota.payload_type=12;m.rawRoute(&ota);assert(ota.tx_radio==mesh::RADIO_TX_PRIMARY);
@@ -181,8 +201,22 @@ int main(){
   m.mode(3);assert(m.response==0&&!m._ota_direct_admin.active(m.timer.now));
   assert(m.getContactTxRadio(m.contact.id)==2);
   m.mode(3);assert(m.response==0); // cleanup is idempotent
+  m.contact.type=ADV_TYPE_ROOM;
+  m.mode(1,nullptr,ADV_TYPE_ROOM);
+  assert(m.response==0&&m._ota_direct_contact.type==ADV_TYPE_ROOM);
+  assert(m._ota_direct_contact.sync_since==0&&m._ota_direct_contact.id.matches(m.contact.id));
+  uint32_t timeout=0;
+  assert(m.sendLogin(m._ota_direct_contact,"secret",timeout)==1);
+  assert(m.type==PAYLOAD_TYPE_ANON_REQ&&m.packet.tx_radio==mesh::RADIO_TX_PRIMARY);
+  assert(m.sent.size()==14&&!memcmp(m.sent.data()+8,"secret",6));
+  for(unsigned i=4;i<8;++i)assert(m.sent[i]==0); // Room login requests no historical cursor.
+  assert(m.contact.type==ADV_TYPE_ROOM&&m.contact.sync_since==987);
+  assert(m.contact.out_path_len==2&&m.contact.out_path[0]==0xAA&&m.contact.tx_radio==2);
+  m.mode(3);assert(m.response==0);
   mesh::Identity fresh;memset(fresh.pub_key,0x22,32);m.mode(1,&fresh);
   assert(m.response==0&&m._ota_direct_transient);
+  assert(m._ota_direct_contact.id.matches(fresh)&&m._ota_direct_contact.type==ADV_TYPE_REPEATER);
+  assert(m._ota_direct_contact.out_path_len==0&&m._ota_direct_contact.tx_radio==mesh::RADIO_TX_PRIMARY);
   assert(m._ota_direct_transient->type==ADV_TYPE_NONE&&m.contact.out_path_len==2);
   m.mode(3,&fresh);assert(m.response==0&&m.clear_calls==1);
   for(auto& slot:m.anonymous){memset(slot.id.pub_key,0x33,32);slot.type=ADV_TYPE_NONE;}
@@ -206,6 +240,8 @@ class CompanionDirectAdminGuardTest(unittest.TestCase):
         binary = extract_braced(source, "bool MyMesh::handleDirectOtaFrame(")
         getter = extract_braced(source, "bool MyMesh::handleDirectOtaCommand(")
         closer = extract_braced(source, "void MyMesh::closeDirectOtaAdmin(")
+        base = (ROOT / "src/helpers/BaseChatMesh.cpp").read_text()
+        send_login = extract_braced(base, "int BaseChatMesh::sendLogin(")
         callbacks = "\n".join(extract_braced(source, signature) for signature in (
             "uint8_t MyMesh::getContactTxRadio(", "void MyMesh::onPeerDataRecv(", "bool MyMesh::onPeerPathRecv("))
         raw = extract_braced(source, "if (pkt->getPayloadType() == PAYLOAD_TYPE_OTA\n            && _ota_direct_admin.active")
@@ -233,6 +269,7 @@ class CompanionDirectAdminGuardTest(unittest.TestCase):
                 code = HARNESS.replace("@BINARY@", (binary + "\n" + getter + "\n" + closer) if full else "")
                 code = code.replace("@RAW_ROUTE@", raw)
                 code = code.replace("@CALLBACKS@", callbacks if full else "")
+                code = code.replace("@SEND_LOGIN@", send_login if full else "")
                 code = code.replace("@REPEATER_PREFIX@", prefix_blocks[0])
                 code = code.replace("@ROOM_PREFIX@", prefix_blocks[1])
                 (folder / "test.cpp").write_text(code)
@@ -255,9 +292,9 @@ class CompanionDirectAdminGuardTest(unittest.TestCase):
         full = ["COMPANION_RADIO_FULL=1", "ENABLE_OTA=1", "ENABLE_USB_INTERFACE=1", "BLE_PIN_CODE=123456"]
         cases = (("esp32", full + ["ESP32_PLATFORM=1", "WIFI_SSID=fixture"], 1, 1),
                  ("nrf52", full + ["NRF52_PLATFORM=1"], 1, 1),
-                 ("rak4631", full + ["NRF52_PLATFORM=1", "RAK_4631=1"], 0, 1),
+                 ("rak4631", full + ["NRF52_PLATFORM=1", "RAK_4631=1"], 1, 1),
                  ("reduced", ["NRF52_PLATFORM=1", "RAK_4631=1"], 0, 0),
-                 ("rak override", full + ["NRF52_PLATFORM=1", "RAK_4631=1", "COMPANION_FEATURE_DIRECT_OTA_ADMIN=1"], 1, 1))
+                 ("rak opt-out", full + ["NRF52_PLATFORM=1", "RAK_4631=1", "COMPANION_FEATURE_DIRECT_OTA_ADMIN=0"], 0, 1))
         code = ('#include "examples/companion_radio/CompanionFeatures.h"\n'
                 'DIRECT_HELPER COMPANION_FEATURE_DIRECT_OTA_ADMIN\n'
                 'EXISTING_OTA COMPANION_FEATURE_OTA_CLI\n'

test/test_companion_reader.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 """Compare the real firmware decoder and CLI output against every source verse."""
 import copy
+import configparser
 import importlib.util
 import json
 import os
@@ -55,6 +56,12 @@ class CompanionReaderTest(unittest.TestCase):
             self.assertEqual(actual, list(self.expected_verses.values()))
 
     def test_arm_reader_decode_frames_fit_existing_loop_stack(self):
+        self.check_arm_reader_stack()
+
+    def test_compact_arm_reader_decode_frames_fit_existing_loop_stack(self):
+        self.check_arm_reader_stack(compact=True)
+
+    def check_arm_reader_stack(self, compact=False):
         # Use the actual Cortex-M4 compiler when installed. This does not
         # invoke PlatformIO or reserve decode buffers in ordinary commands.
         tool = shutil.which("arm-none-eabi-g++")
@@ -74,6 +81,8 @@ class CompanionReaderTest(unittest.TestCase):
                      "-fstack-usage", "-DNRF52_PLATFORM=1", "-DCOMPANION_RADIO_FULL=1",
                      "-DENABLE_USB_INTERFACE=1", "-I" + str(directory),
                      "-I" + str(ROOT / "src"), "-I" + str(FIXTURE)]
+            if compact:
+                flags.append("-DCOMPANION_READER_COMPACT_STREAM=1")
             self.run_checked([tool, "-std=gnu++17", "-fno-exceptions", "-fno-rtti", *flags,
                 "-c", str(ROOT / "src/helpers/CompanionReader.cpp"), "-o", str(directory / "reader.o")])
             self.run_checked([cc, "-std=gnu11", *flags, "-c",
@@ -229,8 +238,10 @@ class CompanionReaderTest(unittest.TestCase):
             self.skipTest("host GCC and G++ required")
         with tempfile.TemporaryDirectory(prefix="meshcore-reader-") as directory:
             directory = Path(directory)
-            for platform in ("ESP32_PLATFORM", "NRF52_PLATFORM"):
+            for platform, compact in (("ESP32_PLATFORM", False), ("NRF52_PLATFORM", False), ("NRF52_PLATFORM", True)):
                 flags = [f"-D{platform}=1", "-DCOMPANION_RADIO_FULL=1", "-DENABLE_USB_INTERFACE=1"]
+                if compact:
+                    flags.append("-DCOMPANION_READER_COMPACT_STREAM=1")
                 obj, binary = directory / "tinf.o", directory / "reader.exe"
                 self.run_checked([cc, "-std=c11", "-Os", *flags, "-c",
                                   str(ROOT / "src/helpers/ota/OtaTinf.c"), "-o", str(obj)])
@@ -242,6 +253,27 @@ class CompanionReaderTest(unittest.TestCase):
                 actual = dict(line.split("\t", 1) for line in result.splitlines())
                 self.assertEqual(actual, self.expected_verses)
 
+    def test_compact_asset_is_opted_into_by_rak4631_full_only(self):
+        config = configparser.ConfigParser(interpolation=None)
+        config.read(ROOT / "variants/rak4631/platformio.ini")
+        opted_in = [section for section in config.sections()
+                    if "-D COMPANION_READER_COMPACT_STREAM=1" in config.get(section, "build_flags", fallback="")]
+        self.assertEqual(opted_in, ["env:RAK_4631_companion_radio_full"])
+        cxx = shutil.which("g++")
+        if not cxx:
+            self.skipTest("host G++ required")
+        for platform, compact, expected in (("NRF52_PLATFORM", False, 3), ("NRF52_PLATFORM", True, 0),
+                                             ("ESP32_PLATFORM", True, None)):
+            flags = [f"-D{platform}=1", "-DCOMPANION_RADIO_FULL=1", "-DENABLE_USB_INTERFACE=1"]
+            if compact:
+                flags.append("-DCOMPANION_READER_COMPACT_STREAM=1")
+            result = self.run_checked([cxx, "-E", "-P", *flags,
+                                      "-I" + str(FIXTURE), str(ROOT / "src/helpers/CompanionReader.cpp")])
+            if expected is None:
+                self.assertIn("using ReaderCorpus = TokenCorpusT<uint16_t>;", result)
+            else:
+                self.assertIn(f"128, kVerseCount, {expected}", result)
+
     def test_other_profiles_exclude_corpus(self):
         cxx = shutil.which("g++")
         if not cxx:

test/test_nrf52_gnss_version_trim.py

@@ -0,0 +1,189 @@
+"""Qualify the private GNSS transform and native scanf replacement behavior."""
+
+import contextlib
+import importlib.util
+import io
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from test_ota_cli_listing import function
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SPEC = importlib.util.spec_from_file_location("gnss_trim", ROOT / "scripts/nrf52_gnss_version_trim.py")
+TRIM = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(TRIM)
+SOURCE = TRIM.INCLUDE + "\nvoid version(const char* ptr) {\n" + TRIM.ORIGINAL_CONVERSION + "\n}\n"
+HEADERS = {name: "// fixture: " + name + "\n" for name in TRIM.HEADERS}
+
+
+class Node:
+    def __init__(self, path):
+        self.path = Path(path)
+
+    def srcnode(self):
+        return self
+
+    def get_abspath(self):
+        return str(self.path)
+
+
+class BuildEnv:
+    def __init__(self, path):
+        self.path = Path(path)
+        self.registrations = []
+
+    def subst(self, value):
+        assert value == "$BUILD_DIR"
+        return str(self.path)
+
+    def File(self, path):
+        return Path(path)
+
+    def AddBuildMiddleware(self, callback, pattern):
+        self.registrations.append((callback, pattern))
+
+
+class GnssVersionTrimTests(unittest.TestCase):
+    def test_private_copy_checks_source_and_all_headers_and_preserves_downloads(self):
+        # Small synthetic dependencies test the transform mechanics; production
+        # fingerprints remain fixed to the complete reviewed library files.
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            source = root / "libdeps/gnss/src" / TRIM.SOURCE_NAME
+            source.parent.mkdir(parents=True)
+            source.write_bytes(SOURCE.replace("\n", "\r\n").encode())
+            for name, text in HEADERS.items():
+                (source.parent / name).write_text(text)
+            original = {path.name: path.read_bytes() for path in source.parent.iterdir()}
+            env = BuildEnv(root / "build")
+            with patch.object(TRIM, "SOURCE_SHA256", TRIM.digest(SOURCE)), patch.object(
+                    TRIM, "HEADERS", {name: TRIM.digest(text) for name, text in HEADERS.items()}):
+                destination = TRIM.replace_gnss_source(env, Node(source))
+                self.assertEqual(destination, env.path / "trimmed-nrf52-gnss" / TRIM.SOURCE_NAME)
+                output = destination.read_text()
+                self.assertNotIn("sscanf", output)
+                self.assertIn(TRIM.VERSION_CONVERSION, output)
+                times = {path.name: path.stat().st_mtime_ns for path in destination.parent.iterdir()}
+                self.assertEqual(TRIM.replace_gnss_source(env, Node(source)), destination)
+                self.assertEqual(times, {path.name: path.stat().st_mtime_ns for path in destination.parent.iterdir()})
+                self.assertEqual(original, {path.name: path.read_bytes() for path in source.parent.iterdir()})
+                for name, text in HEADERS.items():
+                    self.assertEqual((destination.parent / name).read_text(), text)
+            for change in ("source", *HEADERS):
+                with self.subTest(change=change), patch.object(TRIM, "SOURCE_SHA256", TRIM.digest(SOURCE)), patch.object(
+                        TRIM, "HEADERS", {name: TRIM.digest(text) for name, text in HEADERS.items()}):
+                    target = source if change == "source" else source.parent / change
+                    before = target.read_bytes()
+                    target.write_bytes(before + b"// unreviewed\n")
+                    other_env = BuildEnv(root / ("unknown-" + change))
+                    notice = io.StringIO()
+                    node = Node(source)
+                    with contextlib.redirect_stdout(notice):
+                        self.assertIs(TRIM.replace_gnss_source(other_env, node), node)
+                    self.assertIn("ordinary source retained", notice.getvalue())
+                    self.assertFalse(other_env.path.exists())
+                    target.write_bytes(before)
+
+    def test_only_qualified_companion_installs_middleware(self):
+        spec = importlib.util.spec_from_file_location("flash_trim", ROOT / "scripts/nrf52_flash_trim.py")
+        policy = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(policy)
+
+        class Env(dict):
+            def PioPlatform(self):
+                class Platform:
+                    name = "nordicnrf52"
+                    def get_package_dir(self, _name): return ""
+                return Platform()
+            def BoardConfig(self): return {"build.mcu": "nrf52840"}
+            def AppendUnique(self, **kwargs):
+                for name, value in kwargs.items(): self.setdefault(name, []).extend(value)
+            def subst(self, value): return value.replace("$PROJECT_DIR", str(ROOT))
+            def AddBuildMiddleware(self, callback, pattern): self.setdefault("middleware", []).append(pattern)
+
+        for role, flags, expected in (("companion_radio", "", True),
+                ("companion_radio", "-DMESH_NRF52_COMPANION_TRIM=0", False),
+                ("simple_repeater", "", False), ("custom", "", False)):
+            with self.subTest(role=role, flags=flags):
+                env = Env(BUILD_FLAGS=flags, CPPDEFINES=[], PIOENV="custom",
+                          SRC_FILTER="+<../examples/" + role + ">")
+                policy.install(env)
+                self.assertEqual("*" + TRIM.SOURCE_NAME in env.get("middleware", []), expected)
+
+    def test_native_gnss_conversion_parity_and_bounded_cli_tokens(self):
+        compiler = shutil.which("g++") or shutil.which("clang++")
+        if not compiler:
+            self.skipTest("a native C++ compiler is required")
+        cli = (ROOT / "src/helpers/ota/OtaCli.cpp").read_text()
+        native = r'''
+#include <cassert>
+#include <climits>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+struct Version { bool valid; uint8_t high, low; };
+static Version original(const char* ptr) {
+  int protHi = 0, protLo = 0;
+@ORIGINAL@
+  return {scanned == 2, uint8_t(protHi), uint8_t(protLo)};
+}
+static Version trimmed(const char* ptr) {
+  int protHi = 0, protLo = 0;
+@CONVERSION@
+  return {scanned == 2, uint8_t(protHi), uint8_t(protLo)};
+}
+static void parity(const char* text) {
+  const Version before = original(text), after = trimmed(text);
+  assert(before.valid == after.valid);
+  if (before.valid) assert(before.high == after.high && before.low == after.low);
+}
+@TOKENS@
+int main() {
+  for (const char* text : {"18.00", "27.12", "0.0", "255.255", "256.256",
+      "  +27.\t+12 extra", "\t\r\n-1.\v\f-2 trailing", "0018.0000",
+      "", " ", "18", "18.", ".12", "18 .12", "18.x", "x18.12",
+      "+.12", "18.+", "18.12.34", "18.0x20", "0x18.12",
+      "2147483647.2147483647", "-2147483648.-2147483648",
+      "2147483648.2147483648", "-2147483649.-2147483649",
+      "9999999999999999999999999999999.12",
+      "18.-9999999999999999999999999999999"}) parity(text);
+  uint32_t random = 0xB43A1123;
+  for (unsigned n = 0; n < 3000; ++n) {
+    random = random * 1664525U + 1013904223U;
+    char text[96];
+    snprintf(text, sizeof(text), "\t%ld. %ld extra", long(int32_t(random)), long(int32_t(random ^ 0x91D43AA5U)));
+    parity(text);
+  }
+  struct { char first[8]; uint8_t guard1; char second[10]; uint8_t guard2; } words = {};
+  words.guard1 = words.guard2 = 0xA5;
+  assert(parse_two_words("\t1234567\v\fabcdefghi\r\n", words.first, sizeof(words.first), words.second, sizeof(words.second)) == 2);
+  assert(!strcmp(words.first, "1234567") && !strcmp(words.second, "abcdefghi"));
+  for (const char* text : {"12345678 x", "x abcdefghij", "x y z"})
+    assert(parse_two_words(text, words.first, sizeof(words.first), words.second, sizeof(words.second)) == 3);
+  assert(words.guard1 == 0xA5 && words.guard2 == 0xA5);
+  assert(parse_two_words(" \t\r\n", words.first, sizeof(words.first), words.second, sizeof(words.second)) == 0);
+}
+'''.replace("@ORIGINAL@", TRIM.ORIGINAL_CONVERSION).replace("@CONVERSION@", TRIM.VERSION_CONVERSION).replace(
+            "@TOKENS@", function(cli, "parse_two_words"))
+        native = "#include <initializer_list>\n" + native
+        sanitizers = [] if os.name == "nt" else ["-fsanitize=address,undefined", "-fno-sanitize-recover=all", "-fno-pie", "-no-pie"]
+        with tempfile.TemporaryDirectory() as directory:
+            source = Path(directory) / "parity.cpp"
+            source.write_text(native)
+            binary = Path(directory) / ("parity.exe" if os.name == "nt" else "parity")
+            built = subprocess.run([compiler, "-std=c++11", "-O1", "-Wall", "-Wextra", "-Werror",
+                                    *sanitizers, str(source), "-o", str(binary)], capture_output=True, text=True, timeout=60)
+            self.assertEqual(built.returncode, 0, built.stdout + built.stderr)
+            tested = subprocess.run([str(binary)], capture_output=True, text=True, timeout=30)
+            self.assertEqual(tested.returncode, 0, tested.stdout + tested.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()

test/test_ota_folder_resume.py

@@ -20,7 +20,7 @@ class OtaFolderResumeTest(unittest.TestCase):
         start = cli.index('  } else if (is_cmd(a, "pull|get|download", &rest)) {')
         end = cli.index('  // ---- discard the current session', start)
         harness = (ROOT / "test/fixtures/ota_folder_resume.cpp").read_text()
-        helpers = "\n\n".join(function(cli, name) for name in ("is_cmd", "state_word"))
+        helpers = "\n\n".join(function(cli, name) for name in ("is_cmd", "state_word", "parse_two_words"))
         harness = harness.replace("// @CLI_HELPERS@", helpers).replace("// @CLI_PULL@", cli[start:end])
         with tempfile.TemporaryDirectory(prefix="meshcore-ota-folder-resume-") as temporary:
             work = Path(temporary)

test/test_reader_stream.py

@@ -116,6 +116,13 @@ class ReaderStreamTest(unittest.TestCase):
             spec.loader.exec_module(packer)
             document = json.loads(packer.blocks.SOURCE.read_text(encoding="utf-8"))
             self.assertEqual(packer.HEADER.read_text(encoding="ascii"), packer.render(document))
+            self.assertEqual(packer.COMPACT_HEADER.read_text(encoding="ascii"), packer.render(document, compact=True))
+            normal, plain, _, normal_chunks = packer.encode(document)
+            compact, compact_plain, _, compact_chunks = packer.encode(document, compact=True)
+            self.assertEqual((normal_chunks, compact_chunks), (3, 0))
+            self.assertEqual(plain, compact_plain)
+            self.assertEqual(zlib.decompress(compact, wbits=-12), plain)
+            self.assertEqual(len(normal) - len(compact), 661)
             pinned = json.loads(packer.TOKENS.read_text(encoding="ascii"))
             self.assertLessEqual(len(pinned), 128)
             self.assertEqual(len(pinned), len(set(pinned)))
@@ -126,8 +133,8 @@ class ReaderStreamTest(unittest.TestCase):
             sys.path.remove(path)
 
     def test_all_879_verses_match_independently_normalized_source(self):
-        binary = self.compile(r"""
-#include <helpers/bible/ReaderStreamData.generated.h>
+        body = r"""
+#include <helpers/bible/@HEADER@>
 #include <cassert>
 #include <cstdio>
 #include <cstring>
@@ -148,13 +155,16 @@ int main() {
     }
   }
 }
-""", "corpus")
+"""
         document = json.loads((ROOT / "tools/bible/reader-web.json").read_text())
         typography = str.maketrans({"\u00a0": " ", "\u2013": "-", "\u2014": "-",
                                    "\u2018": "'", "\u2019": "'", "\u201c": '\"', "\u201d": '\"'})
         expected = b"".join(text.translate(typography).encode("ascii") + b"\0"
                             for text in document["verses"].values())
-        self.assertEqual(self.run_binary(binary), expected)
+        for name in ("ReaderStreamData.generated.h", "ReaderCompactStreamData.generated.h"):
+            with self.subTest(header=name):
+                binary = self.compile(body.replace("@HEADER@", name), "corpus" + name.split("Data")[0])
+                self.assertEqual(self.run_binary(binary), expected)
 
     def test_ring_bounds_truncation_distances_and_deterministic_mutations(self):
         cases = []
@@ -247,7 +257,7 @@ int main() {
             document = json.loads(packer.blocks.SOURCE.read_text())
             document["verses"] = dict.fromkeys(packer.blocks.references(), None)
             document["verses"]["1:1"] = "q" * 4095
-            rendered = packer.render(document)
+            rendered = packer.render(document, compact=True)
         finally:
             sys.path.remove(path)
         header = self.directory / "literal.h"
@@ -260,6 +270,7 @@ int main() {
   using namespace mesh::bible;
   char scratch[kBlockSize];const char* text=nullptr;
   const auto& c=generated::readerCorpus;
+  assert(c.chunk_count==0);
   assert(lookup(c,{1,1},scratch,sizeof(scratch),text)==LookupResult::Found);
   assert(strlen(text)==4095);for(unsigned n=0;n<4095;++n) assert(text[n]=='q');
   assert(lookup(c,{1,2},scratch,sizeof(scratch),text)==LookupResult::Missing);assert(!text);
@@ -386,6 +397,9 @@ void check(const uint8_t* data,unsigned data_size,unsigned plain_size,const uint
         # against the unchanged block decoder using identical host options.
         binary = self.compile(r"""
 #include <helpers/bible/ReaderStreamData.generated.h>
+#define generated compact_generated
+#include <helpers/bible/ReaderCompactStreamData.generated.h>
+#undef generated
 #define generated old_generated
 #include <helpers/bible/ReaderData.generated.h>
 #undef generated
@@ -405,8 +419,9 @@ template<class Corpus> double bench(const Corpus& corpus) {
   assert(sum>0);
   return std::chrono::duration<double,std::milli>(std::chrono::steady_clock::now()-start).count()/80;
 }
-int main() { printf("Reader last verse host ms: block=%.4f stream=%.4f\n",
-                     bench(mesh::bible::old_generated::readerCorpus),bench(mesh::bible::generated::readerCorpus)); }
+int main() { printf("Reader last verse host ms: block=%.4f stream=%.4f compact=%.4f\n",
+                     bench(mesh::bible::old_generated::readerCorpus),bench(mesh::bible::generated::readerCorpus),
+                     bench(mesh::bible::compact_generated::readerCorpus)); }
 """, "benchmark")
         print(self.run_binary(binary).decode().strip())