Repair CI dependencies and fit STM32 Companion firmware

.github/workflows/run-unit-tests.yml

@@ -16,6 +16,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Setup Build Environment
         uses: ./.github/actions/setup-build-environment
+      - name: Verify nRF52 HEX packaging with installed build dependencies
+        run: python3 -B test/test_nrf52_endf_hex.py -v
       - name: Compile GPS Sensor tracker on nRF52 boards
         run: pio run -e t1000e_sensor -e RAK_WisMesh_Tag_sensor
 
@@ -56,6 +58,11 @@ jobs:
       - name: Setup Build Environment
         uses: ./.github/actions/setup-build-environment
 
+      - name: Verify compact STM32 signature arithmetic
+        run: |
+          python3 -B test/test_ed25519_compact_scalar.py -v
+          python3 -B test/test_ed25519_compact.py -v
+
       - name: Compile ordinary STM32 USB Companion with pinned core and RadioLib
         run: pio run -e RAK_3x72_companion_radio_usb
 

README.md

@@ -74,7 +74,7 @@ For developers:
 
 - Install [PlatformIO](https://docs.platformio.org) in [Visual Studio Code](https://code.visualstudio.com).
 - Clone and open the MeshCore repository in Visual Studio Code.
-- Install the pinned build-time compressors with `python -m pip install -r requirements-build.txt`.
+- Install the pinned build-time dependencies in PlatformIO's Python environment with `python -m pip install -r requirements-build.txt`.
 - See the example applications you can modify and run:
   - [Companion Radio](./examples/companion_radio) - For use with an external chat app, over BLE, USB or Wi-Fi.
   - [KISS Modem](./examples/kiss_modem) - Serial KISS protocol bridge for host applications. ([protocol docs](./docs/kiss_modem_protocol.md))

lib/ed25519/sc.c

@@ -1,6 +1,10 @@
 #include "fixedint.h"
 #include "sc.h"
 
+#if defined(ED25519_COMPACT_SCALAR) && ED25519_COMPACT_SCALAR
+#include "sc_compact.h"
+#else
+
 static uint64_t load_3(const unsigned char *in) {
     uint64_t result;
 
@@ -807,3 +811,5 @@ void sc_muladd(unsigned char *s, const unsigned char *a, const unsigned char *b,
     s[30] = (unsigned char) (s11 >> 9);
     s[31] = (unsigned char) (s11 >> 17);
 }
+
+#endif

lib/ed25519/sc_compact.h

@@ -0,0 +1,88 @@
+/*
+ * Loop form of the ref10 scalar arithmetic in sc.c. The 21-bit limbs,
+ * folding coefficients, and carry order are unchanged. Small STM32
+ * Companion images cannot afford the unrolled reduction and multiply-add.
+ * All loop bounds and memory addresses depend only on public limb indices;
+ * scalar values never select a branch or an array index.
+ */
+
+static void sc_unpack(int64_t *limbs, const unsigned char *bytes, int count) {
+    const int size = count == 24 ? 64 : 32;
+    int i;
+    for (i = 0; i < count; ++i) {
+        const int start = (i * 21) >> 3;
+        const int shift = (i * 21) & 7;
+        uint32_t word = 0;
+        int j;
+        for (j = 0; j < 4 && start + j < size; ++j)
+            word |= (uint32_t)bytes[start + j] << (j * 8);
+        word >>= shift;
+        /* The last limb retains every remaining input bit, as in ref10. */
+        limbs[i] = (int64_t)(i == count - 1 ? word : word & 2097151);
+    }
+}
+
+static void sc_carry(int64_t *limbs, int first, int last, int step, int rounded) {
+    int i;
+    for (i = first; i <= last; i += step) {
+        const int64_t value = (limbs[i] + (rounded ? 1048576 : 0)) >> 21;
+        limbs[i + 1] += value;
+        /* A carry can be negative; multiplication avoids a signed left shift. */
+        limbs[i] -= value * 2097152;
+    }
+}
+
+static void sc_fold(int64_t *limbs, int first, int last) {
+    static const int32_t coefficients[6] = {666643, 470296, 654183, -997805, 136657, -683901};
+    int i;
+    for (i = first; i >= last; --i) {
+        int j;
+        for (j = 0; j < 6; ++j)
+            limbs[i - 12 + j] += limbs[i] * coefficients[j];
+        limbs[i] = 0;
+    }
+}
+
+static void sc_reduce_limbs(unsigned char *out, int64_t limbs[24]) {
+    int i;
+    sc_fold(limbs, 23, 18);
+    sc_carry(limbs, 6, 16, 2, 1);
+    sc_carry(limbs, 7, 15, 2, 1);
+    sc_fold(limbs, 17, 12);
+    sc_carry(limbs, 0, 10, 2, 1);
+    sc_carry(limbs, 1, 11, 2, 1);
+    sc_fold(limbs, 12, 12);
+    sc_carry(limbs, 0, 11, 1, 0);
+    sc_fold(limbs, 12, 12);
+    sc_carry(limbs, 0, 10, 1, 0);
+    /* Encode the same 256 low bits as ref10's byte assignments. */
+    for (i = 0; i < 32; ++i) {
+        const int index = (i * 8) / 21;
+        const int shift = (i * 8) % 21;
+        uint64_t word = (uint64_t)limbs[index] >> shift;
+        if (shift > 13 && index < 11)
+            word |= (uint64_t)limbs[index + 1] << (21 - shift);
+        out[i] = (unsigned char)word;
+    }
+}
+
+void sc_reduce(unsigned char *s) {
+    int64_t limbs[24];
+    sc_unpack(limbs, s, 24);
+    sc_reduce_limbs(s, limbs);
+}
+
+void sc_muladd(unsigned char *s, const unsigned char *a, const unsigned char *b, const unsigned char *c) {
+    int64_t aa[12], bb[12], limbs[24] = {0};
+    int i, j;
+    /* Read all inputs before writing the result; s may alias a, b, or c. */
+    sc_unpack(aa, a, 12);
+    sc_unpack(bb, b, 12);
+    sc_unpack(limbs, c, 12);
+    for (i = 0; i < 12; ++i)
+        for (j = 0; j < 12; ++j)
+            limbs[i + j] += aa[i] * bb[j];
+    sc_carry(limbs, 0, 22, 2, 1);
+    sc_carry(limbs, 1, 21, 2, 1);
+    sc_reduce_limbs(s, limbs);
+}

requirements-build.txt

@@ -1,2 +1,5 @@
+# nRF52 HEX post-build trailer generation.
+intelhex==2.3.0
+
 # Reproducible compressors for firmware-resident and OTA assets.
 zopfli==0.4.3

test/test_cli_settings_contract.py

@@ -182,7 +182,10 @@ class CLISettingsContractTest(unittest.TestCase):
                 'recent.repeaters',       # Discovery/cache query
             }),
             ('simple_room_server', 'MyMesh.cpp', {'acl'}),
-            ('simple_sensor', 'SensorMesh.cpp', {'acl'}),
+            ('simple_sensor', 'SensorMesh.cpp', {
+                'acl',
+                'tracker',  # Aggregate status; set tracker.<key> changes settings
+            }),
         ):
             text = (ROOT/'examples'/role/filename).read_text(encoding='utf-8')
             sets, gets = full_command_keys(text)

test/test_ed25519_compact.py

@@ -116,6 +116,10 @@ class Ed25519CompactTest(unittest.TestCase):
                 ("compact_sha512", ["-DED25519_COMPACT_SHA512=1"]),
                 ("compact_base_sha512", ["-DED25519_COMPACT_BASE=1",
                                          "-DED25519_COMPACT_SHA512=1"]),
+                ("compact_scalar", ["-DED25519_COMPACT_SCALAR=1"]),
+                ("compact_all", ["-DED25519_COMPACT_BASE=1",
+                                 "-DED25519_COMPACT_SHA512=1",
+                                 "-DED25519_COMPACT_SCALAR=1"]),
             ):
                 executable = Path(temp_dir) / label
                 subprocess.run(

test/test_ed25519_compact_scalar.py

@@ -0,0 +1,147 @@
+"""Check compact ref10 scalars against independent integer arithmetic."""
+
+import ctypes
+import itertools
+from pathlib import Path
+import random
+import subprocess
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+ED25519 = ROOT / "lib/ed25519"
+ORDER = 2**252 + 27742317777372353535851937790883648493
+EDGES = (0, 1, ORDER - 1, ORDER, ORDER + 1, 2**256 - 1)
+
+
+def octets(value, size):
+    return value.to_bytes(size, "little")
+
+
+class Ed25519CompactScalarTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.directory = tempfile.TemporaryDirectory()
+        cls.libraries = []
+        for compact in (0, 1):
+            library_path = Path(cls.directory.name) / f"scalar-{compact}.so"
+            subprocess.run(["cc", "-std=c99", "-O2", "-Wall", "-Wextra", "-Werror",
+                            "-shared", "-fPIC", f"-DED25519_COMPACT_SCALAR={compact}",
+                            str(ED25519 / "sc.c"), "-o", str(library_path)], check=True)
+            library = ctypes.CDLL(str(library_path))
+            library.sc_reduce.argtypes = [ctypes.c_void_p]
+            library.sc_reduce.restype = None
+            library.sc_muladd.argtypes = [ctypes.c_void_p] * 4
+            library.sc_muladd.restype = None
+            cls.libraries.append(library)
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.directory.cleanup()
+
+    def test_reduction_matches_integer_oracle_and_standard(self):
+        rng = random.Random(0xED25519)
+        values = list(EDGES) + [2**512 - 1]
+        values += [ORDER * factor + delta for factor in (2, 7, 1000000)
+                   for delta in (-1, 0, 1)]
+        values += [1 << bit for bit in range(512)]
+        values += [rng.getrandbits(512) for _ in range(4000)]
+        for value in values:
+            original = octets(value, 64)
+            expected = octets(value % ORDER, 32)
+            for compact, library in enumerate(self.libraries):
+                with self.subTest(value=value, compact=compact):
+                    data = ctypes.create_string_buffer(original, 64)
+                    library.sc_reduce(data)
+                    self.assertEqual(data.raw[:32], expected)
+                    self.assertEqual(data.raw[32:], original[32:])
+
+    def test_multiply_add_matches_integer_oracle_and_aliases(self):
+        rng = random.Random(0x25519)
+        triples = list(itertools.product(EDGES, repeat=3))
+        triples += [tuple(rng.getrandbits(256) for _ in range(3)) for _ in range(4000)]
+        for a, b, c in triples:
+            expected = octets((a * b + c) % ORDER, 32)
+            for compact, library in enumerate(self.libraries):
+                for alias in range(4):
+                    with self.subTest(a=a, b=b, c=c, compact=compact, alias=alias):
+                        values = [ctypes.create_string_buffer(octets(value, 32), 32)
+                                  for value in (a, b, c)]
+                        output = ctypes.create_string_buffer(32) if alias == 3 else values[alias]
+                        library.sc_muladd(output, *values)
+                        self.assertEqual(output.raw, expected)
+
+    def test_compact_scalars_under_address_and_undefined_sanitizers(self):
+        rng = random.Random(0x8032)
+        reductions = [0, ORDER - 1, ORDER, ORDER + 1, 2**512 - 1]
+        reductions += [rng.getrandbits(512) for _ in range(64)]
+        triples = list(itertools.product(EDGES, repeat=3))
+        triples += [tuple(rng.getrandbits(256) for _ in range(3)) for _ in range(64)]
+
+        def array(value, size):
+            return "{" + ",".join(str(byte) for byte in octets(value, size)) + "}"
+
+        reduce_vectors = ",\n".join(
+            "{" + array(value, 64) + "," + array(value % ORDER, 32) + "}"
+            for value in reductions)
+        muladd_vectors = ",\n".join(
+            "{" + ",".join(array(value, 32) for value in (a, b, c, (a * b + c) % ORDER)) + "}"
+            for a, b, c in triples)
+        source = r'''
+#include <assert.h>
+#include <string.h>
+#include "sc.h"
+struct ReduceVector { unsigned char input[64], expected[32]; };
+struct MuladdVector { unsigned char a[32], b[32], c[32], expected[32]; };
+static const struct ReduceVector reductions[] = { @REDUCTIONS@ };
+static const struct MuladdVector triples[] = { @TRIPLES@ };
+int main(void) {
+    unsigned char data[64], a[32], b[32], c[32], separate[32];
+    unsigned int i, alias;
+    for (i = 0; i < sizeof(reductions) / sizeof(reductions[0]); ++i) {
+        memcpy(data, reductions[i].input, sizeof(data));
+        sc_reduce(data);
+        assert(memcmp(data, reductions[i].expected, 32) == 0);
+        assert(memcmp(data + 32, reductions[i].input + 32, 32) == 0);
+    }
+    for (i = 0; i < sizeof(triples) / sizeof(triples[0]); ++i) {
+        for (alias = 0; alias < 4; ++alias) {
+            unsigned char *out;
+            memcpy(a, triples[i].a, 32);
+            memcpy(b, triples[i].b, 32);
+            memcpy(c, triples[i].c, 32);
+            out = alias == 0 ? a : alias == 1 ? b : alias == 2 ? c : separate;
+            sc_muladd(out, a, b, c);
+            assert(memcmp(out, triples[i].expected, 32) == 0);
+        }
+    }
+    return 0;
+}
+'''.replace("@REDUCTIONS@", reduce_vectors).replace("@TRIPLES@", muladd_vectors)
+        with tempfile.TemporaryDirectory() as directory:
+            harness = Path(directory) / "scalars.c"
+            harness.write_text(source)
+            executable = Path(directory) / "scalars"
+            subprocess.run(["cc", "-std=c99", "-O1", "-g", "-Wall", "-Wextra", "-Werror",
+                            "-fsanitize=address,undefined", "-fno-sanitize-recover=all", "-no-pie",
+                            "-DED25519_COMPACT_SCALAR=1", "-I", str(ED25519),
+                            str(harness), str(ED25519 / "sc.c"), "-o", str(executable)], check=True)
+            subprocess.run([str(executable)], check=True)
+
+    def test_only_stm32_companion_profiles_enable_compact_scalars(self):
+        profiles = []
+        for ini in (ROOT / "variants").glob("*/platformio.ini"):
+            section = ""
+            for line in ini.read_text().splitlines():
+                if line.startswith("["):
+                    section = line
+                if "ED25519_COMPACT_SCALAR" in line:
+                    profiles.append((ini.parent.name, section))
+        self.assertEqual(sorted(profiles), [
+            ("rak3x72", "[env:RAK_3x72_companion_radio_usb]"),
+            ("wio-e5-mini", "[env:wio-e5-mini_companion_radio_usb]"),
+        ])
+
+
+if __name__ == "__main__":
+    unittest.main()

test/test_fleet_location.py

@@ -107,6 +107,8 @@ int main(int argc, char** argv) {
     sensors.seed(47.6062f, -122.3321f);
     sensors.block(true); // Cached reading must not reclaim a bridge-owned UART.
     const unsigned starts = sensors.starts, stops = sensors.stops;
+    // UART ownership setup may consult the provider; cache reads must not.
+    const unsigned providers = sensors.providers, queries = sensors.queries;
     const bool active = sensors.active, receiving = sensors.receiving();
     for (unsigned i = 0; i < 100; ++i) {
       assert(sensors.getCachedGpsPosition(latitude, longitude));
@@ -115,7 +117,7 @@ int main(int argc, char** argv) {
       assert(mesh::readFleetLocation(prefs, sensors, lat, lon));
     }
     assert(sensors.starts == starts && sensors.stops == stops && sensors.active == active);
-    assert(sensors.receiving() == receiving && sensors.providers == 0 && sensors.queries == 0);
+    assert(sensors.receiving() == receiving && sensors.providers == providers && sensors.queries == queries);
     assert(sensors.powersaving_enabled && !sensors.active && !sensors.receiving());
   } else if (scenario == "cache_freshness" || scenario == "cache_wrap") {
     const uint32_t start = scenario == "cache_wrap" ? UINT32_MAX - 1000 : 0;

test/test_nrf52_endf_hex.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Exercise the nRF52 HEX post-build hook with the installed build dependencies."""
+
+from contextlib import redirect_stdout
+import importlib.util
+import io
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+
+from intelhex import IntelHex
+
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "tools" / "mota"))
+import motalib as ml  # noqa: E402
+
+
+class BuildEnvironment(dict):
+    def __init__(self, app_base):
+        super().__init__(
+            PROJECT_DIR=str(ROOT),
+            PIOENV="nrf52_sensor_hex_test",
+            CPPDEFINES=[
+                "ENABLE_OTA", "NRF52_PLATFORM",
+                ("FIRMWARE_VERSION", "v1.17.1.9"),
+                ("MOTA_TARGET_ID", "0x12345678"),
+                ("MOTA_HW_ID", "NRF52_TEST"),
+            ],
+        )
+        self.app_base = app_base
+        self.post_actions = {}
+
+    def AddPostAction(self, target, action):
+        self.post_actions[target] = action
+
+    def BoardConfig(self):
+        return {"upload.maximum_size": ml.NRF52_APP_END - self.app_base}
+
+    def GetProjectOption(self, name, default=""):
+        if name in ("build_src_filter", "src_filter"):
+            return "+<examples/simple_sensor>"
+        return default
+
+
+class Nrf52EndfHexTests(unittest.TestCase):
+    def test_hex_hook_appends_valid_identity_and_preserves_other_segments(self):
+        for app_base in (ml.NRF52_APP_BASE_S140_V6, ml.NRF52_APP_BASE_S140_V7):
+            with self.subTest(app_base=hex(app_base)), tempfile.TemporaryDirectory() as tmp:
+                env = BuildEnvironment(app_base)
+                spec = importlib.util.spec_from_file_location(
+                    "nrf52_endf_hex_test", ROOT / "tools" / "mota" / "pio_endf.py")
+                module = importlib.util.module_from_spec(spec)
+                module.Import = lambda _name: None
+                module.env = env
+                spec.loader.exec_module(module)
+                action = env.post_actions["$BUILD_DIR/${PROGNAME}.hex"]
+
+                path = Path(tmp) / "firmware.hex"
+                firmware = bytes(range(256))
+                uicr_base = 0x10001080
+                uicr = b"\x12\x34\x56\x78"
+                original = IntelHex()
+                original.frombytes(firmware, offset=app_base)
+                original.frombytes(uicr, offset=uicr_base)
+                original.write_hex_file(str(path))
+
+                with redirect_stdout(io.StringIO()):
+                    action([], [path], env)
+                result = IntelHex(str(path))
+                segments = result.segments()
+                self.assertEqual(len(segments), 2)
+                self.assertEqual(segments[0], (
+                    app_base, app_base + len(firmware) + ml.NRF52_LAYOUT_LEN + ml.ENDF_LEN))
+                image = bytes(result.tobinarray(start=app_base, end=segments[0][1] - 1))
+                self.assertEqual(image[:len(firmware)], firmware)
+                self.assertEqual(bytes(result.tobinarray(
+                    start=uicr_base, size=len(uicr))), uicr)
+                self.assertTrue(ml.has_endf(image))
+                self.assertEqual(ml.parse_endf_ident(image), ml.FwIdent(
+                    ml.pack_version("1.17.1.9"), 0x12345678, "NRF52_TEST"))
+                self.assertEqual(ml.parse_nrf52_layout(image), ml.Nrf52Layout(
+                    app_base, ml.NRF52_APP_END, ml.NRF52_APP_END))
+
+                first_pass = path.read_bytes()
+                with redirect_stdout(io.StringIO()):
+                    action([], [path], env)
+                self.assertEqual(path.read_bytes(), first_pass)
+
+
+if __name__ == "__main__":
+    unittest.main()

variants/rak3x72/platformio.ini

@@ -67,6 +67,7 @@ platform_packages = platformio/toolchain-gccarmnoneeabi@^1.140201.0
 build_unflags = -Os
 build_flags = ${rak3x72.build_flags}
   -D ED25519_COMPACT_SHA512=1 ; same hash rounds, compact loop preserves the filesystem boundary
+  -D ED25519_COMPACT_SCALAR=1 ; same ref10 arithmetic with fixed loops instead of unrolled code
   -Oz
   -flto
   -fno-inline-small-functions

variants/wio-e5-mini/platformio.ini

@@ -80,6 +80,7 @@ build_unflags =
   -Os
 build_flags = ${lora_e5_mini.build_flags}
   -D ED25519_COMPACT_SHA512=1 ; same hash rounds, compact loop preserves the filesystem boundary
+  -D ED25519_COMPACT_SCALAR=1 ; same ref10 arithmetic with fixed loops instead of unrolled code
   -Oz
   -flto
   -fno-inline-small-functions