Verify fallback file stages with independent CRC32C

.github/workflows/run-unit-tests.yml

@@ -612,6 +612,7 @@ jobs:
           python3 -B test/test_esp32_cooperative_contacts.py -v
           python3 -B test/test_esp32_contact_writer_batch.py -v
           python3 -B test/test_esp32_contact_crc_batch.py -v
+          python3 -B test/test_contact_file_transaction.py -v
           python3 -B test/test_esp32_contact_publication.py -v
           python3 -B test/test_esp32_advert_cache.py -v
 

src/helpers/ContactFileTransaction.h

@@ -46,6 +46,19 @@ private:
   bool _begin_temp_exists = false;
   bool _owns_temp = false;
 #endif
+  __attribute__((noinline)) static uint32_t updateCRC32C(
+      uint32_t crc, const uint8_t* data, size_t len) {
+    // Persistent images may end in their own IEEE CRC32. Reusing that
+    // polynomial here gives valid images a fixed whole-file residue. This
+    // independent fingerprint is private to the transaction, never persisted.
+    while (len-- > 0) {
+      crc ^= *data++;
+      for (unsigned bit = 0; bit < 8; bit++) {
+        crc = (crc >> 1) ^ (0x82F63B78U & (0U - (crc & 1U)));
+      }
+    }
+    return crc;
+  }
   static bool probe(FILESYSTEM* fs, const char* path, bool& present,
                     PresenceProbe presence) {
     if (presence) return presence(fs, path, present);
@@ -186,7 +199,7 @@ public:
     const size_t wrote = _file.write(data, len);
     _ok = wrote == len;
     _size += wrote;
-    _crc = storage::updateCRC32(_crc, data, wrote);
+    _crc = updateCRC32C(_crc, data, wrote);
     return wrote;
   }
   // The default verification pass reads one 64-byte chunk. ESP background
@@ -242,7 +255,7 @@ public:
               ? _verify_remaining : sizeof(buf);
           ok = _verify.read(buf, chunk) == chunk;
           if (!ok) break;
-          _verify_crc = storage::updateCRC32(_verify_crc, buf, chunk);
+          _verify_crc = updateCRC32C(_verify_crc, buf, chunk);
           _verify_remaining -= chunk;
         } while (--max_verify_chunks != 0 && _verify_remaining != 0);
         if (ok) return CommitProgress::Pending;

src/helpers/FleetChannel.cpp

@@ -72,8 +72,8 @@ bool FleetChannel::save(bool enabled, const uint8_t* key, const uint8_t* control
     progress = writer.serviceCommit();
   } while (progress == ContactFileTransaction::CommitProgress::Pending);
   if (!publishing) return false; // No publication attempt; local retry is safe.
-  // The fallback can report a late rename error or false success. Its IEEE
-  // checksum also cannot distinguish valid images carrying an IEEE CRC tail.
+  // The fallback can report a late rename error or false success. Its staged
+  // fingerprint does not establish which bytes have the final public name.
   // Confirm exact public bytes before accepting new authority or using old RAM.
   auto file = openFileRead(fs_, Path);
   uint8_t verified[ImageSize];

test/fixtures/contact_fingerprint/test.cpp

@@ -0,0 +1,143 @@
+#include <helpers/IdentityStore.h>
+#include <helpers/PersistentStoreFormat.h>
+#include <cassert>
+#include <cstdio>
+
+class FingerprintFS : public MemoryFS {
+public:
+  unsigned renames = 0;
+  bool rename(const char* from, const char* to) {
+    ++renames;
+    return MemoryFS::rename(from, to);
+  }
+};
+#undef FILESYSTEM
+#define FILESYSTEM FingerprintFS
+#include "transaction_under_test.h"
+
+using Writer = mesh::ContactFileTransaction;
+using Progress = Writer::CommitProgress;
+using Bytes = std::vector<uint8_t>;
+constexpr char Target[] = "/fingerprint";
+constexpr char Temp[] = "/fingerprint.tmp";
+constexpr char Backup[] = "/fingerprint.bak";
+
+Bytes image(bool fleet, uint8_t value) {
+  // Match the FCS1 and DTX1 on-disk layouts, including their raw IEEE CRC tail.
+  Bytes bytes(fleet ? 64 : 108, 0);
+  memcpy(bytes.data(), fleet ? "FCS1" : "DTX1", 4);
+  bytes[4] = 1;
+  if (fleet) {
+    memset(bytes.data() + 8, 0x11, 16);
+    memset(bytes.data() + 24, 0x22, 32);
+    mesh::storage::writeLE32(bytes.data() + 56, value);
+  } else {
+    bytes[8] = value; // One one-byte route hash; automatic region selection.
+  }
+  const size_t payload = bytes.size() - 4;
+  mesh::storage::writeLE32(bytes.data() + payload,
+      mesh::storage::updateCRC32(0xffffffff, bytes.data(), payload));
+  assert(mesh::storage::updateCRC32(0xffffffff, bytes.data(), bytes.size()) == 0);
+  return bytes;
+}
+
+void writeImage(Writer& writer, const Bytes& bytes, size_t chunk) {
+  for (size_t offset = 0; offset < bytes.size(); offset += chunk) {
+    const size_t count = std::min(chunk, bytes.size() - offset);
+    assert(writer.write(bytes.data() + offset, count) == count);
+  }
+}
+
+Progress drain(Writer& writer, FingerprintFS& fs, const Bytes& old,
+               unsigned budget, unsigned& passes, bool valid = true) {
+  Progress progress = Progress::Pending;
+  while (progress == Progress::Pending) {
+    assert(++passes < 100);
+    const bool publishing = writer.readyToPublish();
+    progress = writer.serviceCommit(valid, budget);
+    if (progress == Progress::Pending) {
+      assert(fs.files.at(Target) == old && fs.renames == 0);
+    } else if (progress == Progress::Succeeded) {
+      assert(publishing); // No yield between namespace and caller publication.
+    }
+  }
+  return progress;
+}
+
+void healthy() {
+  for (bool fleet : {false, true}) {
+    const auto old = image(fleet, 1), next = image(fleet, 2);
+    for (size_t chunk : {size_t(1), size_t(17), next.size()}) {
+      for (unsigned budget : {1U, 8U}) {
+        FingerprintFS fs;
+        fs.files[Target] = old;
+        Writer writer(&fs, Target);
+        assert(writer);
+        writeImage(writer, next, chunk);
+        unsigned passes = 0;
+        assert(drain(writer, fs, old, budget, passes) == Progress::Succeeded);
+#if !defined(ESP32_PLATFORM)
+        budget = 1; // RP2040 retains one 64-byte verification chunk per pass.
+#endif
+        assert(passes == (next.size() + 64 * budget - 1) / (64 * budget) + 4);
+        assert(fs.files.at(Target) == next && fs.renames == 2);
+        assert(!fs.exists(Temp) && !fs.exists(Backup));
+      }
+    }
+  }
+}
+
+void staleStage(bool fleet) {
+  const auto old = image(fleet, 1), next = image(fleet, 2);
+  assert(old != next && old.size() == next.size());
+  for (size_t chunk : {size_t(1), size_t(17), next.size()}) {
+    for (unsigned budget : {1U, 8U}) {
+      FingerprintFS fs;
+      fs.files[Target] = old;
+      Writer writer(&fs, Target);
+      assert(writer);
+      writeImage(writer, next, chunk);
+      assert(writer.serviceCommit(true, budget) == Progress::Pending); // Flush.
+      fs.files[Temp] = old;
+      unsigned passes = 1;
+      const auto progress = drain(writer, fs, old, budget, passes);
+      assert(progress == Progress::Failed);
+      assert(fs.files.at(Target) == old && fs.renames == 0);
+      assert(!fs.exists(Temp) && !fs.exists(Backup));
+    }
+  }
+}
+
+void prepublicationFailures() {
+  for (bool fleet : {false, true}) {
+    const auto old = image(fleet, 1), next = image(fleet, 2);
+    for (unsigned fault = 0; fault < 6; ++fault) {
+      FingerprintFS fs;
+      fs.files[Target] = old;
+      Writer writer(&fs, Target);
+      assert(writer);
+      if (fault == 1) fs.fail_write_after = 1;
+      const size_t wrote = writer.write(next.data(), next.size());
+      assert(wrote == (fault == 1 ? 1 : next.size()));
+      if (fault == 2) fs.fail_read_open = true;
+      if (fault == 3) fs.fail_read_after = 1;
+      if (fault == 4) fs.files.at(Temp).push_back(0);
+      if (fault == 5) fs.files.at(Temp)[8] ^= 1;
+      unsigned passes = 0;
+      assert(drain(writer, fs, old, 8, passes, fault != 0) == Progress::Failed);
+      assert(fs.files.at(Target) == old && fs.renames == 0);
+      assert(!fs.exists(Temp) && !fs.exists(Backup));
+    }
+  }
+}
+
+int main(int argc, char** argv) {
+  assert(argc == 2);
+  const std::string mode = argv[1];
+  if (mode == "healthy") healthy();
+  else if (mode == "stale_fcs1") staleStage(true);
+  else if (mode == "stale_dtx1") staleStage(false);
+  else if (mode == "failures") prepublicationFailures();
+  else assert(false);
+  printf("contact fingerprint %s passed\n", argv[1]);
+}

test/test_contact_file_transaction.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""Exercise the real transaction fingerprint without changing persisted CRCs."""
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+FIXTURE = ROOT / "test/fixtures/contact_fingerprint/test.cpp"
+
+
+class ContactFingerprintTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        compiler = shutil.which("g++") or shutil.which("clang++")
+        if compiler is None:
+            raise unittest.SkipTest("host C++ compiler required")
+        cls.work = tempfile.TemporaryDirectory(prefix="mesh-contact-fingerprint-")
+        cls.binaries = {}
+        source = (ROOT / "src/helpers/ContactFileTransaction.h").read_text(encoding="ascii")
+        # Replay the old production accumulators as an exact-negative control.
+        legacy = source
+        for call in ("updateCRC32C(_crc,", "updateCRC32C(_verify_crc,"):
+            if legacy.count(call) != 1:
+                raise AssertionError("missing transaction fingerprint call: " + call)
+            legacy = legacy.replace(call, call.replace("updateCRC32C", "storage::updateCRC32"))
+        for platform in ("ESP32_PLATFORM", "RP2040_PLATFORM"):
+            for old_ieee, header in ((False, source), (True, legacy)):
+                work = Path(cls.work.name) / (platform + ("-ieee" if old_ieee else "-crc32c"))
+                work.mkdir()
+                header = header.replace('#include "IdentityStore.h"',
+                                        '#include <helpers/IdentityStore.h>').replace(
+                    '#include "PersistentStoreFormat.h"', '#include <helpers/PersistentStoreFormat.h>')
+                (work / "transaction_under_test.h").write_text(header, encoding="ascii")
+                binary = work / "fixture"
+                result = subprocess.run([
+                    compiler, "-std=c++17", "-O1", "-g", "-Wall", "-Wextra", "-Werror",
+                    "-Wno-sign-compare", "-fsanitize=address,undefined", "-fno-omit-frame-pointer",
+                    "-fno-sanitize-recover=all", "-fno-pie", "-no-pie", "-D" + platform,
+                    "-I", str(work), "-I", str(ROOT / "test/fixtures/radio_profiles/mocks"),
+                    "-I", str(ROOT / "src"), str(FIXTURE), "-o", str(binary),
+                ], capture_output=True, text=True, timeout=60)
+                if result.returncode:
+                    cls.work.cleanup()
+                    raise AssertionError(result.stdout + result.stderr)
+                cls.binaries[(platform, old_ieee)] = binary
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.work.cleanup()
+
+    def scenario(self, mode, old_ieee=False, rejected=False):
+        for platform in ("ESP32_PLATFORM", "RP2040_PLATFORM"):
+            with self.subTest(platform=platform, old_ieee=old_ieee):
+                result = subprocess.run([str(self.binaries[(platform, old_ieee)]), mode],
+                                        capture_output=True, text=True, timeout=10)
+                if rejected:
+                    self.assertNotEqual(result.returncode, 0, result.stdout)
+                    self.assertIn("progress == Progress::Failed", result.stderr)
+                else:
+                    self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+
+    def test_normal_and_chunked_images_keep_publication_cadence(self):
+        self.scenario("healthy")
+
+    def test_same_sized_valid_fcs1_stage_is_rejected_before_publication(self):
+        self.scenario("stale_fcs1")
+
+    def test_same_sized_valid_dtx1_stage_is_rejected_before_publication(self):
+        self.scenario("stale_dtx1")
+
+    def test_known_prepublication_failures_preserve_the_original(self):
+        self.scenario("failures")
+
+    def test_old_ieee_control_still_accepts_healthy_images_and_rejects_known_faults(self):
+        self.scenario("healthy", old_ieee=True)
+        self.scenario("failures", old_ieee=True)
+
+    def test_old_ieee_control_is_caught_by_fcs1_regression(self):
+        self.scenario("stale_fcs1", old_ieee=True, rejected=True)
+
+    def test_old_ieee_control_is_caught_by_dtx1_regression(self):
+        self.scenario("stale_dtx1", old_ieee=True, rejected=True)
+
+
+if __name__ == "__main__":
+    unittest.main()

test/test_fleet_channel_runtime.py

@@ -1052,20 +1052,12 @@ int main(int argc,char** argv){
   fs.substitute_temp=previous;
   assert(config(fleet,"set fleet.controller off").rfind("Err",0)==0);
   assert(fs.files.at(Path)==previous);fs.substitute_temp.clear();
-#if defined(NRF52_PLATFORM)
   // CRC32C detects the substituted IEEE-CRC-tail image before publication.
   assert(config(fleet,"get fleet.channel").find("store=ok")!=std::string::npos);
   assert(config(fleet,"set fleet.controller off").rfind("OK",0)==0);
-#else
-  // The fallback's IEEE whole-image residue matches. Exact public bytes must
-  // reject the old valid image and block any further stale writes until restart.
-  assert(config(fleet,"get fleet.channel").find("store=error")!=std::string::npos);
-  const unsigned writes=fs.writes;
-  assert(config(fleet,"set fleet.channel off").find("unconfirmed")!=std::string::npos);
   Packet packet=command(publisher,mesh.clock.now,"get radio2");
-  assert(apply(fleet,fs,mesh,profiles,packet)==0&&fs.writes==writes&&fs.files.at(Path)==previous);
-  FleetChannel rebooted(&fs);assert(apply(rebooted,fs,mesh,profiles,packet)==1);
-#endif
+  assert(apply(fleet,fs,mesh,profiles,packet)==0);
+  FleetChannel rebooted(&fs);assert(apply(rebooted,fs,mesh,profiles,packet)==0);
  }else if(scenario=="enrollment"){
   Packet packet=command(publisher,mesh.clock.now);assert(apply(fleet,fs,mesh,profiles,packet)==0);
   assert(config(fleet,"get fleet.channel").find("off,controller=unset,store=ok")!=std::string::npos);