diff --git a/examples/companion_radio/DataStore.cpp b/examples/companion_radio/DataStore.cpp index d2863c41c..c895a699c 100644 --- a/examples/companion_radio/DataStore.cpp +++ b/examples/companion_radio/DataStore.cpp @@ -2579,7 +2579,9 @@ bool DataStore::serviceContactWrite(DataStoreHost* host, _contact_path_reader.close(); } #endif - const auto progress = _contact_write->serviceCommit(); + // Keep the 64-byte CRC scratch while verifying at most 512 bytes per + // background pass; transports run between these bounded batches. + const auto progress = _contact_write->serviceCommit(true, 8); if (progress == mesh::ContactFileTransaction::CommitProgress::Pending) { return true; } diff --git a/src/helpers/ContactFileTransaction.h b/src/helpers/ContactFileTransaction.h index fc5014e27..b9c0bc3ec 100644 --- a/src/helpers/ContactFileTransaction.h +++ b/src/helpers/ContactFileTransaction.h @@ -172,10 +172,11 @@ public: _crc = storage::updateCRC32(_crc, data, wrote); return wrote; } - // Each verification pass reads at most one small chunk. The caller can - // service its transports between passes without exposing the rename gap. - // A synchronous durability operation uses commit() to drain the same steps. - CommitProgress serviceCommit(bool valid = true) { + // The default verification pass reads one 64-byte chunk. ESP background + // callers may request up to eight chunks while reusing the same scratch + // buffer. Transports run between passes without exposing the rename gap; + // synchronous commit() and other platforms retain the one-chunk default. + CommitProgress serviceCommit(bool valid = true, unsigned max_verify_chunks = 1) { #if defined(ESP32_PLATFORM) // A premature commit must not publish, clean up, or implicitly aggregate // the deferred setup operations inside this call. @@ -194,8 +195,8 @@ public: #if defined(ESP32_PLATFORM) // fread() otherwise reads ahead up to 4 KiB while the caller requests // only one 64-byte CRC chunk. A bounded two-chunk read-ahead buffer - // halves backend reads without growing that scratch buffer or allowing - // more than one backend refill per pass. Configuration must succeed. + // halves backend reads without growing that scratch buffer. The caller's + // separate chunk budget bounds each pass. Configuration must succeed. ok = ok && _verify.setBufferSize(ESP_VERIFY_BUFFER_SIZE); #endif ok = ok && _verify.size() == _size; @@ -210,12 +211,21 @@ public: const size_t count = _verify_remaining < sizeof(buf) ? _verify_remaining : sizeof(buf); if (count != 0) { - ok = _verify.read(buf, count) == count; - if (ok) { - _verify_crc = storage::updateCRC32(_verify_crc, buf, count); - _verify_remaining -= count; - return CommitProgress::Pending; - } +#if defined(ESP32_PLATFORM) + if (max_verify_chunks == 0) max_verify_chunks = 1; + if (max_verify_chunks > 8) max_verify_chunks = 8; +#else + max_verify_chunks = 1; +#endif + do { + const size_t chunk = _verify_remaining < sizeof(buf) + ? _verify_remaining : sizeof(buf); + ok = _verify.read(buf, chunk) == chunk; + if (!ok) break; + _verify_crc = storage::updateCRC32(_verify_crc, buf, chunk); + _verify_remaining -= chunk; + } while (--max_verify_chunks != 0 && _verify_remaining != 0); + if (ok) return CommitProgress::Pending; } else { _verify.close(); ok = _verify_crc == _crc; diff --git a/test/fixtures/contact_cache/test_contact_cache.cpp b/test/fixtures/contact_cache/test_contact_cache.cpp index 134ed5e41..9c69e0dbf 100644 --- a/test/fixtures/contact_cache/test_contact_cache.cpp +++ b/test/fixtures/contact_cache/test_contact_cache.cpp @@ -110,6 +110,7 @@ public: return _contact_write != nullptr && !static_cast(*_contact_write); } bool activeJob() const { return _contact_write != nullptr; } + bool verifying() const { return _contact_write_verifying; } bool readyToPublish() const { return _contact_write && _contact_write->readyToPublish(); } @@ -469,6 +470,38 @@ static void cooperative_save_services_commands_and_preserves_snapshots() { } } +static void background_crc_verification_uses_bounded_batches() { + Fixture f(347); + SPIFFS.emulate_stdio = true; + snprintf(f.host.contacts[0].name, sizeof(f.host.contacts[0].name), "CRC cadence"); + assert(f.store.markContactDirty(f.host.contacts[0])); + unsigned crc_read_passes = 0, total_crc_reads = 0; + unsigned max_reads = 0, max_backend_reads = 0, passes = 0; + while (f.store.hasPendingContactWrites()) { + const bool verifying = f.store.verifying() && !f.store.readyToPublish(); + const unsigned reads = SPIFFS.reads, backend_reads = SPIFFS.backend_reads; + assert(++passes < 2000); + assert(f.store.serviceContactWrites(&f.host, cachedContactFilter)); + if (!verifying) continue; // serialization also reads cold cached paths + const unsigned read_count = SPIFFS.reads - reads; + const unsigned backend_count = SPIFFS.backend_reads - backend_reads; + if (read_count != 0) ++crc_read_passes; + total_crc_reads += read_count; + max_reads = std::max(max_reads, read_count); + max_backend_reads = std::max(max_backend_reads, backend_count); + assert(read_count <= 8 && backend_count <= 4); + } + assert(total_crc_reads == 825); + assert(crc_read_passes == 104); + assert(max_reads == 8 && max_backend_reads == 4); + assert(!SPIFFS.exists("/contacts3.tmp")); + f.host.contacts.clear(); + f.store.loadContacts(&f.host); + assert(!f.store.hasIncompleteContactLoad() && f.host.contacts.size() == 347); + assert(strcmp(f.host.contacts[0].name, "CRC cadence") == 0); + for (unsigned i = 0; i < 347; ++i) f.check(i, route(i)); +} + static void cooperative_mutations_never_publish_obsolete_indices() { for (unsigned mutation_step : {4u, 46u, 110u, 0u}) { Fixture f(40); @@ -789,6 +822,7 @@ int main() { contact_tx_policy_round_trips_without_growing_records(); #if defined(ESP32_PLATFORM) cooperative_save_services_commands_and_preserves_snapshots(); + background_crc_verification_uses_bounded_batches(); cooperative_mutations_never_publish_obsolete_indices(); cooperative_failures_keep_the_old_file_and_pending_mutations(); synchronous_flush_cancels_pending_jobs_and_stays_durable(); diff --git a/test/fixtures/contact_crc_batch/test.cpp b/test/fixtures/contact_crc_batch/test.cpp new file mode 100644 index 000000000..056025fc9 --- /dev/null +++ b/test/fixtures/contact_crc_batch/test.cpp @@ -0,0 +1,306 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Model the pinned FILE* buffering boundary, not NOR page operations or time. +// The production transaction and CRC implementation are compiled below. +class FileSystem; +struct Stream { + FileSystem* fs; + std::string path; + bool writing; + size_t buffer_size = 4096, position = 0, backend_position = 0, cursor = 0; + std::vector buffer; +}; +class File { + std::shared_ptr stream; +public: + File() = default; + File(FileSystem* fs, const char* path, bool writing) + : stream(std::make_shared(Stream{fs, path, writing})) {} + explicit operator bool() const { return bool(stream); } + bool setBufferSize(size_t); + size_t size() const; + size_t read(uint8_t*, size_t); + size_t write(const uint8_t*, size_t); + void flush(); + void close() { flush(); stream.reset(); } +}; +class FileSystem { +public: + using Files = std::map>; + Files files; + unsigned reads = 0, writes = 0, backend_reads = 0, backend_writes = 0; + unsigned config_calls = 0, closes = 0, renames = 0; + unsigned fail_config = 0, short_read_call = 0, fail_rename = 0; + size_t largest_read = 0, largest_backend_read = 0, max_write = SIZE_MAX; + bool fail_open_read = false, fail_open_write = false; + std::vector rename_snapshots; + bool exists(const char* path) { return files.count(path) != 0; } + bool remove(const char* path) { return files.erase(path) != 0; } + bool rename(const char* from, const char* to) { + ++renames; + if (renames == fail_rename || !exists(from) || exists(to)) return false; + files[to] = files.at(from); + files.erase(from); + rename_snapshots.push_back(files); + return true; + } + File open(const char* path, const char* mode, bool = false) { + const bool writing = *mode == 'w'; + if (writing ? fail_open_write : fail_open_read || !exists(path)) return File(); + if (writing) files[path].clear(); + return File(this, path, writing); + } +}; +bool File::setBufferSize(size_t size) { + if (!stream) return false; + if (++stream->fs->config_calls == stream->fs->fail_config) return false; + assert(stream->buffer.empty() && stream->position == 0); + stream->buffer_size = size; + return true; +} +size_t File::size() const { return stream ? stream->fs->files.at(stream->path).size() : 0; } +void File::flush() { + if (!stream || !stream->writing || stream->buffer.empty()) return; + auto& fs = *stream->fs; + ++fs.backend_writes; + auto& target = fs.files.at(stream->path); + target.insert(target.end(), stream->buffer.begin(), stream->buffer.end()); + stream->buffer.clear(); +} +size_t File::write(const uint8_t* data, size_t length) { + if (!stream || !stream->writing) return 0; + auto& fs = *stream->fs; + ++fs.writes; + const size_t accepted = std::min(length, fs.max_write); + size_t copied = 0; + while (copied < accepted) { + const size_t n = std::min(accepted - copied, stream->buffer_size - stream->buffer.size()); + stream->buffer.insert(stream->buffer.end(), data + copied, data + copied + n); + copied += n; + if (stream->buffer.size() == stream->buffer_size) flush(); + } + return accepted; +} +size_t File::read(uint8_t* data, size_t length) { + if (!stream || stream->writing) return 0; + auto& fs = *stream->fs; + ++fs.reads; + fs.largest_read = std::max(fs.largest_read, length); + if (fs.reads == fs.short_read_call) --length; + const auto& source = fs.files.at(stream->path); + size_t copied = 0; + while (copied < length) { + if (stream->cursor == stream->buffer.size()) { + const size_t available = source.size() - std::min(source.size(), stream->backend_position); + const size_t n = std::min(stream->buffer_size, available); + if (!n) break; + ++fs.backend_reads; + fs.largest_backend_read = std::max(fs.largest_backend_read, n); + stream->buffer.assign(source.begin() + stream->backend_position, + source.begin() + stream->backend_position + n); + stream->backend_position += n; + stream->cursor = 0; + } + const size_t n = std::min(length - copied, stream->buffer.size() - stream->cursor); + memcpy(data + copied, stream->buffer.data() + stream->cursor, n); + stream->cursor += n; + copied += n; + } + stream->position += copied; + return copied; +} +#define FILESYSTEM FileSystem +unsigned fixture_commit_calls = 0; +#include "transaction_under_test.h" +using Writer = mesh::ContactFileTransaction; +using Progress = Writer::CommitProgress; + +std::vector image(size_t bytes) { + std::vector result(bytes); + for (size_t i = 0; i < bytes; ++i) result[i] = (i * 37 + i / 251) & 255; + return result; +} +void writeImage(Writer& writer, const std::vector& bytes) { + for (size_t offset = 0; offset < bytes.size();) { + const size_t count = std::min(size_t(152), bytes.size() - offset); + assert(writer.write(bytes.data() + offset, count) == count); + offset += count; + } +} +unsigned effectiveBudget(unsigned requested) { +#if defined(ESP32_PLATFORM) + return std::max(1U, std::min(8U, requested)); +#else + (void)requested; + return 1; +#endif +} + +struct Measurement { + unsigned passes, logical_reads, backend_reads, max_pass_reads, max_pass_refills; +}; +Measurement verify(size_t bytes, unsigned budget, bool use_default = false) { + FileSystem fs; + const auto old = image(21), next = image(bytes); + fs.files["/contacts3"] = old; + Writer writer(&fs, "/contacts3"); + assert(writer); + writeImage(writer, next); + unsigned passes = 0, max_reads = 0, max_refills = 0, data_passes = 0; + Progress result = Progress::Pending; + while (result == Progress::Pending) { + const unsigned before_reads = fs.reads, before_backend = fs.backend_reads; + const bool was_ready = writer.readyToPublish(); + result = use_default ? writer.serviceCommit() : writer.serviceCommit(true, budget); + assert(++passes < 10000); + const unsigned reads = fs.reads - before_reads, refills = fs.backend_reads - before_backend; + max_reads = std::max(max_reads, reads); + max_refills = std::max(max_refills, refills); + const unsigned bound = use_default ? 1 : effectiveBudget(budget); + assert(reads <= bound); +#if defined(ESP32_PLATFORM) + assert(refills <= (bound + 1) / 2); +#endif + if (reads) { + ++data_passes; + assert(!writer.readyToPublish()); // close/CRC comparison is a later pass + } + if (result == Progress::Pending) assert(fs.files.at("/contacts3") == old); + else assert(was_ready); // publication never shares a CRC read pass + } + assert(result == Progress::Succeeded); + const unsigned bound = use_default ? 1 : effectiveBudget(budget); + assert(data_passes == (bytes + 64 * bound - 1) / (64 * bound)); + assert(passes == data_passes + 4); + assert(fs.files.at("/contacts3") == next); + assert(fs.reads == (bytes + 63) / 64); + assert(fs.largest_read <= 64); +#if defined(ESP32_PLATFORM) + assert(fs.largest_backend_read <= 128); + assert(fs.backend_reads == (bytes + 127) / 128); +#endif + return {passes, fs.reads, fs.backend_reads, max_reads, max_refills}; +} +void limitsAndTails() { + for (size_t bytes : {size_t(0), size_t(1), size_t(63), size_t(64), size_t(65), + size_t(127), size_t(128), size_t(129), size_t(511), + size_t(512), size_t(513), size_t(152 * 347)}) { + verify(bytes, 1, true); + for (unsigned budget : {0U, 1U, 2U, 8U, 9U, UINT32_MAX}) verify(bytes, budget); + } + FileSystem fs; + Writer writer(&fs, "/sync"); + writeImage(writer, image(347 * 152)); + fixture_commit_calls = 0; + assert(writer.commit()); + assert(fixture_commit_calls == 829); // actual commit() drains default-one passes +} +void readFailureAndCorruption() { + const auto old = image(19), next = image(1536); + for (unsigned failed_read = 1; failed_read <= 16; ++failed_read) { + FileSystem fs; + fs.files["/contacts3"] = old; + Writer writer(&fs, "/contacts3"); + writeImage(writer, next); + fs.short_read_call = failed_read; + Progress result = Progress::Pending; + for (unsigned i = 0; result == Progress::Pending && i < 100; ++i) + result = writer.serviceCommit(true, 8); + assert(result == Progress::Failed); + assert(fs.files.at("/contacts3") == old && !fs.exists("/contacts3.tmp")); + } + for (unsigned corruption : {0U, 63U, 64U, 127U, 511U, 512U, 1535U}) { + FileSystem fs; + fs.files["/contacts3"] = old; + Writer writer(&fs, "/contacts3"); + writeImage(writer, next); + assert(writer.serviceCommit(true, 8) == Progress::Pending); // flush + fs.files.at("/contacts3.tmp")[corruption] ^= 1; + Progress result = Progress::Pending; + while (result == Progress::Pending) result = writer.serviceCommit(true, 8); + assert(result == Progress::Failed && fs.files.at("/contacts3") == old); + } +} +void cancellationAndCrash() { + const auto old = image(19), next = image(1536); + for (unsigned stop = 0; stop < 8; ++stop) { + FileSystem fs; + fs.files["/contacts3"] = old; + Writer writer(&fs, "/contacts3"); + writeImage(writer, next); + for (unsigned i = 0; i < stop; ++i) { + const auto result = writer.serviceCommit(true, 8); + if (result == Progress::Succeeded) break; + } + if (fs.files.at("/contacts3") == old) { + const unsigned reads = fs.reads; + assert(writer.serviceCommit(false, 8) == Progress::Failed); + assert(fs.reads == reads && !fs.exists("/contacts3.tmp")); + } + // Simulate boot from the durable map before any destructor can run. + FileSystem reboot; + reboot.files = fs.files; + assert(Writer::recover(&reboot, "/contacts3")); + assert(reboot.files.at("/contacts3") == old || reboot.files.at("/contacts3") == next); + } + for (unsigned failure : {0U, 1U, 2U, 3U}) { + FileSystem fs; + fs.files["/contacts3"] = old; + Writer writer(&fs, "/contacts3"); + writeImage(writer, next); + fs.fail_rename = failure; + Progress result = Progress::Pending; + while (result == Progress::Pending) result = writer.serviceCommit(true, 8); + assert(result == (failure == 1 || failure == 2 ? Progress::Failed : Progress::Succeeded)); + assert(fs.files.at("/contacts3") == (result == Progress::Succeeded ? next : old)); + for (const auto& snapshot : fs.rename_snapshots) { + FileSystem reboot; + reboot.files = snapshot; + assert(Writer::recover(&reboot, "/contacts3")); + assert(reboot.files.at("/contacts3") == old || reboot.files.at("/contacts3") == next); + } + } +} +void setupFailures() { + for (unsigned fail_config : {1U, 2U}) { + FileSystem fs; + fs.files["/contacts3"] = image(19); + fs.fail_config = fail_config; + Writer writer(&fs, "/contacts3"); + if (fail_config == 1) { assert(!writer); continue; } + writeImage(writer, image(512)); + assert(writer.serviceCommit(true, 8) == Progress::Pending); + assert(writer.serviceCommit(true, 8) == Progress::Failed); + assert(fs.reads == 0 && fs.files.at("/contacts3") == image(19)); + } + FileSystem fs; + fs.files["/contacts3"] = image(19); + Writer writer(&fs, "/contacts3"); + writeImage(writer, image(512)); + fs.fail_open_read = true; + assert(writer.serviceCommit(true, 8) == Progress::Pending); + assert(writer.serviceCommit(true, 8) == Progress::Failed); + assert(fs.reads == 0 && fs.files.at("/contacts3") == image(19)); +} +int main() { + limitsAndTails(); + readFailureAndCorruption(); + cancellationAndCrash(); +#if defined(ESP32_PLATFORM) + setupFailures(); +#endif + const auto one = verify(347 * 152, 1), eight = verify(347 * 152, 8); + printf("{\"one_passes\":%u,\"eight_passes\":%u,\"logical_reads\":%u," + "\"backend_reads\":%u,\"max_pass_reads\":%u,\"max_pass_refills\":%u}\n", + one.passes, eight.passes, eight.logical_reads, eight.backend_reads, + eight.max_pass_reads, eight.max_pass_refills); +} diff --git a/test/fixtures/contact_persistence/test_esp32_uncached.cpp b/test/fixtures/contact_persistence/test_esp32_uncached.cpp index da80ab29a..bf31bca39 100644 --- a/test/fixtures/contact_persistence/test_esp32_uncached.cpp +++ b/test/fixtures/contact_persistence/test_esp32_uncached.cpp @@ -135,9 +135,9 @@ unsigned drain(DataStore& store, Host& h, Filter filter = nullptr) { const unsigned startup_operations = SPIFFS.startupOperations() - startup_before; if (starting) assert(startup_operations <= 1); assert(SPIFFS.writes - writes <= 1); - assert(SPIFFS.reads - reads <= 1); + assert(SPIFFS.reads - reads <= 8); assert(SPIFFS.backend_writes - backend_writes <= 1); - assert(SPIFFS.backend_reads - backend_reads <= 1); + assert(SPIFFS.backend_reads - backend_reads <= 4); // Transport service runs here, between every production pass. It must // never observe an unpublished replacement or a target-name gap. if (store.hasPendingContactWrites()) { @@ -169,7 +169,7 @@ void boundedFullTableAndFirstSave() { assert(store.markContactDirty(h.contacts[0])); const auto expected = image(h, keepContact); const auto passes = drain(store, h, keepContact); - assert(passes > expected.size() / 64); + assert(passes > h.contacts.size()); // serialization and CRC both yield assert(SPIFFS.largest_read <= 64); assert(SPIFFS.largest_backend_read <= 128); assert(SPIFFS.largest_backend_write <= 251); @@ -179,6 +179,35 @@ void boundedFullTableAndFirstSave() { bootCheck(h, keepContact); } +void backgroundCRCVerificationUsesBoundedBatches() { + SPIFFS = FakeFilesystem(); + SPIFFS.emulate_stdio = true; + Host h = table(347); + DataStore store(SPIFFS); + assert(store.markContactDirty(h.contacts[0])); + unsigned crc_read_passes = 0, max_reads = 0, max_backend_reads = 0; + unsigned passes = 0; + while (store.hasPendingContactWrites()) { + const unsigned reads = SPIFFS.reads, backend_reads = SPIFFS.backend_reads; + assert(++passes < 2000); + assert(store.serviceContactWrites(&h, nullptr)); + const unsigned read_count = SPIFFS.reads - reads; + const unsigned backend_count = SPIFFS.backend_reads - backend_reads; + if (read_count != 0) ++crc_read_passes; + max_reads = std::max(max_reads, read_count); + max_backend_reads = std::max(max_backend_reads, backend_count); + assert(read_count <= 8 && backend_count <= 4); + } + // These are actual production DataStore calls, not a direct transaction + // benchmark. Default one-chunk verification fails this cadence witness. + assert(SPIFFS.reads == 825); + assert(crc_read_passes == 104); + assert(max_reads == 8 && max_backend_reads == 4); + assert(SPIFFS.largest_read == 64 && SPIFFS.largest_backend_read == 128); + assert(SPIFFS.files.at("/contacts3") == image(h)); + bootCheck(h); +} + void deferredBeginCancellationAndCrashRecovery() { using Writer = mesh::ContactFileTransaction; // Observe/destruct at every startup boundary, including before any recovery @@ -593,6 +622,7 @@ int main() { bufferedAdmissionBenchmark(); #else boundedFullTableAndFirstSave(); + backgroundCRCVerificationUsesBoundedBatches(); boundedCancellationFlush(); deferredBeginCancellationAndCrashRecovery(); deferredBeginFaultsAndSynchronousDefaults(); diff --git a/test/test_contact_cache.py b/test/test_contact_cache.py index bb38c306c..bd3893a68 100644 --- a/test/test_contact_cache.py +++ b/test/test_contact_cache.py @@ -44,7 +44,10 @@ class ContactCacheTest(unittest.TestCase): def test_nrf52_paged_store_migration_and_snapshots(self): self.run_cache_test("NRF52_PLATFORM") - def run_cache_test(self, platform): + def test_esp32_default_crc_cadence_is_detected(self): + self.run_cache_test("ESP32_PLATFORM", disable_crc_batch=True) + + def run_cache_test(self, platform, disable_crc_batch=False): with tempfile.TemporaryDirectory(prefix="mesh-contact-cache-") as temp: temp = Path(temp) store = (ROOT / "examples/companion_radio/DataStore.cpp").read_text() @@ -87,6 +90,11 @@ class ContactCacheTest(unittest.TestCase): functions.append("bool DataStore::writeContactPage(") functions.append("bool DataStore::loadContactPages(") implementation += "\n".join(method(store, signature) for signature in functions) + if disable_crc_batch: + self.assertIn("_contact_write->serviceCommit(true, 8)", implementation) + implementation = implementation.replace( + "_contact_write->serviceCommit(true, 8)", + "_contact_write->serviceCommit()") (temp / "store_under_test.h").write_text(implementation) packet = (ROOT / "src/Packet.cpp").read_text() (temp / "packet_under_test.h").write_text("namespace mesh {\n" + "\n".join( @@ -119,7 +127,11 @@ class ContactCacheTest(unittest.TestCase): ], capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) result = subprocess.run([str(binary)], capture_output=True, text=True) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + if disable_crc_batch: + self.assertNotEqual(result.returncode, 0) + self.assertIn("crc_read_passes == 104", result.stderr) + else: + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) if __name__ == "__main__": diff --git a/test/test_esp32_contact_crc_batch.py b/test/test_esp32_contact_crc_batch.py new file mode 100644 index 000000000..9fcad17cd --- /dev/null +++ b/test/test_esp32_contact_crc_batch.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Execute bounded CRC batching in the production contact transaction.""" +from pathlib import Path +import json +import os +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SIGNATURE = "CommitProgress serviceCommit(bool valid = true, unsigned max_verify_chunks = 1) {" +OPAQUE_FILE = """#pragma once +#include +#include +class File { + void* impl; +public: + File(); + explicit operator bool() const; + size_t read(uint8_t*, size_t); + size_t write(const uint8_t*, size_t); + size_t size() const; + bool setBufferSize(size_t); + void flush(); + void close(); +}; +class FileSystem { +public: + bool exists(const char*); + bool remove(const char*); + bool rename(const char*, const char*); + File open(const char*, const char*, bool = false); +}; +#define FILESYSTEM FileSystem +""" + + +class ContactCRCBatchTest(unittest.TestCase): + def source(self): + source = (ROOT / "src/helpers/ContactFileTransaction.h").read_text() + self.assertEqual(source.count(SIGNATURE), 1) + return source.replace('#include "IdentityStore.h"', + '// Filesystem contract is supplied by this native fixture.').replace( + '#include "PersistentStoreFormat.h"', '#include ') + + def run_native(self, platform="ESP32_PLATFORM", negative=None): + source = self.source() + if negative == "clamp": + anchor = "if (max_verify_chunks > 8) max_verify_chunks = 8;" + self.assertEqual(source.count(anchor), 1) + source = source.replace(anchor, "// Negative control: unbounded budget.") + elif negative == "synchronous": + anchor = "do { progress = serviceCommit(valid); }" + self.assertEqual(source.count(anchor), 1) + source = source.replace(anchor, "do { progress = serviceCommit(valid, 8); }") + elif negative == "stdio": + anchor = "ESP_VERIFY_BUFFER_SIZE = 128;" + self.assertEqual(source.count(anchor), 1) + source = source.replace(anchor, "ESP_VERIFY_BUFFER_SIZE = 4096;") + # Count actual commit() calls without replacing its body or algorithm. + source = source.replace(SIGNATURE, SIGNATURE + "\n ++fixture_commit_calls;") + with tempfile.TemporaryDirectory(prefix="mesh-contact-crc-") as directory: + temp = Path(directory) + (temp / "transaction_under_test.h").write_text(source) + binary = temp / "fixture" + result = subprocess.run([ + "c++", "-std=c++17", "-O1", "-g", "-Wall", "-Wextra", + "-fsanitize=address,undefined", "-fno-omit-frame-pointer", + "-D" + platform + "=1", "-I", str(temp), "-I", str(ROOT / "src"), + str(ROOT / "test/fixtures/contact_crc_batch/test.cpp"), + "-o", str(binary), + ], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + result = subprocess.run([str(binary)], capture_output=True, text=True) + if negative: + self.assertNotEqual(result.returncode, 0, result.stdout) + markers = {"clamp": "reads <= bound", "synchronous": "fixture_commit_calls == 829", + "stdio": "largest_backend_read <= 128"} + self.assertIn(markers[negative], result.stderr) + return + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return json.loads(result.stdout) + + def test_esp_bounds_tails_failures_crc_cancellation_and_crash(self): + result = self.run_native() + self.assertEqual(result, {"one_passes": 829, "eight_passes": 108, + "logical_reads": 825, "backend_reads": 413, + "max_pass_reads": 8, "max_pass_refills": 4}) + + def test_other_platform_ignores_requested_batch(self): + result = self.run_native("RP2040_PLATFORM") + self.assertEqual(result["one_passes"], 829) + self.assertEqual(result["eight_passes"], 829) + self.assertEqual(result["max_pass_reads"], 1) + + def test_unbounded_budget_is_detected(self): + self.run_native(negative="clamp") + + def test_synchronous_batching_is_detected(self): + self.run_native(negative="synchronous") + + def test_large_stdio_read_ahead_is_detected(self): + self.run_native(negative="stdio") + + def test_pinned_xtensa_frame_and_object_do_not_grow(self): + compiler = os.environ.get("CONTACT_CRC_XTENSA_CXX") or shutil.which("xtensa-esp32s3-elf-g++") + if not compiler: + candidate = Path.home() / ".platformio/packages/toolchain-xtensa-esp32s3/bin/xtensa-esp32s3-elf-g++" + if candidate.is_file(): + compiler = str(candidate) + if not compiler: + self.skipTest("Xtensa S3 compiler is not installed") + # Use SDK-compatible opaque File/FS declarations so native vector/mock + # internals cannot distort the target frame. The CFT/CRC bodies are real. + with tempfile.TemporaryDirectory(prefix="mesh-contact-crc-stack-") as directory: + temp = Path(directory) + (temp / "file_contract.h").write_text(OPAQUE_FILE) + source = self.source().replace( + '// Filesystem contract is supplied by this native fixture.', '#include "file_contract.h"') + (temp / "transaction_under_test.h").write_text(source) + (temp / "frame.cpp").write_text( + '#include "transaction_under_test.h"\n' + 'extern "C" int commit_frame(mesh::ContactFileTransaction* p,unsigned n) ' + '{ return int(p->serviceCommit(true,n)); }\n' + 'char transaction_object_size[sizeof(mesh::ContactFileTransaction)];\n') + result = subprocess.run([ + compiler, "-std=c++17", "-Os", "-fno-inline", "-fstack-usage", + "-DESP32_PLATFORM=1", "-I", str(ROOT / "src"), "-c", + str(temp / "frame.cpp"), "-o", str(temp / "frame.o"), + ], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + frames = [line.split("\t") for line in (temp / "frame.su").read_text().splitlines() + if "ContactFileTransaction::serviceCommit(" in line] + self.assertEqual(len(frames), 1) + self.assertEqual(int(frames[0][1]), 112) + nm = str(Path(compiler).with_name("xtensa-esp32s3-elf-nm")) + symbols = subprocess.check_output([nm, "-S", str(temp / "frame.o")], text=True) + size = next(int(line.split()[1], 16) for line in symbols.splitlines() + if line.endswith("transaction_object_size")) + self.assertEqual(size, 144) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_esp32_cooperative_contacts.py b/test/test_esp32_cooperative_contacts.py index 01eca4a2f..7bbb4f6cc 100644 --- a/test/test_esp32_cooperative_contacts.py +++ b/test/test_esp32_cooperative_contacts.py @@ -26,6 +26,9 @@ class UncachedESPContactsTest(unittest.TestCase): def test_synchronous_startup_batch_is_detected(self): self.run_fixture(["-DBOARD_HAS_PSRAM=1"], disable="begin") + def test_default_background_crc_cadence_is_detected(self): + self.run_fixture(["-DBOARD_HAS_PSRAM=1"], disable="crc_batch") + def test_bounded_buffers_reduce_backend_work_without_stack_growth(self): previous = self.run_fixture(["-DBOARD_HAS_PSRAM=1"], benchmark="previous") current = self.run_fixture(["-DBOARD_HAS_PSRAM=1"], benchmark="current") @@ -77,6 +80,11 @@ class UncachedESPContactsTest(unittest.TestCase): implementation = implementation.replace( '"/contacts3", companionPathPresence, true)', '"/contacts3", companionPathPresence, false)') + elif disable == "crc_batch": + self.assertIn("_contact_write->serviceCommit(true, 8)", implementation) + implementation = implementation.replace( + "_contact_write->serviceCommit(true, 8)", + "_contact_write->serviceCommit()") # Only the native SDK stat seam is substituted. The production # presence logic still distinguishes ENOENT from metadata failure. presence = method(source, "static bool companionPathPresence(") @@ -159,8 +167,9 @@ class UncachedESPContactsTest(unittest.TestCase): return measurements else: self.assertNotEqual(result.returncode, 0) - marker = "startup_operations" if disable == "begin" else ( - "largest_backend_" + ("write" if disable == "writer" else "read")) + marker = ("startup_operations" if disable == "begin" else + "crc_read_passes == 104" if disable == "crc_batch" else + "largest_backend_" + ("write" if disable == "writer" else "read")) self.assertIn(marker, result.stderr)