Batch ESP background contact CRC checks without growing scratch storage

This commit is contained in:
mikecarper committed 2026-10-04 09:27:54 -07:00
1 parent 55e4f47338
commit 15bdb7fc4d
8 files changed
+568 -20

No files matched your search

+3 -1
View File
@@ -2579,7 +2579,9 @@ bool DataStore::serviceContactWrite(DataStoreHost* host,
_contact_path_reader.close();
}
#endif
const auto progress = _contact_write->serviceCommit();
// Keep the 64-byte CRC scratch while verifying at most 512 bytes per
// background pass; transports run between these bounded batches.
const auto progress = _contact_write->serviceCommit(true, 8);
if (progress == mesh::ContactFileTransaction::CommitProgress::Pending) {
return true;
}
+22 -12
View File
@@ -172,10 +172,11 @@ public:
_crc = storage::updateCRC32(_crc, data, wrote);
return wrote;
}
// Each verification pass reads at most one small chunk. The caller can
// service its transports between passes without exposing the rename gap.
// A synchronous durability operation uses commit() to drain the same steps.
CommitProgress serviceCommit(bool valid = true) {
// The default verification pass reads one 64-byte chunk. ESP background
// callers may request up to eight chunks while reusing the same scratch
// buffer. Transports run between passes without exposing the rename gap;
// synchronous commit() and other platforms retain the one-chunk default.
CommitProgress serviceCommit(bool valid = true, unsigned max_verify_chunks = 1) {
#if defined(ESP32_PLATFORM)
// A premature commit must not publish, clean up, or implicitly aggregate
// the deferred setup operations inside this call.
@@ -194,8 +195,8 @@ public:
#if defined(ESP32_PLATFORM)
// fread() otherwise reads ahead up to 4 KiB while the caller requests
// only one 64-byte CRC chunk. A bounded two-chunk read-ahead buffer
// halves backend reads without growing that scratch buffer or allowing
// more than one backend refill per pass. Configuration must succeed.
// halves backend reads without growing that scratch buffer. The caller's
// separate chunk budget bounds each pass. Configuration must succeed.
ok = ok && _verify.setBufferSize(ESP_VERIFY_BUFFER_SIZE);
#endif
ok = ok && _verify.size() == _size;
@@ -210,12 +211,21 @@ public:
const size_t count = _verify_remaining < sizeof(buf)
? _verify_remaining : sizeof(buf);
if (count != 0) {
ok = _verify.read(buf, count) == count;
if (ok) {
_verify_crc = storage::updateCRC32(_verify_crc, buf, count);
_verify_remaining -= count;
return CommitProgress::Pending;
}
#if defined(ESP32_PLATFORM)
if (max_verify_chunks == 0) max_verify_chunks = 1;
if (max_verify_chunks > 8) max_verify_chunks = 8;
#else
max_verify_chunks = 1;
#endif
do {
const size_t chunk = _verify_remaining < sizeof(buf)
? _verify_remaining : sizeof(buf);
ok = _verify.read(buf, chunk) == chunk;
if (!ok) break;
_verify_crc = storage::updateCRC32(_verify_crc, buf, chunk);
_verify_remaining -= chunk;
} while (--max_verify_chunks != 0 && _verify_remaining != 0);
if (ok) return CommitProgress::Pending;
} else {
_verify.close();
ok = _verify_crc == _crc;
+34
View File
@@ -110,6 +110,7 @@ public:
return _contact_write != nullptr && !static_cast<bool>(*_contact_write);
}
bool activeJob() const { return _contact_write != nullptr; }
bool verifying() const { return _contact_write_verifying; }
bool readyToPublish() const {
return _contact_write && _contact_write->readyToPublish();
}
@@ -469,6 +470,38 @@ static void cooperative_save_services_commands_and_preserves_snapshots() {
}
}
static void background_crc_verification_uses_bounded_batches() {
Fixture f(347);
SPIFFS.emulate_stdio = true;
snprintf(f.host.contacts[0].name, sizeof(f.host.contacts[0].name), "CRC cadence");
assert(f.store.markContactDirty(f.host.contacts[0]));
unsigned crc_read_passes = 0, total_crc_reads = 0;
unsigned max_reads = 0, max_backend_reads = 0, passes = 0;
while (f.store.hasPendingContactWrites()) {
const bool verifying = f.store.verifying() && !f.store.readyToPublish();
const unsigned reads = SPIFFS.reads, backend_reads = SPIFFS.backend_reads;
assert(++passes < 2000);
assert(f.store.serviceContactWrites(&f.host, cachedContactFilter));
if (!verifying) continue; // serialization also reads cold cached paths
const unsigned read_count = SPIFFS.reads - reads;
const unsigned backend_count = SPIFFS.backend_reads - backend_reads;
if (read_count != 0) ++crc_read_passes;
total_crc_reads += read_count;
max_reads = std::max(max_reads, read_count);
max_backend_reads = std::max(max_backend_reads, backend_count);
assert(read_count <= 8 && backend_count <= 4);
}
assert(total_crc_reads == 825);
assert(crc_read_passes == 104);
assert(max_reads == 8 && max_backend_reads == 4);
assert(!SPIFFS.exists("/contacts3.tmp"));
f.host.contacts.clear();
f.store.loadContacts(&f.host);
assert(!f.store.hasIncompleteContactLoad() && f.host.contacts.size() == 347);
assert(strcmp(f.host.contacts[0].name, "CRC cadence") == 0);
for (unsigned i = 0; i < 347; ++i) f.check(i, route(i));
}
static void cooperative_mutations_never_publish_obsolete_indices() {
for (unsigned mutation_step : {4u, 46u, 110u, 0u}) {
Fixture f(40);
@@ -789,6 +822,7 @@ int main() {
contact_tx_policy_round_trips_without_growing_records();
#if defined(ESP32_PLATFORM)
cooperative_save_services_commands_and_preserves_snapshots();
background_crc_verification_uses_bounded_batches();
cooperative_mutations_never_publish_obsolete_indices();
cooperative_failures_keep_the_old_file_and_pending_mutations();
synchronous_flush_cancels_pending_jobs_and_stays_durable();
+306
View File
@@ -0,0 +1,306 @@
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <limits>
#include <map>
#include <memory>
#include <string>
#include <vector>
// Model the pinned FILE* buffering boundary, not NOR page operations or time.
// The production transaction and CRC implementation are compiled below.
class FileSystem;
struct Stream {
FileSystem* fs;
std::string path;
bool writing;
size_t buffer_size = 4096, position = 0, backend_position = 0, cursor = 0;
std::vector<uint8_t> buffer;
};
class File {
std::shared_ptr<Stream> stream;
public:
File() = default;
File(FileSystem* fs, const char* path, bool writing)
: stream(std::make_shared<Stream>(Stream{fs, path, writing})) {}
explicit operator bool() const { return bool(stream); }
bool setBufferSize(size_t);
size_t size() const;
size_t read(uint8_t*, size_t);
size_t write(const uint8_t*, size_t);
void flush();
void close() { flush(); stream.reset(); }
};
class FileSystem {
public:
using Files = std::map<std::string, std::vector<uint8_t>>;
Files files;
unsigned reads = 0, writes = 0, backend_reads = 0, backend_writes = 0;
unsigned config_calls = 0, closes = 0, renames = 0;
unsigned fail_config = 0, short_read_call = 0, fail_rename = 0;
size_t largest_read = 0, largest_backend_read = 0, max_write = SIZE_MAX;
bool fail_open_read = false, fail_open_write = false;
std::vector<Files> rename_snapshots;
bool exists(const char* path) { return files.count(path) != 0; }
bool remove(const char* path) { return files.erase(path) != 0; }
bool rename(const char* from, const char* to) {
++renames;
if (renames == fail_rename || !exists(from) || exists(to)) return false;
files[to] = files.at(from);
files.erase(from);
rename_snapshots.push_back(files);
return true;
}
File open(const char* path, const char* mode, bool = false) {
const bool writing = *mode == 'w';
if (writing ? fail_open_write : fail_open_read || !exists(path)) return File();
if (writing) files[path].clear();
return File(this, path, writing);
}
};
bool File::setBufferSize(size_t size) {
if (!stream) return false;
if (++stream->fs->config_calls == stream->fs->fail_config) return false;
assert(stream->buffer.empty() && stream->position == 0);
stream->buffer_size = size;
return true;
}
size_t File::size() const { return stream ? stream->fs->files.at(stream->path).size() : 0; }
void File::flush() {
if (!stream || !stream->writing || stream->buffer.empty()) return;
auto& fs = *stream->fs;
++fs.backend_writes;
auto& target = fs.files.at(stream->path);
target.insert(target.end(), stream->buffer.begin(), stream->buffer.end());
stream->buffer.clear();
}
size_t File::write(const uint8_t* data, size_t length) {
if (!stream || !stream->writing) return 0;
auto& fs = *stream->fs;
++fs.writes;
const size_t accepted = std::min(length, fs.max_write);
size_t copied = 0;
while (copied < accepted) {
const size_t n = std::min(accepted - copied, stream->buffer_size - stream->buffer.size());
stream->buffer.insert(stream->buffer.end(), data + copied, data + copied + n);
copied += n;
if (stream->buffer.size() == stream->buffer_size) flush();
}
return accepted;
}
size_t File::read(uint8_t* data, size_t length) {
if (!stream || stream->writing) return 0;
auto& fs = *stream->fs;
++fs.reads;
fs.largest_read = std::max(fs.largest_read, length);
if (fs.reads == fs.short_read_call) --length;
const auto& source = fs.files.at(stream->path);
size_t copied = 0;
while (copied < length) {
if (stream->cursor == stream->buffer.size()) {
const size_t available = source.size() - std::min(source.size(), stream->backend_position);
const size_t n = std::min(stream->buffer_size, available);
if (!n) break;
++fs.backend_reads;
fs.largest_backend_read = std::max(fs.largest_backend_read, n);
stream->buffer.assign(source.begin() + stream->backend_position,
source.begin() + stream->backend_position + n);
stream->backend_position += n;
stream->cursor = 0;
}
const size_t n = std::min(length - copied, stream->buffer.size() - stream->cursor);
memcpy(data + copied, stream->buffer.data() + stream->cursor, n);
stream->cursor += n;
copied += n;
}
stream->position += copied;
return copied;
}
#define FILESYSTEM FileSystem
unsigned fixture_commit_calls = 0;
#include "transaction_under_test.h"
using Writer = mesh::ContactFileTransaction;
using Progress = Writer::CommitProgress;
std::vector<uint8_t> image(size_t bytes) {
std::vector<uint8_t> result(bytes);
for (size_t i = 0; i < bytes; ++i) result[i] = (i * 37 + i / 251) & 255;
return result;
}
void writeImage(Writer& writer, const std::vector<uint8_t>& bytes) {
for (size_t offset = 0; offset < bytes.size();) {
const size_t count = std::min(size_t(152), bytes.size() - offset);
assert(writer.write(bytes.data() + offset, count) == count);
offset += count;
}
}
unsigned effectiveBudget(unsigned requested) {
#if defined(ESP32_PLATFORM)
return std::max(1U, std::min(8U, requested));
#else
(void)requested;
return 1;
#endif
}
struct Measurement {
unsigned passes, logical_reads, backend_reads, max_pass_reads, max_pass_refills;
};
Measurement verify(size_t bytes, unsigned budget, bool use_default = false) {
FileSystem fs;
const auto old = image(21), next = image(bytes);
fs.files["/contacts3"] = old;
Writer writer(&fs, "/contacts3");
assert(writer);
writeImage(writer, next);
unsigned passes = 0, max_reads = 0, max_refills = 0, data_passes = 0;
Progress result = Progress::Pending;
while (result == Progress::Pending) {
const unsigned before_reads = fs.reads, before_backend = fs.backend_reads;
const bool was_ready = writer.readyToPublish();
result = use_default ? writer.serviceCommit() : writer.serviceCommit(true, budget);
assert(++passes < 10000);
const unsigned reads = fs.reads - before_reads, refills = fs.backend_reads - before_backend;
max_reads = std::max(max_reads, reads);
max_refills = std::max(max_refills, refills);
const unsigned bound = use_default ? 1 : effectiveBudget(budget);
assert(reads <= bound);
#if defined(ESP32_PLATFORM)
assert(refills <= (bound + 1) / 2);
#endif
if (reads) {
++data_passes;
assert(!writer.readyToPublish()); // close/CRC comparison is a later pass
}
if (result == Progress::Pending) assert(fs.files.at("/contacts3") == old);
else assert(was_ready); // publication never shares a CRC read pass
}
assert(result == Progress::Succeeded);
const unsigned bound = use_default ? 1 : effectiveBudget(budget);
assert(data_passes == (bytes + 64 * bound - 1) / (64 * bound));
assert(passes == data_passes + 4);
assert(fs.files.at("/contacts3") == next);
assert(fs.reads == (bytes + 63) / 64);
assert(fs.largest_read <= 64);
#if defined(ESP32_PLATFORM)
assert(fs.largest_backend_read <= 128);
assert(fs.backend_reads == (bytes + 127) / 128);
#endif
return {passes, fs.reads, fs.backend_reads, max_reads, max_refills};
}
void limitsAndTails() {
for (size_t bytes : {size_t(0), size_t(1), size_t(63), size_t(64), size_t(65),
size_t(127), size_t(128), size_t(129), size_t(511),
size_t(512), size_t(513), size_t(152 * 347)}) {
verify(bytes, 1, true);
for (unsigned budget : {0U, 1U, 2U, 8U, 9U, UINT32_MAX}) verify(bytes, budget);
}
FileSystem fs;
Writer writer(&fs, "/sync");
writeImage(writer, image(347 * 152));
fixture_commit_calls = 0;
assert(writer.commit());
assert(fixture_commit_calls == 829); // actual commit() drains default-one passes
}
void readFailureAndCorruption() {
const auto old = image(19), next = image(1536);
for (unsigned failed_read = 1; failed_read <= 16; ++failed_read) {
FileSystem fs;
fs.files["/contacts3"] = old;
Writer writer(&fs, "/contacts3");
writeImage(writer, next);
fs.short_read_call = failed_read;
Progress result = Progress::Pending;
for (unsigned i = 0; result == Progress::Pending && i < 100; ++i)
result = writer.serviceCommit(true, 8);
assert(result == Progress::Failed);
assert(fs.files.at("/contacts3") == old && !fs.exists("/contacts3.tmp"));
}
for (unsigned corruption : {0U, 63U, 64U, 127U, 511U, 512U, 1535U}) {
FileSystem fs;
fs.files["/contacts3"] = old;
Writer writer(&fs, "/contacts3");
writeImage(writer, next);
assert(writer.serviceCommit(true, 8) == Progress::Pending); // flush
fs.files.at("/contacts3.tmp")[corruption] ^= 1;
Progress result = Progress::Pending;
while (result == Progress::Pending) result = writer.serviceCommit(true, 8);
assert(result == Progress::Failed && fs.files.at("/contacts3") == old);
}
}
void cancellationAndCrash() {
const auto old = image(19), next = image(1536);
for (unsigned stop = 0; stop < 8; ++stop) {
FileSystem fs;
fs.files["/contacts3"] = old;
Writer writer(&fs, "/contacts3");
writeImage(writer, next);
for (unsigned i = 0; i < stop; ++i) {
const auto result = writer.serviceCommit(true, 8);
if (result == Progress::Succeeded) break;
}
if (fs.files.at("/contacts3") == old) {
const unsigned reads = fs.reads;
assert(writer.serviceCommit(false, 8) == Progress::Failed);
assert(fs.reads == reads && !fs.exists("/contacts3.tmp"));
}
// Simulate boot from the durable map before any destructor can run.
FileSystem reboot;
reboot.files = fs.files;
assert(Writer::recover(&reboot, "/contacts3"));
assert(reboot.files.at("/contacts3") == old || reboot.files.at("/contacts3") == next);
}
for (unsigned failure : {0U, 1U, 2U, 3U}) {
FileSystem fs;
fs.files["/contacts3"] = old;
Writer writer(&fs, "/contacts3");
writeImage(writer, next);
fs.fail_rename = failure;
Progress result = Progress::Pending;
while (result == Progress::Pending) result = writer.serviceCommit(true, 8);
assert(result == (failure == 1 || failure == 2 ? Progress::Failed : Progress::Succeeded));
assert(fs.files.at("/contacts3") == (result == Progress::Succeeded ? next : old));
for (const auto& snapshot : fs.rename_snapshots) {
FileSystem reboot;
reboot.files = snapshot;
assert(Writer::recover(&reboot, "/contacts3"));
assert(reboot.files.at("/contacts3") == old || reboot.files.at("/contacts3") == next);
}
}
}
void setupFailures() {
for (unsigned fail_config : {1U, 2U}) {
FileSystem fs;
fs.files["/contacts3"] = image(19);
fs.fail_config = fail_config;
Writer writer(&fs, "/contacts3");
if (fail_config == 1) { assert(!writer); continue; }
writeImage(writer, image(512));
assert(writer.serviceCommit(true, 8) == Progress::Pending);
assert(writer.serviceCommit(true, 8) == Progress::Failed);
assert(fs.reads == 0 && fs.files.at("/contacts3") == image(19));
}
FileSystem fs;
fs.files["/contacts3"] = image(19);
Writer writer(&fs, "/contacts3");
writeImage(writer, image(512));
fs.fail_open_read = true;
assert(writer.serviceCommit(true, 8) == Progress::Pending);
assert(writer.serviceCommit(true, 8) == Progress::Failed);
assert(fs.reads == 0 && fs.files.at("/contacts3") == image(19));
}
int main() {
limitsAndTails();
readFailureAndCorruption();
cancellationAndCrash();
#if defined(ESP32_PLATFORM)
setupFailures();
#endif
const auto one = verify(347 * 152, 1), eight = verify(347 * 152, 8);
printf("{\"one_passes\":%u,\"eight_passes\":%u,\"logical_reads\":%u,"
"\"backend_reads\":%u,\"max_pass_reads\":%u,\"max_pass_refills\":%u}\n",
one.passes, eight.passes, eight.logical_reads, eight.backend_reads,
eight.max_pass_reads, eight.max_pass_refills);
}
+33 -3
View File
@@ -135,9 +135,9 @@ unsigned drain(DataStore& store, Host& h, Filter filter = nullptr) {
const unsigned startup_operations = SPIFFS.startupOperations() - startup_before;
if (starting) assert(startup_operations <= 1);
assert(SPIFFS.writes - writes <= 1);
assert(SPIFFS.reads - reads <= 1);
assert(SPIFFS.reads - reads <= 8);
assert(SPIFFS.backend_writes - backend_writes <= 1);
assert(SPIFFS.backend_reads - backend_reads <= 1);
assert(SPIFFS.backend_reads - backend_reads <= 4);
// Transport service runs here, between every production pass. It must
// never observe an unpublished replacement or a target-name gap.
if (store.hasPendingContactWrites()) {
@@ -169,7 +169,7 @@ void boundedFullTableAndFirstSave() {
assert(store.markContactDirty(h.contacts[0]));
const auto expected = image(h, keepContact);
const auto passes = drain(store, h, keepContact);
assert(passes > expected.size() / 64);
assert(passes > h.contacts.size()); // serialization and CRC both yield
assert(SPIFFS.largest_read <= 64);
assert(SPIFFS.largest_backend_read <= 128);
assert(SPIFFS.largest_backend_write <= 251);
@@ -179,6 +179,35 @@ void boundedFullTableAndFirstSave() {
bootCheck(h, keepContact);
}
void backgroundCRCVerificationUsesBoundedBatches() {
SPIFFS = FakeFilesystem();
SPIFFS.emulate_stdio = true;
Host h = table(347);
DataStore store(SPIFFS);
assert(store.markContactDirty(h.contacts[0]));
unsigned crc_read_passes = 0, max_reads = 0, max_backend_reads = 0;
unsigned passes = 0;
while (store.hasPendingContactWrites()) {
const unsigned reads = SPIFFS.reads, backend_reads = SPIFFS.backend_reads;
assert(++passes < 2000);
assert(store.serviceContactWrites(&h, nullptr));
const unsigned read_count = SPIFFS.reads - reads;
const unsigned backend_count = SPIFFS.backend_reads - backend_reads;
if (read_count != 0) ++crc_read_passes;
max_reads = std::max(max_reads, read_count);
max_backend_reads = std::max(max_backend_reads, backend_count);
assert(read_count <= 8 && backend_count <= 4);
}
// These are actual production DataStore calls, not a direct transaction
// benchmark. Default one-chunk verification fails this cadence witness.
assert(SPIFFS.reads == 825);
assert(crc_read_passes == 104);
assert(max_reads == 8 && max_backend_reads == 4);
assert(SPIFFS.largest_read == 64 && SPIFFS.largest_backend_read == 128);
assert(SPIFFS.files.at("/contacts3") == image(h));
bootCheck(h);
}
void deferredBeginCancellationAndCrashRecovery() {
using Writer = mesh::ContactFileTransaction;
// Observe/destruct at every startup boundary, including before any recovery
@@ -593,6 +622,7 @@ int main() {
bufferedAdmissionBenchmark();
#else
boundedFullTableAndFirstSave();
backgroundCRCVerificationUsesBoundedBatches();
boundedCancellationFlush();
deferredBeginCancellationAndCrashRecovery();
deferredBeginFaultsAndSynchronousDefaults();
+14 -2
View File
@@ -44,7 +44,10 @@ class ContactCacheTest(unittest.TestCase):
def test_nrf52_paged_store_migration_and_snapshots(self):
self.run_cache_test("NRF52_PLATFORM")
def run_cache_test(self, platform):
def test_esp32_default_crc_cadence_is_detected(self):
self.run_cache_test("ESP32_PLATFORM", disable_crc_batch=True)
def run_cache_test(self, platform, disable_crc_batch=False):
with tempfile.TemporaryDirectory(prefix="mesh-contact-cache-") as temp:
temp = Path(temp)
store = (ROOT / "examples/companion_radio/DataStore.cpp").read_text()
@@ -87,6 +90,11 @@ class ContactCacheTest(unittest.TestCase):
functions.append("bool DataStore::writeContactPage(")
functions.append("bool DataStore::loadContactPages(")
implementation += "\n".join(method(store, signature) for signature in functions)
if disable_crc_batch:
self.assertIn("_contact_write->serviceCommit(true, 8)", implementation)
implementation = implementation.replace(
"_contact_write->serviceCommit(true, 8)",
"_contact_write->serviceCommit()")
(temp / "store_under_test.h").write_text(implementation)
packet = (ROOT / "src/Packet.cpp").read_text()
(temp / "packet_under_test.h").write_text("namespace mesh {\n" + "\n".join(
@@ -119,7 +127,11 @@ class ContactCacheTest(unittest.TestCase):
], capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
result = subprocess.run([str(binary)], capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
if disable_crc_batch:
self.assertNotEqual(result.returncode, 0)
self.assertIn("crc_read_passes == 104", result.stderr)
else:
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
if __name__ == "__main__":
+145
View File
@@ -0,0 +1,145 @@
#!/usr/bin/env python3
"""Execute bounded CRC batching in the production contact transaction."""
from pathlib import Path
import json
import os
import shutil
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
SIGNATURE = "CommitProgress serviceCommit(bool valid = true, unsigned max_verify_chunks = 1) {"
OPAQUE_FILE = """#pragma once
#include <stddef.h>
#include <stdint.h>
class File {
void* impl;
public:
File();
explicit operator bool() const;
size_t read(uint8_t*, size_t);
size_t write(const uint8_t*, size_t);
size_t size() const;
bool setBufferSize(size_t);
void flush();
void close();
};
class FileSystem {
public:
bool exists(const char*);
bool remove(const char*);
bool rename(const char*, const char*);
File open(const char*, const char*, bool = false);
};
#define FILESYSTEM FileSystem
"""
class ContactCRCBatchTest(unittest.TestCase):
def source(self):
source = (ROOT / "src/helpers/ContactFileTransaction.h").read_text()
self.assertEqual(source.count(SIGNATURE), 1)
return source.replace('#include "IdentityStore.h"',
'// Filesystem contract is supplied by this native fixture.').replace(
'#include "PersistentStoreFormat.h"', '#include <helpers/PersistentStoreFormat.h>')
def run_native(self, platform="ESP32_PLATFORM", negative=None):
source = self.source()
if negative == "clamp":
anchor = "if (max_verify_chunks > 8) max_verify_chunks = 8;"
self.assertEqual(source.count(anchor), 1)
source = source.replace(anchor, "// Negative control: unbounded budget.")
elif negative == "synchronous":
anchor = "do { progress = serviceCommit(valid); }"
self.assertEqual(source.count(anchor), 1)
source = source.replace(anchor, "do { progress = serviceCommit(valid, 8); }")
elif negative == "stdio":
anchor = "ESP_VERIFY_BUFFER_SIZE = 128;"
self.assertEqual(source.count(anchor), 1)
source = source.replace(anchor, "ESP_VERIFY_BUFFER_SIZE = 4096;")
# Count actual commit() calls without replacing its body or algorithm.
source = source.replace(SIGNATURE, SIGNATURE + "\n ++fixture_commit_calls;")
with tempfile.TemporaryDirectory(prefix="mesh-contact-crc-") as directory:
temp = Path(directory)
(temp / "transaction_under_test.h").write_text(source)
binary = temp / "fixture"
result = subprocess.run([
"c++", "-std=c++17", "-O1", "-g", "-Wall", "-Wextra",
"-fsanitize=address,undefined", "-fno-omit-frame-pointer",
"-D" + platform + "=1", "-I", str(temp), "-I", str(ROOT / "src"),
str(ROOT / "test/fixtures/contact_crc_batch/test.cpp"),
"-o", str(binary),
], capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
result = subprocess.run([str(binary)], capture_output=True, text=True)
if negative:
self.assertNotEqual(result.returncode, 0, result.stdout)
markers = {"clamp": "reads <= bound", "synchronous": "fixture_commit_calls == 829",
"stdio": "largest_backend_read <= 128"}
self.assertIn(markers[negative], result.stderr)
return
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
return json.loads(result.stdout)
def test_esp_bounds_tails_failures_crc_cancellation_and_crash(self):
result = self.run_native()
self.assertEqual(result, {"one_passes": 829, "eight_passes": 108,
"logical_reads": 825, "backend_reads": 413,
"max_pass_reads": 8, "max_pass_refills": 4})
def test_other_platform_ignores_requested_batch(self):
result = self.run_native("RP2040_PLATFORM")
self.assertEqual(result["one_passes"], 829)
self.assertEqual(result["eight_passes"], 829)
self.assertEqual(result["max_pass_reads"], 1)
def test_unbounded_budget_is_detected(self):
self.run_native(negative="clamp")
def test_synchronous_batching_is_detected(self):
self.run_native(negative="synchronous")
def test_large_stdio_read_ahead_is_detected(self):
self.run_native(negative="stdio")
def test_pinned_xtensa_frame_and_object_do_not_grow(self):
compiler = os.environ.get("CONTACT_CRC_XTENSA_CXX") or shutil.which("xtensa-esp32s3-elf-g++")
if not compiler:
candidate = Path.home() / ".platformio/packages/toolchain-xtensa-esp32s3/bin/xtensa-esp32s3-elf-g++"
if candidate.is_file():
compiler = str(candidate)
if not compiler:
self.skipTest("Xtensa S3 compiler is not installed")
# Use SDK-compatible opaque File/FS declarations so native vector/mock
# internals cannot distort the target frame. The CFT/CRC bodies are real.
with tempfile.TemporaryDirectory(prefix="mesh-contact-crc-stack-") as directory:
temp = Path(directory)
(temp / "file_contract.h").write_text(OPAQUE_FILE)
source = self.source().replace(
'// Filesystem contract is supplied by this native fixture.', '#include "file_contract.h"')
(temp / "transaction_under_test.h").write_text(source)
(temp / "frame.cpp").write_text(
'#include "transaction_under_test.h"\n'
'extern "C" int commit_frame(mesh::ContactFileTransaction* p,unsigned n) '
'{ return int(p->serviceCommit(true,n)); }\n'
'char transaction_object_size[sizeof(mesh::ContactFileTransaction)];\n')
result = subprocess.run([
compiler, "-std=c++17", "-Os", "-fno-inline", "-fstack-usage",
"-DESP32_PLATFORM=1", "-I", str(ROOT / "src"), "-c",
str(temp / "frame.cpp"), "-o", str(temp / "frame.o"),
], capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
frames = [line.split("\t") for line in (temp / "frame.su").read_text().splitlines()
if "ContactFileTransaction::serviceCommit(" in line]
self.assertEqual(len(frames), 1)
self.assertEqual(int(frames[0][1]), 112)
nm = str(Path(compiler).with_name("xtensa-esp32s3-elf-nm"))
symbols = subprocess.check_output([nm, "-S", str(temp / "frame.o")], text=True)
size = next(int(line.split()[1], 16) for line in symbols.splitlines()
if line.endswith("transaction_object_size"))
self.assertEqual(size, 144)
if __name__ == "__main__":
unittest.main()
+11 -2
View File
@@ -26,6 +26,9 @@ class UncachedESPContactsTest(unittest.TestCase):
def test_synchronous_startup_batch_is_detected(self):
self.run_fixture(["-DBOARD_HAS_PSRAM=1"], disable="begin")
def test_default_background_crc_cadence_is_detected(self):
self.run_fixture(["-DBOARD_HAS_PSRAM=1"], disable="crc_batch")
def test_bounded_buffers_reduce_backend_work_without_stack_growth(self):
previous = self.run_fixture(["-DBOARD_HAS_PSRAM=1"], benchmark="previous")
current = self.run_fixture(["-DBOARD_HAS_PSRAM=1"], benchmark="current")
@@ -77,6 +80,11 @@ class UncachedESPContactsTest(unittest.TestCase):
implementation = implementation.replace(
'"/contacts3", companionPathPresence, true)',
'"/contacts3", companionPathPresence, false)')
elif disable == "crc_batch":
self.assertIn("_contact_write->serviceCommit(true, 8)", implementation)
implementation = implementation.replace(
"_contact_write->serviceCommit(true, 8)",
"_contact_write->serviceCommit()")
# Only the native SDK stat seam is substituted. The production
# presence logic still distinguishes ENOENT from metadata failure.
presence = method(source, "static bool companionPathPresence(")
@@ -159,8 +167,9 @@ class UncachedESPContactsTest(unittest.TestCase):
return measurements
else:
self.assertNotEqual(result.returncode, 0)
marker = "startup_operations" if disable == "begin" else (
"largest_backend_" + ("write" if disable == "writer" else "read"))
marker = ("startup_operations" if disable == "begin" else
"crc_read_passes == 104" if disable == "crc_batch" else
"largest_backend_" + ("write" if disable == "writer" else "read"))
self.assertIn(marker, result.stderr)