Coalesce ESP background contact I/O and defer backup cleanup
This commit is contained in:
1 parent
8bc93822ef
commit
9c43efb0bc
11 files changed
+584
-44
No files matched your search
+23
-2
@@ -489,11 +489,11 @@ static void background_crc_verification_uses_bounded_batches() {
|
||||
total_crc_reads += read_count;
|
||||
max_reads = std::max(max_reads, read_count);
|
||||
max_backend_reads = std::max(max_backend_reads, backend_count);
|
||||
assert(read_count <= 8 && backend_count <= 4);
|
||||
assert(read_count <= 8 && backend_count <= 1);
|
||||
}
|
||||
assert(total_crc_reads == 825);
|
||||
assert(crc_read_passes == 104);
|
||||
assert(max_reads == 8 && max_backend_reads == 4);
|
||||
assert(max_reads == 8 && max_backend_reads == 1);
|
||||
assert(!SPIFFS.exists("/contacts3.tmp"));
|
||||
f.host.contacts.clear();
|
||||
f.store.loadContacts(&f.host);
|
||||
@@ -818,9 +818,30 @@ static void contact_tx_policy_round_trips_without_growing_records() {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#if defined(ESP32_PLATFORM)
|
||||
static void deferred_cleanup_publishes_actual_cached_handles() {
|
||||
Fixture f(40);
|
||||
const auto old = SPIFFS.files.at("/contacts3");
|
||||
SPIFFS.files["/contacts3.bak"] = {9, 8, 7};
|
||||
assert(f.host.contacts[0].setRawPath(route(2100).data()));
|
||||
assert(f.store.markContactDirty(f.host.contacts[0]));
|
||||
for (unsigned step = 0; step < 4; ++step)
|
||||
assert(f.store.serviceContactWrites(&f.host, cachedContactFilter));
|
||||
assert(SPIFFS.files.at("/contacts3.bak") == std::vector<uint8_t>({9, 8, 7}));
|
||||
assert(f.store.serviceContactWrites(&f.host, cachedContactFilter));
|
||||
assert(!SPIFFS.exists("/contacts3.bak") && SPIFFS.files.at("/contacts3") == old);
|
||||
drain_cooperative_save(f);
|
||||
assert(!f.store.hasPendingContactWrites() && SPIFFS.files.at("/contacts3.bak") == old);
|
||||
f.check(0, route(2100));
|
||||
for (unsigned i = 1; i < 40; ++i) f.check(i, route(i));
|
||||
}
|
||||
#endif
|
||||
|
||||
int main() {
|
||||
contact_tx_policy_round_trips_without_growing_records();
|
||||
#if defined(ESP32_PLATFORM)
|
||||
deferred_cleanup_publishes_actual_cached_handles();
|
||||
cooperative_save_services_commands_and_preserves_snapshots();
|
||||
background_crc_verification_uses_bounded_batches();
|
||||
cooperative_mutations_never_publish_obsolete_indices();
|
||||
|
||||
+28
-6
@@ -41,6 +41,7 @@ public:
|
||||
unsigned config_calls = 0, closes = 0, renames = 0;
|
||||
unsigned fail_config = 0, short_read_call = 0, fail_rename = 0;
|
||||
size_t largest_read = 0, largest_backend_read = 0, max_write = SIZE_MAX;
|
||||
size_t backend_read_bytes = 0, reader_buffer_size = 0;
|
||||
bool fail_open_read = false, fail_open_write = false;
|
||||
std::vector<Files> rename_snapshots;
|
||||
bool exists(const char* path) { return files.count(path) != 0; }
|
||||
@@ -65,6 +66,7 @@ bool File::setBufferSize(size_t size) {
|
||||
if (++stream->fs->config_calls == stream->fs->fail_config) return false;
|
||||
assert(stream->buffer.empty() && stream->position == 0);
|
||||
stream->buffer_size = size;
|
||||
if (!stream->writing) stream->fs->reader_buffer_size = size;
|
||||
return true;
|
||||
}
|
||||
size_t File::size() const { return stream ? stream->fs->files.at(stream->path).size() : 0; }
|
||||
@@ -104,6 +106,7 @@ size_t File::read(uint8_t* data, size_t length) {
|
||||
const size_t n = std::min(stream->buffer_size, available);
|
||||
if (!n) break;
|
||||
++fs.backend_reads;
|
||||
fs.backend_read_bytes += n;
|
||||
fs.largest_backend_read = std::max(fs.largest_backend_read, n);
|
||||
stream->buffer.assign(source.begin() + stream->backend_position,
|
||||
source.begin() + stream->backend_position + n);
|
||||
@@ -147,6 +150,7 @@ unsigned effectiveBudget(unsigned requested) {
|
||||
|
||||
struct Measurement {
|
||||
unsigned passes, logical_reads, backend_reads, max_pass_reads, max_pass_refills;
|
||||
size_t max_pass_backend_bytes, reader_buffer_size;
|
||||
};
|
||||
Measurement verify(size_t bytes, unsigned budget, bool use_default = false) {
|
||||
FileSystem fs;
|
||||
@@ -156,19 +160,28 @@ Measurement verify(size_t bytes, unsigned budget, bool use_default = false) {
|
||||
assert(writer);
|
||||
writeImage(writer, next);
|
||||
unsigned passes = 0, max_reads = 0, max_refills = 0, data_passes = 0;
|
||||
size_t max_backend_bytes = 0;
|
||||
#if defined(ESP32_PLATFORM)
|
||||
// Selection is made once in Open, independently of the later read loop.
|
||||
const size_t selected = !use_default && budget >= 8 ? 512 : 128;
|
||||
#endif
|
||||
Progress result = Progress::Pending;
|
||||
while (result == Progress::Pending) {
|
||||
const unsigned before_reads = fs.reads, before_backend = fs.backend_reads;
|
||||
const size_t before_backend_bytes = fs.backend_read_bytes;
|
||||
const bool was_ready = writer.readyToPublish();
|
||||
result = use_default ? writer.serviceCommit() : writer.serviceCommit(true, budget);
|
||||
assert(++passes < 10000);
|
||||
const unsigned reads = fs.reads - before_reads, refills = fs.backend_reads - before_backend;
|
||||
max_reads = std::max(max_reads, reads);
|
||||
max_refills = std::max(max_refills, refills);
|
||||
const size_t backend_bytes = fs.backend_read_bytes - before_backend_bytes;
|
||||
max_backend_bytes = std::max(max_backend_bytes, backend_bytes);
|
||||
const unsigned bound = use_default ? 1 : effectiveBudget(budget);
|
||||
assert(reads <= bound);
|
||||
#if defined(ESP32_PLATFORM)
|
||||
assert(refills <= (bound + 1) / 2);
|
||||
assert(refills <= (64 * bound + selected - 1) / selected);
|
||||
assert(backend_bytes <= 512); // FILE refill admissions, not NOR operations or time
|
||||
#endif
|
||||
if (reads) {
|
||||
++data_passes;
|
||||
@@ -185,10 +198,12 @@ Measurement verify(size_t bytes, unsigned budget, bool use_default = false) {
|
||||
assert(fs.reads == (bytes + 63) / 64);
|
||||
assert(fs.largest_read <= 64);
|
||||
#if defined(ESP32_PLATFORM)
|
||||
assert(fs.largest_backend_read <= 128);
|
||||
assert(fs.backend_reads == (bytes + 127) / 128);
|
||||
assert(fs.reader_buffer_size == selected);
|
||||
assert(fs.largest_backend_read <= selected);
|
||||
assert(fs.backend_reads == (bytes + selected - 1) / selected);
|
||||
#endif
|
||||
return {passes, fs.reads, fs.backend_reads, max_reads, max_refills};
|
||||
return {passes, fs.reads, fs.backend_reads, max_reads, max_refills,
|
||||
max_backend_bytes, fs.reader_buffer_size};
|
||||
}
|
||||
void limitsAndTails() {
|
||||
for (size_t bytes : {size_t(0), size_t(1), size_t(63), size_t(64), size_t(65),
|
||||
@@ -203,6 +218,9 @@ void limitsAndTails() {
|
||||
fixture_commit_calls = 0;
|
||||
assert(writer.commit());
|
||||
assert(fixture_commit_calls == 829); // actual commit() drains default-one passes
|
||||
#if defined(ESP32_PLATFORM)
|
||||
assert(fs.reader_buffer_size == 128 && fs.backend_reads == 413);
|
||||
#endif
|
||||
}
|
||||
void readFailureAndCorruption() {
|
||||
const auto old = image(19), next = image(1536);
|
||||
@@ -300,7 +318,11 @@ int main() {
|
||||
#endif
|
||||
const auto one = verify(347 * 152, 1), eight = verify(347 * 152, 8);
|
||||
printf("{\"one_passes\":%u,\"eight_passes\":%u,\"logical_reads\":%u,"
|
||||
"\"backend_reads\":%u,\"max_pass_reads\":%u,\"max_pass_refills\":%u}\n",
|
||||
"\"backend_reads\":%u,\"max_pass_reads\":%u,\"max_pass_refills\":%u,"
|
||||
"\"max_pass_backend_bytes\":%zu,\"reader_buffer_size\":%zu,"
|
||||
"\"one_backend_reads\":%u,\"one_reader_buffer_size\":%zu}\n",
|
||||
one.passes, eight.passes, eight.logical_reads, eight.backend_reads,
|
||||
eight.max_pass_reads, eight.max_pass_refills);
|
||||
eight.max_pass_reads, eight.max_pass_refills,
|
||||
eight.max_pass_backend_bytes, eight.reader_buffer_size,
|
||||
one.backend_reads, one.reader_buffer_size);
|
||||
}
|
||||
+50
-11
@@ -137,7 +137,7 @@ unsigned drain(DataStore& store, Host& h, Filter filter = nullptr) {
|
||||
assert(SPIFFS.writes - writes <= 1);
|
||||
assert(SPIFFS.reads - reads <= 8);
|
||||
assert(SPIFFS.backend_writes - backend_writes <= 1);
|
||||
assert(SPIFFS.backend_reads - backend_reads <= 4);
|
||||
assert(SPIFFS.backend_reads - backend_reads <= 1);
|
||||
// Transport service runs here, between every production pass. It must
|
||||
// never observe an unpublished replacement or a target-name gap.
|
||||
if (store.hasPendingContactWrites()) {
|
||||
@@ -171,8 +171,8 @@ void boundedFullTableAndFirstSave() {
|
||||
const auto passes = drain(store, h, keepContact);
|
||||
assert(passes > h.contacts.size()); // serialization and CRC both yield
|
||||
assert(SPIFFS.largest_read <= 64);
|
||||
assert(SPIFFS.largest_backend_read <= 128);
|
||||
assert(SPIFFS.largest_backend_write <= 251);
|
||||
assert(SPIFFS.largest_backend_read <= 512); // background selected at Open
|
||||
assert(SPIFFS.largest_backend_write <= 502); // background selected at Configure
|
||||
assert(SPIFFS.largest_write == mesh::storage::CONTACT_RECORD_SIZE);
|
||||
assert(SPIFFS.removes == 0); // first publication has no backup/temp to delete
|
||||
assert(h.copies >= h.contacts.size());
|
||||
@@ -196,14 +196,16 @@ void backgroundCRCVerificationUsesBoundedBatches() {
|
||||
if (read_count != 0) ++crc_read_passes;
|
||||
max_reads = std::max(max_reads, read_count);
|
||||
max_backend_reads = std::max(max_backend_reads, backend_count);
|
||||
assert(read_count <= 8 && backend_count <= 4);
|
||||
assert(read_count <= 8 && backend_count <= 1);
|
||||
}
|
||||
// These are actual production DataStore calls, not a direct transaction
|
||||
// benchmark. Default one-chunk verification fails this cadence witness.
|
||||
assert(SPIFFS.reads == 825);
|
||||
assert(crc_read_passes == 104);
|
||||
assert(max_reads == 8 && max_backend_reads == 4);
|
||||
assert(SPIFFS.largest_read == 64 && SPIFFS.largest_backend_read == 128);
|
||||
assert(max_reads == 8 && max_backend_reads == 1);
|
||||
assert(SPIFFS.largest_read == 64 && SPIFFS.largest_backend_read == 512);
|
||||
assert(SPIFFS.backend_reads == 104);
|
||||
assert(SPIFFS.backend_writes == 106); // actual347-contact background502 job
|
||||
assert(SPIFFS.files.at("/contacts3") == image(h));
|
||||
bootCheck(h);
|
||||
}
|
||||
@@ -324,7 +326,7 @@ void deferredBeginFaultsAndSynchronousDefaults() {
|
||||
}
|
||||
|
||||
void mutationDuringEveryBeginStage() {
|
||||
for (unsigned completed = 0; completed <= 7; ++completed) {
|
||||
for (unsigned completed = 0; completed <= 8; ++completed) {
|
||||
SPIFFS = FakeFilesystem();
|
||||
SPIFFS.emulate_stdio = true;
|
||||
Host h = table();
|
||||
@@ -345,8 +347,8 @@ void mutationDuringEveryBeginStage() {
|
||||
assert(store.serviceContactWrites(&h, nullptr)); // guard precedes every begin step
|
||||
assert(store.hasPendingContactWrites());
|
||||
assert(SPIFFS.opens == opens && SPIFFS.buffer_config_calls == configurations);
|
||||
if (completed < 6) assert(SPIFFS.startupOperations() == before_cancel);
|
||||
if (completed < 5) assert(SPIFFS.files.at("/contacts3.tmp") == stale);
|
||||
if (completed < 7) assert(SPIFFS.startupOperations() == before_cancel);
|
||||
if (completed < 6) assert(SPIFFS.files.at("/contacts3.tmp") == stale);
|
||||
else assert(!SPIFFS.exists("/contacts3.tmp"));
|
||||
assert(SPIFFS.files.at("/contacts3") == old);
|
||||
drain(store, h);
|
||||
@@ -474,8 +476,9 @@ void realBackupCleanupFailure() {
|
||||
assert(!success && store.hasPendingContactWrites());
|
||||
assert(SPIFFS.files.at("/contacts3") == old);
|
||||
assert(SPIFFS.files.at("/contacts3.bak") == std::vector<uint8_t>({42}));
|
||||
// Best-effort temp cleanup cannot hide the primary remove failure.
|
||||
assert(SPIFFS.exists("/contacts3.tmp") && SPIFFS.missing_remove_logs == 0);
|
||||
// Retirement fails before this transaction owns or opens a new temp.
|
||||
assert(!SPIFFS.exists("/contacts3.tmp") && SPIFFS.missing_remove_logs == 0);
|
||||
assert(SPIFFS.writes == 0 && SPIFFS.opens == 0 && !store.activeJob());
|
||||
SPIFFS.fail_remove = false;
|
||||
drain(store, h);
|
||||
bootCheck(h);
|
||||
@@ -617,10 +620,46 @@ void boundedCancellationFlush() {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void deferredCleanupThroughActualDataStore() {
|
||||
SPIFFS = FakeFilesystem();
|
||||
SPIFFS.emulate_stdio = true;
|
||||
Host h = table(32);
|
||||
const auto old = image(h);
|
||||
const std::vector<uint8_t> older = {9, 8, 7};
|
||||
SPIFFS.files["/contacts3"] = old;
|
||||
SPIFFS.files["/contacts3.bak"] = older;
|
||||
DataStore store(SPIFFS);
|
||||
h.contacts[0] = contact(904);
|
||||
assert(store.markContactDirty(h.contacts[0]));
|
||||
assert(store.serviceContactWrites(&h, nullptr)); // allocation only
|
||||
for (unsigned step = 0; step < 3; ++step)
|
||||
assert(store.serviceContactWrites(&h, nullptr)); // Target, Backup, Recover
|
||||
assert(SPIFFS.files.at("/contacts3.bak") == older);
|
||||
const unsigned before = SPIFFS.startupOperations();
|
||||
assert(store.serviceContactWrites(&h, nullptr)); // standalone RetireBackup
|
||||
assert(SPIFFS.startupOperations() - before == 1);
|
||||
assert(SPIFFS.files.at("/contacts3") == old && !SPIFFS.exists("/contacts3.bak"));
|
||||
drain(store, h);
|
||||
assert(SPIFFS.exists("/contacts3.bak"));
|
||||
assert(!store.hasPendingContactWrites() && SPIFFS.files.at("/contacts3.bak") == old);
|
||||
// A mutation after the completed job must retire that backup before replacing
|
||||
// the newest authoritative target, with no stale index publication.
|
||||
const auto prior = image(h);
|
||||
h.contacts.erase(h.contacts.begin() + 4);
|
||||
assert(store.releaseContact(contact(4)));
|
||||
h.contacts[0] = contact(905);
|
||||
assert(store.markContactDirty(h.contacts[0]));
|
||||
drain(store, h);
|
||||
assert(SPIFFS.files.at("/contacts3.bak") == prior);
|
||||
bootCheck(h);
|
||||
}
|
||||
|
||||
int main() {
|
||||
#if defined(CONTACT_BUFFER_BENCHMARK)
|
||||
bufferedAdmissionBenchmark();
|
||||
#else
|
||||
deferredCleanupThroughActualDataStore();
|
||||
boundedFullTableAndFirstSave();
|
||||
backgroundCRCVerificationUsesBoundedBatches();
|
||||
boundedCancellationFlush();
|
||||
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
#include "FakeFilesystem.h"
|
||||
#define FILESYSTEM FakeFilesystem
|
||||
#define private public
|
||||
#include "transaction_under_test.h"
|
||||
#undef private
|
||||
#include <cstdio>
|
||||
using Writer=mesh::ContactFileTransaction;
|
||||
using Progress=Writer::CommitProgress;
|
||||
using Bytes=std::vector<uint8_t>;
|
||||
#if defined(ESP32_PLATFORM)
|
||||
#define DEFER_BEGIN ,true
|
||||
#else
|
||||
#define DEFER_BEGIN
|
||||
#endif
|
||||
Bytes image(size_t n,uint8_t salt=0){Bytes out(n);for(size_t i=0;i<n;++i)out[i]=uint8_t(i*29+i/3+salt);return out;}
|
||||
bool presence(FakeFilesystem* fs,const char* name,bool& found){++fs->metadata_probes;if(fs->metadata_error)return false;found=fs->exists(name);return true;}
|
||||
void begin(Writer& w,bool coalesce=false){
|
||||
#if defined(ESP32_PLATFORM)
|
||||
for(unsigned i=0;!w&&i<9;++i){const auto before=SPIFFS.startupOperations();assert(w.serviceBegin(coalesce)!=Writer::BeginProgress::Failed);assert(SPIFFS.startupOperations()-before<=1);}
|
||||
#else
|
||||
(void)coalesce;
|
||||
#endif
|
||||
assert(w);
|
||||
}
|
||||
void write(Writer& w,const Bytes& bytes){for(size_t i=0;i<bytes.size();i+=152){size_t n=std::min(size_t(152),bytes.size()-i);assert(w.write(bytes.data()+i,n)==n);}}
|
||||
unsigned finish(Writer& w,bool defer,unsigned budget=8){Progress result=Progress::Pending;unsigned calls=0;while(result==Progress::Pending){assert(++calls<2000);result=w.serviceCommit(true,budget,defer);}assert(result==Progress::Succeeded);return calls;}
|
||||
void reset(const Bytes& old){SPIFFS=FakeFilesystem();SPIFFS.emulate_stdio=true;SPIFFS.files["/contacts3"]=old;}
|
||||
void contracts(){
|
||||
const auto old=image(304,1),next=image(608,2);
|
||||
for(bool deferred:{false,true}) for(bool coalesce:{false,true}) {
|
||||
reset(old);{
|
||||
Writer w(&SPIFFS,"/contacts3",presence DEFER_BEGIN);begin(w,coalesce);write(w,next);
|
||||
const unsigned removes=SPIFFS.removes;
|
||||
finish(w,deferred);
|
||||
assert(SPIFFS.files.at("/contacts3")==next);
|
||||
#if defined(ESP32_PLATFORM)
|
||||
assert(SPIFFS.exists("/contacts3.bak")==deferred);
|
||||
assert(SPIFFS.removes-removes==unsigned(!deferred));
|
||||
#else
|
||||
assert(!SPIFFS.exists("/contacts3.bak"));assert(SPIFFS.removes-removes==1);
|
||||
#endif
|
||||
}
|
||||
#if defined(ESP32_PLATFORM)
|
||||
if(deferred)assert(SPIFFS.files.at("/contacts3.bak")==old);
|
||||
#endif
|
||||
assert(!SPIFFS.exists("/contacts3.tmp"));
|
||||
}
|
||||
// Default synchronous failures retain the pre-existing prior-generation backup.
|
||||
#if defined(ESP32_PLATFORM)
|
||||
reset(old);const auto older=image(152,9);SPIFFS.files["/contacts3.bak"]=older;SPIFFS.fail_buffer_config=1;
|
||||
{Writer w(&SPIFFS,"/contacts3",presence);assert(!w);}
|
||||
assert(SPIFFS.files.at("/contacts3")==old&&SPIFFS.files.at("/contacts3.bak")==older);
|
||||
// RecoverFALSE skips retirement even when cleanupTRUE at Publish.
|
||||
reset(old);SPIFFS.files["/contacts3.bak"]=older;
|
||||
{Writer w(&SPIFFS,"/contacts3",presence,true);begin(w,false);assert(SPIFFS.files.at("/contacts3.bak")==older);write(w,next);finish(w,true);}
|
||||
assert(SPIFFS.files.at("/contacts3.bak")==old);
|
||||
// RecoverTRUE latches the retirement stage; ConfigureFALSE retains writer251.
|
||||
reset(old);SPIFFS.files["/contacts3.bak"]=older;
|
||||
{Writer w(&SPIFFS,"/contacts3",presence,true);
|
||||
for(unsigned i=0;i<3;++i)assert(w.serviceBegin(true)==Writer::BeginProgress::Pending);
|
||||
assert(w.serviceBegin(false)==Writer::BeginProgress::Pending);assert(!SPIFFS.exists("/contacts3.bak"));
|
||||
begin(w,false);write(w,next);finish(w,true);assert(SPIFFS.largest_backend_write==251);}
|
||||
#endif
|
||||
// Default synchronous commit still cleans the old backup.
|
||||
reset(old);{Writer w(&SPIFFS,"/contacts3",presence);assert(w);write(w,next);assert(w.commit());}
|
||||
assert(!SPIFFS.exists("/contacts3.bak")&&SPIFFS.files.at("/contacts3")==next);
|
||||
}
|
||||
void retirement(){
|
||||
#if defined(ESP32_PLATFORM)
|
||||
const auto old=image(304,1),older=image(152,2);reset(old);SPIFFS.files["/contacts3.bak"]=older;
|
||||
{
|
||||
Writer w(&SPIFFS,"/contacts3",presence,true);
|
||||
for(unsigned i=0;i<3;++i)assert(w.serviceBegin(true)==Writer::BeginProgress::Pending);
|
||||
assert(SPIFFS.files.at("/contacts3")==old&&SPIFFS.files.at("/contacts3.bak")==older);
|
||||
const auto removes=SPIFFS.removes,probes=SPIFFS.metadata_probes,opens=SPIFFS.opens;
|
||||
assert(w.serviceBegin(true)==Writer::BeginProgress::Pending);
|
||||
assert(SPIFFS.removes-removes==1&&SPIFFS.metadata_probes==probes&&SPIFFS.opens==opens);
|
||||
assert(!SPIFFS.exists("/contacts3.bak")&&SPIFFS.files.at("/contacts3")==old);
|
||||
}
|
||||
// Missing target recovers backup, updates flags and performs no removal.
|
||||
SPIFFS=FakeFilesystem();SPIFFS.files["/contacts3.bak"]=old;
|
||||
{
|
||||
Writer w(&SPIFFS,"/contacts3",presence,true);
|
||||
for(unsigned i=0;i<3;++i)assert(w.serviceBegin(true)==Writer::BeginProgress::Pending);
|
||||
assert(w._begin_target_exists&&!w._begin_backup_exists);
|
||||
assert(SPIFFS.files.at("/contacts3")==old&&!SPIFFS.exists("/contacts3.bak"));
|
||||
const auto removes=SPIFFS.removes;
|
||||
assert(w.serviceBegin(true)==Writer::BeginProgress::Pending);
|
||||
assert(SPIFFS.removes==removes&&SPIFFS.missing_remove_logs==0);
|
||||
}
|
||||
// Failed retirement stops before new/temp ownership; prior temp is untouched.
|
||||
reset(old);SPIFFS.files["/contacts3.bak"]=older;SPIFFS.files["/contacts3.tmp"]={9,8,7};SPIFFS.fail_remove=true;
|
||||
{
|
||||
Writer w(&SPIFFS,"/contacts3",presence,true);
|
||||
for(unsigned i=0;i<3;++i)assert(w.serviceBegin(true)==Writer::BeginProgress::Pending);
|
||||
assert(w.serviceBegin(true)==Writer::BeginProgress::Failed&&!w&&!w._owns_temp);
|
||||
assert(!w.commit()&&w.write(old.data(),old.size())==0);
|
||||
assert(SPIFFS.opens==0&&SPIFFS.writes==0);
|
||||
}
|
||||
assert(SPIFFS.files.at("/contacts3")==old&&SPIFFS.files.at("/contacts3.bak")==older);
|
||||
assert(SPIFFS.files.at("/contacts3.tmp")==Bytes({9,8,7}));
|
||||
#endif
|
||||
}
|
||||
void repeatedAndCancel(){
|
||||
#if defined(ESP32_PLATFORM)
|
||||
const auto old=image(152,1),next=image(608,2),last=image(760,3);reset(old);
|
||||
{Writer w(&SPIFFS,"/contacts3",presence,true);begin(w,true);write(w,next);finish(w,true);}
|
||||
assert(SPIFFS.files.at("/contacts3.bak")==old);
|
||||
for(unsigned completed=0;completed<=8;++completed) {
|
||||
reset(next);SPIFFS.files["/contacts3.bak"]=old;
|
||||
{
|
||||
Writer w(&SPIFFS,"/contacts3",presence,true);
|
||||
for(unsigned i=0;i<completed;++i){const auto n=SPIFFS.startupOperations();assert(w.serviceBegin(true)!=Writer::BeginProgress::Failed);assert(SPIFFS.startupOperations()-n<=1);}
|
||||
if(w)write(w,last);
|
||||
assert(SPIFFS.files.at("/contacts3")==next);
|
||||
}
|
||||
assert(SPIFFS.files.at("/contacts3")==next);
|
||||
assert(SPIFFS.exists("/contacts3.bak")==bool(completed<4));
|
||||
FakeFilesystem boot;boot.files=SPIFFS.files;assert(Writer::recover(&boot,"/contacts3",presence));assert(boot.files.at("/contacts3")==next);
|
||||
}
|
||||
reset(next);SPIFFS.files["/contacts3.bak"]=old;
|
||||
{Writer w(&SPIFFS,"/contacts3",presence,true);begin(w,true);write(w,last);finish(w,true);}
|
||||
assert(SPIFFS.files.at("/contacts3")==last&&SPIFFS.files.at("/contacts3.bak")==next);
|
||||
#endif
|
||||
}
|
||||
void faultsAndCrash(){
|
||||
const auto old=image(304,1),next=image(608,2);
|
||||
for(bool deferred:{false,true})for(unsigned fault:{0U,1U,2U}) {
|
||||
reset(old);
|
||||
{
|
||||
Writer w(&SPIFFS,"/contacts3",presence DEFER_BEGIN);begin(w,true);write(w,next);
|
||||
if(fault)SPIFFS.fail_rename=fault;
|
||||
Progress result=Progress::Pending;unsigned n=0;
|
||||
while(result==Progress::Pending){assert(++n<2000);result=w.serviceCommit(true,8,deferred);}
|
||||
assert(result==(fault?Progress::Failed:Progress::Succeeded));
|
||||
}
|
||||
assert(SPIFFS.files.at("/contacts3")== (fault?old:next));
|
||||
const auto snapshots=SPIFFS.rename_snapshots;
|
||||
for(const auto& snapshot:snapshots){FakeFilesystem boot;boot.files=snapshot;assert(Writer::recover(&boot,"/contacts3",presence));const auto& bytes=boot.files.at("/contacts3");assert(bytes==old||bytes==next);}
|
||||
}
|
||||
}
|
||||
void counts(){
|
||||
#if defined(ESP32_PLATFORM)
|
||||
for(bool deferred:{false,true})for(bool coalesce:{false,true}) {
|
||||
reset(image(152,9));const auto next=image(347*152,3);
|
||||
Writer w(&SPIFFS,"/contacts3",presence,true);begin(w,coalesce);write(w,next);
|
||||
assert(finish(w,deferred)==108);assert(SPIFFS.files.at("/contacts3")==next);
|
||||
assert(SPIFFS.backend_writes==(coalesce?106U:211U));
|
||||
assert(SPIFFS.largest_backend_write==(coalesce?502U:251U));
|
||||
assert(SPIFFS.reads==825&&SPIFFS.backend_reads==104);
|
||||
}
|
||||
reset(image(152,9));Writer w(&SPIFFS,"/contacts3",presence);assert(w);write(w,image(347*152));
|
||||
assert(finish(w,false,1)==829&&SPIFFS.backend_writes==211&&SPIFFS.backend_reads==413);
|
||||
#endif
|
||||
}
|
||||
int main(int argc,char**argv){const std::string mode=argc>1?argv[1]:"all";if(mode=="contracts")contracts();else if(mode=="retire")retirement();else if(mode=="cancel")repeatedAndCancel();else if(mode=="faults")faultsAndCrash();else if(mode=="counts")counts();else assert(false);}
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
// Executes the real CFT with the existing stdio-admission model.
|
||||
#include "FakeFilesystem.h"
|
||||
#define FILESYSTEM FakeFilesystem
|
||||
#include "transaction_under_test.h"
|
||||
#include <cstdio>
|
||||
|
||||
using Writer = mesh::ContactFileTransaction;
|
||||
using Progress = Writer::CommitProgress;
|
||||
using Bytes = std::vector<uint8_t>;
|
||||
Bytes image(size_t bytes) {
|
||||
Bytes result(bytes);
|
||||
for (size_t i = 0; i < bytes; ++i) result[i] = uint8_t(i * 23 + i / 7);
|
||||
return result;
|
||||
}
|
||||
bool presence(FakeFilesystem* fs, const char* path, bool& found) {
|
||||
++fs->metadata_probes;
|
||||
if (fs->metadata_error) return false;
|
||||
found = fs->exists(path);
|
||||
return true;
|
||||
}
|
||||
void begin(Writer& writer, bool background) {
|
||||
for (unsigned i = 0; !writer && i < 8; ++i)
|
||||
assert(writer.serviceBegin(background) != Writer::BeginProgress::Failed);
|
||||
assert(writer);
|
||||
}
|
||||
void records(Writer& writer, const Bytes& bytes, size_t cap) {
|
||||
assert(bytes.size() % 152 == 0);
|
||||
for (size_t offset = 0; offset < bytes.size(); offset += 152) {
|
||||
const unsigned before = SPIFFS.backend_writes;
|
||||
assert(writer.write(bytes.data() + offset, 152) == 152);
|
||||
assert(SPIFFS.backend_writes - before <= 1);
|
||||
assert(SPIFFS.largest_backend_write <= cap);
|
||||
}
|
||||
}
|
||||
Progress finish(Writer& writer, unsigned budget = 8) {
|
||||
Progress progress = Progress::Pending;
|
||||
for (unsigned i = 0; progress == Progress::Pending && i < 2000; ++i)
|
||||
progress = writer.serviceCommit(true, budget);
|
||||
assert(progress != Progress::Pending);
|
||||
return progress;
|
||||
}
|
||||
void measure(unsigned count, bool background) {
|
||||
SPIFFS = FakeFilesystem(); SPIFFS.emulate_stdio = true;
|
||||
const Bytes bytes = image(count * 152);
|
||||
Writer writer(&SPIFFS, "/contacts3", presence, true);
|
||||
begin(writer, background);
|
||||
const unsigned configurations = SPIFFS.buffer_config_calls;
|
||||
assert(writer.serviceBegin(!background) == Writer::BeginProgress::Ready);
|
||||
assert(SPIFFS.buffer_config_calls == configurations); // selection latched
|
||||
const size_t cap = background ? 502 : 251;
|
||||
records(writer, bytes, cap);
|
||||
assert(SPIFFS.writes == count);
|
||||
assert(SPIFFS.backend_writes == bytes.size() / cap);
|
||||
assert(finish(writer) == Progress::Succeeded);
|
||||
assert(SPIFFS.files.at("/contacts3") == bytes);
|
||||
assert(SPIFFS.backend_writes == (bytes.size() + cap - 1) / cap);
|
||||
assert(SPIFFS.largest_backend_write <= cap);
|
||||
assert(SPIFFS.missing_remove_logs == 0);
|
||||
printf("{\"records\":%u,\"background\":%s,\"bytes\":%zu,\"logical_writes\":%u,"
|
||||
"\"backend_writes\":%u,\"largest_backend_write\":%zu}\n",
|
||||
count, background ? "true" : "false", bytes.size(), SPIFFS.writes,
|
||||
SPIFFS.backend_writes, SPIFFS.largest_backend_write);
|
||||
}
|
||||
void synchronousDefault() {
|
||||
SPIFFS = FakeFilesystem(); SPIFFS.emulate_stdio = true;
|
||||
const Bytes bytes = image(347 * 152);
|
||||
Writer writer(&SPIFFS, "/contacts3", presence);
|
||||
assert(writer); records(writer, bytes, 251);
|
||||
assert(writer.commit());
|
||||
assert(SPIFFS.backend_writes == 211 && SPIFFS.largest_backend_write == 251);
|
||||
assert(SPIFFS.files.at("/contacts3") == bytes);
|
||||
}
|
||||
void configureTime(bool earlier, bool configure) {
|
||||
SPIFFS = FakeFilesystem(); SPIFFS.emulate_stdio = true;
|
||||
Writer writer(&SPIFFS, "/contacts3", presence, true);
|
||||
// Reach the open stream without configuring it, regardless of whether
|
||||
// background preparation also retires an earlier backup.
|
||||
unsigned passes = 0;
|
||||
while (SPIFFS.opens == 0) {
|
||||
assert(++passes < 16);
|
||||
assert(writer.serviceBegin(earlier) == Writer::BeginProgress::Pending);
|
||||
}
|
||||
assert(SPIFFS.buffer_config_calls == 0 && SPIFFS.writes == 0);
|
||||
assert(writer.serviceBegin(configure) == Writer::BeginProgress::Ready);
|
||||
const unsigned configurations = SPIFFS.buffer_config_calls;
|
||||
assert(writer.serviceBegin(!configure) == Writer::BeginProgress::Ready);
|
||||
assert(SPIFFS.buffer_config_calls == configurations);
|
||||
records(writer, image(4 * 152), configure ? 502 : 251);
|
||||
assert(SPIFFS.largest_backend_write == (configure ? 502 : 251));
|
||||
}
|
||||
void cancellation() {
|
||||
// gcd(152,502)=2:251 records are exactly76 full502 buffers.
|
||||
for (bool background : {false,true}) for (unsigned n : {1U,2U,3U,4U,250U,251U,252U,347U}) {
|
||||
SPIFFS = FakeFilesystem(); SPIFFS.emulate_stdio = true;
|
||||
const Bytes old = image(152), next = image(n * 152);
|
||||
SPIFFS.files["/contacts3"] = old;
|
||||
const size_t cap = background ? 502 : 251;
|
||||
unsigned before_close = 0;
|
||||
{
|
||||
Writer writer(&SPIFFS, "/contacts3", presence, true); begin(writer,background);
|
||||
records(writer,next,cap);
|
||||
assert(SPIFFS.backend_writes == next.size() / cap);
|
||||
before_close = SPIFFS.backend_writes;
|
||||
assert(SPIFFS.files.at("/contacts3") == old);
|
||||
}
|
||||
assert(SPIFFS.backend_writes-before_close == unsigned(next.size()%cap != 0));
|
||||
assert(SPIFFS.backend_writes == (next.size()+cap-1)/cap);
|
||||
assert(SPIFFS.largest_backend_write <= cap);
|
||||
assert(SPIFFS.files.at("/contacts3") == old && !SPIFFS.exists("/contacts3.tmp"));
|
||||
}
|
||||
}
|
||||
void setupFailures() {
|
||||
for (bool background : {false,true}) for (unsigned fault : {1U,2U,3U}) {
|
||||
SPIFFS=FakeFilesystem(); SPIFFS.emulate_stdio=true;
|
||||
const Bytes old=image(152); SPIFFS.files["/contacts3"]=old;
|
||||
if (fault==1) SPIFFS.fail_buffer_config=1;
|
||||
if (fault==2) SPIFFS.metadata_error=true;
|
||||
if (fault==3) SPIFFS.fail_create=SPIFFS.partial_create_failure=true;
|
||||
{
|
||||
Writer writer(&SPIFFS,"/contacts3",presence,true);
|
||||
auto result=Writer::BeginProgress::Pending;
|
||||
for (unsigned i=0;result==Writer::BeginProgress::Pending && i<8;++i)
|
||||
result=writer.serviceBegin(background);
|
||||
assert(result==Writer::BeginProgress::Failed && !writer);
|
||||
assert(writer.write(old.data(),old.size())==0);
|
||||
assert(writer.serviceCommit()==Progress::Failed);
|
||||
assert(SPIFFS.writes==0 && SPIFFS.backend_writes==0 && SPIFFS.reads==0);
|
||||
}
|
||||
assert(SPIFFS.files.at("/contacts3")==old);
|
||||
if(fault!=2) assert(!SPIFFS.exists("/contacts3.tmp"));
|
||||
}
|
||||
}
|
||||
void faultsAndCrash() {
|
||||
for (bool background : {false,true}) for(unsigned fault : {0U,1U,2U,3U,4U,5U}) {
|
||||
SPIFFS=FakeFilesystem(); SPIFFS.emulate_stdio=true;
|
||||
const Bytes old=image(152),next=image(4*152);SPIFFS.files["/contacts3"]=old;
|
||||
{
|
||||
Writer writer(&SPIFFS,"/contacts3",presence,true);begin(writer,background);
|
||||
if(fault==1) {
|
||||
SPIFFS.max_write=151;
|
||||
assert(writer.write(next.data(),152)==151);
|
||||
} else records(writer,next,background?502:251);
|
||||
if(fault==2) SPIFFS.fail_read="/contacts3.tmp";
|
||||
if(fault==3) SPIFFS.fail_rename=1;
|
||||
if(fault==4) SPIFFS.fail_rename=2;
|
||||
if(fault==5) {
|
||||
assert(writer.serviceCommit()==Progress::Pending); // flush before corrupting temp
|
||||
SPIFFS.files.at("/contacts3.tmp")[30]^=1;
|
||||
}
|
||||
const auto result=finish(writer);
|
||||
assert(result==(fault==0?Progress::Succeeded:Progress::Failed));
|
||||
}
|
||||
assert(SPIFFS.files.at("/contacts3")== (fault==0?next:old));
|
||||
assert(!SPIFFS.exists("/contacts3.tmp"));
|
||||
const auto snapshots=SPIFFS.rename_snapshots;
|
||||
for (const auto& snapshot:snapshots) {
|
||||
SPIFFS=FakeFilesystem();SPIFFS.files=snapshot;
|
||||
assert(Writer::recover(&SPIFFS,"/contacts3",presence));
|
||||
const auto& recovered=SPIFFS.files.at("/contacts3");
|
||||
assert(recovered==old || recovered==next);
|
||||
}
|
||||
}
|
||||
}
|
||||
int main(int argc,char** argv) {
|
||||
const std::string mode=argc>1?argv[1]:"all";
|
||||
if(mode=="measure") {
|
||||
for(unsigned n : {347U,349U,350U}) for(bool background : {false,true}) measure(n,background);
|
||||
} else if(mode=="default") synchronousDefault();
|
||||
else if(mode=="configure") {configureTime(true,false);configureTime(false,true);}
|
||||
else if(mode=="cancel") cancellation();
|
||||
else if(mode=="faults") {setupFailures();faultsAndCrash();}
|
||||
else assert(false && "unknown fixture mode");
|
||||
}
|
||||
Reference in new issue
Block a user