diff --git a/docs/fleet_management.md b/docs/fleet_management.md index 78e3405a8..0436b5e74 100644 --- a/docs/fleet_management.md +++ b/docs/fleet_management.md @@ -67,7 +67,13 @@ is unknown. Fleet reception and further fleet settings writes are disabled, and `get fleet.channel` reports `store=error`. Enrollment shown in RAM is not active authority while held. Restart to load and validate the public file, then check enrollment before retrying. A `not saved; retry later` error occurred before -publication and does not impose this hold. +publication or after a confirmed rollback, and does not impose this hold. On +ESP32/RP2040, rename return values alone are not sufficient: the transaction +checks both names before accepting the move or rollback. Fleet also checks the +exact public settings image before acknowledging new authority. A Fleet hold +does not stop the node's normal radio, USB, or individually authenticated admin +services; it prevents uncertain credentials or replay state from granting remote +Fleet control. `get fleet.stats` includes `parts`, the number of fragments currently buffered across the two bounded, incomplete two-packet command slots. diff --git a/docs/research/companion_contact_cache.md b/docs/research/companion_contact_cache.md index 184716d62..2fe191962 100644 --- a/docs/research/companion_contact_cache.md +++ b/docs/research/companion_contact_cache.md @@ -36,8 +36,15 @@ before replacing the file or recovering a saved transaction. Changing a copied contact preserves the old path for a pending app response or rollback. Dirty entries are saved before eviction. If storage cannot be read, written, or represented completely, the operation fails instead of -silently losing a contact or sending along a different route. Reboot retries -an incomplete load from the preserved files. The path checksum also rejects +sending along a different route. Contact I/O failures request bounded recovery +between callbacks: reload a readable live image, otherwise a readable backup, +and discard only unrecoverable contact data if neither works. nRF52 preserves +healthy pages when another page is damaged. No archival `.bad` files are kept. +If damaged contacts cannot be removed, the node remains usable with volatile +contacts; explicit saves still report failure, while reboot/shutdown remain +available. Identity, credentials, channels and unrelated settings are not erased. +Allocation/capacity failures and shared filesystem-migration authority errors +are not corrupt-contact evidence and do not authorize this reset. The path checksum also rejects a cold path changed underneath a live handle; it does not add a new checksum to the legacy ESP32 on-disk format. diff --git a/examples/companion_radio/DataStore.cpp b/examples/companion_radio/DataStore.cpp index 42940eca7..d009f93f3 100644 --- a/examples/companion_radio/DataStore.cpp +++ b/examples/companion_radio/DataStore.cpp @@ -829,6 +829,7 @@ bool DataStore::formatFileSystem() { || _fsExtra == nullptr || _fsExtra->format(); #endif const bool success = primary_success && secondary_success; + if (success) _contact_recovery = 0; #if defined(NRF52_PLATFORM) // A successful explicit erase creates a new empty authoritative store. A // failed erase must keep the boot's load quarantine latched. @@ -854,6 +855,7 @@ bool DataStore::formatFileSystem() { #elif defined(RP2040_PLATFORM) const bool success = LittleFS.format(); if (success) { + _contact_recovery = 0; _identity_creation_blocked = false; _prefs_load_incomplete = false; _channel_load_incomplete = false; @@ -864,6 +866,7 @@ bool DataStore::formatFileSystem() { bool fs_success = ((fs::SPIFFSFS *)_fs)->format(); esp_err_t nvs_err = nvs_flash_erase(); // no need to reinit, will be done by reboot if (fs_success && nvs_err == ESP_OK) { + _contact_recovery = 0; _advert_write.clear(); _identity_creation_blocked = false; _prefs_load_incomplete = false; @@ -1128,10 +1131,15 @@ bool DataStore::loadPrefsInt(const char *filename, // actually emitted by those releases, but reject a truncated field/group // or an unknown tail before any value reaches the live preferences. static const uint32_t MIN_PREFS_SIZE = 84; -#if defined(STM32_PLATFORM) - // All published STM32 image lengths fit in one byte. The file size below +#if defined(STM32_PLATFORM) || defined(NRF52_PLATFORM) + // All published ARM image lengths fit in one byte. The file size below // remains wide so oversized files cannot alias an accepted legacy length. using PrefsImageSize = uint8_t; + static_assert(236 + mesh::companion::traffic::FooterSize +#if defined(RAK_4631) + + mesh::peripherals::FooterSize +#endif + <= UINT8_MAX, "published preference sizes fit the compact allowlist"); #else using PrefsImageSize = uint16_t; #endif @@ -1234,7 +1242,7 @@ bool DataStore::loadPrefsInt(const char *filename, return false; } bool success = true; -#if defined(STM32_PLATFORM) && defined(__GNUC__) +#if (defined(STM32_PLATFORM) || defined(NRF52_PLATFORM)) && defined(__GNUC__) using namespace mesh::companion_prefs; uint8_t pad[4]; for (size_t i = 0; i < FIELD_COUNT; ++i) { @@ -1380,17 +1388,18 @@ bool DataStore::loadPrefsInt(const char *filename, readOptionalField(&loaded_prefs.usb_debug_enabled, sizeof(loaded_prefs.usb_debug_enabled)); readOptionalField(&loaded_prefs.lost_reply, sizeof(loaded_prefs.lost_reply)); +#endif #if defined(RAK_4631) if (file.available() != 0) { uint8_t peripheral_footer[mesh::peripherals::FooterSize] = {}; - success = success && readField(peripheral_footer, sizeof(peripheral_footer)) + success = success + && file.read(peripheral_footer, sizeof(peripheral_footer)) == sizeof(peripheral_footer) && mesh::peripherals::decodeFooter(peripheral_footer, sizeof(peripheral_footer), loaded_prefs.peripherals_shared); } -#endif #endif - // The compact STM32 table and the explicit readers both end at the + // The compact ARM table and the explicit readers both end at the // unchanged historical prefix. Consume the common footer only afterward. if (file.available() != 0) { uint8_t traffic_footer[mesh::companion::traffic::FooterSize] = {}; @@ -1451,6 +1460,18 @@ bool DataStore::savePrefs(const CompanionNodePrefs& _prefs, double node_lat, dou } bool success = used == sizeof(image) && file.write(image, sizeof(image)) == sizeof(image); +#elif defined(NRF52_PLATFORM) && defined(__GNUC__) + // nRF52 retains streaming writes and its existing small stack footprint. + // The table is the same validated wire order used by the scalar emitter. + using namespace mesh::companion_prefs; + const uint8_t pad[4] = {}; + bool success = true; + for (const PrefField& field : PREF_FIELDS) { + if (!success) break; + success = file.write(static_cast( + prefValue(field, _prefs, node_lat, node_lon, pad)), field.length) + == field.length; + } #else #if defined(STM32_PLATFORM) // Keep the explicit wire field list, but make one bounded atomic write. @@ -1593,16 +1614,16 @@ bool DataStore::savePrefs(const CompanionNodePrefs& _prefs, double node_lat, dou memcpy(tail + 6, &_prefs.gps_sync_interval_hours, sizeof(_prefs.gps_sync_interval_hours)); success = success && writeField(tail, sizeof(tail)); -#if defined(RAK_4631) - uint8_t peripheral_footer[mesh::peripherals::FooterSize]; - mesh::peripherals::encodeFooter(peripheral_footer, _prefs.peripherals_shared); - success = success && writeField(peripheral_footer, sizeof(peripheral_footer)); -#endif - #if defined(STM32_PLATFORM) success = success && image_length == sizeof(image) && file.write(image, sizeof(image)) == sizeof(image); #endif +#endif +#if defined(RAK_4631) + uint8_t peripheral_footer[mesh::peripherals::FooterSize]; + mesh::peripherals::encodeFooter(peripheral_footer, _prefs.peripherals_shared); + success = success && file.write(peripheral_footer, sizeof(peripheral_footer)) + == sizeof(peripheral_footer); #endif uint8_t traffic_footer[mesh::companion::traffic::FooterSize]; mesh::companion::traffic::encodeFooter(traffic_footer, @@ -1788,7 +1809,7 @@ bool DataStore::loadContactPages(DataStoreHost* host, uint16_t minimum_slot, uint32_t expected_page_mask) { bool any_page_file = false; FILESYSTEM* fs = _getContactsChannelsFS(); - auto quarantineUnreadPage = [this](uint8_t page) { + auto quarantineUnreadPage = [this](uint8_t page, bool storage_error = true) { // Fail closed if an otherwise present page cannot be read. Reserving its // slots prevents a new contact from overwriting records which a later boot // may recover successfully. @@ -1797,7 +1818,10 @@ bool DataStore::loadContactPages(DataStoreHost* host, uint16_t minimum_slot, _contact_slots.reserve( (uint16_t)page * mesh::storage::CONTACTS_PER_PAGE + index); } - _unread_contact_pages.mark(page); + if (storage_error) { + _unread_contact_pages.mark(page); + queueContactRecovery(); + } _contact_load_incomplete = true; }; @@ -1877,7 +1901,7 @@ bool DataStore::loadContactPages(DataStoreHost* host, uint16_t minimum_slot, MESH_DEBUG_PRINTLN( "DataStore: no memory to load contact page %u", page); file.close(); - quarantineUnreadPage(page); + quarantineUnreadPage(page, false); continue; } const bool payload_read = @@ -2070,7 +2094,190 @@ bool DataStore::writeContactPage(DataStoreHost* host, uint8_t page, } #endif +void DataStore::clearContactLoadFailure() { +#if MESH_CONTACT_CACHE + _cache_load_incomplete = false; +#elif !defined(NRF52_PLATFORM) + _uncached_contact_load_incomplete = false; +#endif +#if defined(NRF52_PLATFORM) + _contact_load_incomplete = false; + _unread_contact_pages.clearAll(); +#endif +} + +#if !defined(NRF52_PLATFORM) +// Validate a complete image before retiring its alternative. Legacy contacts +// have no file CRC; retain their existing record/route repair semantics. +static bool readableContactFile(FILESYSTEM* fs, const char* path) { + File file = mesh::openFileRead(fs, path); + bool valid = file && !file.isDirectory() + && file.size() % mesh::storage::CONTACT_RECORD_SIZE == 0; + uint8_t record[mesh::storage::CONTACT_RECORD_SIZE]; + size_t remaining = valid ? file.size() : 0; + while (valid && remaining != 0) { + valid = file.read(record, sizeof(record)) == sizeof(record); + remaining -= sizeof(record); + } + file.close(); + return valid; +} + +const char* DataStore::selectContactFile(bool& present) { + FILESYSTEM* fs = _getContactsChannelsFS(); + present = false; + bool failed = false; +#if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM) + _contact_recovery &= ~ContactRecoveryBackup; + const char* paths[] = {"/contacts3", "/contacts3.bak"}; +#else + const char* paths[] = {"/contacts3"}; +#endif + for (const char* path : paths) { + for (uint8_t attempt = 0; attempt < 2; ++attempt) { + bool exists = false; + if (!companionPathPresence(fs, path, exists)) { failed = true; continue; } + if (!exists) break; + failed = true; + if (!readableContactFile(fs, path)) continue; + present = true; +#if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM) + if (strcmp(path, "/contacts3") != 0) { + _contact_recovery |= ContactRecoveryBackup; + // A valid backup remains usable even if its promotion cannot write. + if (prepareContactSave()) return "/contacts3"; + } +#endif + return path; + } + } + if (failed) { + queueContactRecovery(true); + return nullptr; + } + return "/contacts3"; +} + +bool DataStore::prepareContactSave() { + if (contactsAreVolatile()) return false; +#if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM) + if (!(_contact_recovery & ContactRecoveryBackup)) return true; +#if defined(ESP32_PLATFORM) && MESH_CONTACT_CACHE + _contact_path_reader.close(); +#endif + FILESYSTEM* fs = _getContactsChannelsFS(); + bool present = false, backup = false; + if (!companionPathPresence(fs, "/contacts3", present) + || !companionPathPresence(fs, "/contacts3.bak", backup)) return false; + // The previous restore can have moved the backup despite an unreadable + // acknowledgement. Never delete its sole public copy on a later retry. + if (!backup) { + if (!present || !readableContactFile(fs, "/contacts3")) return false; + _contact_recovery &= ~ContactRecoveryBackup; + return true; + } + if (!readableContactFile(fs, "/contacts3.bak")) return false; + if (present) { + fs->remove("/contacts3"); + if (!companionPathPresence(fs, "/contacts3", present) || present) return false; + } + mesh::ContactFileTransaction::recover(fs, "/contacts3", companionPathPresence); + backup = true; + if (!companionPathPresence(fs, "/contacts3", present) || !present + || !companionPathPresence(fs, "/contacts3.bak", backup) || backup) return false; + _contact_recovery &= ~ContactRecoveryBackup; + return true; +#else + return true; +#endif +} +#endif + +void DataStore::recoverContacts(DataStoreHost* host) { + if (!contactRecoveryPending() || !host->resetContactsForRecovery()) return; +#if defined(ESP32_PLATFORM) + cancelContactWrite(); + _contact_write_requested = false; +#if MESH_CONTACT_CACHE + _contact_path_reader.close(); +#endif +#endif + _contact_recovery = 0; + clearContactLoadFailure(); + loadContacts(host); // One fresh complete load before discarding anything. + if (!contactRecoveryPending()) return; // Includes non-destructive resource refusal. +#if !defined(NRF52_PLATFORM) + if (!host->resetContactsForRecovery()) return; +#endif + FILESYSTEM* fs = _getContactsChannelsFS(); + bool removed = true; + auto discard = [fs, &removed](const char* path) { + // Reconcile false acknowledgements without treating them as successful + // reset. Failed media must not turn this into a format/reboot loop. + bool present = true; +#if defined(NRF52_PLATFORM) + if (!contactPathPresence(fs, path, present)) { removed = false; return; } + if (!present) return; + fs->remove(path); + removed = contactPathPresence(fs, path, present) && !present && removed; +#else + if (!companionPathPresence(fs, path, present)) { removed = false; return; } + if (!present) return; + fs->remove(path); + removed = companionPathPresence(fs, path, present) && !present && removed; +#endif + }; +#if defined(NRF52_PLATFORM) + // Without a trusted migration marker, pages can be stale downgrade + // leftovers, not a valid tail of the unreadable legacy image. + const uint32_t discard_pages = (_contact_recovery & ContactRecoveryLegacy) + && !_legacy_migration_ready + ? (1UL << mesh::storage::CONTACT_PAGE_COUNT) - 1 + : _unread_contact_pages.bits(); + for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; ++page) { + if ((discard_pages & (1UL << page)) == 0) continue; + char path[24]; + makeContactPagePath(page, path); + discard(path); + char temp[28]; + snprintf(temp, sizeof(temp), "%s.tmp", path); + discard(temp); + } + if (_contact_recovery & ContactRecoveryLegacy) { + discard("/contacts3"); + discard("/contacts3.tmp"); + if (_contact_recovery & ContactMigrationDeferred) { + // Bad legacy plus unknown marker has no trusted alternate contact view. + // Clear this logical contact store, never the shared ExtraFS journal. + discard(CONTACT_MIGRATION_MARKER); + discard("/contacts4.mig.tmp"); + } + } +#else + for (const char* path : {"/contacts3", "/contacts3.bak", "/contacts3.tmp"}) discard(path); +#endif + _contact_recovery = removed ? 0 : ContactStorageUnavailable; + clearContactLoadFailure(); +#if defined(NRF52_PLATFORM) + // Failed deletion of one damaged page need not discard the healthy pages + // already loaded by the retry. Persistence remains disabled, not the node. + if (removed && !host->resetContactsForRecovery()) return; +#endif + if (removed) loadContacts(host); + if (contactRecoveryPending()) { + // A still-failing device is usable without persistent contacts. Never + // rewrite it from a partially loaded table or erase unrelated stores. + host->resetContactsForRecovery(); + _contact_recovery = ContactStorageUnavailable; +#if defined(NRF52_PLATFORM) + resetContactPageState(); +#endif + clearContactLoadFailure(); + } +} + void DataStore::loadContacts(DataStoreHost* host) { + if (_contact_recovery & ContactStorageUnavailable) return; #if defined(ESP32_PLATFORM) cancelContactWrite(); #endif @@ -2090,21 +2297,8 @@ void DataStore::loadContacts(DataStoreHost* host) { #endif #if !defined(NRF52_PLATFORM) bool contacts_exist = false; - bool contacts_metadata_ready = companionPathPresence( - _getContactsChannelsFS(), "/contacts3", contacts_exist); -#endif -#if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM) - bool backup_exists = false; - contacts_metadata_ready = contacts_metadata_ready && companionPathPresence( - _getContactsChannelsFS(), "/contacts3.bak", backup_exists); - const bool contacts_required = contacts_exist || backup_exists; - contacts_metadata_ready = contacts_metadata_ready - && mesh::ContactFileTransaction::recover(_getContactsChannelsFS(), "/contacts3") - && companionPathPresence(_getContactsChannelsFS(), "/contacts3", contacts_exist) - && (!contacts_required || contacts_exist); -#endif -#if !defined(NRF52_PLATFORM) - if (!contacts_metadata_ready) { + const char* contact_path = selectContactFile(contacts_exist); + if (contact_path == nullptr) { MESH_DEBUG_PRINTLN("DataStore: contact transaction recovery failed"); #if MESH_CONTACT_CACHE _cache_load_incomplete = true; @@ -2120,11 +2314,11 @@ void DataStore::loadContacts(DataStoreHost* host) { // contact table cannot collide with the reload. resetContactPageState(); if (_contact_load_incomplete) { - // A failed load is latched for this boot. loadContacts() is also called by - // identity import, but only a reboot may retry storage and re-enable - // migration/mutation; otherwise an unrelated command could clear the veto. + // A failed load stays latched for ordinary reloads, including identity + // import. Only explicit safe-boundary recovery or a reboot may retry it; + // an unrelated command must not clear the incomplete-view veto. MESH_DEBUG_PRINTLN( - "DataStore: contact load remains quarantined until reboot"); + "DataStore: contacts need safe recovery"); return; } FILESYSTEM* contacts_fs = _getContactsChannelsFS(); @@ -2175,11 +2369,15 @@ void DataStore::loadContacts(DataStoreHost* host) { MESH_DEBUG_PRINTLN( "DataStore: HIL injecting contact page %u stat failure", page); _contact_load_incomplete = true; + _unread_contact_pages.mark(page); + queueContactRecovery(); return; } #endif if (!contactPathPresence(contacts_fs, path, page_exists)) { _contact_load_incomplete = true; + _unread_contact_pages.mark(page); + queueContactRecovery(); return; } if (page_exists) { @@ -2188,12 +2386,19 @@ void DataStore::loadContacts(DataStoreHost* host) { } bool legacy_exists = false; bool migration_marker_exists = false; - if (!contactPathPresence(contacts_fs, "/contacts3", legacy_exists) - || !contactPathPresence(contacts_fs, CONTACT_MIGRATION_MARKER, - migration_marker_exists)) { + if (!contactPathPresence(contacts_fs, "/contacts3", legacy_exists)) { _contact_load_incomplete = true; + queueContactRecovery(true); return; } + if (!contactPathPresence(contacts_fs, CONTACT_MIGRATION_MARKER, + migration_marker_exists) + && !contactPathPresence(contacts_fs, CONTACT_MIGRATION_MARKER, + migration_marker_exists)) { + // An unreadable migration marker is not evidence that any contact is bad. + // Use the readable legacy view without migrating/rewriting healthy pages. + _contact_recovery |= ContactMigrationDeferred; + } const mesh::storage::ContactStoreSource source = mesh::storage::chooseContactStoreSource( legacy_exists, contact_page_presence != 0); @@ -2222,9 +2427,14 @@ void DataStore::loadContacts(DataStoreHost* host) { _legacy_migration_ready = migration_marker_exists; #endif +#if defined(NRF52_PLATFORM) File file = openRead(_getContactsChannelsFS(), "/contacts3"); +#else + File file = openRead(_getContactsChannelsFS(), contact_path); +#endif #if !defined(NRF52_PLATFORM) if (!file && contacts_exist) { + queueContactRecovery(true); #if MESH_CONTACT_CACHE _cache_load_incomplete = true; #else @@ -2240,6 +2450,7 @@ void DataStore::loadContacts(DataStoreHost* host) { // file, but this boot must not accept a mutation into a competing page. MESH_DEBUG_PRINTLN("DataStore: legacy contacts exist but could not be read"); _contact_load_incomplete = true; + queueContactRecovery(true); return; } @@ -2250,6 +2461,7 @@ void DataStore::loadContacts(DataStoreHost* host) { (unsigned long)legacy_size); file.close(); _contact_load_incomplete = true; + queueContactRecovery(true); return; } #endif @@ -2283,6 +2495,7 @@ void DataStore::loadContacts(DataStoreHost* host) { ) { uint8_t record[mesh::storage::CONTACT_RECORD_SIZE]; if (file.read(record, sizeof(record)) != sizeof(record)) { + queueContactRecovery(true); #if MESH_CONTACT_CACHE _cache_load_incomplete = true; #elif !defined(NRF52_PLATFORM) @@ -2357,7 +2570,7 @@ void DataStore::loadContacts(DataStoreHost* host) { // Only modify stale page artifacts after the complete authoritative legacy // file has been loaded. A transient read failure above therefore leaves // every possible recovery source untouched. - if (!mesh::storage::trustMigratedContactPages( + if (!contactsAreVolatile() && !mesh::storage::trustMigratedContactPages( true, _legacy_migration_ready)) { if (!prepareLegacyContactMigration()) { _contact_load_incomplete = true; @@ -2377,6 +2590,16 @@ void DataStore::loadContacts(DataStoreHost* host) { if (_legacy_migration_ready) { loadContactPages(host, _legacy_contact_count, contact_page_presence); } +#else + // Only a completely loaded public image makes its old alternatives expendable. + if (!hasIncompleteContactLoad() +#if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM) + && !(_contact_recovery & ContactRecoveryBackup) +#endif + ) { + for (const char* path : {"/contacts3.bak", "/contacts3.tmp"}) + if (_getContactsChannelsFS()->exists(path)) _getContactsChannelsFS()->remove(path); + } #endif } @@ -2385,6 +2608,10 @@ bool DataStore::saveContacts(DataStoreHost* host, bool (*filter)(const ContactIn cancelContactWrite(); #endif if (hasIncompleteContactLoad()) return false; + if (contactsAreVolatile()) return false; +#if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM) + if (!prepareContactSave()) return false; +#endif #if defined(NRF52_PLATFORM) bool success = true; for (uint32_t idx = 0;; idx++) { @@ -2436,6 +2663,18 @@ bool DataStore::saveContacts(DataStoreHost* host, bool (*filter)(const ContactIn _contact_path_reader.close(); #endif success = writer.commit(success); + const bool uncertain = writer.outcome() + == mesh::ContactFileTransaction::Outcome::Indeterminate; + if (uncertain) { + // Old RAM indices may no longer describe the public file. Preserve dirty + // work until the outer loop can safely reset/reload the contact table. + queueContactRecovery(true); +#if MESH_CONTACT_CACHE + _cache_load_incomplete = true; +#else + _uncached_contact_load_incomplete = true; +#endif + } #if MESH_CONTACT_CACHE if (success) { uint16_t index = 0; @@ -2446,7 +2685,7 @@ bool DataStore::saveContacts(DataStoreHost* host, bool (*filter)(const ContactIn paths.publish(c->path_ref.handle(), index++); } } - paths.endCommit(success); + paths.endCommit(success || uncertain); #endif _contact_write_requested = !success; return success; @@ -2488,11 +2727,20 @@ bool DataStore::saveContacts(DataStoreHost* host, bool (*filter)(const ContactIn } success = file.commit(success); } + if (file.outcome() == decltype(file)::Outcome::Indeterminate) { + queueContactRecovery(true); +#if MESH_CONTACT_CACHE + _cache_load_incomplete = true; +#else + _uncached_contact_load_incomplete = true; +#endif + } return success; #endif } bool DataStore::markContactDirty(const ContactInfo& contact) { + if (contactsAreVolatile()) return true; #if defined(NRF52_PLATFORM) if (hasIncompleteContactLoad()) return false; uint16_t slot = contact.storage_slot; @@ -2513,6 +2761,7 @@ bool DataStore::markContactDirty(const ContactInfo& contact) { } bool DataStore::releaseContact(const ContactInfo& contact) { + if (contactsAreVolatile()) return true; #if defined(NRF52_PLATFORM) if (hasIncompleteContactLoad()) return false; const uint16_t slot = contact.storage_slot; @@ -2595,11 +2844,23 @@ bool DataStore::truncateLegacyContacts(uint16_t remaining_contacts) { #if defined(ESP32_PLATFORM) void DataStore::cancelContactWrite() { if (_contact_write == nullptr) return; + const bool uncertain = _contact_write->outcome() + == mesh::ContactFileTransaction::Outcome::Indeterminate; + if (uncertain) { + queueContactRecovery(true); +#if MESH_CONTACT_CACHE + _cache_load_incomplete = true; +#else + _uncached_contact_load_incomplete = true; +#endif + } delete _contact_write; _contact_write = nullptr; #if MESH_CONTACT_CACHE _contact_path_reader.close(); - mesh::contactPathStorage().endCommit(false); + // Detach retained snapshots from names which may now contain another + // generation. Current clean handles are protected by the read/write hold. + mesh::contactPathStorage().endCommit(uncertain); #endif _contact_write_host = nullptr; _contact_write_filter = nullptr; @@ -2641,6 +2902,7 @@ bool DataStore::serviceContactWrite(DataStoreHost* host, auto& paths = mesh::contactPathStorage(); #endif if (_contact_write == nullptr) { + if (!prepareContactSave()) return false; #if MESH_CONTACT_CACHE paths.beginCommit(); for (uint32_t i = 0;; ++i) { @@ -2753,10 +3015,11 @@ bool DataStore::serviceContactWrite(DataStoreHost* host, bool DataStore::serviceContactWrites(DataStoreHost* host, bool (*filter)(const ContactInfo& c)) { + if (contactsAreVolatile()) return false; #if defined(NRF52_PLATFORM) if (hasIncompleteContactLoad()) { - // Reboot is the recovery operation. Never migrate or rewrite from a RAM - // table which is known not to represent every durable contact. + // Recovery must first rebuild a complete view at the safe host boundary. + // Resource and shared-authority failures never authorize a destructive reset. return false; } if (_legacy_contacts_pending_cleanup) { @@ -2816,7 +3079,7 @@ bool DataStore::flushContactWrites(DataStoreHost* host, #if defined(ESP32_PLATFORM) cancelContactWrite(); #endif - if (hasIncompleteContactLoad()) return false; + if (hasIncompleteContactLoad() || contactsAreVolatile()) return false; #if defined(NRF52_PLATFORM) while (hasPendingContactWrites()) { if (!serviceContactWrites(host, filter)) return false; @@ -2828,6 +3091,7 @@ bool DataStore::flushContactWrites(DataStoreHost* host, } bool DataStore::hasPendingContactWrites() const { + if (contactsAreVolatile()) return false; #if defined(NRF52_PLATFORM) return !hasIncompleteContactLoad() && (!_dirty_contact_pages.empty() || _legacy_contacts_pending_cleanup); @@ -2839,6 +3103,7 @@ bool DataStore::hasPendingContactWrites() const { } bool DataStore::hasIncompleteContactLoad() const { + if (contactRecoveryPending()) return true; #if !defined(NRF52_PLATFORM) if (_channel_load_incomplete) return true; #if !MESH_CONTACT_CACHE @@ -4307,6 +4572,9 @@ bool validSecretRecord(const uint8_t record[SECRET_RECORD_SIZE]) { bool DataStore::readStoredPath(uint16_t source, uint8_t path[64]) { if (source == mesh::ContactPathStorage::NONE || hasIncompleteContactLoad()) return false; char filename[24] = "/contacts3"; +#if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM) + if (_contact_recovery & ContactRecoveryBackup) strcpy(filename, "/contacts3.bak"); +#endif size_t offset = static_cast(source) * mesh::storage::CONTACT_RECORD_SIZE; #if defined(ESP32_PLATFORM) if (!(source & mesh::ContactPathStorage::PAGED)) { @@ -4318,7 +4586,10 @@ bool DataStore::readStoredPath(uint16_t source, uint8_t path[64]) { File& file = _contact_path_reader; const bool ok = file && file.size() >= offset + mesh::storage::CONTACT_RECORD_SIZE && file.seek(offset + 76) && file.read(path, 64) == 64; - if (!ok) file.close(); + if (!ok) { + file.close(); + queueContactRecovery(true); + } return ok; } #endif @@ -4337,6 +4608,16 @@ bool DataStore::readStoredPath(uint16_t source, uint8_t path[64]) { const bool ok = file && file.size() >= offset + mesh::storage::CONTACT_RECORD_SIZE && file.seek(offset + 76) && file.read(path, 64) == 64; if (file) file.close(); + if (!ok) { +#if defined(NRF52_PLATFORM) + if (source & mesh::ContactPathStorage::PAGED) + _unread_contact_pages.mark((source & ~mesh::ContactPathStorage::PAGED) + / mesh::storage::CONTACTS_PER_PAGE); + queueContactRecovery((source & mesh::ContactPathStorage::PAGED) == 0); +#else + queueContactRecovery(true); +#endif + } // The pool checks the CRC captured when this immutable path value was bound. return ok; } diff --git a/examples/companion_radio/DataStore.h b/examples/companion_radio/DataStore.h index 4cc231c93..849e9792f 100644 --- a/examples/companion_radio/DataStore.h +++ b/examples/companion_radio/DataStore.h @@ -22,6 +22,8 @@ public: virtual bool getContactForSave(uint32_t idx, ContactInfo& contact) =0; virtual ContactInfo* getContactForStore(uint32_t idx) =0; virtual void onContactCacheFlushed() {} + // Only called by explicit recovery between packet/app callbacks. + virtual bool resetContactsForRecovery() { return false; } virtual bool onChannelLoaded(uint8_t channel_idx, const ChannelDetails& ch) =0; virtual bool getChannelForSave(uint8_t channel_idx, ChannelDetails& ch) =0; }; @@ -44,6 +46,21 @@ class DataStore IdentityStore identity_store; bool _identity_creation_blocked = false; bool _prefs_load_incomplete = false; + // Contact-only recovery must not reinterpret a channel/migration error as + // permission to discard contacts. No recovery files or boot counters needed. + enum : uint8_t { ContactRecoveryPending = 1, ContactRecoveryLegacy = 2, + ContactRecoveryBackup = 4, ContactStorageUnavailable = 8, + ContactMigrationDeferred = 16 }; + uint8_t _contact_recovery = 0; + void queueContactRecovery(bool legacy = false) { + _contact_recovery |= ContactRecoveryPending + | (legacy ? ContactRecoveryLegacy : 0); + } + void clearContactLoadFailure(); +#if !defined(NRF52_PLATFORM) + const char* selectContactFile(bool& present); + bool prepareContactSave(); +#endif #if defined(ESP32_PLATFORM) const char* _prefs_recovery_source = nullptr; const char* _channel_recovery_source = nullptr; @@ -207,6 +224,13 @@ public: bool saveReaderBookmark(mesh::bible::Position pos); #endif void loadContacts(DataStoreHost* host); + bool contactRecoveryPending() const { + return (_contact_recovery & ContactRecoveryPending) != 0; + } + bool contactsAreVolatile() const { + return (_contact_recovery & (ContactStorageUnavailable | ContactMigrationDeferred)) != 0; + } + void recoverContacts(DataStoreHost* host); bool saveContacts(DataStoreHost* host, bool (*filter)(const ContactInfo& c) = NULL); bool markContactDirty(const ContactInfo& contact); bool releaseContact(const ContactInfo& contact); diff --git a/examples/companion_radio/MyMesh.cpp b/examples/companion_radio/MyMesh.cpp index 9229d8926..1578a92e4 100644 --- a/examples/companion_radio/MyMesh.cpp +++ b/examples/companion_radio/MyMesh.cpp @@ -3202,6 +3202,7 @@ void MyMesh::begin(bool has_display, bool radio_available, resetContacts(); _store->loadContacts(this); + _store->recoverContacts(this); if (_store->hasPendingContactWrites()) { dirty_contacts_expiry = mesh::nonzeroLazyPersistenceDeadline( futureMillis(LAZY_CONTACTS_WRITE_DELAY)); @@ -5158,7 +5159,7 @@ bool MyMesh::isWebConfigWiFiRecoveryActive() const { } void MyMesh::rebootNow() { - if (flushContactsBeforeReboot()) { + if (MyMesh::prepareForUiShutdown()) { board.reboot(); } else { MESH_DEBUG_PRINTLN("Reboot refused: pending contacts could not be saved"); @@ -6642,7 +6643,7 @@ void MyMesh::handleCmdFrame(size_t len) { else writeErrFrame(ERR_CODE_FILE_IO_ERROR); } } else if (cmd_frame[0] == CMD_REBOOT && len >= 7 && memcmp(&cmd_frame[1], "reboot", 6) == 0) { - if (!flushContactsBeforeReboot()) { + if (!MyMesh::prepareForUiShutdown()) { writeErrFrame(ERR_CODE_FILE_IO_ERROR); return; } @@ -6695,9 +6696,9 @@ void MyMesh::handleCmdFrame(size_t len) { // re-load contacts, to invalidate ecdh shared_secrets stopContactsIterator(); resetContacts(); - mesh::resetLazyPersistenceAfterSuccess( - dirty_contacts_expiry, dirty_contacts_failures); + MyMesh::onContactCacheFlushed(); _store->loadContacts(this); + _store->recoverContacts(this); if (_store->hasPendingContactWrites()) { mesh::scheduleLazyPersistenceMutation( dirty_contacts_expiry, dirty_contacts_failures, @@ -7517,13 +7518,25 @@ static bool save_filter(const ContactInfo& c) { return c.type != ADV_TYPE_NONE; // don't save the transient/anon entries } +__attribute__((noinline)) void MyMesh::onContactCacheFlushed() { - // Eviction already saved the pending mutations. Avoid writing them again - // when the old five-second lazy-save deadline arrives. + // Share the deadline reset after successful persistence or deliberate + // retirement of the old RAM table. Internal calls bypass virtual overrides. mesh::resetLazyPersistenceAfterSuccess(dirty_contacts_expiry, dirty_contacts_failures); } +bool MyMesh::resetContactsForRecovery() { + stopContactsIterator(); + resetContacts(); + MyMesh::onContactCacheFlushed(); + return true; +} + void MyMesh::saveContacts() { + if (_store->contactsAreVolatile()) { + MyMesh::onContactCacheFlushed(); + return; + } const bool success = _store->saveContacts(this, save_filter); if (!success) { scheduleContactWriteRetry(); @@ -7532,8 +7545,7 @@ void MyMesh::saveContacts() { dirty_contacts_expiry = mesh::nonzeroLazyPersistenceDeadline( futureMillis(CONTACT_PAGE_WRITE_GAP)); } else { - mesh::resetLazyPersistenceAfterSuccess( - dirty_contacts_expiry, dirty_contacts_failures); + MyMesh::onContactCacheFlushed(); } } @@ -7556,6 +7568,8 @@ bool MyMesh::isContactWriteDue() const { } bool MyMesh::flushContactsBeforeReboot() { + // This helper also backs explicit durability promises such as tx.contact. + if (_store->contactsAreVolatile()) return false; #if defined(ESP32_PLATFORM) const uint32_t now = _ms->getMillis(); if (_store->hasPendingAdvertWrites() && !_store->isAdvertWriteDue(now)) return false; @@ -7568,12 +7582,12 @@ bool MyMesh::flushContactsBeforeReboot() { scheduleContactWriteRetry(); return false; } - mesh::resetLazyPersistenceAfterSuccess( - dirty_contacts_expiry, dirty_contacts_failures); + MyMesh::onContactCacheFlushed(); return true; } bool MyMesh::prepareForOtaReboot() { + if (_store->contactsAreVolatile()) return true; // Once a write has failed, honor its bounded retry deadline instead of // letting the armed OTA apply hammer the same filesystem every loop. if (dirty_contacts_failures != 0 && dirty_contacts_expiry != 0 @@ -7583,8 +7597,9 @@ bool MyMesh::prepareForOtaReboot() { return flushContactsBeforeReboot(); } +__attribute__((noinline)) bool MyMesh::prepareForUiShutdown() { - return flushContactsBeforeReboot(); + return _store->contactsAreVolatile() || flushContactsBeforeReboot(); } bool MyMesh::scheduleContactWrite(const ContactInfo& contact) { @@ -7592,6 +7607,7 @@ bool MyMesh::scheduleContactWrite(const ContactInfo& contact) { // deliberately excluded from contact storage. Never allocate a persistent // slot for one when a path/message callback updates its runtime state. if (contact.type == ADV_TYPE_NONE || isTransientContact(contact)) return true; + if (_store->contactsAreVolatile()) return true; if (!_store->markContactDirty(contact)) { MESH_DEBUG_PRINTLN( "Contact persistence scheduling failed: no persistent slot"); @@ -7615,6 +7631,7 @@ bool MyMesh::scheduleContactWriteAfterRelease(const ContactInfo& contact, "Contact removal scheduling failed: persistent slot was not reserved"); return false; } + if (_store->contactsAreVolatile()) return true; mesh::scheduleLazyPersistenceMutation( dirty_contacts_expiry, dirty_contacts_failures, futureMillis(LAZY_CONTACTS_WRITE_DELAY)); @@ -7664,16 +7681,14 @@ void MyMesh::repairInternalExtraFS(Stream& output) { && (capacity_kib % 4) == 0; if (contacts_saved && channels_saved && capacity_ready && !_store->hasPendingContactWrites()) { - mesh::resetLazyPersistenceAfterSuccess( - dirty_contacts_expiry, dirty_contacts_failures); + MyMesh::onContactCacheFlushed(); output.printf(" > internal ExtraFS repaired (%lu KiB); contacts and channels rebuilt\n", (unsigned long)_store->getStorageTotalKb()); } else { if (!contacts_saved || _store->hasPendingContactWrites()) { scheduleContactWriteRetry(); } else { - mesh::resetLazyPersistenceAfterSuccess( - dirty_contacts_expiry, dirty_contacts_failures); + MyMesh::onContactCacheFlushed(); } output.println(" Error: ExtraFS is active, but data rebuild/verification failed; retry repair extrafs"); } @@ -10520,7 +10535,7 @@ bool MyMesh::handleDirectCommand(const char* command, char* reply, size_t reply_ if (_store->formatFileSystem()) { stopContactsIterator(); resetContacts(); - mesh::resetLazyPersistenceAfterSuccess(dirty_contacts_expiry, dirty_contacts_failures); + MyMesh::onContactCacheFlushed(); #if MESH_ENABLE_ETHERNET_TUNNEL if (!eraseEthernetSettings()) { snprintf(reply, reply_size, "File system erased; Error: Ethernet settings erase failed"); @@ -11069,8 +11084,7 @@ complete_line: contacts_saved && !_store->hasPendingContactWrites(); if (contacts_complete) { - mesh::resetLazyPersistenceAfterSuccess( - dirty_contacts_expiry, dirty_contacts_failures); + MyMesh::onContactCacheFlushed(); } else { scheduleContactWriteRetry(); } @@ -11241,7 +11255,7 @@ complete_line: } } else if (strcmp(cli_command, "reboot") == 0) { - if (flushContactsBeforeReboot()) { + if (MyMesh::prepareForUiShutdown()) { board.reboot(); // doesn't return } else { output.println(" Error: pending contacts could not be saved; reboot cancelled"); @@ -11333,6 +11347,16 @@ void MyMesh::checkSerialInterface() { } void MyMesh::loop() { + // Storage callbacks can request recovery while holding ContactInfo/path + // references. Retire those references only here, between callbacks. + if (_store->contactRecoveryPending() && !mesh::morseTimingActive(_radio)) { + _store->recoverContacts(this); + updateGpsTelemetryPolicy(); + if (_store->hasPendingContactWrites()) { + dirty_contacts_expiry = mesh::nonzeroLazyPersistenceDeadline( + futureMillis(LAZY_CONTACTS_WRITE_DELAY)); + } + } #if MESH_ENABLE_MORSE_RX if (_morse_reply.pending()) { const ContactInfo* actor = lookupContactByPubKey(_morse_reply.actor(), PUB_KEY_SIZE); @@ -11368,7 +11392,7 @@ void MyMesh::loop() { && (_scheduled_reboot_at == _ms->getMillis() || millisHasNowPassed(_scheduled_reboot_at))) { _scheduled_reboot_at = 0; - if (flushContactsBeforeReboot()) { + if (MyMesh::prepareForUiShutdown()) { board.reboot(); } else { MESH_DEBUG_PRINTLN("Scheduled reboot cancelled: pending contacts could not be saved"); @@ -11467,8 +11491,7 @@ void MyMesh::loop() { dirty_contacts_expiry = mesh::nonzeroLazyPersistenceDeadline( futureMillis(CONTACT_PAGE_WRITE_GAP)); } else { - mesh::resetLazyPersistenceAfterSuccess( - dirty_contacts_expiry, dirty_contacts_failures); + MyMesh::onContactCacheFlushed(); } } #endif @@ -11512,8 +11535,7 @@ void MyMesh::servicePersistence() { dirty_contacts_expiry = mesh::nonzeroLazyPersistenceDeadline( futureMillis(CONTACT_PAGE_WRITE_GAP)); } else { - mesh::resetLazyPersistenceAfterSuccess( - dirty_contacts_expiry, dirty_contacts_failures); + MyMesh::onContactCacheFlushed(); } } } @@ -11599,6 +11621,7 @@ bool MyMesh::hasPendingWork() const { #endif && (!radio_apply_retry_at || millisHasNowPassed(radio_apply_retry_at))) || contact_write_needs_polling + || _store->contactRecoveryPending() #if defined(ESP32_PLATFORM) || _store->hasPendingAdvertWrites() #endif diff --git a/examples/companion_radio/MyMesh.h b/examples/companion_radio/MyMesh.h index 7d5461460..c48eb00e1 100644 --- a/examples/companion_radio/MyMesh.h +++ b/examples/companion_radio/MyMesh.h @@ -543,6 +543,7 @@ protected: bool getContactForSave(uint32_t idx, ContactInfo& contact) override { return getContactByIdx(idx, contact); } ContactInfo* getContactForStore(uint32_t idx) override { return getContactPtrByIdx(idx); } void onContactCacheFlushed() override; + bool resetContactsForRecovery() override; // A legacy malformed alias must not quarantine every other saved channel. // The raw slot remains visible to the app so it can be renamed or deleted. bool onChannelLoaded(uint8_t channel_idx, const ChannelDetails& ch) override { return BaseChatMesh::setChannel(channel_idx, ch); } diff --git a/examples/companion_radio/PrefsStorageLayout.h b/examples/companion_radio/PrefsStorageLayout.h index 180c9197d..33d177c49 100644 --- a/examples/companion_radio/PrefsStorageLayout.h +++ b/examples/companion_radio/PrefsStorageLayout.h @@ -1,12 +1,12 @@ #pragma once -#if defined(STM32_PLATFORM) && defined(__GNUC__) +#if (defined(STM32_PLATFORM) || defined(NRF52_PLATFORM)) && defined(__GNUC__) #include "NodePrefs.h" #include #include #if defined(TBEAM_1W) || defined(RP2040_PLATFORM) -#error "The compact STM32 preference table needs an explicit board layout" +#error "The compact preference table needs an explicit board layout" #endif namespace mesh { @@ -17,7 +17,7 @@ enum : uint8_t { PrefsLat = 253, PrefsLon = 254, PrefsPad = 255 }; static constexpr size_t MANDATORY_FIELDS = 21; // GNU supports offsetof for this adapter-bearing, non-standard-layout class. -// Use it only on the STM32 GNU ABI, and verify every individual value field; +// Use it only on the supported ARM GNU ABIs, and verify every individual value field; // neither the class padding nor either runtime adapter is a wire image. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Winvalid-offsetof" @@ -123,7 +123,7 @@ constexpr size_t wireSize(size_t count) { return count == 0 ? 0 : PREF_FIELDS[count - 1].length + wireSize(count - 1); } static_assert(wireSize(MANDATORY_FIELDS) == 84, "published mandatory image size"); -static_assert(wireSize(FIELD_COUNT) == 236, "published STM32 preference image size"); +static_assert(wireSize(FIELD_COUNT) == 236, "published preference prefix size"); static const void* prefValue(const PrefField& field, const CompanionNodePrefs& values, diff --git a/src/helpers/ContactFileTransaction.h b/src/helpers/ContactFileTransaction.h index 275a250ac..60bb3279e 100644 --- a/src/helpers/ContactFileTransaction.h +++ b/src/helpers/ContactFileTransaction.h @@ -1,8 +1,8 @@ #pragma once // SPIFFS cannot replace an existing name in one rename. Preserve the previous -// file as .bak until the verified replacement has its final name. Recovery of -// the rename gap happens before loading contacts on the next boot. +// file as .bak until the verified replacement has its final name. Recover the +// rename gap before loading a store, including an explicit runtime reload. #if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM) #include "IdentityStore.h" #include "PersistentStoreFormat.h" @@ -12,6 +12,11 @@ namespace mesh { class ContactFileTransaction { public: using PresenceProbe = bool (*)(FILESYSTEM*, const char*, bool&); + enum class CommitProgress : uint8_t { Pending, Succeeded, Failed }; + // Committed combines staged readback with an observed namespace move, not + // fresh public-byte readback. PresenceProbe must report metadata errors; + // the bare exists() fallback cannot distinguish those errors from absence. + enum class Outcome : uint8_t { Pending, NotCommitted, Committed, Indeterminate }; private: FILESYSTEM* _fs; const char* _target; @@ -22,7 +27,7 @@ private: size_t _size = 0; uint32_t _crc = 0xffffffff; bool _ok = false; - bool _finished = false; + Outcome _outcome = Outcome::Pending; enum class CommitStage : uint8_t { Flush, Open, Verify, Publish }; CommitStage _commit_stage = CommitStage::Flush; uint32_t _verify_crc = 0xffffffff; @@ -38,7 +43,9 @@ private: static_assert(storage::CONTACT_RECORD_SIZE <= ESP_WRITE_BUFFER_SIZE, "Contact records must fit one bounded writer buffer"); enum class BeginStage : uint8_t { - Target, Backup, Recover, RetireBackup, Temp, Remove, Open, Configure, Ready, Failed + Target, Backup, Recover, RecoverTarget, RecoverBackup, + RecoverTargetRetire, RecoverBackupRetire, RetireBackup, + Temp, Remove, Open, Configure, Ready, Failed }; BeginStage _begin_stage = BeginStage::Target; bool _begin_target_exists = false; @@ -65,13 +72,81 @@ private: present = fs->exists(path); return true; } - static bool removeIfPresent(FILESYSTEM* fs, const char* path, - PresenceProbe presence) { + // Do not add another frame to the bounded publication/cleanup call chain. + __attribute__((always_inline)) static bool removeIfPresent( + FILESYSTEM* fs, const char* path, PresenceProbe presence) { bool present = false; return probe(fs, path, present, presence) && (!present || fs->remove(path)); } + enum class MoveState : uint8_t { Moved, Unchanged, Unknown }; + __attribute__((always_inline)) static MoveState moveState( + FILESYSTEM* fs, const char* from, const char* to, PresenceProbe presence) { + bool source = false, destination = false; + if (!probe(fs, from, source, presence) + || !probe(fs, to, destination, presence)) return MoveState::Unknown; + if (!source && destination) return MoveState::Moved; + if (source && !destination) return MoveState::Unchanged; + return MoveState::Unknown; + } + __attribute__((noinline)) CommitProgress finishCommit(bool valid, + bool defer_backup_cleanup) { + _outcome = Outcome::NotCommitted; + bool backup_exists = false, target_exists = false; + bool ok = valid && probe(_fs, _backup, backup_exists, _presence) + && probe(_fs, _target, target_exists, _presence); + if (ok && backup_exists && !target_exists) { + // A prior public file is no longer where construction found it. Never + // retire the only recoverable copy or replace unknown authority. + _outcome = Outcome::Indeterminate; + return CommitProgress::Failed; + } + if (ok && backup_exists) { + ok = _fs->remove(_backup) + && probe(_fs, _backup, backup_exists, _presence) && !backup_exists; + } + bool backed_up = false; + if (ok && target_exists) { + _fs->rename(_target, _backup); + const MoveState moved = moveState(_fs, _target, _backup, _presence); + if (moved == MoveState::Unknown) { + _outcome = Outcome::Indeterminate; + return CommitProgress::Failed; + } + ok = backed_up = moved == MoveState::Moved; + } + if (ok) { + _fs->rename(_temp, _target); + const MoveState moved = moveState(_fs, _temp, _target, _presence); + if (moved == MoveState::Moved) { + _outcome = Outcome::Committed; +#if defined(ESP32_PLATFORM) + if (!defer_backup_cleanup) +#else + (void)defer_backup_cleanup; +#endif + removeIfPresent(_fs, _backup, _presence); + return CommitProgress::Succeeded; + } + // An unknown publication may already have installed the candidate. + // In particular, Pico rename can replace a live destination: never + // blindly roll an old backup over a possibly committed public file. + if (moved == MoveState::Unknown) { + _outcome = Outcome::Indeterminate; + return CommitProgress::Failed; + } + if (backed_up) { + _fs->rename(_backup, _target); + if (moveState(_fs, _backup, _target, _presence) != MoveState::Moved) { + _outcome = Outcome::Indeterminate; + return CommitProgress::Failed; + } + } + } + removeIfPresent(_fs, _temp, _presence); + return CommitProgress::Failed; + } public: - enum class CommitProgress : uint8_t { Pending, Succeeded, Failed }; + Outcome outcome() const { return _outcome; } static bool recover(FILESYSTEM* fs, const char* target, PresenceProbe presence = nullptr) { char backup[48]; @@ -80,7 +155,9 @@ public: if (!probe(fs, target, target_exists, presence) || !probe(fs, backup, backup_exists, presence)) return false; if (target_exists) return true; - return !backup_exists || fs->rename(backup, target); + if (!backup_exists) return true; + fs->rename(backup, target); + return moveState(fs, backup, target, presence) == MoveState::Moved; } ContactFileTransaction(FILESYSTEM* fs, const char* target, PresenceProbe presence = nullptr @@ -98,7 +175,12 @@ public: while (serviceBegin() == BeginProgress::Pending) {} } #else - if (!recover(fs, target, presence)) return; + if (!recover(fs, target, presence)) { + // Recovery may already have moved the old backup. We do not own any + // prior scratch and must not remove it when that result is unknown. + _outcome = Outcome::Indeterminate; + return; + } if (!removeIfPresent(fs, _temp, presence)) return; #if defined(RP2040_PLATFORM) _file = fs->open(_temp, "w"); @@ -111,7 +193,7 @@ public: ~ContactFileTransaction() { if (_file) _file.close(); if (_verify) _verify.close(); - if (!_finished + if (_outcome == Outcome::Pending #if defined(ESP32_PLATFORM) && _owns_temp #endif @@ -133,15 +215,32 @@ public: break; case BeginStage::Recover: if (!_begin_target_exists && _begin_backup_exists) { - ok = _fs->rename(_backup, _target); - if (ok) { - _begin_target_exists = true; - _begin_backup_exists = false; - } + _fs->rename(_backup, _target); + _begin_stage = background_contact_write + ? BeginStage::RecoverTargetRetire : BeginStage::RecoverTarget; + break; } if (ok) _begin_stage = background_contact_write ? BeginStage::RetireBackup : BeginStage::Temp; break; + case BeginStage::RecoverTarget: + case BeginStage::RecoverTargetRetire: { + const bool retire = _begin_stage == BeginStage::RecoverTargetRetire; + ok = probe(_fs, _target, _begin_target_exists, _presence); + if (!ok) _outcome = Outcome::Indeterminate; + if (ok) _begin_stage = retire + ? BeginStage::RecoverBackupRetire : BeginStage::RecoverBackup; + break; + } + case BeginStage::RecoverBackup: + case BeginStage::RecoverBackupRetire: { + const bool retire = _begin_stage == BeginStage::RecoverBackupRetire; + ok = probe(_fs, _backup, _begin_backup_exists, _presence) + && _begin_target_exists && !_begin_backup_exists; + if (!ok) _outcome = Outcome::Indeterminate; + if (ok) _begin_stage = retire ? BeginStage::RetireBackup : BeginStage::Temp; + break; + } case BeginStage::RetireBackup: // A successful prior publication may retain its old backup. Retire it // in a separate setup pass only when the original target was present. @@ -192,10 +291,10 @@ public: } #endif bool readyToPublish() const { - return !_finished && _commit_stage == CommitStage::Publish; + return _outcome == Outcome::Pending && _commit_stage == CommitStage::Publish; } size_t write(const uint8_t* data, size_t len) { - if (!_ok || _finished || _commit_stage != CommitStage::Flush) return 0; + if (!_ok || _outcome != Outcome::Pending || _commit_stage != CommitStage::Flush) return 0; const size_t wrote = _file.write(data, len); _ok = wrote == len; _size += wrote; @@ -213,7 +312,7 @@ public: // the deferred setup operations inside this call. if (_begin_stage != BeginStage::Ready) return CommitProgress::Failed; #endif - if (_finished) return CommitProgress::Failed; + if (_outcome != Outcome::Pending) return CommitProgress::Failed; bool ok = _ok && valid; if (ok && _commit_stage == CommitStage::Flush) { if (_file) { _file.flush(); _file.close(); } @@ -272,31 +371,7 @@ public: // not run between removing the target name and installing its replacement. if (_file) _file.close(); if (_verify) _verify.close(); - bool backup_exists = false, target_exists = false; - if (ok) ok = probe(_fs, _backup, backup_exists, _presence) - && probe(_fs, _target, target_exists, _presence); - if (ok && backup_exists) ok = _fs->remove(_backup); - bool backed_up = false; - if (ok && target_exists) { - ok = _fs->rename(_target, _backup); - backed_up = ok; - } - if (ok) ok = _fs->rename(_temp, _target); - if (!ok && backed_up) _fs->rename(_backup, _target); - // Cleanup remains best effort after the publish/rollback result, but an - // absent file must not trigger the SDK's error log on the shared USB port. - if (ok) { -#if defined(ESP32_PLATFORM) - // Background contacts publish the target and its path handles now. - // A later transaction retires this recoverable old backup separately. - if (!defer_backup_cleanup) -#else - (void)defer_backup_cleanup; // Other platforms retain immediate cleanup. -#endif - removeIfPresent(_fs, _backup, _presence); - } else removeIfPresent(_fs, _temp, _presence); - _finished = true; - return ok ? CommitProgress::Succeeded : CommitProgress::Failed; + return finishCommit(ok, defer_backup_cleanup); } bool commit(bool valid = true) { CommitProgress progress; diff --git a/src/helpers/FleetChannel.cpp b/src/helpers/FleetChannel.cpp index 47a8b0130..2cdc98224 100644 --- a/src/helpers/FleetChannel.cpp +++ b/src/helpers/FleetChannel.cpp @@ -64,16 +64,15 @@ bool FleetChannel::save(bool enabled, const uint8_t* key, const uint8_t* control return saved; #else ContactFileTransaction writer(fs_, Path, filePresence); - if (!writer || writer.write(image, sizeof(image)) != sizeof(image)) return false; - ContactFileTransaction::CommitProgress progress; - bool publishing; - do { - publishing = writer.readyToPublish(); - progress = writer.serviceCommit(); - } while (progress == ContactFileTransaction::CommitProgress::Pending); - if (!publishing) return false; // No publication attempt; local retry is safe. - // The fallback can report a late rename error or false success. Its staged - // fingerprint does not establish which bytes have the final public name. + if (!writer || writer.write(image, sizeof(image)) != sizeof(image)) { + if (writer.outcome() == ContactFileTransaction::Outcome::Indeterminate) + healthy_ = false; + return false; + } + writer.commit(); + if (writer.outcome() == ContactFileTransaction::Outcome::NotCommitted) return false; + // Namespace checks establish the move of the prevalidated stage, but do not + // establish a fresh public-byte readback. Reconcile uncertain moves too. // Confirm exact public bytes before accepting new authority or using old RAM. auto file = openFileRead(fs_, Path); uint8_t verified[ImageSize]; diff --git a/test/fixtures/advert_cache/test.cpp b/test/fixtures/advert_cache/test.cpp index 301ffc970..415401897 100644 --- a/test/fixtures/advert_cache/test.cpp +++ b/test/fixtures/advert_cache/test.cpp @@ -26,6 +26,7 @@ public: bool contact_pending=false; unsigned contact_passes=0; #include "advert_state.h" +#include "contact_recovery.h" explicit DataStore(FakeFilesystem& fs):_fs(&fs) {} File openRead(FakeFilesystem* fs,const char* path){return fs->open(path);} void cancelContactWrite(){} @@ -81,6 +82,7 @@ public: DelayedReplies _delayed_replies; bool putBlobByKey(const uint8_t[],int,const uint8_t[],int); void servicePersistence(); + void onContactCacheFlushed(); bool flushContactsBeforeReboot(); bool hasPendingWork() const; bool canRecoverUsbLogging() const; @@ -265,9 +267,12 @@ void failuresAndLifecycle() { enqueue(store,k,b);until(store,fs,Stage::Write);assert(fs.handles==1); store.begin();assert(fs.handles==0 && !store.hasPendingAdvertWrites()); enqueue(store,k,b);until(store,fs,Stage::Write);fs.fail_format=true; + store._contact_recovery=DataStore::ContactStorageUnavailable; assert(!store.formatFileSystem());assert(fs.handles==0 && get(store,k)==b); + assert(store.contactsAreVolatile()); fs.fail_format=false;drain(store,fs);enqueue(store,k,blob(2)); assert(store.formatFileSystem());assert(!store.hasPendingAdvertWrites() && get(store,k).empty()); + assert(!store.contactsAreVolatile() && !store.contactRecoveryPending()); // Reboot and identity-import paths use the real MyMesh durability gate. Clock clock;MyMesh mesh{&store,&clock};enqueue(store,k,b); assert(mesh.flushContactsBeforeReboot());assert(get(store,k)==b && fs.handles==0); @@ -293,6 +298,9 @@ void pendingWorkAndRecovery() { FakeFilesystem fs;DataStore store(fs);Clock clock;MyMesh mesh{&store,&clock}; const auto k=key(1);const auto b=blob(1); assert(!mesh.hasPendingWork() && mesh.canRecoverUsbLogging()); + store._contact_recovery=DataStore::ContactRecoveryPending; + assert(mesh.hasPendingWork()); + store._contact_recovery=0; enqueue(store,k,b); assert(mesh.hasPendingWork() && !mesh.canRecoverUsbLogging()); fs.fail_open=true; diff --git a/test/fixtures/companion_uncached_storage/test.cpp b/test/fixtures/companion_uncached_storage/test.cpp index 5c3e2de58..9d525e4fd 100644 --- a/test/fixtures/companion_uncached_storage/test.cpp +++ b/test/fixtures/companion_uncached_storage/test.cpp @@ -21,7 +21,7 @@ public: explicit File(FakeFilesystem& filesystem) : fs(&filesystem) {} File(FakeFilesystem* filesystem, const char* name, bool write) : fs(filesystem), path(name), writing(write) {} - explicit operator bool() const { return fs != nullptr; } + explicit operator bool() const { return fs != nullptr && !path.empty(); } bool isDirectory() const { return false; } bool open(const char* name, uint8_t mode); size_t read(uint8_t* bytes, size_t length); @@ -42,6 +42,7 @@ public: std::string fail_open; unsigned fail_open_remaining = std::numeric_limits::max(); int stat_error = 0; + bool fail_remove = false; unsigned fail_rename = 0, renames = 0, writes = 0; unsigned read_opens = 0; std::vector snapshots; @@ -52,7 +53,7 @@ public: return files.count(path) != 0; } bool mkdir(const char*) { return true; } - bool remove(const char* path) { return files.erase(path) != 0; } + bool remove(const char* path) { return !fail_remove && files.erase(path) != 0; } void _lockFS() {} void _unlockFS() {} FakeFilesystem* _getFS() { return this; } @@ -106,7 +107,7 @@ size_t File::write(const uint8_t* bytes, size_t length) { fs->snapshots.push_back(fs->files); return count; } -size_t File::size() const { return fs ? fs->files.at(path).size() : 0; } +size_t File::size() const { return *this ? fs->files.at(path).size() : 0; } #if defined(RP2040_PLATFORM) #include @@ -117,6 +118,7 @@ size_t File::size() const { return fs ? fs->files.at(path).size() : 0; } #include #include #include +#include #define ATOMIC_FILE_WRITER_IMPLEMENTATION #include @@ -142,7 +144,9 @@ static const char* identity_path = "/identity/_main.id"; struct Host { size_t capacity = 100; + unsigned resets = 0; std::vector contacts; + bool resetContactsForRecovery() { ++resets; contacts.clear(); return true; } bool getContactForSave(uint32_t index, ContactInfo& c) { if (index >= contacts.size()) return false; c = contacts[index]; @@ -161,6 +165,7 @@ class DataStore { bool _identity_creation_blocked = false; bool _channel_load_incomplete = false; bool _uncached_contact_load_incomplete = false; +#include "contact_recovery_state_under_test.h" #if defined(ESP32_PLATFORM) #include "contact_write_state_under_test.h" #endif @@ -187,6 +192,8 @@ class DataStore { } } identity_store; public: +#include "contact_recovery_api_under_test.h" + void recoverContacts(DataStoreHost*); #if defined(ESP32_PLATFORM) ~DataStore(); bool markContactDirty(const ContactInfo&); @@ -343,7 +350,72 @@ static void contact_checks() { assert(filesystem.files.at("/contacts3").empty()); } +static void recovery_checks() { + const auto original = make_original(); + for (unsigned fault = 0; fault < 3; ++fault) { + filesystem = FakeFilesystem(); + filesystem.files["/contacts3"] = original; + filesystem.files["/contacts3"].pop_back(); + filesystem.files[identity_path] = std::vector(96, 7); + filesystem.files["/channels2"] = {3, 4, 5}; + filesystem.files["/prefs"] = {6, 7}; + const auto identity = filesystem.files.at(identity_path); + const auto channels = filesystem.files.at("/channels2"); + const auto prefs = filesystem.files.at("/prefs"); + if (fault == 1) filesystem.fail_remove = true; + DataStore store; + Host host; + store.loadContacts(&host); + assert(store.hasIncompleteContactLoad() && store.contactRecoveryPending()); + if (fault == 2) filesystem.files["/contacts3"] = original; + store.recoverContacts(&host); + assert(!store.hasIncompleteContactLoad() && !store.contactRecoveryPending()); + assert(host.resets >= 1 && store.contactsAreVolatile() == (fault == 1)); + assert(host.contacts.size() == (fault == 2 ? 2U : 0U)); + assert(filesystem.files.at(identity_path) == identity); + assert(filesystem.files.at("/channels2") == channels); + assert(filesystem.files.at("/prefs") == prefs); + for (const auto& item : filesystem.files) assert(item.first.find(".bad") == std::string::npos); + host.contacts.push_back(contact(5)); + const auto before = filesystem.files; + const auto writes = filesystem.writes; + assert(store.saveContacts(&host) == (fault != 1)); + if (fault == 1) assert(filesystem.files == before && filesystem.writes == writes); + } + { + filesystem = FakeFilesystem(); + filesystem.files["/contacts3"] = original; + DataStore store; + Host host; + host.capacity = 1; + store.loadContacts(&host); + assert(store.hasIncompleteContactLoad() && !store.contactRecoveryPending()); + store.recoverContacts(&host); + assert(host.resets == 0 && host.contacts.size() == 1); + assert(!store.contactsAreVolatile() && filesystem.files.at("/contacts3") == original); + } +#if !defined(STM32_PLATFORM) + for (bool blocked_promotion : {false, true}) { + filesystem = FakeFilesystem(); + filesystem.files["/contacts3"] = {0xff}; + filesystem.files["/contacts3.bak"] = original; + if (blocked_promotion) filesystem.fail_rename = 1; + DataStore store; + Host host; + store.loadContacts(&host); + assert(!store.hasIncompleteContactLoad() && host.contacts.size() == 2); + if (blocked_promotion) assert(filesystem.files.at("/contacts3.bak") == original); + else assert(filesystem.files.at("/contacts3") == original); + host.contacts.push_back(contact(6)); + assert(store.saveContacts(&host)); + assert(!filesystem.files.count("/contacts3.bak")); + assert(filesystem.files.at("/contacts3").size() == 3 * mesh::storage::CONTACT_RECORD_SIZE); + } +#endif +} + int main() { identity_checks(); contact_checks(); + recovery_checks(); } diff --git a/test/fixtures/contact_cache/test_contact_cache.cpp b/test/fixtures/contact_cache/test_contact_cache.cpp index be163fa26..9ffd9b80a 100644 --- a/test/fixtures/contact_cache/test_contact_cache.cpp +++ b/test/fixtures/contact_cache/test_contact_cache.cpp @@ -4,15 +4,26 @@ #include #include #include +#if !defined(NRF52_PLATFORM) +#include +#endif #if defined(NRF52_PLATFORM) #define ATOMIC_FILE_WRITER_IMPLEMENTATION #include #endif #include "packet_under_test.h" #define ADV_TYPE_NONE 0 +#if defined(NRF52_PLATFORM) +static bool fail_contact_page_allocation = false; +static void* allocateContactPagePayload(size_t size) { + return fail_contact_page_allocation ? nullptr : malloc(size); +} +#endif struct Host { unsigned cache_flushes = 0; + unsigned recovery_resets = 0; + bool recovery_reset_allowed = true; size_t capacity = 350; std::vector contacts; bool onContactLoaded(const ContactInfo& c) { @@ -21,6 +32,12 @@ struct Host { return true; } void onContactCacheFlushed() { ++cache_flushes; } + bool resetContactsForRecovery() { + if (!recovery_reset_allowed) return false; + ++recovery_resets; + contacts.clear(); + return true; + } ContactInfo* getContactForStore(uint32_t index) { return index < contacts.size() ? &contacts[index] : nullptr; } @@ -29,12 +46,15 @@ using DataStoreHost = Host; #if defined(NRF52_PLATFORM) static const char* CONTACT_MIGRATION_MARKER = "/contacts4.mig"; +static bool fail_contact_migration_marker = false; bool contactPathPresence(FakeFilesystem* fs, const char* path, bool& present) { + if (fail_contact_migration_marker && !strcmp(path, CONTACT_MIGRATION_MARKER)) return false; present = fs->exists(path); return true; } #else bool companionPathPresence(FakeFilesystem* fs, const char* path, bool& present) { + if (fs->metadata_error) return false; present = fs->exists(path); return true; } @@ -46,11 +66,13 @@ bool companionPathPresence(FakeFilesystem* fs, const char* path, bool& present) class DataStore : public mesh::ContactPathBackend, public mesh::ContactSecretBackend { DataStoreHost* _cache_host; bool _cache_load_incomplete = false; +#include "contact_recovery_state_under_test.h" #if defined(ESP32_PLATFORM) // Advert queue lifecycle has a dedicated production fixture. The contact // reinitialization test needs only this unrelated hardware lifecycle seam. struct { void clear() {} } _advert_write; File _contact_path_reader; + bool _channel_load_incomplete = false; #include "contact_write_state_under_test.h" FakeFilesystem* _fs = &SPIFFS; FakeFilesystem* _fsExtra = nullptr; @@ -64,7 +86,7 @@ class DataStore : public mesh::ContactPathBackend, public mesh::ContactSecretBac uint16_t _legacy_contact_count = 0; mesh::storage::ContactSlotMap _contact_slots; mesh::storage::DirtyPageSet _dirty_contact_pages, _unread_contact_pages; - void resetContactPageState() { _contact_slots.clear(); } + void resetContactPageState(bool clear_incomplete = false); bool prepareLegacyContactMigration() { _legacy_migration_ready = true; return true; } #endif public: @@ -80,14 +102,10 @@ public: #endif } File openRead(FakeFilesystem* fs, const char* path) { return fs->open(path); } - bool hasIncompleteContactLoad() const { - return _cache_load_incomplete -#if defined(NRF52_PLATFORM) - || _contact_load_incomplete || !_unread_contact_pages.empty() -#endif - ; - } + bool hasIncompleteContactLoad() const; void loadContacts(DataStoreHost*); + void recoverContacts(DataStoreHost*); +#include "contact_recovery_api_under_test.h" bool saveContacts(DataStoreHost*, bool (*filter)(const ContactInfo&)); bool flushContactWrites(DataStoreHost*, bool (*filter)(const ContactInfo&)); bool readStoredPath(uint16_t, uint8_t[64]) override; @@ -328,6 +346,9 @@ static void storage_failures_preserve_routes() { assert(!f.store.saveContacts(&f.host, cachedContactFilter)); assert(SPIFFS.files.at("/contacts3") == original); SPIFFS.fail_read.clear(); + assert(f.store.contactRecoveryPending()); + f.store.recoverContacts(&f.host); + assert(!f.store.contactRecoveryPending() && !f.store.hasIncompleteContactLoad()); SPIFFS.files["/contacts3"][100 * 152 + 76] ^= 1; assert(!f.host.contacts[100].getPath()); // CRC protects a cold route after boot assert(!f.store.saveContacts(&f.host, cachedContactFilter)); @@ -399,11 +420,16 @@ static void failed_path_reads_fail_sends_without_leaking_packets() { Fixture f(30); mesh::contactSecretCache().attach(&f.store); BaseChatMesh mesh; - const auto& recipient = f.host.contacts[29]; uint8_t request[] = {1}; uint32_t tag, timeout; for (bool fail : {true, false}) { SPIFFS.fail_read = fail ? "/contacts3" : ""; + if (!fail) { + assert(f.store.contactRecoveryPending() && f.host.contacts.size() == 30); + f.store.recoverContacts(&f.host); // Safe boundary: no recipient reference is held. + assert(!f.store.contactRecoveryPending() && f.host.contacts.size() == 30); + } + const auto& recipient = f.host.contacts[29]; const int expected = fail ? MSG_SEND_FAILED : MSG_SEND_SENT_DIRECT; assert(mesh.sendLogin(recipient, "test", timeout) == expected); assert(mesh.sendAnonReq(recipient, request, 1, tag, timeout) == expected); @@ -539,6 +565,8 @@ static void contact_reload_preserves_disk_on_cache_exhaustion() { f.store.loadContacts(&f.host); assert(f.store.hasIncompleteContactLoad()); assert(f.host.contacts.empty()); + assert(!f.store.contactRecoveryPending()); // Handle pressure is not corrupt data. + f.store.recoverContacts(&f.host); assert(!f.store.saveContacts(&f.host, cachedContactFilter)); assert(SPIFFS.files == durable && SPIFFS.writes == writes); pending.clear(); @@ -550,6 +578,55 @@ static void contact_reload_preserves_disk_on_cache_exhaustion() { for (unsigned i = 0; i < 350; ++i) f.check(i, route(i)); } +static void contact_recovery_retries_and_discards_only_unread_contacts() { + for (unsigned fault = 0; fault < 3; ++fault) { + const bool persistent_failure = fault != 0; + const bool removal_failure = fault == 2; + Fixture f(30); +#if defined(NRF52_PLATFORM) + assert(f.store.writeContactPage(&f.host, 0, cachedContactFilter)); + assert(f.store.writeContactPage(&f.host, 1, cachedContactFilter)); + SPIFFS.remove("/contacts3"); + const char* failed_path = "/contacts4_00"; +#else + const char* failed_path = "/contacts3"; +#endif + for (const char* name : {"/_main.id", "/new_prefs", "/channels2", "/fleet_channel", + "/mesh_mt", "/mesh_eth"}) SPIFFS.files[name] = {7, 3, 9}; + const auto durable = SPIFFS.files; + f.host.contacts.clear(); + SPIFFS.fail_read = failed_path; + f.store.loadContacts(&f.host); + assert(f.store.hasIncompleteContactLoad() && f.store.contactRecoveryPending()); + assert(f.host.recovery_resets == 0); // The load callback never resets its host. + f.host.recovery_reset_allowed = false; + f.store.recoverContacts(&f.host); + assert(f.store.contactRecoveryPending() && SPIFFS.files == durable); + assert(f.host.recovery_resets == 0); // No discard without the safe host boundary. + f.host.recovery_reset_allowed = true; + if (!persistent_failure) SPIFFS.fail_read.clear(); + SPIFFS.fail_remove = removal_failure; + f.store.recoverContacts(&f.host); + assert(!f.store.contactRecoveryPending() && !f.store.hasIncompleteContactLoad()); + assert(f.store.contactsAreVolatile() == removal_failure); + assert(f.host.recovery_resets > 0); + auto expected = durable; + if (persistent_failure && !removal_failure) expected.erase(failed_path); + assert(SPIFFS.files == expected); // No archive or unrelated settings reset. +#if defined(NRF52_PLATFORM) + assert(f.host.contacts.size() == (persistent_failure ? 5u : 30u)); + for (unsigned i = 0; i < f.host.contacts.size(); ++i) + f.check(i, route(i + (persistent_failure ? 25u : 0u))); +#else + assert(f.host.contacts.size() == (persistent_failure ? 0u : 30u)); + for (unsigned i = 0; i < f.host.contacts.size(); ++i) f.check(i, route(i)); +#endif + SPIFFS.fail_read.clear(); SPIFFS.fail_remove = false; + assert(f.store.saveContacts(&f.host, cachedContactFilter) == !removal_failure); + if (removal_failure) assert(SPIFFS.files == expected); + } +} + #if defined(ESP32_PLATFORM) static unsigned drain_cooperative_save(Fixture& f) { unsigned serviced_commands = 0; @@ -786,17 +863,60 @@ static void lifecycle_cancels_jobs_and_closes_prior_filesystem_readers() { } static void partial_loads_cannot_replace_the_complete_file() { - for (bool capacity_failure : {false, true}) { + Fixture f(40); + f.host.contacts.clear(); + const auto durable = SPIFFS.files; + const unsigned removes = SPIFFS.removes, writes = SPIFFS.writes; + f.host.capacity = 20; + f.store.loadContacts(&f.host); + assert(f.store.hasIncompleteContactLoad()); + assert(!f.store.contactRecoveryPending()); // Host refusal is not a storage fault. + f.store.recoverContacts(&f.host); + assert(!f.store.saveContacts(&f.host, cachedContactFilter)); + assert(SPIFFS.files == durable && SPIFFS.removes == removes && SPIFFS.writes == writes); +} + +static void valid_backup_routes_survive_failed_promotion() { + for (bool removal_failure : {false, true}) { Fixture f(40); + const auto valid = SPIFFS.files.at("/contacts3"); f.host.contacts.clear(); - const auto durable = SPIFFS.files; - if (capacity_failure) f.host.capacity = 20; - else SPIFFS.fail_read = "/contacts3"; + SPIFFS.files["/contacts3"] = {9}; + SPIFFS.files["/contacts3.bak"] = valid; + SPIFFS.fail_remove = removal_failure; + if (!removal_failure) SPIFFS.fail_rename = SPIFFS.renames + 1; f.store.loadContacts(&f.host); - assert(f.store.hasIncompleteContactLoad()); + assert(!f.store.hasIncompleteContactLoad() && !f.store.contactRecoveryPending()); + assert(f.host.contacts.size() == 40 && f.host.recovery_resets == 0); + assert(SPIFFS.files.at("/contacts3.bak") == valid); + for (unsigned i = 0; i < 40; ++i) f.check(i, route(i)); + if (!removal_failure) SPIFFS.fail_rename = SPIFFS.renames + 1; assert(!f.store.saveContacts(&f.host, cachedContactFilter)); - assert(SPIFFS.files == durable); + assert(SPIFFS.files.at("/contacts3.bak") == valid); + assert(!f.store.contactRecoveryPending() && f.host.recovery_resets == 0); + SPIFFS.fail_remove = false; SPIFFS.fail_rename = 0; + assert(f.store.saveContacts(&f.host, cachedContactFilter)); + assert(SPIFFS.files.at("/contacts3") == valid); + for (unsigned i = 0; i < 40; ++i) f.check(i, route(i)); } + Fixture f(40); + const auto valid = SPIFFS.files.at("/contacts3"); + f.host.contacts.clear(); + SPIFFS.files["/contacts3"] = {9}; + SPIFFS.files["/contacts3.bak"] = valid; + SPIFFS.fail_rename = SPIFFS.renames + 1; + f.store.loadContacts(&f.host); + assert(f.host.contacts.size() == 40 && !f.store.hasIncompleteContactLoad()); + SPIFFS.fail_rename = 0; + SPIFFS.after_rename = [](FakeFilesystem& fs, const char* path) { + if (!strcmp(path, "/contacts3")) fs.metadata_error = true; + }; + assert(!f.store.saveContacts(&f.host, cachedContactFilter)); + assert(SPIFFS.files.at("/contacts3") == valid && !SPIFFS.exists("/contacts3.bak")); + SPIFFS.metadata_error = false; SPIFFS.after_rename = {}; + assert(f.store.saveContacts(&f.host, cachedContactFilter)); + assert(SPIFFS.files.at("/contacts3") == valid); + for (unsigned i = 0; i < 40; ++i) f.check(i, route(i)); } #endif @@ -926,6 +1046,78 @@ static void nrf52_migration_snapshots_and_failed_page_replacement() { assert(!f.store.writeContactPage(&f.host, 0, cachedContactFilter)); } +static void nrf52_resource_refusals_never_discard_contact_pages() { + for (bool allocation_failure : {false, true}) { + Fixture f(30); + assert(f.store.writeContactPage(&f.host, 0, cachedContactFilter)); + assert(f.store.writeContactPage(&f.host, 1, cachedContactFilter)); + SPIFFS.remove("/contacts3"); + const auto durable = SPIFFS.files; + const unsigned writes = SPIFFS.writes, removes = SPIFFS.removes; + f.host.contacts.clear(); + fail_contact_page_allocation = allocation_failure; + if (!allocation_failure) f.host.capacity = 20; + f.store.loadContacts(&f.host); + assert(f.store.hasIncompleteContactLoad() && !f.store.contactRecoveryPending()); + f.store.recoverContacts(&f.host); + assert(f.host.recovery_resets == 0); + assert(!f.store.saveContacts(&f.host, cachedContactFilter)); + assert(SPIFFS.files == durable && SPIFFS.writes == writes && SPIFFS.removes == removes); + fail_contact_page_allocation = false; + f.host.capacity = 350; + f.host.contacts.clear(); + DataStore rebooted(f.host); + rebooted.loadContacts(&f.host); + assert(!rebooted.hasIncompleteContactLoad() && f.host.contacts.size() == 30); + for (unsigned i = 0; i < 30; ++i) f.check(i, route(i)); + } +} + +static void nrf52_marker_error_keeps_a_usable_contact_view() { + Fixture f(30); + const auto durable = SPIFFS.files; + f.host.contacts.clear(); + fail_contact_migration_marker = true; + f.store.loadContacts(&f.host); + fail_contact_migration_marker = false; + assert(!f.store.hasIncompleteContactLoad() && !f.store.contactRecoveryPending()); + assert(f.store.contactsAreVolatile() && f.host.contacts.size() == 30); + for (unsigned i = 0; i < 30; ++i) f.check(i, route(i)); + const unsigned writes = SPIFFS.writes, removes = SPIFFS.removes; + f.store.recoverContacts(&f.host); + assert(f.host.recovery_resets == 0); + assert(!f.store.saveContacts(&f.host, cachedContactFilter)); + assert(SPIFFS.files == durable && SPIFFS.writes == writes && SPIFFS.removes == removes); +} + +static void nrf52_unread_legacy_and_unknown_marker_cannot_resurrect_stale_pages() { + Fixture f(30); + assert(f.store.writeContactPage(&f.host, 0, cachedContactFilter)); + assert(f.store.writeContactPage(&f.host, 1, cachedContactFilter)); + SPIFFS.files["/contacts3"] = {9}; + SPIFFS.files["/contacts4_00.tmp"] = {8}; + SPIFFS.files["/contacts4.mig"] = {}; + const FakeFilesystem::Files unrelated = { + {"/_main.id", {1, 3}}, {"/new_prefs", {2, 4}}, {"/channels2", {5, 7}}, + {"/.extrafs.mig", {'M', 'C', 'X', 'F', 1, 2}}}; + for (const auto& file : unrelated) SPIFFS.files[file.first] = file.second; + f.host.contacts.clear(); + fail_contact_migration_marker = true; + f.store.loadContacts(&f.host); + assert(f.store.contactRecoveryPending()); + f.store.recoverContacts(&f.host); + assert(!f.store.contactRecoveryPending() && !f.store.hasIncompleteContactLoad()); + assert(f.host.contacts.empty()); + assert(!SPIFFS.exists("/contacts3") && !SPIFFS.exists("/contacts4_00") + && !SPIFFS.exists("/contacts4_01") && !SPIFFS.exists("/contacts4_00.tmp")); + for (const auto& file : unrelated) assert(SPIFFS.files.at(file.first) == file.second); + fail_contact_migration_marker = false; + DataStore rebooted(f.host); + rebooted.loadContacts(&f.host); + assert(!rebooted.hasIncompleteContactLoad() && !rebooted.contactRecoveryPending()); + assert(f.host.contacts.empty() && SPIFFS.files == unrelated); +} + static void nrf52_uncertain_page_publication_preserves_snapshots() { for (unsigned fault = 0; fault < 3; ++fault) { Fixture f; @@ -1022,17 +1214,23 @@ int main() { encoded_paths_and_copy_assignment(); failed_path_reads_fail_sends_without_leaking_packets(); contact_reload_preserves_disk_on_cache_exhaustion(); + contact_recovery_retries_and_discards_only_unread_contacts(); partial_loads_cannot_replace_the_complete_file(); + valid_backup_routes_survive_failed_promotion(); secrets_load_from_flash_instead_of_recalculating(); persisted_secret_matches_the_real_peer_key_exchange(); secret_corruption_reordering_and_full_storage(); #else nrf52_secret_cache_leaves_contact_storage_reserve(); nrf52_migration_snapshots_and_failed_page_replacement(); + nrf52_resource_refusals_never_discard_contact_pages(); + nrf52_marker_error_keeps_a_usable_contact_view(); + nrf52_unread_legacy_and_unknown_marker_cannot_resurrect_stale_pages(); nrf52_uncertain_page_publication_preserves_snapshots(); encoded_paths_and_copy_assignment(); failed_path_reads_fail_sends_without_leaking_packets(); contact_reload_preserves_disk_on_cache_exhaustion(); + contact_recovery_retries_and_discards_only_unread_contacts(); secrets_load_from_flash_instead_of_recalculating(); persisted_secret_matches_the_real_peer_key_exchange(); #endif diff --git a/test/fixtures/contact_persistence/test_esp32_uncached.cpp b/test/fixtures/contact_persistence/test_esp32_uncached.cpp index 3c5a25fb6..469c24cc2 100644 --- a/test/fixtures/contact_persistence/test_esp32_uncached.cpp +++ b/test/fixtures/contact_persistence/test_esp32_uncached.cpp @@ -4,6 +4,7 @@ #define FILESYSTEM FakeFilesystem #include #include +#include #include "packet_under_test.h" static_assert(MESH_CONTACT_CACHE == 0, @@ -25,12 +26,20 @@ int fixtureStat(FakeFilesystem* fs, const char* vfs_path, struct stat*) { struct Host { std::vector contacts; unsigned copies = 0; + unsigned recovery_resets = 0; + bool recovery_reset_allowed = true; size_t capacity = 350; bool onContactLoaded(const ContactInfo& c) { if (contacts.size() >= capacity) return false; contacts.push_back(c); return true; } + bool resetContactsForRecovery() { + if (!recovery_reset_allowed) return false; + ++recovery_resets; + contacts.clear(); + return true; + } bool getContactForSave(uint32_t index, ContactInfo& contact) { ++copies; if (index >= contacts.size()) return false; @@ -55,6 +64,7 @@ class DataStore { void retireAdvertWrite(const uint8_t[], int) {} bool _channel_load_incomplete = false; bool _uncached_contact_load_incomplete = false; +#include "contact_recovery_state_under_test.h" #include "contact_write_state_under_test.h" public: explicit DataStore(FakeFilesystem& fs) : _fs(&fs) {} @@ -63,6 +73,8 @@ public: File openRead(FakeFilesystem* fs, const char* path) { return fs->open(path); } bool hasIncompleteContactLoad() const; void loadContacts(DataStoreHost*); + void recoverContacts(DataStoreHost*); +#include "contact_recovery_api_under_test.h" bool saveContacts(DataStoreHost*, bool (*)(const ContactInfo&)); bool flushContactWrites(DataStoreHost*, bool (*)(const ContactInfo&)); bool markContactDirty(const ContactInfo&); @@ -266,7 +278,8 @@ void deferredBeginCancellationAndCrashRecovery() { SPIFFS.files["/contacts3.bak"] = old; { Writer writer(&SPIFFS, "/contacts3", companionPathPresence, true); - for (unsigned pass = 0; pass < 7; ++pass) { + // Restoring a backup also verifies both names in separate setup passes. + for (unsigned pass = 0; pass < 9; ++pass) { assert(writer.serviceBegin() != Writer::BeginProgress::Failed); FakeFilesystem rebooted; rebooted.files = SPIFFS.files; @@ -513,10 +526,115 @@ void rebootAbortRecoveryAndDurableRemoval() { DataStore failed_boot(SPIFFS); failed_boot.loadContacts(&incomplete); assert(failed_boot.hasIncompleteContactLoad()); + assert(failed_boot.contactRecoveryPending()); const auto before = SPIFFS.files; assert(!failed_boot.serviceContactWrites(&incomplete, nullptr)); assert(!failed_boot.flushContactWrites(&incomplete, nullptr)); assert(SPIFFS.files == before); + SPIFFS.fail_read.clear(); + failed_boot.recoverContacts(&incomplete); + assert(!failed_boot.hasIncompleteContactLoad() && !failed_boot.contactRecoveryPending()); + // A fully readable prior backup may have replaced the unread target. + assert(image(incomplete) == before.at("/contacts3") && SPIFFS.files == before); +} + +void contactRecoveryPreservesSettingsAndResourceRefusals() { + for (unsigned fault = 0; fault < 3; ++fault) { + SPIFFS = FakeFilesystem(); + Host original = table(32), loaded; + SPIFFS.files["/contacts3"] = image(original); + for (const char* name : {"/_main.id", "/new_prefs", "/channels2", "/fleet_channel", + "/mesh_mt", "/mesh_eth"}) SPIFFS.files[name] = {7, 3, 9}; + if (fault == 1) { + SPIFFS.files["/contacts3"] = {8, 9}; + SPIFFS.files["/contacts3.bak"] = {4}; + } else SPIFFS.fail_read = "/contacts3"; + const auto durable = SPIFFS.files; + DataStore store(SPIFFS); + store.loadContacts(&loaded); + assert(store.hasIncompleteContactLoad() && store.contactRecoveryPending()); + assert(loaded.recovery_resets == 0); + loaded.recovery_reset_allowed = false; + store.recoverContacts(&loaded); + assert(store.contactRecoveryPending() && SPIFFS.files == durable); + loaded.recovery_reset_allowed = true; + SPIFFS.fail_remove = fault == 2; + store.recoverContacts(&loaded); + assert(!store.contactRecoveryPending() && !store.hasIncompleteContactLoad()); + assert(loaded.contacts.empty() && loaded.recovery_resets > 0); + auto expected = durable; + if (fault != 2) { + expected.erase("/contacts3"); + expected.erase("/contacts3.bak"); + } + assert(SPIFFS.files == expected); // No archive or broad settings reset. + const unsigned removes = SPIFFS.removes; + store.recoverContacts(&loaded); + assert(SPIFFS.removes == removes && SPIFFS.files == expected); + SPIFFS.fail_read.clear(); SPIFFS.fail_remove = false; + if (fault == 2) { + assert(!store.saveContacts(&loaded, nullptr)); // Failed media stays volatile this boot. + assert(SPIFFS.files == expected); + } else { + loaded.contacts.push_back(contact(909)); + assert(store.saveContacts(&loaded, nullptr)); + bootCheck(loaded); + } + } + SPIFFS = FakeFilesystem(); + Host original = table(32), limited; + SPIFFS.files["/contacts3"] = image(original); + const auto durable = SPIFFS.files; + limited.capacity = 20; + DataStore store(SPIFFS); + store.loadContacts(&limited); + assert(store.hasIncompleteContactLoad() && !store.contactRecoveryPending()); + const unsigned writes = SPIFFS.writes, removes = SPIFFS.removes; + store.recoverContacts(&limited); + assert(!store.saveContacts(&limited, nullptr)); + assert(SPIFFS.files == durable && SPIFFS.writes == writes && SPIFFS.removes == removes); + assert(limited.recovery_resets == 0); +} + +void validBackupSurvivesFailedPromotion() { + for (bool removal_failure : {false, true}) { + SPIFFS = FakeFilesystem(); + Host original = table(32), loaded; + const auto valid = image(original); + SPIFFS.files["/contacts3"] = {9}; + SPIFFS.files["/contacts3.bak"] = valid; + SPIFFS.fail_remove = removal_failure; + if (!removal_failure) SPIFFS.fail_rename = SPIFFS.renames + 1; + DataStore store(SPIFFS); + store.loadContacts(&loaded); + assert(!store.hasIncompleteContactLoad() && !store.contactRecoveryPending()); + assert(image(loaded) == valid && loaded.recovery_resets == 0); + assert(SPIFFS.files.at("/contacts3.bak") == valid); + if (!removal_failure) SPIFFS.fail_rename = SPIFFS.renames + 1; + assert(!store.saveContacts(&loaded, nullptr)); + assert(SPIFFS.files.at("/contacts3.bak") == valid); + SPIFFS.fail_remove = false; SPIFFS.fail_rename = 0; + assert(store.saveContacts(&loaded, nullptr)); + bootCheck(loaded); + } + SPIFFS = FakeFilesystem(); + Host original = table(32), loaded; + const auto valid = image(original); + SPIFFS.files["/contacts3"] = {9}; + SPIFFS.files["/contacts3.bak"] = valid; + SPIFFS.fail_rename = SPIFFS.renames + 1; + DataStore store(SPIFFS); + store.loadContacts(&loaded); + assert(image(loaded) == valid && !store.hasIncompleteContactLoad()); + SPIFFS.fail_rename = 0; + SPIFFS.after_rename = [](FakeFilesystem& fs, const char* path) { + if (!strcmp(path, "/contacts3")) fs.metadata_error = true; + }; + assert(!store.saveContacts(&loaded, nullptr)); + assert(SPIFFS.files.at("/contacts3") == valid && !SPIFFS.exists("/contacts3.bak")); + SPIFFS.metadata_error = false; SPIFFS.after_rename = {}; + assert(store.saveContacts(&loaded, nullptr)); + bootCheck(loaded); } void expectedAbsenceAndRealDeleteFailures() { @@ -671,6 +789,8 @@ int main() { syncFlushAndFilterChange(); realBackupCleanupFailure(); rebootAbortRecoveryAndDurableRemoval(); + contactRecoveryPreservesSettingsAndResourceRefusals(); + validBackupSurvivesFailedPromotion(); expectedAbsenceAndRealDeleteFailures(); #endif } diff --git a/test/fixtures/contact_publication/test.cpp b/test/fixtures/contact_publication/test.cpp index 7962fd72a..cdbc7f33d 100644 --- a/test/fixtures/contact_publication/test.cpp +++ b/test/fixtures/contact_publication/test.cpp @@ -6,6 +6,7 @@ #include using Writer=mesh::ContactFileTransaction; using Progress=Writer::CommitProgress; +using Outcome=Writer::Outcome; using Bytes=std::vector; #if defined(ESP32_PLATFORM) #define DEFER_BEGIN ,true @@ -13,10 +14,10 @@ using Bytes=std::vector; #define DEFER_BEGIN #endif Bytes image(size_t n,uint8_t salt=0){Bytes out(n);for(size_t i=0;imetadata_probes;if(fs->metadata_error)return false;found=fs->exists(name);return true;} +bool presence(FakeFilesystem* fs,const char* name,bool& found){++fs->metadata_probes;if(fs->metadata_error||fs->metadata_probes==fs->fail_metadata_probe)return false;found=fs->exists(name);return true;} void begin(Writer& w,bool coalesce=false){ #if defined(ESP32_PLATFORM) - for(unsigned i=0;!w&&i<9;++i){const auto before=SPIFFS.startupOperations();assert(w.serviceBegin(coalesce)!=Writer::BeginProgress::Failed);assert(SPIFFS.startupOperations()-before<=1);} + for(unsigned i=0;!w&&i<10;++i){const auto before=SPIFFS.startupOperations();assert(w.serviceBegin(coalesce)!=Writer::BeginProgress::Failed);assert(SPIFFS.startupOperations()-before<=1);} #else (void)coalesce; #endif @@ -81,7 +82,7 @@ void retirement(){ SPIFFS=FakeFilesystem();SPIFFS.files["/contacts3.bak"]=old; { Writer w(&SPIFFS,"/contacts3",presence,true); - for(unsigned i=0;i<3;++i)assert(w.serviceBegin(true)==Writer::BeginProgress::Pending); + for(unsigned i=0;i<5;++i){const auto before=SPIFFS.startupOperations();assert(w.serviceBegin(true)==Writer::BeginProgress::Pending);assert(SPIFFS.startupOperations()-before<=1);} assert(w._begin_target_exists&&!w._begin_backup_exists); assert(SPIFFS.files.at("/contacts3")==old&&!SPIFFS.exists("/contacts3.bak")); const auto removes=SPIFFS.removes; @@ -153,4 +154,227 @@ void counts(){ assert(finish(w,false,1)==829&&SPIFFS.backend_writes==211&&SPIFFS.backend_reads==413); #endif } -int main(int argc,char**argv){const std::string mode=argc>1?argv[1]:"all";if(mode=="contracts")contracts();else if(mode=="retire")retirement();else if(mode=="cancel")repeatedAndCancel();else if(mode=="faults")faultsAndCrash();else if(mode=="counts")counts();else assert(false);} +void ready(Writer& writer,const Bytes& old,bool present=true){ + unsigned passes=0; + while(!writer.readyToPublish()){ + const auto reads=SPIFFS.reads; + assert(writer.serviceCommit(true,8,true)==Progress::Pending); + assert(++passes<100&&SPIFFS.renames==0); +#if defined(ESP32_PLATFORM) + assert(SPIFFS.reads-reads<=8); + assert(SPIFFS.largest_backend_read<=512); +#else + assert(SPIFFS.reads-reads<=1); +#endif + assert(SPIFFS.largest_read<=64); + assert(SPIFFS.exists("/contacts3")==present); + if(present)assert(SPIFFS.files.at("/contacts3")==old); + } +} +void terminal(Writer& writer,Outcome expected){ + assert(writer.outcome()==expected&&!writer.readyToPublish()); + const auto files=SPIFFS.files; + const auto operations=SPIFFS.startupOperations(),reads=SPIFFS.reads,writes=SPIFFS.writes; + const uint8_t byte=7; + assert(!writer.commit()&&writer.serviceCommit()==Progress::Failed); + assert(writer.write(&byte,1)==0&&writer.outcome()==expected); + assert(SPIFFS.files==files&&SPIFFS.startupOperations()==operations + &&SPIFFS.reads==reads&&SPIFFS.writes==writes); +} +void crashSnapshots(const Bytes& old,const Bytes& next){ + for(const auto& snapshot:SPIFFS.rename_snapshots){ + FakeFilesystem boot;boot.files=snapshot; + assert(Writer::recover(&boot,"/contacts3",presence)); + const auto& bytes=boot.files.at("/contacts3");assert(bytes==old||bytes==next); + } +} +void outcomes(){ + const auto old=image(304,1),next=image(608,2); + for(bool deferred:{false,true})for(unsigned fault=0;fault<7;++fault){ + reset(old);FakeFilesystem::Files final; + const bool committed=fault==1||fault==3; + const bool uncertain=fault==4||fault==6; + const Outcome expected=committed?Outcome::Committed:uncertain?Outcome::Indeterminate:Outcome::NotCommitted; + { + Writer writer(&SPIFFS,"/contacts3",presence DEFER_BEGIN);begin(writer);write(writer,next);ready(writer,old); + if(fault==0)SPIFFS.false_success_rename=1; // Backup rename acknowledged without moving. + if(fault==1)SPIFFS.late_rename_error=1; + if(fault==2)SPIFFS.false_success_rename=2; // Publication did not happen; restore old. + if(fault==3)SPIFFS.late_rename_error=2; // Candidate is public; never replace it with backup. + if(fault==4||fault==5){SPIFFS.fail_rename=2; + if(fault==4)SPIFFS.false_success_rename=3;else SPIFFS.late_rename_error=3;} + if(fault==6){SPIFFS.false_success_rename=2;SPIFFS.fail_rename=3;} + const auto reads=SPIFFS.reads; + assert(writer.serviceCommit(true,8,deferred)==(committed?Progress::Succeeded:Progress::Failed)); + assert(SPIFFS.reads==reads); // No payload reads or yield inside publication. + terminal(writer,expected); + assert(SPIFFS.exists("/contacts3")==!uncertain); + if(!uncertain)assert(SPIFFS.files.at("/contacts3")== (committed?next:old)); + if(uncertain){assert(SPIFFS.files.at("/contacts3.bak")==old);assert(SPIFFS.files.at("/contacts3.tmp")==next);} + else { + assert(!SPIFFS.exists("/contacts3.tmp")); +#if defined(ESP32_PLATFORM) + assert(SPIFFS.exists("/contacts3.bak")== (committed&&deferred)); +#else + assert(!SPIFFS.exists("/contacts3.bak")); +#endif + } + if(fault==3)assert(SPIFFS.renames==2); + final=SPIFFS.files; + } + assert(SPIFFS.files==final);crashSnapshots(old,next); + } +} +void metadata(){ + const auto old=image(304,1),next=image(608,2); + for(bool deferred:{false,true})for(unsigned moved=1;moved<=3;++moved)for(unsigned probe=1;probe<=2;++probe){ + reset(old);FakeFilesystem::Files final; + { + Writer writer(&SPIFFS,"/contacts3",presence DEFER_BEGIN);begin(writer);write(writer,next);ready(writer,old); + if(moved==3)SPIFFS.fail_rename=2; // Force the rollback branch. + SPIFFS.after_rename=[=](FakeFilesystem& fs,const char*){ + if(fs.renames==moved)fs.fail_metadata_probe=fs.metadata_probes+probe; + }; + const auto reads=SPIFFS.reads; + assert(writer.serviceCommit(true,8,deferred)==Progress::Failed&&SPIFFS.reads==reads); + assert(SPIFFS.metadata_probes>=SPIFFS.fail_metadata_probe); + terminal(writer,Outcome::Indeterminate); + assert(SPIFFS.renames==moved); // Unknown publication cannot blindly enter rollback. + assert(SPIFFS.exists("/contacts3")== (moved!=1)); + if(moved!=1)assert(SPIFFS.files.at("/contacts3")== (moved==2?next:old)); + assert(SPIFFS.exists("/contacts3.bak")== (moved!=3)); + assert(SPIFFS.exists("/contacts3.tmp")== (moved!=2)); + final=SPIFFS.files; + } + assert(SPIFFS.files==final);crashSnapshots(old,next); + } + // Probe failures before any move remain ordinary safe failures. + for(unsigned probe=1;probe<=2;++probe){ + reset(old);Writer writer(&SPIFFS,"/contacts3",presence DEFER_BEGIN);begin(writer);write(writer,next);ready(writer,old); + SPIFFS.fail_metadata_probe=SPIFFS.metadata_probes+probe; + assert(writer.serviceCommit()==Progress::Failed);terminal(writer,Outcome::NotCommitted); + assert(SPIFFS.renames==0&&SPIFFS.files.at("/contacts3")==old&&!SPIFFS.exists("/contacts3.tmp")); + } +} +void fresh(){ + const auto next=image(608,2); + for(unsigned fault=0;fault<5;++fault){ + SPIFFS=FakeFilesystem();FakeFilesystem::Files final; + { + Writer writer(&SPIFFS,"/contacts3",presence DEFER_BEGIN);begin(writer);write(writer,next);ready(writer,{},false); + if(fault==1)SPIFFS.false_success_rename=1; + if(fault==2)SPIFFS.late_rename_error=1; + if(fault==3)SPIFFS.after_rename=[](FakeFilesystem& fs,const char*){fs.fail_metadata_probe=fs.metadata_probes+1;}; + if(fault==4)SPIFFS.fail_rename=1; + const Outcome expected=fault==3?Outcome::Indeterminate: + fault==1||fault==4?Outcome::NotCommitted:Outcome::Committed; + const auto reads=SPIFFS.reads; + assert(writer.serviceCommit(true,8,true)==(expected==Outcome::Committed?Progress::Succeeded:Progress::Failed)); + assert(SPIFFS.reads==reads);terminal(writer,expected); + assert(SPIFFS.exists("/contacts3")== (expected!=Outcome::NotCommitted)); + if(SPIFFS.exists("/contacts3"))assert(SPIFFS.files.at("/contacts3")==next); + assert(!SPIFFS.exists("/contacts3.bak")&&!SPIFFS.exists("/contacts3.tmp")); + final=SPIFFS.files; + } + assert(SPIFFS.files==final); + } +} +void ambiguous(){ + const auto old=image(304,1),next=image(608,2); + for(bool deferred:{false,true})for(unsigned moved=1;moved<=2;++moved)for(bool duplicate:{false,true}){ + reset(old);FakeFilesystem::Files final; + { + Writer writer(&SPIFFS,"/contacts3",presence DEFER_BEGIN);begin(writer);write(writer,next);ready(writer,old); + SPIFFS.after_rename=[=](FakeFilesystem& fs,const char*){ + if(fs.renames!=moved)return; + if(moved==1){ + if(duplicate)fs.files["/contacts3"]=old;else fs.files.erase("/contacts3.bak"); + }else{ + if(duplicate)fs.files["/contacts3.tmp"]=next;else fs.files.erase("/contacts3"); + } + }; + const auto reads=SPIFFS.reads,removes=SPIFFS.removes; + assert(writer.serviceCommit(true,8,deferred)==Progress::Failed); + terminal(writer,Outcome::Indeterminate); + assert(SPIFFS.renames==moved&&SPIFFS.reads==reads&&SPIFFS.removes==removes); + assert(SPIFFS.exists("/contacts3")==duplicate); + if(duplicate)assert(SPIFFS.files.at("/contacts3")== (moved==1?old:next)); + assert(SPIFFS.exists("/contacts3.bak")== (moved==2||duplicate)); + if(SPIFFS.exists("/contacts3.bak"))assert(SPIFFS.files.at("/contacts3.bak")==old); + assert(SPIFFS.exists("/contacts3.tmp")== (moved==1||duplicate)); + if(SPIFFS.exists("/contacts3.tmp"))assert(SPIFFS.files.at("/contacts3.tmp")==next); + final=SPIFFS.files; + } + assert(SPIFFS.files==final); + } +} +void recovery(){ + const auto old=image(304,1),next=image(608,2); + for(unsigned fault=0;fault<8;++fault){ + FakeFilesystem fs;fs.files["/contacts3.bak"]=old;fs.files["/contacts3.tmp"]=next; + if(fault==1)fs.false_success_rename=1; + if(fault==2)fs.late_rename_error=1; + if(fault==3)fs.fail_rename=1; + if(fault>=4)fs.fail_metadata_probe=fault-3; // Before rename and each confirmation probe. + assert(Writer::recover(&fs,"/contacts3",presence)==(fault==0||fault==2)); + const bool moved=fault==0||fault==2||fault>=6; + assert(fs.exists("/contacts3")==moved&&fs.exists("/contacts3.bak")==!moved); + if(moved)assert(fs.files.at("/contacts3")==old); + assert(fs.files.at("/contacts3.tmp")==next); + fs.false_success_rename=fs.late_rename_error=fs.fail_rename=fs.fail_metadata_probe=0; + assert(Writer::recover(&fs,"/contacts3",presence)&&fs.files.at("/contacts3")==old); + } + // A present candidate wins over the retained old backup; scratch is never promoted. + {FakeFilesystem fs;fs.files["/contacts3"]=next;fs.files["/contacts3.bak"]=old; + assert(Writer::recover(&fs,"/contacts3",presence)&&fs.renames==0&&fs.files.at("/contacts3")==next&&fs.files.at("/contacts3.bak")==old);} +#if defined(ESP32_PLATFORM) + for(bool background:{false,true})for(unsigned fault=0;fault<6;++fault){ + SPIFFS=FakeFilesystem();SPIFFS.files["/contacts3.bak"]=old;SPIFFS.files["/contacts3.tmp"]=next; + FakeFilesystem::Files final; + { + Writer writer(&SPIFFS,"/contacts3",presence,true); + if(fault==1)SPIFFS.false_success_rename=1; + if(fault==2)SPIFFS.late_rename_error=1; + if(fault==3)SPIFFS.fail_rename=1; + if(fault>=4)SPIFFS.after_rename=[=](FakeFilesystem& fs,const char*){fs.fail_metadata_probe=fs.metadata_probes+fault-3;}; + auto result=Writer::BeginProgress::Pending;unsigned passes=0; + while(result==Writer::BeginProgress::Pending){ + const auto before=SPIFFS.startupOperations();result=writer.serviceBegin(background); + assert(++passes<=10&&SPIFFS.startupOperations()-before<=1); + } + const bool recovered=fault==0||fault==2; + assert(result==(recovered?Writer::BeginProgress::Ready:Writer::BeginProgress::Failed)); + if(recovered){assert(writer&&passes==unsigned(background?10:9));assert(writer.outcome()==Outcome::Pending);} + else {assert(!writer&&!writer._owns_temp);terminal(writer,Outcome::Indeterminate); + assert(SPIFFS.files.at("/contacts3.tmp")==next);} + final=SPIFFS.files; + } + if(fault==0||fault==2){final.erase("/contacts3.tmp");assert(SPIFFS.files==final&&SPIFFS.files.at("/contacts3")==old);} + else assert(SPIFFS.files==final); + } + // Cancellation at every recovery/setup boundary retains old authority and + // never deletes prior scratch until its explicit Remove stage. + for(unsigned completed=0;completed<=10;++completed){ + SPIFFS=FakeFilesystem();SPIFFS.files["/contacts3.bak"]=old;SPIFFS.files["/contacts3.tmp"]=next; + {Writer writer(&SPIFFS,"/contacts3",presence,true); + for(unsigned i=0;i=2)SPIFFS.fail_metadata_probe=fault-1; + FakeFilesystem::Files final; + {Writer writer(&SPIFFS,"/contacts3",presence);assert(!writer);terminal(writer,Outcome::Indeterminate); + assert(SPIFFS.files.at("/contacts3.tmp")==next);final=SPIFFS.files;} + assert(SPIFFS.files==final); + } +#endif +} +int main(int argc,char**argv){const std::string mode=argc>1?argv[1]:"all";if(mode=="contracts")contracts();else if(mode=="retire")retirement();else if(mode=="cancel")repeatedAndCancel();else if(mode=="faults")faultsAndCrash();else if(mode=="counts")counts();else if(mode=="outcomes")outcomes();else if(mode=="metadata")metadata();else if(mode=="fresh")fresh();else if(mode=="ambiguous")ambiguous();else if(mode=="recovery")recovery();else assert(false);} diff --git a/test/test_companion_contact_sleep_contract.py b/test/test_companion_contact_sleep_contract.py index b9e1b6e1e..9c09e7061 100644 --- a/test/test_companion_contact_sleep_contract.py +++ b/test/test_companion_contact_sleep_contract.py @@ -27,6 +27,103 @@ def function_body(source: str, signature: str) -> str: class CompanionContactSleepContractTest(unittest.TestCase): + def test_recovery_resets_at_outer_boundary_and_volatile_saves_stay_truthful(self): + mesh = (ROOT / "examples/companion_radio/MyMesh.cpp").read_text() + store = (ROOT / "examples/companion_radio/DataStore.cpp").read_text() + loop = function_body(mesh, "void MyMesh::loop()") + self.assertLess(loop.index("_store->recoverContacts(this)"), + loop.index("BaseChatMesh::loop()")) + self.assertIn("contactRecoveryPending() && !mesh::morseTimingActive(_radio)", loop) + self.assertIn("|| _store->contactRecoveryPending()", + function_body(mesh, "bool MyMesh::hasPendingWork() const")) + reset = function_body(mesh, "bool MyMesh::resetContactsForRecovery()") + self.assertLess(reset.index("stopContactsIterator()"), reset.index("resetContacts()")) + self.assertIn("MyMesh::onContactCacheFlushed()", reset) + clear = function_body(mesh, "void MyMesh::onContactCacheFlushed()") + self.assertIn("mesh::resetLazyPersistenceAfterSuccess(dirty_contacts_expiry, dirty_contacts_failures)", clear) + recovery = function_body(store, "void DataStore::recoverContacts(") + self.assertNotIn("format(", recovery) + self.assertNotIn("reboot(", recovery) + self.assertNotIn('"/.extrafs.mig"', recovery) + signatures = ( + "void MyMesh::onContactCacheFlushed()", + "bool MyMesh::flushContactsBeforeReboot()", + "bool MyMesh::prepareForUiShutdown()", + "void MyMesh::rebootNow()", + "void MyMesh::saveContacts()", + ) + methods = "\n".join(signature + " {" + function_body(mesh, signature) + "}" + for signature in signatures) + fixture = r''' +#include +#include +#define MESH_DEBUG_PRINTLN(...) do {} while (0) +namespace mesh { +void resetLazyPersistenceAfterSuccess(uint32_t& deadline, uint8_t& failures) { + deadline = 0; failures = 0; +} +uint32_t nonzeroLazyPersistenceDeadline(uint32_t value) { return value ? value : 1; } +} +constexpr unsigned CONTACT_PAGE_WRITE_GAP = 10; +bool save_filter(int) { return true; } +struct Board { unsigned resets = 0; void reboot() { ++resets; } } board; +struct Store { + bool vol = false, saved = true, pending = true; + unsigned writes = 0; + bool contactsAreVolatile() const { return vol; } + bool hasPendingContactWrites() const { return pending; } + bool flushContactWrites(void*, bool (*)(int)) { ++writes; return saved; } + bool saveContacts(void*, bool (*)(int)) { ++writes; return saved; } +}; +struct MyMesh { + Store store; Store* _store = &store; + uint32_t dirty_contacts_expiry = 1; + uint8_t dirty_contacts_failures = 1; + unsigned retries = 0; + uint32_t futureMillis(unsigned value) { return 100 + value; } + void scheduleContactWriteRetry() { ++retries; } + bool flushContactsBeforeReboot(); bool prepareForUiShutdown(); + virtual void onContactCacheFlushed(); + void rebootNow(); void saveContacts(); +}; +@METHODS@ +int main() { + MyMesh node; + node.store.vol = true; + assert(!node.flushContactsBeforeReboot()); // tx.contact must not promise durability + assert(node.store.writes == 0 && node.retries == 0); + assert(node.prepareForUiShutdown()); + node.rebootNow(); assert(board.resets == 1); + node.saveContacts(); + assert(node.dirty_contacts_expiry == 0 && node.dirty_contacts_failures == 0); + assert(node.store.writes == 0 && node.retries == 0); + node.store.vol = false; node.store.saved = false; + node.dirty_contacts_expiry = 1; + assert(!node.flushContactsBeforeReboot()); + assert(node.store.writes == 1 && node.retries == 1); + node.rebootNow(); assert(board.resets == 1); + node.store.saved = true; node.store.pending = false; + assert(node.flushContactsBeforeReboot()); + assert(node.dirty_contacts_expiry == 0 && node.dirty_contacts_failures == 0); + node.rebootNow(); assert(board.resets == 2); + struct Derived : MyMesh { + unsigned overrides = 0; + void onContactCacheFlushed() override { ++overrides; } + } derived; + derived.store.vol = true; + derived.saveContacts(); + assert(derived.overrides == 0 && derived.dirty_contacts_expiry == 0); + assert(derived.dirty_contacts_failures == 0 && derived.store.writes == 0); +} +'''.replace("@METHODS@", methods) + with tempfile.TemporaryDirectory(prefix="companion-contact-recovery-") as directory: + source, binary = Path(directory) / "recovery.cpp", Path(directory) / "recovery" + source.write_text(fixture, encoding="ascii") + subprocess.run([os.environ.get("CXX", "g++"), "-std=c++17", "-Wall", "-Wextra", + "-Werror", "-fsanitize=address,undefined", "-fno-pie", "-no-pie", + str(source), "-o", str(binary)], check=True, timeout=30) + subprocess.run([str(binary)], check=True, timeout=5) + def test_initial_write_polls_but_failed_backoff_can_sleep(self): mesh = (ROOT / "examples/companion_radio/MyMesh.cpp").read_text() body = function_body(mesh, "bool MyMesh::hasPendingWork() const") @@ -111,7 +208,7 @@ class CompanionContactSleepContractTest(unittest.TestCase): self.assertIn("retry in %lu ms", retry) for body in (save, loop): self.assertIn("scheduleContactWriteRetry();", body) - self.assertIn("mesh::resetLazyPersistenceAfterSuccess(", body) + self.assertIn("MyMesh::onContactCacheFlushed()", body) def test_main_loop_gates_sleep_on_pending_work(self): main = (ROOT / "examples/companion_radio/main.cpp").read_text() @@ -184,25 +281,27 @@ int main() { flush = function_body(mesh, "bool MyMesh::flushContactsBeforeReboot()") self.assertIn("_store->flushContactWrites(this, save_filter)", flush) self.assertIn("scheduleContactWriteRetry();", flush) - self.assertIn("mesh::resetLazyPersistenceAfterSuccess(", flush) + self.assertIn("MyMesh::onContactCacheFlushed()", flush) framed = function_body(mesh, "void MyMesh::handleCmdFrame(") reboot_at = framed.index("cmd_frame[0] == CMD_REBOOT") framed_reboot = framed[reboot_at : framed.index( "cmd_frame[0] == CMD_GET_BATT_AND_STORAGE", reboot_at )] - self.assertIn("flushContactsBeforeReboot()", framed_reboot) + self.assertIn("MyMesh::prepareForUiShutdown()", framed_reboot) rescue = function_body(mesh, "void MyMesh::checkCLIRescueCmd()") rescue_reboot = rescue[rescue.index('strcmp(cli_command, "reboot")') :] - self.assertIn("flushContactsBeforeReboot()", rescue_reboot) + self.assertIn("MyMesh::prepareForUiShutdown()", rescue_reboot) callback = function_body(mesh, "void MyMesh::rebootNow()") - self.assertIn("flushContactsBeforeReboot()", callback) + self.assertIn("MyMesh::prepareForUiShutdown()", callback) loop = function_body(mesh, "void MyMesh::loop()") scheduled = loop[: loop.index("#if COMPANION_FEATURE_TEMP_RADIO")] - self.assertIn("flushContactsBeforeReboot()", scheduled) + self.assertIn("MyMesh::prepareForUiShutdown()", scheduled) + shutdown = function_body(mesh, "bool MyMesh::prepareForUiShutdown()") + self.assertIn("_store->contactsAreVolatile() || flushContactsBeforeReboot()", shutdown) pending = function_body(mesh, "bool MyMesh::hasPendingWork() const") self.assertIn("|| _scheduled_reboot_at != 0", pending) @@ -338,7 +437,7 @@ int main() { save_identity_at = identity_import.index("_store->saveMainIdentity(") load_at = identity_import.index("_store->loadContacts(this)") reset_at = identity_import.index( - "mesh::resetLazyPersistenceAfterSuccess(", save_identity_at + "MyMesh::onContactCacheFlushed()", save_identity_at ) rearm_at = identity_import.index( "mesh::scheduleLazyPersistenceMutation(", load_at diff --git a/test/test_companion_prefs_image.py b/test/test_companion_prefs_image.py index 0c8163cda..6e0672390 100644 --- a/test/test_companion_prefs_image.py +++ b/test/test_companion_prefs_image.py @@ -1,4 +1,4 @@ -"""Check the explicit preference wire layout and bounded STM32 staging.""" +"""Check explicit preference bytes, STM32 staging and nRF field streaming.""" from pathlib import Path import os import subprocess @@ -63,6 +63,9 @@ static void seed(CompanionNodePrefs& prefs, uint32_t& random) { @SEED@ prefs.tx_media = random % 5; prefs.tx_ssh = (random >> 8) % 5; +#if defined(RAK_4631) + prefs.peripherals_shared = uint8_t(random >> 16); +#endif } static std::vector golden(const CompanionNodePrefs& prefs, @@ -79,6 +82,18 @@ static std::vector golden(const CompanionNodePrefs& prefs, assert(bytes.size() == 333); #else assert(bytes.size() == 236); +#endif +#if defined(RAK_4631) + bytes.insert(bytes.end(), {'P', 'M', 'O', 1, + uint8_t(prefs.peripherals_shared == 0 ? 0 : 1), 0}); + uint16_t peripheral_crc = 0xffff; + for (size_t i = bytes.size() - 6; i < bytes.size(); ++i) { + peripheral_crc ^= uint16_t(bytes[i]) << 8; + for (unsigned bit = 0; bit < 8; ++bit) + peripheral_crc = (peripheral_crc & 0x8000) + ? uint16_t((peripheral_crc << 1) ^ 0x1021) : uint16_t(peripheral_crc << 1); + } + bytes.push_back(uint8_t(peripheral_crc >> 8)); bytes.push_back(uint8_t(peripheral_crc)); #endif // Independent wire definition of the new checked traffic footer. Keep // every historical byte above unchanged and encode the public policy tags. @@ -111,9 +126,24 @@ int main() { // and migration remain the loader/CLI's existing responsibility. seed(prefs, random); DataStore store; + WRITE_CALLS.clear(); assert(store.savePrefs(prefs, 47.125, -122.25)); assert(store.fs.files.at("/new_prefs") == golden(prefs, 47.125, -122.25)); assert(!store.fs.exists("/new_prefs.tmp")); +#if defined(NRF52_PLATFORM) + const std::vector streaming = {@WRITE_SIZES@ +#if defined(RAK_4631) + , 8 +#endif + , 8}; + assert(WRITE_CALLS == streaming); // Direct fields, never a 236-byte staging write. +#if defined(RAK_4631) + assert(store.fs.files.at("/new_prefs").size() == 252); +#endif +#endif + DataStore reference; + assert(reference.savePrefsLegacy(prefs, 47.125, -122.25)); + assert(reference.fs.files.at("/new_prefs") == store.fs.files.at("/new_prefs")); ++comparisons; } @@ -127,10 +157,17 @@ int main() { original.lost_reply = 2; original.tx_media = mesh::RADIO_TX_SECONDARY; original.tx_ssh = mesh::RADIO_TX_OFF; +#if defined(RAK_4631) + original.peripherals_shared = 1; +#endif DataStore initial; assert(initial.savePrefs(original, 47.125, -122.25)); const auto before = golden(original, 47.125, -122.25); - const size_t historical_size=before.size()-mesh::companion::traffic::FooterSize; + const size_t historical_size=before.size()-mesh::companion::traffic::FooterSize +#if defined(RAK_4631) + - mesh::peripherals::FooterSize +#endif + ; assert(initial.fs.files.at("/new_prefs") == before); // Every complete historical wire boundary stays readable, and an old @@ -172,6 +209,9 @@ int main() { // if the underlying file size were accidentally narrowed with the table. std::vector accepted_sizes = legacy_sizes; accepted_sizes.push_back(historical_size); +#if defined(RAK_4631) + accepted_sizes.push_back(historical_size + mesh::peripherals::FooterSize); +#endif accepted_sizes.push_back(before.size()); for (size_t length = 0; length <= before.size() + 256; ++length) { DataStore reboot; @@ -191,6 +231,9 @@ int main() { assert(live.lost_reply == (length >= historical_size ? 2 : 0)); assert(live.tx_media == (length == before.size() ? mesh::RADIO_TX_SECONDARY : mesh::RADIO_TX_AUTO)); assert(live.tx_ssh == (length == before.size() ? mesh::RADIO_TX_OFF : mesh::RADIO_TX_AUTO)); +#if defined(RAK_4631) + assert(live.peripherals_shared == (length >= 244 ? 1 : 0)); +#endif assert(lat == 47.125 && lon == -122.25); } else { assert(live.freq == 920.5f && live.lost_reply == 1); @@ -255,6 +298,11 @@ int main() { write.fs.fail_write_after = static_cast(quota); assert(!write.savePrefs(changed, 42.25, -121.125)); unchanged(write, before); + DataStore reference; + reference.fs.files["/new_prefs"] = before; + reference.fs.fail_write_after = static_cast(quota); + assert(!reference.savePrefsLegacy(changed, 42.25, -121.125)); + unchanged(reference, before); write.fs.fail_write_after = -1; assert(write.savePrefs(changed, 42.25, -121.125)); assert(write.fs.files.at("/new_prefs") == after); @@ -265,6 +313,22 @@ int main() { unchanged(read, before); ++comparisons; } +#if defined(RAK_4631) + // All checked peripheral-footer bits are validation inputs, including the + // boundary immediately before the independent traffic footer. + for (unsigned bit = 0; bit < 64; ++bit) { + DataStore corrupt; + auto candidate = before; + candidate[236 + bit / 8] ^= uint8_t(1u << (bit % 8)); + corrupt.fs.files["/new_prefs"] = candidate; + CompanionNodePrefs live; live.freq = 920.5f; live.peripherals_shared = 0; + double lat = 1, lon = 2; + assert(!corrupt.loadPrefsInt("/new_prefs", live, lat, lon)); + assert(live.freq == 920.5f && live.peripherals_shared == 0 && lat == 1 && lon == 2); + assert(corrupt.fs.files.at("/new_prefs") == candidate); + ++comparisons; + } +#endif for (unsigned fault = 0; fault < 3; ++fault) { DataStore store; store.fs.files["/new_prefs"] = before; @@ -335,10 +399,28 @@ class CompanionPrefsImageTests(unittest.TestCase): def test_explicit_wire_layout_and_atomic_staging(self): source = (ROOT / 'examples/companion_radio/DataStore.cpp').read_text() save = method(source, 'bool DataStore::savePrefs(') + nrf_branch_start = save.index('#elif defined(NRF52_PLATFORM) && defined(__GNUC__)') + nrf_branch = save[nrf_branch_start:save.index('\n#else', nrf_branch_start)] + self.assertIn('file.write(', nrf_branch) + self.assertIn('prefValue(', nrf_branch) + self.assertNotIn('image[', nrf_branch) + self.assertNotIn('memcpy', nrf_branch) + legacy_writer = (ROOT / 'test/fixtures/companion_prefs_storage/legacy_writer.inc').read_text() + commit = ' success = file.commit(success);' + self.assertEqual(legacy_writer.count(commit), 1) + legacy_writer = legacy_writer.replace(commit, '''#if defined(RAK_4631) + uint8_t peripheral_footer[8]; + mesh::peripherals::encodeFooter(peripheral_footer, _prefs.peripherals_shared); + success = success && file.write(peripheral_footer, sizeof(peripheral_footer)) == sizeof(peripheral_footer); +#endif + uint8_t traffic_footer[8]; + mesh::companion::traffic::encodeFooter(traffic_footer, _prefs.tx_media, _prefs.tx_ssh); + success = success && file.write(traffic_footer, sizeof(traffic_footer)) == sizeof(traffic_footer); +''' + commit) methods = esp_recovery_helpers(source) + '\n'.join( method(source, signature) for signature in ( 'bool DataStore::loadPrefs(', 'bool DataStore::loadPrefsInt(') - ) + '\n' + save + ) + '\n' + save + '\n' + legacy_writer for suffix, capacity in (('TooSmall', 235), ('TooLarge', 237)): variant = save.replace('DataStore::savePrefs(', 'DataStore::savePrefs' + suffix + '(', 1) self.assertIn('uint8_t image[236];', variant) @@ -347,6 +429,7 @@ class CompanionPrefsImageTests(unittest.TestCase): code = HARNESS.split('int main(')[0].replace('@METHODS@', methods) code = code.replace(' bool savePrefs(const CompanionNodePrefs&, double, double);', ' bool savePrefs(const CompanionNodePrefs&, double, double);\n' + ' bool savePrefsLegacy(const CompanionNodePrefs&, double, double);\n' ' bool savePrefsTooSmall(const CompanionNodePrefs&, double, double);\n' ' bool savePrefsTooLarge(const CompanionNodePrefs&, double, double);') self.assertEqual(sum(size for _, size in WIRE_FIELDS), 226) @@ -361,6 +444,8 @@ class CompanionPrefsImageTests(unittest.TestCase): tests = TESTS for placeholder, value in generated.items(): tests = tests.replace(placeholder, value) + tests = tests.replace('@WRITE_SIZES@', ', '.join( + str(size) for _, size in WIRE_FIELDS + TAIL_FIELDS)) with tempfile.TemporaryDirectory(prefix='meshcore-prefs-image-') as directory: work = Path(directory) (work / 'Utils.h').write_text('''#pragma once @@ -383,6 +468,11 @@ inline char* utoa(unsigned int value,char* output,int base){ # Reuse the existing memory-filesystem boundary, adding only an # observable open failure needed to exercise the actual writer. fs = (ROOT / 'test/fixtures/radio_profiles/mocks/helpers/IdentityStore.h').read_text() + fs = fs.replace('class MemoryFS;', + 'static std::vector WRITE_CALLS;\nclass MemoryFS;') + fs = fs.replace('inline size_t File::write(const uint8_t* data, size_t size) {', + 'inline size_t File::write(const uint8_t* data, size_t size) {\n' + ' WRITE_CALLS.push_back(size);') fs = fs.replace(' bool fail_write = false;', ' bool fail_write = false, fail_write_open = false;') fs = fs.replace(' if (*mode == \'w\') files[path].clear();', @@ -392,7 +482,7 @@ inline char* utoa(unsigned int value,char* output,int base){ (work / 'helpers/IdentityStore.h').write_text(fs) cpp = work / 'test.cpp' cpp.write_text(code + tests) - for platform in ('STM32_PLATFORM', 'NRF52_PLATFORM', 'ESP32_PLATFORM', + for platform in ('STM32_PLATFORM', 'NRF52_PLATFORM', 'NRF52_PLATFORM,RAK_4631', 'ESP32_PLATFORM', 'RP2040_PLATFORM', 'ESP32_PLATFORM,TBEAM_1W', 'RP2040_PLATFORM,ENABLE_WIFI_INTERFACE'): with self.subTest(platform=platform): diff --git a/test/test_companion_prefs_values.py b/test/test_companion_prefs_values.py index 2a97961b9..0a0ee2e53 100644 --- a/test/test_companion_prefs_values.py +++ b/test/test_companion_prefs_values.py @@ -143,6 +143,9 @@ int main() { #endif #if defined(RP2040_PLATFORM) && defined(ENABLE_WIFI_INTERFACE) optional_fields += 2; +#endif +#if defined(RAK_4631) + optional_fields += 1; // Checked peripheral footer has its own EOF probe. #endif // Every absent optional field observes EOF, then the completed // image receives the existing final trailing-byte check. @@ -179,6 +182,28 @@ class CompanionPrefsValuesTests(unittest.TestCase): # is the authority for all old/torn images and read failure traces. legacy = (ROOT / 'test/fixtures/companion_prefs_storage/legacy_loader.inc').read_text() self.assertNotIn('copyPersistedValuesFrom', legacy) + # Keep the frozen scalar prefix independent of the new GNU table. + # The RAK-only checked tail was published later; add it explicitly + # to this test-local reference without rewriting the frozen fixture. + legacy = legacy.replace(' loaded_prefs.lost_reply = 0;', + ' loaded_prefs.lost_reply = 0;\n' + '#if defined(RAK_4631)\n loaded_prefs.peripherals_shared = 0;\n#endif') + size_check = ' if (prefs_size < MIN_PREFS_SIZE || !known_size) {' + self.assertEqual(legacy.count(size_check), 1) + legacy = legacy.replace(size_check, + '#if defined(RAK_4631)\n' + ' known_size = known_size || prefs_size == 244;\n' + '#endif\n' + size_check) + trailing_check = ' // Any bytes left over form only part of a historically appended field.' + self.assertEqual(legacy.count(trailing_check), 1) + legacy = legacy.replace(trailing_check, + '#if defined(RAK_4631)\n' + ' if (file.available() != 0) {\n' + ' uint8_t footer[8] = {};\n' + ' success = success && readField(footer, sizeof(footer))\n' + ' && mesh::peripherals::decodeFooter(footer, sizeof(footer),\n' + ' loaded_prefs.peripherals_shared);\n' + ' }\n#endif\n\n' + trailing_check) methods = esp_recovery_helpers(source) + '\n'.join((loader, legacy, method(source, 'bool DataStore::savePrefs('))) code = HARNESS.split('int main(')[0].replace('@METHODS@', methods) @@ -251,7 +276,7 @@ inline char* utoa(unsigned int value,char* output,int base){ (work / 'helpers/IdentityStore.h').write_text(fs) cpp = work / 'test.cpp' cpp.write_text(code + tests) - for platform in ('STM32_PLATFORM', 'NRF52_PLATFORM', 'ESP32_PLATFORM', 'RP2040_PLATFORM', + for platform in ('STM32_PLATFORM', 'NRF52_PLATFORM', 'NRF52_PLATFORM,RAK_4631', 'ESP32_PLATFORM', 'RP2040_PLATFORM', 'ESP32_PLATFORM,TBEAM_1W', 'RP2040_PLATFORM,ENABLE_WIFI_INTERFACE'): with self.subTest(platform=platform): binary = work / 'test' diff --git a/test/test_companion_tx_routing.py b/test/test_companion_tx_routing.py index dad8f35b5..42de6258d 100644 --- a/test/test_companion_tx_routing.py +++ b/test/test_companion_tx_routing.py @@ -48,6 +48,9 @@ struct DataStore { FILESYSTEM fs; bool _channel_load_incomplete=false; bool _uncached_contact_load_incomplete=false; + enum : uint8_t { ContactRecoveryPending=1 }; + uint8_t _contact_recovery=0; + @RECOVERY_METHOD@ const char* _channel_recovery_source=nullptr; bool hasIncompleteContactLoad() const; FILESYSTEM* _getContactsChannelsFS() { return &fs; } @@ -104,6 +107,10 @@ int main() { node.radio.config.secondary_temporary=true; char reply[160]; auto cmd=[&](const char* text) { assert(node.handleTxRoutingCommand(text,reply,sizeof(reply))); }; + node.store._contact_recovery=DataStore::ContactRecoveryPending; + cmd("set tx.channel 0 off"); assert(strstr(reply,"storage unavailable")); + assert(node.writes==0 && node.channels[0].channel.tx_radio==mesh::RADIO_TX_AUTO); + node.store._contact_recovery=0; cmd("set tx.user \"Alice Jones\" radio2"); assert(!strncmp(reply,"OK",2)); assert(strstr(reply,"active=radio2") && node.contacts[1].tx_radio==mesh::RADIO_TX_SECONDARY); assert(node.contacts[2].tx_radio==mesh::RADIO_TX_AUTO); @@ -271,6 +278,7 @@ class CompanionTxRoutingTest(unittest.TestCase): def test_cli_resolution_persistence_and_failures(self): companion = (ROOT / 'examples/companion_radio/MyMesh.cpp').read_text() store = (ROOT / 'examples/companion_radio/DataStore.cpp').read_text() + store_header = (ROOT / 'examples/companion_radio/DataStore.h').read_text() methods = esp_recovery_helpers(store) + method(companion, 'bool MyMesh::handleTxRoutingCommand(') methods += '\n' + method(store, 'void DataStore::loadChannels(') methods += '\n' + method(store, 'bool DataStore::saveChannels(') @@ -282,7 +290,8 @@ class CompanionTxRoutingTest(unittest.TestCase): transaction = (ROOT / 'src/helpers/ContactFileTransaction.h').read_text() (work / 'ContactFileTransaction.h').write_text(transaction.replace( '#include "IdentityStore.h"', '#include ')) - (work / 'test.cpp').write_text(HARNESS.replace('@METHODS@', methods)) + (work / 'test.cpp').write_text(HARNESS.replace('@METHODS@', methods).replace( + '@RECOVERY_METHOD@', method(store_header, ' bool contactRecoveryPending() const'))) for platform in ('ESP32_PLATFORM', 'RP2040_PLATFORM', 'STM32_PLATFORM'): with self.subTest(platform=platform): build = subprocess.run(['g++', '-std=c++17', '-Wall', '-Wextra', '-D'+platform+'=1', diff --git a/test/test_companion_uncached_storage.py b/test/test_companion_uncached_storage.py index e4fcd9ead..c3e237048 100644 --- a/test/test_companion_uncached_storage.py +++ b/test/test_companion_uncached_storage.py @@ -24,6 +24,11 @@ class CompanionUncachedStorageTest(unittest.TestCase): source = (ROOT / "examples/companion_radio/DataStore.cpp").read_text() signatures = [ "static bool companionPathPresence(", + "static bool readableContactFile(", + "const char* DataStore::selectContactFile(", + "bool DataStore::prepareContactSave(", + "void DataStore::clearContactLoadFailure(", + "void DataStore::recoverContacts(", "bool DataStore::loadMainIdentity(", "bool DataStore::canCreateMainIdentity() const", "bool DataStore::saveMainIdentity(", @@ -55,6 +60,13 @@ class CompanionUncachedStorageTest(unittest.TestCase): temp = Path(temp) (temp / "store_under_test.h").write_text(generated) header = (ROOT / "examples/companion_radio/DataStore.h").read_text() + start = header.index(" // Contact-only recovery") + end = header.index("#if defined(ESP32_PLATFORM)", start) + (temp / "contact_recovery_state_under_test.h").write_text(header[start:end]) + (temp / "contact_recovery_api_under_test.h").write_text("\n".join( + extract_braced(header, signature) for signature in ( + " bool contactRecoveryPending() const", + " bool contactsAreVolatile() const"))) start = header.index(" mesh::ContactFileTransaction* _contact_write") end = header.index("\n#endif", start) (temp / "contact_write_state_under_test.h").write_text(header[start:end]) @@ -63,6 +75,8 @@ class CompanionUncachedStorageTest(unittest.TestCase): (temp / "FS.h").write_text( "#pragma once\nnamespace fs { using FS = FakeFilesystem; }\n" ) + (temp / "InternalFileSystem.h").write_text( + "#pragma once\n#define InternalFS filesystem\n") binary = temp / "test" compiled = subprocess.run([ "c++", "-std=c++17", "-O1", "-g", "-Wall", "-Wextra", diff --git a/test/test_contact_cache.py b/test/test_contact_cache.py index b75e934af..c9e9eea1c 100644 --- a/test/test_contact_cache.py +++ b/test/test_contact_cache.py @@ -47,13 +47,19 @@ class ContactCacheTest(unittest.TestCase): def test_esp32_default_crc_cadence_is_detected(self): self.run_cache_test("ESP32_PLATFORM", disable_crc_batch=True) - def run_cache_test(self, platform, disable_crc_batch=False): + def test_esp32_disabled_contact_recovery_is_detected(self): + self.run_cache_test("ESP32_PLATFORM", disable_recovery=True) + + def run_cache_test(self, platform, disable_crc_batch=False, disable_recovery=False): with tempfile.TemporaryDirectory(prefix="mesh-contact-cache-") as temp: temp = Path(temp) store = (ROOT / "examples/companion_radio/DataStore.cpp").read_text() functions = [ "static bool serializeContactRecord(", "static bool deserializeContactRecord(", + "void DataStore::clearContactLoadFailure(", + "void DataStore::recoverContacts(", + "bool DataStore::hasIncompleteContactLoad() const", "void DataStore::loadContacts(", "bool DataStore::saveContacts(", "bool DataStore::flushContactWrites(", @@ -67,6 +73,13 @@ class ContactCacheTest(unittest.TestCase): helper_end = store.index("bool DataStore::readStoredPath(", helper_start) implementation = store[helper_start:helper_end] header = (ROOT / "examples/companion_radio/DataStore.h").read_text() + recovery_start = header.index(" // Contact-only recovery") + recovery_end = header.index("#if defined(ESP32_PLATFORM)", recovery_start) + (temp / "contact_recovery_state_under_test.h").write_text( + header[recovery_start:recovery_end]) + (temp / "contact_recovery_api_under_test.h").write_text( + method(header, " bool contactRecoveryPending() const") + "\n" + + method(header, " bool contactsAreVolatile() const")) state_start = header.index(" mesh::ContactFileTransaction* _contact_write") state_end = header.index("\n#endif", state_start) (temp / "contact_write_state_under_test.h").write_text( @@ -74,6 +87,9 @@ class ContactCacheTest(unittest.TestCase): functions.append("DataStore::~DataStore()") if platform == "ESP32_PLATFORM": functions.extend(( + "static bool readableContactFile(", + "const char* DataStore::selectContactFile(", + "bool DataStore::prepareContactSave(", "void DataStore::cancelContactWrite(", "bool DataStore::cancelCooperativeWrite(", "bool DataStore::serviceContactWrite(", @@ -89,7 +105,21 @@ class ContactCacheTest(unittest.TestCase): implementation += method(store, "static void discardInvalidContactPage(") + "\n" functions.append("bool DataStore::writeContactPage(") functions.append("bool DataStore::loadContactPages(") + functions.append("void DataStore::resetContactPageState(") implementation += "\n".join(method(store, signature) for signature in functions) + if disable_recovery: + recovery = method(store, "void DataStore::recoverContacts(") + self.assertIn(recovery, implementation) + implementation = implementation.replace( + recovery, "void DataStore::recoverContacts(DataStoreHost*) {}") + if platform == "NRF52_PLATFORM": + # Fault only the page snapshot allocation; keep production + # loading/recovery logic and unrelated crypto allocations. + allocation = "(uint8_t*)malloc(mesh::storage::CONTACT_PAGE_PAYLOAD_SIZE)" + self.assertEqual(implementation.count(allocation), 1) + implementation = implementation.replace( + allocation, + "(uint8_t*)allocateContactPagePayload(mesh::storage::CONTACT_PAGE_PAYLOAD_SIZE)") if disable_crc_batch: self.assertIn("_contact_write->serviceCommit(true, 8, true)", implementation) implementation = implementation.replace( @@ -129,7 +159,10 @@ class ContactCacheTest(unittest.TestCase): ], capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) result = subprocess.run([str(binary)], capture_output=True, text=True) - if disable_crc_batch: + if disable_recovery: + self.assertNotEqual(result.returncode, 0) + self.assertIn("!f.store.contactRecoveryPending()", result.stderr) + elif disable_crc_batch: self.assertNotEqual(result.returncode, 0) self.assertIn("crc_read_passes == 104", result.stderr) else: diff --git a/test/test_esp32_advert_cache.py b/test/test_esp32_advert_cache.py index 0c703da2a..534897fbc 100644 --- a/test/test_esp32_advert_cache.py +++ b/test/test_esp32_advert_cache.py @@ -104,6 +104,12 @@ class AdvertCacheTest(unittest.TestCase): start = header.index(' struct AdvertWriteState {') end = header.index('\n#endif', start) state = header[start:end] + recovery_start = header.index(' enum : uint8_t { ContactRecoveryPending') + recovery_end = header.index(' uint8_t _contact_recovery = 0;', recovery_start) + recovery_end += len(' uint8_t _contact_recovery = 0;') + recovery_state = header[recovery_start:recovery_end] + recovery_state += '\n' + '\n'.join(method(header, signature) for signature in ( + ' bool contactRecoveryPending() const', ' bool contactsAreVolatile() const')) signatures = ('static File openWrite(', 'inline void makeBlobPath(', 'bool DataStore::queueAdvertByKey(', 'void DataStore::retireAdvertWrite(', 'void DataStore::invalidateAdvertWrite(', 'bool DataStore::consumeSynchronousAdvertIO(', @@ -127,6 +133,7 @@ class AdvertCacheTest(unittest.TestCase): mesh_source = (ROOT / 'examples/companion_radio/MyMesh.cpp').read_text() scheduling = method(mesh_source, 'void MyMesh::servicePersistence()') flush = method(mesh_source, 'bool MyMesh::flushContactsBeforeReboot()') + flushed = method(mesh_source, 'void MyMesh::onContactCacheFlushed()') work = method(mesh_source, 'bool MyMesh::hasPendingWork() const') recovery = method(mesh_source, 'bool MyMesh::canRecoverUsbLogging() const') # The entire production scheduling body runs below. This order check @@ -137,9 +144,10 @@ class AdvertCacheTest(unittest.TestCase): with tempfile.TemporaryDirectory(prefix='mesh-advert-jobs-') as directory: temp = Path(directory) (temp / 'advert_state.h').write_text(state) + (temp / 'contact_recovery.h').write_text(recovery_state) (temp / 'presence.h').write_text(presence) (temp / 'store.h').write_text(implementation) - (temp / 'mesh.h').write_text('\n'.join((route, scheduling, flush, work, recovery))) + (temp / 'mesh.h').write_text('\n'.join((route, scheduling, flushed, flush, work, recovery))) binary = temp / 'test' result = subprocess.run([ 'c++', '-std=c++17', '-O1', '-g', '-Wall', '-Wextra', diff --git a/test/test_esp32_contact_crc_batch.py b/test/test_esp32_contact_crc_batch.py index 223340a54..05e335e26 100644 --- a/test/test_esp32_contact_crc_batch.py +++ b/test/test_esp32_contact_crc_batch.py @@ -150,10 +150,29 @@ class ContactCRCBatchTest(unittest.TestCase): str(temp / "frame.cpp"), "-o", str(temp / "frame.o"), ], capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) - frames = [line.split("\t") for line in (temp / "frame.su").read_text().splitlines() + usage = (temp / "frame.su").read_text().splitlines() + frames = [line.split("\t") for line in usage if "ContactFileTransaction::serviceCommit(" in line] self.assertEqual(len(frames), 1) - self.assertEqual(int(frames[0][1]), 128) + self.assertEqual(int(frames[0][1]), 112) + # Publication is separate from the 64-byte read buffer. Include + # every known nested helper, not just the smaller caller frame. + # The old opaque fixture peaked at 128 + 48 + 32 = 208 bytes; + # the new chain retains that bound. SDK/callback frames remain + # outside this isolated contract, as does the full firmware ABI. + helper_frames = {} + for name, expected in (("finishCommit", 64), ("probe", 32), ("updateCRC32C", 32)): + found = [line.split("\t") for line in usage + if "ContactFileTransaction::" + name + "(" in line] + self.assertEqual(len(found), 1, name) + helper_frames[name] = int(found[0][1]) + self.assertEqual(helper_frames[name], expected, name) + self.assertFalse(any("ContactFileTransaction::moveState(" in line + or "ContactFileTransaction::removeIfPresent(" in line + for line in usage)) + self.assertEqual(int(frames[0][1]) + max( + helper_frames["updateCRC32C"], + helper_frames["finishCommit"] + helper_frames["probe"]), 208) nm = str(Path(compiler).with_name("xtensa-esp32s3-elf-nm")) symbols = subprocess.check_output([nm, "-S", str(temp / "frame.o")], text=True) size = next(int(line.split()[1], 16) for line in symbols.splitlines() diff --git a/test/test_esp32_contact_publication.py b/test/test_esp32_contact_publication.py index c5d1cfaa3..662568926 100644 --- a/test/test_esp32_contact_publication.py +++ b/test/test_esp32_contact_publication.py @@ -9,15 +9,21 @@ class Publication(unittest.TestCase): if negative=='cleanup': a='if (!defer_backup_cleanup)';self.assertEqual(source.count(a),1);source=source.replace(a,'if (true)') elif negative=='retirement': - a='ok = _fs->remove(_backup);';self.assertEqual(source.count(a),2);source=source.replace(a,'ok = true; // negative: leave previous backup',1) + a='ok = _fs->remove(_backup);';self.assertEqual(source.count(a),1);source=source.replace(a,'ok = true; // negative: leave previous backup',1) + elif negative=='movement': + a='if (!source && destination) return MoveState::Moved;';self.assertEqual(source.count(a),1) + source=source.replace(a,'return MoveState::Moved; // negative: ignore observed names') with tempfile.TemporaryDirectory(prefix='mesh-publication-') as folder: p=Path(folder);(p/'transaction_under_test.h').write_text(source);binary=p/'test' - result=subprocess.run(['c++','-std=c++17','-O1','-g','-Wall','-Wextra','-fsanitize=address,undefined','-fno-omit-frame-pointer','-D'+platform+'=1','-I',str(p),'-I',str(ROOT/'src'),'-I',str(ROOT/'test/fixtures/contact_cache'),'-I',str(ROOT/'test/fixtures/contact_cache/mocks'),str(FIXTURE),'-o',str(binary)],text=True,capture_output=True) + result=subprocess.run(['c++','-std=c++17','-O1','-g','-Wall','-Wextra','-fsanitize=address,undefined','-fno-sanitize-recover=all','-fno-omit-frame-pointer','-fno-pie','-no-pie','-D'+platform+'=1','-I',str(p),'-I',str(ROOT/'src'),'-I',str(ROOT/'test/fixtures/contact_cache'),'-I',str(ROOT/'test/fixtures/contact_cache/mocks'),str(FIXTURE),'-o',str(binary)],text=True,capture_output=True) self.assertEqual(result.returncode,0,result.stderr) + if negative=='movement': + healthy=subprocess.run([str(binary),'contracts'],text=True,capture_output=True) + self.assertEqual(healthy.returncode,0,healthy.stdout+healthy.stderr) result=subprocess.run([str(binary),mode],text=True,capture_output=True) if negative: self.assertNotEqual(result.returncode,0,result.stdout) - self.assertIn('SPIFFS.',result.stderr) + self.assertIn('writer.serviceCommit(true,8,deferred)' if negative=='movement' else 'SPIFFS.',result.stderr) else:self.assertEqual(result.returncode,0,result.stdout+result.stderr) def test_default_and_deferred_success_contracts(self):self.run_native('contracts') def test_retirement_and_recovery_failure_before_temp_ownership(self):self.run_native('retire') @@ -27,4 +33,22 @@ class Publication(unittest.TestCase): def test_other_platform_retains_cleanup_despite_true_arg(self):self.run_native('contracts','RP2040_PLATFORM') def test_removed_defer_guard_is_detected(self):self.run_native('contracts',negative='cleanup') def test_omitted_retirement_is_detected(self):self.run_native('retire',negative='retirement') + def test_rename_outcomes_follow_observed_names_and_remain_terminal(self): + for platform in ('ESP32_PLATFORM','RP2040_PLATFORM'): + with self.subTest(platform=platform):self.run_native('outcomes',platform) + def test_forced_move_success_is_detected_without_breaking_healthy_publication(self): + for platform in ('ESP32_PLATFORM','RP2040_PLATFORM'): + with self.subTest(platform=platform):self.run_native('outcomes',platform,negative='movement') + def test_post_move_metadata_errors_preserve_all_remaining_authorities(self): + for platform in ('ESP32_PLATFORM','RP2040_PLATFORM'): + with self.subTest(platform=platform):self.run_native('metadata',platform) + def test_ambiguous_names_never_trigger_speculative_rollback_or_cleanup(self): + for platform in ('ESP32_PLATFORM','RP2040_PLATFORM'): + with self.subTest(platform=platform):self.run_native('ambiguous',platform) + def test_missing_original_target_uses_the_same_publication_contract(self): + for platform in ('ESP32_PLATFORM','RP2040_PLATFORM'): + with self.subTest(platform=platform):self.run_native('fresh',platform) + def test_static_and_constructor_recovery_confirm_moves_without_losing_scratch(self): + for platform in ('ESP32_PLATFORM','RP2040_PLATFORM'): + with self.subTest(platform=platform):self.run_native('recovery',platform) if __name__=='__main__':unittest.main() diff --git a/test/test_esp32_cooperative_contacts.py b/test/test_esp32_cooperative_contacts.py index ec72814f4..c302a4029 100644 --- a/test/test_esp32_cooperative_contacts.py +++ b/test/test_esp32_cooperative_contacts.py @@ -29,6 +29,9 @@ class UncachedESPContactsTest(unittest.TestCase): def test_default_background_crc_cadence_is_detected(self): self.run_fixture(["-DBOARD_HAS_PSRAM=1"], disable="crc_batch") + def test_disabled_contact_recovery_is_detected(self): + self.run_fixture(["-DBOARD_HAS_PSRAM=1"], disable="recovery") + def test_bounded_buffers_reduce_backend_work_without_stack_growth(self): previous = self.run_fixture(["-DBOARD_HAS_PSRAM=1"], benchmark="previous") current = self.run_fixture(["-DBOARD_HAS_PSRAM=1"], benchmark="current") @@ -55,12 +58,24 @@ class UncachedESPContactsTest(unittest.TestCase): temp = Path(directory) source = (ROOT / "examples/companion_radio/DataStore.cpp").read_text() header = (ROOT / "examples/companion_radio/DataStore.h").read_text() + recovery_start = header.index(" // Contact-only recovery") + recovery_end = header.index("#if defined(ESP32_PLATFORM)", recovery_start) + (temp / "contact_recovery_state_under_test.h").write_text( + header[recovery_start:recovery_end]) + (temp / "contact_recovery_api_under_test.h").write_text( + method(header, " bool contactRecoveryPending() const") + "\n" + + method(header, " bool contactsAreVolatile() const")) start = header.index(" mesh::ContactFileTransaction* _contact_write") end = header.index("\n#endif", start) (temp / "contact_write_state_under_test.h").write_text(header[start:end]) signatures = ( "static bool serializeContactRecord(", "static bool deserializeContactRecord(", + "static bool readableContactFile(", + "const char* DataStore::selectContactFile(", + "bool DataStore::prepareContactSave(", + "void DataStore::clearContactLoadFailure(", + "void DataStore::recoverContacts(", "DataStore::~DataStore()", "void DataStore::cancelContactWrite(", "bool DataStore::serviceContactWrite(", @@ -75,7 +90,12 @@ class UncachedESPContactsTest(unittest.TestCase): "inline void makeBlobPath(", ) implementation = "\n".join(method(source, signature) for signature in signatures) - if disable == "begin": + if disable == "recovery": + recovery = method(source, "void DataStore::recoverContacts(") + self.assertIn(recovery, implementation) + implementation = implementation.replace( + recovery, "void DataStore::recoverContacts(DataStoreHost*) {}") + elif disable == "begin": self.assertIn('"/contacts3", companionPathPresence, true)', implementation) implementation = implementation.replace( '"/contacts3", companionPathPresence, true)', @@ -171,7 +191,8 @@ class UncachedESPContactsTest(unittest.TestCase): return measurements else: self.assertNotEqual(result.returncode, 0) - marker = ("SPIFFS.startupOperations() - before == 1" if disable == "begin" else + marker = ("!failed_boot.hasIncompleteContactLoad()" if disable == "recovery" else + "SPIFFS.startupOperations() - before == 1" if disable == "begin" else "crc_read_passes == 104" if disable == "crc_batch" else "largest_backend_" + ("write" if disable == "writer" else "read")) self.assertIn(marker, result.stderr) diff --git a/test/test_fleet_channel_runtime.py b/test/test_fleet_channel_runtime.py index 9bfff81ea..4c943724b 100644 --- a/test/test_fleet_channel_runtime.py +++ b/test/test_fleet_channel_runtime.py @@ -1015,38 +1015,59 @@ int main(int argc,char** argv){ const std::string reply=config(receiver,"set fleet.controller off"); assert(reply.rfind(false_success?"Err":"OK",0)==0); disk.false_success_rename=disk.late_publish_error=disk.fail_rollback=false; - assert(config(receiver,"get fleet.channel").find(false_success?"store=error":"store=ok")!=std::string::npos); +#if defined(RP2040_PLATFORM) + // Both public names prove that the rejected backup move left the old + // enrollment untouched. This failed local edit remains retryable. + const bool held=false; + if(false_success)assert(reply.find("retry later")!=std::string::npos); +#else + const bool held=false_success; +#endif + assert(config(receiver,"get fleet.channel").find(held?"store=error":"store=ok")!=std::string::npos); Packet packet=command(publisher,transport.clock.now,"get radio2"); const auto published=disk.files.at(Path);const unsigned writes=disk.writes; - assert(apply(receiver,disk,transport,radio,packet)==0&&disk.writes==writes&&disk.files.at(Path)==published); + const bool allowed=false_success&&!held; + assert(apply(receiver,disk,transport,radio,packet)==unsigned(allowed)); + if(!allowed)assert(disk.writes==writes&&disk.files.at(Path)==published); assert((published==previous)==false_success); FleetChannel rebooted(&disk); - assert(apply(rebooted,disk,transport,radio,packet)==unsigned(false_success)); + assert(apply(rebooted,disk,transport,radio,packet)==unsigned(false_success&&!allowed)); } }else if(scenario=="local_config_missing_public"){ - enroll(fleet,publisher);const auto previous=fs.files.at(Path);fs.false_success_publish=true; - assert(config(fleet,"set fleet.controller off").rfind("Err",0)==0); - fs.false_success_publish=false; - assert(config(fleet,"get fleet.channel").find("store=error")!=std::string::npos); + for(bool rollback_fault:{false,true}){ + fs::FS disk;Mesh transport;RadioProfileCLI radio;FleetChannel receiver(&disk);enroll(receiver,publisher); + const auto previous=disk.files.at(Path); + disk.false_success_publish=true;disk.fail_rollback=rollback_fault; + const auto reply=config(receiver,"set fleet.controller off");assert(reply.rfind("Err",0)==0); + disk.false_success_publish=disk.fail_rollback=false; #if defined(RP2040_PLATFORM) - // The fallback moved the old target to backup, then trusted the no-op - // publication and removed that backup. Fleet must quarantine the missing name. - assert(!fs.exists(Path)&&!fs.exists("/fleet_channel.bak")); + const bool held=rollback_fault; + if(held){ + assert(!disk.exists(Path)&&disk.files.at("/fleet_channel.bak")==previous); + assert(disk.exists("/fleet_channel.tmp")); + }else{ + // A no-op publication must not delete the only prior enrollment. A + // confirmed rollback keeps the old controller usable without a reboot. + assert(disk.files.at(Path)==previous&&!disk.exists("/fleet_channel.bak")); + assert(!disk.exists("/fleet_channel.tmp")); + assert(reply.find("retry later")!=std::string::npos); + } #else - assert(fs.files.at(Path)==previous); -#endif - const auto published=fs.files;const unsigned writes=fs.writes,renames=fs.renames; - Packet packet=command(publisher,mesh.clock.now,"get radio2"); - assert(apply(fleet,fs,mesh,profiles,packet)==0); - assert(config(fleet,"set fleet.channel off").find("unconfirmed")!=std::string::npos); - assert(fs.files==published&&fs.writes==writes&&fs.renames==renames); - FleetChannel rebooted(&fs); -#if defined(RP2040_PLATFORM) - assert(config(rebooted,"get fleet.channel").find("off,controller=unset,store=ok")!=std::string::npos); - assert(apply(rebooted,fs,mesh,profiles,packet)==0); -#else - assert(apply(rebooted,fs,mesh,profiles,packet)==1); + const bool held=true; + assert(disk.files.at(Path)==previous); #endif + assert(config(receiver,"get fleet.channel").find(held?"store=error":"store=ok")!=std::string::npos); + const auto published=disk.files;const unsigned writes=disk.writes,renames=disk.renames; + Packet packet=command(publisher,transport.clock.now,"get radio2"); + assert(apply(receiver,disk,transport,radio,packet)==unsigned(!held)); + if(held){ + assert(config(receiver,"set fleet.channel off").find("unconfirmed")!=std::string::npos); + assert(disk.files==published&&disk.writes==writes&&disk.renames==renames); + } + FleetChannel rebooted(&disk); + assert(config(rebooted,"get fleet.channel").find("store=ok")!=std::string::npos); + assert(apply(rebooted,disk,transport,radio,packet)==unsigned(held)); + } }else if(scenario=="local_config_staged_substitution"){ enroll(fleet,publisher);const auto previous=fs.files.at(Path); fs.substitute_temp=previous; @@ -1566,7 +1587,7 @@ class SHA256 : public SHA256Base {{ public: def test_valid_same_crc_tail_staged_substitution_cannot_acknowledge_revocation(self): self.scenario("local_config_staged_substitution") - def test_false_success_final_rename_missing_public_image_blocks_live_writes(self): + def test_noop_final_rename_restores_old_enrollment_or_retains_backup_on_rollback_failure(self): self.scenario("local_config_missing_public") def test_corrupt_short_long_unreadable_images_fail_closed_and_backup_recovers(self): diff --git a/test/test_local_cli_access.py b/test/test_local_cli_access.py index 51eaa91e4..405c2c4cf 100644 --- a/test/test_local_cli_access.py +++ b/test/test_local_cli_access.py @@ -159,6 +159,7 @@ struct MyMesh : CompanionFleetFixture { return false; } void stopContactsIterator(){} + void onContactCacheFlushed(); void cancelAllFloodRetries(){} void resetContacts(){++resets;} bool savePrefs(){++saves;return save_ok;} @@ -271,11 +272,14 @@ int main() { node.onCLICommandRecv(ContactInfo{},nullptr,0,"erase",reply); assert(node.store.erased==0); assert(node.handleCommand("erase",0,reply)); - assert(node.store.erased==1 && node.resets==1 && node.dirty_contacts_expiry==0); + assert(node.store.erased==1 && node.resets==1 && node.dirty_contacts_expiry==0 + && node.dirty_contacts_failures==0); assert(strstr(reply,"OK")); node.store.success=false; + node.dirty_contacts_expiry=2; node.dirty_contacts_failures=3; assert(node.handleCommand("erase",0,reply)); - assert(node.resets==1 && strstr(reply,"Err")); + assert(node.resets==1 && strstr(reply,"Err") && node.dirty_contacts_expiry==2 + && node.dirty_contacts_failures==3); for(const char* command:{"poweroff","shutdown"}) { const int previous=board.power_offs; node.handleCommand(command,100,reply); @@ -397,6 +401,7 @@ int main() { "static bool isCompanionRadioPrefsCommand(", "MyMesh::RadioSettingResult MyMesh::applyAndSaveTxPower(", "bool MyMesh::saveAdvertName(", + "void MyMesh::onContactCacheFlushed()", "bool MyMesh::handleDirectCommand(", "bool MyMesh::handleCommand(", "void MyMesh::onCLICommandRecv(")) for export, webconfig in ((0, False), (1, False), (0, True), (1, True)): diff --git a/test/test_nrf52_extrafs_contract.py b/test/test_nrf52_extrafs_contract.py index 6e2d4f868..32c87e128 100644 --- a/test/test_nrf52_extrafs_contract.py +++ b/test/test_nrf52_extrafs_contract.py @@ -38,6 +38,16 @@ def function_body(source: str, signature: str) -> str: class Nrf52ExtraFsContractTest(unittest.TestCase): + def test_verified_factory_erase_clears_contact_recovery_state(self): + store = STORE.read_text(encoding="utf-8") + body = function_body(store, "bool DataStore::formatFileSystem()") + self.assertEqual(body.count("_contact_recovery = 0;"), 3) + self.assertIn("if (success) _contact_recovery = 0;", body) + rp = body[body.index("const bool success = LittleFS.format();"):] + self.assertIn("if (success) {\n _contact_recovery = 0;", rp) + esp = body[body.index("if (fs_success && nvs_err == ESP_OK)"):] + self.assertTrue(esp.startswith("if (fs_success && nvs_err == ESP_OK) {\n _contact_recovery = 0;")) + def test_uncertain_journal_publication_preserves_storage_authority(self): store = STORE.read_text(encoding="utf-8") # Compile the complete production methods, including their platform branches.