diff --git a/docs/companion_contact_cache.md b/docs/companion_contact_cache.md new file mode 100644 index 000000000..b1343a50d --- /dev/null +++ b/docs/companion_contact_cache.md @@ -0,0 +1,176 @@ +# Full Companion contact caches + +Full Companions on ESP32 and nRF52 without PSRAM now keep 16 outgoing paths +and 16 shared secrets in RAM. Every contact remains in the contact table; +selecting a different contact loads its saved path as needed. This recovers +about 25.4 KiB of internal RAM in the 350-contact ESP32 qualification builds. + +PSRAM boards retain their existing complete contact table in external RAM. +Legacy USB/BLE/WiFi Companion profiles also retain their inline paths and +secrets. The build flag `MESH_CONTACT_CACHE=0` or `1` overrides this policy for +qualification; it is not a runtime feature switch. + +## Paths and persistence + +The 16 resident paths use least-recently-used replacement. Contact records +keep small handles to their saved paths, including 1-, 2-, and 3-byte hop +hashes. The existing 152-byte contact record and Companion app frame formats +are unchanged. ESP32 uses `/contacts3`; nRF52 also supports its paged contact +store and migration from `/contacts3`. An erase is not required. + +A hot path needs no flash read. A cold path is read from its existing contact +record. ESP32 reuses one open contact-file reader to avoid repeated SPIFFS +metadata scans during synchronization and saving. It closes that reader +before replacing the file or recovering a saved transaction. + +Changing a copied contact preserves the old path for a pending app response +or rollback. Dirty entries are saved before eviction. If storage cannot be +read, written, or represented completely, the operation fails instead of +silently losing a contact or sending along a different route. Reboot retries +an incomplete load from the preserved files. The path checksum also rejects +a cold path changed underneath a live handle; it does not add a new checksum +to the legacy ESP32 on-disk format. + +## Shared secrets and power + +Both platforms reuse the 16 most recent secrets from RAM. Cache misses use a +platform-specific policy: + +| Platform | Miss behavior | +| --- | --- | +| ESP32 | Recalculate the secret without reading or writing a secret file. | +| nRF52 | Try a saved LittleFS entry, then recalculate if it is unavailable. | + +On the V4, a measured SPIFFS secret lookup took 132,427 microseconds while +key exchange took 21,932 microseconds. ESP32 therefore uses recalculation on +a RAM-cache miss. It avoids secret-file writes and reduces time awake for +this measured workload. These are elapsed times, not measurements of energy +in joules; other ESP32 hardware has not been physically benchmarked. + +On a RAK3401, a LittleFS lookup took 7,812 microseconds versus 31,250 +microseconds for key exchange. The nRF52 backend therefore uses saved flash +entries to avoid repeat key exchange when a usable entry exists, including +after reboot. Saved entries match the full +peer public key and a fingerprint of the local key pair, and have a record +checksum. Importing a different private or public key invalidates old results. +A corrupt or missing saved entry is recalculated. The native tests use the +firmware's actual Ed25519 key-exchange library and verify agreement with the +peer's independently calculated secret. + +Derived secrets are expendable. Their storage keeps a reserve for contacts, +preferences and filesystem metadata; a full filesystem leaves the calculated +secret usable in RAM. nRF52 packs 56 entries into a 3,844-byte page to use its +4 KiB flash blocks efficiently. Small filesystems may not have space to persist +secrets for all 350 contacts. A miss without a usable saved entry still +performs key exchange. + +`MESH_CONTACT_SECRET_FLASH_CACHE=0` or `1` overrides the miss policy for +qualification builds. The optional ESP32 flash backend groups eight entries +per file. The normal ESP32 policy does not create or use those files. + +## Inspect the cache + +From the Full Companion's ASCII USB terminal or authenticated local CLI: + +```text +get contact.cache +``` + +Cached builds report `paths=16 secrets=16`, `miss=calculate` or `miss=flash`, +plus these boot-session counters: + +| Field | Meaning | +| --- | --- | +| `ram_hits` | Shared secret reused from RAM. | +| `flash_hits` | Shared secret loaded from flash without recalculation. | +| `calculations` | Shared-secret key exchanges performed. | +| `save_skips` | Calculated secrets that could not be persisted, including storage reserve/backoff. | + +Uncached builds report `paths=inline secrets=inline`. The normal `memory` +command on supported ESP32 Full Companions shows live heap and queue capacity. + +`get contact.cache.timing` reports the last successful flash lookup and last +key calculation in microseconds. Zero means no such operation has completed +since boot. These timings exclude secret writes and are useful for comparing +the work avoided on a valid flash hit. + +## Offline messages during mOTA + +Contact caches are independent of the offline message queue. Builds using +the shared mOTA queue retain **256 frames normally and 128 while mOTA owns +its workspace**, then restore 256. No unread frames are discarded to start +mOTA: synchronize the queue and retry if it exceeds the retained capacity. +PSRAM-backed queues keep their existing capacity and allocation policy. + +Each queue frame occupies 177 bytes. Retaining 32 instead of 128 would make +another 16,992 bytes (16.6 KiB) available inside the shared storage. It would +not increase free heap by itself: the normal 256-frame storage is statically +reserved, and the current mOTA context already fits in the loaned half. More +mOTA scratch data would have to share that space to obtain an additional RAM +saving. These changes keep the existing 128-frame mOTA limit. + +## Build the six capacity trials + +The optional environments in `platformio.nimble.ini` combine NimBLE, these +contact caches, and the shared mOTA queue at **350 contacts, 40 channels and +256 normal offline frames**. They remain outside the ordinary release matrix. +The ordinary six constrained release recipes still use their established +150-contact limits pending wider hardware qualification. + +Enable `platformio.nimble.ini` as described in the +[NimBLE trial guide](nimble_companion_trial.md), then build one environment at +a time. For example: + +```sh +OUTPUT_DIR=.releases/contact-cache-v3 bash build.sh build-firmware \ + Heltec_v3_ram_trial_companion_radio_full_nimble \ + --firmware-version v1.17.1.5-halo-keymind-cascade-cache-trial \ + --radio-preset usa-cascadia --profile cascade --standard --require-ota +``` + +| Hardware | Trial environment | +| --- | --- | +| Heltec V3 | `Heltec_v3_ram_trial_companion_radio_full_nimble` | +| Wireless Tracker | `Heltec_Wireless_Tracker_ram_trial_companion_radio_full_nimble` | +| Tracker V2, FEM on | `heltec_tracker_v2_ram_trial_companion_radio_full_femon_nimble` | +| Heltec CT62 | `Heltec_ct62_ram_trial_companion_radio_full_nimble` | +| XIAO ESP32-C3 | `Xiao_C3_ram_trial_companion_radio_full_nimble` | +| Generic ESP-NOW | `Generic_ESPNOW_ram_trial_companion_radio_full_nimble` | + +The firmware RAM guard remains enabled. A successful linked budget is distinct +from live free heap and physical Bluetooth/LoRa qualification on each board. + +## Regression checks + +```sh +python3 -B test/test_contact_cache.py +python3 -B test/test_companion_contact_persistence_contract.py +python3 -B test/test_companion_contact_stream_contract.py +python3 -B test/test_nrf52_extrafs_contract.py +python3 -B test/test_shared_mota_queue.py +python3 -B test/test_t096_full_memory.py +python3 -B test/test_firmware_ram.py +``` + +The cache tests compile the production path/secret cache code and extracted +production persistence, routing and request functions with filesystem/radio +adapters. They exercise 350 routes, eviction, snapshot rollback, page +migration, identity changes, read/write/rename faults, handle exhaustion, +packet release on a failed path read, and recovery after a simulated reset +between file renames. They do not simulate a physical flash power cut. + +`tools/hil/contact_cache_serial_stress.py` checks all 350 paths over USB, +replaces them with a different generation, and optionally exercises secret +eviction. Use an otherwise empty test node and exclusive access to its +Companion data port. `--populate` writes test contacts; `--secrets` transmits +23 directed LoRa datagrams to synthetic peers. For an ESP32 cache trial: + +```sh +python3 tools/hil/contact_cache_serial_stress.py --port /dev/ttyACM0 \ + --populate --generation 1 --secrets --expect-calculate --reboot +python3 tools/hil/contact_cache_serial_stress.py --port /dev/ttyACM0 \ + --generation 1 --secrets --expect-calculate +``` + +Use the actual device path. Omit `--expect-calculate` for the nRF52 flash +policy. The second invocation verifies the persisted paths after reboot. diff --git a/docs/firmware_memory_budget.md b/docs/firmware_memory_budget.md index f7315c65e..b518301a0 100644 --- a/docs/firmware_memory_budget.md +++ b/docs/firmware_memory_budget.md @@ -49,6 +49,11 @@ buffers. The display and simultaneous USB, Bluetooth and WiFi remain enabled. ## Release evidence and regression tests +Full Companions without PSRAM also use +[16-entry path and shared-secret caches](companion_contact_cache.md). +The optional NimBLE capacity trials retain the same RAM guards at 350 +contacts and 256 normal offline frames. + Each newly built firmware has a matching `.memory.json` report. It records the linked ELF SHA-256, available internal RAM, required RAM, largest region, and SHA-256 hashes for the actual firmware files and capability manifest. diff --git a/docs/index.md b/docs/index.md index 2f959cfd0..0703d07f6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,6 +17,7 @@ Below are a few quick start guides. - [Telemetry Decoder](./telemetry_decoder.md) - [CLI Availability by Firmware Build](./cli_build_matrix.md) - [Classic ESP32 image memory budget](./esp32_memory_budget.md) +- [Full Companion contact caches and capacity trials](./companion_contact_cache.md) - [Easy LoRa OTA: ESP32 and nRF52 firmware updates](./ota_easy.md) - [Scripted LoRa OTA: Bash and PowerShell](./lora_ota_automation.md) - [nRF52 repeater OTA with external QSPI](./ota_nrf52_qspi.md) diff --git a/docs/nimble_companion_trial.md b/docs/nimble_companion_trial.md index aba8c551c..d00a8b511 100644 --- a/docs/nimble_companion_trial.md +++ b/docs/nimble_companion_trial.md @@ -7,6 +7,9 @@ release matrix and firmware picker. See the [2026-09-08 hardware results](nimble_companion_trial_results.md) for measured RAM, MAC-policy checks, WiFi updates, and remaining qualification. +The [contact-cache guide](companion_contact_cache.md) adds six optional +350-contact trials for ESP32 boards without PSRAM, including Heltec V3. + | Hardware | Trial environment | Contacts | Channels | Offline queue | | --- | --- | ---: | ---: | ---: | | Heltec V4.2/V4.3 OLED, FEM on | `heltec_v4_2_v4_3_companion_radio_full_femon_nimble` | 350 | 40 | 512 | diff --git a/examples/companion_radio/DataStore.cpp b/examples/companion_radio/DataStore.cpp index c095025cd..3a75e0670 100644 --- a/examples/companion_radio/DataStore.cpp +++ b/examples/companion_radio/DataStore.cpp @@ -2,6 +2,10 @@ #include #include "DataStore.h" #include +#include +#if MESH_CONTACT_CACHE && defined(ESP32_PLATFORM) +#include +#endif #if defined(NRF52_PLATFORM) #include @@ -420,6 +424,9 @@ bool DataStore::removeFile(FILESYSTEM* fs, const char* filename) { } bool DataStore::formatFileSystem() { +#if MESH_CONTACT_CACHE && defined(ESP32_PLATFORM) + _contact_path_reader.close(); +#endif #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) #if defined(NRF52_PLATFORM) resetContactPageState(); @@ -440,6 +447,9 @@ bool DataStore::formatFileSystem() { // failed erase must keep the boot's load quarantine latched. if (success) { _contact_load_incomplete = false; +#if MESH_CONTACT_CACHE + _cache_load_incomplete = false; +#endif _identity_creation_blocked = false; _prefs_load_incomplete = false; } @@ -858,7 +868,7 @@ bool DataStore::savePrefs(const CompanionNodePrefs& _prefs, double node_lat, dou return false; } -static void serializeContactRecord(const ContactInfo& c, +static bool serializeContactRecord(const ContactInfo& c, uint8_t out[mesh::storage::CONTACT_RECORD_SIZE]) { size_t offset = 0; const uint8_t unused = 0; @@ -870,14 +880,18 @@ static void serializeContactRecord(const ContactInfo& c, memcpy(&out[offset], &c.sync_since, 4); offset += 4; out[offset++] = c.out_path_len; memcpy(&out[offset], &c.last_advert_timestamp, 4); offset += 4; - memcpy(&out[offset], c.out_path, 64); offset += 64; + if (!c.copyPathTo(&out[offset])) return false; + offset += 64; memcpy(&out[offset], &c.lastmod, 4); offset += 4; memcpy(&out[offset], &c.gps_lat, 4); offset += 4; memcpy(&out[offset], &c.gps_lon, 4); + return true; } static bool deserializeContactRecord( - const uint8_t in[mesh::storage::CONTACT_RECORD_SIZE], ContactInfo& c) { + const uint8_t in[mesh::storage::CONTACT_RECORD_SIZE], ContactInfo& c, + uint16_t path_source, bool* path_unavailable = nullptr) { + if (path_unavailable) *path_unavailable = false; size_t offset = 0; uint8_t pub_key[32]; memcpy(pub_key, &in[offset], 32); offset += 32; @@ -889,7 +903,16 @@ static bool deserializeContactRecord( memcpy(&c.sync_since, &in[offset], 4); offset += 4; c.out_path_len = in[offset++]; memcpy(&c.last_advert_timestamp, &in[offset], 4); offset += 4; - memcpy(c.out_path, &in[offset], 64); offset += 64; +#if MESH_CONTACT_CACHE + if (!c.path_ref.bind(path_source, &in[offset])) { + if (path_unavailable) *path_unavailable = true; + return false; + } +#else + (void)path_source; + if (!c.setRawPath(&in[offset])) return false; +#endif + offset += 64; memcpy(&c.lastmod, &in[offset], 4); offset += 4; memcpy(&c.gps_lat, &in[offset], 4); offset += 4; memcpy(&c.gps_lon, &in[offset], 4); @@ -957,7 +980,12 @@ void DataStore::resetContactPageState(bool clear_incomplete) { _contact_slots.clear(); _dirty_contact_pages.clearAll(); _unread_contact_pages.clearAll(); - if (clear_incomplete) _contact_load_incomplete = false; + if (clear_incomplete) { + _contact_load_incomplete = false; +#if MESH_CONTACT_CACHE + _cache_load_incomplete = false; +#endif + } memset(_contact_page_generations, 0, sizeof(_contact_page_generations)); _legacy_contacts_pending_cleanup = false; _legacy_migration_ready = false; @@ -1153,7 +1181,18 @@ bool DataStore::loadContactPages(DataStoreHost* host, uint16_t minimum_slot, } ContactInfo contact; - if (!deserializeContactRecord(record, contact) || !_contact_slots.reserve(slot)) { + bool path_unavailable = false; + const bool valid = deserializeContactRecord(record, contact, + 0x8000 | slot, &path_unavailable); + if (path_unavailable) { + // Cache exhaustion is not a corrupt record. Keep the complete page + // on disk and refuse changes until it can be loaded after reboot. + _contact_load_incomplete = true; + _dirty_contact_pages.clearAll(); + free(payload); + return true; + } + if (!valid || !_contact_slots.reserve(slot)) { MESH_DEBUG_PRINTLN("DataStore: contact page %u slot %u is invalid/duplicate", page, index); _dirty_contact_pages.mark(page); continue; @@ -1209,12 +1248,28 @@ bool DataStore::writeContactPage(DataStoreHost* host, uint8_t page, occupied |= 1UL << page_slot; } +#if MESH_CONTACT_CACHE + auto& paths = mesh::contactPathStorage(); + paths.beginCommit(); + for (auto* contact : page_contacts) if (contact) paths.mark(contact->path_ref.handle()); + const uint16_t first_slot = page * mesh::storage::CONTACTS_PER_PAGE; + if (!paths.preserveSnapshots(0x8000 | first_slot, mesh::storage::CONTACTS_PER_PAGE) + || !paths.preserveSnapshots(first_slot, mesh::storage::CONTACTS_PER_PAGE)) { + paths.endCommit(false); + return false; + } +#endif uint32_t payload_crc = 0xFFFFFFFFUL; uint8_t record[mesh::storage::CONTACT_RECORD_SIZE]; for (uint8_t slot = 0; slot < mesh::storage::CONTACTS_PER_PAGE; slot++) { memset(record, 0, sizeof(record)); if (page_contacts[slot] != nullptr) { - serializeContactRecord(*page_contacts[slot], record); + if (!serializeContactRecord(*page_contacts[slot], record)) { +#if MESH_CONTACT_CACHE + mesh::contactPathStorage().endCommit(false); +#endif + return false; + } } payload_crc = mesh::storage::updateCRC32(payload_crc, record, sizeof(record)); } @@ -1235,21 +1290,50 @@ bool DataStore::writeContactPage(DataStoreHost* host, uint8_t page, for (uint8_t slot = 0; wrote && slot < mesh::storage::CONTACTS_PER_PAGE; slot++) { memset(record, 0, sizeof(record)); if (page_contacts[slot] != nullptr) { - serializeContactRecord(*page_contacts[slot], record); + if (!serializeContactRecord(*page_contacts[slot], record)) { +#if MESH_CONTACT_CACHE + mesh::contactPathStorage().endCommit(false); +#endif + return false; + } } wrote = writer.write(record, sizeof(record)) == sizeof(record); } if (!writer.commit(wrote)) { +#if MESH_CONTACT_CACHE + paths.endCommit(false); +#endif MESH_DEBUG_PRINTLN("DataStore: atomic contact page %u write failed", page); return false; } +#if MESH_CONTACT_CACHE + for (auto* contact : page_contacts) if (contact) + paths.publish(contact->path_ref.handle(), 0x8000 | contact->storage_slot); + paths.endCommit(true); +#endif _contact_page_generations[page] = header.generation; return true; } #endif void DataStore::loadContacts(DataStoreHost* host) { +#if MESH_CONTACT_CACHE + _cache_host = host; + mesh::contactPathStorage().attach(this); +#if MESH_CONTACT_SECRET_FLASH_CACHE + mesh::contactSecretCache().attach(this); +#endif + if (_cache_load_incomplete) return; +#if defined(ESP32_PLATFORM) + _contact_path_reader.close(); + if (!mesh::ContactFileTransaction::recover(_getContactsChannelsFS(), "/contacts3")) { + MESH_DEBUG_PRINTLN("DataStore: contact transaction recovery failed"); + _cache_load_incomplete = true; + return; + } +#endif +#endif #if defined(NRF52_PLATFORM) // loadContacts() is also used after an identity import. Rebuild runtime // slot ownership from disk so stale pointers/slots from the previous in-RAM @@ -1359,6 +1443,12 @@ void DataStore::loadContacts(DataStoreHost* host) { #endif File file = openRead(_getContactsChannelsFS(), "/contacts3"); +#if MESH_CONTACT_CACHE && defined(ESP32_PLATFORM) + if (!file && _getContactsChannelsFS()->exists("/contacts3")) { + _cache_load_incomplete = true; + return; + } +#endif #if defined(NRF52_PLATFORM) if (!file) { // Never delete, truncate, or write alongside a legacy source that could @@ -1381,20 +1471,33 @@ void DataStore::loadContacts(DataStoreHost* host) { #endif if (file) { bool full = false; + uint16_t record_index = 0; +#if MESH_CONTACT_CACHE && defined(ESP32_PLATFORM) + if (file.size() % mesh::storage::CONTACT_RECORD_SIZE != 0) { + _cache_load_incomplete = true; + file.close(); + return; + } +#endif #if defined(NRF52_PLATFORM) _legacy_contact_count = mesh::storage::legacyContactCountForSize(legacy_size); - uint16_t record_index = 0; bool legacy_read_failed = false; bool legacy_host_refused = false; #endif while (!full +#if MESH_CONTACT_CACHE && defined(ESP32_PLATFORM) + && record_index < file.size() / mesh::storage::CONTACT_RECORD_SIZE +#endif #if defined(NRF52_PLATFORM) && record_index < _legacy_contact_count #endif ) { uint8_t record[mesh::storage::CONTACT_RECORD_SIZE]; if (file.read(record, sizeof(record)) != sizeof(record)) { +#if MESH_CONTACT_CACHE + _cache_load_incomplete = true; +#endif #if defined(NRF52_PLATFORM) legacy_read_failed = true; #endif @@ -1402,20 +1505,34 @@ void DataStore::loadContacts(DataStoreHost* host) { } ContactInfo contact; - if (!deserializeContactRecord(record, contact)) { + bool path_unavailable = false; + if (!deserializeContactRecord(record, contact, record_index, &path_unavailable)) { + if (path_unavailable) { +#if MESH_CONTACT_CACHE + _cache_load_incomplete = true; +#endif +#if defined(NRF52_PLATFORM) + legacy_read_failed = true; +#endif + break; + } // Preserve the contact while containing corrupt legacy routing data. contact.out_path_len = OUT_PATH_UNKNOWN; } #if defined(NRF52_PLATFORM) - const uint16_t slot = record_index++; + const uint16_t slot = record_index; if (!_contact_slots.reserve(slot)) { legacy_read_failed = true; break; } contact.storage_slot = slot; #endif + ++record_index; if (!host->onContactLoaded(contact)) { full = true; +#if MESH_CONTACT_CACHE + _cache_load_incomplete = true; +#endif #if defined(NRF52_PLATFORM) _contact_slots.release(slot); legacy_host_refused = true; @@ -1470,6 +1587,7 @@ void DataStore::loadContacts(DataStoreHost* host) { } bool DataStore::saveContacts(DataStoreHost* host, bool (*filter)(const ContactInfo& c)) { + if (hasIncompleteContactLoad()) return false; #if defined(NRF52_PLATFORM) bool success = true; for (uint32_t idx = 0;; idx++) { @@ -1479,6 +1597,44 @@ bool DataStore::saveContacts(DataStoreHost* host, bool (*filter)(const ContactIn success = markContactDirty(*contact) && success; } return flushContactWrites(host, filter) && success; +#elif MESH_CONTACT_CACHE && defined(ESP32_PLATFORM) + auto& paths = mesh::contactPathStorage(); + paths.beginCommit(); + for (uint32_t i = 0;; ++i) { + auto* c = host->getContactForStore(i); + if (!c) break; + if (!filter || filter(*c)) paths.mark(c->path_ref.handle()); + } + if (!paths.preserveSnapshots(0, 0x8000)) { + paths.endCommit(false); + return false; + } + // The cold-path reader reuses one SPIFFS File during this streaming write. + // Close it before replacing the original name so later reads reopen the + // committed file. No complete contact-table copy is needed in RAM. + mesh::ContactFileTransaction writer(_getContactsChannelsFS(), "/contacts3"); + bool success = writer; + uint8_t record[mesh::storage::CONTACT_RECORD_SIZE]; + for (uint32_t i = 0; success; ++i) { + auto* c = host->getContactForStore(i); + if (!c) break; + if (filter && !filter(*c)) continue; + success = serializeContactRecord(*c, record) + && writer.write(record, sizeof(record)) == sizeof(record); + } + _contact_path_reader.close(); + success = writer.commit(success); + if (success) { + uint16_t index = 0; + for (uint32_t i = 0;; ++i) { + auto* c = host->getContactForStore(i); + if (!c) break; + if (filter && !filter(*c)) continue; + paths.publish(c->path_ref.handle(), index++); + } + } + paths.endCommit(success); + return success; #else File file = openWrite(_getContactsChannelsFS(), "/contacts3"); bool success = (bool)file; @@ -1500,7 +1656,8 @@ bool DataStore::saveContacts(DataStoreHost* host, bool (*filter)(const ContactIn success = success && (file.write((uint8_t *)&c.sync_since, 4) == 4); success = success && (file.write((uint8_t *)&c.out_path_len, 1) == 1); success = success && (file.write((uint8_t *)&c.last_advert_timestamp, 4) == 4); - success = success && (file.write(c.out_path, 64) == 64); + uint8_t path[64]; + success = success && c.copyPathTo(path) && (file.write(path, 64) == 64); success = success && (file.write((uint8_t *)&c.lastmod, 4) == 4); success = success && (file.write((uint8_t *)&c.gps_lat, 4) == 4); success = success && (file.write((uint8_t *)&c.gps_lon, 4) == 4); @@ -1667,6 +1824,7 @@ bool DataStore::serviceContactWrites(DataStoreHost* host, bool DataStore::flushContactWrites(DataStoreHost* host, bool (*filter)(const ContactInfo& c)) { + if (hasIncompleteContactLoad()) return false; #if defined(NRF52_PLATFORM) while (hasPendingContactWrites()) { if (!serviceContactWrites(host, filter)) return false; @@ -1687,6 +1845,9 @@ bool DataStore::hasPendingContactWrites() const { } bool DataStore::hasIncompleteContactLoad() const { +#if MESH_CONTACT_CACHE + if (_cache_load_incomplete) return true; +#endif #if defined(NRF52_PLATFORM) return _contact_load_incomplete || !_unread_contact_pages.empty(); #else @@ -2719,3 +2880,183 @@ bool DataStore::deleteBlobByKey(const uint8_t key[], int key_len) { return true; // return true even if file did not exist } #endif + +#if MESH_CONTACT_CACHE +namespace { +bool cachedContactFilter(const ContactInfo& c) { return c.type != ADV_TYPE_NONE; } +#if MESH_CONTACT_SECRET_FLASH_CACHE +#if defined(NRF52_PLATFORM) +// ExtraFS allocates 4 KiB data blocks. Pack derived keys into almost a full +// block; tiny pages would consume the available flash after only a few peers. +constexpr size_t SECRET_PAGE_SLOTS = 56; +#else +constexpr size_t SECRET_PAGE_SLOTS = 8; +#endif +constexpr size_t SECRET_HEADER_SIZE = 36; +constexpr size_t SECRET_RECORD_SIZE = 68; +void secretPagePath(uint16_t slot, char path[24]) { + snprintf(path, 24, "/csecret%03x", slot / SECRET_PAGE_SLOTS); +} +void secretHeader(uint8_t header[SECRET_HEADER_SIZE], const uint8_t identity[32]) { + memcpy(header, "MCS1", 4); + memcpy(header + 4, identity, 32); +} +bool validSecretRecord(const uint8_t record[SECRET_RECORD_SIZE]) { + return mesh::storage::readLE32(record + 64) == + mesh::storage::updateCRC32(0xffffffff, record, 64); +} +#endif +} + +bool DataStore::readStoredPath(uint16_t source, uint8_t path[64]) { + if (source == mesh::ContactPathStorage::NONE || hasIncompleteContactLoad()) return false; + char filename[24] = "/contacts3"; + size_t offset = static_cast(source) * mesh::storage::CONTACT_RECORD_SIZE; +#if defined(ESP32_PLATFORM) + if (!(source & mesh::ContactPathStorage::PAGED)) { + // SPIFFS open scans metadata. Doing it for every contact made bulk saves + // and app synchronization take tens of seconds. Share one read handle; + // loadContacts/saveContacts close it before recovery or replacement. + if (!_contact_path_reader) + _contact_path_reader = openRead(_getContactsChannelsFS(), filename); + File& file = _contact_path_reader; + const bool ok = file && file.size() >= offset + mesh::storage::CONTACT_RECORD_SIZE + && file.seek(offset + 76) && file.read(path, 64) == 64; + if (!ok) file.close(); + return ok; + } +#endif + if (source & mesh::ContactPathStorage::PAGED) { +#if defined(NRF52_PLATFORM) + const uint16_t slot = source & ~mesh::ContactPathStorage::PAGED; + if (slot >= mesh::storage::CONTACT_PAGE_COUNT * mesh::storage::CONTACTS_PER_PAGE) return false; + makeContactPagePath(slot / mesh::storage::CONTACTS_PER_PAGE, filename); + offset = mesh::storage::CONTACT_PAGE_HEADER_SIZE + + (slot % mesh::storage::CONTACTS_PER_PAGE) * mesh::storage::CONTACT_RECORD_SIZE; +#else + return false; +#endif + } + File file = openRead(_getContactsChannelsFS(), filename); + const bool ok = file && file.size() >= offset + mesh::storage::CONTACT_RECORD_SIZE + && file.seek(offset + 76) && file.read(path, 64) == 64; + if (file) file.close(); + // The pool checks the CRC captured when this immutable path value was bound. + return ok; +} + +bool DataStore::flushCachedPaths() { + if (!_cache_host || hasIncompleteContactLoad() + || !flushContactWrites(_cache_host, cachedContactFilter)) return false; + _cache_host->onContactCacheFlushed(); + return true; +} + +#if MESH_CONTACT_SECRET_FLASH_CACHE +uint16_t DataStore::secretSlot(const uint8_t peer[32]) const { + if (!_cache_host || hasIncompleteContactLoad()) return mesh::storage::CONTACT_SLOT_NONE; + uint16_t persistent_index = 0; + for (uint32_t i = 0;; ++i) { + const auto* c = _cache_host->getContactForStore(i); + if (!c) break; + if (c->type == ADV_TYPE_NONE) continue; + if (memcmp(c->id.pub_key, peer, 32) == 0) { +#if defined(NRF52_PLATFORM) + return c->storage_slot; +#else + return persistent_index; +#endif + } + ++persistent_index; + } + return mesh::storage::CONTACT_SLOT_NONE; +} + +bool DataStore::readSavedSecret(const uint8_t peer[32], const uint8_t identity[32], + uint8_t secret[32]) { + const uint16_t slot = secretSlot(peer); + if (slot == mesh::storage::CONTACT_SLOT_NONE) return false; + char path[24]; + secretPagePath(slot, path); +#if defined(ESP32_PLATFORM) + if (!mesh::ContactFileTransaction::recover(_getContactsChannelsFS(), path)) return false; +#endif + File file = openRead(_getContactsChannelsFS(), path); + uint8_t expected[SECRET_HEADER_SIZE], header[SECRET_HEADER_SIZE]; + secretHeader(expected, identity); + uint8_t record[SECRET_RECORD_SIZE]; + const bool ok = file && file.size() == SECRET_HEADER_SIZE + SECRET_PAGE_SLOTS * SECRET_RECORD_SIZE + && file.read(header, sizeof(header)) == sizeof(header) + && memcmp(header, expected, sizeof(header)) == 0 + && file.seek(SECRET_HEADER_SIZE + (slot % SECRET_PAGE_SLOTS) * SECRET_RECORD_SIZE) + && file.read(record, sizeof(record)) == sizeof(record) + && memcmp(record, peer, 32) == 0 && validSecretRecord(record); + if (file) file.close(); + if (ok) memcpy(secret, record + 32, 32); + return ok; +} + +bool DataStore::saveSecret(const uint8_t peer[32], const uint8_t identity[32], + const uint8_t secret[32]) { + const uint16_t slot = secretSlot(peer); + if (slot == mesh::storage::CONTACT_SLOT_NONE || + (_secret_retry_at && static_cast(millis() - _secret_retry_at) < 0)) return false; + + // Derived keys are expendable. Keep room for contact replacement, preferences + // and filesystem metadata instead of letting this cache prevent a real save. + size_t free_bytes = 0; + size_t reserve_bytes = 0; +#if defined(NRF52_PLATFORM) + const auto* config = _getContactsChannelsFS()->_getFS()->cfg; + const int used = _getLfsUsedBlockCount(_getContactsChannelsFS()); + if (used < 0 || static_cast(used) > config->block_count) return false; + free_bytes = (config->block_count - used) * config->block_size; + reserve_bytes = 4 * config->block_size; +#else + const size_t total = SPIFFS.totalBytes(), used = SPIFFS.usedBytes(); + if (used > total) return false; + free_bytes = total - used; + reserve_bytes = MAX_CONTACTS * mesh::storage::CONTACT_RECORD_SIZE + 16384; +#endif + if (free_bytes < reserve_bytes + 2 * (SECRET_HEADER_SIZE + SECRET_PAGE_SLOTS * SECRET_RECORD_SIZE)) { + _secret_retry_at = millis() + 30000; + return false; + } + + char path[24]; + secretPagePath(slot, path); + uint8_t header[SECRET_HEADER_SIZE], previous_header[SECRET_HEADER_SIZE]; + secretHeader(header, identity); +#if defined(ESP32_PLATFORM) + if (!mesh::ContactFileTransaction::recover(_getContactsChannelsFS(), path)) return false; +#endif + File old = openRead(_getContactsChannelsFS(), path); + bool reuse = old && old.size() == SECRET_HEADER_SIZE + SECRET_PAGE_SLOTS * SECRET_RECORD_SIZE + && old.read(previous_header, sizeof(previous_header)) == sizeof(previous_header) + && memcmp(previous_header, header, sizeof(header)) == 0; +#if defined(NRF52_PLATFORM) + mesh::AtomicFileWriter writer(_getContactsChannelsFS(), path); +#else + mesh::ContactFileTransaction writer(_getContactsChannelsFS(), path); +#endif + bool ok = writer && writer.write(header, sizeof(header)) == sizeof(header); + uint8_t record[SECRET_RECORD_SIZE]; + for (size_t i = 0; ok && i < SECRET_PAGE_SLOTS; ++i) { + memset(record, 0, sizeof(record)); + if (reuse && (old.read(record, sizeof(record)) != sizeof(record) || !validSecretRecord(record))) + memset(record, 0, sizeof(record)); + if (i == slot % SECRET_PAGE_SLOTS) { + memcpy(record, peer, 32); + memcpy(record + 32, secret, 32); + mesh::storage::writeLE32(record + 64, + mesh::storage::updateCRC32(0xffffffff, record, 64)); + } + ok = writer.write(record, sizeof(record)) == sizeof(record); + } + if (old) old.close(); + ok = writer.commit(ok); + if (!ok) _secret_retry_at = millis() + 30000; + return ok; +} +#endif +#endif diff --git a/examples/companion_radio/DataStore.h b/examples/companion_radio/DataStore.h index d41677949..576ff3ddd 100644 --- a/examples/companion_radio/DataStore.h +++ b/examples/companion_radio/DataStore.h @@ -5,17 +5,28 @@ #include #include #include "NodePrefs.h" +#if MESH_CONTACT_CACHE +#include +#endif class DataStoreHost { public: virtual bool onContactLoaded(const ContactInfo& contact) =0; virtual bool getContactForSave(uint32_t idx, ContactInfo& contact) =0; virtual ContactInfo* getContactForStore(uint32_t idx) =0; + virtual void onContactCacheFlushed() {} virtual bool onChannelLoaded(uint8_t channel_idx, const ChannelDetails& ch) =0; virtual bool getChannelForSave(uint8_t channel_idx, ChannelDetails& ch) =0; }; -class DataStore { +class DataStore +#if MESH_CONTACT_CACHE + : public mesh::ContactPathBackend +#if MESH_CONTACT_SECRET_FLASH_CACHE + , public mesh::ContactSecretBackend +#endif +#endif +{ FILESYSTEM* _fs; FILESYSTEM* _fsExtra; // Keep the configured secondary even when normal I/O falls back to the @@ -24,6 +35,23 @@ class DataStore { FILESYSTEM* _configuredFsExtra; mesh::RTCClock* _clock; IdentityStore identity_store; +#if MESH_CONTACT_CACHE + DataStoreHost* _cache_host = nullptr; + bool _cache_load_incomplete = false; +#if defined(ESP32_PLATFORM) + File _contact_path_reader; +#endif + bool readStoredPath(uint16_t source, uint8_t path[64]) override; + bool flushCachedPaths() override; +#if MESH_CONTACT_SECRET_FLASH_CACHE + uint32_t _secret_retry_at = 0; + bool readSavedSecret(const uint8_t peer[32], const uint8_t identity[32], + uint8_t secret[32]) override; + bool saveSecret(const uint8_t peer[32], const uint8_t identity[32], + const uint8_t secret[32]) override; + uint16_t secretSlot(const uint8_t peer[32]) const; +#endif +#endif #if defined(NRF52_PLATFORM) mesh::storage::ContactSlotMap _contact_slots; diff --git a/examples/companion_radio/MyMesh.cpp b/examples/companion_radio/MyMesh.cpp index 02b39fc1e..25b62c485 100644 --- a/examples/companion_radio/MyMesh.cpp +++ b/examples/companion_radio/MyMesh.cpp @@ -323,7 +323,7 @@ bool MyMesh::writeContactRespFrame(uint8_t code, const ContactInfo &contact) { out_frame[i++] = contact.type; out_frame[i++] = contact.flags; out_frame[i++] = contact.out_path_len; - memcpy(&out_frame[i], contact.out_path, MAX_PATH_SIZE); + if (!contact.copyPathTo(&out_frame[i])) return false; i += MAX_PATH_SIZE; StrHelper::strzcpy((char *)&out_frame[i], contact.name, 32); i += 32; @@ -339,6 +339,7 @@ bool MyMesh::writeContactRespFrame(uint8_t code, const ContactInfo &contact) { } void MyMesh::stopContactsIterator() { + _iter_pending_contact = ContactInfo(); if (!_iter_started) return; _iter_started = false; _iter_start_pending = false; @@ -383,7 +384,7 @@ bool MyMesh::updateContactFromFrame(ContactInfo &contact, uint32_t& last_mod, co contact.type = frame[i++]; contact.flags = frame[i++]; contact.out_path_len = frame[i++]; - memcpy(contact.out_path, &frame[i], MAX_PATH_SIZE); + if (!contact.setRawPath(&frame[i])) return false; i += MAX_PATH_SIZE; memcpy(contact.name, &frame[i], 32); contact.name[sizeof(contact.name) - 1] = 0; @@ -2326,6 +2327,29 @@ bool MyMesh::handleLocalControlCommand(const char* command, char* reply, return true; } + if (strcmp(command, "get contact.cache.timing") == 0) { +#if MESH_CONTACT_CACHE + const auto& cache = mesh::contactSecretCache(); + snprintf(reply, reply_size, "last_flash_us=%lu last_calculation_us=%lu", + (unsigned long)cache.last_flash_read_us, + (unsigned long)cache.last_calculation_us); +#else + snprintf(reply, reply_size, "contact cache disabled"); +#endif + return true; + } + if (strcmp(command, "get contact.cache") == 0) { +#if MESH_CONTACT_CACHE + const auto& cache = mesh::contactSecretCache(); + snprintf(reply, reply_size, "paths=16 secrets=16 miss=%s ram_hits=%lu flash_hits=%lu calculations=%lu save_skips=%lu", + MESH_CONTACT_SECRET_FLASH_CACHE ? "flash" : "calculate", + (unsigned long)cache.ram_hits, (unsigned long)cache.flash_hits, + (unsigned long)cache.calculations, (unsigned long)cache.save_failures); +#else + snprintf(reply, reply_size, "paths=inline secrets=inline"); +#endif + return true; + } #if COMPANION_FEATURE_MEMORY_DIAGNOSTICS if (strcmp(command, "memory") == 0) { const mesh::CompanionMemoryDiagnostics diagnostics = { @@ -5217,6 +5241,12 @@ static bool save_filter(const ContactInfo& c) { return c.type != ADV_TYPE_NONE; // don't save the transient/anon entries } +void MyMesh::onContactCacheFlushed() { + // Eviction already saved the pending mutations. Avoid writing them again + // when the old five-second lazy-save deadline arrives. + mesh::resetLazyPersistenceAfterSuccess(dirty_contacts_expiry, dirty_contacts_failures); +} + void MyMesh::saveContacts() { const bool success = _store->saveContacts(this, save_filter); if (!success) { @@ -6583,10 +6613,15 @@ void MyMesh::printTerminalPath(const ContactInfo& recipient) { return; } + uint8_t path_bytes[MAX_PATH_SIZE]; + if (!recipient.copyPathTo(path_bytes)) { + terminalOutput().print("unavailable (storage read failed)\r\n"); + return; + } for (uint8_t hop = 0; hop < hop_count; hop++) { if (hop != 0) terminalOutput().print(','); mesh::Utils::printHex(terminalOutput(), - &recipient.out_path[(size_t)hop * hash_size], + &path_bytes[(size_t)hop * hash_size], hash_size); } terminalOutput().printf(" (%u %s, %u-byte hashes; used by DIRECT sends)\r\n", @@ -6604,7 +6639,7 @@ void MyMesh::handleTerminalPath(ContactInfo& recipient, mesh::cli::TerminalPath path; const mesh::cli::TerminalPathParseResult parsed = mesh::cli::parseTerminalPath(path_spec, _terminal_tmp_buf, - sizeof(recipient.out_path), 63, path); + MAX_PATH_SIZE, 63, path); switch (parsed) { case mesh::cli::TerminalPathParseResult::Valid: break; @@ -6629,14 +6664,9 @@ void MyMesh::handleTerminalPath(ContactInfo& recipient, } const ContactInfo previous = recipient; - memset(recipient.out_path, 0, sizeof(recipient.out_path)); - if (path.mode == mesh::cli::TerminalPathMode::Clear) { - recipient.out_path_len = OUT_PATH_UNKNOWN; - } else { - recipient.out_path_len = mesh::Packet::copyPath( - recipient.out_path, _terminal_tmp_buf, path.encoded_len); - } - if (!scheduleContactWrite(recipient)) { + if (!recipient.setPath(_terminal_tmp_buf, + path.mode == mesh::cli::TerminalPathMode::Clear ? OUT_PATH_UNKNOWN : path.encoded_len) + || !scheduleContactWrite(recipient)) { recipient = previous; terminalOutput().print( " ERROR: contact storage is unavailable; reboot and retry\r\n"); @@ -6793,10 +6823,13 @@ void MyMesh::printTerminalSendStatus(const char* operation, } else { const uint8_t hash_size = (recipient.out_path_len >> 6) + 1; terminalOutput().print("DIRECT via path "); - for (uint8_t hop = 0; hop < hop_count; hop++) { + uint8_t path_bytes[MAX_PATH_SIZE]; + const bool path_ready = recipient.copyPathTo(path_bytes); + if (!path_ready) terminalOutput().print("unavailable"); + for (uint8_t hop = 0; path_ready && hop < hop_count; hop++) { if (hop != 0) terminalOutput().print(','); mesh::Utils::printHex( - terminalOutput(), &recipient.out_path[(size_t)hop * hash_size], hash_size); + terminalOutput(), &path_bytes[(size_t)hop * hash_size], hash_size); } terminalOutput().printf("; %u %s, %u-byte hashes", (unsigned)hop_count, hop_count == 1 ? "hop" : "hops", (unsigned)hash_size); @@ -7052,8 +7085,9 @@ void MyMesh::sendTerminalTrace(ContactInfo& recipient) { const bool include_endpoint = recipient.type == ADV_TYPE_REPEATER || recipient.type == ADV_TYPE_ROOM; mesh::RoundTripTracePath route; - if (!mesh::buildRoundTripTracePath( - recipient.out_path, recipient.out_path_len, recipient.id.pub_key, + uint8_t path_bytes[MAX_PATH_SIZE]; + if (!recipient.copyPathTo(path_bytes) || !mesh::buildRoundTripTracePath( + path_bytes, recipient.out_path_len, recipient.id.pub_key, include_endpoint, _terminal_tmp_buf, MAX_PACKET_PAYLOAD - 9, route)) { terminalOutput().print(" ERROR: recipient has no traceable round-trip path\r\n"); diff --git a/examples/companion_radio/MyMesh.h b/examples/companion_radio/MyMesh.h index 64309997e..3b1162695 100644 --- a/examples/companion_radio/MyMesh.h +++ b/examples/companion_radio/MyMesh.h @@ -305,6 +305,7 @@ protected: bool onContactLoaded(const ContactInfo& contact) override { return addContact(contact); } bool getContactForSave(uint32_t idx, ContactInfo& contact) override { return getContactByIdx(idx, contact); } ContactInfo* getContactForStore(uint32_t idx) override { return getContactPtrByIdx(idx); } + void onContactCacheFlushed() override; bool onChannelLoaded(uint8_t channel_idx, const ChannelDetails& ch) override { return setChannel(channel_idx, ch); } bool getChannelForSave(uint8_t channel_idx, ChannelDetails& ch) override { return getChannel(channel_idx, ch); } diff --git a/platformio.nimble.ini b/platformio.nimble.ini index 68470bc0f..c5a1fee39 100644 --- a/platformio.nimble.ini +++ b/platformio.nimble.ini @@ -28,3 +28,81 @@ lib_deps = ${env:Xiao_S3_WIO_companion_radio_wifi.lib_deps} ${companion_nimble_trial.lib_deps} lib_ignore = ${companion_nimble_trial.lib_ignore} + +; Non-PSRAM Full Companions: compare at the original 350/40/256 capacity. +; The ram_trial infix keeps the ordinary release's 150-contact overlay out of +; these explicit qualification profiles. Firmware RAM guards remain enabled. +[companion_nimble_ram_trial] +build_flags = + ${companion_nimble_trial.build_flags} + -D MESH_CONTACT_CACHE=1 + -D MAX_CONTACTS=350 + -D MAX_GROUP_CHANNELS=40 + -D OFFLINE_QUEUE_SIZE=256 + -D OTA_SHARED_COMPANION_QUEUE=1 + -D MESHCORE_REQUIRES_COMPANION_RADIO_FULL=1 + +[env:Heltec_v3_ram_trial_companion_radio_full_nimble] +extends = env:Heltec_v3_companion_radio_full +board_build.flash_mode = dio +build_flags = + ${env:Heltec_v3_companion_radio_full.build_flags} + ${companion_nimble_ram_trial.build_flags} +lib_deps = + ${env:Heltec_v3_companion_radio_full.lib_deps} + ${companion_nimble_trial.lib_deps} +lib_ignore = ${companion_nimble_trial.lib_ignore} + +[env:Heltec_Wireless_Tracker_ram_trial_companion_radio_full_nimble] +extends = env:Heltec_Wireless_Tracker_companion_radio_ble +board_build.flash_mode = dio +build_flags = + ${env:Heltec_Wireless_Tracker_companion_radio_ble.build_flags} + ${companion_nimble_ram_trial.build_flags} +lib_deps = + ${env:Heltec_Wireless_Tracker_companion_radio_ble.lib_deps} + ${companion_nimble_trial.lib_deps} +lib_ignore = ${companion_nimble_trial.lib_ignore} + +[env:heltec_tracker_v2_ram_trial_companion_radio_full_femon_nimble] +extends = env:heltec_tracker_v2_companion_radio_full_femon +board_build.flash_mode = dio +build_flags = + ${env:heltec_tracker_v2_companion_radio_full_femon.build_flags} + ${companion_nimble_ram_trial.build_flags} +lib_deps = + ${env:heltec_tracker_v2_companion_radio_full_femon.lib_deps} + ${companion_nimble_trial.lib_deps} +lib_ignore = ${companion_nimble_trial.lib_ignore} + +[env:Heltec_ct62_ram_trial_companion_radio_full_nimble] +extends = env:Heltec_ct62_companion_radio_ble +build_flags = + ${env:Heltec_ct62_companion_radio_ble.build_flags} + ${companion_nimble_ram_trial.build_flags} +lib_deps = + ${env:Heltec_ct62_companion_radio_ble.lib_deps} + ${companion_nimble_trial.lib_deps} +lib_ignore = ${companion_nimble_trial.lib_ignore} + +[env:Xiao_C3_ram_trial_companion_radio_full_nimble] +extends = env:Xiao_C3_companion_radio_wifi +build_flags = + ${env:Xiao_C3_companion_radio_wifi.build_flags} + ${companion_nimble_ram_trial.build_flags} +lib_deps = + ${env:Xiao_C3_companion_radio_wifi.lib_deps} + ${companion_nimble_trial.lib_deps} +lib_ignore = ${companion_nimble_trial.lib_ignore} + +[env:Generic_ESPNOW_ram_trial_companion_radio_full_nimble] +extends = env:Generic_ESPNOW_comp_radio_usb +build_flags = + ${env:Generic_ESPNOW_comp_radio_usb.build_flags} + ${companion_nimble_ram_trial.build_flags} + -D MESH_ESPNOW_RADIO=1 + -D MESH_ESPNOW_CHANNEL=1 +lib_deps = + ${env:Generic_ESPNOW_comp_radio_usb.lib_deps} + ${companion_nimble_trial.lib_deps} +lib_ignore = ${companion_nimble_trial.lib_ignore} diff --git a/src/helpers/BaseChatMesh.cpp b/src/helpers/BaseChatMesh.cpp index 1ac3f9024..521b9595d 100644 --- a/src/helpers/BaseChatMesh.cpp +++ b/src/helpers/BaseChatMesh.cpp @@ -44,7 +44,15 @@ bool BaseChatMesh::initializeContactStorage() { return false; } +#if MESH_CONTACT_CACHE + for (int i = 0; i < requested_capacity; ++i) new (&expanded_contacts[i]) ContactInfo(); + for (int i = 0; i < num_contacts; ++i) { + expanded_contacts[i] = contacts[i]; + contacts[i] = ContactInfo(); + } +#else memcpy(expanded_contacts, contacts, sizeof(ContactInfo) * num_contacts); +#endif contacts = expanded_contacts; sort_array = expanded_sort_array; contact_capacity = requested_capacity; @@ -84,12 +92,12 @@ void BaseChatMesh::sendAckTo(const ContactInfo& dest, const uint8_t* ack_hash, u uint32_t d = TXT_ACK_DELAY; if (getExtraAckTransmitCount() > 0) { mesh::Packet* a1 = createMultiAck(ack_hash, ack_len, 1); - if (a1) sendDirect(a1, dest.out_path, dest.out_path_len, d); + if (a1) sendDirect(a1, dest.getPath(), dest.out_path_len, d); d += 300; } mesh::Packet* a2 = createAck(ack_hash, ack_len); - if (a2) sendDirect(a2, dest.out_path, dest.out_path_len, d); + if (a2) sendDirect(a2, dest.getPath(), dest.out_path_len, d); } } @@ -154,7 +162,7 @@ ContactInfo* BaseChatMesh::allocateContactSlot(bool transient_only) { } void BaseChatMesh::populateContactFromAdvert(ContactInfo& ci, const mesh::Identity& id, const AdvertDataParser& parser, uint32_t timestamp) { - memset(&ci, 0, sizeof(ci)); + ci = ContactInfo(); #if defined(NRF52_PLATFORM) ci.storage_slot = mesh::storage::CONTACT_SLOT_NONE; #endif @@ -205,7 +213,7 @@ void BaseChatMesh::onAdvertRecv(mesh::Packet* packet, const mesh::Identity& id, } if (from && from->type == ADV_TYPE_NONE) { // already in contacts, but from a temporary ANON_REQ ? - memset(from, 0, sizeof(*from)); // clear the anon/temp slot + *from = ContactInfo(); // release the anon/temp path as well #if defined(NRF52_PLATFORM) from->storage_slot = mesh::storage::CONTACT_SLOT_NONE; #endif @@ -352,7 +360,7 @@ void BaseChatMesh::onPeerDataRecv(mesh::Packet* packet, uint8_t type, int sender if (from.out_path_len == OUT_PATH_UNKNOWN) { sendFloodScoped(from, reply_pkt, CLI_REPLY_DELAY_MILLIS); } else { - sendDirect(reply_pkt, from.out_path, from.out_path_len, CLI_REPLY_DELAY_MILLIS); + sendDirect(reply_pkt, from.getPath(), from.out_path_len, CLI_REPLY_DELAY_MILLIS); } } } @@ -393,7 +401,7 @@ void BaseChatMesh::onPeerDataRecv(mesh::Packet* packet, uint8_t type, int sender mesh::Packet* reply = createDatagram(PAYLOAD_TYPE_RESPONSE, from.id, secret, temp_buf, reply_len); if (reply) { if (from.out_path_len != OUT_PATH_UNKNOWN) { // we have an out_path, so send DIRECT - sendDirect(reply, from.out_path, from.out_path_len, SERVER_RESPONSE_DELAY); + sendDirect(reply, from.getPath(), from.out_path_len, SERVER_RESPONSE_DELAY); } else { sendFloodScoped(from, reply, SERVER_RESPONSE_DELAY); } @@ -425,7 +433,7 @@ bool BaseChatMesh::onContactPathRecv(ContactInfo& from, uint8_t* in_path, uint8_ // NOTE: default impl, we just replace the current 'out_path' regardless, whenever sender sends us a new out_path. // FUTURE: could store multiple out_paths per contact, and try to find which is the 'best'(?) if (canMutateContacts()) { - from.out_path_len = mesh::Packet::copyPath(from.out_path, out_path, out_path_len); // store a copy of path, for sendDirect() + if (!from.setPath(out_path, out_path_len)) return false; from.lastmod = getRTCClock()->getCurrentTime(); onContactPathUpdated(from); @@ -459,7 +467,7 @@ void BaseChatMesh::handleReturnPathRetry(const ContactInfo& contact, const uint8 // NOTE: simplest impl is just to re-send a reciprocal return path to sender (DIRECTLY) // override this method in various firmwares, if there's a better strategy mesh::Packet* rpath = createPathReturn(contact.id, contact.getSharedSecret(self_id), path, path_len, 0, NULL, 0); - if (rpath) sendDirect(rpath, contact.out_path, contact.out_path_len, 3000); // 3 second delay + if (rpath) sendDirect(rpath, contact.getPath(), contact.out_path_len, 3000); // 3 second delay } #ifdef MAX_GROUP_CHANNELS @@ -565,7 +573,7 @@ int BaseChatMesh::sendMessage(const ContactInfo& recipient, uint32_t timestamp, est_timeout = calcFloodTimeoutMillisFor(t); rc = MSG_SEND_SENT_FLOOD; } else { - sent = sendDirect(pkt, recipient.out_path, recipient.out_path_len); + sent = sendDirect(pkt, recipient.getPath(), recipient.out_path_len); est_timeout = calcDirectTimeoutMillisFor(t, recipient.out_path_len); rc = MSG_SEND_SENT_DIRECT; } @@ -632,7 +640,7 @@ int BaseChatMesh::sendCommandData(const ContactInfo& recipient, sent = sendFloodScoped(recipient, pkt); rc = MSG_SEND_SENT_FLOOD; } else { - sent = sendDirect(pkt, recipient.out_path, recipient.out_path_len); + sent = sendDirect(pkt, recipient.getPath(), recipient.out_path_len); rc = MSG_SEND_SENT_DIRECT; } if (!sent) { @@ -755,7 +763,7 @@ int BaseChatMesh::sendLogin(const ContactInfo& recipient, const char* password, est_timeout = calcFloodTimeoutMillisFor(t); return MSG_SEND_SENT_FLOOD; } else { - sendDirect(pkt, recipient.out_path, recipient.out_path_len); + if (!sendDirect(pkt, recipient.getPath(), recipient.out_path_len)) return MSG_SEND_FAILED; est_timeout = calcDirectTimeoutMillisFor(t, recipient.out_path_len); return MSG_SEND_SENT_DIRECT; } @@ -780,7 +788,7 @@ int BaseChatMesh::sendAnonReq(const ContactInfo& recipient, const uint8_t* data, est_timeout = calcFloodTimeoutMillisFor(t); return MSG_SEND_SENT_FLOOD; } else { - sendDirect(pkt, recipient.out_path, recipient.out_path_len); + if (!sendDirect(pkt, recipient.getPath(), recipient.out_path_len)) return MSG_SEND_FAILED; est_timeout = calcDirectTimeoutMillisFor(t, recipient.out_path_len); return MSG_SEND_SENT_DIRECT; } @@ -807,7 +815,7 @@ int BaseChatMesh::sendRequest(const ContactInfo& recipient, const uint8_t* req_ est_timeout = calcFloodTimeoutMillisFor(t); return MSG_SEND_SENT_FLOOD; } else { - sendDirect(pkt, recipient.out_path, recipient.out_path_len); + if (!sendDirect(pkt, recipient.getPath(), recipient.out_path_len)) return MSG_SEND_FAILED; est_timeout = calcDirectTimeoutMillisFor(t, recipient.out_path_len); return MSG_SEND_SENT_DIRECT; } @@ -834,7 +842,7 @@ int BaseChatMesh::sendRequest(const ContactInfo& recipient, uint8_t req_type, u est_timeout = calcFloodTimeoutMillisFor(t); return MSG_SEND_SENT_FLOOD; } else { - sendDirect(pkt, recipient.out_path, recipient.out_path_len); + if (!sendDirect(pkt, recipient.getPath(), recipient.out_path_len)) return MSG_SEND_FAILED; est_timeout = calcDirectTimeoutMillisFor(t, recipient.out_path_len); return MSG_SEND_SENT_DIRECT; } @@ -950,7 +958,7 @@ void BaseChatMesh::checkConnections() { auto pkt = createDatagram(PAYLOAD_TYPE_REQ, contact->id, contact->getSharedSecret(self_id), data, 9); if (pkt) { - sendDirect(pkt, contact->out_path, contact->out_path_len); + sendDirect(pkt, contact->getPath(), contact->out_path_len); } // schedule next KEEP_ALIVE @@ -1033,7 +1041,7 @@ bool BaseChatMesh::isTransientContact(const ContactInfo& contact) const { bool BaseChatMesh::clearTransientContact(ContactInfo& contact) { if (!isTransientContact(contact)) return false; - memset(&contact, 0, sizeof(contact)); + contact = ContactInfo(); #if defined(NRF52_PLATFORM) contact.storage_slot = mesh::storage::CONTACT_SLOT_NONE; #endif @@ -1065,6 +1073,7 @@ bool BaseChatMesh::removeContact(ContactInfo& contact) { contacts[idx] = contacts[idx + 1]; idx++; } + contacts[num_contacts] = ContactInfo(); // release the retired tail path contact_table_revision++; return true; // Success } diff --git a/src/helpers/BaseChatMesh.h b/src/helpers/BaseChatMesh.h index 03ed484af..c0085ba1d 100644 --- a/src/helpers/BaseChatMesh.h +++ b/src/helpers/BaseChatMesh.h @@ -84,7 +84,7 @@ class BaseChatMesh : public mesh::Mesh { ContactInfo contacts[MAX_CONTACTS+MAX_ANON_CONTACTS]; int sort_array[MAX_CONTACTS+MAX_ANON_CONTACTS]; #endif - int num_contacts; + int num_contacts = 0; uint32_t contact_table_revision; int matching_peer_indexes[MAX_SEARCH_RESULTS]; unsigned long txt_send_timeout; @@ -124,7 +124,7 @@ protected: bool initializeContactStorage(); void resetContacts() { - memset(contacts, 0, sizeof(contacts[0])*MAX_ANON_CONTACTS); // set all to have type = ADV_TYPE_NONE(0) + for (int i = 0; i < num_contacts; ++i) contacts[i] = ContactInfo(); #if defined(NRF52_PLATFORM) for (int i = 0; i < MAX_ANON_CONTACTS; i++) { contacts[i].storage_slot = mesh::storage::CONTACT_SLOT_NONE; diff --git a/src/helpers/ContactCachePolicy.h b/src/helpers/ContactCachePolicy.h new file mode 100644 index 000000000..7225bba74 --- /dev/null +++ b/src/helpers/ContactCachePolicy.h @@ -0,0 +1,24 @@ +#pragma once + +// PSRAM boards already keep the complete contact table outside internal RAM. +// An explicit 0/1 override is useful for matched qualification builds. +#ifndef MESH_CONTACT_CACHE +#if defined(COMPANION_RADIO_FULL) && !defined(BOARD_HAS_PSRAM) && \ + (defined(ESP32_PLATFORM) || defined(NRF52_PLATFORM)) +#define MESH_CONTACT_CACHE 1 +#else +#define MESH_CONTACT_CACHE 0 +#endif +#endif + +// A measured ESP32-S3 SPIFFS lookup took ~132 ms versus ~22 ms for key +// exchange. Recalculate cold ESP32 entries instead of spending more time and +// writes on flash. On a RAK3401, LittleFS took ~8 ms versus ~31 ms for key +// exchange, so nRF52 reuses saved secrets when available. +#ifndef MESH_CONTACT_SECRET_FLASH_CACHE +#if defined(NRF52_PLATFORM) +#define MESH_CONTACT_SECRET_FLASH_CACHE 1 +#else +#define MESH_CONTACT_SECRET_FLASH_CACHE 0 +#endif +#endif diff --git a/src/helpers/ContactFileTransaction.h b/src/helpers/ContactFileTransaction.h new file mode 100644 index 000000000..17a74e0e7 --- /dev/null +++ b/src/helpers/ContactFileTransaction.h @@ -0,0 +1,81 @@ +#pragma once + +// SPIFFS cannot replace an existing name in one rename. Preserve the previous +// file as .bak until the verified replacement has its final name. Recovery of +// the rename gap happens before loading contacts on the next boot. +#if defined(ESP32_PLATFORM) +#include "IdentityStore.h" +#include "PersistentStoreFormat.h" +#include + +namespace mesh { +class ContactFileTransaction { + FILESYSTEM* _fs; + const char* _target; + char _temp[48]; + char _backup[48]; + File _file; + size_t _size = 0; + uint32_t _crc = 0xffffffff; + bool _ok = false; + bool _finished = false; +public: + static bool recover(FILESYSTEM* fs, const char* target) { + char backup[48]; + snprintf(backup, sizeof(backup), "%s.bak", target); + if (fs->exists(target)) return true; + return !fs->exists(backup) || fs->rename(backup, target); + } + ContactFileTransaction(FILESYSTEM* fs, const char* target) : _fs(fs), _target(target) { + snprintf(_temp, sizeof(_temp), "%s.tmp", target); + snprintf(_backup, sizeof(_backup), "%s.bak", target); + if (!recover(fs, target)) return; + if (fs->exists(_temp) && !fs->remove(_temp)) return; + _file = fs->open(_temp, "w", true); + _ok = static_cast(_file); + } + ~ContactFileTransaction() { + if (_file) _file.close(); + if (!_finished) _fs->remove(_temp); + } + operator bool() const { return _ok; } + size_t write(const uint8_t* data, size_t len) { + if (!_ok) return 0; + const size_t wrote = _file.write(data, len); + _ok = wrote == len; + _size += wrote; + _crc = storage::updateCRC32(_crc, data, wrote); + return wrote; + } + bool commit(bool valid = true) { + if (_finished) return false; + if (_file) { _file.flush(); _file.close(); } + bool ok = _ok && valid; + File verify = _fs->open(_temp, "r"); + ok = ok && verify && verify.size() == _size; + uint32_t crc = 0xffffffff; + size_t remaining = _size; + uint8_t buf[64]; + while (ok && remaining) { + const size_t count = remaining < sizeof(buf) ? remaining : sizeof(buf); + ok = verify.read(buf, count) == count; + if (ok) { crc = storage::updateCRC32(crc, buf, count); remaining -= count; } + } + if (verify) verify.close(); + ok = ok && crc == _crc; + if (ok && _fs->exists(_backup)) ok = _fs->remove(_backup); + bool backed_up = false; + if (ok && _fs->exists(_target)) { + ok = _fs->rename(_target, _backup); + backed_up = ok; + } + if (ok) ok = _fs->rename(_temp, _target); + if (!ok && backed_up) _fs->rename(_backup, _target); + if (ok) _fs->remove(_backup); + else _fs->remove(_temp); + _finished = true; + return ok; + } +}; +} // namespace mesh +#endif diff --git a/src/helpers/ContactInfo.cpp b/src/helpers/ContactInfo.cpp new file mode 100644 index 000000000..537269df6 --- /dev/null +++ b/src/helpers/ContactInfo.cpp @@ -0,0 +1,28 @@ +#include "ContactInfo.h" + +#if MESH_CONTACT_CACHE +#include "ContactSecretCache.h" + +#ifndef MAX_CONTACTS +#define MAX_CONTACTS 32 +#endif + +namespace mesh { + +ContactPathStorage& contactPathStorage() { + // Eight anonymous peers plus room for independent in-flight snapshots. + static ContactPathPool pool; + return pool; +} + +ContactSecretCache& contactSecretCache() { + static ContactSecretCache cache; + return cache; +} + +} // namespace mesh + +const uint8_t* ContactInfo::getSharedSecret(const mesh::LocalIdentity& self_id) const { + return mesh::contactSecretCache().get(self_id, id.pub_key); +} +#endif diff --git a/src/helpers/ContactInfo.h b/src/helpers/ContactInfo.h index 54ff96d2d..4f8af0973 100644 --- a/src/helpers/ContactInfo.h +++ b/src/helpers/ContactInfo.h @@ -2,6 +2,10 @@ #include #include +#include "ContactCachePolicy.h" +#if MESH_CONTACT_CACHE +#include "ContactPathCache.h" +#endif #if defined(NRF52_PLATFORM) #include "PersistentStoreFormat.h" #endif @@ -10,16 +14,20 @@ struct ContactInfo { mesh::Identity id; - char name[32]; - uint8_t type; // on of ADV_TYPE_* - uint8_t flags; - uint8_t out_path_len; - mutable bool shared_secret_valid; // flag to indicate if shared_secret has been calculated + char name[32] = {}; + uint8_t type = 0; // one of ADV_TYPE_* + uint8_t flags = 0; + uint8_t out_path_len = 0; + mutable bool shared_secret_valid = false; +#if MESH_CONTACT_CACHE + mesh::ContactPathRef path_ref; +#else uint8_t out_path[MAX_PATH_SIZE]; - uint32_t last_advert_timestamp; // by THEIR clock - uint32_t lastmod; // by OUR clock - int32_t gps_lat, gps_lon; // 6 dec places - uint32_t sync_since; +#endif + uint32_t last_advert_timestamp = 0; // by THEIR clock + uint32_t lastmod = 0; // by OUR clock + int32_t gps_lat = 0, gps_lon = 0; // 6 dec places + uint32_t sync_since = 0; // Runtime-only position in the paged companion contact store. It is not // part of the on-disk record; the record's page/slot supplies it on load. @@ -28,6 +36,47 @@ struct ContactInfo { mutable uint16_t storage_slot = mesh::storage::CONTACT_SLOT_NONE; #endif + // The pointer is a short-lived borrow. Copy bytes for a queued operation; + // another contact-cache access may replace the resident entry. + const uint8_t* getPath() const { +#if MESH_CONTACT_CACHE + return path_ref.view(); +#else + return out_path; +#endif + } + bool copyPathTo(uint8_t path[MAX_PATH_SIZE]) const { +#if MESH_CONTACT_CACHE + return path_ref.read(path); +#else + memcpy(path, out_path, MAX_PATH_SIZE); + return true; +#endif + } + bool setRawPath(const uint8_t path[MAX_PATH_SIZE]) { +#if MESH_CONTACT_CACHE + return path_ref.set(path); +#else + memcpy(out_path, path, MAX_PATH_SIZE); + return true; +#endif + } + bool setPath(const uint8_t* path, uint8_t encoded_len) { + if (encoded_len != OUT_PATH_UNKNOWN && + !mesh::Packet::isValidPathLen(encoded_len)) return false; + uint8_t normalized[MAX_PATH_SIZE] = {}; + if (encoded_len != OUT_PATH_UNKNOWN && (encoded_len & 63)) { + if (!path) return false; + mesh::Packet::copyPath(normalized, path, encoded_len); + } + if (!setRawPath(normalized)) return false; + out_path_len = encoded_len; + return true; + } + +#if MESH_CONTACT_CACHE + const uint8_t* getSharedSecret(const mesh::LocalIdentity& self_id) const; +#else const uint8_t* getSharedSecret(const mesh::LocalIdentity& self_id) const { if (!shared_secret_valid) { self_id.calcSharedSecret(shared_secret, id.pub_key); @@ -35,6 +84,7 @@ struct ContactInfo { } return shared_secret; } +#endif bool isFav() const { return flags & 0x01; } bool isTelemBaseAllowed() const { return flags & 0x02; } @@ -43,5 +93,7 @@ struct ContactInfo { bool isRemoteCLIAllowed() const { return flags & 0x10; } private: +#if !MESH_CONTACT_CACHE mutable uint8_t shared_secret[PUB_KEY_SIZE]; +#endif }; diff --git a/src/helpers/ContactPathCache.h b/src/helpers/ContactPathCache.h new file mode 100644 index 000000000..5949326d9 --- /dev/null +++ b/src/helpers/ContactPathCache.h @@ -0,0 +1,267 @@ +#pragma once + +#include +#include +#include + +namespace mesh { + +// Paths are immutable values shared by ContactInfo copies. Editing a copy uses +// a new handle, so a pending app response or a rollback keeps its original path. +// Only the 16 most recently used path byte arrays are resident. Small handles +// retain the location of the other paths in the existing contact files. +class ContactPathBackend { +public: + virtual bool readStoredPath(uint16_t source, uint8_t path[64]) = 0; + virtual bool flushCachedPaths() = 0; +}; + +class ContactPathStorage { +public: + static const uint16_t NONE = 0xffff; + static const uint16_t PAGED = 0x8000; + virtual void attach(ContactPathBackend* backend) = 0; + virtual uint16_t bind(uint16_t source, const uint8_t path[64]) = 0; + virtual void retain(uint16_t handle) = 0; + virtual void release(uint16_t handle) = 0; + virtual bool read(uint16_t handle, uint8_t path[64]) = 0; + virtual const uint8_t* view(uint16_t handle) = 0; + virtual bool replace(uint16_t& handle, const uint8_t path[64]) = 0; + virtual uint16_t source(uint16_t handle) const = 0; + virtual void beginCommit() = 0; + virtual void mark(uint16_t handle) = 0; + virtual bool preserveSnapshots(uint16_t first, uint16_t count) = 0; + virtual void publish(uint16_t handle, uint16_t source) = 0; + virtual void endCommit(bool success) = 0; +}; + +ContactPathStorage& contactPathStorage(); + +class ContactPathRef { + uint16_t _handle = 0; +public: + ContactPathRef() = default; + ContactPathRef(const ContactPathRef& other) : _handle(other._handle) { + if (_handle) contactPathStorage().retain(_handle); + } + ContactPathRef& operator=(const ContactPathRef& other) { + if (this != &other) { + if (other._handle) contactPathStorage().retain(other._handle); + if (_handle) contactPathStorage().release(_handle); + _handle = other._handle; + } + return *this; + } + ~ContactPathRef() { if (_handle) contactPathStorage().release(_handle); } + bool bind(uint16_t source, const uint8_t path[64]) { + uint16_t next = contactPathStorage().bind(source, path); + if (next == ContactPathStorage::NONE) return false; + if (_handle) contactPathStorage().release(_handle); + _handle = next; + return true; + } + bool set(const uint8_t path[64]) { + return contactPathStorage().replace(_handle, path); + } + bool read(uint8_t path[64]) const { + return contactPathStorage().read(_handle, path); + } + const uint8_t* view() const { return contactPathStorage().view(_handle); } + uint16_t handle() const { return _handle; } +}; + +template +class ContactPathPool : public ContactPathStorage { + static_assert(Handles > Resident && Handles < 0xffff, "Invalid path handle capacity"); + static_assert(Resident > 0 && Resident < 255, "Invalid path cache capacity"); + enum : uint8_t { DIRTY = 1, MARKED = 2, DETACH = 4 }; + struct Slot { + uint32_t checksum = 0; + uint16_t refs = 0; + uint16_t location = NONE; + uint8_t resident = 255; + uint8_t flags = 0; + } _slots[Handles]; + struct Entry { + uint8_t path[64] = {}; + uint16_t handle = 0; + uint8_t age = 0; + } _entries[Resident]; + ContactPathBackend* _backend = nullptr; + bool _committing = false; + bool _flushing = false; + + static uint32_t checksum(const uint8_t path[64]) { + uint32_t crc = 0xffffffff; + for (size_t i = 0; i < 64; ++i) { + crc ^= path[i]; + for (unsigned bit = 0; bit < 8; ++bit) + crc = (crc >> 1) ^ ((crc & 1) ? 0xedb88320UL : 0); + } + return crc; + } + + Slot* slot(uint16_t h) { + return h > 0 && h <= Handles && _slots[h - 1].refs ? &_slots[h - 1] : nullptr; + } + void touch(size_t index) { + // Bounded ranks avoid millis/counter wrap affecting LRU ordering. + for (size_t i = 0; i < Resident; ++i) { + if (_entries[i].handle && _entries[i].age < Resident) ++_entries[i].age; + } + _entries[index].age = 0; + } + int availableEntry() { + int oldest = -1; + for (size_t i = 0; i < Resident; ++i) { + Slot* s = slot(_entries[i].handle); + if (!s) return static_cast(i); + if (s->location != NONE && !(s->flags & (DIRTY | DETACH)) && + (oldest < 0 || _entries[i].age > _entries[oldest].age)) oldest = i; + } + return oldest; + } + int makeRoom() { + int index = availableEntry(); + if (index < 0 && _backend && !_committing && !_flushing) { + _flushing = true; + _backend->flushCachedPaths(); + _flushing = false; + index = availableEntry(); + } + if (index >= 0) { + Slot* old = slot(_entries[index].handle); + if (old) old->resident = 255; + _entries[index].handle = 0; + } + return index; + } + uint16_t allocate() { + for (size_t i = 0; i < Handles; ++i) { + if (!_slots[i].refs) { + _slots[i] = Slot(); + _slots[i].refs = 1; + return static_cast(i + 1); + } + } + return NONE; + } + int load(uint16_t h) { + Slot* s = slot(h); + if (!s) return -1; + if (s->resident != 255) { touch(s->resident); return s->resident; } + // A flush in makeRoom() may move this source. Read its current location + // afterward. read() itself never allocates or starts a persistence write. + if (!_backend || s->location == NONE) return -1; + int index = makeRoom(); + if (index < 0) return -1; + if (!_backend->readStoredPath(s->location, _entries[index].path) || + checksum(_entries[index].path) != s->checksum) return -1; + _entries[index].handle = h; + s->resident = index; + touch(index); + return index; + } + +public: + void attach(ContactPathBackend* backend) override { _backend = backend; } + uint16_t bind(uint16_t location, const uint8_t path[64]) override { + if (location == NONE) return NONE; + const uint16_t h = allocate(); + if (h != NONE) { + _slots[h - 1].location = location; + _slots[h - 1].checksum = checksum(path); + } + return h; + } + void retain(uint16_t h) override { Slot* s = slot(h); if (s) ++s->refs; } + void release(uint16_t h) override { + Slot* s = slot(h); + if (!s || --s->refs) return; + if (s->resident != 255) _entries[s->resident].handle = 0; + *s = Slot(); + } + bool read(uint16_t h, uint8_t path[64]) override { + if (!h) { memset(path, 0, 64); return true; } + Slot* s = slot(h); + if (!s) return false; + if (s->resident != 255) { + memcpy(path, _entries[s->resident].path, 64); + touch(s->resident); + return true; + } + // Serializing a whole contact table must not evict a dirty entry and + // recursively rewrite the table it is currently reading. + return _backend && s->location != NONE && + _backend->readStoredPath(s->location, path) && checksum(path) == s->checksum; + } + const uint8_t* view(uint16_t h) override { + static const uint8_t empty[64] = {}; + if (!h) return empty; + const int index = load(h); + return index < 0 ? nullptr : _entries[index].path; + } + bool replace(uint16_t& h, const uint8_t path[64]) override { + uint8_t previous[64]; + if (read(h, previous) && memcmp(previous, path, 64) == 0) return true; + bool empty = true; + for (size_t i = 0; i < 64; ++i) empty = empty && path[i] == 0; + if (empty) { release(h); h = 0; return true; } + Slot* old = slot(h); + // In-place mutation is safe only when there is no ContactInfo snapshot. + if (old && old->refs == 1 && old->resident != 255) { + memcpy(_entries[old->resident].path, path, 64); + old->flags |= DIRTY; + old->checksum = checksum(path); + touch(old->resident); + return true; + } + const uint16_t next = allocate(); + if (next == NONE) return false; + const int index = makeRoom(); + if (index < 0) { release(next); return false; } + Slot& s = _slots[next - 1]; + s.location = old ? old->location : NONE; + s.flags = DIRTY; + s.checksum = checksum(path); + s.resident = index; + memcpy(_entries[index].path, path, 64); + _entries[index].handle = next; + touch(index); + release(h); + h = next; + return true; + } + uint16_t source(uint16_t h) const override { + return h && h <= Handles && _slots[h - 1].refs ? _slots[h - 1].location : NONE; + } + void beginCommit() override { + _committing = true; + for (auto& s : _slots) s.flags &= ~(MARKED | DETACH); + } + void mark(uint16_t h) override { Slot* s = slot(h); if (s) s->flags |= MARKED; } + bool preserveSnapshots(uint16_t first, uint16_t count) override { + for (size_t i = 0; i < Handles; ++i) { + Slot& s = _slots[i]; + if (!s.refs || s.location == NONE || (s.flags & MARKED) || + s.location < first || static_cast(s.location) >= + static_cast(first) + count) continue; + if (load(i + 1) < 0) return false; + s.flags |= DETACH; + } + return true; + } + void publish(uint16_t h, uint16_t location) override { + Slot* s = slot(h); + if (s) { s->location = location; s->flags &= ~DIRTY; } + } + void endCommit(bool success) override { + for (auto& s : _slots) { + if (success && (s.flags & DETACH)) s.location = NONE; + s.flags &= ~(MARKED | DETACH); + } + _committing = false; + } +}; + +} // namespace mesh diff --git a/src/helpers/ContactSecretCache.h b/src/helpers/ContactSecretCache.h new file mode 100644 index 000000000..5e5ced4f4 --- /dev/null +++ b/src/helpers/ContactSecretCache.h @@ -0,0 +1,107 @@ +#pragma once + +#include +#include "ContactCachePolicy.h" +#include +#include +#include +#include + +namespace mesh { + +class ContactSecretBackend { +public: + virtual bool readSavedSecret(const uint8_t peer[32], const uint8_t identity[32], + uint8_t secret[32]) = 0; + virtual bool saveSecret(const uint8_t peer[32], const uint8_t identity[32], + const uint8_t secret[32]) = 0; +}; + +class ContactSecretCache { + struct Entry { + uint8_t peer[32] = {}; + uint8_t secret[32] = {}; + uint8_t age = 0; + bool valid = false; + } _entries[16]; + uint8_t _identity[PRV_KEY_SIZE + PUB_KEY_SIZE] = {}; +#if MESH_CONTACT_SECRET_FLASH_CACHE + uint8_t _identity_hash[32] = {}; +#endif + bool _has_identity = false; +#if MESH_CONTACT_SECRET_FLASH_CACHE + ContactSecretBackend* _backend = nullptr; +#endif +public: + uint32_t ram_hits = 0; + uint32_t flash_hits = 0; + uint32_t calculations = 0; + uint32_t save_failures = 0; + uint32_t last_flash_read_us = 0; + uint32_t last_calculation_us = 0; + + void attach(ContactSecretBackend* backend) { +#if MESH_CONTACT_SECRET_FLASH_CACHE + _backend = backend; +#else + (void)backend; +#endif + } + const uint8_t* get(const LocalIdentity& self, const uint8_t peer[32]) { + uint8_t identity[sizeof(_identity)]; + self.writeTo(identity, sizeof(identity)); + // Compare the complete key pair, including private material. An identity + // import must invalidate RAM and flash lookups even if its public bytes + // happen to be unchanged. Only a SHA-256 fingerprint is stored with a page. + if (!_has_identity || memcmp(identity, _identity, sizeof(identity)) != 0) { + for (auto& e : _entries) { memset(e.secret, 0, sizeof(e.secret)); e.valid = false; } + memcpy(_identity, identity, sizeof(identity)); +#if MESH_CONTACT_SECRET_FLASH_CACHE + Utils::sha256(_identity_hash, sizeof(_identity_hash), identity, sizeof(identity)); +#endif + _has_identity = true; + } + int chosen = -1; + bool hit = false; + for (int i = 0; i < 16; ++i) { + if (_entries[i].valid && memcmp(_entries[i].peer, peer, 32) == 0) { + chosen = i; + hit = true; + break; + } + if (chosen < 0 || !_entries[i].valid || + (_entries[chosen].valid && _entries[i].age > _entries[chosen].age)) chosen = i; + } + Entry& e = _entries[chosen]; + if (hit) { + ++ram_hits; + } else { + e.valid = false; + memcpy(e.peer, peer, 32); +#if MESH_CONTACT_SECRET_FLASH_CACHE + const uint32_t started = micros(); + if (_backend && _backend->readSavedSecret(peer, _identity_hash, e.secret)) { + last_flash_read_us = micros() - started; + ++flash_hits; + } else +#endif + { + const uint32_t started = micros(); + self.calcSharedSecret(e.secret, peer); + last_calculation_us = micros() - started; + ++calculations; +#if MESH_CONTACT_SECRET_FLASH_CACHE + if (_backend && !_backend->saveSecret(peer, _identity_hash, e.secret)) ++save_failures; +#endif + } + e.valid = true; + } + for (auto& other : _entries) if (other.valid && other.age < 16) ++other.age; + e.age = 0; + return e.secret; + } +}; + +ContactSecretCache& contactSecretCache(); + +} // namespace mesh diff --git a/test/fixtures/contact_cache/mocks/Adafruit_LittleFS.h b/test/fixtures/contact_cache/mocks/Adafruit_LittleFS.h new file mode 100644 index 000000000..d97e761dd --- /dev/null +++ b/test/fixtures/contact_cache/mocks/Adafruit_LittleFS.h @@ -0,0 +1,4 @@ +#pragma once +class FakeFilesystem; +using Adafruit_LittleFS = FakeFilesystem; +namespace Adafruit_LittleFS_Namespace {} diff --git a/test/fixtures/contact_cache/mocks/Arduino.h b/test/fixtures/contact_cache/mocks/Arduino.h new file mode 100644 index 000000000..d17c8636d --- /dev/null +++ b/test/fixtures/contact_cache/mocks/Arduino.h @@ -0,0 +1,7 @@ +#pragma once +#include +#include +#include +#include +inline uint32_t millis() { return 100000; } +inline uint32_t micros() { return 100000000; } diff --git a/test/fixtures/contact_cache/mocks/Identity.h b/test/fixtures/contact_cache/mocks/Identity.h new file mode 100644 index 000000000..f518766c7 --- /dev/null +++ b/test/fixtures/contact_cache/mocks/Identity.h @@ -0,0 +1,26 @@ +#pragma once +#include "Utils.h" +#include +namespace mesh { +class Identity { +public: + uint8_t pub_key[32] = {}; + Identity() = default; + explicit Identity(const uint8_t* key) { memcpy(pub_key, key, 32); } +}; +class LocalIdentity : public Identity { +public: + uint8_t private_key[64] = {}; + mutable unsigned derivations = 0; + size_t writeTo(uint8_t* out, size_t size) const { + if (size < 96) return 0; + memcpy(out, private_key, 64); + memcpy(out + 64, pub_key, 32); + return 96; + } + void calcSharedSecret(uint8_t* out, const uint8_t* peer) const { + ++derivations; + ed25519_key_exchange(out, peer, private_key); + } +}; +} diff --git a/test/fixtures/contact_cache/mocks/Mesh.h b/test/fixtures/contact_cache/mocks/Mesh.h new file mode 100644 index 000000000..76e75d17b --- /dev/null +++ b/test/fixtures/contact_cache/mocks/Mesh.h @@ -0,0 +1,4 @@ +#pragma once +#include "Identity.h" +#include + diff --git a/test/fixtures/contact_cache/mocks/Utils.h b/test/fixtures/contact_cache/mocks/Utils.h new file mode 100644 index 000000000..6797975ae --- /dev/null +++ b/test/fixtures/contact_cache/mocks/Utils.h @@ -0,0 +1,18 @@ +#pragma once +#include +#include +#include +#include +#define PUB_KEY_SIZE 32 +#define PRV_KEY_SIZE 64 +namespace mesh { +class Utils { +public: + static void sha256(uint8_t* out, size_t size, const uint8_t* data, int length) { + uint8_t digest[32]; + SHA256(data, length, digest); + memcpy(out, digest, size); + } +}; +} + diff --git a/test/fixtures/contact_cache/test_contact_cache.cpp b/test/fixtures/contact_cache/test_contact_cache.cpp new file mode 100644 index 000000000..fb67baa87 --- /dev/null +++ b/test/fixtures/contact_cache/test_contact_cache.cpp @@ -0,0 +1,665 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +enum { FILE_O_READ = 0, FILE_O_WRITE = 1 }; + +class FakeFilesystem; +class File { + FakeFilesystem* _fs = nullptr; + std::string _path; + size_t _position = 0; + bool _write = false; +public: + File() = default; + explicit File(FakeFilesystem& fs) : _fs(&fs) {} + File(FakeFilesystem* fs, const char* path, bool write); + bool open(const char* path, uint8_t mode); + explicit operator bool() const { return _fs != nullptr; } + size_t read(uint8_t* bytes, size_t length); + size_t write(const uint8_t* bytes, size_t length); + size_t size() const; + bool seek(size_t position); + void flush() {} + void close() { _fs = nullptr; } +}; + +class FakeFilesystem { + struct Config { uint32_t block_count = 256, block_size = 4096; } _config; + struct Lfs { const Config* cfg; } _lfs{&_config}; +public: + using Files = std::map>; + Files files; + size_t capacity = 1024 * 1024; + size_t max_write = std::numeric_limits::max(); + std::string fail_read; + unsigned fail_rename = 0, renames = 0, writes = 0, reads = 0, opens = 0; + std::vector rename_snapshots; + bool exists(const char* path) const { return files.count(path) != 0; } + bool mkdir(const char*) { return true; } + bool remove(const char* path) { return files.erase(path) != 0; } + File open(const char* path, const char* mode = "r", bool = false) { + ++opens; + if (*mode != 'r') { + files[path].clear(); + return File(this, path, true); + } + if (fail_read == path || !exists(path)) return File(); + return File(this, path, false); + } + bool rename(const char* from, const char* to) { + ++renames; + if (renames == fail_rename || !exists(from)) return false; +#if defined(ESP32_PLATFORM) + if (exists(to)) return false; +#endif + files[to] = files.at(from); + files.erase(from); + rename_snapshots.push_back(files); + return true; + } + size_t totalBytes() const { return capacity; } + void setBlockCount(uint32_t count) { _config.block_count = count; } + Lfs* _getFS() { _lfs.cfg = &_config; return &_lfs; } + size_t usedBytes() const { + size_t total = 0; + for (const auto& f : files) total += f.second.size(); + return total; + } +} SPIFFS; + +int _getLfsUsedBlockCount(FakeFilesystem* fs) { + unsigned used = 2; // directory metadata pair + for (const auto& file : fs->files) used += (file.second.size() + 4095) / 4096; + return used; +} + +File::File(FakeFilesystem* fs, const char* path, bool write) + : _fs(fs), _path(path), _write(write) {} +bool File::open(const char* path, uint8_t mode) { + if (!_fs) return false; + *this = _fs->open(path, mode == FILE_O_WRITE ? "w" : "r"); + return static_cast(*this); +} +size_t File::read(uint8_t* bytes, size_t length) { + if (!_fs || _write || _fs->fail_read == _path) return 0; + ++_fs->reads; + auto& data = _fs->files[_path]; + const size_t count = std::min(length, data.size() - std::min(data.size(), _position)); + if (count) memcpy(bytes, data.data() + _position, count); + _position += count; + return count; +} +size_t File::write(const uint8_t* bytes, size_t length) { + if (!_fs || !_write) return 0; + ++_fs->writes; + const size_t count = std::min(length, _fs->max_write); + auto& data = _fs->files[_path]; + data.resize(_position + count); + memcpy(data.data() + _position, bytes, count); + _position += count; + return count; +} +size_t File::size() const { return _fs ? _fs->files.at(_path).size() : 0; } +bool File::seek(size_t pos) { + if (!_fs || pos > size()) return false; + _position = pos; + return true; +} + +#define FILESYSTEM FakeFilesystem +#include +#include +#include +#if defined(NRF52_PLATFORM) +#define ATOMIC_FILE_WRITER_IMPLEMENTATION +#include +#endif +#include "packet_under_test.h" +#define ADV_TYPE_NONE 0 + +struct Host { + unsigned cache_flushes = 0; + size_t capacity = 350; + std::vector contacts; + bool onContactLoaded(const ContactInfo& c) { + if (contacts.size() >= capacity) return false; + contacts.push_back(c); + return true; + } + void onContactCacheFlushed() { ++cache_flushes; } + ContactInfo* getContactForStore(uint32_t index) { + return index < contacts.size() ? &contacts[index] : nullptr; + } +}; +using DataStoreHost = Host; + +#if defined(NRF52_PLATFORM) +static const char* CONTACT_MIGRATION_MARKER = "/contacts4.mig"; +bool contactPathPresence(FakeFilesystem* fs, const char* path, bool& present) { + present = fs->exists(path); + return true; +} +#endif + +// The narrow hardware adapter below supplies the same filesystem/host seams. +// Serialization, save/eviction and secret I/O bodies are extracted unchanged +// from DataStore.cpp and compiled below, including their platform branches. +class DataStore : public mesh::ContactPathBackend, public mesh::ContactSecretBackend { + DataStoreHost* _cache_host; + bool _cache_load_incomplete = false; +#if defined(ESP32_PLATFORM) + File _contact_path_reader; +#endif + uint32_t _secret_retry_at = 0; +#if defined(NRF52_PLATFORM) + uint32_t _contact_page_generations[14] = {}; + bool _dirty[14] = {}; + bool _contact_load_incomplete = false; + bool _legacy_contacts_pending_cleanup = false, _legacy_migration_ready = false; + uint16_t _legacy_contact_count = 0; + mesh::storage::ContactSlotMap _contact_slots; + mesh::storage::DirtyPageSet _dirty_contact_pages, _unread_contact_pages; + void resetContactPageState() { _contact_slots.clear(); } + bool prepareLegacyContactMigration() { _legacy_migration_ready = true; return true; } +#endif +public: + explicit DataStore(Host& host) : _cache_host(&host) { + mesh::contactPathStorage().attach(this); + } + FakeFilesystem* _getContactsChannelsFS() const { return &SPIFFS; } + File openRead(FakeFilesystem* fs, const char* path) { return fs->open(path); } + bool hasIncompleteContactLoad() const { + return _cache_load_incomplete +#if defined(NRF52_PLATFORM) + || _contact_load_incomplete || !_unread_contact_pages.empty() +#endif + ; + } + void loadContacts(DataStoreHost*); + bool saveContacts(DataStoreHost*, bool (*filter)(const ContactInfo&)); + bool flushContactWrites(DataStoreHost*, bool (*filter)(const ContactInfo&)); + bool readStoredPath(uint16_t, uint8_t[64]) override; + bool flushCachedPaths() override; + uint16_t secretSlot(const uint8_t[32]) const; + bool readSavedSecret(const uint8_t[32], const uint8_t[32], uint8_t[32]) override; + bool saveSecret(const uint8_t[32], const uint8_t[32], const uint8_t[32]) override; +#if defined(NRF52_PLATFORM) + bool loadContactPages(DataStoreHost*, uint16_t, uint32_t); + bool writeContactPage(DataStoreHost*, uint8_t, bool (*filter)(const ContactInfo&)); + bool markContactDirty(const ContactInfo& c) { + if (hasIncompleteContactLoad() || c.storage_slot >= 350) return false; + _dirty[c.storage_slot / 25] = true; + return true; + } + bool hasPendingContactWrites() const { + for (bool dirty : _dirty) if (dirty) return true; + return false; + } + bool serviceContactWrites(DataStoreHost* host, bool (*filter)(const ContactInfo&)) { + if (hasIncompleteContactLoad()) return false; + for (uint8_t i = 0; i < 14; ++i) if (_dirty[i]) { + if (!writeContactPage(host, i, filter)) return false; + _dirty[i] = false; + return true; + } + return true; + } +#endif +}; +#include "store_under_test.h" + +// Exercise the real Mesh::sendDirect and all request/login callers with only +// the radio queue and datagram constructor replaced. A flash miss must release +// the packet and report failure to the app, then recover on a successful read. +namespace mesh { +class Mesh { +protected: + struct Tables { void markSent(Packet*) {} } tables; + Tables* _tables = &tables; + void maybeScheduleDirectRetry(Packet*, uint8_t) {} + uint8_t getTraceDirectPriority(Packet*) { return 0; } + const char* getLogDateTime() { return "test"; } +public: + Packet packet; + unsigned released = 0, queued = 0; + void releasePacket(Packet*) { ++released; } + bool sendPacket(Packet*, uint8_t, uint32_t) { ++queued; return true; } + bool sendDirect(Packet*, const uint8_t*, uint8_t, uint32_t = 0); +}; +} + +#define MSG_SEND_FAILED 0 +#define MSG_SEND_SENT_FLOOD 1 +#define MSG_SEND_SENT_DIRECT 2 +#define ADV_TYPE_ROOM 3 +class BaseChatMesh : public mesh::Mesh { + struct Radio { uint32_t getEstAirtimeFor(int) { return 20; } } radio; + struct Clock { uint32_t getCurrentTimeUnique() { return 100; } } clock; + struct Rng { void random(uint8_t* data, size_t size) { memset(data, 42, size); } } rng; + Radio* _radio = &radio; + mesh::LocalIdentity self_id; + Clock* getRTCClock() { return &clock; } + Rng* getRNG() { return &rng; } + mesh::Packet* createDatagram(uint8_t type, const mesh::Identity&, + const uint8_t*, const uint8_t*, size_t length) { + packet = mesh::Packet(); + packet.header = type << PH_TYPE_SHIFT; + packet.payload_len = length; + return &packet; + } + mesh::Packet* createAnonDatagram(uint8_t type, const mesh::LocalIdentity&, + const mesh::Identity& peer, const uint8_t* secret, const uint8_t* data, size_t length) { + return createDatagram(type, peer, secret, data, length); + } + bool sendFloodScoped(const ContactInfo&, mesh::Packet* p) { return sendPacket(p, 0, 0); } + uint32_t calcFloodTimeoutMillisFor(uint32_t t) { return t * 2; } + uint32_t calcDirectTimeoutMillisFor(uint32_t t, uint8_t n) { return t * (n + 1); } +public: + int sendLogin(const ContactInfo&, const char*, uint32_t&); + int sendAnonReq(const ContactInfo&, const uint8_t*, uint8_t, uint32_t&, uint32_t&); + int sendRequest(const ContactInfo&, const uint8_t*, uint8_t, uint32_t&, uint32_t&); + int sendRequest(const ContactInfo&, uint8_t, uint32_t&, uint32_t&); +}; +#include "send_under_test.h" + +using Path = std::array; +Path route(unsigned index) { + Path path; + for (size_t i = 0; i < path.size(); ++i) path[i] = index * 19 + i + (index >> 8); + return path; +} + +struct Fixture { + Host host; + DataStore store{host}; + explicit Fixture(size_t count = 350) { + SPIFFS = FakeFilesystem(); + std::vector records(count * mesh::storage::CONTACT_RECORD_SIZE, 0); + host.contacts.reserve(count); + for (size_t i = 0; i < count; ++i) { + uint8_t* r = records.data() + i * mesh::storage::CONTACT_RECORD_SIZE; + memset(r, 0x21, 32); // identical prefixes exercise full peer-key matching + r[30] = i >> 8; + r[31] = i; + snprintf(reinterpret_cast(r + 32), 32, "Contact %u", unsigned(i)); + r[64] = 1; + r[71] = 63; + const Path path = route(i); + memcpy(r + 76, path.data(), path.size()); + } + SPIFFS.files["/contacts3"] = records; + for (size_t i = 0; i < count; ++i) { + ContactInfo c; + assert(deserializeContactRecord(records.data() + i * 152, c, i)); +#if defined(NRF52_PLATFORM) + c.storage_slot = i; +#endif + host.contacts.push_back(c); + } + } + void check(unsigned index, const Path& expected) { + const uint8_t* actual = host.contacts[index].getPath(); + assert(actual && memcmp(actual, expected.data(), 64) == 0); + } +}; + +static void routes_survive_eviction_and_full_sync() { + Fixture f; + assert(sizeof(ContactInfo) <= 100); // all 350 records retain compact handles + const unsigned reads = SPIFFS.reads; + f.check(0, route(0)); + assert(SPIFFS.reads == reads + 1); + f.check(0, route(0)); + assert(SPIFFS.reads == reads + 1); // hot route consumes no flash I/O + for (unsigned i = 1; i < 350; ++i) f.check(i, route(i)); + f.check(0, route(0)); // actual LRU miss after 350 different contacts + const auto original = SPIFFS.files.at("/contacts3"); + const unsigned opens = SPIFFS.opens; + assert(f.store.saveContacts(&f.host, cachedContactFilter)); + assert(SPIFFS.opens - opens == 2); // reuse source; open writer and verifier + assert(SPIFFS.files.at("/contacts3") == original); + for (unsigned i = 0; i < 350; ++i) f.check(i, route(i)); +} + +static void snapshot_rollback_and_dirty_eviction() { + Fixture f; + ContactInfo pending_response = f.host.contacts[0]; + assert(f.host.contacts[0].setRawPath(route(900).data())); + assert(f.store.saveContacts(&f.host, cachedContactFilter)); + Path old; + assert(pending_response.copyPathTo(old.data()) && old == route(0)); + for (unsigned i = 1; i < 80; ++i) { + assert(f.host.contacts[i].setRawPath(route(900 + i).data())); + } + assert(SPIFFS.writes > 350); // full cache forced safe dirty writeback + for (unsigned i = 0; i < 80; ++i) f.check(i, route(900 + i)); + assert(pending_response.copyPathTo(old.data()) && old == route(0)); + f.host.contacts[0] = pending_response; // rollback after the old file changed + assert(f.store.saveContacts(&f.host, cachedContactFilter)); + f.check(0, route(0)); + // Deletion compacts records; every remaining handle follows its new offset. + f.host.contacts.erase(f.host.contacts.begin() + 20); + assert(f.store.saveContacts(&f.host, cachedContactFilter)); + for (unsigned i = 20; i < 79; ++i) f.check(i, route(901 + i)); + for (unsigned i = 79; i < 349; ++i) f.check(i, route(i + 1)); +} + +static void storage_failures_preserve_routes() { + Fixture f; + const auto original = SPIFFS.files.at("/contacts3"); + ContactInfo previous = f.host.contacts[0]; + assert(f.host.contacts[0].setRawPath(route(810).data())); + SPIFFS.max_write = 13; + assert(!f.store.saveContacts(&f.host, cachedContactFilter)); + assert(SPIFFS.files.at("/contacts3") == original); + f.check(0, route(810)); + f.host.contacts[0] = previous; + f.check(0, route(0)); + SPIFFS.max_write = std::numeric_limits::max(); + for (unsigned fail : {1u, 2u}) { + SPIFFS.renames = 0; + SPIFFS.fail_rename = fail; + assert(!f.store.saveContacts(&f.host, cachedContactFilter)); + assert(SPIFFS.files.at("/contacts3") == original); + } + SPIFFS.fail_rename = 0; + SPIFFS.fail_read = "/contacts3"; + assert(!f.host.contacts[100].getPath()); + assert(!f.store.saveContacts(&f.host, cachedContactFilter)); + assert(SPIFFS.files.at("/contacts3") == original); + SPIFFS.fail_read.clear(); + SPIFFS.files["/contacts3"][100 * 152 + 76] ^= 1; + assert(!f.host.contacts[100].getPath()); // CRC protects a cold route after boot + assert(!f.store.saveContacts(&f.host, cachedContactFilter)); + SPIFFS.files["/contacts3"] = original; + assert(f.store.saveContacts(&f.host, cachedContactFilter)); +} + +#if defined(ESP32_PLATFORM) +static void rename_gaps_recover_a_complete_contact_file() { + Fixture f(30); + auto old = SPIFFS.files.at("/contacts3"); + assert(f.host.contacts[0].setRawPath(route(800).data())); + assert(f.store.saveContacts(&f.host, cachedContactFilter)); + auto updated = SPIFFS.files.at("/contacts3"); + const auto snapshots = SPIFFS.rename_snapshots; + assert(snapshots.size() == 2); + for (const auto& snapshot : snapshots) { + FakeFilesystem rebooted; + rebooted.files = snapshot; + assert(mesh::ContactFileTransaction::recover(&rebooted, "/contacts3")); + const auto& recovered = rebooted.files.at("/contacts3"); + assert(recovered == old || recovered == updated); + } +} +#endif + +static void pinned_snapshots_bound_ram_without_losing_updates() { + Fixture f; + std::vector snapshots; + snapshots.reserve(16); + for (unsigned i = 0; i < 16; ++i) { + snapshots.push_back(f.host.contacts[i]); + assert(snapshots.back().setRawPath(route(1000 + i).data())); + } + const ContactInfo previous = f.host.contacts[100]; + assert(!f.host.contacts[100].setRawPath(route(1500).data())); + Path path; + assert(f.host.contacts[100].copyPathTo(path.data()) && path == route(100)); + snapshots.clear(); + assert(f.host.contacts[100].setRawPath(route(1500).data())); + assert(f.store.saveContacts(&f.host, cachedContactFilter)); + assert(previous.copyPathTo(path.data()) && path == route(100)); +} + +static void encoded_paths_and_copy_assignment() { + Fixture f(40); + auto& c = f.host.contacts[0]; + const Path raw = route(700); + for (uint8_t len : {uint8_t(63), uint8_t(0x60), uint8_t(0x95)}) { + assert(c.setPath(raw.data(), len)); + assert(c.out_path_len == len); + const auto* p = c.getPath(); + const unsigned bytes = (len & 63) * ((len >> 6) + 1); + assert(p && memcmp(p, raw.data(), bytes) == 0); + } + const ContactInfo old = c; + assert(!c.setPath(raw.data(), 0x61)); // 33 two-byte hashes exceed 64 bytes + assert(!c.setPath(raw.data(), 0xc1)); // reserved hash encoding + assert(c.setPath(nullptr, 0)); + assert(c.out_path_len == 0); + assert(c.setPath(nullptr, OUT_PATH_UNKNOWN)); + assert(c.out_path_len == OUT_PATH_UNKNOWN); + c = old; + assert(c.out_path_len == old.out_path_len); + assert(f.store.saveContacts(&f.host, cachedContactFilter)); +} + +static void failed_path_reads_fail_sends_without_leaking_packets() { + Fixture f(30); + mesh::contactSecretCache().attach(&f.store); + BaseChatMesh mesh; + const auto& recipient = f.host.contacts[29]; + uint8_t request[] = {1}; + uint32_t tag, timeout; + for (bool fail : {true, false}) { + SPIFFS.fail_read = fail ? "/contacts3" : ""; + const int expected = fail ? MSG_SEND_FAILED : MSG_SEND_SENT_DIRECT; + assert(mesh.sendLogin(recipient, "test", timeout) == expected); + assert(mesh.sendAnonReq(recipient, request, 1, tag, timeout) == expected); + assert(mesh.sendRequest(recipient, request, 1, tag, timeout) == expected); + assert(mesh.sendRequest(recipient, 1, tag, timeout) == expected); + assert(mesh.released == 4); + assert(mesh.queued == (fail ? 0u : 4u)); + if (!fail) assert(memcmp(mesh.packet.path, route(29).data(), 63) == 0); + } +} + +static void contact_reload_preserves_disk_on_cache_exhaustion() { + Fixture f; +#if defined(NRF52_PLATFORM) + for (unsigned page = 0; page < 14; ++page) + assert(f.store.writeContactPage(&f.host, page, cachedContactFilter)); + SPIFFS.remove("/contacts3"); +#endif + f.host.contacts.clear(); + f.store.loadContacts(&f.host); + assert(!f.store.hasIncompleteContactLoad() && f.host.contacts.size() == 350); + for (unsigned i = 0; i < 350; ++i) f.check(i, route(i)); + f.host.contacts.clear(); + const auto durable = SPIFFS.files; + const unsigned writes = SPIFFS.writes; + + // Occupy every handle with independent pending values. A failed load must + // not turn a valid persisted record into an empty/invalid contact repair. + std::vector pending(390); + for (auto& path : pending) assert(path.bind(0, route(0).data())); + f.store.loadContacts(&f.host); + assert(f.store.hasIncompleteContactLoad()); + assert(f.host.contacts.empty()); + assert(!f.store.saveContacts(&f.host, cachedContactFilter)); + assert(SPIFFS.files == durable && SPIFFS.writes == writes); + pending.clear(); + f.store.loadContacts(&f.host); + assert(f.host.contacts.empty()); // latched until a fresh boot/store + DataStore rebooted(f.host); + rebooted.loadContacts(&f.host); + assert(!rebooted.hasIncompleteContactLoad() && f.host.contacts.size() == 350); + for (unsigned i = 0; i < 350; ++i) f.check(i, route(i)); +} + +#if defined(ESP32_PLATFORM) +static void partial_loads_cannot_replace_the_complete_file() { + for (bool capacity_failure : {false, true}) { + Fixture f(40); + f.host.contacts.clear(); + const auto durable = SPIFFS.files; + if (capacity_failure) f.host.capacity = 20; + else SPIFFS.fail_read = "/contacts3"; + f.store.loadContacts(&f.host); + assert(f.store.hasIncompleteContactLoad()); + assert(!f.store.saveContacts(&f.host, cachedContactFilter)); + assert(SPIFFS.files == durable); + } +} +#endif + +static void secrets_load_from_flash_instead_of_recalculating() { + Fixture f; + mesh::LocalIdentity self; + self.private_key[0] = 42; + self.pub_key[0] = 23; + mesh::ContactSecretCache cache; + cache.attach(&f.store); + std::array first; + memcpy(first.data(), cache.get(self, f.host.contacts[0].id.pub_key), 32); + assert(self.derivations == 1); + const unsigned writes = SPIFFS.writes; + assert(memcmp(first.data(), cache.get(self, f.host.contacts[0].id.pub_key), 32) == 0); + assert(self.derivations == 1 && SPIFFS.writes == writes); + for (unsigned i = 1; i <= 20; ++i) cache.get(self, f.host.contacts[i].id.pub_key); + const unsigned before_miss = self.derivations, before_writes = SPIFFS.writes; + assert(memcmp(first.data(), cache.get(self, f.host.contacts[0].id.pub_key), 32) == 0); + assert(self.derivations == before_miss && SPIFFS.writes == before_writes); + assert(cache.flash_hits == 1); + // Reboot loses the RAM cache; the persisted identity-bound key is reusable. + mesh::ContactSecretCache rebooted; + rebooted.attach(&f.store); + assert(memcmp(first.data(), rebooted.get(self, f.host.contacts[0].id.pub_key), 32) == 0); + assert(self.derivations == before_miss); + self.private_key[1] ^= 2; // also invalidate when public bytes did not change + assert(memcmp(first.data(), rebooted.get(self, f.host.contacts[0].id.pub_key), 32) != 0); + assert(self.derivations == before_miss + 1); + self.pub_key[3] ^= 4; + rebooted.get(self, f.host.contacts[0].id.pub_key); + assert(self.derivations == before_miss + 2); +} + +static void persisted_secret_matches_the_real_peer_key_exchange() { + Fixture f(30); + mesh::LocalIdentity self, peer; + uint8_t seed[32] = {1}; + ed25519_create_keypair(self.pub_key, self.private_key, seed); + seed[0] = 2; + ed25519_create_keypair(peer.pub_key, peer.private_key, seed); + f.host.contacts[0].id = peer; + uint8_t reciprocal[32]; + peer.calcSharedSecret(reciprocal, self.pub_key); + mesh::ContactSecretCache cache; + cache.attach(&f.store); + assert(memcmp(reciprocal, cache.get(self, peer.pub_key), 32) == 0); + for (unsigned i = 1; i < 30; ++i) cache.get(self, f.host.contacts[i].id.pub_key); + const unsigned derivations = self.derivations; + assert(memcmp(reciprocal, cache.get(self, peer.pub_key), 32) == 0); + assert(self.derivations == derivations && cache.flash_hits == 1); +} + +static void secret_corruption_reordering_and_full_storage() { + Fixture f(40); + mesh::LocalIdentity self; + mesh::ContactSecretCache cache; + cache.attach(&f.store); + std::array first; + memcpy(first.data(), cache.get(self, f.host.contacts[0].id.pub_key), 32); + SPIFFS.files["/csecret000"][36 + 32] ^= 4; + mesh::ContactSecretCache cold; + cold.attach(&f.store); + assert(memcmp(first.data(), cold.get(self, f.host.contacts[0].id.pub_key), 32) == 0); + assert(self.derivations == 2); // corrupted key was recomputed, never used + std::swap(f.host.contacts[0], f.host.contacts[1]); + mesh::ContactSecretCache moved; + moved.attach(&f.store); + assert(memcmp(first.data(), moved.get(self, f.host.contacts[0].id.pub_key), 32) != 0); + assert(self.derivations == 3); // same slot, different full peer key + SPIFFS.capacity = SPIFFS.usedBytes() + 10000; + const unsigned writes = SPIFFS.writes; + moved.get(self, f.host.contacts[30].id.pub_key); + assert(self.derivations == 4 && SPIFFS.writes == writes); + moved.get(self, f.host.contacts[30].id.pub_key); + assert(self.derivations == 4); // still cached in RAM when flash has no reserve + assert(f.store.saveContacts(&f.host, cachedContactFilter)); +} + +#if defined(NRF52_PLATFORM) +static void nrf52_secret_cache_leaves_contact_storage_reserve() { + Fixture f; + for (unsigned page = 0; page < 14; ++page) + assert(f.store.writeContactPage(&f.host, page, cachedContactFilter)); + SPIFFS.remove("/contacts3"); + const auto contacts = SPIFFS.files; + SPIFFS.setBlockCount(25); // 100 KiB ExtraFS + mesh::LocalIdentity self; + mesh::ContactSecretCache cache; + cache.attach(&f.store); + for (unsigned i = 0; i < 350; i += 56) cache.get(self, f.host.contacts[i].id.pub_key); + assert(cache.save_failures > 0); // low space remains a best-effort cache + assert(_getLfsUsedBlockCount(&SPIFFS) <= 21); // keep at least four blocks + for (const auto& file : contacts) assert(SPIFFS.files.at(file.first) == file.second); + assert(f.host.contacts[0].setRawPath(route(999).data())); + assert(f.store.saveContacts(&f.host, cachedContactFilter)); + f.check(0, route(999)); +} + +static void nrf52_migration_snapshots_and_failed_page_replacement() { + Fixture f; + ContactInfo snapshot = f.host.contacts[349]; + assert(f.host.contacts[349].setRawPath(route(950).data())); + // Commit one page at a time from the legacy tail, exactly as migration does. + for (int page = 13; page >= 0; --page) { + assert(f.store.writeContactPage(&f.host, page, cachedContactFilter)); + SPIFFS.files["/contacts3"].resize(page * 25 * 152); + for (unsigned i = page * 25; i < 350; ++i) + f.check(i, route(i == 349 ? 950 : i)); + } + SPIFFS.remove("/contacts3"); + Path path; + assert(snapshot.copyPathTo(path.data()) && path == route(349)); + const auto original = SPIFFS.files.at("/contacts4_00"); + assert(f.host.contacts[0].setRawPath(route(990).data())); + SPIFFS.max_write = 13; + assert(!f.store.writeContactPage(&f.host, 0, cachedContactFilter)); + assert(SPIFFS.files.at("/contacts4_00") == original); + f.check(0, route(990)); + SPIFFS.max_write = std::numeric_limits::max(); + assert(f.store.saveContacts(&f.host, cachedContactFilter)); + f.check(0, route(990)); + assert(snapshot.copyPathTo(path.data()) && path == route(349)); + for (unsigned i = 1; i < 350; ++i) f.check(i, route(i == 349 ? 950 : i)); + SPIFFS.files["/contacts4_00"][20 + 3 * 152 + 76] ^= 8; + assert(!f.host.contacts[3].getPath()); + assert(!f.store.writeContactPage(&f.host, 0, cachedContactFilter)); +} +#endif + +int main() { +#if defined(ESP32_PLATFORM) + routes_survive_eviction_and_full_sync(); + snapshot_rollback_and_dirty_eviction(); + storage_failures_preserve_routes(); + rename_gaps_recover_a_complete_contact_file(); + pinned_snapshots_bound_ram_without_losing_updates(); + encoded_paths_and_copy_assignment(); + failed_path_reads_fail_sends_without_leaking_packets(); + contact_reload_preserves_disk_on_cache_exhaustion(); + partial_loads_cannot_replace_the_complete_file(); + secrets_load_from_flash_instead_of_recalculating(); + persisted_secret_matches_the_real_peer_key_exchange(); + secret_corruption_reordering_and_full_storage(); +#else + nrf52_secret_cache_leaves_contact_storage_reserve(); + nrf52_migration_snapshots_and_failed_page_replacement(); + encoded_paths_and_copy_assignment(); + failed_path_reads_fail_sends_without_leaking_packets(); + contact_reload_preserves_disk_on_cache_exhaustion(); + secrets_load_from_flash_instead_of_recalculating(); + persisted_secret_matches_the_real_peer_key_exchange(); +#endif + return 0; +} diff --git a/test/fixtures/contact_cache/test_secret_power.cpp b/test/fixtures/contact_cache/test_secret_power.cpp new file mode 100644 index 000000000..70f557a17 --- /dev/null +++ b/test/fixtures/contact_cache/test_secret_power.cpp @@ -0,0 +1,44 @@ +#include +#include +#include + +class NoIoBackend : public mesh::ContactSecretBackend { +public: + bool readSavedSecret(const uint8_t*, const uint8_t*, uint8_t*) override { + assert(false && "ESP32 cache misses must not read flash"); + return false; + } + bool saveSecret(const uint8_t*, const uint8_t*, const uint8_t*) override { + assert(false && "ESP32 cache misses must not write flash"); + return false; + } +}; + +int main() { + NoIoBackend backend; + mesh::ContactSecretCache cache; + cache.attach(&backend); + mesh::LocalIdentity self, peer; + uint8_t seed[32] = {1}; + ed25519_create_keypair(self.pub_key, self.private_key, seed); + seed[0] = 2; + ed25519_create_keypair(peer.pub_key, peer.private_key, seed); + uint8_t expected[32]; + peer.calcSharedSecret(expected, self.pub_key); + assert(memcmp(cache.get(self, peer.pub_key), expected, 32) == 0); + assert(memcmp(cache.get(self, peer.pub_key), expected, 32) == 0); + assert(self.derivations == 1 && cache.ram_hits == 1); + for (unsigned i = 3; i < 20; ++i) { + mesh::LocalIdentity other; + seed[0] = i; + ed25519_create_keypair(other.pub_key, other.private_key, seed); + cache.get(self, other.pub_key); + } + const unsigned before = self.derivations; + assert(memcmp(cache.get(self, peer.pub_key), expected, 32) == 0); + assert(self.derivations == before + 1); + assert(cache.flash_hits == 0 && cache.save_failures == 0); + self.private_key[1] ^= 2; + assert(memcmp(cache.get(self, peer.pub_key), expected, 32) != 0); + assert(self.derivations == before + 2); +} diff --git a/test/test_contact_cache.py b/test/test_contact_cache.py new file mode 100644 index 000000000..35b99b512 --- /dev/null +++ b/test/test_contact_cache.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Exercise real contact handles, cache policy and persistence under sanitizers.""" +from pathlib import Path +import subprocess +import tempfile +import unittest +from test_t096_full_memory import method + +ROOT = Path(__file__).resolve().parents[1] + + +def crypto_objects(temp): + objects = [] + for name in ("fe", "ge", "sc", "sha512", "keypair", "key_exchange"): + obj = temp / (name + ".o") + subprocess.run([ + "cc", "-O1", "-g", "-fsanitize=address", "-fwrapv", "-c", + str(ROOT / "lib/ed25519" / (name + ".c")), "-o", str(obj), + ], capture_output=True, text=True, check=True) + objects.append(str(obj)) + return objects + + +class ContactCacheTest(unittest.TestCase): + def test_esp32_misses_recalculate_without_flash_io(self): + with tempfile.TemporaryDirectory(prefix="mesh-secret-power-") as temp: + temp = Path(temp) + binary = temp / "test" + result = subprocess.run([ + "c++", "-std=c++17", "-O1", "-g", "-fsanitize=address,undefined", + "-DESP32_PLATFORM=1", "-DCOMPANION_RADIO_FULL=1", + "-I", str(ROOT / "test/fixtures/contact_cache/mocks"), + "-I", str(ROOT / "lib/ed25519"), "-I", str(ROOT / "src"), + str(ROOT / "test/fixtures/contact_cache/test_secret_power.cpp"), + *crypto_objects(temp), "-lcrypto", "-o", str(binary), + ], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + result = subprocess.run([str(binary)], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_eviction_snapshots_transactions_and_identity_changes(self): + self.run_cache_test("ESP32_PLATFORM") + + def test_nrf52_paged_store_migration_and_snapshots(self): + self.run_cache_test("NRF52_PLATFORM") + + def run_cache_test(self, platform): + with tempfile.TemporaryDirectory(prefix="mesh-contact-cache-") as temp: + temp = Path(temp) + store = (ROOT / "examples/companion_radio/DataStore.cpp").read_text() + functions = [ + "static bool serializeContactRecord(", + "static bool deserializeContactRecord(", + "void DataStore::loadContacts(", + "bool DataStore::saveContacts(", + "bool DataStore::flushContactWrites(", + "bool DataStore::readStoredPath(", + "bool DataStore::flushCachedPaths(", + "uint16_t DataStore::secretSlot(", + "bool DataStore::readSavedSecret(", + "bool DataStore::saveSecret(", + ] + helper_start = store.index("namespace {\nbool cachedContactFilter") + helper_end = store.index("bool DataStore::readStoredPath(", helper_start) + implementation = store[helper_start:helper_end] + if platform == "NRF52_PLATFORM": + implementation += method(store, "static void makeContactPagePath(") + "\n" + implementation += method(store, "static void discardInvalidContactPage(") + "\n" + functions.append("bool DataStore::writeContactPage(") + functions.append("bool DataStore::loadContactPages(") + implementation += "\n".join(method(store, signature) for signature in functions) + (temp / "store_under_test.h").write_text(implementation) + packet = (ROOT / "src/Packet.cpp").read_text() + (temp / "packet_under_test.h").write_text("namespace mesh {\n" + "\n".join( + method(packet, signature) for signature in ( + "Packet::Packet()", "int Packet::getRawLength() const", + "bool Packet::isValidPathLen(", "size_t Packet::writePath(", + "uint8_t Packet::copyPath(")) + "\n}\n") + chat = (ROOT / "src/helpers/BaseChatMesh.cpp").read_text() + mesh = (ROOT / "src/Mesh.cpp").read_text() + (temp / "send_under_test.h").write_text( + "namespace mesh {\n" + method(mesh, "bool Mesh::sendDirect(") + "\n}\n" + + "\n".join(method(chat, signature) for signature in ( + "int BaseChatMesh::sendLogin(", "int BaseChatMesh::sendAnonReq(", + "int BaseChatMesh::sendRequest(const ContactInfo& recipient, const uint8_t*", + "int BaseChatMesh::sendRequest(const ContactInfo& recipient, uint8_t")) + ) + binary = temp / "test" + result = subprocess.run([ + "c++", "-std=c++17", "-O1", "-g", "-Wall", "-Wextra", + "-fsanitize=address,undefined", "-fno-omit-frame-pointer", + "-DMESH_CONTACT_CACHE=1", "-DCOMPANION_RADIO_FULL=1", + "-DMESH_CONTACT_SECRET_FLASH_CACHE=1", + "-D" + platform + "=1", "-DMAX_CONTACTS=350", + "-I", str(ROOT / "test/fixtures/contact_cache/mocks"), + "-I", str(ROOT / "lib/ed25519"), + "-I", str(ROOT / "src"), "-I", str(temp), + str(ROOT / "test/fixtures/contact_cache/test_contact_cache.cpp"), + str(ROOT / "src/helpers/ContactInfo.cpp"), *crypto_objects(temp), + "-lcrypto", "-o", str(binary), + ], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + result = subprocess.run([str(binary)], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_nrf52_extrafs_contract.py b/test/test_nrf52_extrafs_contract.py index 6113ccc22..d8d28ec43 100644 --- a/test/test_nrf52_extrafs_contract.py +++ b/test/test_nrf52_extrafs_contract.py @@ -3,6 +3,7 @@ from pathlib import Path import re +import subprocess import unittest @@ -115,7 +116,14 @@ class Nrf52ExtraFsContractTest(unittest.TestCase): self.assertLess(incomplete_at, source_at) self.assertLess(legacy_stat_at, source_at) self.assertLess(marker_stat_at, source_at) - self.assertNotIn("->exists(", load_contacts) + # ESP32 has a separate SPIFFS recovery path. Check the actual nRF52 + # branch so its tri-state discovery cannot regress to exists(). + nrf_load = subprocess.run( + ["c++", "-E", "-P", "-x", "c++", "-DNRF52_PLATFORM=1", + "-DMESH_CONTACT_CACHE=1", "-"], + input=load_contacts, text=True, capture_output=True, check=True, + ).stdout + self.assertNotIn("->exists(", nrf_load) for body in (prepare, truncate, service): self.assertIn("contactPathPresence(", body) @@ -231,7 +239,8 @@ class Nrf52ExtraFsContractTest(unittest.TestCase): terminal_path = function_body(mesh, "void MyMesh::handleTerminalPath(") self.assertIn("const ContactInfo previous = recipient", terminal_path) - self.assertIn("if (!scheduleContactWrite(recipient))", terminal_path) + self.assertIn("if (!recipient.setPath(", terminal_path) + self.assertIn("|| !scheduleContactWrite(recipient)", terminal_path) self.assertIn("recipient = previous", terminal_path) terminal = function_body(mesh, "void MyMesh::handleTerminalCommand(") diff --git a/tools/hil/contact_cache_serial_stress.py b/tools/hil/contact_cache_serial_stress.py new file mode 100644 index 000000000..66de5c497 --- /dev/null +++ b/tools/hil/contact_cache_serial_stress.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Qualify a 350-contact Full Companion's path and shared-secret caches. + +Requires pyserial and exclusive access to the specified Companion data port. +--populate writes 350 deterministic contacts and changes the node's name; +use a dedicated test node. --secrets sends 23 directed LoRa CLI datagrams to +synthetic peers to exercise eviction. Without either flag, only local reads +are performed. --reboot restarts the node after validation. + +Run again without --populate after reboot to check persistence. The generation +must match the previous population. Results are newline-delimited JSON. +""" + +import argparse +import hashlib +import json +import re +import struct +import time + +import serial + +from esp32_companion_serial_stress import ( + DeviceFrameReader, + TransportCounters, + encode_host_frame, + validate_core_stats_response, +) + + +def report(event, **values): + print(json.dumps(dict(event=event, **values)), flush=True) + + +def secret_counters(reply): + return { + key: int(value) + for key, value in re.findall( + r"(ram_hits|flash_hits|calculations|save_skips)=(\d+)", reply + ) + } + + +def run(args): + port = serial.Serial(None, 115200, timeout=0.05, write_timeout=10) + port.dtr = False + port.rts = False + port.port = args.port + port.open() + try: + port.dtr = True + time.sleep(4) + reader = DeviceFrameReader(TransportCounters()) + + def send(payload): + port.write(encode_host_frame(payload)) + port.flush() + + def request(payload, expected, timeout=12): + send(payload) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + value = reader.read_frame(port, deadline - time.monotonic()) + if value[0] == expected: + return value + if value[0] == 1: + raise RuntimeError(f"Command {payload[0]} failed: {value.hex()}") + raise TimeoutError(f"Command {payload[0]}") + + def cli(command): + reply = request(b"\x42" + command.encode("ascii"), 29)[1:].decode("ascii") + report("cli", command=command, reply=reply) + return reply + + def fixture(index): + key = hashlib.sha256(f"contact-cache-hil-{index}".encode()).digest() + route = bytes( + (index * 19 + j + (index >> 8) + args.generation * 37) % 256 + for j in range(64) + ) + name = f"Cache test {index:03d}".encode("ascii").ljust(32, b"\0") + return key, route, name + + version = request(b"\x16\x0e", 13) + report("device", board=version[20:60].split(b"\0")[0].decode("ascii"), + contacts=version[2] * 2, channels=version[3]) + assert version[2] * 2 == 350 and version[3] == 40 + request(b"\x01" + b"\0" * 7 + b"Contact Cache HIL", 5) + policy = cli("get contact.cache") + assert "paths=16 secrets=16" in policy + assert ("miss=calculate" if args.expect_calculate else "miss=flash") in policy + cli("memory") # Some nRF52 builds do not implement this diagnostic. + + if args.populate: + request(b"\x08" + args.name.encode("ascii"), 0) + for index in range(350): + key, route, name = fixture(index) + data = (b"\x09" + key + b"\x01\x00\x3f" + route + name + + b"\0" * 12 + b"\x01\0\0\0") + try: + request(data, 0, timeout=60) + except Exception: + report("insert_failed", index=index) + raise + if index % 25 == 24: + stats = validate_core_stats_response(request(b"\x38\x00", 24)) + report("inserted", count=index + 1, **stats) + time.sleep(6) + + start = request(b"\x04", 2, timeout=30) + assert int.from_bytes(start[1:5], "little") == 350 + expected = {fixture(i)[0]: fixture(i) for i in range(350)} + seen = set() + while True: + value = reader.read_frame(port, 30) + if value[0] == 4: + break + if value[0] != 3: + continue + key = value[1:33] + assert len(value) == 148 and key in expected and key not in seen + assert value[35] == 63 and value[36:100] == expected[key][1] + assert value[100:132] == expected[key][2] + seen.add(key) + assert seen == expected.keys() + report("sync", contacts=len(seen), verified_path_bytes=64) + for index in range(350): + key, route, _ = fixture(index) + value = request(b"\x1e" + key, 3) + assert value[36:100] == route + report("individual_reads", contacts=350) + + if args.secrets: + before = secret_counters(cli("get contact.cache")) + peers = list(range(args.secret_start, args.secret_start + 21)) + for index in peers + [args.secret_start, args.secret_start]: + key, _, _ = fixture(index) + # CLI datagrams do not occupy an ACK/correlation slot. + message = (b"\x02\x01\x00" + struct.pack("= 1 and delta["save_skips"] == 0 + if args.expect_calculate: + assert delta["flash_hits"] == 0 and delta["calculations"] >= 22 + else: + assert delta["flash_hits"] >= 1 + report("secret_eviction", **delta) + cli("get contact.cache.timing") + + cli("memory") + report("stats", **validate_core_stats_response(request(b"\x38\x00", 24))) + if args.reboot: + send(b"\x13reboot") + report("reboot_requested") + report("passed") + finally: + port.close() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--port", required=True) + parser.add_argument("--populate", action="store_true") + parser.add_argument("--secrets", action="store_true") + parser.add_argument("--reboot", action="store_true") + parser.add_argument("--generation", type=int, default=0) + parser.add_argument("--secret-start", type=int, default=0) + parser.add_argument("--expect-calculate", action="store_true") + parser.add_argument("--name", default="Contact-Cache-HIL") + args = parser.parse_args() + if not 0 <= args.secret_start <= 329: + parser.error("--secret-start must leave 21 peers within contacts 0..349") + if not args.name.isascii() or len(args.name) > 31: + parser.error("--name must contain at most 31 ASCII characters") + run(args) + + +if __name__ == "__main__": + main()