#include <Arduino.h>
#include <stdlib.h>
#include <initializer_list>
#include <new>
#include "DataStore.h"
#include "PrefsStorageLayout.h"
#if defined(RAK_4631)
#include <helpers/PeripheralModePrefs.h>
#endif
#include <helpers/FileRead.h>
#include <helpers/AdvertDataHelpers.h>
#include <helpers/LazyPersistence.h>
#if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM)
#include <helpers/ContactFileTransaction.h>
#endif
#if defined(ESP32_PLATFORM)
#include <errno.h>
#include <sys/stat.h>
#endif
#if COMPANION_FEATURE_READER
#include <helpers/bible/ReaderBookmarkFiles.h>
#endif

#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
#include <helpers/AtomicFileWriter.h>
#if defined(NRF52_PLATFORM) && defined(EXTRAFS) && !defined(QSPIFLASH)
#include <helpers/nrf52/InternalSecondaryFsRepair.h>
#include "ResilientInternalExtraFS.h"
#endif
#if defined(NRF52_PLATFORM)
#include <helpers/nrf52/RamFallbackFileSystem.h>
#endif
#endif

// Linked presence of this symbol is the authoritative signal that this firmware actually mounts the
// internal 0xD4000 ExtraFS. OTA layout code references it weakly, so non-companion roles can reclaim the
// reserved range even though nrf52_base defines EXTRAFS globally.
#if defined(NRF52_PLATFORM) && defined(EXTRAFS) && !defined(QSPIFLASH)
extern "C" __attribute__((used)) const uint8_t g_meshcore_internal_extrafs = 1u;
extern "C" uint32_t __flash_arduino_end[];
#endif

#if defined(EXTRAFS) || defined(QSPIFLASH)
  #define MAX_BLOBRECS 100
#else
  #define MAX_BLOBRECS 20
#endif

DataStore::DataStore(FILESYSTEM& fs, mesh::RTCClock& clock) : _fs(&fs), _fsExtra(nullptr),
    _configuredFsExtra(nullptr), _clock(&clock),
#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
    identity_store(fs, "")
#elif defined(RP2040_PLATFORM)
    identity_store(fs, "/identity")
#else
    identity_store(fs, "/identity")
#endif
{
}

DataStore::~DataStore() {
#if defined(ESP32_PLATFORM)
  cancelContactWrite();
#endif
}

#if defined(EXTRAFS) || defined(QSPIFLASH)
DataStore::DataStore(FILESYSTEM& fs, FILESYSTEM& fsExtra, mesh::RTCClock& clock) : _fs(&fs), _fsExtra(&fsExtra),
    _configuredFsExtra(&fsExtra), _clock(&clock),
#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
    identity_store(fs, "")
#elif defined(RP2040_PLATFORM)
    identity_store(fs, "/identity")
#else
    identity_store(fs, "/identity")
#endif
{
}
#endif

#if !defined(NRF52_PLATFORM)
// Unlike ESP32 FS::exists(), a metadata probe must not treat failure to open
// an existing file as evidence that its durable contents are absent.
static bool companionPathPresence(FILESYSTEM* fs, const char* path,
                                  bool& present) {
#if defined(ESP32_PLATFORM)
  (void)fs; // Companion storage is mounted at the default SPIFFS VFS path.
  char vfs_path[96];
  const int length = snprintf(vfs_path, sizeof(vfs_path), "/spiffs%s", path);
  if (length < 0 || static_cast<size_t>(length) >= sizeof(vfs_path)) return false;
  struct stat info;
  const int result = ::stat(vfs_path, &info);
  if (result != 0 && errno != ENOENT) return false;
  present = result == 0;
#elif defined(STM32_PLATFORM)
  struct lfs_info info;
  fs->_lockFS();
  const int result = lfs_stat(fs->_getFS(), path, &info);
  fs->_unlockFS();
  if (result != 0 && result != LFS_ERR_NOENT) return false;
  present = result == 0;
#else
  // Arduino-Pico's exists() uses lfs_stat(), not an open operation.
  // Its public API hides metadata errors; existing-file open/read failures
  // are distinguishable below, but metadata I/O failure is not.
  present = fs->exists(path);
#endif
  return true;
}
#if defined(ESP32_PLATFORM)
// Only called after the alternate image has been read and validated in full.
// Irrecoverable settings/channels may be replaced, but the verified recovery
// source stays intact until its rename has actually succeeded.
static bool promoteCompanionRecoveryFile(FILESYSTEM* fs, const char* target,
                                         const char*& source) {
  if (source == nullptr) return true;
  bool target_exists = false, source_exists = false;
  if (!companionPathPresence(fs, target, target_exists)
      || !companionPathPresence(fs, source, source_exists) || !source_exists) {
    return false;
  }
  if (target_exists && !fs->remove(target)) return false;
  if (!fs->rename(source, target)) return false;
  source = nullptr;
  return true;
}
#endif
#endif

static File openWrite(FILESYSTEM* fs, const char* filename) {
#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
  fs->remove(filename);
  return fs->open(filename, FILE_O_WRITE);
#elif defined(RP2040_PLATFORM)
  return fs->open(filename, "w");
#else
  return fs->open(filename, "w", true);
#endif
}

#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
static bool validateLfsFilesystem(FILESYSTEM* fs);
#endif
#if defined(NRF52_PLATFORM)
static void cleanupAtomicTempFiles(FILESYSTEM* fs);
static bool contactPathPresence(FILESYSTEM* fs, const char* path,
                                bool& present, uint32_t* size = nullptr);
#endif

void DataStore::begin() {
#if defined(ESP32_PLATFORM)
  _advert_write.clear();
  // Reinitialization must not retain a streaming transaction or an inode
  // opened through an earlier filesystem route. Dirty mutations stay pending.
  cancelContactWrite();
#if MESH_CONTACT_CACHE
  _contact_path_reader.close();
#endif
#endif
#if defined(RP2040_PLATFORM)
  identity_store.begin();
#endif

#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
#if defined(NRF52_PLATFORM)
  resetContactPageState(true);
  if (_secondary_authority_unknown) {
    _contact_load_incomplete = true;
    _prefs_load_incomplete = true;
  }
  if (_primary_storage_unavailable) {
    // The base mount already failed. Do not traverse or open this unmounted
    // filesystem; both can re-enter corrupt metadata before explicit repair.
    _identity_creation_blocked = true;
    _contact_load_incomplete = true;
    _fsExtra = nullptr;
    return;
  }

  bool primary_ready = validateLfsFilesystem(_fs);
  if (!primary_ready) {
    for (uint8_t retry = 0; retry < 3 && !primary_ready; ++retry) {
      delay(3000);
      primary_ready = validateLfsFilesystem(_fs);
    }
    if (!primary_ready) {
      MESH_DEBUG_PRINTLN(
          "DataStore: primary metadata remains unreadable after delayed retries; reinitializing storage");
      primary_ready = _fs->format() && validateLfsFilesystem(_fs);
      if (!primary_ready) {
        mesh::storage::RamFallbackFileSystem* ram_primary =
            mesh::storage::createRamFallbackFileSystem();
        if (ram_primary == nullptr) {
          _primary_storage_unavailable = true;
          _identity_creation_blocked = true;
          _contact_load_incomplete = true;
          _prefs_load_incomplete = true;
          _fsExtra = nullptr;
          return;
        }
        MESH_DEBUG_PRINTLN(
            "DataStore: verified primary erase/write failed; using volatile RAM filesystem");
        useVolatilePrimaryFS(ram_primary->filesystem());
        primary_ready = true;
      } else {
        _identity_creation_blocked = false;
        _contact_load_incomplete = false;
        _prefs_load_incomplete = false;
#if defined(EXTRAFS) && !defined(QSPIFLASH)
        if (_configuredFsExtra != nullptr) {
          ResilientInternalExtraFS* extra =
              static_cast<ResilientInternalExtraFS*>(_configuredFsExtra);
          const bool scan = !extra->pageMapReady();
          if (scan) extra->resetPageMapForDestructiveRecovery();
          else extra->requirePageMapRewrite();
          if (!reinitializeInternalExtraFS(scan)) disableSecondaryFS(false);
        }
#endif
      }
    }
  }

  // A mounted filesystem can still contain an unreadable identity record.
  // Keep the normal bounded reads, then try three more times three seconds
  // apart. Only after those fail may startup wipe the unusable primary store.
  mesh::LocalIdentity identity_probe;
  IdentityLoadResult identity_state =
      identity_store.loadResult("_main", identity_probe);
  if (identity_state == IdentityLoadResult::Unreadable) {
    for (uint8_t retry = 0; retry < 3; ++retry) {
      delay(3000);
      identity_state = identity_store.loadResult("_main", identity_probe);
      if (identity_state != IdentityLoadResult::Unreadable) break;
    }
  }
  if (identity_state == IdentityLoadResult::Unreadable) {
    MESH_DEBUG_PRINTLN(
        "DataStore: identity remains unreadable after delayed retries; reinitializing primary storage");
    const bool primary_reinitialized = _fs->format()
        && validateLfsFilesystem(_fs);
    if (!primary_reinitialized) {
      mesh::storage::RamFallbackFileSystem* ram_primary =
          mesh::storage::createRamFallbackFileSystem();
      if (ram_primary == nullptr) {
        _primary_storage_unavailable = true;
        _identity_creation_blocked = true;
        _contact_load_incomplete = true;
        _prefs_load_incomplete = true;
        _fsExtra = nullptr;
        return;
      }
      MESH_DEBUG_PRINTLN(
          "DataStore: verified primary erase/write failed; using volatile RAM filesystem");
      useVolatilePrimaryFS(ram_primary->filesystem());
    } else {
      _identity_creation_blocked = false;
      _contact_load_incomplete = false;
      _prefs_load_incomplete = false;
#if defined(EXTRAFS) && !defined(QSPIFLASH)
      if (_configuredFsExtra != nullptr) {
        ResilientInternalExtraFS* extra =
            static_cast<ResilientInternalExtraFS*>(_configuredFsExtra);
        const bool scan = !extra->pageMapReady();
        if (scan) extra->resetPageMapForDestructiveRecovery();
        else extra->requirePageMapRewrite();
        if (!reinitializeInternalExtraFS(scan)) {
          // The primary identity is now a fresh authoritative install. An
          // unusable old secondary must not block that new identity.
          disableSecondaryFS(false);
        }
      }
#endif
    }
  }
#if defined(EXTRAFS) && !defined(QSPIFLASH)
  // Validate primary first: automatic secondary recovery must never hide a
  // primary/identity fault. Retry or rebuild before loading any RAM state.
  recoverInternalExtraFSOnBoot();
#else
  if (_fsExtra != nullptr && !validateLfsFilesystem(_fsExtra)) {
    // Automatic destructive recovery is restricted to reserved internal
    // ExtraFS. Removable/external QSPI still requires an explicit repair.
    MESH_DEBUG_PRINTLN("DataStore: secondary LittleFS metadata is corrupt; preserving it and using primary storage");
    disableSecondaryFS(true);
  }
#endif
  if (primary_ready) cleanupAtomicTempFiles(_fs);
  if (_fsExtra != nullptr) cleanupAtomicTempFiles(_fsExtra);
#endif
  #if defined(EXTRAFS) || defined(QSPIFLASH)
  if (_fsExtra != nullptr && !migrateToSecondaryFS()) {
    MESH_DEBUG_PRINTLN("DataStore: one or more secondary filesystem migrations remain pending");
  }
  #endif
#if defined(NRF52_PLATFORM)
  // A journal-stat failure leaves the contact/channel authority unknowable.
  // Do not mutate the reconstructable advert cache on either candidate store
  // while that fail-closed latch is active.
  if (!_contact_load_incomplete) checkAdvBlobFile();
#else
  checkAdvBlobFile();
#endif
#else
  // init 'blob store' support
  _fs->mkdir("/bl");
#endif
}

#if defined(ESP32)
  #include <SPIFFS.h>
  #include <nvs_flash.h>
#elif defined(RP2040_PLATFORM)
  #include <LittleFS.h>
#elif defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
  #if defined(QSPIFLASH)
    #include <CustomLFS_QSPIFlash.h>
  #elif defined(EXTRAFS)
    #include "ResilientInternalExtraFS.h"
  #else 
    #include <InternalFileSystem.h>
  #endif
#endif

#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
struct LfsTraversalState {
  lfs_size_t visited;
  lfs_size_t block_count;
};

int _countLfsBlock(void *p, lfs_block_t block){
  LfsTraversalState* state = (LfsTraversalState*)p;
  // Valid blocks are [0, block_count).  A traversal longer than block_count
  // indicates a metadata cycle even if each individual block number is valid.
  if (block >= state->block_count || state->visited >= state->block_count) {
    MESH_DEBUG_PRINTLN("ERROR: LittleFS traversal out of bounds/cyclic at block %lu",
                       (unsigned long)block);
    return LFS_ERR_CORRUPT;
  }
  state->visited++;
  return 0;
}

lfs_ssize_t _getLfsUsedBlockCount(FILESYSTEM* fs) {
  if (fs == nullptr) return -1;
  // Raw LittleFS traversal touches the same metadata/cache as file writes.
  // Hold this filesystem's lock, separately from the physical flash lock.
  fs->_lockFS();
  lfs_t* raw_fs = fs->_getFS();
  if (raw_fs == nullptr || raw_fs->cfg == nullptr
      || raw_fs->cfg->block_count == 0) {
    fs->_unlockFS();
    return -1;
  }
  LfsTraversalState state = {0, raw_fs->cfg->block_count};
  int err = lfs_traverse(raw_fs, _countLfsBlock, &state);
  fs->_unlockFS();
  if (err) {
    MESH_DEBUG_PRINTLN("ERROR: lfs_traverse() error: %d", err);
    return -1;
  }
  return state.visited;
}

static bool validateLfsFilesystem(FILESYSTEM* fs) {
  return fs != nullptr && _getLfsUsedBlockCount(fs) >= 0;
}
#endif

#if defined(NRF52_PLATFORM)
static void cleanupAtomicTempFiles(FILESYSTEM* fs) {
  if (fs == nullptr) return;

  static const char* fixed_temp_paths[] = {
      "/_main.id.tmp", "/new_prefs.tmp", "/channels2.tmp",
      "/contacts3.tmp", "/contacts4.mig.tmp", "/adv_blobs.tmp",
      "/.extrafs.mig.tmp", "/extrafs.badpages.tmp",
      "/extrafs.badpages.bak.tmp", "/mesh_mt.tmp"};
  for (size_t i = 0; i < sizeof(fixed_temp_paths) / sizeof(fixed_temp_paths[0]); i++) {
    if (fs->exists(fixed_temp_paths[i])) fs->remove(fixed_temp_paths[i]);
  }

  // Include the old ten-bucket range as well as the current five-bucket
  // layout so interrupted preview builds cannot strand full LittleFS blocks.
  for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) {
    char path[28];
    snprintf(path, sizeof(path), "/contacts4_%02u.tmp", (unsigned)page);
    if (fs->exists(path)) fs->remove(path);
  }
  for (uint8_t bucket = 0; bucket < 10; bucket++) {
    char path[24];
    snprintf(path, sizeof(path), "/adv4_%02u.tmp", (unsigned)bucket);
    if (fs->exists(path)) fs->remove(path);
  }
}

#if defined(EXTRAFS) && !defined(QSPIFLASH)
bool DataStore::recoverInternalExtraFSOnBoot() {
  if (_configuredFsExtra == nullptr) return false;

  ResilientInternalExtraFS* extra =
      static_cast<ResilientInternalExtraFS*>(_configuredFsExtra);
  if (_primary_storage_unavailable
      || !mesh::storage::isExpectedInternalExtraFsGeometry(
          extra->getFlashAddr(), extra->getFlashSize(), extra->getBlockSize())
      || !mesh::storage::isInternalExtraFsReservedByApplication(
          (uint32_t)(uintptr_t)__flash_arduino_end)) {
    disableSecondaryFS(true);
    MESH_DEBUG_PRINTLN("DataStore: refusing automatic ExtraFS recovery outside reserved 100 KiB region");
    return false;
  }

  if (!extra->pageMapReady()) {
    MESH_DEBUG_PRINTLN(
        "DataStore: both ExtraFS page maps are unreadable; scanning and rebuilding reserved secondary storage");
    extra->resetPageMapForDestructiveRecovery();
    if (!reinitializeInternalExtraFS(true)) {
      disableSecondaryFS(true);
      return false;
    }
    _secondary_authority_unknown = false;
    _contact_load_incomplete = false;
    _identity_creation_blocked = false;
    _prefs_load_incomplete = false;
    return true;
  }

  if (extra->recoveryPending()) {
    MESH_DEBUG_PRINTLN("DataStore: running requested boot-time ExtraFS scan");
    if (!reinitializeInternalExtraFS(true)) {
      disableSecondaryFS(true);
      return false;
    }
    _secondary_authority_unknown = false;
    _contact_load_incomplete = false;
    _identity_creation_blocked = false;
    _prefs_load_incomplete = false;
    return true;
  }

  const mesh::storage::InternalSecondaryFsRecoveryResult result =
      mesh::storage::recoverInternalSecondaryFilesystem(
          [this]() -> bool {
            return _fsExtra == _configuredFsExtra
                && validateLfsFilesystem(_fsExtra);
          },
          [this, extra]() -> bool {
            MESH_DEBUG_PRINTLN("DataStore: retrying internal ExtraFS mount before recovery");
            _fsExtra = nullptr;
            extra->end();
            if (!extra->Adafruit_LittleFS::begin()) return false;
            _fsExtra = _configuredFsExtra;
            return true;
          },
          [this]() -> bool {
            MESH_DEBUG_PRINTLN("DataStore: internal ExtraFS remains unusable after mount retry; scanning and rebuilding (secondary-only data may be lost)");
            return reinitializeInternalExtraFS(true);
          });
  if (result == mesh::storage::InternalSecondaryFsRecoveryResult::Failed) {
    disableSecondaryFS(true);
    return false;
  }

  extra->acknowledgeRecoveredBootHint();
  if (extra->pageMapNeedsSave() && !extra->savePageMap(*_fs)) {
    disableSecondaryFS(true);
    return false;
  }

  // This runs only before migration and user-data loading. Clear the initial
  // mount quarantine, not errors from a later incomplete contact/prefs load.
  // Migration below establishes authority again and can re-latch any error.
  _secondary_authority_unknown = false;
  _contact_load_incomplete = false;
  _identity_creation_blocked = false;
  _prefs_load_incomplete = false;
  return true;
}

bool DataStore::reinitializeInternalExtraFS(bool scan_physical_pages) {
  if (_configuredFsExtra == nullptr) return false;

  ResilientInternalExtraFS* extra =
      static_cast<ResilientInternalExtraFS*>(_configuredFsExtra);
  if (!mesh::storage::isExpectedInternalExtraFsGeometry(
          extra->getFlashAddr(), extra->getFlashSize(),
          extra->getBlockSize())
      || !mesh::storage::isInternalExtraFsReservedByApplication(
          (uint32_t)(uintptr_t)__flash_arduino_end)) {
    _fsExtra = nullptr;
    MESH_DEBUG_PRINTLN("DataStore: refusing internal ExtraFS repair with unexpected geometry");
    return false;
  }
  if (!extra->pageMapReady()) {
    if (!scan_physical_pages) {
      _fsExtra = nullptr;
      return false;
    }
    extra->resetPageMapForDestructiveRecovery();
  }

  // A physical test is destructive and must run only during early boot, before
  // radio/UI tasks can issue competing SoftDevice flash operations. Ordinary
  // USB repair and factory-reset paths retain the already-verified page map.
  if ((scan_physical_pages && !extra->scanAndRetireBadPages())
      || (extra->pageMapNeedsSave() && !extra->savePageMap(*_fs))) {
    _fsExtra = nullptr;
    MESH_DEBUG_PRINTLN("DataStore: ExtraFS physical scan or bad-page map save failed");
    return false;
  }

  const mesh::storage::InternalSecondaryFsRepairResult result =
      mesh::storage::repairInternalSecondaryFilesystem(
          [extra]() -> bool {
            // Avoid CustomLFS::formatRegion(): it probes corrupt metadata with
            // open("/") before deciding whether to unmount. end() is safe for
            // both mounted and unmounted Adafruit LittleFS instances.
            extra->end();
            return extra->format();
          },
          [extra]() -> bool {
            // Call the base mount explicitly. CustomLFS::begin() would erase
            // the complete region again if this mount failed.
            return extra->Adafruit_LittleFS::begin();
          },
          [this]() -> bool {
            return validateLfsFilesystem(_configuredFsExtra);
          });
  if (result != mesh::storage::InternalSecondaryFsRepairResult::Repaired) {
    _fsExtra = nullptr;
    switch (result) {
      case mesh::storage::InternalSecondaryFsRepairResult::FormatFailed:
        extra->setStage(ResilientInternalExtraFS::Stage::FormatFailed);
        MESH_DEBUG_PRINTLN("DataStore: internal ExtraFS repair format failed");
        break;
      case mesh::storage::InternalSecondaryFsRepairResult::MountFailed:
        extra->setStage(ResilientInternalExtraFS::Stage::MountFailed);
        MESH_DEBUG_PRINTLN("DataStore: internal ExtraFS repair mount failed");
        break;
      case mesh::storage::InternalSecondaryFsRepairResult::ValidationFailed:
        extra->setStage(ResilientInternalExtraFS::Stage::ValidationFailed);
        MESH_DEBUG_PRINTLN("DataStore: internal ExtraFS repair validation failed");
        break;
      default:
        break;
    }
    return false;
  }

  _fsExtra = _configuredFsExtra;
  extra->setStage(ResilientInternalExtraFS::Stage::Repaired);
  MESH_DEBUG_PRINTLN("DataStore: internal ExtraFS repaired and reactivated");
  return true;
}
#endif
#endif

void DataStore::markPrimaryFSUnavailable() {
#if defined(NRF52_PLATFORM)
  _primary_storage_unavailable = true;
  _identity_creation_blocked = true;
  _contact_load_incomplete = true;
  _prefs_load_incomplete = true;
#endif
}

#if defined(NRF52_PLATFORM)
void DataStore::useVolatilePrimaryFS(FILESYSTEM& fs) {
  _fs = &fs;
  identity_store.useFileSystem(fs);
  _fsExtra = nullptr;
  _configuredFsExtra = nullptr;
  _volatile_primary_fs = true;
  _primary_storage_unavailable = false;
  _secondary_authority_unknown = false;
  _identity_creation_blocked = false;
  _contact_load_incomplete = false;
  _prefs_load_incomplete = false;
}

bool DataStore::loadMeshtextSettings(uint8_t image[40], size_t& size) {
  size = 0;
  if (image == nullptr || _fs == nullptr || _primary_storage_unavailable
      || _volatile_primary_fs) return false;
  bool present = false;
  uint32_t stored_size = 0;
  if (!contactPathPresence(_fs, "/mesh_mt", present, &stored_size)) return false;
  if (!present) return true;
  // This backend has never stored the legacy ESP32-only NVS formats. Do not
  // accept a truncated image, directory or uncommitted .tmp as fresh defaults.
  if (stored_size != 40) return false;
  for (uint8_t attempt = 0; attempt < 3; ++attempt) {
    File file = openRead(_fs, "/mesh_mt");
    if (!file) continue;
    uint8_t loaded[40];
    const bool ok = !file.isDirectory() && file.size() == sizeof(loaded)
        && file.read(loaded, sizeof(loaded)) == sizeof(loaded);
    file.close();
    if (ok) {
      memcpy(image, loaded, sizeof(loaded));
      size = sizeof(loaded);
      return true;
    }
  }
  return false;
}

bool DataStore::saveMeshtextSettings(const uint8_t image[40]) {
  if (image == nullptr || _fs == nullptr || _primary_storage_unavailable
      || _volatile_primary_fs) return false;
  bool present = false;
  if (!contactPathPresence(_fs, "/mesh_mt", present)) return false;
  if (!contactPathPresence(_fs, "/mesh_mt.tmp", present)) return false;
  if (present && !_fs->remove("/mesh_mt.tmp")) return false;
  mesh::AtomicFileWriter file(_fs, "/mesh_mt");
  return file && file.write(image, 40) == 40 && file.commit();
}

bool DataStore::eraseMeshtextSettings() {
  if (_fs == nullptr || _primary_storage_unavailable || _volatile_primary_fs) return false;
  bool present = false;
  // A staging file must never become a recovery source after explicit erase.
  for (const char* path : {"/mesh_mt.tmp", "/mesh_mt"}) {
    if (!contactPathPresence(_fs, path, present)) return false;
    if (present && !_fs->remove(path)) return false;
    if (!contactPathPresence(_fs, path, present) || present) return false;
  }
  return true;
}
#endif

void DataStore::disableSecondaryFS(bool authority_unknown) {
#if defined(ESP32_PLATFORM)
  cancelContactWrite();
#if MESH_CONTACT_CACHE
  _contact_path_reader.close();
#endif
#endif
  _fsExtra = nullptr;
#if defined(NRF52_PLATFORM)
  if (authority_unknown) {
    // Without reading the secondary journal, primary may be a retired source
    // rather than an empty fresh-install store. Keep this fact across begin()'s
    // runtime-state reset and block every authority-changing write.
    _secondary_authority_unknown = true;
    _contact_load_incomplete = true;
    _identity_creation_blocked = true;
    _prefs_load_incomplete = true;
  }
#else
  (void)authority_unknown;
#endif
}

uint32_t DataStore::getStorageUsedKb() const {
#if defined(ESP32)
  return SPIFFS.usedBytes() / 1024;
#elif defined(RP2040_PLATFORM)
  FSInfo info;
  info.usedBytes = 0;
  _fs->info(info);
  return info.usedBytes / 1024;
#elif defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
  const lfs_config* config = _getContactsChannelsFS()->_getFS()->cfg;
  int usedBlockCount = _getLfsUsedBlockCount(_getContactsChannelsFS());
  if (usedBlockCount < 0) return 0;
  int usedBytes = config->block_size * usedBlockCount;
  return usedBytes / 1024;
#else
  return 0;
#endif
}

uint32_t DataStore::getStorageTotalKb() const {
#if defined(ESP32)
  return SPIFFS.totalBytes() / 1024;
#elif defined(RP2040_PLATFORM)
  FSInfo info;
  info.totalBytes = 0;
  _fs->info(info);
  return info.totalBytes / 1024;
#elif defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
  const lfs_config* config = _getContactsChannelsFS()->_getFS()->cfg;
  int totalBytes = config->block_size * config->block_count;
  return totalBytes / 1024;
#else
  return 0;
#endif
}

File DataStore::openRead(const char* filename) {
  return openRead(_fs, filename);
}

#if COMPANION_FEATURE_READER
namespace {
class ReaderBookmarkFiles {
  FILESYSTEM* _fs;
public:
  explicit ReaderBookmarkFiles(FILESYSTEM* fs) : _fs(fs) {}
  bool exists(const char* path) {
#if defined(NRF52_PLATFORM)
    bool present = false;
    // Unknown presence must take the read/abort path, never the new-file path.
    return !contactPathPresence(_fs, path, present) || present;
#else
    return _fs->exists(path);
#endif
  }
  bool remove(const char* path) {
#if defined(NRF52_PLATFORM)
    bool present = false;
    if (!contactPathPresence(_fs, path, present)) return false;
    return !present || _fs->remove(path);
#else
    return !exists(path) || _fs->remove(path);
#endif
  }
  bool rename(const char* from, const char* to) { return _fs->rename(from, to); }
  bool read(const char* path, uint8_t (&data)[mesh::bible::kBookmarkBytes]) {
    File file = mesh::openFileRead(_fs, path);
    if (!file) return false;
    if (file.size() != sizeof(data)) {
      // Known malformed record, not an I/O failure. Let the codec reject it
      // and permit a later verified replacement instead of trapping resume.
      memset(data, 0, sizeof(data));
      file.close();
      return true;
    }
    const bool ok = file.read(data, sizeof(data)) == sizeof(data);
    file.close();
    return ok;
  }
  bool write(const char* path, const uint8_t (&data)[mesh::bible::kBookmarkBytes]) {
    File file = openWrite(_fs, path);
    if (!file) return false;
    const bool ok = file.write(data, sizeof(data)) == sizeof(data);
    file.flush();
    file.close();
    return ok;
  }
};
} // namespace

bool DataStore::loadReaderBookmark(mesh::bible::Position& pos) {
  pos = mesh::bible::Position{};
#if defined(NRF52_PLATFORM)
  if (_primary_storage_unavailable) return false;
#endif
  ReaderBookmarkFiles files(_fs);
  return mesh::bible::loadReaderBookmark(files, pos);
}

bool DataStore::saveReaderBookmark(mesh::bible::Position pos) {
#if defined(NRF52_PLATFORM)
  if (_primary_storage_unavailable) return false;
#endif
  ReaderBookmarkFiles files(_fs);
  return mesh::bible::saveReaderBookmark(files, pos);
}
#endif

File DataStore::openRead(FILESYSTEM* fs, const char* filename) {
  return mesh::openFileRead(fs, filename);
}

File DataStore::openDirectory(const char* path) {
  return openDirectory(_fs, path);
}

File DataStore::openDirectory(FILESYSTEM* fs, const char* path) {
  if (fs == nullptr || path == nullptr) return mesh::emptyFile(fs);
  // SPIFFS has virtual directories: do not use the regular-file existence
  // check here. Only the explicit listing API may return directory handles.
#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
  File directory = fs->open(path, FILE_O_READ);
#elif defined(RP2040_PLATFORM)
  File directory = fs->open(path, "r");
#else
  File directory = fs->open(path, "r", false);
#endif
  if (directory && !directory.isDirectory()) {
    directory.close();
    return mesh::emptyFile(fs);
  }
  return directory;
}

bool DataStore::removeFile(const char* filename) {
  return _fs->remove(filename);
}

bool DataStore::removeFile(FILESYSTEM* fs, const char* filename) {
  return fs->remove(filename);
}

bool DataStore::formatFileSystem() {
#if defined(ESP32_PLATFORM)
  // A failed format keeps the latest accepted RAM packets available to retry.
  _advert_write.pause();
  cancelContactWrite();
#if MESH_CONTACT_CACHE
  _contact_path_reader.close();
#endif
#endif
#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
  #if defined(NRF52_PLATFORM)
  resetContactPageState();
  #endif
  const bool primary_success = _fs->format();
#if defined(NRF52_PLATFORM) && defined(EXTRAFS) && !defined(QSPIFLASH)
  // Factory reset/rebuild is already an explicit destructive operation. Use
  // the configured pointer so it also clears and reactivates an ExtraFS which
  // normal boot deliberately quarantined after failed traversal validation.
  bool secondary_success = true;
  if (primary_success && _configuredFsExtra != nullptr) {
    ResilientInternalExtraFS* extra =
        static_cast<ResilientInternalExtraFS*>(_configuredFsExtra);
    if (!extra->pageMapReady()) {
      extra->resetPageMapForDestructiveRecovery();
    } else {
      // The primary format erased both map files. Recreate them before the
      // secondary format so a reboot never guesses a retired page is healthy.
      extra->requirePageMapRewrite();
    }
    secondary_success = reinitializeInternalExtraFS();
  }
#else
  const bool secondary_success = !primary_success
      || _fsExtra == nullptr || _fsExtra->format();
#endif
  const bool success = primary_success && secondary_success;
#if defined(NRF52_PLATFORM)
  // A successful explicit erase creates a new empty authoritative store. A
  // failed erase must keep the boot's load quarantine latched.
  if (success) {
    _contact_load_incomplete = false;
#if MESH_CONTACT_CACHE
    _cache_load_incomplete = false;
#endif
    _identity_creation_blocked = false;
    _prefs_load_incomplete = false;
  }
#else
  if (success) {
    _identity_creation_blocked = false;
    _prefs_load_incomplete = false;
    _channel_load_incomplete = false;
#if !MESH_CONTACT_CACHE
    _uncached_contact_load_incomplete = false;
#endif
  }
#endif
  return success;
#elif defined(RP2040_PLATFORM)
  const bool success = LittleFS.format();
  if (success) {
    _identity_creation_blocked = false;
    _prefs_load_incomplete = false;
    _channel_load_incomplete = false;
    _uncached_contact_load_incomplete = false;
  }
  return success;
#elif defined(ESP32)
  bool fs_success = ((fs::SPIFFSFS *)_fs)->format();
  esp_err_t nvs_err = nvs_flash_erase(); // no need to reinit, will be done by reboot
  if (fs_success && nvs_err == ESP_OK) {
    _advert_write.clear();
    _identity_creation_blocked = false;
    _prefs_load_incomplete = false;
    _channel_load_incomplete = false;
    _prefs_recovery_source = nullptr;
    _channel_recovery_source = nullptr;
    _contact_write_requested = false;
#if MESH_CONTACT_CACHE
    _cache_load_incomplete = false;
#else
    _uncached_contact_load_incomplete = false;
#endif
  }
  return fs_success && (nvs_err == ESP_OK);
#else
  #error "need to implement format()"
#endif
}

bool DataStore::repairInternalExtraFS() {
#if defined(NRF52_PLATFORM) && defined(EXTRAFS) && !defined(QSPIFLASH)
  // A healthy active ExtraFS must never be erased. Re-running the explicit
  // command is still useful, though: it retries any verified migration which
  // failed after a previously repaired filesystem was activated.
  if (_fsExtra != nullptr) {
    CustomLFS* extra = static_cast<CustomLFS*>(_fsExtra);
    if (_fsExtra != _configuredFsExtra
        || !mesh::storage::isExpectedInternalExtraFsGeometry(
            extra->getFlashAddr(), extra->getFlashSize(), extra->getBlockSize())
        || !mesh::storage::isInternalExtraFsReservedByApplication(
            (uint32_t)(uintptr_t)__flash_arduino_end)) {
      MESH_DEBUG_PRINTLN("DataStore: refusing internal ExtraFS migration with unexpected geometry");
      return false;
    }
    cleanupAtomicTempFiles(_fsExtra);
    return migrateToSecondaryFS();
  }
  if (!reinitializeInternalExtraFS()) return false;

  cleanupAtomicTempFiles(_fsExtra);
  // The fallback primary remains intact until each destination file has been
  // copied, read back byte-for-byte, and committed by migrateToSecondaryFS().
  return migrateToSecondaryFS();
#else
  return false;
#endif
}

#if defined(NRF52_PLATFORM) && defined(EXTRAFS) && !defined(QSPIFLASH)
bool DataStore::requestInternalExtraFSBootScan() {
  if (_primary_storage_unavailable || _configuredFsExtra == nullptr) {
    return false;
  }
  ResilientInternalExtraFS* extra =
      static_cast<ResilientInternalExtraFS*>(_configuredFsExtra);
  return extra->requestBootScan(true);
}

bool DataStore::formatInternalExtraFSHealth(char* reply,
                                             size_t reply_size) const {
  if (reply == nullptr || reply_size == 0 || _configuredFsExtra == nullptr) {
    return false;
  }
  const ResilientInternalExtraFS* extra =
      static_cast<const ResilientInternalExtraFS*>(_configuredFsExtra);
  snprintf(reply, reply_size,
           "ExtraFS %s; bad=0x%07lX; detected=0x%07lX; pending=0x%07lX; usable=%luK; mount=%s; last=%s; scanned=%u; boot=0x%02X",
           extra->pageMapReady() ? "mapped" : "map-unreadable",
           (unsigned long)extra->badPages(),
           (unsigned long)extra->detectedBadPages(),
           (unsigned long)extra->pendingPages(),
           (unsigned long)(extra->usableBytes() / 1024),
           _fsExtra == _configuredFsExtra ? "active" : "unavailable",
           ResilientInternalExtraFS::stageName(extra->stage()),
           (unsigned)extra->scannedPages(),
           (unsigned)extra->bootMarkerAtInit());
  return true;
}
#endif

bool DataStore::loadMainIdentity(mesh::LocalIdentity &identity) {
#if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM)
  if (!identity_store.recover("_main")) {
    _identity_creation_blocked = true;
    return false;
  }
#endif
#if defined(NRF52_PLATFORM)
  if (_primary_storage_unavailable) return false;

  bool identity_exists = false;
  if (!contactPathPresence(_fs, "/_main.id", identity_exists)) {
    _identity_creation_blocked = true;
    return false;
  }
  if (!identity_exists) return false;

  if (!identity_store.load("_main", identity)) {
    // A present identity which cannot be read or decoded is not a fresh
    // install. Never replace it with a newly generated key this boot.
    _identity_creation_blocked = true;
    return false;
  }
  // A verified primary identity is canonical even when only contact/channel
  // authority on an unavailable secondary remains unknown.
  _identity_creation_blocked = false;
  return true;
#else
  bool identity_exists = false;
#if defined(STM32_PLATFORM)
  const char* path = "/_main.id";
#else
  const char* path = "/identity/_main.id";
#endif
  if (!companionPathPresence(_fs, path, identity_exists)) {
    _identity_creation_blocked = true;
    return false;
  }
  if (!identity_exists) return false;
  if (!identity_store.load("_main", identity)) {
    _identity_creation_blocked = true;
    return false;
  }
  _identity_creation_blocked = false;
  return true;
#endif
}

bool DataStore::canCreateMainIdentity() const {
  return !_identity_creation_blocked;
}

bool DataStore::saveMainIdentity(const mesh::LocalIdentity &identity) {
  if (_identity_creation_blocked) return false;
  return identity_store.save("_main", identity);
}

bool DataStore::loadPrefs(CompanionNodePrefs& prefs, double& node_lat,
                          double& node_lon) {
#if defined(ESP32_PLATFORM)
  // A failed open is not absence on ESP32. Try each complete, committed
  // source twice; a temporary I/O failure must not erase saved radio/PIN data.
  // Unpublished .tmp candidates never supersede a previously saved image.
  _prefs_load_incomplete = false;
  _prefs_recovery_source = nullptr;
  for (const char* path : {"/new_prefs", "/new_prefs.bak", "/node_prefs"}) {
    for (unsigned attempt = 0; attempt < 2; ++attempt) {
      bool present = false;
      if (!companionPathPresence(_fs, path, present)) continue;
      if (!present) break;
      if (!loadPrefsInt(path, prefs, node_lat, node_lon)) continue;
      if (strcmp(path, "/new_prefs") != 0) {
        _prefs_recovery_source = path;
        promoteCompanionRecoveryFile(_fs, "/new_prefs", _prefs_recovery_source);
      }
      return true;
    }
  }
  // No usable saved settings remain. Continue with the caller's defaults and
  // permit a later verified save, instead of permanently disabling settings.
  MESH_DEBUG_PRINTLN("DataStore: no recoverable preferences; defaults may replace the old image");
  return true;
#elif defined(NRF52_PLATFORM)
  if (_primary_storage_unavailable
      || (_prefs_load_incomplete && !_secondary_authority_unknown)) {
    _prefs_load_incomplete = true;
    return false;
  }

  bool new_prefs_exists = false;
  if (!contactPathPresence(_fs, "/new_prefs", new_prefs_exists)) {
    _prefs_load_incomplete = true;
    return false;
  }
  if (new_prefs_exists) {
    const bool loaded = loadPrefsInt(
        "/new_prefs", prefs, node_lat, node_lon);
    _prefs_load_incomplete = !loaded;
    return loaded;
  }

  bool legacy_prefs_exists = false;
  if (!contactPathPresence(_fs, "/node_prefs", legacy_prefs_exists)) {
    _prefs_load_incomplete = true;
    return false;
  }
  if (!legacy_prefs_exists) {
    if (_secondary_authority_unknown) {
      _prefs_load_incomplete = true;
      return false;
    }
    return true;
  }
#else
  if (_prefs_load_incomplete) return false;
#if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM)
  if (!mesh::ContactFileTransaction::recover(_fs, "/new_prefs")) {
    _prefs_load_incomplete = true;
    return false;
  }
#endif
  if (_fs->exists("/new_prefs")) {
    const bool loaded = loadPrefsInt("/new_prefs", prefs, node_lat, node_lon);
    _prefs_load_incomplete = !loaded;
    return loaded;
  }
  if (!_fs->exists("/node_prefs")) return true;
#endif

  if (!loadPrefsInt("/node_prefs", prefs, node_lat, node_lon)) {
    _prefs_load_incomplete = true;
    return false;
  }
#if defined(NRF52_PLATFORM)
  // A verified primary legacy image is authoritative even if an unavailable
  // secondary might contain a duplicate. Permit its atomic in-place upgrade.
  _prefs_load_incomplete = false;
#endif
  if (savePrefs(prefs, node_lat, node_lon)) {
    _fs->remove("/node_prefs"); // remove old only after verified replacement
  }
  return true;
}

bool DataStore::loadPrefsInt(const char *filename,
                             CompanionNodePrefs& _prefs, double& node_lat,
                             double& node_lon) {
  File file = openRead(_fs, filename);
  if (file) {
    CompanionNodePrefs loaded_prefs;
    if (!loaded_prefs.copyPersistedValuesFrom(_prefs)) {
      file.close();
      return false;
    }
    // Files written before this tail existed always started Bluetooth at boot.
    // Do not inherit a runtime off value when loading one of those images.
    loaded_prefs.bluetooth_enabled = 1;
    loaded_prefs.gps_sync_interval_hours = 0;
    // Older images have no independent debug preference. Never inherit a
    // runtime-on value when loading one of those images.
    loaded_prefs.usb_debug_enabled = 0;
    // Old images cannot opt in using an unrelated runtime preference value.
    loaded_prefs.lost_reply = 0;
#if defined(RAK_4631)
    loaded_prefs.peripherals_shared = 0;
#endif
    double loaded_lat = node_lat;
    double loaded_lon = node_lon;
    // The original image ended after ble_pin at byte 84. Later releases only
    // appended fields, sometimes as an indivisible group. Accept every format
    // actually emitted by those releases, but reject a truncated field/group
    // or an unknown tail before any value reaches the live preferences.
    static const uint32_t MIN_PREFS_SIZE = 84;
#if defined(STM32_PLATFORM)
    // All published STM32 image lengths fit in one byte. The file size below
    // remains wide so oversized files cannot alias an accepted legacy length.
    using PrefsImageSize = uint8_t;
#else
    using PrefsImageSize = uint16_t;
#endif
    static const PrefsImageSize KNOWN_PREFS_SIZES[] = {
        84, 85, 90, 91, 92, 93, 140, 141, 142, 143, 144, 155,
        156, 157, 158, 159,
        159 + sizeof(loaded_prefs.bluetooth_name),
        161 + sizeof(loaded_prefs.bluetooth_name),
        168 + sizeof(loaded_prefs.bluetooth_name),
        168 + sizeof(loaded_prefs.bluetooth_name)
            + sizeof(loaded_prefs.bluetooth_mac_mode)
            + sizeof(loaded_prefs.bluetooth_mac),
        168 + sizeof(loaded_prefs.bluetooth_name)
            + sizeof(loaded_prefs.bluetooth_mac_mode)
            + sizeof(loaded_prefs.bluetooth_mac)
            + sizeof(loaded_prefs.bluetooth_stealth_peer_type)
            + sizeof(loaded_prefs.bluetooth_stealth_peer),
        168 + sizeof(loaded_prefs.bluetooth_name)
            + sizeof(loaded_prefs.bluetooth_mac_mode)
            + sizeof(loaded_prefs.bluetooth_mac)
            + sizeof(loaded_prefs.bluetooth_stealth_peer_type)
            + sizeof(loaded_prefs.bluetooth_stealth_peer)
            + sizeof(loaded_prefs.bluetooth_stealth_mode),
        226,  // prior 215-byte image plus radio timing, interference, AGC and timezone
#ifdef TBEAM_1W
        233,  // fan mode and low/high thermistor thresholds
#endif
#if defined(RP2040_PLATFORM) && defined(ENABLE_WIFI_INTERFACE)
        323,  // Pico W additionally persists its SSID/password (ESP32 uses NVS)
#endif
        230,  // Companion flood retry controls, appended after prior fields
#ifdef TBEAM_1W
        237,
#endif
#if defined(RP2040_PLATFORM) && defined(ENABLE_WIFI_INTERFACE)
        327,
#endif
        231,  // One-key DM consent policy, appended after flood retry controls
#ifdef TBEAM_1W
        238,
#endif
#if defined(RP2040_PLATFORM) && defined(ENABLE_WIFI_INTERFACE)
        328,
#endif
#ifdef TBEAM_1W
        239,  // Bluetooth enable preference, appended after one-key DM policy
#elif defined(RP2040_PLATFORM) && defined(ENABLE_WIFI_INTERFACE)
        329,
#else
        232,
#endif
#ifdef TBEAM_1W
        241,  // GPS time-sync cadence in hours, appended after Bluetooth enable
#elif defined(RP2040_PLATFORM) && defined(ENABLE_WIFI_INTERFACE)
        331,
#else
        234,
#endif
#ifdef TBEAM_1W
        242,  // Independent USB debug preference, after the published GPS tail
#elif defined(RP2040_PLATFORM) && defined(ENABLE_WIFI_INTERFACE)
        332,
#else
        235,
#endif
#ifdef TBEAM_1W
        243,  // Opt-in lost reply, appended after independent USB debug
#elif defined(RP2040_PLATFORM) && defined(ENABLE_WIFI_INTERFACE)
        333,
#else
        236,
#endif
    };
    const uint32_t prefs_size = file.size();
    bool known_size = false;
    for (size_t i = 0;
         i < sizeof(KNOWN_PREFS_SIZES) / sizeof(KNOWN_PREFS_SIZES[0]); i++) {
      if (prefs_size == KNOWN_PREFS_SIZES[i]) {
        known_size = true;
        break;
      }
    }
#if defined(RAK_4631)
    // Only the RAK4631 publishes this footer. Other targets retain their
    // exact existing image sizes, including the compact STM32 wire table.
    known_size = known_size || prefs_size == 236 + mesh::peripherals::FooterSize;
#endif
    if (prefs_size < MIN_PREFS_SIZE || !known_size) {
      file.close();
      return false;
    }
    bool success = true;
#if defined(STM32_PLATFORM) && defined(__GNUC__)
    using namespace mesh::companion_prefs;
    uint8_t pad[4];
    for (size_t i = 0; i < FIELD_COUNT; ++i) {
      const PrefField& field = PREF_FIELDS[i];
      if (i >= MANDATORY_FIELDS && file.available() == 0) continue;
      // Only this normally constructed, mutable snapshot is a destination.
      void* destination = const_cast<void*>(prefValue(field, loaded_prefs,
          loaded_lat, loaded_lon, pad));
      if (success && file.read(static_cast<uint8_t*>(destination), field.length)
          != field.length) success = false;
    }
#else
    auto readField = [&file, &success](void* dest, size_t size) -> bool {
      if (!success
          || file.read(static_cast<uint8_t*>(dest), size) != size) {
        success = false;
        return false;
      }
      return true;
    };
    auto readOptionalField = [&file, &readField](void* dest,
                                                 size_t size) -> bool {
      return file.available() == 0 || readField(dest, size);
    };
    uint8_t pad[8];

    readField(&loaded_prefs.airtime_factor, sizeof(float));                                // 0
    readField(loaded_prefs.node_name, sizeof(loaded_prefs.node_name));                      // 4
    readField(pad, 4);                                                                      // 36
    readField(&loaded_lat, sizeof(loaded_lat));                                             // 40
    readField(&loaded_lon, sizeof(loaded_lon));                                             // 48
    readField(&loaded_prefs.freq, sizeof(loaded_prefs.freq));                               // 56
    readField(&loaded_prefs.sf, sizeof(loaded_prefs.sf));                                   // 60
    readField(&loaded_prefs.cr, sizeof(loaded_prefs.cr));                                   // 61
    readField(&loaded_prefs.client_repeat, sizeof(loaded_prefs.client_repeat));             // 62
    readField(&loaded_prefs.manual_add_contacts, sizeof(loaded_prefs.manual_add_contacts)); // 63
    readField(&loaded_prefs.bw, sizeof(loaded_prefs.bw));                                   // 64
    readField(&loaded_prefs.tx_power_dbm, sizeof(loaded_prefs.tx_power_dbm));               // 68
    readField(&loaded_prefs.telemetry_mode_base, sizeof(loaded_prefs.telemetry_mode_base)); // 69
    readField(&loaded_prefs.telemetry_mode_loc, sizeof(loaded_prefs.telemetry_mode_loc));   // 70
    readField(&loaded_prefs.telemetry_mode_env, sizeof(loaded_prefs.telemetry_mode_env));   // 71
    readField(&loaded_prefs.rx_delay_base, sizeof(loaded_prefs.rx_delay_base));             // 72
    readField(&loaded_prefs.advert_loc_policy, sizeof(loaded_prefs.advert_loc_policy));     // 76
    readField(&loaded_prefs.multi_acks, sizeof(loaded_prefs.multi_acks));                   // 77
    readField(&loaded_prefs.path_hash_mode, sizeof(loaded_prefs.path_hash_mode));           // 78
    readField(pad, 1);                                                                      // 79
    readField(&loaded_prefs.ble_pin, sizeof(loaded_prefs.ble_pin));                         // 80
    readOptionalField(&loaded_prefs.buzzer_quiet,
                      sizeof(loaded_prefs.buzzer_quiet));                                  // 84
    readOptionalField(&loaded_prefs.gps_enabled,
                      sizeof(loaded_prefs.gps_enabled));                                   // 85
    readOptionalField(&loaded_prefs.gps_interval,
                      sizeof(loaded_prefs.gps_interval));                                  // 86
    readOptionalField(&loaded_prefs.autoadd_config,
                      sizeof(loaded_prefs.autoadd_config));                                // 90
    readOptionalField(&loaded_prefs.autoadd_max_hops,
                      sizeof(loaded_prefs.autoadd_max_hops));                              // 91
    readOptionalField(&loaded_prefs.rx_boosted_gain,
                      sizeof(loaded_prefs.rx_boosted_gain));                               // 92
    readOptionalField(loaded_prefs.default_scope_name,
                      sizeof(loaded_prefs.default_scope_name));                            // 93
    readOptionalField(loaded_prefs.default_scope_key,
                      sizeof(loaded_prefs.default_scope_key));                             // 124
    readOptionalField(&loaded_prefs.radio_fem_rxgain,
                      sizeof(loaded_prefs.radio_fem_rxgain));                              // 140
    readOptionalField(&loaded_prefs.radio_fem_rxgain_override,
                      sizeof(loaded_prefs.radio_fem_rxgain_override));                     // 141
    readOptionalField(&loaded_prefs.vibe_quiet,
                      sizeof(loaded_prefs.vibe_quiet));                                    // 142
    readOptionalField(&loaded_prefs.radio_fem_txgain,
                      sizeof(loaded_prefs.radio_fem_txgain));                              // 143
    readOptionalField(&loaded_prefs.rx_powersaving_enabled,
                      sizeof(loaded_prefs.rx_powersaving_enabled));                        // 144
    readOptionalField(&loaded_prefs.rx_ps_rx_us,
                      sizeof(loaded_prefs.rx_ps_rx_us));                                   // 145
    readOptionalField(&loaded_prefs.rx_ps_sleep_us,
                      sizeof(loaded_prefs.rx_ps_sleep_us));                                // 149
    readOptionalField(&loaded_prefs.rx_ps_level,
                      sizeof(loaded_prefs.rx_ps_level));                                   // 153
    readOptionalField(&loaded_prefs.rx_ps_preamble,
                      sizeof(loaded_prefs.rx_ps_preamble));                                // 154
    readOptionalField(&loaded_prefs.powersaving_enabled,
                      sizeof(loaded_prefs.powersaving_enabled));                           // 155
    readOptionalField(&loaded_prefs.wifi_enabled,
                      sizeof(loaded_prefs.wifi_enabled));                                  // 156
    readOptionalField(&loaded_prefs.powersaving_policy_version,
                      sizeof(loaded_prefs.powersaving_policy_version));                    // 157
    readOptionalField(&loaded_prefs.usb_logging_enabled,
                      sizeof(loaded_prefs.usb_logging_enabled));                           // 158
    readOptionalField(loaded_prefs.bluetooth_name,
                      sizeof(loaded_prefs.bluetooth_name));                               // 159
    readOptionalField(&loaded_prefs.display_rotation_degrees,
                      sizeof(loaded_prefs.display_rotation_degrees));                      // 191
    readOptionalField(&loaded_prefs.cad_enabled,
                      sizeof(loaded_prefs.cad_enabled));                                   // 193
    readOptionalField(&loaded_prefs.cad_scan_timeout_ms,
                      sizeof(loaded_prefs.cad_scan_timeout_ms));                           // 194
    readOptionalField(&loaded_prefs.cad_retry_delay_ms,
                      sizeof(loaded_prefs.cad_retry_delay_ms));                            // 196
    readOptionalField(&loaded_prefs.cad_max_duration_ms,
                      sizeof(loaded_prefs.cad_max_duration_ms));                           // 198
    readOptionalField(&loaded_prefs.bluetooth_mac_mode,
                      sizeof(loaded_prefs.bluetooth_mac_mode));                           // 200
    readOptionalField(loaded_prefs.bluetooth_mac,
                      sizeof(loaded_prefs.bluetooth_mac));                                // 201
    readOptionalField(&loaded_prefs.bluetooth_stealth_peer_type,
                      sizeof(loaded_prefs.bluetooth_stealth_peer_type));                  // 207
    readOptionalField(loaded_prefs.bluetooth_stealth_peer,
                      sizeof(loaded_prefs.bluetooth_stealth_peer));                       // 208
    readOptionalField(&loaded_prefs.bluetooth_stealth_mode,
                      sizeof(loaded_prefs.bluetooth_stealth_mode));                       // 214
    readOptionalField(&loaded_prefs.tx_delay_factor, sizeof(loaded_prefs.tx_delay_factor));
    readOptionalField(&loaded_prefs.direct_tx_delay_factor, sizeof(loaded_prefs.direct_tx_delay_factor));
    readOptionalField(&loaded_prefs.interference_threshold, sizeof(loaded_prefs.interference_threshold));
    readOptionalField(&loaded_prefs.agc_reset_interval, sizeof(loaded_prefs.agc_reset_interval));
    readOptionalField(&loaded_prefs.tz_offset, sizeof(loaded_prefs.tz_offset));
#ifdef TBEAM_1W
    readOptionalField(loaded_prefs.fan_mode, sizeof(loaded_prefs.fan_mode));
    readOptionalField(&loaded_prefs.fan_lo, sizeof(loaded_prefs.fan_lo));
    readOptionalField(&loaded_prefs.fan_hi, sizeof(loaded_prefs.fan_hi));
    loaded_prefs.fan_mode[sizeof(loaded_prefs.fan_mode) - 1] = 0;
#endif
#if defined(RP2040_PLATFORM) && defined(ENABLE_WIFI_INTERFACE)
    readOptionalField(loaded_prefs.wifi_ssid, sizeof(loaded_prefs.wifi_ssid));
    readOptionalField(loaded_prefs.wifi_pwd, sizeof(loaded_prefs.wifi_pwd));
    loaded_prefs.wifi_ssid[sizeof(loaded_prefs.wifi_ssid) - 1] = 0;
    loaded_prefs.wifi_pwd[sizeof(loaded_prefs.wifi_pwd) - 1] = 0;
#endif
    readOptionalField(&loaded_prefs.flood_retry_attempts,
                      sizeof(loaded_prefs.flood_retry_attempts));
    readOptionalField(&loaded_prefs.flood_retry_max_path,
                      sizeof(loaded_prefs.flood_retry_max_path));
    readOptionalField(&loaded_prefs.flood_retry_group_max_path,
                      sizeof(loaded_prefs.flood_retry_group_max_path));
    readOptionalField(&loaded_prefs.flood_retry_advert_enabled,
                      sizeof(loaded_prefs.flood_retry_advert_enabled));
    readOptionalField(&loaded_prefs.one_key_dm_enabled,
                      sizeof(loaded_prefs.one_key_dm_enabled));
    readOptionalField(&loaded_prefs.bluetooth_enabled,
                      sizeof(loaded_prefs.bluetooth_enabled));
    readOptionalField(&loaded_prefs.gps_sync_interval_hours,
                      sizeof(loaded_prefs.gps_sync_interval_hours));
    readOptionalField(&loaded_prefs.usb_debug_enabled,
                      sizeof(loaded_prefs.usb_debug_enabled));
    readOptionalField(&loaded_prefs.lost_reply, sizeof(loaded_prefs.lost_reply));
#if defined(RAK_4631)
    if (file.available() != 0) {
      uint8_t peripheral_footer[mesh::peripherals::FooterSize] = {};
      success = success && readField(peripheral_footer, sizeof(peripheral_footer))
          && mesh::peripherals::decodeFooter(peripheral_footer,
              sizeof(peripheral_footer), loaded_prefs.peripherals_shared);
    }
#endif
#endif

    // Any bytes left over form only part of a historically appended field.
    // Preserve the file and defaults rather than treating that tail as EOF.
    success = success && file.available() == 0;
    success = success && loaded_prefs.bluetooth_enabled <= 1;
    file.close();
    if (!success) return false;
    loaded_prefs.usb_debug_enabled = loaded_prefs.usb_debug_enabled == 1 ? 1 : 0;
    loaded_prefs.lost_reply = loaded_prefs.lost_reply <= 2 ? loaded_prefs.lost_reply : 0;
    if (!_prefs.copyPersistedValuesFrom(loaded_prefs)) return false;
    node_lat = loaded_lat;
    node_lon = loaded_lon;
    return true;
  }
  return false;
}

bool DataStore::savePrefs(const CompanionNodePrefs& _prefs, double node_lat, double node_lon) {
  if (_prefs_load_incomplete) return false;
#if defined(ESP32_PLATFORM)
  if (!promoteCompanionRecoveryFile(_fs, "/new_prefs", _prefs_recovery_source)) return false;
#endif
#if defined(NRF52_PLATFORM)
  if (_primary_storage_unavailable) return false;
#endif
#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
  mesh::AtomicFileWriter file(_fs, "/new_prefs");
#elif defined(ESP32_PLATFORM)
  mesh::ContactFileTransaction file(_fs, "/new_prefs", companionPathPresence);
#elif defined(RP2040_PLATFORM)
  mesh::ContactFileTransaction file(_fs, "/new_prefs");
#else
  File file = openWrite(_fs, "/new_prefs");
#endif
  if (file) {
#if defined(STM32_PLATFORM) && defined(__GNUC__)
    using namespace mesh::companion_prefs;
    uint8_t image[236];
    const uint8_t pad[4] = {};
    size_t used = 0;
    for (const PrefField& field : PREF_FIELDS) {
      if (field.length > sizeof(image) - used) {
        // Latch failure and stop before any out-of-bounds pointer arithmetic.
        used = sizeof(image) + 1;
        break;
      }
      memcpy(image + used, prefValue(field, _prefs, node_lat, node_lon, pad),
          field.length);
      used += field.length;
    }
    bool success = used == sizeof(image)
        && file.write(image, sizeof(image)) == sizeof(image);
#else
#if defined(STM32_PLATFORM)
    // Keep the explicit wire field list, but make one bounded atomic write.
    // Other platforms retain direct writes and their existing stack footprint.
    uint8_t image[236];
    size_t image_length = 0;
    const auto writeField = [&](const void* data, size_t size) -> bool {
      if (image_length > sizeof(image)
          || size > sizeof(image) - image_length) {
        // Latch failure without reading data or advancing past the buffer.
        // The return value only preserves the common emitter interface;
        // the exact-length check below is the authoritative write gate.
        image_length = sizeof(image) + 1;
        return true;
      }
      memcpy(image + image_length, data, size);
      image_length += size;
      return true;
    };
#else
    const auto writeField = [&file](const void* data, size_t size) -> bool {
      return file.write(static_cast<const uint8_t*>(data), size) == size;
    };
#endif
    uint8_t pad[8];
    memset(pad, 0, sizeof(pad));

    bool success = writeField((uint8_t *)&_prefs.airtime_factor, sizeof(float)); // 0
    success = success && writeField((uint8_t *)_prefs.node_name, sizeof(_prefs.node_name)); // 4
    success = success && writeField(pad, 4);                                            // 36
    success = success && writeField((uint8_t *)&node_lat, sizeof(node_lat)); // 40
    success = success && writeField((uint8_t *)&node_lon, sizeof(node_lon)); // 48
    success = success && writeField((uint8_t *)&_prefs.freq, sizeof(_prefs.freq)); // 56
    success = success && writeField((uint8_t *)&_prefs.sf, sizeof(_prefs.sf)); // 60
    success = success && writeField((uint8_t *)&_prefs.cr, sizeof(_prefs.cr)); // 61
    success = success && writeField((uint8_t *)&_prefs.client_repeat, sizeof(_prefs.client_repeat)); // 62
    success = success && writeField((uint8_t *)&_prefs.manual_add_contacts, sizeof(_prefs.manual_add_contacts)); // 63
    success = success && writeField((uint8_t *)&_prefs.bw, sizeof(_prefs.bw)); // 64
    success = success && writeField((uint8_t *)&_prefs.tx_power_dbm, sizeof(_prefs.tx_power_dbm)); // 68
    success = success && writeField((uint8_t *)&_prefs.telemetry_mode_base, sizeof(_prefs.telemetry_mode_base)); // 69
    success = success && writeField((uint8_t *)&_prefs.telemetry_mode_loc, sizeof(_prefs.telemetry_mode_loc)); // 70
    success = success && writeField((uint8_t *)&_prefs.telemetry_mode_env, sizeof(_prefs.telemetry_mode_env)); // 71
    success = success && writeField((uint8_t *)&_prefs.rx_delay_base, sizeof(_prefs.rx_delay_base)); // 72
    success = success && writeField((uint8_t *)&_prefs.advert_loc_policy, sizeof(_prefs.advert_loc_policy)); // 76
    success = success && writeField((uint8_t *)&_prefs.multi_acks, sizeof(_prefs.multi_acks)); // 77
    success = success && writeField((uint8_t *)&_prefs.path_hash_mode, sizeof(_prefs.path_hash_mode)); // 78
    success = success && writeField(pad, 1);                                            // 79
    success = success && writeField((uint8_t *)&_prefs.ble_pin, sizeof(_prefs.ble_pin)); // 80
    success = success && writeField((uint8_t *)&_prefs.buzzer_quiet, sizeof(_prefs.buzzer_quiet)); // 84
    success = success && writeField((uint8_t *)&_prefs.gps_enabled, sizeof(_prefs.gps_enabled)); // 85
    success = success && writeField((uint8_t *)&_prefs.gps_interval, sizeof(_prefs.gps_interval)); // 86
    success = success && writeField((uint8_t *)&_prefs.autoadd_config, sizeof(_prefs.autoadd_config)); // 87
    success = success && writeField((uint8_t *)&_prefs.autoadd_max_hops, sizeof(_prefs.autoadd_max_hops)); // 88
    success = success && writeField((uint8_t *)&_prefs.rx_boosted_gain, sizeof(_prefs.rx_boosted_gain)); // 89
    success = success && writeField((uint8_t *)_prefs.default_scope_name, sizeof(_prefs.default_scope_name)); // 90
    success = success && writeField((uint8_t *)_prefs.default_scope_key, sizeof(_prefs.default_scope_key)); // 121
    success = success && writeField((uint8_t *)&_prefs.radio_fem_rxgain, sizeof(_prefs.radio_fem_rxgain)); // 122
    success = success && writeField((uint8_t *)&_prefs.radio_fem_rxgain_override,
               sizeof(_prefs.radio_fem_rxgain_override)); // 123
    success = success && writeField((uint8_t *)&_prefs.vibe_quiet,
               sizeof(_prefs.vibe_quiet));                    // 124
    success = success && writeField((uint8_t *)&_prefs.radio_fem_txgain,
               sizeof(_prefs.radio_fem_txgain));        // 125
    success = success && writeField((uint8_t *)&_prefs.rx_powersaving_enabled,
               sizeof(_prefs.rx_powersaving_enabled)); // 126
    success = success && writeField((uint8_t *)&_prefs.rx_ps_rx_us,
               sizeof(_prefs.rx_ps_rx_us));                  // 127
    success = success && writeField((uint8_t *)&_prefs.rx_ps_sleep_us,
               sizeof(_prefs.rx_ps_sleep_us));            // 131
    success = success && writeField((uint8_t *)&_prefs.rx_ps_level,
               sizeof(_prefs.rx_ps_level));                  // 135
    success = success && writeField((uint8_t *)&_prefs.rx_ps_preamble,
               sizeof(_prefs.rx_ps_preamble));            // 136
    success = success && writeField((uint8_t *)&_prefs.powersaving_enabled,
               sizeof(_prefs.powersaving_enabled)); // 137
    success = success && writeField((uint8_t *)&_prefs.wifi_enabled,
               sizeof(_prefs.wifi_enabled));               // 138
    success = success && writeField((uint8_t *)&_prefs.powersaving_policy_version,
               sizeof(_prefs.powersaving_policy_version));                              // 139
    success = success && writeField((uint8_t *)&_prefs.usb_logging_enabled,
               sizeof(_prefs.usb_logging_enabled));                                    // 140
    success = success && writeField((uint8_t *)_prefs.bluetooth_name,
               sizeof(_prefs.bluetooth_name));          // 141
    success = success && writeField(
               (uint8_t *)&_prefs.display_rotation_degrees,
               sizeof(_prefs.display_rotation_degrees));
    success = success && writeField((uint8_t *)&_prefs.cad_enabled,
               sizeof(_prefs.cad_enabled));
    success = success && writeField((uint8_t *)&_prefs.cad_scan_timeout_ms,
               sizeof(_prefs.cad_scan_timeout_ms));
    success = success && writeField((uint8_t *)&_prefs.cad_retry_delay_ms,
               sizeof(_prefs.cad_retry_delay_ms));
    success = success && writeField((uint8_t *)&_prefs.cad_max_duration_ms,
               sizeof(_prefs.cad_max_duration_ms));
    success = success && writeField((uint8_t *)&_prefs.bluetooth_mac_mode,
               sizeof(_prefs.bluetooth_mac_mode));
    success = success && writeField((uint8_t *)_prefs.bluetooth_mac,
               sizeof(_prefs.bluetooth_mac));
    success = success && writeField(
               (uint8_t *)&_prefs.bluetooth_stealth_peer_type,
               sizeof(_prefs.bluetooth_stealth_peer_type));
    success = success && writeField(
               (uint8_t *)_prefs.bluetooth_stealth_peer,
               sizeof(_prefs.bluetooth_stealth_peer));
    success = success && writeField(
               (uint8_t *)&_prefs.bluetooth_stealth_mode,
               sizeof(_prefs.bluetooth_stealth_mode));

    success = success && writeField((uint8_t *)&_prefs.tx_delay_factor,
        sizeof(_prefs.tx_delay_factor));
    success = success && writeField((uint8_t *)&_prefs.direct_tx_delay_factor,
        sizeof(_prefs.direct_tx_delay_factor));
    success = success && writeField((uint8_t *)&_prefs.interference_threshold,
        sizeof(_prefs.interference_threshold));
    success = success && writeField((uint8_t *)&_prefs.agc_reset_interval,
        sizeof(_prefs.agc_reset_interval));
    success = success && writeField((uint8_t *)&_prefs.tz_offset,
        sizeof(_prefs.tz_offset));
#ifdef TBEAM_1W
    success = success && writeField((uint8_t *)_prefs.fan_mode,
        sizeof(_prefs.fan_mode));
    success = success && writeField((uint8_t *)&_prefs.fan_lo,
        sizeof(_prefs.fan_lo));
    success = success && writeField((uint8_t *)&_prefs.fan_hi,
        sizeof(_prefs.fan_hi));
#endif
#if defined(RP2040_PLATFORM) && defined(ENABLE_WIFI_INTERFACE)
    success = success && writeField((uint8_t *)_prefs.wifi_ssid,
        sizeof(_prefs.wifi_ssid));
    success = success && writeField((uint8_t *)_prefs.wifi_pwd,
        sizeof(_prefs.wifi_pwd));
#endif
    // Pack only the explicit persisted tail, never the class's padded layout.
    // memcpy retains the GPS field's established native byte representation.
    uint8_t tail[10] = {
        _prefs.flood_retry_attempts, _prefs.flood_retry_max_path,
        _prefs.flood_retry_group_max_path, _prefs.flood_retry_advert_enabled,
        _prefs.one_key_dm_enabled, _prefs.bluetooth_enabled, 0, 0,
        _prefs.usb_debug_enabled, _prefs.lost_reply};
    memcpy(tail + 6, &_prefs.gps_sync_interval_hours,
        sizeof(_prefs.gps_sync_interval_hours));
    success = success && writeField(tail, sizeof(tail));
#if defined(RAK_4631)
    uint8_t peripheral_footer[mesh::peripherals::FooterSize];
    mesh::peripherals::encodeFooter(peripheral_footer, _prefs.peripherals_shared);
    success = success && writeField(peripheral_footer, sizeof(peripheral_footer));
#endif

#if defined(STM32_PLATFORM)
    success = success && image_length == sizeof(image)
        && file.write(image, sizeof(image)) == sizeof(image);
#endif
#endif
#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) || defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM)
    success = file.commit(success);
    if (!success) MESH_DEBUG_PRINTLN("DataStore: atomic preferences write failed");
#else
    file.close();
#endif
    return success;
  }
  return false;
}

static bool serializeContactRecord(const ContactInfo& c,
                                   uint8_t out[mesh::storage::CONTACT_RECORD_SIZE]) {
  size_t offset = 0;
  memcpy(&out[offset], c.id.pub_key, 32); offset += 32;
  memcpy(&out[offset], c.name, 32); offset += 32;
  out[offset++] = c.type;
  out[offset++] = c.flags;
  out[offset++] = mesh::encodeRadioTxPolicy(c.tx_radio);
  memcpy(&out[offset], &c.sync_since, 4); offset += 4;
  out[offset++] = c.out_path_len;
  memcpy(&out[offset], &c.last_advert_timestamp, 4); offset += 4;
  if (!c.copyPathTo(&out[offset])) return false;
  offset += 64;
  memcpy(&out[offset], &c.lastmod, 4); offset += 4;
  memcpy(&out[offset], &c.gps_lat, 4); offset += 4;
  memcpy(&out[offset], &c.gps_lon, 4);
  return true;
}

static bool deserializeContactRecord(
    const uint8_t in[mesh::storage::CONTACT_RECORD_SIZE], ContactInfo& c,
    uint16_t path_source, bool* path_unavailable = nullptr) {
  if (path_unavailable) *path_unavailable = false;
  size_t offset = 0;
  uint8_t pub_key[32];
  memcpy(pub_key, &in[offset], 32); offset += 32;
  memcpy(c.name, &in[offset], 32); offset += 32;
  c.name[sizeof(c.name) - 1] = 0;
  c.type = in[offset++];
  c.flags = in[offset++];
  c.tx_radio = mesh::decodeRadioTxPolicy(in[offset++]);
  memcpy(&c.sync_since, &in[offset], 4); offset += 4;
  c.out_path_len = in[offset++];
  memcpy(&c.last_advert_timestamp, &in[offset], 4); offset += 4;
#if MESH_CONTACT_CACHE
  if (!c.path_ref.bind(path_source, &in[offset])) {
    if (path_unavailable) *path_unavailable = true;
    return false;
  }
#else
  (void)path_source;
  if (!c.setRawPath(&in[offset])) return false;
#endif
  offset += 64;
  memcpy(&c.lastmod, &in[offset], 4); offset += 4;
  memcpy(&c.gps_lat, &in[offset], 4); offset += 4;
  memcpy(&c.gps_lon, &in[offset], 4);
  c.id = mesh::Identity(pub_key);
  c.shared_secret_valid = false;
  return c.out_path_len == OUT_PATH_UNKNOWN
      || mesh::Packet::isValidPathLen(c.out_path_len);
}

#if defined(NRF52_PLATFORM)
static const char* CONTACT_MIGRATION_MARKER = "/contacts4.mig";
static const char* SECONDARY_MIGRATION_JOURNAL = "/.extrafs.mig";
static const uint8_t SECONDARY_MIGRATION_PENDING = 1;
static const uint8_t SECONDARY_MIGRATION_COMMITTED = 2;
#if defined(MESHCORE_EXTRAFS_HIL)
static const char* HIL_CONTACT_PAGE_FAILURE_MARKER = "/__hil.readfail";
static const uint8_t HIL_CONTACT_STAT_FAILURE_FLAG = 0x80;
#endif

static mesh::storage::ContactPathState statContactPath(
    FILESYSTEM* fs, const char* path, uint32_t* size) {
  if (fs == nullptr || path == nullptr) {
    return mesh::storage::ContactPathState::IO_ERROR;
  }

  struct lfs_info info;
  fs->_lockFS();
  const int result = lfs_stat(fs->_getFS(), path, &info);
  fs->_unlockFS();
  const mesh::storage::ContactPathState state =
      mesh::storage::classifyContactPathStat(result, LFS_ERR_NOENT);
  if (state == mesh::storage::ContactPathState::PRESENT && size != nullptr) {
    *size = info.size;
  }
  if (state == mesh::storage::ContactPathState::IO_ERROR) {
    MESH_DEBUG_PRINTLN(
        "DataStore: storage path stat failed for %s: %d", path, result);
  }
  return state;
}

static bool contactPathPresence(FILESYSTEM* fs, const char* path,
                                bool& present, uint32_t* size) {
  const mesh::storage::ContactPathState state = statContactPath(fs, path, size);
  present = state == mesh::storage::ContactPathState::PRESENT;
  return state != mesh::storage::ContactPathState::IO_ERROR;
}

static void makeContactPagePath(uint8_t page, char path[24]) {
  snprintf(path, 24, "/contacts4_%02u", (unsigned)page);
}

static void discardInvalidContactPage(FILESYSTEM* fs, const char* path,
                                      uint8_t page) {
  // The page has already failed size/format/CRC validation and cannot be a
  // recovery source. Remove it so the atomic replacement only needs one free
  // page; retaining full-size .bad copies can otherwise exhaust a 100 KiB
  // ExtraFS and make self-repair impossible.
  if (!fs->remove(path)) {
    MESH_DEBUG_PRINTLN("DataStore: could not remove invalid contact page %u", page);
  }
}

void DataStore::resetContactPageState(bool clear_incomplete) {
  _contact_slots.clear();
  _dirty_contact_pages.clearAll();
  _unread_contact_pages.clearAll();
  if (clear_incomplete) {
    _contact_load_incomplete = false;
#if MESH_CONTACT_CACHE
    _cache_load_incomplete = false;
#endif
  }
  memset(_contact_page_generations, 0, sizeof(_contact_page_generations));
  _legacy_contacts_pending_cleanup = false;
  _legacy_migration_ready = false;
  _legacy_contact_count = 0;
}

bool DataStore::prepareLegacyContactMigration() {
  FILESYSTEM* fs = _getContactsChannelsFS();
  bool marker_exists = false;
  if (!contactPathPresence(fs, CONTACT_MIGRATION_MARKER, marker_exists)) {
    return false;
  }
  if (marker_exists) {
    _legacy_migration_ready = true;
    return true;
  }

  // With no marker, page files can be leftovers from a newer firmware followed
  // by a downgrade that rewrote /contacts3. The complete legacy file remains
  // authoritative while these are removed, so a reset at any point is safe.
  bool clean = true;
  for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) {
    char path[24];
    makeContactPagePath(page, path);
    bool path_exists = false;
    if (!contactPathPresence(fs, path, path_exists)) return false;
    if (path_exists && !fs->remove(path)) clean = false;

    char temp_path[28];
    snprintf(temp_path, sizeof(temp_path), "%s.tmp", path);
    bool temp_exists = false;
    if (!contactPathPresence(fs, temp_path, temp_exists)) return false;
    if (temp_exists && !fs->remove(temp_path)) clean = false;
  }
  if (!clean) {
    MESH_DEBUG_PRINTLN("DataStore: could not clear stale contact pages before migration");
    return false;
  }

  mesh::AtomicFileWriter marker(fs, CONTACT_MIGRATION_MARKER);
  _legacy_migration_ready = marker.commit(true);
  if (!_legacy_migration_ready) {
    MESH_DEBUG_PRINTLN("DataStore: could not create contact migration marker");
  }
  return _legacy_migration_ready;
}

bool DataStore::loadContactPages(DataStoreHost* host, uint16_t minimum_slot,
                                 uint32_t expected_page_mask) {
  bool any_page_file = false;
  FILESYSTEM* fs = _getContactsChannelsFS();
  auto quarantineUnreadPage = [this](uint8_t page) {
    // Fail closed if an otherwise present page cannot be read. Reserving its
    // slots prevents a new contact from overwriting records which a later boot
    // may recover successfully.
    for (uint8_t index = 0;
         index < mesh::storage::CONTACTS_PER_PAGE; index++) {
      _contact_slots.reserve(
          (uint16_t)page * mesh::storage::CONTACTS_PER_PAGE + index);
    }
    _unread_contact_pages.mark(page);
    _contact_load_incomplete = true;
  };

  for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) {
    if ((expected_page_mask & (1UL << page)) == 0) continue;

    char path[24];
    makeContactPagePath(page, path);
    // The first source-selection pass proved this page existed. Do not ask
    // exists() a second time and silently reinterpret a transient failure as
    // an empty slot range; opening the captured page either succeeds or puts
    // that range into fail-closed quarantine.
    any_page_file = true;

#if defined(MESHCORE_EXTRAFS_HIL)
    // Direct-attached HIL can request one fail-closed boot without damaging
    // the page. Consume the marker before quarantine so the next reboot
    // exercises normal recovery from the untouched source.
    if (fs->exists(HIL_CONTACT_PAGE_FAILURE_MARKER)) {
      File marker = openRead(fs, HIL_CONTACT_PAGE_FAILURE_MARKER);
      const int fail_page = marker ? marker.read() : -1;
      if (marker) marker.close();
      if (fail_page == page && fs->remove(HIL_CONTACT_PAGE_FAILURE_MARKER)
          && !fs->exists(HIL_CONTACT_PAGE_FAILURE_MARKER)) {
        MESH_DEBUG_PRINTLN(
            "DataStore: HIL quarantining contact page %u", page);
        quarantineUnreadPage(page);
        continue;
      }
    }
#endif

    File file = openRead(fs, path);
    if (!file) {
      MESH_DEBUG_PRINTLN("DataStore: contact page %u could not be opened", page);
      quarantineUnreadPage(page);
      continue;
    }
    if (file.size() != mesh::storage::CONTACT_PAGE_FILE_SIZE) {
      MESH_DEBUG_PRINTLN("DataStore: ignoring invalid contact page %u", page);
      file.close();
      discardInvalidContactPage(fs, path, page);
      _dirty_contact_pages.mark(page);
      continue;
    }

    uint8_t raw_header[mesh::storage::CONTACT_PAGE_HEADER_SIZE];
    mesh::storage::ContactPageHeader header;
    const bool header_read =
        file.read(raw_header, sizeof(raw_header)) == sizeof(raw_header);
    const bool header_valid = header_read
        && mesh::storage::decodeContactPageHeader(raw_header, page, header);
    if (!header_read) {
      // A short read is not evidence that the atomically committed page is
      // corrupt. Leave it untouched so a later boot can retry it.
      MESH_DEBUG_PRINTLN(
          "DataStore: contact page %u header could not be read", page);
      file.close();
      quarantineUnreadPage(page);
      continue;
    }
    if (!header_valid) {
      MESH_DEBUG_PRINTLN("DataStore: contact page %u failed CRC/format validation", page);
      file.close();
      discardInvalidContactPage(fs, path, page);
      _dirty_contact_pages.mark(page);
      continue;
    }

    // Keep a verified page snapshot in heap memory while constructing the RAM
    // table. A second series of filesystem seeks could fail after only part of
    // the page had been loaded; later rewriting that partial RAM view would
    // permanently delete otherwise valid contacts.
    uint8_t* payload =
        (uint8_t*)malloc(mesh::storage::CONTACT_PAGE_PAYLOAD_SIZE);
    if (payload == nullptr) {
      MESH_DEBUG_PRINTLN(
          "DataStore: no memory to load contact page %u", page);
      file.close();
      quarantineUnreadPage(page);
      continue;
    }
    const bool payload_read =
        file.read(payload, mesh::storage::CONTACT_PAGE_PAYLOAD_SIZE)
        == mesh::storage::CONTACT_PAGE_PAYLOAD_SIZE;
    file.close();
    if (!payload_read) {
      MESH_DEBUG_PRINTLN(
          "DataStore: contact page %u payload could not be read", page);
      free(payload);
      quarantineUnreadPage(page);
      continue;
    }
    const uint32_t crc = mesh::storage::updateCRC32(
        0xFFFFFFFFUL, payload, mesh::storage::CONTACT_PAGE_PAYLOAD_SIZE);
    if (crc != header.payload_crc) {
      MESH_DEBUG_PRINTLN("DataStore: contact page %u failed CRC/format validation", page);
      free(payload);
      discardInvalidContactPage(fs, path, page);
      _dirty_contact_pages.mark(page);
      continue;
    }

    if ((header.occupied & ~mesh::storage::contactPageValidSlotMask()) != 0) {
      MESH_DEBUG_PRINTLN(
          "DataStore: repairing contact page %u high occupancy bits", page);
      _dirty_contact_pages.mark(page);
    }

    _contact_page_generations[page] = header.generation;
    for (uint8_t index = 0; index < mesh::storage::CONTACTS_PER_PAGE; index++) {
      const uint16_t slot = (uint16_t)page * mesh::storage::CONTACTS_PER_PAGE + index;
      const uint8_t* record =
          &payload[(uint16_t)index * mesh::storage::CONTACT_RECORD_SIZE];

      // The payload CRC covers every record, so occupancy can be reconstructed
      // without trusting the separate header bitmask. This recovers a contact
      // hidden by a flipped mask bit and ignores an empty slot exposed by one;
      // the dirty page is later rewritten with the repaired mask.
      const bool has_data = mesh::storage::contactRecordHasData(record);
      const bool marked_occupied =
          (header.occupied & (1UL << index)) != 0;
      if (has_data != marked_occupied) {
        MESH_DEBUG_PRINTLN(
            "DataStore: repairing contact page %u occupancy slot %u",
            page, index);
        _dirty_contact_pages.mark(page);
      }
      if (!has_data
          || !mesh::storage::loadSlotFromMigratedPage(slot, minimum_slot)) {
        continue;
      }

      ContactInfo contact;
      bool path_unavailable = false;
      const bool valid = deserializeContactRecord(record, contact,
          0x8000 | slot, &path_unavailable);
      if (path_unavailable) {
        // Cache exhaustion is not a corrupt record. Keep the complete page
        // on disk and refuse changes until it can be loaded after reboot.
        _contact_load_incomplete = true;
        _dirty_contact_pages.clearAll();
        free(payload);
        return true;
      }
      if (!valid || !_contact_slots.reserve(slot)) {
        MESH_DEBUG_PRINTLN("DataStore: contact page %u slot %u is invalid/duplicate", page, index);
        _dirty_contact_pages.mark(page);
        continue;
      }
      contact.storage_slot = slot;
      if (!host->onContactLoaded(contact)) {
        _contact_slots.release(slot);
        // A host capacity refusal is not a filesystem read error, but the RAM
        // table is still not a complete representation of the durable page.
        // Preserve the page and veto every mutation/rewrite until reboot.
        MESH_DEBUG_PRINTLN(
            "DataStore: contact host refused page %u slot %u; load incomplete",
            page, index);
        _contact_load_incomplete = true;
        _dirty_contact_pages.clearAll();
        free(payload);
        return true;
      }
    }
    free(payload);
  }
  if (hasIncompleteContactLoad()) {
    // Occupancy/header repairs discovered elsewhere in this pass must wait for
    // a clean reboot too. Rewriting any page from an incomplete contact table
    // can turn a transient read failure into durable cross-page data loss.
    _dirty_contact_pages.clearAll();
  }
  return any_page_file;
}

bool DataStore::writeContactPage(DataStoreHost* host, uint8_t page,
                                 bool (*filter)(const ContactInfo& c)) {
  if (page >= mesh::storage::CONTACT_PAGE_COUNT) return false;

  // The nRF52 Arduino loop task has only a 4 KiB stack.  Keep pointers to this
  // page's contacts and stream one 152-byte record at a time instead of
  // allocating the complete 3.8 KiB payload on that stack.
  ContactInfo* page_contacts[mesh::storage::CONTACTS_PER_PAGE];
  memset(page_contacts, 0, sizeof(page_contacts));
  uint32_t occupied = 0;

  for (uint32_t index = 0;; index++) {
    ContactInfo* contact = host->getContactForStore(index);
    if (contact == NULL) break;
    if ((filter && !filter(*contact))
        || contact->storage_slot == mesh::storage::CONTACT_SLOT_NONE
        || contact->storage_slot / mesh::storage::CONTACTS_PER_PAGE != page) {
      continue;
    }

    const uint8_t page_slot = contact->storage_slot % mesh::storage::CONTACTS_PER_PAGE;
    page_contacts[page_slot] = contact;
    occupied |= 1UL << page_slot;
  }

#if MESH_CONTACT_CACHE
  auto& paths = mesh::contactPathStorage();
  paths.beginCommit();
  for (auto* contact : page_contacts) if (contact) paths.mark(contact->path_ref.handle());
  const uint16_t first_slot = page * mesh::storage::CONTACTS_PER_PAGE;
  if (!paths.preserveSnapshots(0x8000 | first_slot, mesh::storage::CONTACTS_PER_PAGE)
      || !paths.preserveSnapshots(first_slot, mesh::storage::CONTACTS_PER_PAGE)) {
    paths.endCommit(false);
    return false;
  }
#endif
  uint32_t payload_crc = 0xFFFFFFFFUL;
  uint8_t record[mesh::storage::CONTACT_RECORD_SIZE];
  for (uint8_t slot = 0; slot < mesh::storage::CONTACTS_PER_PAGE; slot++) {
    memset(record, 0, sizeof(record));
    if (page_contacts[slot] != nullptr) {
      if (!serializeContactRecord(*page_contacts[slot], record)) {
#if MESH_CONTACT_CACHE
        mesh::contactPathStorage().endCommit(false);
#endif
        return false;
      }
    }
    payload_crc = mesh::storage::updateCRC32(payload_crc, record, sizeof(record));
  }

  mesh::storage::ContactPageHeader header;
  header.page_index = page;
  header.occupied = occupied;
  header.generation = _contact_page_generations[page] + 1;
  header.payload_crc = payload_crc;
  uint8_t raw_header[mesh::storage::CONTACT_PAGE_HEADER_SIZE];
  mesh::storage::encodeContactPageHeader(raw_header, header);

  char path[24];
  makeContactPagePath(page, path);
  mesh::AtomicFileWriter writer(_getContactsChannelsFS(), path);
  bool wrote = writer
      && writer.write(raw_header, sizeof(raw_header)) == sizeof(raw_header);
  for (uint8_t slot = 0; wrote && slot < mesh::storage::CONTACTS_PER_PAGE; slot++) {
    memset(record, 0, sizeof(record));
    if (page_contacts[slot] != nullptr) {
      if (!serializeContactRecord(*page_contacts[slot], record)) {
#if MESH_CONTACT_CACHE
        mesh::contactPathStorage().endCommit(false);
#endif
        return false;
      }
    }
    wrote = writer.write(record, sizeof(record)) == sizeof(record);
  }
  if (!writer.commit(wrote)) {
#if MESH_CONTACT_CACHE
    paths.endCommit(false);
#endif
    MESH_DEBUG_PRINTLN("DataStore: atomic contact page %u write failed", page);
    return false;
  }

#if MESH_CONTACT_CACHE
  for (auto* contact : page_contacts) if (contact)
    paths.publish(contact->path_ref.handle(), 0x8000 | contact->storage_slot);
  paths.endCommit(true);
#endif
  _contact_page_generations[page] = header.generation;
  return true;
}
#endif

void DataStore::loadContacts(DataStoreHost* host) {
#if defined(ESP32_PLATFORM)
  cancelContactWrite();
#endif
#if !defined(NRF52_PLATFORM) && !MESH_CONTACT_CACHE
  if (_uncached_contact_load_incomplete) return;
#endif
#if MESH_CONTACT_CACHE
  _cache_host = host;
  mesh::contactPathStorage().attach(this);
#if MESH_CONTACT_SECRET_FLASH_CACHE
  mesh::contactSecretCache().attach(this);
#endif
  if (_cache_load_incomplete) return;
#if defined(ESP32_PLATFORM)
  _contact_path_reader.close();
#endif
#endif
#if !defined(NRF52_PLATFORM)
  bool contacts_exist = false;
  bool contacts_metadata_ready = companionPathPresence(
      _getContactsChannelsFS(), "/contacts3", contacts_exist);
#endif
#if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM)
  bool backup_exists = false;
  contacts_metadata_ready = contacts_metadata_ready && companionPathPresence(
      _getContactsChannelsFS(), "/contacts3.bak", backup_exists);
  const bool contacts_required = contacts_exist || backup_exists;
  contacts_metadata_ready = contacts_metadata_ready
      && mesh::ContactFileTransaction::recover(_getContactsChannelsFS(), "/contacts3")
      && companionPathPresence(_getContactsChannelsFS(), "/contacts3", contacts_exist)
      && (!contacts_required || contacts_exist);
#endif
#if !defined(NRF52_PLATFORM)
  if (!contacts_metadata_ready) {
    MESH_DEBUG_PRINTLN("DataStore: contact transaction recovery failed");
#if MESH_CONTACT_CACHE
    _cache_load_incomplete = true;
#else
    _uncached_contact_load_incomplete = true;
#endif
    return;
  }
#endif
#if defined(NRF52_PLATFORM)
  // loadContacts() is also used after an identity import.  Rebuild runtime
  // slot ownership from disk so stale pointers/slots from the previous in-RAM
  // contact table cannot collide with the reload.
  resetContactPageState();
  if (_contact_load_incomplete) {
    // A failed load is latched for this boot. loadContacts() is also called by
    // identity import, but only a reboot may retry storage and re-enable
    // migration/mutation; otherwise an unrelated command could clear the veto.
    MESH_DEBUG_PRINTLN(
        "DataStore: contact load remains quarantined until reboot");
    return;
  }
  FILESYSTEM* contacts_fs = _getContactsChannelsFS();
#if defined(MESHCORE_EXTRAFS_HIL)
  // Reuse the read-failure marker's high bit for a one-boot source-discovery
  // stat failure.  Consume and verify the marker before injecting the error so
  // the untouched page is recoverable on the very next reboot.  Low-bit values
  // remain armed for loadContactPages(), preserving the existing read hook.
  int hil_stat_failure_page = -1;
  bool hil_failure_marker_exists = false;
  if (!contactPathPresence(contacts_fs, HIL_CONTACT_PAGE_FAILURE_MARKER,
                           hil_failure_marker_exists)) {
    _contact_load_incomplete = true;
    return;
  }
  if (hil_failure_marker_exists) {
    File marker = openRead(contacts_fs, HIL_CONTACT_PAGE_FAILURE_MARKER);
    const int encoded_failure = marker && marker.size() == 1
        ? marker.read() : -1;
    if (marker) marker.close();
    if (encoded_failure >= 0
        && (encoded_failure & HIL_CONTACT_STAT_FAILURE_FLAG) != 0) {
      const uint8_t page = static_cast<uint8_t>(
          encoded_failure & ~HIL_CONTACT_STAT_FAILURE_FLAG);
      bool marker_remains = true;
      const bool removed = contacts_fs->remove(
          HIL_CONTACT_PAGE_FAILURE_MARKER);
      const bool verified = contactPathPresence(
          contacts_fs, HIL_CONTACT_PAGE_FAILURE_MARKER, marker_remains);
      if (page >= mesh::storage::CONTACT_PAGE_COUNT || !removed || !verified
          || marker_remains) {
        MESH_DEBUG_PRINTLN(
            "DataStore: HIL stat-failure marker could not be consumed safely");
        _contact_load_incomplete = true;
        return;
      }
      hil_stat_failure_page = page;
    }
  }
#endif
  uint32_t contact_page_presence = 0;
  for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) {
    char path[24];
    makeContactPagePath(page, path);
    bool page_exists = false;
#if defined(MESHCORE_EXTRAFS_HIL)
    if (page == hil_stat_failure_page) {
      MESH_DEBUG_PRINTLN(
          "DataStore: HIL injecting contact page %u stat failure", page);
      _contact_load_incomplete = true;
      return;
    }
#endif
    if (!contactPathPresence(contacts_fs, path, page_exists)) {
      _contact_load_incomplete = true;
      return;
    }
    if (page_exists) {
      contact_page_presence |= 1UL << page;
    }
  }
  bool legacy_exists = false;
  bool migration_marker_exists = false;
  if (!contactPathPresence(contacts_fs, "/contacts3", legacy_exists)
      || !contactPathPresence(contacts_fs, CONTACT_MIGRATION_MARKER,
                              migration_marker_exists)) {
    _contact_load_incomplete = true;
    return;
  }
  const mesh::storage::ContactStoreSource source =
      mesh::storage::chooseContactStoreSource(
          legacy_exists, contact_page_presence != 0);
  if (source == mesh::storage::ContactStoreSource::PAGED) {
    loadContactPages(host, 0, contact_page_presence);
    // A reset after the final legacy removal may leave this harmless marker.
    // Retire it only after every page captured above loaded successfully; an
    // unread page must leave all recovery metadata untouched for reboot retry.
    if (!hasIncompleteContactLoad() && migration_marker_exists) {
      contacts_fs->remove(CONTACT_MIGRATION_MARKER);
    }
    return;
  }
  if (source == mesh::storage::ContactStoreSource::EMPTY) {
    if (migration_marker_exists) {
      contacts_fs->remove(CONTACT_MIGRATION_MARKER);
    }
    return;
  }

  // Migrate /contacts3 from the tail so the legacy prefix and completed pages
  // never need enough room to coexist in full.  A page is committed first,
  // then the corresponding legacy tail is truncated.  On a reset between
  // those operations the still-present legacy prefix wins overlapping slots.
  _legacy_contacts_pending_cleanup = true;
  _legacy_migration_ready = migration_marker_exists;
#endif

  File file = openRead(_getContactsChannelsFS(), "/contacts3");
#if !defined(NRF52_PLATFORM)
  if (!file && contacts_exist) {
#if MESH_CONTACT_CACHE
    _cache_load_incomplete = true;
#else
    _uncached_contact_load_incomplete = true;
#endif
    return;
  }
#endif
#if defined(NRF52_PLATFORM)
  if (!file) {
    // Never delete, truncate, or write alongside a legacy source that could
    // not be opened. A later reboot can retry the untouched authoritative
    // file, but this boot must not accept a mutation into a competing page.
    MESH_DEBUG_PRINTLN("DataStore: legacy contacts exist but could not be read");
    _contact_load_incomplete = true;
    return;
  }

  const size_t legacy_size = file.size();
  if (!mesh::storage::isValidLegacyContactFileSize(legacy_size)) {
    MESH_DEBUG_PRINTLN(
        "DataStore: legacy contacts size %lu is invalid; preserving source",
        (unsigned long)legacy_size);
    file.close();
    _contact_load_incomplete = true;
    return;
  }
#endif
  if (file) {
    bool full = false;
    uint16_t record_index = 0;
#if !defined(NRF52_PLATFORM)
    if (file.size() % mesh::storage::CONTACT_RECORD_SIZE != 0) {
#if MESH_CONTACT_CACHE
      _cache_load_incomplete = true;
#else
      _uncached_contact_load_incomplete = true;
#endif
      file.close();
      return;
    }
#endif
#if defined(NRF52_PLATFORM)
    _legacy_contact_count =
        mesh::storage::legacyContactCountForSize(legacy_size);
    bool legacy_read_failed = false;
    bool legacy_host_refused = false;
#endif
    while (!full
#if !defined(NRF52_PLATFORM)
           && record_index < file.size() / mesh::storage::CONTACT_RECORD_SIZE
#endif
#if defined(NRF52_PLATFORM)
           && record_index < _legacy_contact_count
#endif
    ) {
      uint8_t record[mesh::storage::CONTACT_RECORD_SIZE];
      if (file.read(record, sizeof(record)) != sizeof(record)) {
#if MESH_CONTACT_CACHE
        _cache_load_incomplete = true;
#elif !defined(NRF52_PLATFORM)
        _uncached_contact_load_incomplete = true;
#endif
#if defined(NRF52_PLATFORM)
        legacy_read_failed = true;
#endif
        break;
      }

      ContactInfo contact;
      bool path_unavailable = false;
      if (!deserializeContactRecord(record, contact, record_index, &path_unavailable)) {
        if (path_unavailable) {
#if MESH_CONTACT_CACHE
          _cache_load_incomplete = true;
#elif !defined(NRF52_PLATFORM)
          _uncached_contact_load_incomplete = true;
#endif
#if defined(NRF52_PLATFORM)
          legacy_read_failed = true;
#endif
          break;
        }
        // Preserve the contact while containing corrupt legacy routing data.
        contact.out_path_len = OUT_PATH_UNKNOWN;
      }
#if defined(NRF52_PLATFORM)
      const uint16_t slot = record_index;
      if (!_contact_slots.reserve(slot)) {
        legacy_read_failed = true;
        break;
      }
      contact.storage_slot = slot;
#endif
      ++record_index;
      if (!host->onContactLoaded(contact)) {
        full = true;
#if MESH_CONTACT_CACHE
        _cache_load_incomplete = true;
#elif !defined(NRF52_PLATFORM)
        _uncached_contact_load_incomplete = true;
#endif
#if defined(NRF52_PLATFORM)
        _contact_slots.release(slot);
        legacy_host_refused = true;
#endif
      }
    }
    file.close();

#if defined(NRF52_PLATFORM)
    if (legacy_read_failed || legacy_host_refused
        || record_index != _legacy_contact_count) {
      if (legacy_host_refused) {
        // Capacity refusal is semantically distinct from an I/O error, but the
        // durable source still contains contacts absent from RAM. Preserve it
        // and use the same mutation/migration veto until a clean load succeeds.
        MESH_DEBUG_PRINTLN(
            "DataStore: contact host capacity refused legacy record %u; load incomplete",
            (unsigned)record_index);
      } else {
        MESH_DEBUG_PRINTLN(
            "DataStore: short/incomplete legacy contact read at record %u of %u",
            (unsigned)record_index, (unsigned)_legacy_contact_count);
      }
      _contact_load_incomplete = true;
      return;
    }

    // Only modify stale page artifacts after the complete authoritative legacy
    // file has been loaded. A transient read failure above therefore leaves
    // every possible recovery source untouched.
    if (!mesh::storage::trustMigratedContactPages(
            true, _legacy_migration_ready)) {
      if (!prepareLegacyContactMigration()) {
        _contact_load_incomplete = true;
        return;
      }
      // prepareLegacyContactMigration() intentionally removed every stale page
      // observed in the first pass before committing the marker.
      contact_page_presence = 0;
    }
#endif
  }

#if defined(NRF52_PLATFORM)
  // Pages at and beyond the remaining legacy prefix have already committed.
  // Loading both sources this way resumes safely after every possible reset
  // point, including a reset after page rename but before legacy truncation.
  if (_legacy_migration_ready) {
    loadContactPages(host, _legacy_contact_count, contact_page_presence);
  }
#endif
}

bool DataStore::saveContacts(DataStoreHost* host, bool (*filter)(const ContactInfo& c)) {
#if defined(ESP32_PLATFORM)
  cancelContactWrite();
#endif
  if (hasIncompleteContactLoad()) return false;
#if defined(NRF52_PLATFORM)
  bool success = true;
  for (uint32_t idx = 0;; idx++) {
    ContactInfo* contact = host->getContactForStore(idx);
    if (contact == NULL) break;
    if (filter && !filter(*contact)) continue;
    success = markContactDirty(*contact) && success;
  }
  return flushContactWrites(host, filter) && success;
#elif defined(ESP32_PLATFORM)
  // Explicit save/flush callers require a fully durable current table. An
  // unfinished lazy transaction may contain an earlier revision: discard it
  // before synchronously writing the current contacts through the same format.
#if MESH_CONTACT_CACHE
  auto& paths = mesh::contactPathStorage();
  paths.beginCommit();
  for (uint32_t i = 0;; ++i) {
    auto* c = host->getContactForStore(i);
    if (!c) break;
    if (!filter || filter(*c)) paths.mark(c->path_ref.handle());
  }
  if (!paths.preserveSnapshots(0, 0x8000)) {
    paths.endCommit(false);
    _contact_write_requested = true;
    return false;
  }
#endif
  // The cold-path reader reuses one SPIFFS File during this streaming write.
  // Close it before replacing the original name so later reads reopen the
  // committed file. No complete contact-table copy is needed in RAM.
  mesh::ContactFileTransaction writer(_getContactsChannelsFS(), "/contacts3",
                                      companionPathPresence);
  bool success = writer;
  uint8_t record[mesh::storage::CONTACT_RECORD_SIZE];
  for (uint32_t i = 0; success; ++i) {
#if MESH_CONTACT_CACHE
    auto* c = host->getContactForStore(i);
    if (!c) break;
#else
    ContactInfo snapshot;
    if (!host->getContactForSave(i, snapshot)) break;
    const auto* c = &snapshot;
#endif
    if (filter && !filter(*c)) continue;
    success = serializeContactRecord(*c, record)
        && writer.write(record, sizeof(record)) == sizeof(record);
  }
#if MESH_CONTACT_CACHE
  _contact_path_reader.close();
#endif
  success = writer.commit(success);
#if MESH_CONTACT_CACHE
  if (success) {
    uint16_t index = 0;
    for (uint32_t i = 0;; ++i) {
      auto* c = host->getContactForStore(i);
      if (!c) break;
      if (filter && !filter(*c)) continue;
      paths.publish(c->path_ref.handle(), index++);
    }
  }
  paths.endCommit(success);
#endif
  _contact_write_requested = !success;
  return success;
#else
#if defined(STM32_PLATFORM)
  mesh::AtomicFileWriter file(_getContactsChannelsFS(), "/contacts3");
#else
  mesh::ContactFileTransaction file(_getContactsChannelsFS(), "/contacts3");
#endif
  bool success = (bool)file;
  if (file) {
    uint32_t idx = 0;
    ContactInfo c;
    uint8_t tx_radio = 0;

    while (host->getContactForSave(idx, c)) {
      if (filter && !filter(c)) {
        idx++;  // advance to next contact
        continue;
      }
      success = (file.write(c.id.pub_key, 32) == 32);
      success = success && (file.write((uint8_t *)&c.name, 32) == 32);
      success = success && (file.write(&c.type, 1) == 1);
      success = success && (file.write(&c.flags, 1) == 1);
      tx_radio = mesh::encodeRadioTxPolicy(c.tx_radio);
      success = success && (file.write(&tx_radio, 1) == 1);
      success = success && (file.write((uint8_t *)&c.sync_since, 4) == 4);
      success = success && (file.write((uint8_t *)&c.out_path_len, 1) == 1);
      success = success && (file.write((uint8_t *)&c.last_advert_timestamp, 4) == 4);
      uint8_t path[64];
      success = success && c.copyPathTo(path) && (file.write(path, 64) == 64);
      success = success && (file.write((uint8_t *)&c.lastmod, 4) == 4);
      success = success && (file.write((uint8_t *)&c.gps_lat, 4) == 4);
      success = success && (file.write((uint8_t *)&c.gps_lon, 4) == 4);

      if (!success) break; // write failed

      idx++;  // advance to next contact
    }
    success = file.commit(success);
  }
  return success;
#endif
}

bool DataStore::markContactDirty(const ContactInfo& contact) {
#if defined(NRF52_PLATFORM)
  if (hasIncompleteContactLoad()) return false;
  uint16_t slot = contact.storage_slot;
  if (!_contact_slots.isUsed(slot)) {
    slot = _contact_slots.allocate();
    if (slot == mesh::storage::CONTACT_SLOT_NONE) return false;
    contact.storage_slot = slot;
  }
  return _dirty_contact_pages.mark(slot / mesh::storage::CONTACTS_PER_PAGE);
#else
  (void)contact;
#if defined(ESP32_PLATFORM)
  ++_contact_write_revision;
  _contact_write_requested = true;
#endif
  return true;
#endif
}

bool DataStore::releaseContact(const ContactInfo& contact) {
#if defined(NRF52_PLATFORM)
  if (hasIncompleteContactLoad()) return false;
  const uint16_t slot = contact.storage_slot;
  if (!_contact_slots.release(slot)) return false;
  contact.storage_slot = mesh::storage::CONTACT_SLOT_NONE;
  return _dirty_contact_pages.mark(slot / mesh::storage::CONTACTS_PER_PAGE);
#else
  (void)contact;
#if defined(ESP32_PLATFORM)
  ++_contact_write_revision;
  _contact_write_requested = true;
#endif
  return true;
#endif
}

bool DataStore::restoreContactSlot(const ContactInfo& contact, uint16_t slot) {
#if defined(NRF52_PLATFORM)
  if (hasIncompleteContactLoad()) return false;
  // Roll back a release into the exact slot it vacated. Allocating the first
  // free slot here could move the record to another page; a reset between the
  // two resulting page writes would then leave duplicate records on disk.
  if (!_contact_slots.reserve(slot)) return false;
  if (!_dirty_contact_pages.mark(
          slot / mesh::storage::CONTACTS_PER_PAGE)) {
    _contact_slots.release(slot);
    return false;
  }
  contact.storage_slot = slot;
  return true;
#else
  (void)contact;
  (void)slot;
  return true;
#endif
}

#if defined(NRF52_PLATFORM)
bool DataStore::truncateLegacyContacts(uint16_t remaining_contacts) {
  FILESYSTEM* fs = _getContactsChannelsFS();
  if (remaining_contacts == 0) {
    bool legacy_exists = false;
    if (!contactPathPresence(fs, "/contacts3", legacy_exists)) return false;
    if (legacy_exists && !fs->remove("/contacts3")) return false;

    bool marker_exists = false;
    if (!contactPathPresence(fs, CONTACT_MIGRATION_MARKER, marker_exists)) {
      return false;
    }
    if (marker_exists && !fs->remove(CONTACT_MIGRATION_MARKER)) return false;

    _legacy_contact_count = 0;
    _legacy_contacts_pending_cleanup = false;
    _legacy_migration_ready = false;
    return true;
  }

  File file = fs->open("/contacts3", FILE_O_WRITE);
  if (!file) return false;

  const uint32_t expected_size =
      (uint32_t)remaining_contacts * mesh::storage::CONTACT_RECORD_SIZE;
  const bool truncated = file.truncate(expected_size);
  if (truncated) file.flush();
  file.close();
  if (!truncated) return false;

  // Verify the committed length before advancing the in-memory transaction.
  // If verification itself fails, retrying the same page/truncate is harmless.
  File verify = openRead(fs, "/contacts3");
  const bool valid = verify && verify.size() == expected_size;
  if (verify) verify.close();
  if (!valid) return false;

  _legacy_contact_count = remaining_contacts;
  return true;
}
#endif

#if defined(ESP32_PLATFORM)
void DataStore::cancelContactWrite() {
  if (_contact_write == nullptr) return;
  delete _contact_write;
  _contact_write = nullptr;
#if MESH_CONTACT_CACHE
  _contact_path_reader.close();
  mesh::contactPathStorage().endCommit(false);
#endif
  _contact_write_host = nullptr;
  _contact_write_filter = nullptr;
}

#if MESH_CONTACT_CACHE
bool DataStore::cancelCooperativeWrite() {
  // Cache pressure inside serialization/snapshot preservation must keep the
  // reentrancy veto. Only an app/radio path update between service passes can
  // abandon the lazy job before the ordinary guarded eviction flush.
  if (_contact_write == nullptr || _contact_write_servicing) return false;
  cancelContactWrite();
  return true;
}
#endif

bool DataStore::serviceContactWrite(DataStoreHost* host,
                                   bool (*filter)(const ContactInfo&)) {
  struct ServiceGuard {
    bool& active;
    explicit ServiceGuard(bool& value) : active(value) { active = true; }
    ~ServiceGuard() { active = false; }
  } guard(_contact_write_servicing);
  if (hasIncompleteContactLoad()) {
    cancelContactWrite();
    return false;
  }
  _contact_write_requested = true;
  if (_contact_write != nullptr
      && (_contact_write_active_revision != _contact_write_revision
          || _contact_write_host != host || _contact_write_filter != filter)) {
    // A mutation can move records or replace path handles. Never publish the
    // indices of an obsolete streaming snapshot. Keep dirty state armed and
    // start the current revision on a later loop pass.
    cancelContactWrite();
    return true;
  }
#if MESH_CONTACT_CACHE
  auto& paths = mesh::contactPathStorage();
#endif
  if (_contact_write == nullptr) {
#if MESH_CONTACT_CACHE
    paths.beginCommit();
    for (uint32_t i = 0;; ++i) {
      auto* contact = host->getContactForStore(i);
      if (contact == nullptr) break;
      if (!filter || filter(*contact)) paths.mark(contact->path_ref.handle());
    }
    if (!paths.preserveSnapshots(0, 0x8000)) {
      paths.endCommit(false);
      return false;
    }
#endif
    _contact_write = new (std::nothrow) mesh::ContactFileTransaction(
        _getContactsChannelsFS(), "/contacts3", companionPathPresence, true);
    if (_contact_write == nullptr) {
#if MESH_CONTACT_CACHE
      paths.endCommit(false);
#endif
      return false;
    }
    _contact_write_host = host;
    _contact_write_filter = filter;
    _contact_write_active_revision = _contact_write_revision;
    _contact_write_index = 0;
    _contact_write_verifying = false;
    return true;
  }
  if (!static_cast<bool>(*_contact_write)) {
    // The revision/session guard above also applies to every setup step.
    // Keep initialization separate from serialization, including its final
    // buffer configuration, so transports are serviced between operations.
    const auto progress = _contact_write->serviceBegin(true);
    if (progress == mesh::ContactFileTransaction::BeginProgress::Failed) {
      cancelContactWrite();
      return false;
    }
    return true;
  }
  if (!_contact_write_verifying) {
#if MESH_CONTACT_CACHE
    ContactInfo* contact;
    do {
      contact = host->getContactForStore(_contact_write_index++);
    } while (contact != nullptr && filter && !filter(*contact));
#else
    ContactInfo snapshot;
    bool available;
    do {
      available = host->getContactForSave(_contact_write_index++, snapshot);
    } while (available && filter && !filter(snapshot));
    const ContactInfo* contact = available ? &snapshot : nullptr;
#endif
    if (contact != nullptr) {
      uint8_t record[mesh::storage::CONTACT_RECORD_SIZE];
      if (serializeContactRecord(*contact, record)
          && _contact_write->write(record, sizeof(record)) == sizeof(record)) {
        return true;
      }
      cancelContactWrite();
      return false;
    }
#if MESH_CONTACT_CACHE
    _contact_path_reader.close();
#endif
    _contact_write_verifying = true;
    return true;
  }
#if MESH_CONTACT_CACHE
  if (_contact_write->readyToPublish()
      && !paths.preserveSnapshots(0, 0x8000)) {
    cancelContactWrite();
    return false;
  }
  if (_contact_write->readyToPublish()) {
    // Cold app reads and final snapshot preservation can reopen the original
    // inode during verification. Close it at the actual publication boundary.
    _contact_path_reader.close();
  }
#endif
  // Keep the 64-byte CRC scratch while verifying at most 512 bytes per
  // background pass; transports run between these bounded batches.
  const auto progress = _contact_write->serviceCommit(true, 8, true);
  if (progress == mesh::ContactFileTransaction::CommitProgress::Pending) {
    return true;
  }
  if (progress == mesh::ContactFileTransaction::CommitProgress::Failed) {
    cancelContactWrite();
    return false;
  }
#if MESH_CONTACT_CACHE
  // serviceCommit's final name replacement and this handle publication are
  // one non-yielding section. Every later cold read sees the corresponding
  // committed file and source indices, including retained ContactInfo copies.
  uint16_t index = 0;
  for (uint32_t i = 0;; ++i) {
    auto* contact = host->getContactForStore(i);
    if (contact == nullptr) break;
    if (!filter || filter(*contact)) paths.publish(contact->path_ref.handle(), index++);
  }
  paths.endCommit(true);
#endif
  delete _contact_write;
  _contact_write = nullptr;
  _contact_write_host = nullptr;
  _contact_write_filter = nullptr;
  _contact_write_requested = false;
  return true;
}
#endif

bool DataStore::serviceContactWrites(DataStoreHost* host,
                                     bool (*filter)(const ContactInfo& c)) {
#if defined(NRF52_PLATFORM)
  if (hasIncompleteContactLoad()) {
    // Reboot is the recovery operation. Never migrate or rewrite from a RAM
    // table which is known not to represent every durable contact.
    return false;
  }
  if (_legacy_contacts_pending_cleanup) {
    FILESYSTEM* fs = _getContactsChannelsFS();
    bool legacy_exists = false;
    if (!contactPathPresence(fs, "/contacts3", legacy_exists)) return false;
    if (!legacy_exists) {
      bool marker_exists = false;
      if (!contactPathPresence(fs, CONTACT_MIGRATION_MARKER,
                               marker_exists)) {
        return false;
      }
      if (marker_exists && !fs->remove(CONTACT_MIGRATION_MARKER)) return false;
      _legacy_contacts_pending_cleanup = false;
      _legacy_migration_ready = false;
      _legacy_contact_count = 0;
    } else if (!_legacy_migration_ready
               && !prepareLegacyContactMigration()) {
      return false;
    } else if (_legacy_contact_count == 0) {
      return truncateLegacyContacts(0);
    } else {
      const uint8_t page =
          mesh::storage::legacyMigrationPage(_legacy_contact_count);
      if ((_unread_contact_pages.bits() & (1UL << page)) != 0) {
        // The legacy prefix cannot reconstruct post-prefix records from an
        // unread committed page. Keep both sources untouched for reboot retry.
        return false;
      }
      if (page >= mesh::storage::CONTACT_PAGE_COUNT
          || !writeContactPage(host, page, filter)) {
        return false;
      }

      const uint16_t remaining =
          mesh::storage::legacyCountAfterMigratingPage(page);
      if (!truncateLegacyContacts(remaining)) return false;
      _dirty_contact_pages.clear(page);
      return true;
    }
  }

  const int page = _dirty_contact_pages.first();
  if (page < 0) return true;
  if (!writeContactPage(host, (uint8_t)page, filter)) return false;
  _dirty_contact_pages.clear((uint8_t)page);
  return true;
#elif defined(ESP32_PLATFORM)
  return serviceContactWrite(host, filter);
#else
  return saveContacts(host, filter);
#endif
}

bool DataStore::flushContactWrites(DataStoreHost* host,
                                   bool (*filter)(const ContactInfo& c)) {
#if defined(ESP32_PLATFORM)
  cancelContactWrite();
#endif
  if (hasIncompleteContactLoad()) return false;
#if defined(NRF52_PLATFORM)
  while (hasPendingContactWrites()) {
    if (!serviceContactWrites(host, filter)) return false;
  }
  return true;
#else
  return saveContacts(host, filter);
#endif
}

bool DataStore::hasPendingContactWrites() const {
#if defined(NRF52_PLATFORM)
  return !hasIncompleteContactLoad()
      && (!_dirty_contact_pages.empty() || _legacy_contacts_pending_cleanup);
#elif defined(ESP32_PLATFORM)
  return _contact_write_requested || _contact_write != nullptr;
#else
  return false;
#endif
}

bool DataStore::hasIncompleteContactLoad() const {
#if !defined(NRF52_PLATFORM)
  if (_channel_load_incomplete) return true;
#if !MESH_CONTACT_CACHE
  if (_uncached_contact_load_incomplete) return true;
#endif
#endif
#if MESH_CONTACT_CACHE
  if (_cache_load_incomplete) return true;
#endif
#if defined(NRF52_PLATFORM)
  return _contact_load_incomplete || !_unread_contact_pages.empty();
#else
  return false;
#endif
}

void DataStore::loadChannels(DataStoreHost* host) {
#if defined(ESP32_PLATFORM)
  _channel_load_incomplete = false;
  _channel_recovery_source = nullptr;
  FILESYSTEM* fs = _getContactsChannelsFS();
  for (const char* path : {"/channels2", "/channels2.bak"}) {
    for (unsigned attempt = 0; attempt < 2; ++attempt) {
      bool present = false;
      if (!companionPathPresence(fs, path, present)) continue;
      if (!present) break;
      File file = openRead(fs, path);
      if (!file) continue;
      static const uint32_t RECORD_SIZE = 4 + 32 + 32;
      const size_t size = file.size();
      if (size % RECORD_SIZE != 0 || size / RECORD_SIZE > MAX_GROUP_CHANNELS) {
        file.close();
        break;
      }
      const uint8_t count = size / RECORD_SIZE;
      ChannelDetails* loaded = count == 0 ? nullptr
          : static_cast<ChannelDetails*>(malloc(sizeof(ChannelDetails) * count));
      if (count != 0 && loaded == nullptr) {
        file.close();
        // Heap pressure is not evidence that the durable data is damaged.
        _channel_load_incomplete = true;
        return;
      }
      bool valid = true;
      for (uint8_t i = 0; valid && i < count; ++i) {
        uint8_t unused[4];
        valid = file.read(unused, sizeof(unused)) == sizeof(unused)
            && file.read(reinterpret_cast<uint8_t*>(loaded[i].name), 32) == 32
            && file.read(loaded[i].channel.secret, 32) == 32;
        if (valid) {
          loaded[i].name[31] = 0;
          loaded[i].channel.tx_radio = mesh::decodeRadioTxPolicy(unused[0]);
        }
      }
      file.close();
      if (!valid) { free(loaded); continue; }
      for (uint8_t i = 0; i < count; ++i) {
        if (!host->onChannelLoaded(i, loaded[i])) {
          free(loaded);
          _channel_load_incomplete = true;
          return;
        }
      }
      free(loaded);
      if (strcmp(path, "/channels2") != 0) {
        _channel_recovery_source = path;
        promoteCompanionRecoveryFile(fs, "/channels2", _channel_recovery_source);
      }
      return;
    }
  }
  MESH_DEBUG_PRINTLN("DataStore: no recoverable channels; defaults may replace the old image");
  return;
#else
#if defined(NRF52_PLATFORM)
  bool& incomplete = _contact_load_incomplete;
#else
  bool& incomplete = _channel_load_incomplete;
#endif
  if (incomplete) return;
  FILESYSTEM* contacts_fs = _getContactsChannelsFS();
#if defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM)
  if (!mesh::ContactFileTransaction::recover(contacts_fs, "/channels2")) {
    MESH_DEBUG_PRINTLN("DataStore: channel transaction recovery failed; storage quarantined");
    incomplete = true;
    return;
  }
#endif
  uint32_t channels_size = 0;
#if defined(NRF52_PLATFORM)
  bool channels_exist = false;
  if (!contactPathPresence(contacts_fs, "/channels2", channels_exist,
                           &channels_size)) {
    incomplete = true;
    return;
  }
  if (!channels_exist) return;
#else
  if (!contacts_fs->exists("/channels2")) return;
#endif
  File file = openRead(contacts_fs, "/channels2");
  if (!file) {
    MESH_DEBUG_PRINTLN("DataStore: channels file open failed; storage quarantined");
    incomplete = true;
    return;
  }
#if defined(NRF52_PLATFORM)
  if (file.size() != channels_size) {
    file.close();
    MESH_DEBUG_PRINTLN("DataStore: channels file changed while opening; storage quarantined");
    incomplete = true;
    return;
  }
#else
  channels_size = file.size();
#endif
  static const uint32_t CHANNEL_RECORD_SIZE = 4 + 32 + 32;
  if ((channels_size % CHANNEL_RECORD_SIZE) != 0
      || channels_size / CHANNEL_RECORD_SIZE > MAX_GROUP_CHANNELS) {
    file.close();
    MESH_DEBUG_PRINTLN("DataStore: invalid channels file size; storage quarantined");
    incomplete = true;
    return;
  }
  const uint8_t channel_count =
      static_cast<uint8_t>(channels_size / CHANNEL_RECORD_SIZE);
  ChannelDetails* loaded = channel_count == 0 ? nullptr
      : static_cast<ChannelDetails*>(
          malloc(sizeof(ChannelDetails) * channel_count));
  if (channel_count != 0 && loaded == nullptr) {
    file.close();
    MESH_DEBUG_PRINTLN(
        "DataStore: no memory for channels snapshot; storage quarantined");
    incomplete = true;
    return;
  }
  bool success = true;
  for (uint8_t channel_idx = 0; channel_idx < channel_count; channel_idx++) {
    uint8_t unused[4];
    success = file.read(unused, sizeof(unused)) == sizeof(unused)
        && file.read(reinterpret_cast<uint8_t*>(loaded[channel_idx].name),
                     sizeof(loaded[channel_idx].name))
            == sizeof(loaded[channel_idx].name)
        && file.read(loaded[channel_idx].channel.secret,
                     sizeof(loaded[channel_idx].channel.secret))
            == sizeof(loaded[channel_idx].channel.secret);
    if (!success) break;
    loaded[channel_idx].name[sizeof(loaded[channel_idx].name) - 1] = 0;
    loaded[channel_idx].channel.tx_radio = mesh::decodeRadioTxPolicy(unused[0]);
  }
  file.close();
  if (!success) {
    free(loaded);
    MESH_DEBUG_PRINTLN(
        "DataStore: channels file read failed; storage quarantined");
    incomplete = true;
    return;
  }
  for (uint8_t channel_idx = 0; channel_idx < channel_count; channel_idx++) {
    if (!host->onChannelLoaded(channel_idx, loaded[channel_idx])) {
      free(loaded);
      MESH_DEBUG_PRINTLN(
          "DataStore: channel host refused durable record; storage quarantined");
      incomplete = true;
      return;
    }
  }
  free(loaded);
#endif
}

bool DataStore::saveChannels(DataStoreHost* host) {
#if defined(ESP32_PLATFORM)
  if (!promoteCompanionRecoveryFile(_getContactsChannelsFS(), "/channels2",
                                    _channel_recovery_source)) return false;
#endif
#if !defined(NRF52_PLATFORM)
  if (_channel_load_incomplete) return false;
#endif
#if defined(NRF52_PLATFORM)
  if (_contact_load_incomplete) return false;
#endif
#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
  mesh::AtomicFileWriter file(_getContactsChannelsFS(), "/channels2");
#elif defined(ESP32_PLATFORM)
  mesh::ContactFileTransaction file(_getContactsChannelsFS(), "/channels2", companionPathPresence);
#elif defined(RP2040_PLATFORM)
  mesh::ContactFileTransaction file(_getContactsChannelsFS(), "/channels2");
#else
  File file = openWrite(_getContactsChannelsFS(), "/channels2");
#endif
  bool success = (bool)file;
  if (file) {
    uint8_t channel_idx = 0;
    ChannelDetails ch;
    uint8_t unused[4];
    memset(unused, 0, 4);

    while (success && host->getChannelForSave(channel_idx, ch)) {
      unused[0] = mesh::encodeRadioTxPolicy(ch.channel.tx_radio);
      success = (file.write(unused, 4) == 4);
      success = success && (file.write((uint8_t *)ch.name, 32) == 32);
      success = success && (file.write((uint8_t *)ch.channel.secret, 32) == 32);

      if (!success) break; // write failed
      channel_idx++;
    }
#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) || defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM)
    success = file.commit(success);
    if (!success) MESH_DEBUG_PRINTLN("DataStore: atomic channels write failed");
#else
    file.close();
#endif
  }
  return success;
}

#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)

#define MAX_ADVERT_PKT_LEN   (2 + 32 + PUB_KEY_SIZE + 4 + SIGNATURE_SIZE + MAX_ADVERT_DATA_SIZE)

struct BlobRec {
  uint32_t timestamp;
  uint8_t  key[7];
  uint8_t  len;
  uint8_t  data[MAX_ADVERT_PKT_LEN];
};

#if !defined(NRF52_PLATFORM)
static void normalizeBlobKey(const uint8_t key[], int key_len, uint8_t normalized[7]) {
  memset(normalized, 0, 7);
  if (key == NULL || key_len <= 0) return;
  if (key_len > 7) key_len = 7;
  memcpy(normalized, key, key_len);
}
#endif

void DataStore::checkAdvBlobFile() {
#if defined(NRF52_PLATFORM)
  // Advert packets are a disposable cache and are learned again over the air.
  // Retaining the old 18 KiB monolithic cache alongside atomic buckets can
  // exhaust the 100 KiB ExtraFS and prevent a contact-page commit, so retire
  // it once on upgrade. Contact records and identity data are not affected.
  if (_fs->exists("/adv_blobs") && !_fs->remove("/adv_blobs")) {
    MESH_DEBUG_PRINTLN("DataStore: could not retire internal legacy advert cache");
  }
  if (_fsExtra != nullptr && _fsExtra->exists("/adv_blobs")
      && !_fsExtra->remove("/adv_blobs")) {
    MESH_DEBUG_PRINTLN("DataStore: could not retire secondary legacy advert cache");
  }
  return;
#else
  if (!_getContactsChannelsFS()->exists("/adv_blobs")) {
    File file = openWrite(_getContactsChannelsFS(), "/adv_blobs");
    if (file) {
      BlobRec zeroes;
      memset(&zeroes, 0, sizeof(zeroes));
      for (int i = 0; i < MAX_BLOBRECS; i++) {     // pre-allocate to fixed size
        file.write((uint8_t *) &zeroes, sizeof(zeroes));
      }
      file.close();
    }
  }
#endif
}

bool DataStore::migrateToSecondaryFS() {
#if defined(ESP32_PLATFORM)
  cancelContactWrite();
#endif
  if (_fsExtra == nullptr) return false;

  // Implemented below through verified copy transactions. On nRF52, all
  // primary contact/channel files are copied before an atomic journal commit;
  // no source is removed before that commit makes the complete secondary
  // snapshot authoritative across a reset.
#if defined(NRF52_PLATFORM)
  // /adv_blobs is a reconstructable cache retired by checkAdvBlobFile(); do
  // not spend time and temporary space atomically copying it first.
  static const char* to_secondary[] = {
      "/contacts3", "/contacts4.mig", "/channels2"};
#else
  static const char* to_secondary[] = {"/adv_blobs", "/contacts3", "/channels2"};
#endif
  static const char* to_primary[] = {"/_main.id", "/new_prefs"};

  auto filesEqual = [this](FILESYSTEM* left_fs, FILESYSTEM* right_fs,
                           const char* path) -> bool {
    File left = openRead(left_fs, path);
    File right = openRead(right_fs, path);
    const uint32_t expected_size = left ? left.size() : 0;
    if (!left || !right || expected_size != right.size()) {
      if (left) left.close();
      if (right) right.close();
      return false;
    }
    uint8_t left_buf[64], right_buf[64];
    bool equal = true;
    uint32_t compared = 0;
    while (equal) {
      int left_count = left.read(left_buf, sizeof(left_buf));
      int right_count = right.read(right_buf, sizeof(right_buf));
      if (left_count < 0 || right_count < 0) {
        equal = false;
      } else if (left_count != right_count) {
        equal = false;
      } else if (left_count <= 0) {
        break;
      } else if (memcmp(left_buf, right_buf, left_count) != 0) {
        equal = false;
      } else {
        compared += (uint32_t)left_count;
      }
    }
    left.close();
    right.close();
    return equal && compared == expected_size;
  };

  auto copy = [this, &filesEqual](FILESYSTEM* source_fs, FILESYSTEM* dest_fs,
                                  const char* path,
                                  bool exact_snapshot) -> bool {
#if defined(NRF52_PLATFORM)
    bool source_exists = false;
    if (!contactPathPresence(source_fs, path, source_exists)) return false;
    if (!source_exists) {
      if (!exact_snapshot) return true;

      // A reset during a previous Pending attempt can leave a destination-only
      // file. Primary is still authoritative until COMMITTED, so the retry
      // must remove that stale file rather than blessing it into the snapshot.
      bool destination_exists = false;
      if (!contactPathPresence(dest_fs, path, destination_exists)) return false;
      if (!destination_exists) return true;
      if (!dest_fs->remove(path)) return false;
      bool destination_remains = true;
      return contactPathPresence(dest_fs, path, destination_remains)
          && !destination_remains;
    }

    bool destination_exists = false;
    if (!contactPathPresence(dest_fs, path, destination_exists)) return false;
#else
    if (!source_fs->exists(path)) return true;
    const bool destination_exists = dest_fs->exists(path);
#endif
    if (destination_exists) {
      if (filesEqual(source_fs, dest_fs, path)) {
        return true;
      }
      if (!exact_snapshot) {
        MESH_DEBUG_PRINTLN("DataStore: migration conflict for %s; preserving both copies", path);
        return false;
      }
      // Pending migration destinations are non-authoritative. Fall through to
      // AtomicFileWriter so this stale copy is replaced as one transaction.
    }

    File source = openRead(source_fs, path);
    if (!source) return false;
    const uint32_t expected_size = source.size();
    bool success = true;
    uint8_t buf[64];

#if defined(NRF52_PLATFORM)
    mesh::AtomicFileWriter destination(dest_fs, path);
    success = (bool)destination;
    while (success) {
      int count = source.read(buf, sizeof(buf));
      if (count < 0) {
        success = false;
      } else if (count == 0) {
        break;
      } else {
        success = destination.write(buf, count) == (size_t)count;
      }
    }
    source.close();
    success = destination.commit(success && destination.bytesWritten() == expected_size);
#else
    char temp_path[64];
    snprintf(temp_path, sizeof(temp_path), "%s.tmp", path);
    File destination = openWrite(dest_fs, temp_path);
    success = (bool)destination;
    uint32_t written = 0;
    while (success) {
      int count = source.read(buf, sizeof(buf));
      if (count < 0) {
        success = false;
      } else if (count == 0) {
        break;
      } else {
        success = destination.write(buf, count) == (size_t)count;
        written += success ? count : 0;
      }
    }
    source.close();
    if (destination) destination.close();
    success = success && written == expected_size;
    if (success) {
      File verify = openRead(dest_fs, temp_path);
      success = verify && verify.size() == expected_size;
      if (verify) verify.close();
    }
    if (success) success = dest_fs->rename(temp_path, path);
    if (!success) dest_fs->remove(temp_path);
#endif

    if (!success || !filesEqual(source_fs, dest_fs, path)) {
      MESH_DEBUG_PRINTLN("DataStore: verified migration failed for %s", path);
      return false;
    }
    return true;
  };

#if !defined(NRF52_PLATFORM)
  auto move = [&copy](FILESYSTEM* source_fs, FILESYSTEM* dest_fs,
                      const char* path) -> bool {
    if (!source_fs->exists(path)) return true;
    return copy(source_fs, dest_fs, path, false) && source_fs->remove(path);
  };
#endif

  auto migratePrimarySources = [this, &copy, &filesEqual]() -> bool {
    bool success = true;
    for (size_t i = 0; i < sizeof(to_primary) / sizeof(to_primary[0]); i++) {
      const char* path = to_primary[i];
#if defined(NRF52_PLATFORM)
      const bool is_identity = strcmp(path, "/_main.id") == 0;
      const bool is_preferences = strcmp(path, "/new_prefs") == 0;
#endif
      bool primary_exists = false;
#if defined(NRF52_PLATFORM)
      if (!contactPathPresence(_fs, path, primary_exists)) {
        if (is_identity) _identity_creation_blocked = true;
        if (is_preferences) _prefs_load_incomplete = true;
        return false;
      }

      bool secondary_exists = false;
      if (!contactPathPresence(_fsExtra, path, secondary_exists)) {
        // Identity and preferences on primary are canonical. If that copy is
        // already present, a secondary metadata error only prevents optional
        // duplicate cleanup and is irrelevant to their availability.
        if (primary_exists) continue;
        if (is_identity) _identity_creation_blocked = true;
        if (is_preferences) _prefs_load_incomplete = true;
        return false;
      }
#else
      primary_exists = _fs->exists(path);
      const bool secondary_exists = _fsExtra->exists(path);
#endif
      if (!secondary_exists) continue;

      if (primary_exists) {
        if (!filesEqual(_fsExtra, _fs, path)) {
          // Identity/preferences are canonical on primary. A differing legacy
          // secondary copy is useful forensic data, but it must not prevent
          // contact/channel migration or make secondary storage disappear.
          MESH_DEBUG_PRINTLN("DataStore: preserving conflicting legacy secondary %s; primary remains authoritative", path);
          continue;
        }
      } else {
        // The required source can disappear between its discovery probe and
        // copy()'s own stat. Only a verified destination proves recovery.
        bool recovered_primary = copy(_fsExtra, _fs, path, false);
#if defined(NRF52_PLATFORM)
        bool recovered_primary_exists = false;
        recovered_primary = recovered_primary
            && contactPathPresence(_fs, path, recovered_primary_exists)
            && recovered_primary_exists;
#else
        recovered_primary = recovered_primary && _fs->exists(path);
#endif
        if (!recovered_primary) {
          // Without any primary copy MyMesh would generate a replacement
          // identity or persist default preferences, so this is the only
          // primary-source condition which must block the storage handoff.
#if defined(NRF52_PLATFORM)
          if (is_identity) _identity_creation_blocked = true;
          if (is_preferences) _prefs_load_incomplete = true;
#endif
          success = false;
          continue;
        }
      }

      // At this point primary has an exact verified copy. Failure to retire a
      // duplicate is harmless and can be retried on a later boot.
      if (!_fsExtra->remove(path)) {
        MESH_DEBUG_PRINTLN("DataStore: could not retire verified legacy secondary %s", path);
      }
    }
    return success;
  };

#if defined(NRF52_PLATFORM)
  enum class MigrationJournalState : uint8_t {
    None,
    Pending,
    Committed,
    Invalid,
    IoError,
  };
  static const uint8_t journal_magic[] = {'M', 'C', 'X', 'F', 1};

  auto readJournal = [this]() -> MigrationJournalState {
    bool journal_exists = false;
    if (!contactPathPresence(_fsExtra, SECONDARY_MIGRATION_JOURNAL,
                             journal_exists)) {
      return MigrationJournalState::IoError;
    }
    if (!journal_exists) {
      return MigrationJournalState::None;
    }
    File journal = openRead(_fsExtra, SECONDARY_MIGRATION_JOURNAL);
    uint8_t payload[sizeof(journal_magic) + 1];
    const bool valid_size = journal && journal.size() == sizeof(payload);
    const int count = valid_size ? journal.read(payload, sizeof(payload)) : 0;
    if (journal) journal.close();
    if (count != (int)sizeof(payload)
        || memcmp(payload, journal_magic, sizeof(journal_magic)) != 0) {
      return MigrationJournalState::Invalid;
    }
    if (payload[sizeof(journal_magic)] == SECONDARY_MIGRATION_PENDING) {
      return MigrationJournalState::Pending;
    }
    if (payload[sizeof(journal_magic)] == SECONDARY_MIGRATION_COMMITTED) {
      return MigrationJournalState::Committed;
    }
    return MigrationJournalState::Invalid;
  };

  auto writeJournal = [this](uint8_t state) -> bool {
    uint8_t payload[sizeof(journal_magic) + 1];
    memcpy(payload, journal_magic, sizeof(journal_magic));
    payload[sizeof(journal_magic)] = state;
    mesh::AtomicFileWriter journal(_fsExtra, SECONDARY_MIGRATION_JOURNAL);
    return journal
        && journal.write(payload, sizeof(payload)) == sizeof(payload)
        && journal.commit();
  };

  auto hasSecondarySources = [this](bool& present) -> bool {
    present = false;
    for (size_t i = 0; i < sizeof(to_secondary) / sizeof(to_secondary[0]); i++) {
      bool path_exists = false;
      if (!contactPathPresence(_fs, to_secondary[i], path_exists)) return false;
      if (path_exists) {
        present = true;
        return true;
      }
    }
    for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) {
      char path[24];
      makeContactPagePath(page, path);
      bool path_exists = false;
      if (!contactPathPresence(_fs, path, path_exists)) return false;
      if (path_exists) {
        present = true;
        return true;
      }
    }
    for (uint8_t bucket = 0; bucket < 10; bucket++) {
      char path[20];
      snprintf(path, sizeof(path), "/adv4_%02u", (unsigned)bucket);
      bool path_exists = false;
      if (!contactPathPresence(_fs, path, path_exists)) return false;
      if (path_exists) {
        present = true;
        return true;
      }
    }
    return true;
  };

  auto copySecondarySources = [this, &copy]() -> bool {
    bool success = true;
    for (size_t i = 0; i < sizeof(to_secondary) / sizeof(to_secondary[0]); i++) {
      if (!copy(_fs, _fsExtra, to_secondary[i], true)) success = false;
    }
    for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) {
      char path[24];
      makeContactPagePath(page, path);
      if (!copy(_fs, _fsExtra, path, true)) success = false;
    }
    for (uint8_t bucket = 0; bucket < 10; bucket++) {
      char path[20];
      snprintf(path, sizeof(path), "/adv4_%02u", (unsigned)bucket);
      if (!copy(_fs, _fsExtra, path, true)) success = false;
    }
    return success;
  };

  auto retireSecondarySources = [this]() -> bool {
    bool fixed_present[sizeof(to_secondary) / sizeof(to_secondary[0])] = {};
    uint32_t page_presence = 0;
    uint16_t bucket_presence = 0;

    // Capture the complete removal set before deleting anything. A metadata
    // I/O error must leave every committed primary source in place for the
    // next reboot, rather than being mistaken for an absent file midway
    // through cleanup.
    for (size_t i = 0; i < sizeof(to_secondary) / sizeof(to_secondary[0]); i++) {
      if (!contactPathPresence(_fs, to_secondary[i], fixed_present[i])) {
        return false;
      }
    }
    for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) {
      char path[24];
      makeContactPagePath(page, path);
      bool path_exists = false;
      if (!contactPathPresence(_fs, path, path_exists)) return false;
      if (path_exists) page_presence |= 1UL << page;
    }
    for (uint8_t bucket = 0; bucket < 10; bucket++) {
      char path[20];
      snprintf(path, sizeof(path), "/adv4_%02u", (unsigned)bucket);
      bool path_exists = false;
      if (!contactPathPresence(_fs, path, path_exists)) return false;
      if (path_exists) bucket_presence |= (uint16_t)1U << bucket;
    }

    bool success = true;
    auto retire = [this, &success](const char* path, bool path_exists) {
      if (!path_exists) return;
      // The committed journal is the authority switch: every destination was
      // already verified before it was written. Do not compare again here,
      // because the secondary may legitimately advance while cleanup retries.
      if (!_fs->remove(path)) success = false;
    };
    for (size_t i = 0; i < sizeof(to_secondary) / sizeof(to_secondary[0]); i++) {
      retire(to_secondary[i], fixed_present[i]);
    }
    for (uint8_t page = 0; page < mesh::storage::CONTACT_PAGE_COUNT; page++) {
      char path[24];
      makeContactPagePath(page, path);
      retire(path, (page_presence & (1UL << page)) != 0);
    }
    for (uint8_t bucket = 0; bucket < 10; bucket++) {
      char path[20];
      snprintf(path, sizeof(path), "/adv4_%02u", (unsigned)bucket);
      retire(path, (bucket_presence & ((uint16_t)1U << bucket)) != 0);
    }
    return success;
  };

  MigrationJournalState journal_state = readJournal();
  // Identity and preferences are always read from the primary filesystem.
  // Recover legacy/test-layout copies before any journal result can select or
  // quarantine the contact/channel store. The journal has already been read,
  // however, so a failure here can never silently select the wrong authority.
  if (!migratePrimarySources()) {
    _contact_load_incomplete = true;
    MESH_DEBUG_PRINTLN(
        "DataStore: primary identity/preferences migration failed; storage quarantined until reboot");
    return false;
  }

  if (journal_state == MigrationJournalState::IoError) {
    // The journal is the authority switch. If its presence cannot be
    // classified, neither filesystem can safely be selected for contact or
    // channel mutations this boot: Pending means primary is authoritative,
    // while Committed means secondary may already be the only complete copy.
    _contact_load_incomplete = true;
    MESH_DEBUG_PRINTLN(
        "DataStore: ExtraFS migration journal stat failed; storage quarantined until reboot");
    return false;
  }
  if (journal_state == MigrationJournalState::Invalid) {
    // Atomic journal contents should always be one of the two valid states.
    // Corruption cannot reveal whether source retirement had begun, so it has
    // the same authority-unknown policy as a journal I/O error.
    _contact_load_incomplete = true;
    MESH_DEBUG_PRINTLN(
        "DataStore: invalid ExtraFS migration journal; storage quarantined until explicit erase");
    return false;
  }

  bool secondary_sources_present = false;
  if (journal_state == MigrationJournalState::None
      && !hasSecondarySources(secondary_sources_present)) {
    // A clean post-migration boot also has no journal and no primary sources,
    // so a source-stat error makes authority unknowable. Preserve both stores
    // and block contact/channel use until a reboot can classify every path.
    _contact_load_incomplete = true;
    MESH_DEBUG_PRINTLN(
        "DataStore: could not inspect primary migration sources; storage quarantined until reboot");
    return false;
  }
  if (journal_state != MigrationJournalState::Committed
      && (journal_state != MigrationJournalState::None
          || secondary_sources_present)) {
    if (journal_state == MigrationJournalState::None
        && !writeJournal(SECONDARY_MIGRATION_PENDING)) {
      MESH_DEBUG_PRINTLN("DataStore: could not start ExtraFS migration transaction");
      _fsExtra = nullptr;
      return false;
    }
    if (!copySecondarySources()
        || !writeJournal(SECONDARY_MIGRATION_COMMITTED)) {
      // Pending means no source has been retired, so primary remains a complete
      // fallback even if the destination contains harmless partial copies.
      MESH_DEBUG_PRINTLN("DataStore: ExtraFS migration copy incomplete; using primary storage");
      _fsExtra = nullptr;
      return false;
    }
    journal_state = MigrationJournalState::Committed;
  }

  if (journal_state == MigrationJournalState::Committed) {
    if (!retireSecondarySources()) {
      // The committed secondary contains the full verified snapshot. Keep it
      // active and keep the journal so cleanup can resume without copying a
      // stale primary source over newer secondary data.
      MESH_DEBUG_PRINTLN("DataStore: ExtraFS migration committed; source cleanup remains pending");
      return false;
    }
    // readJournal() found it, or writeJournal() committed it in this call, so
    // a second presence probe cannot add safety. Remove the known journal
    // directly after all committed sources have been retired.
    if (!_fsExtra->remove(SECONDARY_MIGRATION_JOURNAL)) {
      MESH_DEBUG_PRINTLN("DataStore: ExtraFS migration journal cleanup failed");
      return false;
    }
  }
#else
  bool success = true;
  for (size_t i = 0; i < sizeof(to_secondary) / sizeof(to_secondary[0]); i++) {
    if (!move(_fs, _fsExtra, to_secondary[i])) success = false;
  }
  if (!success) return false;
  return migratePrimarySources();
#endif

  return true;
}

#if defined(NRF52_PLATFORM)
// Keep every bucket below one 4 KiB LittleFS block. Five 20-record buckets use
// about the same flash as the old 100-record file; smaller buckets would each
// consume a full block and leave no room for contact-page transactions.
static const uint8_t BLOB_BUCKET_COUNT = MAX_BLOBRECS > 20 ? 5 : 1;
static const uint8_t BLOB_BUCKET_SLOTS =
    (MAX_BLOBRECS + BLOB_BUCKET_COUNT - 1) / BLOB_BUCKET_COUNT;
static const uint8_t BLOB_BUCKET_HEADER_SIZE = 16;
static const uint8_t BLOB_BUCKET_MAGIC[4] = {'M', 'C', 'B', '4'};
static_assert(BLOB_BUCKET_HEADER_SIZE + sizeof(BlobRec) * BLOB_BUCKET_SLOTS < 4096,
              "advert bucket must fit in one LittleFS block");

static void normalizeBlobKey(const uint8_t key[], int key_len, uint8_t normalized[7]) {
  memset(normalized, 0, 7);
  if (key == NULL || key_len <= 0) return;
  if (key_len > 7) key_len = 7;
  memcpy(normalized, key, key_len);
}

static uint8_t blobBucketFor(const uint8_t key[7]) {
  uint32_t hash = 2166136261UL;
  for (uint8_t i = 0; i < 7; i++) {
    hash ^= key[i];
    hash *= 16777619UL;
  }
  return hash % BLOB_BUCKET_COUNT;
}

static void makeBlobBucketPath(uint8_t bucket, char path[20]) {
  snprintf(path, 20, "/adv4_%02u", (unsigned)bucket);
}

static bool loadBlobBucket(FILESYSTEM* fs, uint8_t bucket,
                           BlobRec records[BLOB_BUCKET_SLOTS]) {
  memset(records, 0, sizeof(BlobRec) * BLOB_BUCKET_SLOTS);
  char path[20];
  makeBlobBucketPath(bucket, path);
  if (!fs->exists(path)) return true;

  File file = fs->open(path, FILE_O_READ);
  if (!file) return false;
  const size_t payload_size = sizeof(BlobRec) * BLOB_BUCKET_SLOTS;
  if (file.size() != BLOB_BUCKET_HEADER_SIZE + payload_size) {
    file.close();
    return false;
  }

  uint8_t header[BLOB_BUCKET_HEADER_SIZE];
  bool valid = file.read(header, sizeof(header)) == sizeof(header)
      && memcmp(header, BLOB_BUCKET_MAGIC, sizeof(BLOB_BUCKET_MAGIC)) == 0
      && header[4] == 1 && header[5] == bucket
      && header[6] == BLOB_BUCKET_SLOTS
      && mesh::storage::readLE16(&header[8]) == sizeof(BlobRec)
      && file.read((uint8_t*)records, payload_size) == (int)payload_size;
  file.close();
  if (!valid) return false;

  const uint32_t expected_crc = mesh::storage::readLE32(&header[12]);
  const uint32_t actual_crc = mesh::storage::updateCRC32(
      0xFFFFFFFFUL, (const uint8_t*)records, payload_size);
  if (actual_crc != expected_crc) return false;
  for (uint8_t i = 0; i < BLOB_BUCKET_SLOTS; i++) {
    if (records[i].len > MAX_ADVERT_PKT_LEN) return false;
  }
  return true;
}

static bool saveBlobBucket(FILESYSTEM* fs, uint8_t bucket,
                           const BlobRec records[BLOB_BUCKET_SLOTS]) {
  const size_t payload_size = sizeof(BlobRec) * BLOB_BUCKET_SLOTS;
  uint8_t header[BLOB_BUCKET_HEADER_SIZE];
  memset(header, 0, sizeof(header));
  memcpy(header, BLOB_BUCKET_MAGIC, sizeof(BLOB_BUCKET_MAGIC));
  header[4] = 1;
  header[5] = bucket;
  header[6] = BLOB_BUCKET_SLOTS;
  mesh::storage::writeLE16(&header[8], sizeof(BlobRec));
  mesh::storage::writeLE32(&header[12], mesh::storage::updateCRC32(
      0xFFFFFFFFUL, (const uint8_t*)records, payload_size));

  char path[20];
  makeBlobBucketPath(bucket, path);
  mesh::AtomicFileWriter writer(fs, path);
  const bool wrote = writer
      && writer.write(header, sizeof(header)) == sizeof(header)
      && writer.write((const uint8_t*)records, payload_size) == payload_size;
  return writer.commit(wrote);
}

static bool findBlobInBucket(FILESYSTEM* fs, const uint8_t key[7],
                             uint8_t dest_buf[], uint8_t& length) {
  // Twenty records are roughly 3.6 KiB, too large for the nRF Arduino loop's
  // 4 KiB stack once callers are included. Use a short-lived heap buffer.
  BlobRec* records = (BlobRec*)malloc(sizeof(BlobRec) * BLOB_BUCKET_SLOTS);
  if (records == nullptr) return false;
  const uint8_t bucket = blobBucketFor(key);
  if (!loadBlobBucket(fs, bucket, records)) {
    free(records);
    return false;
  }
  for (uint8_t i = 0; i < BLOB_BUCKET_SLOTS; i++) {
    if (memcmp(records[i].key, key, sizeof(records[i].key)) == 0
        && (records[i].timestamp != 0 || records[i].len != 0)) {
      length = records[i].len; // zero is an intentional tombstone
      if (length > 0) memcpy(dest_buf, records[i].data, length);
      free(records);
      return true;
    }
  }
  free(records);
  return false;
}

// Once a v4 bucket contains a key, the old monolithic cache entry is no
// longer needed.  Clear just that disposable legacy record so an eventually
// evicted bucket/tombstone can never expose stale advert data again.  This is
// intentionally a bounded in-place write: atomically rewriting the complete
// legacy cache is the multi-second operation the bucket format avoids.
static bool clearLegacyBlobRecord(FILESYSTEM* fs, const uint8_t key[7]) {
  if (!fs->exists("/adv_blobs")) return true;

  File file = fs->open("/adv_blobs", FILE_O_WRITE);
  if (!file) return false;

  BlobRec record;
  uint32_t position = 0;
  bool success = true;
  bool found = false;
  file.seek(0);
  while (file.read((uint8_t*)&record, sizeof(record)) == sizeof(record)) {
    if (record.len <= MAX_ADVERT_PKT_LEN
        && memcmp(record.key, key, sizeof(record.key)) == 0
        && (record.timestamp != 0 || record.len != 0)) {
      found = true;
      memset(&record, 0, sizeof(record));
      success = file.seek(position)
          && file.write((uint8_t*)&record, sizeof(record)) == sizeof(record);
      if (success) file.flush();
      break;
    }
    position += sizeof(record);
  }
  file.close();
  return !found || success;
}
#endif

uint8_t DataStore::getBlobByKey(const uint8_t key[], int key_len, uint8_t dest_buf[]) {
#if defined(NRF52_PLATFORM)
  uint8_t normalized[7], length = 0;
  normalizeBlobKey(key, key_len, normalized);
  if (findBlobInBucket(_getContactsChannelsFS(), normalized, dest_buf, length)) {
    return length;
  }
  // checkAdvBlobFile() retires the old monolithic nRF cache at boot. If that
  // best-effort removal failed, do not fall back to it: a durable bucket
  // tombstone must remain authoritative even when its bucket is temporarily
  // unreadable, and adverts are a reconstructable cache.
  return 0;
#endif

#if !defined(NRF52_PLATFORM)
  File file = openRead(_getContactsChannelsFS(), "/adv_blobs");
  uint8_t len = 0;  // 0 = not found
  if (file) {
    BlobRec tmp;
    while (file.read((uint8_t *) &tmp, sizeof(tmp)) == sizeof(tmp)) {
      uint8_t normalized[7];
      normalizeBlobKey(key, key_len, normalized);
      if (tmp.len <= MAX_ADVERT_PKT_LEN
          && memcmp(normalized, tmp.key, sizeof(tmp.key)) == 0) {  // only match by 7 byte prefix
        len = tmp.len;
        memcpy(dest_buf, tmp.data, len);
        break;
      }
    }
    file.close();
  }
  return len;
#endif
}

bool DataStore::putBlobByKey(const uint8_t key[], int key_len, const uint8_t src_buf[], uint8_t len) {
  if (len < PUB_KEY_SIZE+4+SIGNATURE_SIZE || len > MAX_ADVERT_PKT_LEN) return false;
#if defined(NRF52_PLATFORM)
  uint8_t normalized[7];
  normalizeBlobKey(key, key_len, normalized);
  const uint8_t bucket = blobBucketFor(normalized);
  BlobRec* records = (BlobRec*)malloc(sizeof(BlobRec) * BLOB_BUCKET_SLOTS);
  if (records == nullptr) return false;
  if (!loadBlobBucket(_getContactsChannelsFS(), bucket, records)) {
    MESH_DEBUG_PRINTLN("DataStore: advert bucket %u corrupt; replacing on next write", bucket);
    memset(records, 0, sizeof(BlobRec) * BLOB_BUCKET_SLOTS);
  }

  uint8_t selected = 0;
  uint32_t oldest = 0xFFFFFFFFUL;
  for (uint8_t i = 0; i < BLOB_BUCKET_SLOTS; i++) {
    if (memcmp(records[i].key, normalized, sizeof(records[i].key)) == 0
        && (records[i].timestamp != 0 || records[i].len != 0)) {
      selected = i;
      break;
    }
    if (records[i].timestamp < oldest) {
      oldest = records[i].timestamp;
      selected = i;
    }
  }
  BlobRec& record = records[selected];
  memset(&record, 0, sizeof(record));
  memcpy(record.key, normalized, sizeof(record.key));
  memcpy(record.data, src_buf, len);
  record.len = len;
  record.timestamp = _clock->getCurrentTime();
  if (record.timestamp == 0) record.timestamp = 1;
  const bool saved = saveBlobBucket(_getContactsChannelsFS(), bucket, records);
  free(records);
  if (!saved) return false;
  if (!clearLegacyBlobRecord(_getContactsChannelsFS(), normalized)) {
    // The nRF reader never falls back to this reconstructable legacy cache.
    MESH_DEBUG_PRINTLN(
        "DataStore: advert saved; legacy cache cleanup deferred");
  }
  return true;
#else
  checkAdvBlobFile();
  File file = _getContactsChannelsFS()->open("/adv_blobs", FILE_O_WRITE);
  if (file) {
    uint32_t pos = 0, found_pos = 0;
    uint32_t min_timestamp = 0xFFFFFFFF;

    // search for matching key OR evict by oldest timestamp
    BlobRec tmp;
    file.seek(0);
    while (file.read((uint8_t *) &tmp, sizeof(tmp)) == sizeof(tmp)) {
      if (memcmp(key, tmp.key, sizeof(tmp.key)) == 0) {  // only match by 7 byte prefix
        found_pos = pos;
        break;
      }
      if (tmp.timestamp < min_timestamp) {
        min_timestamp = tmp.timestamp;
        found_pos = pos;
      }

      pos += sizeof(tmp);
    }

    memcpy(tmp.key, key, sizeof(tmp.key));  // just record 7 byte prefix of key
    memcpy(tmp.data, src_buf, len);
    tmp.len = len;
    tmp.timestamp = _clock->getCurrentTime();

    file.seek(found_pos);
    file.write((uint8_t *) &tmp, sizeof(tmp));

    file.close();
    return true;
  }
  return false; // error
#endif
}
bool DataStore::deleteBlobByKey(const uint8_t key[], int key_len) {
#if defined(NRF52_PLATFORM)
  uint8_t normalized[7];
  normalizeBlobKey(key, key_len, normalized);
  const uint8_t bucket = blobBucketFor(normalized);
  BlobRec* records = (BlobRec*)malloc(sizeof(BlobRec) * BLOB_BUCKET_SLOTS);
  if (records == nullptr) return false;
  const bool bucket_valid = loadBlobBucket(
      _getContactsChannelsFS(), bucket, records);
  if (!bucket_valid) {
    memset(records, 0, sizeof(BlobRec) * BLOB_BUCKET_SLOTS);
  }

  uint8_t selected = 0;
  uint32_t oldest = 0xFFFFFFFFUL;
  bool found = false;
  for (uint8_t i = 0; i < BLOB_BUCKET_SLOTS; i++) {
    if (memcmp(records[i].key, normalized, sizeof(records[i].key)) == 0
        && (records[i].timestamp != 0 || records[i].len != 0)) {
      selected = i;
      found = true;
      break;
    }
    if (records[i].timestamp < oldest) {
      oldest = records[i].timestamp;
      selected = i;
    }
  }
  const bool legacy_exists =
      _getContactsChannelsFS()->exists("/adv_blobs");
  if (!legacy_exists && bucket_valid
      && (!found || records[selected].len == 0)) {
    // Only a successfully verified bucket can prove the key is absent or
    // already tombstoned. A failed bucket read may be transient and must not
    // turn a stale cached advert into an acknowledged deletion.
    free(records);
    return true;
  }
  BlobRec& tombstone = records[selected];
  memset(&tombstone, 0, sizeof(tombstone));
  memcpy(tombstone.key, normalized, sizeof(tombstone.key));
  tombstone.timestamp = _clock->getCurrentTime();
  if (tombstone.timestamp == 0) tombstone.timestamp = 1;
  const bool saved = saveBlobBucket(_getContactsChannelsFS(), bucket, records);
  free(records);
  if (!saved) return false;
  if (!clearLegacyBlobRecord(_getContactsChannelsFS(), normalized)) {
    // The durable zero-length bucket record already masks the legacy entry,
    // and checkAdvBlobFile() retires the reconstructable legacy cache on the
    // next boot. The cache mutation cannot be rolled back after this commit,
    // so report deletion success and keep the live contact table consistent
    // with the authoritative tombstone.
    MESH_DEBUG_PRINTLN(
        "DataStore: deleted advert tombstoned; legacy cleanup deferred");
  }
  return true;
#else
  return true; // this is just a stub on NRF52/STM32 platforms
#endif
}
#else
inline void makeBlobPath(const uint8_t key[], int key_len, char* path, size_t path_size) {
  char fname[18];
  if (key_len > 8) key_len = 8; // just use first 8 bytes (prefix)
  mesh::Utils::toHex(fname, key, key_len);
  sprintf(path, "/bl/%s", fname);
}

uint8_t DataStore::getBlobByKey(const uint8_t key[], int key_len, uint8_t dest_buf[]) {
  char path[64];
#if defined(ESP32_PLATFORM)
  if (key_len <= 0) return 0;
#endif
  makeBlobPath(key, key_len, path, sizeof(path));

#if defined(ESP32_PLATFORM)
  const int prefix_len = key_len > 8 ? 8 : key_len;
  for (const auto& slot : _advert_write.slots) {
    if (slot.len && slot.key_len == prefix_len
        && memcmp(slot.key, key, prefix_len) == 0) {
      memcpy(dest_buf, slot.bytes, slot.len);
      return slot.len;
    }
  }
  // SPIFFS cannot rename over an existing file. A reset between the staged
  // target->backup and temp->target renames leaves the prior valid cache here.
  _advert_write.synchronous_io = true;
  bool present = false;
  if (!companionPathPresence(_fs, path, present)) return 0;
  if (!present) {
    const size_t used = strlen(path);
    strcpy(path + used, ".bak");
    if (!companionPathPresence(_fs, path, present)) return 0;
  }
#else
  const bool present = _fs->exists(path);
#endif
  if (present) {
    File f = openRead(_fs, path);
    if (f) {
      int len = f.read(dest_buf, 255); // currently MAX 255 byte blob len supported!!
      f.close();
      return len;
    }
  }
  return 0; // not found
}

bool DataStore::putBlobByKey(const uint8_t key[], int key_len, const uint8_t src_buf[], uint8_t len) {
  char path[64];
  makeBlobPath(key, key_len, path, sizeof(path));
#if defined(ESP32_PLATFORM)
  _advert_write.synchronous_io = true;
  invalidateAdvertWrite(key, key_len);
#endif

  File f = openWrite(_fs, path);
  if (f) {
    int n = f.write(src_buf, len);
    f.close();
    if (n == len) {
#if defined(ESP32_PLATFORM)
      retireAdvertWrite(key, key_len);
#endif
      return true;
    }

    _fs->remove(path); // blob was only partially written!
  }
  return false; // error
}

bool DataStore::deleteBlobByKey(const uint8_t key[], int key_len) {
  char path[64];
  makeBlobPath(key, key_len, path, sizeof(path));

#if defined(ESP32_PLATFORM)
  _advert_write.synchronous_io = true;
  invalidateAdvertWrite(key, key_len);
  // VFS remove() logs an error for normal absence, contaminating the binary
  // USB stream. Distinguish absence from metadata failure before calling it;
  // a real deletion failure must preserve the live contact for rollback.
  char backup[68];
  snprintf(backup, sizeof(backup), "%s.bak", path);
  bool present = false, backup_present = false;
  // Probe both before mutating either; removing the backup first preserves
  // the authoritative target if a later deletion fails and the contact rolls
  // back. A successful deletion invalidates publication before staged cleanup.
  if (!companionPathPresence(_fs, path, present)
      || !companionPathPresence(_fs, backup, backup_present)) return false;
  if (backup_present && !_fs->remove(backup)) return false;
  if (present && !_fs->remove(path)) return false;
  retireAdvertWrite(key, key_len);
  return true;
#else
  _fs->remove(path);
  
  return true; // return true even if file did not exist
#endif
}
#endif

#if defined(ESP32_PLATFORM)
bool DataStore::queueAdvertByKey(const uint8_t key[], int key_len,
                                const uint8_t src_buf[], uint8_t len) {
  if (key_len <= 0 || len == 0) return false;
  const uint8_t prefix_len = key_len > 8 ? 8 : key_len;
  AdvertWriteState::Slot* chosen = nullptr;
  for (auto& slot : _advert_write.slots) {
    if (slot.len && slot.key_len == prefix_len
        && memcmp(slot.key, key, prefix_len) == 0) {
      chosen = &slot;
      break;
    }
    if (!slot.len && chosen == nullptr) chosen = &slot;
  }
  // Never force a filesystem flush or allocate another packet on overflow.
  if (chosen == nullptr) return false;
  memcpy(chosen->key, key, prefix_len);
  memcpy(chosen->bytes, src_buf, len);
  chosen->key_len = prefix_len;
  chosen->len = len;
  chosen->revision = ++_advert_write.revision;
  return true;
}

void DataStore::retireAdvertWrite(const uint8_t key[], int key_len) {
  if (key_len <= 0) return;
  const int prefix_len = key_len > 8 ? 8 : key_len;
  for (auto& slot : _advert_write.slots) {
    if (slot.len && slot.key_len == prefix_len
        && memcmp(slot.key, key, prefix_len) == 0) {
      slot.len = 0;
      slot.revision = ++_advert_write.revision;
    }
  }
}

void DataStore::invalidateAdvertWrite(const uint8_t key[], int key_len) {
  if (key_len <= 0) return;
  const int prefix_len = key_len > 8 ? 8 : key_len;
  auto& job = _advert_write;
  for (auto& slot : job.slots) {
    if (slot.len && slot.key_len == prefix_len
        && memcmp(slot.key, key, prefix_len) == 0) {
      slot.revision = ++job.revision;
    }
  }
  if (job.stage != AdvertWriteState::Stage::Idle
      && job.active_key_len == prefix_len
      && memcmp(job.active_key, key, prefix_len) == 0) {
    // A synchronous API may have changed target/backup even when it returned
    // failure. Re-probe the independent active key before recovery/publication.
    job.cancelling = true;
    job.stage = AdvertWriteState::Stage::CancelClose;
    job.retry_at = 0;
  }
}

bool DataStore::consumeSynchronousAdvertIO() {
  const bool result = _advert_write.synchronous_io;
  _advert_write.synchronous_io = false;
  return result;
}

bool DataStore::hasPendingAdvertWrites() const {
  if (_advert_write.stage != AdvertWriteState::Stage::Idle) return true;
  for (const auto& slot : _advert_write.slots) if (slot.len) return true;
  return false;
}

bool DataStore::isAdvertWriteDue(uint32_t now) const {
  return hasPendingAdvertWrites() && (_advert_write.retry_at == 0
      || static_cast<int32_t>(now - _advert_write.retry_at) >= 0);
}

bool DataStore::serviceAdvertWrites(uint32_t now) {
  auto& job = _advert_write;
  using Stage = AdvertWriteState::Stage;
  static const char* const temp = "/advert.tmp";
  if (!isAdvertWriteDue(now)) return true;
  job.retry_at = 0;
  if (job.stage == Stage::Idle) {
    for (uint8_t n = 0; n < AdvertWriteState::CAPACITY; ++n) {
      const uint8_t i = (job.next + n) % AdvertWriteState::CAPACITY;
      if (!job.slots[i].len) continue;
      job.active = i;
      job.next = (i + 1) % AdvertWriteState::CAPACITY;
      job.active_revision = job.slots[i].revision;
      job.active_key_len = job.slots[i].key_len;
      memcpy(job.active_key, job.slots[i].key, job.active_key_len);
      makeBlobPath(job.slots[i].key, job.slots[i].key_len,
                   job.target, sizeof(job.target));
      snprintf(job.backup, sizeof(job.backup), "%s.bak", job.target);
      job.offset = 0;
      job.cancelling = false;
      job.stage = Stage::TargetProbe;
      break;
    }
    if (job.stage == Stage::Idle) return true;
  }
  auto& slot = job.slots[job.active];
  if (!job.cancelling && (!slot.len || slot.revision != job.active_revision)) {
    job.cancelling = true;
    job.stage = Stage::CancelClose;
  }
  const auto retry = [&]() {
    const uint32_t delay = mesh::recordLazyPersistenceSaveFailure(
        job.failures, 1000, mesh::LAZY_PERSISTENCE_MAX_RETRY_DELAY_MILLIS);
    job.retry_at = now + delay;
    if (job.retry_at == 0) job.retry_at = 1;
    return false;
  };
  const auto fail = [&]() {
    job.cancelling = true;
    job.stage = Stage::CancelClose;
    return retry();
  };
  const auto prepared = [&]() {
    if (job.cancelling) {
      job.stage = Stage::Idle;
      job.active = AdvertWriteState::CAPACITY;
    } else job.stage = Stage::OpenWrite;
  };
  // Exactly one filesystem API operation in each arm. In particular, open,
  // fclose (which may flush stdio), crypto in BaseChatMesh, and contact-file
  // work must not accumulate in the same serviced loop pass. SPIFFS may still
  // spend an unbounded amount of wall time inside one individual operation.
  switch (job.stage) {
    case Stage::TargetProbe:
      if (!companionPathPresence(_fs, job.target, job.target_present)) return fail();
      job.stage = Stage::BackupProbe;
      break;
    case Stage::BackupProbe:
      if (!companionPathPresence(_fs, job.backup, job.backup_present)) return fail();
      job.stage = job.backup_present ? (job.target_present ? Stage::RemoveBackup
                                                         : Stage::RestoreBackup)
                                     : Stage::TempProbe;
      break;
    case Stage::RestoreBackup:
      if (!_fs->rename(job.backup, job.target)) return fail();
      job.stage = Stage::TempProbe;
      break;
    case Stage::RemoveBackup:
      if (!_fs->remove(job.backup)) return fail();
      job.stage = Stage::TempProbe;
      break;
    case Stage::TempProbe:
      if (!companionPathPresence(_fs, temp, job.temp_present)) return fail();
      if (job.temp_present) job.stage = Stage::RemoveTemp;
      else prepared();
      break;
    case Stage::RemoveTemp:
      if (!_fs->remove(temp)) return fail();
      prepared();
      break;
    case Stage::OpenWrite:
      job.file = openWrite(_fs, temp);
      if (!job.file) return fail();
      job.stage = Stage::ConfigureWrite;
      break;
    case Stage::ConfigureWrite:
      if (!job.file.setBufferSize(sizeof(job.scratch))) return fail();
      job.stage = Stage::Write;
      break;
    case Stage::Write: {
      const uint8_t count = static_cast<size_t>(slot.len - job.offset) > sizeof(job.scratch)
          ? sizeof(job.scratch) : slot.len - job.offset;
      if (job.file.write(slot.bytes + job.offset, count) != count) return fail();
      job.offset += count;
      if (job.offset == slot.len) job.stage = Stage::CloseWrite;
      break;
    }
    case Stage::CloseWrite:
      job.file.close();
      job.stage = Stage::OpenVerify;
      break;
    case Stage::OpenVerify:
      job.file = openRead(_fs, temp);
      if (!job.file) return fail();
      job.stage = Stage::ConfigureVerify;
      break;
    case Stage::ConfigureVerify:
      if (!job.file.setBufferSize(sizeof(job.scratch))) return fail();
      job.stage = Stage::VerifySize;
      break;
    case Stage::VerifySize:
      if (job.file.size() != slot.len) return fail();
      job.offset = 0;
      job.stage = Stage::Verify;
      break;
    case Stage::Verify: {
      const uint8_t count = static_cast<size_t>(slot.len - job.offset) > sizeof(job.scratch)
          ? sizeof(job.scratch) : slot.len - job.offset;
      if (job.file.read(job.scratch, count) != count
          || memcmp(job.scratch, slot.bytes + job.offset, count) != 0) return fail();
      job.offset += count;
      if (job.offset == slot.len) job.stage = Stage::CloseVerify;
      break;
    }
    case Stage::CloseVerify:
      job.file.close();
      job.stage = Stage::PublishTargetProbe;
      break;
    case Stage::PublishTargetProbe:
      if (!companionPathPresence(_fs, job.target, job.target_present)) return fail();
      job.stage = Stage::PublishBackupProbe;
      break;
    case Stage::PublishBackupProbe:
      if (!companionPathPresence(_fs, job.backup, job.backup_present)) return fail();
      job.stage = job.target_present ? (job.backup_present ? Stage::PublishRemoveBackup
                                                         : Stage::PublishBackup)
                                     : Stage::Publish;
      break;
    case Stage::PublishRemoveBackup:
      if (!_fs->remove(job.backup)) return fail();
      job.stage = Stage::PublishBackup;
      break;
    case Stage::PublishBackup:
      if (!_fs->rename(job.target, job.backup)) return fail();
      job.backup_present = true;
      job.stage = Stage::Publish;
      break;
    case Stage::Publish:
      // SPIFFS rejects rename over an existing destination. The two renames
      // have an explicit gap; RAM serves the new packet and disk readers after
      // a reset use the old backup. No unverified temp is ever a visible blob.
      if (!_fs->rename(temp, job.target)) return fail();
      job.stage = Stage::PublishCleanup;
      break;
    case Stage::PublishCleanup:
      if (job.backup_present && !_fs->remove(job.backup)) return retry();
      slot.len = 0;
      job.failures = 0;
      job.stage = Stage::Idle;
      job.active = AdvertWriteState::CAPACITY;
      break;
    case Stage::CancelClose:
      job.file.close();
      job.stage = Stage::TargetProbe;
      break;
    case Stage::Idle:
      break;
  }
  return true;
}

bool DataStore::flushAdvertWrites(uint32_t now) {
  // Explicit reboot/shutdown/import paths require durability and may block.
  // On failure keep accepted packets in RAM, but never leave a file open.
  _advert_write.synchronous_io = hasPendingAdvertWrites()
      || _advert_write.synchronous_io;
  for (unsigned pass = 0; hasPendingAdvertWrites() && pass < 512; ++pass) {
    _advert_write.retry_at = 0;
    if (!serviceAdvertWrites(now)) {
      const uint32_t retry_at = _advert_write.retry_at;
      _advert_write.pause();
      _advert_write.retry_at = retry_at;
      return false;
    }
  }
  if (hasPendingAdvertWrites()) {
    _advert_write.pause();
    return false;
  }
  return true;
}
#endif

#if MESH_CONTACT_CACHE
namespace {
bool cachedContactFilter(const ContactInfo& c) { return c.type != ADV_TYPE_NONE; }
#if MESH_CONTACT_SECRET_FLASH_CACHE
#if defined(NRF52_PLATFORM)
// ExtraFS allocates 4 KiB data blocks. Pack derived keys into almost a full
// block; tiny pages would consume the available flash after only a few peers.
constexpr size_t SECRET_PAGE_SLOTS = 56;
#else
constexpr size_t SECRET_PAGE_SLOTS = 8;
#endif
constexpr size_t SECRET_HEADER_SIZE = 36;
constexpr size_t SECRET_RECORD_SIZE = 68;
void secretPagePath(uint16_t slot, char path[24]) {
  snprintf(path, 24, "/csecret%03x", slot / SECRET_PAGE_SLOTS);
}
void secretHeader(uint8_t header[SECRET_HEADER_SIZE], const uint8_t identity[32]) {
  memcpy(header, "MCS1", 4);
  memcpy(header + 4, identity, 32);
}
bool validSecretRecord(const uint8_t record[SECRET_RECORD_SIZE]) {
  return mesh::storage::readLE32(record + 64) ==
      mesh::storage::updateCRC32(0xffffffff, record, 64);
}
#endif
}

bool DataStore::readStoredPath(uint16_t source, uint8_t path[64]) {
  if (source == mesh::ContactPathStorage::NONE || hasIncompleteContactLoad()) return false;
  char filename[24] = "/contacts3";
  size_t offset = static_cast<size_t>(source) * mesh::storage::CONTACT_RECORD_SIZE;
#if defined(ESP32_PLATFORM)
  if (!(source & mesh::ContactPathStorage::PAGED)) {
    // SPIFFS open scans metadata. Doing it for every contact made bulk saves
    // and app synchronization take tens of seconds. Share one read handle;
    // loadContacts/saveContacts close it before recovery or replacement.
    if (!_contact_path_reader)
      _contact_path_reader = openRead(_getContactsChannelsFS(), filename);
    File& file = _contact_path_reader;
    const bool ok = file && file.size() >= offset + mesh::storage::CONTACT_RECORD_SIZE
        && file.seek(offset + 76) && file.read(path, 64) == 64;
    if (!ok) file.close();
    return ok;
  }
#endif
  if (source & mesh::ContactPathStorage::PAGED) {
#if defined(NRF52_PLATFORM)
    const uint16_t slot = source & ~mesh::ContactPathStorage::PAGED;
    if (slot >= mesh::storage::CONTACT_PAGE_COUNT * mesh::storage::CONTACTS_PER_PAGE) return false;
    makeContactPagePath(slot / mesh::storage::CONTACTS_PER_PAGE, filename);
    offset = mesh::storage::CONTACT_PAGE_HEADER_SIZE +
        (slot % mesh::storage::CONTACTS_PER_PAGE) * mesh::storage::CONTACT_RECORD_SIZE;
#else
    return false;
#endif
  }
  File file = openRead(_getContactsChannelsFS(), filename);
  const bool ok = file && file.size() >= offset + mesh::storage::CONTACT_RECORD_SIZE
      && file.seek(offset + 76) && file.read(path, 64) == 64;
  if (file) file.close();
  // The pool checks the CRC captured when this immutable path value was bound.
  return ok;
}

bool DataStore::flushCachedPaths() {
  if (!_cache_host || hasIncompleteContactLoad()
      || !flushContactWrites(_cache_host, cachedContactFilter)) return false;
  _cache_host->onContactCacheFlushed();
  return true;
}

#if MESH_CONTACT_SECRET_FLASH_CACHE
uint16_t DataStore::secretSlot(const uint8_t peer[32]) const {
  if (!_cache_host || hasIncompleteContactLoad()) return mesh::storage::CONTACT_SLOT_NONE;
  uint16_t persistent_index = 0;
  for (uint32_t i = 0;; ++i) {
    const auto* c = _cache_host->getContactForStore(i);
    if (!c) break;
    if (c->type == ADV_TYPE_NONE) continue;
    if (memcmp(c->id.pub_key, peer, 32) == 0) {
#if defined(NRF52_PLATFORM)
      return c->storage_slot;
#else
      return persistent_index;
#endif
    }
    ++persistent_index;
  }
  return mesh::storage::CONTACT_SLOT_NONE;
}

bool DataStore::readSavedSecret(const uint8_t peer[32], const uint8_t identity[32],
                                uint8_t secret[32]) {
  const uint16_t slot = secretSlot(peer);
  if (slot == mesh::storage::CONTACT_SLOT_NONE) return false;
  char path[24];
  secretPagePath(slot, path);
#if defined(ESP32_PLATFORM)
  if (!mesh::ContactFileTransaction::recover(_getContactsChannelsFS(), path)) return false;
#endif
  File file = openRead(_getContactsChannelsFS(), path);
  uint8_t expected[SECRET_HEADER_SIZE], header[SECRET_HEADER_SIZE];
  secretHeader(expected, identity);
  uint8_t record[SECRET_RECORD_SIZE];
  const bool ok = file && file.size() == SECRET_HEADER_SIZE + SECRET_PAGE_SLOTS * SECRET_RECORD_SIZE
      && file.read(header, sizeof(header)) == sizeof(header)
      && memcmp(header, expected, sizeof(header)) == 0
      && file.seek(SECRET_HEADER_SIZE + (slot % SECRET_PAGE_SLOTS) * SECRET_RECORD_SIZE)
      && file.read(record, sizeof(record)) == sizeof(record)
      && memcmp(record, peer, 32) == 0 && validSecretRecord(record);
  if (file) file.close();
  if (ok) memcpy(secret, record + 32, 32);
  return ok;
}

bool DataStore::saveSecret(const uint8_t peer[32], const uint8_t identity[32],
                           const uint8_t secret[32]) {
  const uint16_t slot = secretSlot(peer);
  if (slot == mesh::storage::CONTACT_SLOT_NONE ||
      (_secret_retry_at && static_cast<int32_t>(millis() - _secret_retry_at) < 0)) return false;

  // Derived keys are expendable. Keep room for contact replacement, preferences
  // and filesystem metadata instead of letting this cache prevent a real save.
  size_t free_bytes = 0;
  size_t reserve_bytes = 0;
#if defined(NRF52_PLATFORM)
  const auto* config = _getContactsChannelsFS()->_getFS()->cfg;
  const int used = _getLfsUsedBlockCount(_getContactsChannelsFS());
  if (used < 0 || static_cast<uint32_t>(used) > config->block_count) return false;
  free_bytes = (config->block_count - used) * config->block_size;
  reserve_bytes = 4 * config->block_size;
#else
  const size_t total = SPIFFS.totalBytes(), used = SPIFFS.usedBytes();
  if (used > total) return false;
  free_bytes = total - used;
  reserve_bytes = MAX_CONTACTS * mesh::storage::CONTACT_RECORD_SIZE + 16384;
#endif
  if (free_bytes < reserve_bytes + 2 * (SECRET_HEADER_SIZE + SECRET_PAGE_SLOTS * SECRET_RECORD_SIZE)) {
    _secret_retry_at = millis() + 30000;
    return false;
  }

  char path[24];
  secretPagePath(slot, path);
  uint8_t header[SECRET_HEADER_SIZE], previous_header[SECRET_HEADER_SIZE];
  secretHeader(header, identity);
#if defined(ESP32_PLATFORM)
  if (!mesh::ContactFileTransaction::recover(_getContactsChannelsFS(), path)) return false;
#endif
  File old = openRead(_getContactsChannelsFS(), path);
  bool reuse = old && old.size() == SECRET_HEADER_SIZE + SECRET_PAGE_SLOTS * SECRET_RECORD_SIZE
      && old.read(previous_header, sizeof(previous_header)) == sizeof(previous_header)
      && memcmp(previous_header, header, sizeof(header)) == 0;
#if defined(NRF52_PLATFORM)
  mesh::AtomicFileWriter writer(_getContactsChannelsFS(), path);
#else
  mesh::ContactFileTransaction writer(_getContactsChannelsFS(), path);
#endif
  bool ok = writer && writer.write(header, sizeof(header)) == sizeof(header);
  uint8_t record[SECRET_RECORD_SIZE];
  for (size_t i = 0; ok && i < SECRET_PAGE_SLOTS; ++i) {
    memset(record, 0, sizeof(record));
    if (reuse && (old.read(record, sizeof(record)) != sizeof(record) || !validSecretRecord(record)))
      memset(record, 0, sizeof(record));
    if (i == slot % SECRET_PAGE_SLOTS) {
      memcpy(record, peer, 32);
      memcpy(record + 32, secret, 32);
      mesh::storage::writeLE32(record + 64,
          mesh::storage::updateCRC32(0xffffffff, record, 64));
    }
    ok = writer.write(record, sizeof(record)) == sizeof(record);
  }
  if (old) old.close();
  ok = writer.commit(ok);
  if (!ok) _secret_retry_at = millis() + 30000;
  return ok;
}
#endif
#endif