diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 056287ad..a5436981 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1,5 +1,9 @@ #include "MyMesh.h" +#if defined(ESP32_PLATFORM) +#include +#endif #include +#include #include #if MESH_ENABLE_TELEMETRY_HISTORY #include @@ -3706,14 +3710,20 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc // OTA mesh-integration (receive/begin/loop) is centralized in mesh::Mesh - no per-example wiring. void MyMesh::begin(FILESYSTEM *fs) { + mesh::hilStartupTrace("mesh_dispatcher_begin"); mesh::Mesh::begin(); // also starts OTA (ota_ctx().begin) for all roles + mesh::hilStartupTrace("mesh_dispatcher_ready"); _fs = fs; // load persisted prefs + mesh::hilStartupTrace("mesh_prefs_begin"); _cli.loadPrefs(_fs); + mesh::hilStartupTrace("mesh_prefs_ready"); + mesh::hilStartupTrace("mesh_management_begin"); _cli.beginManagement(*this, _fs); #if MESH_ENABLE_TELEMETRY_HISTORY loadTelemetryHistoryTxPrefs(); #endif + mesh::hilStartupTrace("mesh_management_ready"); #if defined(SIM_WIFI_SSID) && defined(WITH_MQTT_BRIDGE) // Emulator builds (Wokwi) boot with fresh NVS every run. Seed WiFi so the @@ -3734,9 +3744,12 @@ void MyMesh::begin(FILESYSTEM *fs) { } #endif + mesh::hilStartupTrace("mesh_acl_begin"); acl.load(_fs, self_id); + mesh::hilStartupTrace("mesh_acl_ready"); // TODO: key_store.begin(); region_map.load(_fs); + mesh::hilStartupTrace("mesh_regions_ready"); #if !defined(PORTABLE_MQTT_OBSERVER) #if MESH_ENABLE_FLOOD_RULE_ENGINE bool flood_filters_loaded = loadFloodPacketFilters(); @@ -3755,6 +3768,7 @@ void MyMesh::begin(FILESYSTEM *fs) { loadClockSyncPrefs(); #endif #endif + mesh::hilStartupTrace("mesh_filters_ready"); // establish default-scope { @@ -3776,6 +3790,11 @@ void MyMesh::begin(FILESYSTEM *fs) { } } +#if defined(ESP32_PLATFORM) + // Bridge/WebConfig startup can create tasks that read or write this FS. + mesh::endEsp32BootFileInventory(); +#endif + mesh::hilStartupTrace("mesh_bridge_begin"); #if defined(WITH_BRIDGE) if (_prefs.bridge_enabled) { #ifdef WITH_MQTT_BRIDGE @@ -3849,6 +3868,7 @@ void MyMesh::begin(FILESYSTEM *fs) { #if defined(WITH_MQTT_BRIDGE) && defined(WITH_ESPNOW_BRIDGE) if (_prefs.espnow_bridge_enabled) setEspNowBridgeState(true); #endif + mesh::hilStartupTrace("mesh_bridge_ready"); // Wire fault-alert reporter. begin() is safe regardless of bridge state. // Passing `this` as the callbacks lets the reporter resolve a TransportKey @@ -3859,6 +3879,7 @@ void MyMesh::begin(FILESYSTEM *fs) { _alerter.setBridge(mqtt_bridge); #endif + mesh::hilStartupTrace("mesh_webconfig_begin"); #if defined(WITH_WEBCONFIG) && !defined(WEBCONFIG_NO_AUTO_AP) bool start_webui = WebConfigServer::loadEnabled(false); #ifdef WITH_MQTT_BRIDGE @@ -3880,6 +3901,8 @@ void MyMesh::begin(FILESYSTEM *fs) { } #endif + mesh::hilStartupTrace("mesh_webconfig_ready"); + mesh::hilStartupTrace("mesh_radio_begin"); saved_radio_apply_pending = !applySavedRadioParams(); MESH_DEBUG_PRINTLN("RX Boosted Gain Mode: %s", radio_driver.getRxBoostedGainMode() ? "Enabled" : "Disabled"); @@ -3890,6 +3913,7 @@ void MyMesh::begin(FILESYSTEM *fs) { } board.setLoRaFemPaGainEnabled(_prefs.radio_fem_txgain); setRxPowerSaving(_prefs.rx_powersaving_enabled, _prefs.rx_ps_rx_us, _prefs.rx_ps_sleep_us); + mesh::hilStartupTrace("mesh_radio_ready"); board.attachDynamicPrefs(_prefs.getCustom()); @@ -3898,6 +3922,7 @@ void MyMesh::begin(FILESYSTEM *fs) { next_recent_repeater_sweep = futureMillis(RECENT_REPEATER_SWEEP_INTERVAL_MILLIS); #if ENV_INCLUDE_GPS == 1 + mesh::hilStartupTrace("mesh_gps_begin"); applyGpsPrefs(); #if MESH_ENABLE_TELEMETRY_HISTORY if (sensors.getLocationProvider() != NULL) { @@ -3905,6 +3930,7 @@ void MyMesh::begin(FILESYSTEM *fs) { MESH_DEBUG_PRINTLN("Telemetry GPS retention: %u days", (unsigned)gps_days); } #endif + mesh::hilStartupTrace("mesh_gps_ready"); #endif #if MESH_ENABLE_TELEMETRY_HISTORY @@ -3920,6 +3946,7 @@ void MyMesh::begin(FILESYSTEM *fs) { (unsigned)external_voltage_history.storageBytes()); } #endif + mesh::hilStartupTrace("mesh_history_ready"); } bool MyMesh::sendFloodScoped(const TransportKey& scope, mesh::Packet* pkt, uint32_t delay_millis, uint8_t path_hash_size) { diff --git a/examples/simple_repeater/main.cpp b/examples/simple_repeater/main.cpp index bcab6458..e0252436 100644 --- a/examples/simple_repeater/main.cpp +++ b/examples/simple_repeater/main.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #if MESH_PACKET_LOGGING #include #endif @@ -15,6 +16,7 @@ #endif #if defined(ESP32_PLATFORM) #include + #include #endif #if defined(NRF52_PLATFORM) #include @@ -105,12 +107,16 @@ void setup() { #if MESH_PACKET_LOGGING mesh::serialLogBegin(); #endif + mesh::hilStartupTrace("serial_ready"); + mesh::hilStartupTrace("board_begin"); board.begin(); + mesh::hilStartupTrace("board_ready"); #ifdef HAS_EXTERNAL_WATCHDOG external_watchdog.begin(); #endif + mesh::hilStartupTrace("fs_begin"); FILESYSTEM* fs; #if defined(NRF52_PLATFORM) bool volatile_primary_fs = false; @@ -146,7 +152,9 @@ void setup() { #else #error "need to define filesystem" #endif + mesh::hilStartupTrace("fs_ready"); #ifdef DISPLAY_CLASS + mesh::hilStartupTrace("display_begin"); mesh::ui::loadDisplayPowerSettings(fs, false); mesh::ui::StartupScreen startup_screen; display_ready = display.begin(); @@ -154,6 +162,7 @@ void setup() { startup_screen.begin(&display, board.isExternalPowered() || board.isUsbHostConnected()); } + mesh::hilStartupTrace("display_ready"); #endif // Let serial settle after the display is already showing startup. @@ -165,6 +174,7 @@ void setup() { delay(5000); #endif + mesh::hilStartupTrace("radio_begin"); int radioinit_attempts = 0; while (!radio_init()) { ++radioinit_attempts; @@ -198,9 +208,11 @@ void setup() { } delay(500); } + mesh::hilStartupTrace("radio_ready"); fast_rng.begin(radio_driver.getRngSeed()); + mesh::hilStartupTrace("identity_begin"); #if defined(NRF52_PLATFORM) IdentityLoadResult identity_load = volatile_primary_fs ? store.loadResult("_main", the_mesh.self_id) @@ -250,6 +262,15 @@ void setup() { board.reboot(); return; } + mesh::hilStartupTrace("identity_ready"); + +#if defined(ESP32_PLATFORM) + // Identity recovery/persistence has finished before taking the inventory. + // The mesh retains this FS view after setup, so the view has static lifetime. + static mesh::Esp32BootFileSystem boot_fs(*fs); + fs = &boot_fs; + boot_fs.beginInventory(); +#endif // Print the running firmware version at boot so it's visible after an OTA // reboot without having to issue `ver` manually. @@ -276,10 +297,15 @@ void setup() { } #endif + mesh::hilStartupTrace("sensors_begin"); sensors.begin(); + mesh::hilStartupTrace("sensors_ready"); + mesh::hilStartupTrace("mesh_begin"); the_mesh.begin(fs); + mesh::hilStartupTrace("mesh_ready"); + mesh::hilStartupTrace("watchdog_begin"); #if defined(NRF52_PLATFORM) mesh::loadUsbLoggingWatchdog(fs, !volatile_primary_fs, []() -> uint32_t { return rtc_clock.getCurrentTime(); }); @@ -287,6 +313,7 @@ void setup() { mesh::loadUsbLoggingWatchdog(fs, true, []() -> uint32_t { return rtc_clock.getCurrentTime(); }); #endif + mesh::hilStartupTrace("watchdog_ready"); #if defined(NRF52_PLATFORM) if (volatile_primary_fs) { @@ -300,6 +327,7 @@ void setup() { #ifdef DISPLAY_CLASS if (display_ready) { + mesh::hilStartupTrace("display_task_begin"); #ifdef WITH_MQTT_BRIDGE ui_task.setObserverPrefs(the_mesh.getObserverPrefs()); #endif @@ -307,6 +335,7 @@ void setup() { ui_task.setActivityWindow(the_mesh.getActivityWindow()); #endif ui_task.begin(the_mesh.getNodePrefs(), FIRMWARE_BUILD_DATE, FIRMWARE_VERSION); + mesh::hilStartupTrace("display_task_ready"); } #endif @@ -319,10 +348,25 @@ void setup() { the_mesh.sendSelfAdvertisement(16000, false); #endif + mesh::hilStartupTrace("boot_complete_begin"); +#if defined(ESP32_PLATFORM) + mesh::endEsp32BootFileInventory(); +#endif board.onBootComplete(); + mesh::hilStartupTrace("boot_complete_ready"); } static void __attribute__((noinline)) serviceCommandInterfaces() { +#if defined(MESH_HIL_STARTUP_TRACE) && MESH_HIL_STARTUP_TRACE + static bool first_cli_entry = true; + static bool first_cli_ready = true; + static bool first_cli_blocked = true; + static bool first_cli_rx = true; + if (first_cli_entry) { + first_cli_entry = false; + mesh::hilStartupTrace("cli_first_entry"); + } +#endif bool usb_ready = true; #if MESH_USB_CONSOLE_COOPERATIVE mesh::serviceUsbLoggingPort(); @@ -339,6 +383,19 @@ static void __attribute__((noinline)) serviceCommandInterfaces() { && mesh::canAcceptUsbConsoleCommand(); #endif Stream& console = mesh::usbConsolePort(); +#if defined(MESH_HIL_STARTUP_TRACE) && MESH_HIL_STARTUP_TRACE + if (usb_ready && first_cli_ready) { + first_cli_ready = false; + mesh::hilStartupTrace("cli_ready"); + } else if (!usb_ready && first_cli_blocked) { + first_cli_blocked = false; + mesh::hilStartupTrace("cli_gate_blocked"); + } + if (usb_ready && first_cli_rx && console.available() > 0) { + first_cli_rx = false; + mesh::hilStartupTrace("cli_rx_seen"); + } +#endif // Handle Serial CLI int len = strlen(command); bool line_complete = false; diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 60d532b4..d85a2741 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -1,4 +1,7 @@ #include "MyMesh.h" +#if defined(ESP32_PLATFORM) +#include +#endif #include #if defined(ENABLE_OTA) #include @@ -1418,6 +1421,10 @@ void MyMesh::begin(FILESYSTEM *fs) { #if ENV_INCLUDE_GPS == 1 applyGpsPrefs(); #endif +#if defined(ESP32_PLATFORM) + // MQTT/WebConfig startup below may create tasks that access persisted state. + mesh::endEsp32BootFileInventory(); +#endif #ifdef WITH_MQTT_BRIDGE if (_prefs.bridge_enabled) { // Defer construction to avoid static init crashes on ESP32 classic diff --git a/examples/simple_room_server/main.cpp b/examples/simple_room_server/main.cpp index e5587836..2eae50a0 100644 --- a/examples/simple_room_server/main.cpp +++ b/examples/simple_room_server/main.cpp @@ -13,6 +13,7 @@ #if defined(ESP32_PLATFORM) #include + #include #endif #if defined(NRF52_PLATFORM) #include @@ -173,6 +174,13 @@ void setup() { return; } +#if defined(ESP32_PLATFORM) + // Identity writes have finished; this FS view remains alive after setup. + static mesh::Esp32BootFileSystem boot_fs(*fs); + fs = &boot_fs; + boot_fs.beginInventory(); +#endif + Stream& console = mesh::usbConsolePort(); console.print("Room ID: "); mesh::Utils::printHex(console, the_mesh.self_id.pub_key, PUB_KEY_SIZE); console.println(); @@ -225,6 +233,9 @@ void setup() { the_mesh.sendSelfAdvertisement(16000, false); #endif +#if defined(ESP32_PLATFORM) + mesh::endEsp32BootFileInventory(); +#endif board.onBootComplete(); } diff --git a/platformio.ini b/platformio.ini index 05cb0bde..3cc730c8 100644 --- a/platformio.ini +++ b/platformio.ini @@ -123,6 +123,7 @@ build_src_filter = ${arduino_base.build_src_filter} + + + + + + + diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 2965b9e6..40887b61 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -2,6 +2,7 @@ #include "CommonCLI.h" #include "PrefsSaveReplyGuard.h" #include "UsbLoggingWatchdog.h" +#include "HilStartupTrace.h" #include #include "CLICommandUtils.h" #include "GpsPowerPolicy.h" @@ -745,15 +746,23 @@ static void formatSnrDbX4Short(char* dest, size_t dest_len, int16_t snr_x4) { } void CommonCLI::loadPrefs(FILESYSTEM* fs) { + mesh::hilStartupTrace("prefs_profiles_begin"); _radio_profiles.begin(fs, _callbacks->getProfileRadio(), _rtc, true); + mesh::hilStartupTrace("prefs_profiles_ready"); _prefs->primary_radio_preamble = _radio_profiles.primaryPreamble(); #if !defined(WITH_MQTT_BRIDGE) && (defined(ESP32_PLATFORM) || defined(RP2040_PLATFORM)) + mesh::hilStartupTrace("prefs_recovery_begin"); mesh::ContactFileTransaction::recover(fs, "/com_prefs"); + mesh::hilStartupTrace("prefs_recovery_ready"); #endif #if defined(ENABLE_OTA) + mesh::hilStartupTrace("prefs_ota_speed_begin"); mesh::ota::beginSpeedConfig(fs); + mesh::hilStartupTrace("prefs_ota_speed_ready"); #endif + mesh::hilStartupTrace("prefs_display_begin"); const bool display_settings_loaded = mesh::ui::loadDisplayPowerSettings(fs, false); + mesh::hilStartupTrace("prefs_display_ready"); (void)display_settings_loaded; bool is_fresh_install = false; bool is_upgrade = false; @@ -777,11 +786,14 @@ void CommonCLI::loadPrefs(FILESYSTEM* fs) { #ifdef WITH_MQTT_BRIDGE bool node_prefs_needs_migration = false; + mesh::hilStartupTrace("prefs_recovery_begin"); if (!recoverCommonPrefsFiles(fs)) { MESH_DEBUG_PRINTLN("Prefs: common preference recovery is incomplete"); } + mesh::hilStartupTrace("prefs_recovery_ready"); #endif + mesh::hilStartupTrace("prefs_common_begin"); if (fs->exists("/com_prefs")) { loadPrefsInt(fs, "/com_prefs"); loaded = true; // new filename } else if (fs->exists("/node_prefs")) { @@ -835,13 +847,16 @@ void CommonCLI::loadPrefs(FILESYSTEM* fs) { _prefs->powersaving_enabled = DEFAULT_POWERSAVING_ENABLED && !dual_radio_active ? 1 : 0; _prefs->bridge_pkt_src = 1; // Default to RX (logRx) for new installs } + mesh::hilStartupTrace("prefs_common_ready"); #ifdef WITH_MQTT_BRIDGE // Load observer preferences (MQTT/WiFi/timezone/SNMP/alert) from /mqtt_prefs. // Readers (MQTTBridge, AlertReporter, observer CLI) use _mqtt_prefs directly - // these fields no longer exist in NodePrefs, so there is nothing to sync. MQTTPrefsAtomicStore::LegacyUpgradeGate legacy_upgrade( _com_prefs_needs_upgrade || node_prefs_needs_migration); + mesh::hilStartupTrace("prefs_mqtt_begin"); loadMQTTPrefs(fs, &legacy_upgrade); + mesh::hilStartupTrace("prefs_mqtt_ready"); if (!display_settings_loaded && _mqtt_prefs.display_timeout_secs != DISPLAY_TIMEOUT_DEFAULT_SECS) mesh::ui::migrateLegacyDisplayTimeout(_mqtt_prefs.display_timeout_secs); if (_mqtt_prefs_hold) legacy_upgrade.holdMqttSource(); @@ -868,6 +883,7 @@ void CommonCLI::loadPrefs(FILESYSTEM* fs) { // set by setMQTTPrefsDefaults(). No explicit migration needed. #endif + mesh::hilStartupTrace("prefs_migration_begin"); #ifdef WITH_MQTT_BRIDGE if (node_prefs_needs_migration) { if (legacy_upgrade.mayRewriteComPrefs()) { @@ -905,15 +921,18 @@ void CommonCLI::loadPrefs(FILESYSTEM* fs) { _com_prefs_needs_upgrade = false; } #endif + mesh::hilStartupTrace("prefs_migration_ready"); #if defined(ENABLE_OTA) if (loaded) syncOtaConfigFromPrefs(); // persisted OTA policy/keys -> OtaContext (else keep safe defaults) #endif + mesh::hilStartupTrace("prefs_apply_begin"); #if MESH_USB_LOGGING_AVAILABLE mesh::setUsbDebugEnabled(_prefs->usb_debug_enabled != 0); mesh::setUsbLoggingEnabled(_prefs->usb_logging_enabled != 0); #endif _radio_profiles.adoptPrimaryPreamble(_prefs->primary_radio_preamble); _radio_profiles.stagePrimary(_prefs->primary_radio_preamble, false); + mesh::hilStartupTrace("prefs_apply_ready"); } #if defined(ENABLE_OTA) @@ -939,6 +958,7 @@ void CommonCLI::syncOtaConfigFromPrefs() { #endif void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) { + mesh::hilStartupTrace("prefs_image_open_begin"); // Older or truncated preference images must never inherit these opt-ins // from a previous load of a newer image. _prefs->usb_debug_enabled = 0; @@ -950,6 +970,7 @@ void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) { #else File file = fs->open(filename); #endif + mesh::hilStartupTrace("prefs_image_open_ready"); if (file) { #if defined(WITH_RS232_BRIDGE) && defined(RS232_BRIDGE_MERGED) \ && !defined(RS232_BRIDGE_DEFAULT_ON) @@ -967,6 +988,7 @@ void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) { } uint8_t pad[8]; + mesh::hilStartupTrace("prefs_image_core_begin"); file.read((uint8_t *)&_prefs->airtime_factor, sizeof(_prefs->airtime_factor)); // 0 file.read((uint8_t *)&_prefs->node_name, sizeof(_prefs->node_name)); // 4 file.read(pad, 4); // 36 @@ -1010,6 +1032,7 @@ void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) { file.read((uint8_t *)&_prefs->discovery_mod_timestamp, sizeof(_prefs->discovery_mod_timestamp)); // 162 file.read((uint8_t *)&_prefs->adc_multiplier, sizeof(_prefs->adc_multiplier)); // 166 file.read((uint8_t *)_prefs->owner_info, sizeof(_prefs->owner_info)); // 170 + mesh::hilStartupTrace("prefs_image_core_ready"); _prefs->node_name[sizeof(_prefs->node_name) - 1] = '\0'; _prefs->password[sizeof(_prefs->password) - 1] = '\0'; _prefs->guest_password[sizeof(_prefs->guest_password) - 1] = '\0'; diff --git a/src/helpers/CommonCLI_Management.cpp b/src/helpers/CommonCLI_Management.cpp index f538bf83..5110b2b8 100644 --- a/src/helpers/CommonCLI_Management.cpp +++ b/src/helpers/CommonCLI_Management.cpp @@ -2,6 +2,7 @@ #include "ManagementReporter.h" #include "FileRead.h" #include "PersistentStoreFormat.h" +#include "HilStartupTrace.h" #include #include #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) @@ -245,17 +246,23 @@ void CommonCLI::beginManagement(mesh::Mesh& mesh, FILESYSTEM* fs) { _management_mesh = &mesh; _management_fs = fs; _management_last_ms = millis(); _management_uptime_ms = _management_last_ms; if (!_data_route) { + mesh::hilStartupTrace("management_route_begin"); _data_route = new (std::nothrow) mesh::DataRouteState; if (_data_route) { _data_route->fs = fs; _data_route->regions = _region_map; _data_route->healthy = mesh::dataRouteLoad(*_data_route); } + mesh::hilStartupTrace("management_route_ready"); } // No reporter/history allocation for the default unconfigured installation. + mesh::hilStartupTrace("management_probe_begin"); if (fs->exists("/management") || fs->exists("/management.bak")) { + mesh::hilStartupTrace("management_reporter_begin"); _management = new (std::nothrow) mesh::ManagementReporter( mesh, *_board, *_sensors, *_acl, *_prefs, *_callbacks, *this, fs); + mesh::hilStartupTrace("management_reporter_ready"); } + mesh::hilStartupTrace("management_probe_ready"); } void CommonCLI::loopManagement() { diff --git a/src/helpers/FilePresence.h b/src/helpers/FilePresence.h index 65a87011..db53dc51 100644 --- a/src/helpers/FilePresence.h +++ b/src/helpers/FilePresence.h @@ -4,6 +4,7 @@ #if defined(ESP32_PLATFORM) #include #include +#include #endif namespace mesh { @@ -16,6 +17,10 @@ bool filePresence(Filesystem* fs, const char* path, bool& present) { char absolute[128]; const int length = snprintf(absolute, sizeof(absolute), "/spiffs%s", path); if (length < 0 || static_cast(length) >= sizeof(absolute)) return false; + if (esp32BootFileKnownAbsent(fs, path)) { + present = false; + return true; + } struct stat info; const int result = ::stat(absolute, &info); if (result != 0 && errno != ENOENT) return false; diff --git a/src/helpers/HilStartupTrace.h b/src/helpers/HilStartupTrace.h new file mode 100644 index 00000000..df83f2ae --- /dev/null +++ b/src/helpers/HilStartupTrace.h @@ -0,0 +1,18 @@ +#pragma once + +// Optional hardware-test timestamps only. Normal firmware has no extra output +// or timing work; phases are fixed literals and never contain node data. +#if defined(MESH_HIL_STARTUP_TRACE) && MESH_HIL_STARTUP_TRACE +#include +#include +namespace mesh { +inline void hilStartupTrace(const char* phase) { + usbConsolePort().printf("HIL_BOOT phase=%s at=%lu\r\n", phase, + static_cast(millis())); +} +} +#else +namespace mesh { +inline void hilStartupTrace(const char*) {} +} +#endif diff --git a/src/helpers/esp32/BootFilePresence.h b/src/helpers/esp32/BootFilePresence.h new file mode 100644 index 00000000..e0c2598a --- /dev/null +++ b/src/helpers/esp32/BootFilePresence.h @@ -0,0 +1,25 @@ +#pragma once + +namespace mesh { +namespace detail { +struct Esp32BootAbsenceRegistry { + const void* owner = nullptr; + const void* context = nullptr; + bool (*known_absent)(const void*, const char*) = nullptr; +}; + +inline Esp32BootAbsenceRegistry& esp32BootAbsenceRegistry() { + static Esp32BootAbsenceRegistry registry; + return registry; +} +} // namespace detail + +// The inventory belongs to one FS view, and is only active during synchronous +// startup. Unrelated filesystems and native positive/error probes are untouched. +inline bool esp32BootFileKnownAbsent(const void* filesystem, const char* path) { + const detail::Esp32BootAbsenceRegistry& registry = detail::esp32BootAbsenceRegistry(); + return filesystem != nullptr && registry.owner == filesystem + && registry.known_absent != nullptr + && registry.known_absent(registry.context, path); +} +} // namespace mesh diff --git a/src/helpers/esp32/BootFileSystem.cpp b/src/helpers/esp32/BootFileSystem.cpp new file mode 100644 index 00000000..6a47d711 --- /dev/null +++ b/src/helpers/esp32/BootFileSystem.cpp @@ -0,0 +1,188 @@ +#if defined(ESP32_PLATFORM) +#include "BootFileSystem.h" +#include "BootFilePresence.h" +#include +#include +#include +#include +#include +#include + +namespace mesh { +namespace { +constexpr size_t InventoryCapacity = 8192; +#if defined(CONFIG_SPIFFS_OBJ_NAME_LEN) +constexpr size_t MaximumPathLength = CONFIG_SPIFFS_OBJ_NAME_LEN - 1; +#else +constexpr size_t MaximumPathLength = 0; +#endif +Esp32BootFileSystem* active_inventory = nullptr; + +// SPIFFS has flat names with implicit directories. Ambiguous path spellings +// must use the backend, which owns their interpretation. +bool normalPath(const char* path) { + if (path == nullptr || path[0] != '/' || path[1] == 0) return false; + const size_t length = strnlen(path, MaximumPathLength + 1); + if (length > MaximumPathLength || path[length - 1] == '/') return false; + const char* segment = path + 1; + for (const char* cursor = segment;; ++cursor) { + if (*cursor == '/' || *cursor == 0) { + const size_t count = cursor - segment; + if (count == 0 || (count == 1 && segment[0] == '.') + || (count == 2 && segment[0] == '.' && segment[1] == '.')) return false; + if (*cursor == 0) return true; + segment = cursor + 1; + } + } +} +} // namespace + +Esp32BootFileSystem::Delegate::Delegate(Esp32BootFileSystem& owner, + fs::FSImplPtr backend) + : owner_(owner), backend_(backend) { + if (backend_) mountpoint(backend_->mountpoint()); +} + +Esp32BootFileSystem::Esp32BootFileSystem(fs::FS& source) + : fs::FS(source), delegate_(*this, _impl) { + // Aliasing reuses the original shared_ptr control block; constructing the + // view requires no allocation. Its delegate has static lifetime in the app. + if (_impl) _impl = fs::FSImplPtr(_impl, &delegate_); +} + +bool Esp32BootFileSystem::beginInventory() { + // Never rebuild after a mutation: a returned writable handle can continue + // changing the backend after open() has returned. + if (inventory_attempted_) return false; + inventory_attempted_ = true; + if (active_inventory != nullptr || MaximumPathLength < 2 + || delegate_.mountpoint() == nullptr + || strcmp(delegate_.mountpoint(), "/spiffs") != 0) return false; + char* names = static_cast(::malloc(InventoryCapacity)); + if (names == nullptr) return false; + DIR* directory = ::opendir(delegate_.mountpoint()); + if (directory == nullptr) { + ::free(names); + return false; + } + size_t used = 0; + bool complete = false; + while (true) { + errno = 0; + struct dirent* entry = ::readdir(directory); + if (entry == nullptr) { + complete = errno == 0; + break; + } + const size_t raw_length = strnlen(entry->d_name, sizeof(entry->d_name)); + if (raw_length == 0 || raw_length == sizeof(entry->d_name)) break; + const char* name = entry->d_name; + if (name[0] == '/') ++name; + const size_t length = raw_length - (name != entry->d_name ? 1 : 0); + if (length == 0 || length + 1 > MaximumPathLength + || length + 2 > InventoryCapacity - used) break; + names[used] = '/'; + memcpy(names + used + 1, name, length + 1); + if (!normalPath(names + used)) break; + used += length + 2; + } + if (::closedir(directory) != 0) complete = false; + if (!complete) { + ::free(names); + return false; + } + // Release the enumeration scratch before allocating persisted-state owners. + // An empty inventory needs no retained storage; a small one keeps only names. + if (used == 0) { + ::free(names); + names = nullptr; + } else { + char* compact = static_cast(::realloc(names, used)); + if (compact == nullptr) { + ::free(names); + return false; + } + names = compact; + } + names_ = names; + names_used_ = used; + inventory_active_ = true; + active_inventory = this; + detail::Esp32BootAbsenceRegistry& registry = detail::esp32BootAbsenceRegistry(); + registry.owner = static_cast(this); + registry.context = this; + registry.known_absent = &Esp32BootFileSystem::queryAbsent; + return true; +} + +void Esp32BootFileSystem::endInventory() { + inventory_attempted_ = true; + inventory_active_ = false; + if (active_inventory == this) { + active_inventory = nullptr; + detail::Esp32BootAbsenceRegistry& registry = detail::esp32BootAbsenceRegistry(); + registry = detail::Esp32BootAbsenceRegistry(); + } + ::free(names_); + names_ = nullptr; + names_used_ = 0; +} + +void endEsp32BootFileInventory() { + if (active_inventory != nullptr) active_inventory->endInventory(); +} + +bool Esp32BootFileSystem::queryAbsent(const void* context, const char* path) { + return static_cast(context)->knownAbsent(path); +} + +bool Esp32BootFileSystem::knownAbsent(const char* path) const { + if (!inventory_active_ || !normalPath(path)) return false; + const size_t length = strlen(path); + for (size_t offset = 0; offset < names_used_;) { + const char* name = names_ + offset; + const size_t name_length = strlen(name); + // Exact filenames are case sensitive. Native SPIFFS directory prefix + // enumeration is case insensitive, so keep its virtual directories too. + if (strcmp(name, path) == 0 + || (name_length > length && name[length] == '/' + && strncasecmp(name, path, length) == 0)) return false; + offset += name_length + 1; + } + return true; +} + +fs::FileImplPtr Esp32BootFileSystem::Delegate::open(const char* path, + const char* mode, bool create) { + const bool readonly = mode != nullptr + && (strcmp(mode, "r") == 0 || strcmp(mode, "rb") == 0) && !create; + if (!readonly) owner_.endInventory(); + else if (owner_.knownAbsent(path)) return fs::FileImplPtr(); + return backend_->open(path, mode, create); +} + +bool Esp32BootFileSystem::Delegate::exists(const char* path) { + return !owner_.knownAbsent(path) && backend_->exists(path); +} + +bool Esp32BootFileSystem::Delegate::rename(const char* from, const char* to) { + owner_.endInventory(); + return backend_->rename(from, to); +} + +bool Esp32BootFileSystem::Delegate::remove(const char* path) { + owner_.endInventory(); + return backend_->remove(path); +} + +bool Esp32BootFileSystem::Delegate::mkdir(const char* path) { + owner_.endInventory(); + return backend_->mkdir(path); +} + +bool Esp32BootFileSystem::Delegate::rmdir(const char* path) { + owner_.endInventory(); + return backend_->rmdir(path); +} +} // namespace mesh +#endif diff --git a/src/helpers/esp32/BootFileSystem.h b/src/helpers/esp32/BootFileSystem.h new file mode 100644 index 00000000..38896576 --- /dev/null +++ b/src/helpers/esp32/BootFileSystem.h @@ -0,0 +1,49 @@ +#pragma once + +#if defined(ESP32_PLATFORM) +#include +#include + +namespace mesh { + +// This view must outlive every user of the filesystem (use static storage). +// Only known absence is cached; files and FileImpl handles stay SDK-owned. +class Esp32BootFileSystem : public fs::FS { + public: + explicit Esp32BootFileSystem(fs::FS& source); + ~Esp32BootFileSystem() { endInventory(); } + Esp32BootFileSystem(const Esp32BootFileSystem&) = delete; + Esp32BootFileSystem& operator=(const Esp32BootFileSystem&) = delete; + bool beginInventory(); + void endInventory(); + bool isInventoryActive() const { return inventory_active_; } + + private: + class Delegate : public fs::FSImpl { + public: + Delegate(Esp32BootFileSystem& owner, fs::FSImplPtr backend); + fs::FileImplPtr open(const char* path, const char* mode, bool create) override; + bool exists(const char* path) override; + bool rename(const char* from, const char* to) override; + bool remove(const char* path) override; + bool mkdir(const char* path) override; + bool rmdir(const char* path) override; + + private: + Esp32BootFileSystem& owner_; + fs::FSImplPtr backend_; + }; + + static bool queryAbsent(const void* context, const char* path); + bool knownAbsent(const char* path) const; + Delegate delegate_; + char* names_ = nullptr; + size_t names_used_ = 0; + bool inventory_active_ = false; + bool inventory_attempted_ = false; +}; + +// Stop before starting any task that can access or modify persisted state. +void endEsp32BootFileInventory(); +} // namespace mesh +#endif diff --git a/src/helpers/ota/OtaSpeedConfig.cpp b/src/helpers/ota/OtaSpeedConfig.cpp index d59f9ca4..2a2ad160 100644 --- a/src/helpers/ota/OtaSpeedConfig.cpp +++ b/src/helpers/ota/OtaSpeedConfig.cpp @@ -7,6 +7,7 @@ #if defined(ESP32_PLATFORM) #include #include +#include #endif namespace mesh { namespace ota { @@ -36,6 +37,10 @@ bool pathPresence(const char* path, bool& present) { char vfs_path[48]; const int length = snprintf(vfs_path, sizeof(vfs_path), "/spiffs%s", path); if (length < 0 || static_cast(length) >= sizeof(vfs_path)) return false; + if (esp32BootFileKnownAbsent(settings_fs, path)) { + present = false; + return true; + } struct stat info; const int result = ::stat(vfs_path, &info); if (result != 0 && errno != ENOENT) return false; diff --git a/test/test_esp32_boot_file_inventory.py b/test/test_esp32_boot_file_inventory.py new file mode 100644 index 00000000..4e1403c0 --- /dev/null +++ b/test/test_esp32_boot_file_inventory.py @@ -0,0 +1,626 @@ +#!/usr/bin/env python3 +"""Run the production boot-only SPIFFS inventory over SDK/POSIX doubles. + +The actual FS decorator, owner registry, metadata helper and transaction +recovery run unchanged. Directory and flash failures occur only at peripheral +boundaries. The reusable deadline workload reports a 129 ms/name-scan model, +not physical timing or a qualification of flash corruption. +""" +from pathlib import Path +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + +from test_replay_reset_integration import extract_braced + +ROOT = Path(__file__).resolve().parents[1] +PRODUCTION = ROOT / "src/helpers/esp32/BootFileSystem.cpp" + +FS_HEADER = r''' +#pragma once +#include +#include +#include +#include +using String = std::string; +using boolean = bool; +namespace fs { +class FileImpl { +public: + bool directory = false; + std::string name; + virtual ~FileImpl() = default; +}; +using FileImplPtr = std::shared_ptr; +class File { +public: + FileImplPtr impl; + File(FileImplPtr value = {}) : impl(value) {} + explicit operator bool() const { return bool(impl); } + bool isDirectory() const { return impl && impl->directory; } +}; +class FSImpl { +protected: + const char* _mountpoint = nullptr; +public: + virtual ~FSImpl() = default; + virtual FileImplPtr open(const char*, const char*, bool) = 0; + virtual bool exists(const char*) = 0; + virtual bool rename(const char*, const char*) = 0; + virtual bool remove(const char*) = 0; + virtual bool mkdir(const char*) = 0; + virtual bool rmdir(const char*) = 0; + void mountpoint(const char* value) { _mountpoint = value; } + const char* mountpoint() { return _mountpoint; } +}; +using FSImplPtr = std::shared_ptr; +class FS { +protected: + FSImplPtr _impl; +public: + explicit FS(FSImplPtr impl) : _impl(impl) {} + File open(const char* path, const char* mode = "r", bool create = false) { + return File(_impl->open(path, mode, create)); + } + bool exists(const char* path) { return _impl->exists(path); } + bool remove(const char* path) { return _impl->remove(path); } + bool rename(const char* from, const char* to) { return _impl->rename(from, to); } + bool mkdir(const char* path) { return _impl->mkdir(path); } + bool rmdir(const char* path) { return _impl->rmdir(path); } +}; +} +using fs::File; +''' + +DIRENT_HEADER = r''' +#pragma once +#include +struct DIR { size_t cursor = 0; }; +struct dirent { unsigned char d_type = 1; char d_name[256] = {}; }; +#define DT_REG 1 +#define DT_DIR 2 +DIR* opendir(const char*); +struct dirent* readdir(DIR*); +int closedir(DIR*); +''' + +STAT_HEADER = r''' +#pragma once +struct stat { unsigned st_mode = 0; }; +int stat(const char*, struct stat*); +''' + +HARNESS = r''' +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static unsigned new_calls = 0; +void* operator new(size_t count) { + ++new_calls; + if (void* address = std::malloc(count)) return address; + throw std::bad_alloc(); +} +void operator delete(void* address) noexcept { std::free(address); } +void operator delete(void* address, size_t) noexcept { std::free(address); } + +struct Peripherals { + std::vector names; + unsigned scans = 0, directory_opens = 0, directory_reads = 0, directory_closes = 0; + unsigned native_exists = 0, native_opens = 0, stats = 0, allocations = 0; + unsigned mutations = 0, frees = 0, reallocations = 0; + size_t shrink_bytes = 0; + bool fail_open = false, fail_close = false, fail_malloc = false, metadata_error = false; + bool fail_realloc = false; + bool dirty_success_errno = false, unterminated_name = false; + int fail_read_at = -1; + const void* owner = nullptr; +} peripheral; +static std::map inventory_buffers; + +void* inventory_malloc(size_t count) { + ++peripheral.allocations; + assert(count == 8192); + if (peripheral.fail_malloc) return nullptr; + void* address = std::malloc(count); + if (address) inventory_buffers[address] = count; + return address; +} +void* inventory_realloc(void* address, size_t count) { + ++peripheral.reallocations; + peripheral.shrink_bytes = count; + assert(inventory_buffers.count(address) && inventory_buffers[address] == 8192); + assert(count > 0 && count < 8192); + if (peripheral.fail_realloc) return nullptr; + const auto found = inventory_buffers.find(address); + void* compact = std::realloc(address, count); + if (compact) { + inventory_buffers.erase(found); + inventory_buffers[compact] = count; + } + return compact; +} +void inventory_free(void* address) { + if (address) { + assert(inventory_buffers.erase(address) == 1); + ++peripheral.frees; + } + std::free(address); +} +DIR* opendir(const char* path) { + ++peripheral.directory_opens; + ++peripheral.scans; // One complete SPIFFS lookup-table traversal, when valid. + assert(std::string(path) == "/spiffs" || std::string(path) == "/spiffs/"); + if (peripheral.fail_open) { errno = EIO; return nullptr; } + return new DIR; +} +struct dirent* readdir(DIR* directory) { + static struct dirent entry; + ++peripheral.directory_reads; + if (int(directory->cursor) == peripheral.fail_read_at) { errno = EIO; return nullptr; } + if (directory->cursor == peripheral.names.size()) return nullptr; + entry = {}; + const std::string& name = peripheral.names[directory->cursor++]; + assert(name.size() < sizeof(entry.d_name)); + memcpy(entry.d_name, name.c_str(), name.size() + 1); + if (peripheral.unterminated_name) memset(entry.d_name, 'x', sizeof(entry.d_name)); + if (peripheral.dirty_success_errno) errno = EIO; + return &entry; +} +int closedir(DIR* directory) { + ++peripheral.directory_closes; + delete directory; + if (peripheral.fail_close) { errno = EIO; return -1; } + return 0; +} + +class NativeFSImpl : public fs::FSImpl { +public: + std::set files; + std::set directories; + fs::FileImplPtr last_open; + bool fail_mutation = false, unreadable = false; + NativeFSImpl() { mountpoint("/spiffs"); } + static void requireInvalidated() { + ++peripheral.mutations; + assert(!mesh::esp32BootFileKnownAbsent(peripheral.owner, "/never-created")); + } + fs::FileImplPtr open(const char* path, const char* mode, bool create) override { + ++peripheral.native_opens; + const bool write = !mode || (strcmp(mode, "r") && strcmp(mode, "rb")) || create; + if (write) { + requireInvalidated(); + if (fail_mutation) return {}; + if (path) files.insert(path); + } + if (path && strlen(path) > 31) { errno = ENAMETOOLONG; return {}; } + const bool present = path && files.count(path); + peripheral.scans += present ? 3 : 2; // pinned VFS stat + constructor stat (+ fopen). + if (unreadable && present) return {}; + last_open = std::make_shared(); + last_open->directory = !present || directories.count(path); + last_open->name = path ? path : ""; + return last_open; // Missing r-open is a truthy SPIFFS Directory. + } + bool exists(const char* path) override { + ++peripheral.native_exists; + const bool present = path && files.count(path); + peripheral.scans += present ? 3 : 2; + return present && !unreadable; + } + bool rename(const char* from, const char* to) override { + requireInvalidated(); + if (fail_mutation || !files.count(from) || files.count(to)) return false; + files.erase(from); files.insert(to); return true; + } + bool remove(const char* path) override { + requireInvalidated(); + return !fail_mutation && files.erase(path); + } + bool mkdir(const char*) override { requireInvalidated(); return !fail_mutation; } + bool rmdir(const char*) override { requireInvalidated(); return !fail_mutation; } +}; +static NativeFSImpl* metadata_fs; +int stat(const char* absolute, struct stat* info) { + ++peripheral.stats; ++peripheral.scans; + if (peripheral.metadata_error) { errno = EIO; return -1; } + assert(strncmp(absolute, "/spiffs", 7) == 0); + if (!metadata_fs->files.count(absolute + 7)) { errno = ENOENT; return -1; } + info->st_mode = 0100000; + return 0; +} + +// This shell executes the real transaction recovery's probes/rename policy. +#define FILESYSTEM fs::FS +namespace mesh { +class ContactFileTransaction { +public: + using PresenceProbe = bool (*)(FILESYSTEM*, const char*, bool&); + @PROBE@ + @RECOVER@ +}; +} +struct CommonCLI { bool recoverCommonPrefsFiles(fs::FS*); }; +@COMMON_RECOVER@ + +static void populate(NativeFSImpl& native, const std::vector& names) { + peripheral.names = names; + for (const std::string& name : names) + native.files.insert(name[0] == '/' ? name : "/" + name); +} +static void requireInactiveFallback(mesh::Esp32BootFileSystem& view) { + assert(!view.isInventoryActive()); + assert(!mesh::esp32BootFileKnownAbsent(&view, "/never-created")); + const unsigned calls = peripheral.native_exists; + assert(!view.exists("/never-created")); + assert(peripheral.native_exists == calls + 1); + const unsigned directory_calls = peripheral.directory_opens; + assert(!view.beginInventory()); + assert(peripheral.directory_opens == directory_calls); +} + +static void deadlineWorkload(fs::FS& fs) { + assert(mesh::ContactFileTransaction::recover(&fs, "/com_prefs")); + assert(!fs.exists("/com_prefs") && !fs.exists("/node_prefs") && !fs.exists("/prefs.json")); + assert(mesh::ContactFileTransaction::recover(&fs, "/data_tx")); + assert(!fs.exists("/data_tx") && !fs.exists("/management") && !fs.exists("/management.bak")); + assert(mesh::ContactFileTransaction::recover( + &fs, "/telemetry_history_tx", mesh::filePresence)); + // Repeated metadata calls follow the empty ACL/replay startup recovery shape. + for (const char* path : {"/s_contacts", "/s_contacts.bak", "/client_login", + "/client_login.bak", "/client_login.tmp", "/s_contacts", + "/s_contacts.bak", "/s_contacts.tmp", "/s_contacts", + "/telemetry_history_tx"}) { + bool present = true; + assert(mesh::filePresence(&fs, path, present) && !present); + } + for (const char* path : {"/flood_filters", "/flood_filters.tmp", "/flood_filters.bak", + "/clock_sync"}) assert(!fs.exists(path)); +} + +int main(int argc, char** argv) { + assert(argc == 2); + const std::string which = argv[1]; + auto native = std::make_shared(); + metadata_fs = native.get(); + fs::FS original(native); + const unsigned construction_allocations = new_calls; + mesh::Esp32BootFileSystem view(original); + assert(new_calls == construction_allocations); // Embedded alias reuses SDK control block. + peripheral.owner = &view; + + if (which == "timing_on" || which == "timing_off") { + if (which == "timing_on") assert(view.beginInventory()); + deadlineWorkload(view); + const unsigned elapsed = 1000 + 129 * peripheral.scans; + std::cout << "{\"elapsed_ms\":" << elapsed << ",\"metadata_lookups\":" << peripheral.scans + << ",\"native_exists\":" << peripheral.native_exists + << ",\"native_opens\":" << peripheral.native_opens + << ",\"snapshot_reads\":" << peripheral.directory_reads << "}\n"; + return 0; + } + + if (which == "basic" || which == "owner" || which == "positive_fault" || which == "positive_directory") { + populate(*native, {"/identity/_main.id", "com_prefs", "com_prefs.tmp", + "com_prefs.bak", "mqtt_prefs", "s_contacts", "corrupt", "Dir/leaf"}); + assert(view.beginInventory() && view.isInventoryActive()); + size_t expected_bytes = 0; + for (const std::string& name : peripheral.names) + expected_bytes += name.size() + (name[0] == '/' ? 1 : 2); + assert(peripheral.allocations == 1 && peripheral.reallocations == 1); + assert(peripheral.shrink_bytes == expected_bytes); + assert(inventory_buffers.size() == 1 && inventory_buffers.begin()->second == expected_bytes); + assert(!view.exists("/never-created")); + assert(!view.open("/never-created", "r")); + assert(!view.open("/never-created", "rb")); + assert(peripheral.native_exists == 0 && peripheral.native_opens == 0); + bool present = true; + assert(mesh::filePresence(&view, "/never-created", present) && !present); + assert(peripheral.stats == 0); + if (which == "owner") { + assert(!mesh::esp32BootFileKnownAbsent(&original, "/never-created")); + assert(!mesh::esp32BootFileKnownAbsent(nullptr, "/never-created")); + assert(mesh::filePresence(&original, "/never-created", present) && !present); + assert(peripheral.stats == 1); + assert(original.open("/never-created").isDirectory()); + mesh::Esp32BootFileSystem other_view(original); + const unsigned directory_calls = peripheral.directory_opens; + assert(!other_view.beginInventory()); + assert(peripheral.directory_opens == directory_calls); + assert(!mesh::esp32BootFileKnownAbsent(&other_view, "/never-created")); + assert(mesh::esp32BootFileKnownAbsent(&view, "/never-created")); + } else if (which == "positive_fault") { + peripheral.metadata_error = true; + assert(!mesh::filePresence(&view, "/com_prefs", present)); + native->unreadable = true; + assert(!view.open("/com_prefs")); + assert(peripheral.stats == 1 && peripheral.native_opens == 1); + } else if (which == "positive_directory") { + native->directories.insert("/corrupt"); + const auto file = view.open("/corrupt"); + assert(file.isDirectory() && file.impl == native->last_open); + assert(peripheral.native_opens == 1); + } else { + for (const char* path : {"/com_prefs", "/com_prefs.tmp", "/com_prefs.bak", + "/mqtt_prefs", "/s_contacts", "/corrupt"}) { + assert(view.exists(path)); + auto file = view.open(path); + assert(file.impl == native->last_open && !file.isDirectory()); + } + assert(mesh::filePresence(&view, "/com_prefs", present) && present); + assert(peripheral.stats == 1); // Positive metadata remains a real stat. + const unsigned before = peripheral.native_opens; + for (const char* path : {"/", "/identity", "/IDENTITY", "/dir", "/Dir/", "/Dir//leaf", + "/./missing", "/../missing", "relative"}) { + assert(view.open(path)); // Delegate to original, preserving SDK directory behavior. + } + assert(peripheral.native_opens == before + 9); + assert(!view.exists("/COM_PREFS")); // Exact file lookup stays case-sensitive. + const std::string long_path = "/" + std::string(31, 'x'); + const unsigned long_before = peripheral.native_opens; + errno = 0; + assert(!view.open(long_path.c_str()) && errno == ENAMETOOLONG); + assert(peripheral.native_opens == long_before + 1); + assert(!mesh::esp32BootFileKnownAbsent(&view, long_path.c_str())); + } + assert(view.isInventoryActive()); + view.endInventory(); + assert(inventory_buffers.empty()); + requireInactiveFallback(view); + } else if (which == "rename_backup" || which == "rename_temp") { + const char* source = which == "rename_backup" ? "/com_prefs.bak" : "/com_prefs.tmp"; + populate(*native, {source + 1}); + assert(view.beginInventory()); + assert(!view.exists("/com_prefs") && view.exists(source)); + if (which == "rename_backup") { + assert(mesh::ContactFileTransaction::recover(&view, "/com_prefs")); + assert(view.exists("/com_prefs") && !view.exists(source)); + assert(view.open("/com_prefs").impl == native->last_open); + } else { + CommonCLI cli; + assert(cli.recoverCommonPrefsFiles(&view)); // First-save temp is uncommitted. + assert(!view.exists("/com_prefs") && !view.exists(source)); + } + requireInactiveFallback(view); + } else if (which == "mutation_before_begin" || which == "end_before_begin") { + if (which == "mutation_before_begin") view.open("/new", "w"); + else view.endInventory(); + assert(!view.beginInventory()); + assert(peripheral.directory_opens == 0 && peripheral.allocations == 0); + } else if (which.rfind("mutation_", 0) == 0) { + assert(view.beginInventory()); + native->fail_mutation = which.find("fail") != std::string::npos; + const std::string operation = which.substr(9); + if (operation == "rplus") view.open("/new", "r+"); + else if (operation == "rbplus") view.open("/new", "rb+"); + else if (operation == "write") view.open("/new", "w"); + else if (operation == "append") view.open("/new", "a"); + else if (operation == "unknown") view.open("/new", "unexpected"); + else if (operation == "null") view.open("/new", nullptr); + else if (operation == "create") view.open("/new", "r", true); + else if (operation == "open_fail") view.open("/new", "w"); + else if (operation == "rename_fail") assert(!view.rename("/absent", "/new")); + else if (operation == "remove_fail") assert(!view.remove("/absent")); + else if (operation == "mkdir") view.mkdir("/dir"); + else if (operation == "rmdir") view.rmdir("/dir"); + else assert(false); + assert(peripheral.mutations == 1); + requireInactiveFallback(view); + } else if (which == "end" || which == "global_end") { + assert(view.beginInventory()); + if (which == "end") view.endInventory(); + else mesh::endEsp32BootFileInventory(); + requireInactiveFallback(view); + } else if (which == "null_backend") { + fs::FS null_fs(fs::FSImplPtr{}); + mesh::Esp32BootFileSystem null_view(null_fs); + assert(!null_view.beginInventory() && !null_view.isInventoryActive()); + assert(peripheral.directory_opens == 0 && peripheral.allocations == 0); + } else if (which == "other_mount") { + native->mountpoint("/sd"); + mesh::Esp32BootFileSystem other_view(original); + assert(!other_view.beginInventory() && !other_view.isInventoryActive()); + assert(peripheral.directory_opens == 0 && peripheral.allocations == 0); + } else if (which == "dirty_errno") { + populate(*native, {"present"}); + peripheral.dirty_success_errno = true; + assert(view.beginInventory()); // Stale errno from a successful read is not an EOF error. + assert(view.isInventoryActive() && !view.exists("/missing")); + assert(peripheral.directory_closes == 1); + } else if (which == "empty") { + assert(view.beginInventory()); + assert(view.isInventoryActive() && !view.exists("/missing") && !view.open("/missing")); + assert(peripheral.native_exists == 0 && peripheral.native_opens == 0); + assert(peripheral.allocations == 1 && peripheral.frees == 1 && peripheral.reallocations == 0); + assert(inventory_buffers.empty()); + } else { + populate(*native, {"present"}); + if (which == "open_error") peripheral.fail_open = true; + else if (which == "close_error") peripheral.fail_close = true; + else if (which == "read_error_first") peripheral.fail_read_at = 0; + else if (which == "read_error_partial") peripheral.fail_read_at = 1; + else if (which == "oom") peripheral.fail_malloc = true; + else if (which == "shrink_failure") peripheral.fail_realloc = true; + else if (which == "overflow") { + peripheral.names.clear(); + for (unsigned i = 0; i < 400; ++i) + peripheral.names.push_back(std::string(25, 'x') + std::to_string(i)); + } else if (which == "bad_name_empty") peripheral.names = {""}; + else if (which == "bad_name_dot") peripheral.names = {"../identity"}; + else if (which == "bad_name_repeat") peripheral.names = {"identity//key"}; + else if (which == "bad_name_absolute") peripheral.names = {"//identity/key"}; + else if (which == "bad_name_long") peripheral.names = {std::string(31, 'x')}; + else if (which == "bad_name_unterminated") peripheral.unterminated_name = true; + else assert(false); + assert(!view.beginInventory()); + assert(inventory_buffers.empty()); + if (which == "shrink_failure") { + assert(peripheral.reallocations == 1 && peripheral.frees == 1); + assert(peripheral.shrink_bytes == strlen("/present") + 1); + } + requireInactiveFallback(view); + if (peripheral.directory_opens && !peripheral.fail_open) + assert(peripheral.directory_closes == 1); + } +} +''' + + +def compile_boot_inventory_harness(work, mutation=None): + """Return a compiled actual-class harness; all generated files live in work.""" + work = Path(work) + compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++") + if compiler is None: + raise unittest.SkipTest("a host C++17 compiler is required") + (work / "sys").mkdir(exist_ok=True) + (work / "FS.h").write_text(FS_HEADER, encoding="utf-8") + (work / "FSImpl.h").write_text('#include "FS.h"\n', encoding="utf-8") + (work / "Arduino.h").write_text('#include "FS.h"\n', encoding="utf-8") + (work / "sdkconfig.h").write_text('#define CONFIG_SPIFFS_OBJ_NAME_LEN 32\n', encoding="utf-8") + (work / "dirent.h").write_text(DIRENT_HEADER, encoding="utf-8") + (work / "sys/stat.h").write_text(STAT_HEADER, encoding="utf-8") + transaction = (ROOT / "src/helpers/ContactFileTransaction.h").read_text(encoding="utf-8") + probe = extract_braced(transaction, "static bool probe(") + recover = extract_braced(transaction, "static bool recover(") + common_recover = extract_braced( + (ROOT / "src/helpers/CommonCLI.cpp").read_text(encoding="utf-8"), + "bool CommonCLI::recoverCommonPrefsFiles(") + harness = HARNESS.replace("@PROBE@", probe).replace("@RECOVER@", recover).replace( + "@COMMON_RECOVER@", common_recover) + cpp = work / "harness.cpp" + cpp.write_text(harness, encoding="utf-8") + # Include real production under allocation-boundary substitution. Standard + # declarations are read first; no production logic is replaced by a stub. + production = PRODUCTION.read_text(encoding="utf-8") + if mutation is not None: + production = mutation(production) + wrapped = work / "production.cpp" + wrapped.write_text( + '#include \n#include \n' + 'void* inventory_malloc(size_t);\nvoid* inventory_realloc(void*, size_t);\n' + 'void inventory_free(void*);\n' + 'namespace std { using ::inventory_malloc; using ::inventory_realloc; using ::inventory_free; }\n' + '#define malloc inventory_malloc\n#define realloc inventory_realloc\n#define free inventory_free\n' + production, + encoding="utf-8") + binary = work / "inventory" + compiled = subprocess.run([ + compiler, "-std=c++17", "-Wall", "-Wextra", "-Werror", "-DESP32_PLATFORM=1", + *(["-fsanitize=address,undefined", "-fno-sanitize-recover=all", "-fno-pie", "-no-pie"] + if sys.platform.startswith("linux") else []), + f"-I{work}", f"-I{ROOT / 'src'}", f"-I{PRODUCTION.parent}", + str(cpp), str(wrapped), "-o", str(binary), + ], capture_output=True, text=True, timeout=60) + if compiled.returncode != 0: + raise AssertionError(compiled.stdout + compiled.stderr) + return binary + + +def run_boot_inventory_timing(disable_inventory=False): + """Return modeled readiness and native lookup counts for GUI deadline tests.""" + with tempfile.TemporaryDirectory(prefix="boot-inventory-timing-") as folder: + binary = compile_boot_inventory_harness(folder) + case = "timing_off" if disable_inventory else "timing_on" + result = subprocess.run([str(binary), case], cwd=folder, + capture_output=True, text=True, timeout=10) + if result.returncode != 0: + raise AssertionError(result.stdout + result.stderr) + return json.loads(result.stdout) + + +class Esp32BootFileInventoryTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.folder = tempfile.TemporaryDirectory(prefix="boot-inventory-") + cls.addClassCleanup(cls.folder.cleanup) + cls.binary = compile_boot_inventory_harness(cls.folder.name) + + def run_case(self, case): + result = subprocess.run([str(self.binary), case], cwd=self.folder.name, + capture_output=True, text=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return result.stdout + + def test_missing_positive_transaction_and_directory_paths(self): + for case in ("basic", "empty", "rename_backup", "rename_temp", "positive_fault", "positive_directory"): + with self.subTest(case=case): + self.run_case(case) + + def test_exact_owner_identity_and_permanent_end(self): + for case in ("owner", "end", "global_end", "null_backend", "other_mount", "end_before_begin"): + with self.subTest(case=case): + self.run_case(case) + + def test_mutation_invalidates_before_delegation_even_failure(self): + self.run_case("mutation_before_begin") + for kind in ("rplus", "rbplus", "write", "append", "unknown", "null", "create", + "open_fail", "rename_fail", "remove_fail", "mkdir", "rmdir"): + with self.subTest(kind=kind): + self.run_case("mutation_" + kind) + + def test_complete_eof_is_required_and_stale_errno_is_cleared(self): + for case in ("open_error", "read_error_first", "read_error_partial", "close_error", "dirty_errno"): + with self.subTest(case=case): + self.run_case(case) + + def test_oom_capacity_and_malformed_names_fall_back(self): + for case in ("oom", "shrink_failure", "overflow", "bad_name_empty", "bad_name_dot", "bad_name_repeat", + "bad_name_absolute", "bad_name_long", "bad_name_unterminated"): + with self.subTest(case=case): + self.run_case(case) + + def test_native_lookup_cost_crosses_unchanged_gui_deadline(self): + active = json.loads(self.run_case("timing_on")) + disabled = json.loads(self.run_case("timing_off")) + self.assertLess(active["elapsed_ms"], 5000) + self.assertGreaterEqual(disabled["elapsed_ms"], 5000) + self.assertEqual(active["native_exists"], 0) + self.assertEqual(active["metadata_lookups"], 1) + self.assertGreater(disabled["native_exists"], 0) + + def test_compiled_negative_controls_detect_missing_guards(self): + controls = ( + ("skip_negative_open", + "else if (owner_.knownAbsent(path)) return fs::FileImplPtr();", + "else if (false) return fs::FileImplPtr();", "basic", "!view.open"), + ("skip_mutation_invalidation", "owner_.endInventory();", "(void)0;", + "mutation_write", "!mesh::esp32BootFileKnownAbsent"), + ("accept_partial_scan", "complete = errno == 0;", "complete = true;", + "read_error_partial", "!view.beginInventory()"), + ("hide_positive_file", "strcmp(name, path) == 0", "false", + "basic", "view.exists(path)"), + ) + for name, before, after, case, expected_assertion in controls: + with self.subTest(control=name): + def mutate(source, before=before, after=after): + self.assertIn(before, source) + return source.replace(before, after) + + with tempfile.TemporaryDirectory(prefix="boot-inventory-negative-") as folder: + binary = compile_boot_inventory_harness(folder, mutate) + result = subprocess.run([str(binary), case], cwd=folder, + capture_output=True, text=True, timeout=10) + self.assertNotEqual(result.returncode, 0) + self.assertIn(expected_assertion, result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_esp32_boot_file_lifecycle.py b/test/test_esp32_boot_file_lifecycle.py new file mode 100644 index 00000000..00578d7c --- /dev/null +++ b/test/test_esp32_boot_file_lifecycle.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Keep the boot-only FS view outside identity writes and asynchronous tasks.""" +from pathlib import Path +import configparser +import re +import unittest + +from test_replay_reset_integration import extract_braced + +ROOT = Path(__file__).resolve().parents[1] +ROLES = ("simple_repeater", "simple_room_server") + + +def role_sources(role): + path = ROOT / "examples" / role + return (extract_braced((path / "main.cpp").read_text(), "void setup()"), + extract_braced((path / "MyMesh.cpp").read_text(), + "void MyMesh::begin(FILESYSTEM *fs)")) + + +def require_safe_lifecycle(main, mesh): + # Cache contents witness the completed durable identity, including recovery + # and a possible first write. The failed-identity path returns beforehand. + identity = main.index("if (!identity_ready)") + failure = extract_braced(main, "if (!identity_ready)") + assert "board.reboot();" in failure and "return;" in failure + inventory = main.index("static mesh::Esp32BootFileSystem boot_fs(*fs);") + assert identity + len(failure) < inventory + assert inventory < main.index("fs = &boot_fs;") < main.index("boot_fs.beginInventory();") + assert main.index("boot_fs.beginInventory();") < main.index("the_mesh.begin(fs);") + assert main[main.rfind("#if", 0, inventory):].startswith("#if defined(ESP32_PLATFORM)") + # The main fallback runs before boot completion; MyMesh must end earlier + # because its bridge may already own a task when begin() returns. + assert main.index("mesh::endEsp32BootFileInventory();") < main.index("board.onBootComplete();") + stop = mesh.index("mesh::endEsp32BootFileInventory();") + assert mesh[mesh.rfind("#if", 0, stop):].startswith("#if defined(ESP32_PLATFORM)") + for token in ("new MQTTBridge(", "active_bridge->begin();", "bridge->begin();", + "beginRS232Bridge()", "setEspNowBridgeState(true)", + "startWebConfig(false, wc_reply)"): + if token in mesh: + assert stop < mesh.index(token), token + assert mesh.index("_cli.loadPrefs(_fs);") < stop + assert mesh.index("acl.load(_fs, self_id);") < stop + + +class BootFileLifecycleTests(unittest.TestCase): + def test_ordinary_esp32_role_filters_include_the_inventory_implementation(self): + config = configparser.ConfigParser(interpolation=None, strict=False) + config.read([str(ROOT / "platformio.ini"), + *(str(path) for path in sorted((ROOT / "variants").glob("*/platformio.ini")))]) + + def resolve(section, option): + if option in config[section]: + value = config[section][option] + else: + parent = config[section].get("extends", "").split(",")[0].strip() + assert parent, (section, option) + return resolve(parent, option) + return re.sub(r"\$\{([^}]+)\}", + lambda match: resolve(*match.group(1).rsplit(".", 1)), value) + + targets = ("env:Xiao_C6_repeater_", "env:Heltec_v2_repeater", + "env:heltec_v4_room_server", "env:Xiao_S3_WIO_repeater") + for target in targets: + with self.subTest(target=target): + source_filter = resolve(target, "build_src_filter") + assert "+" in source_filter + + def test_both_roles_keep_static_view_after_identity_and_before_tasks(self): + for role in ROLES: + with self.subTest(role=role): + require_safe_lifecycle(*role_sources(role)) + + def test_missing_task_boundary_is_detected(self): + for role in ROLES: + main, mesh = role_sources(role) + mesh = mesh.replace("mesh::endEsp32BootFileInventory();", "") + with self.subTest(role=role), self.assertRaises((ValueError, AssertionError)): + require_safe_lifecycle(main, mesh) + + def test_short_lived_view_is_detected(self): + for role in ROLES: + main, mesh = role_sources(role) + main = main.replace("static mesh::Esp32BootFileSystem", "mesh::Esp32BootFileSystem") + with self.subTest(role=role), self.assertRaises((ValueError, AssertionError)): + require_safe_lifecycle(main, mesh) + + def test_inventory_before_identity_is_detected(self): + for role in ROLES: + main, mesh = role_sources(role) + token = "static mesh::Esp32BootFileSystem boot_fs(*fs);" + main = main.replace(token, "").replace("void setup() {", "void setup() {\n" + token) + with self.subTest(role=role), self.assertRaises((ValueError, AssertionError)): + require_safe_lifecycle(main, mesh) + + def test_missing_boot_complete_fallback_is_detected(self): + for role in ROLES: + main, mesh = role_sources(role) + main = main.replace("mesh::endEsp32BootFileInventory();", "") + with self.subTest(role=role), self.assertRaises((ValueError, AssertionError)): + require_safe_lifecycle(main, mesh) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_identity_and_settings_recovery.py b/test/test_identity_and_settings_recovery.py index e23cd664..f91d4bd5 100644 --- a/test/test_identity_and_settings_recovery.py +++ b/test/test_identity_and_settings_recovery.py @@ -69,6 +69,10 @@ class IdentityAndSettingsRecovery(unittest.TestCase): source = (ROOT / 'src/helpers/IdentityStore.cpp').read_text() presence = (ROOT / 'src/helpers/FilePresence.h').read_text() program = fixture_prefix() + '\n#include \n' + if platform == 'ESP32_PLATFORM': + # Identity startup precedes the inventory. Include its real owner + # registry so the extracted metadata probe takes the normal path. + program += '#include \n' if platform == 'ESP32_PLATFORM': program += r''' struct MigrationNvs {