Suspend failed ESP32 MQTT and NTP uplinks for power saving

Share one bounded WiFi outage and recovery schedule across MQTT, automatic NTP, Companion station ownership and WebConfig. Preserve enabled settings and foreground access while allowing normal sleep after confirmed physical shutdown.

Honor failed SDK client stops before handing off WiFi, protect clean teardown and reconfiguration, and retry transient ESP-NOW channel restoration without repeated disconnects. Add production-method, SDK failure and real sleep regressions to CI.

Validation: native MQTT/NTP/clock/reconnect suites; production power and wrapper failure tests; browser, room HTTP, OTA and combined bridge regressions; Full Heltec V4 repeater, room and companion builds with capability, flash and linked-memory checks.
This commit is contained in:
mikecarper committed 2026-10-12 14:57:01 -07:00
1 parent f62cc7b527
commit 6b3f7df4d0
37 files changed
+3258 -82

No files matched your search

+10 -2
View File
@@ -7,6 +7,7 @@
#include "helpers/AlertFaultPolicy.h"
#include "helpers/MQTTConnectionPolicy.h"
#include "helpers/WiFiPowerSave.h"
#include "helpers/WiFiNetworkPowerPolicy.h"
#define MQTT_DEBUG_PRINTLN(...) ((void)0)
#define pdMS_TO_TICKS(value) (value)
using TickType_t=uint32_t;
@@ -69,6 +70,9 @@ namespace mesh{namespace wifi{
int s_wifi_disconnect_reason=0;uint32_t s_wifi_disconnect_time=0;
unsigned long s_wifi_connected_at=0;
struct MQTTBridge{
mesh::wifi::NetworkPowerPolicy _network_power_policy;
std::atomic<bool>_network_suspended{false};
bool _network_espnow_restore_pending=false;
bool _wifi_event_registered=false,_manage_wifi=true,_ntp_sync_pending=false;
int _wifi_reconnect_backoff_attempt=0;
std::atomic<bool>_ntp_synced{true},_stop_requested{false},_stop_acked{false};
@@ -78,13 +82,14 @@ struct MQTTBridge{
void setWifiOutage(AlertFaultPolicy::OutageSnapshot value){outage=value;}
char _wifi_ssid[8]="mesh",_wifi_password[8]="secret";
void beginWiFiStation();
void applyTimezoneRules() {}
bool _wifi_status_initialized=true;
int _last_wifi_status=WL_DISCONNECTED;
unsigned long _last_wifi_check=0,_last_wifi_reconnect_attempt=0;
uint8_t _wifi_power_save=1;
struct{bool canonical_wifi=true;}_node_info;
struct{uint8_t wifi_power_save=1;}prefs,*_obs=&prefs;
struct Client{void disconnect(){}};
struct Client{void disconnect(){}bool isStarted()const{return false;}};
struct Slot{Client* client=nullptr;bool connected=false;}_slots[RUNTIME_MQTT_SLOTS];
bool _slot_attempt_pending[RUNTIME_MQTT_SLOTS]={};
std::vector<std::string> cleanup;
@@ -96,8 +101,11 @@ struct MQTTBridge{
bool handleWiFiConnection(unsigned long now);
void mqttTaskLoop(){
@TASK_START@
while(true){
@TASK_STOP@
++slot_work;
++slot_work;
break;
}
}
};
@INITIALIZE@
+35 -2
View File
@@ -113,6 +113,7 @@ struct MQTTNodeInfo {
class MQTTBridge : public AbstractBridge {
bool running = false;
bool stopping = false;
bool network_suspended = false;
public:
inline static unsigned live = 0, fail_starts = 0;
unsigned begins = 0, ends = 0, loops = 0, sent = 0, received = 0;
@@ -127,12 +128,21 @@ public:
assert(filters); ++begins;
if (fail_starts) { --fail_starts; return; }
running = true;
network_suspended = false;
}
void end() override { ++ends; running = stopping = false; }
void end() override { ++ends; running = stopping = network_suspended = false; }
void requestStop() { stopping = running; }
bool isStopping() const { return stopping; }
bool isRunning() const override { return running; }
void loop() override { ++loops; if (stopping) { ++ends; running = stopping = false; } }
bool isNetworkSuspended() const { return network_suspended; }
void setNetworkSuspended(bool paused) {
assert(running && !stopping);
network_suspended = paused;
}
void loop() override {
++loops;
if (stopping) { ++ends; running = stopping = network_suspended = false; }
}
void sendPacket(mesh::Packet* p) override { if (allow(p)) ++sent; }
void onPacketReceived(mesh::Packet* p) { if (allow(p)) ++received; }
};
@@ -270,6 +280,29 @@ int main() {
expect_get(cli, "bridge.type", "> mqtt+rs232+espnow");
expect_get(cli, "rs232.enabled", "> off");
expect_get(cli, "rs232.running", "> off");
fresh.mqtt_bridge->setNetworkSuspended(true);
assert(fresh.isMqttBridgeRunning() && fresh._prefs.bridge_enabled);
assert(fresh.bridgesPreventSleep()); // ESP-NOW retains its radio owner.
assert(fresh.setEspNowBridgeState(false));
assert(!fresh.bridgesPreventSleep()); // Paused MQTT is still configured.
assert(fresh.setRs232BridgeState(true));
assert(fresh.bridgesPreventSleep()); // An active UART remains independent.
assert(fresh.setRs232BridgeState(false));
assert(!fresh.bridgesPreventSleep());
fresh.mqtt_bridge->setNetworkSuspended(false);
assert(fresh.bridgesPreventSleep()); // The recovery probe owns the radio.
char pause_reply[256] = {};
cli.set("espnow.enabled off", pause_reply);
assert(strcmp(pause_reply, "OK") == 0);
fresh.mqtt_bridge->setNetworkSuspended(true);
assert(fresh.requestMqttBridgeStop());
assert(fresh.isMqttBridgeStopping() && fresh.bridgesPreventSleep());
fresh.serviceBridges();
assert(!fresh.isMqttBridgeRunning() && !fresh.bridgesPreventSleep());
assert(fresh.setBridgeState(true));
assert(fresh.isMqttBridgeRunning() && fresh.bridgesPreventSleep());
cli.set("espnow.enabled on", pause_reply);
assert(strcmp(pause_reply, "OK") == 0 && fresh.isEspNowBridgeRunning());
}
MyMesh mesh; mesh.defaults(); mesh.boot(); CommonCLI cli(mesh);
char reply[256] = {};
+2
View File
@@ -337,6 +337,8 @@ struct WebConfigServer {
bool auth_allowed = true;
~WebConfigServer() { delete _dns; }
void createServer() { ++servers; }
void requestNetworkAccess() {}
void refreshNetworkPowerHold(uint32_t) {}
bool startSetupMode(char reply[]);
bool checkAuth(AsyncWebServerRequest*) const { return auth_allowed; }
void handleScan(AsyncWebServerRequest* req);
+13 -1
View File
@@ -5,11 +5,14 @@
#include <cstring>
#include <initializer_list>
#include <helpers/RoomWebMailbox.h>
#include <helpers/WiFiNetworkPowerPolicy.h>
static uint32_t millis() { return 123; }
struct WebConfigBatch {
static uint32_t scheduleAt(uint32_t now, uint32_t delay) { return now + delay; }
static bool deadlineReached(uint32_t now, uint32_t at) { return int32_t(now - at) >= 0; }
};
struct WCLock { explicit WCLock(void*) {} };
constexpr int WIFI_STA = 1;
constexpr int WIFI_AP = 2;
struct WiFiFake {
@@ -29,7 +32,7 @@ struct DNS { void stop() {} };
struct Callbacks { unsigned stops = 0; void onWebConfigStopped() { ++stops; } };
struct WebConfigServer {
enum Mode { MODE_OFF, MODE_LAN, MODE_SETUP };
enum { STOP_WARN_MS = 1000, BATCH_IDLE = 0, BATCH_CONFIG = 0 };
enum { STOP_WARN_MS = 1000, BATCH_IDLE = 0, BATCH_PENDING = 1, BATCH_CONFIG = 0 };
Server server;
Server* _server = &server;
DNS* _dns = nullptr;
@@ -46,11 +49,18 @@ struct WebConfigServer {
uint32_t _setup_reconnect_deadline = 0, _reboot_at = 0, _batch_state = 0;
uint32_t _batch_next = 0, _setup_wifi_handoff_deadline = 0, _batch_kind = 0;
uint32_t _board_cmds = 0;
uint32_t _network_activity_until = 0;
void* _mux = nullptr;
struct Terminal { bool running = false; struct { bool pending = false; } queue; };
Terminal* _terminal = nullptr;
char _wifi_ssid[32] = {}, _setup_wifi_handoff_ip[16] = {};
char _session_token[16] = {}, _stats_json[16] = {};
bool routes = true;
void detachRoutes() { routes = false; }
void closeTerminal() {}
uint32_t handlerRefCount() const { return 0; }
bool isNetworkForegroundActive(uint32_t) const;
void refreshNetworkPowerHold(uint32_t);
bool isRunning() const { return _mode != MODE_OFF; }
bool isStopping() const { return _stopping; }
void requestStop();
@@ -75,6 +85,7 @@ int main() {
assert(portal.stopForOTA(reply));
assert(strcmp(reply, "WebConfig stopped") == 0);
assert(!portal.routes && !portal.server.listening && portal.isStopping());
assert(mesh::wifi::networkPower().foregroundActive());
assert(WiFi.sta && !WiFi.ap && WiFi.stops == 0);
assert(WiFi.connected == !setup);
// OTA can now bind port 80 and create its own AP, while HTTP request
@@ -86,6 +97,7 @@ int main() {
assert(WiFi.ap && WiFi.sta && WiFi.stops == 0);
assert(WiFi.ap_stops == unsigned(setup));
assert(portal.callbacks.stops == 1 && !portal._keep_wifi_on_stop);
assert(!mesh::wifi::networkPower().foregroundActive());
}
}
WebConfigServer ordinary;
+5
View File
@@ -47,6 +47,8 @@ struct WebConfigServer {
uint32_t _last_activity=0, _session_last_seen=0, _login_lock_until=0;
int _login_fails=0; Callback callback; Callback* _cb=&callback;
WebConfigServer() { strcpy(callback.node.admin_password, "existing-secret"); }
unsigned network_activity = 0;
void noteNetworkActivity() { ++network_activity; }
bool checkAuth(AsyncWebServerRequest*);
void handleLogin(AsyncWebServerRequest*);
void handleLogout(AsyncWebServerRequest*);
@@ -58,12 +60,15 @@ int main() {
attacker.body="{\"set\":{\"password\":\"attacker\",\"wifi.ssid\":\"EvilNet\"}}";
attacker._tempObject=attacker.body.data();
assert(!portal.checkAuth(&attacker));
assert(portal.network_activity==0);
portal.checkConfigPost(&attacker);
assert(attacker.status==401 && !portal.config_accepted);
attacker.password("wrong"); portal.handleLogin(&attacker);
assert(attacker.status==401 && portal._session_token[0]==0);
assert(portal.network_activity==0);
AsyncWebServerRequest admin; admin.password("existing-secret");
portal.handleLogin(&admin); assert(admin.status==200);
assert(portal.network_activity>0);
assert(strlen(portal._session_token)==32);
assert(admin.response.headers["Set-Cookie"].find("HttpOnly; SameSite=Lax")!=std::string::npos);
admin.header.cookies=std::string("wcs=")+portal._session_token;
+1
View File
@@ -23,6 +23,7 @@ struct WebConfigServer {
bool isRunning() const { return false; }
bool isStopping() const { return false; }
void updateWiFiOwnership(bool owner) { owns = owner; }
void requestNetworkAccess() {}
bool startAutoMode(char* reply) { strcpy(reply, "starting"); return true; }
bool startSetupMode(char* reply) { strcpy(reply, "setup"); return true; }
};
+30
View File
@@ -6,6 +6,7 @@
#include <string>
#include <helpers/RoomWebMailbox.h>
#include <helpers/WiFiReconnectPolicy.h>
#include <helpers/WiFiNetworkPowerPolicy.h>
static uint32_t clock_ms = 1000;
uint32_t millis() { return clock_ms; }
void delay(unsigned) {}
@@ -96,6 +97,7 @@ public:
bool _stopping = false, _retry_saved_wifi_in_setup = false;
bool _setup_reconnect_in_progress = false, _board_cmds_probed = false;
bool _was_setup_ap = false, _initial_setup = false;
bool foreground = false;
uint32_t _setup_reconnect_deadline = 0, _connect_deadline = 0;
uint32_t _setup_started_at = 0, _last_activity = 0;
WiFiReconnectPolicy::Tracker _wifi_reconnect_tracker;
@@ -134,6 +136,7 @@ public:
void probeBoardCommands() { _board_cmds_probed = true; }
void serviceTerminal(uint32_t) {}
void serviceRoom(uint32_t) {}
bool isNetworkForegroundActive(uint32_t) const { return foreground || _mode == MODE_SETUP; }
bool bluetoothWiFiCoexistenceRequired() { return false; }
bool acceptWiFi(Entry& e, const char* value) {
@CREDENTIAL_BATCH@
@@ -197,6 +200,33 @@ int main() {
companion.tick(16000);
assert(companion.setup_starts == 1 && companion._retry_saved_wifi_in_setup);
assert(!companion._runtime_wifi_ownership_managed);
#elif defined(TEST_BACKGROUND_SUSPEND)
auto& power = mesh::wifi::networkPower();
power.setManaged(true); power.setSuspended(true);
portal._mode = WebConfigServer::MODE_LAN;
clock_ms = 301000;
portal.tick(clock_ms);
assert(WiFi.begins == 0 && portal.setup_starts == 0);
// A connecting listener must not reopen an AP or acquire STA while the
// background owner has deliberately stopped MQTT and NTP for the cooldown.
portal._mode = WebConfigServer::MODE_CONNECTING;
portal._connect_deadline = 16000;
portal.tick(clock_ms);
assert(WiFi.begins == 0 && portal.setup_starts == 0);
portal._connect_deadline = 0;
portal.foreground = true;
portal.tick(++clock_ms);
assert(WiFi.begins == 1 && portal._connect_deadline == clock_ms + 15000);
portal.foreground = false;
clock_ms += 16000;
portal.tick(clock_ms);
assert(portal.setup_starts == 0 && WiFi.begins == 1);
// Resuming the worker permits ordinary ownership/reconnect logic again.
power.setSuspended(false);
portal._mode = WebConfigServer::MODE_LAN;
clock_ms += 301000;
portal.tick(clock_ms);
assert(WiFi.begins == 2);
#elif defined(TEST_EXTERNAL_OWNER)
portal.updateWiFiOwnership(false);
assert(portal._runtime_wifi_ownership_managed);
+474
View File
@@ -0,0 +1,474 @@
"""Execute Companion's production outage transitions without hardware."""
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import unittest
from test_replay_reset_integration import extract_braced
ROOT = Path(__file__).resolve().parents[1]
MAIN = ROOT / "examples/companion_radio/main.cpp"
HARNESS = r'''
#include <cassert>
#include <cstdint>
#include <cstring>
#include <helpers/WiFiNetworkPowerPolicy.h>
namespace mesh { namespace wifi {
bool espnow = false, station_allowed = true;
int protocols = 0, channels = 0, associations = 0, restore_result = 0;
bool espNowChannelConstrained() { return espnow; }
bool stationMutationAllowed() { return station_allowed; }
void applyProtocolMask(int) { ++protocols; }
int restoreEspNowChannel() { ++channels; return restore_result; }
void setStationAutoReconnect(bool);
void beginStation(const char*, const char*) { ++associations; }
}}
constexpr int WIFI_STA = 1, WIFI_OFF = 0, WIFI_IF_STA = 0, TCP_PORT = 5000;
constexpr int WL_CONNECTED = 3, WL_DISCONNECTED = 6;
constexpr int ESP_OK = 0;
uint32_t test_now = 10000;
uint32_t millis() { return test_now; }
struct Board {
bool ota = false, inhibit = true;
bool isOTAUpdateRunning() const { return ota; }
void setInhibitSleep(bool value) { inhibit = value; }
} board;
struct Interface {
bool connected = false, enabled = true;
int ends = 0, begins = 0;
bool isConnected() const { return connected; }
void end() { enabled = false; ++ends; }
void begin(int) { ++begins; }
void enable() { enabled = true; }
} wifi_interface;
struct ConsoleClient {
bool active = false;
bool connected() const { return active; }
} ota_console_client;
struct Wifi {
int current_mode = WIFI_STA;
int current_status = WL_DISCONNECTED;
bool reconnect = true, last_disconnect_stopped = false;
int disconnects = 0;
void setAutoReconnect(bool value) { reconnect = value; }
void disconnect(bool stop, bool) {
last_disconnect_stopped = stop;
++disconnects;
}
void mode(int value) { current_mode = value; }
int status() const { return current_status; }
} WiFi;
void mesh::wifi::setStationAutoReconnect(bool value) { WiFi.setAutoReconnect(value); }
struct Tracker { void noteDisconnected(uint32_t) {} } wifi_reconnect_tracker;
bool companion_wifi_active = true, companion_wifi_requested = true;
bool companion_wifi_background_stopped = false;
bool companion_wifi_espnow_restore_pending = false;
uint32_t companion_wifi_espnow_restore_at = 0;
bool companion_wifi_webconfig_start_pending = false;
bool companion_wifi_credential_reload_pending = false;
bool companion_wifi_setup_requested = false;
bool companion_wifi_has_credentials = true;
char configured_wifi_ssid[] = "saved-network";
char configured_wifi_password[] = "saved-password";
int ntp_cancels = 0, resets = 0, ps_applied = 0, console_starts = 0, console_stops = 0;
void cancelCompanionWiFiNtp(bool retry) { assert(!retry); ++ntp_cancels; }
void resetCompanionWiFiRecoveryState() { ++resets; }
void applyCompanionWiFiPowerSave() { ++ps_applied; }
void ota_console_start() { ++console_starts; }
void ota_console_stop() { ++console_stops; }
#define WIFI_DEBUG_PRINTLN(...) do {} while (0)
struct WebConfigServer {
static bool enabled;
static bool loadEnabled(bool) { return enabled; }
};
bool WebConfigServer::enabled = true;
struct Mesh {
bool web_active = false, last_force_ap = false;
int web_starts = 0;
bool isWebConfigActiveOrStopping() const { return web_active; }
bool startWebConfig(bool force_ap, char*) {
++web_starts; last_force_ap = force_ap; web_active = true; return true;
}
} the_mesh;
@METHODS@
void reset() {
auto& power = mesh::wifi::networkPower();
power.setManaged(false);
power.setHold(mesh::wifi::NetworkPowerHold::CompanionClient, false);
power.setManaged(true);
power.setSuspended(false);
mesh::wifi::espnow = false;
mesh::wifi::station_allowed = true;
mesh::wifi::protocols = mesh::wifi::channels = mesh::wifi::associations = 0;
mesh::wifi::restore_result = 0;
test_now = 10000;
board = {}; wifi_interface = {}; ota_console_client = {}; WiFi = {};
companion_wifi_active = companion_wifi_requested = true;
companion_wifi_background_stopped = false;
companion_wifi_espnow_restore_pending = false;
companion_wifi_espnow_restore_at = 0;
companion_wifi_webconfig_start_pending = false;
companion_wifi_credential_reload_pending = companion_wifi_setup_requested = false;
companion_wifi_has_credentials = true;
ntp_cancels = resets = ps_applied = console_starts = console_stops = 0;
the_mesh = {}; WebConfigServer::enabled = true;
}
int main() {
reset();
// Merely listening for a client does not hold the radio on.
serviceCompanionWiFiNetworkPowerHold();
assert(!mesh::wifi::networkPower().foregroundActive());
wifi_interface.connected = true;
serviceCompanionWiFiNetworkPowerHold();
assert(mesh::wifi::networkPower().foregroundActive());
wifi_interface.connected = false;
#ifdef ENABLE_OTA
ota_console_client.active = true;
serviceCompanionWiFiNetworkPowerHold();
assert(mesh::wifi::networkPower().foregroundActive());
ota_console_client.active = false;
#endif
companion_wifi_credential_reload_pending = true;
serviceCompanionWiFiNetworkPowerHold();
assert(mesh::wifi::networkPower().foregroundActive());
companion_wifi_credential_reload_pending = false;
companion_wifi_setup_requested = true;
serviceCompanionWiFiNetworkPowerHold();
assert(mesh::wifi::networkPower().foregroundActive());
companion_wifi_setup_requested = false;
board.ota = true;
serviceCompanionWiFiNetworkPowerHold();
assert(mesh::wifi::networkPower().foregroundActive());
reset();
mesh::wifi::networkPower().setSuspended(true);
serviceCompanionWiFiBackgroundPower();
assert(companion_wifi_background_stopped);
assert(companion_wifi_active && companion_wifi_requested);
assert(!board.inhibit && !WiFi.reconnect);
assert(WiFi.current_mode == WIFI_OFF && WiFi.last_disconnect_stopped);
assert(wifi_interface.ends == 1 && ntp_cancels == 1);
#ifdef ENABLE_OTA
assert(console_stops == 1);
#endif
for (int i = 0; i < 100; ++i) serviceCompanionWiFiBackgroundPower();
assert(wifi_interface.ends == 1 && ntp_cancels == 1);
assert(mesh::wifi::associations == 0);
assert(strcmp(configured_wifi_ssid, "saved-network") == 0);
assert(strcmp(configured_wifi_password, "saved-password") == 0);
// The worker owns the cooldown. Once it opens a recovery window, the
// externally owned Companion station starts exactly one saved association.
mesh::wifi::networkPower().setSuspended(false);
mesh::wifi::station_allowed = false;
serviceCompanionWiFiBackgroundPower();
assert(companion_wifi_background_stopped && mesh::wifi::associations == 0);
mesh::wifi::station_allowed = true;
serviceCompanionWiFiBackgroundPower();
assert(!companion_wifi_background_stopped && board.inhibit);
assert(wifi_interface.enabled && wifi_interface.begins == 1);
assert(mesh::wifi::associations == 1 && WiFi.reconnect);
assert(WiFi.current_mode == WIFI_STA && ps_applied == 1);
#ifdef ENABLE_OTA
assert(console_starts == 1);
#endif
for (int i = 0; i < 100; ++i) serviceCompanionWiFiBackgroundPower();
assert(mesh::wifi::associations == 1 && wifi_interface.begins == 1);
// An active updater remains powered even if it starts between the worker's
// last ownership check and publication of the suspension flag.
reset();
mesh::wifi::networkPower().setSuspended(true);
board.ota = true;
serviceCompanionWiFiBackgroundPower();
assert(!companion_wifi_background_stopped && board.inhibit);
assert(wifi_interface.ends == 0 && WiFi.disconnects == 0);
// Primary and dynamically enabled ESP-NOW both need their driver/channel.
reset();
mesh::wifi::networkPower().setSuspended(true);
mesh::wifi::espnow = true;
serviceCompanionWiFiBackgroundPower();
assert(companion_wifi_background_stopped && !WiFi.reconnect);
assert(board.inhibit);
assert(WiFi.current_mode == WIFI_STA && !WiFi.last_disconnect_stopped);
assert(mesh::wifi::protocols == 1 && mesh::wifi::channels == 1);
// Disconnect is asynchronous. The first restoration can fail until the
// station/scan ends; keep retrying the channel, never the disconnect.
reset();
mesh::wifi::networkPower().setSuspended(true);
mesh::wifi::espnow = true;
mesh::wifi::restore_result = 1;
serviceCompanionWiFiBackgroundPower();
assert(companion_wifi_espnow_restore_pending && WiFi.disconnects == 1);
for (int i = 0; i < 100; ++i) serviceCompanionWiFiBackgroundPower();
assert(mesh::wifi::channels == 1 && WiFi.disconnects == 1);
test_now += 999;
serviceCompanionWiFiBackgroundPower();
assert(mesh::wifi::channels == 1);
++test_now;
serviceCompanionWiFiBackgroundPower();
assert(mesh::wifi::channels == 2 && companion_wifi_espnow_restore_pending);
assert(WiFi.disconnects == 1 && mesh::wifi::protocols == 1);
test_now += 1000;
mesh::wifi::station_allowed = false; // a foreground AP owns the channel
serviceCompanionWiFiBackgroundPower();
assert(mesh::wifi::channels == 2);
mesh::wifi::station_allowed = true;
board.ota = true;
serviceCompanionEspNowChannelRestore();
assert(mesh::wifi::channels == 2);
board.ota = false;
mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::CompanionClient, true);
serviceCompanionEspNowChannelRestore();
assert(mesh::wifi::channels == 2);
mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::CompanionClient, false);
mesh::wifi::restore_result = ESP_OK;
serviceCompanionWiFiBackgroundPower();
assert(mesh::wifi::channels == 3 && !companion_wifi_espnow_restore_pending);
test_now += 10000;
serviceCompanionWiFiBackgroundPower();
assert(mesh::wifi::channels == 3 && WiFi.disconnects == 1);
// Retry deadlines remain safe when millis wraps during the disconnect.
reset();
test_now = UINT32_MAX - 500;
mesh::wifi::networkPower().setSuspended(true);
mesh::wifi::espnow = true;
mesh::wifi::restore_result = 1;
serviceCompanionWiFiBackgroundPower();
test_now += 999;
serviceCompanionWiFiBackgroundPower();
assert(mesh::wifi::channels == 1);
++test_now;
mesh::wifi::restore_result = ESP_OK;
serviceCompanionWiFiBackgroundPower();
assert(mesh::wifi::channels == 2 && !companion_wifi_espnow_restore_pending);
// The same deferred repair runs after an intentional WiFi disable, without
// restarting that station or its listeners. A removed ESP-NOW owner cancels it.
reset();
mesh::wifi::espnow = true;
mesh::wifi::restore_result = 1;
stopCompanionInfrastructureWiFi();
companion_wifi_requested = companion_wifi_active = false;
test_now += 1000;
mesh::wifi::restore_result = ESP_OK;
serviceCompanionWiFiBackgroundPower();
assert(mesh::wifi::channels == 2 && !companion_wifi_espnow_restore_pending);
assert(WiFi.disconnects == 1 && mesh::wifi::associations == 0);
mesh::wifi::restore_result = 1;
stopCompanionInfrastructureWiFi();
mesh::wifi::espnow = false;
test_now += 1000;
serviceCompanionWiFiBackgroundPower();
assert(!companion_wifi_espnow_restore_pending && mesh::wifi::channels == 3);
// Beginning a recovery association cancels a delayed channel write.
reset();
mesh::wifi::networkPower().setSuspended(true);
mesh::wifi::espnow = true;
mesh::wifi::restore_result = 1;
serviceCompanionWiFiBackgroundPower();
mesh::wifi::networkPower().setSuspended(false);
serviceCompanionWiFiBackgroundPower();
assert(!companion_wifi_espnow_restore_pending && mesh::wifi::associations == 1);
// A client can be accepted immediately before a stale worker suspension
// flag becomes visible. Its fresh foreground hold prevents teardown.
reset();
mesh::wifi::networkPower().setSuspended(true);
wifi_interface.connected = true;
serviceCompanionWiFiNetworkPowerHold();
serviceCompanionWiFiBackgroundPower();
assert(!companion_wifi_background_stopped && WiFi.disconnects == 0);
assert(wifi_interface.ends == 0 && board.inhibit);
// User-disabled or not-yet-active WiFi is never restarted by recovery.
reset();
companion_wifi_requested = false;
mesh::wifi::networkPower().setSuspended(true);
serviceCompanionWiFiBackgroundPower();
assert(WiFi.disconnects == 0 && !companion_wifi_background_stopped);
companion_wifi_requested = true;
companion_wifi_active = false;
serviceCompanionWiFiBackgroundPower();
assert(WiFi.disconnects == 0 && !companion_wifi_background_stopped);
// A plain WiFi Companion with no managed MQTT worker keeps its existing
// connection and standalone NTP policy; a stale flag cannot suspend it.
reset();
mesh::wifi::networkPower().setSuspended(true);
mesh::wifi::networkPower().setManaged(false);
serviceCompanionWiFiBackgroundPower();
assert(WiFi.disconnects == 0 && !companion_wifi_background_stopped);
// A saved-SSID boot cannot start the portal's independent fifteen-second
// AP fallback before the MQTT worker acquires its network policy.
reset();
mesh::wifi::networkPower().setManaged(false);
startCompanionWiFiAutoWebConfig();
assert(companion_wifi_webconfig_start_pending && the_mesh.web_starts == 0);
for (int i = 0; i < 100; ++i) serviceCompanionWiFiAutoWebConfig();
assert(companion_wifi_webconfig_start_pending && the_mesh.web_starts == 0);
mesh::wifi::networkPower().setManaged(true);
assert(!companionWiFiAutomaticRecoveryAllowed());
for (int i = 0; i < 100; ++i) serviceCompanionWiFiAutoWebConfig();
assert(the_mesh.web_starts == 0);
WiFi.current_status = WL_CONNECTED;
serviceCompanionWiFiAutoWebConfig();
assert(!companion_wifi_webconfig_start_pending && the_mesh.web_starts == 1);
assert(!the_mesh.last_force_ap);
serviceCompanionWiFiAutoWebConfig();
assert(the_mesh.web_starts == 1);
// Unconfigured devices still open setup, and unmanaged standalone WiFi
// retains its ordinary recovery AP. Explicit manual setup is independent.
reset();
companion_wifi_has_credentials = false;
startCompanionWiFiAutoWebConfig();
assert(the_mesh.web_starts == 1 && the_mesh.last_force_ap);
assert(!companion_wifi_webconfig_start_pending);
mesh::wifi::networkPower().setManaged(false);
assert(companionWiFiAutomaticRecoveryAllowed());
}
'''
class CompanionWiFiNetworkPowerTests(unittest.TestCase):
def test_router_down_boot_starts_the_configured_mqtt_scheduler_once(self):
source = (ROOT / "examples/companion_radio/MyMesh.cpp").read_text(encoding="ascii")
actual = extract_braced(source, "static void copyMqttString(")
actual += extract_braced(source, "void MyMesh::serviceMQTT(")
fixture = r'''
#define WITH_WEBCONFIG 1
#include <cassert>
#include <cstring>
#include <helpers/WiFiNetworkPowerPolicy.h>
constexpr int WL_CONNECTED = 3, WL_DISCONNECTED = 6;
struct { int status() const { return WL_DISCONNECTED; } } WiFi;
namespace mesh { struct Log { void println(const char*) {} };
inline Log& usbLoggingPort() { static Log log; return log; } }
struct Bridge {
int starts = 0; bool running = false;
void begin() { ++starts; running = true; mesh::wifi::networkPower().setManaged(true); }
bool isRunning() const { return running; }
};
struct MyMesh {
bool _mqtt_enabled = true, _wc_batch_active = false;
bool _mqtt_started = false, _mqtt_configured = true, setup = false;
int stops = 0;
struct { char wifi_ssid[33] = {}, wifi_password[65] = {}; } _mqtt_prefs;
Bridge* _mqtt_bridge;
void stopMQTT() { ++stops; }
bool isWebConfigSetupActive() const { return setup; }
void serviceMQTT(const char*, const char*);
};
'''
scenarios = r'''
int main() {
Bridge bridge; MyMesh node; node._mqtt_bridge = &bridge;
assert(WiFi.status() == WL_DISCONNECTED);
node._wc_batch_active = true;
node.serviceMQTT("saved-network", "saved-password");
assert(bridge.starts == 0);
node._wc_batch_active = false; node.setup = true;
node.serviceMQTT("saved-network", "saved-password");
assert(bridge.starts == 0);
node.setup = false; node._mqtt_configured = false;
node.serviceMQTT("saved-network", "saved-password");
assert(bridge.starts == 0);
node._mqtt_configured = true;
node.serviceMQTT("saved-network", "saved-password");
assert(bridge.starts == 1 && node._mqtt_started);
assert(mesh::wifi::networkPower().managed());
assert(strcmp(node._mqtt_prefs.wifi_ssid, "saved-network") == 0);
assert(strcmp(node._mqtt_prefs.wifi_password, "saved-password") == 0);
node.serviceMQTT("saved-network", "saved-password");
assert(bridge.starts == 1);
node._mqtt_enabled = false;
node.serviceMQTT("saved-network", "saved-password");
assert(node.stops == 1);
}
'''
with tempfile.TemporaryDirectory(prefix="companion-router-outage-") as directory:
path = Path(directory)
cpp, binary = path / "fixture.cpp", path / "fixture"
cpp.write_text(fixture + actual + scenarios, encoding="ascii")
result = subprocess.run(["g++", "-std=c++11", "-Wall", "-Wextra", "-Werror",
"-I", str(ROOT / "src"), str(cpp), "-o", str(binary)],
capture_output=True, text=True, timeout=60)
self.assertEqual(result.returncode, 0, result.stderr)
result = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10)
self.assertEqual(result.returncode, 0, result.stderr)
def test_production_transitions_with_and_without_ota_console(self):
compiler = os.environ.get("CXX") or shutil.which("g++")
self.assertIsNotNone(compiler)
source = MAIN.read_text(encoding="ascii")
methods = "\n".join(extract_braced(source, signature) for signature in (
"static bool companionWiFiBackgroundSuspended(",
"static bool companionWiFiAutomaticRecoveryAllowed(",
"static void startCompanionWiFiAutoWebConfig(",
"static void serviceCompanionWiFiAutoWebConfig(",
"static void serviceCompanionWiFiNetworkPowerHold(",
"static void stopCompanionInfrastructureWiFi(",
"static void serviceCompanionEspNowChannelRestore(",
"static void serviceCompanionWiFiBackgroundPower(",
))
sanitizer = (["-fsanitize=address,undefined", "-fno-sanitize-recover=all",
"-fno-pie", "-no-pie"] if sys.platform.startswith("linux") else [])
with tempfile.TemporaryDirectory(prefix="companion-network-power-") as directory:
path = Path(directory)
cpp = path / "fixture.cpp"
cpp.write_text(HARNESS.replace("@METHODS@", methods), encoding="ascii")
for ota in (False, True):
with self.subTest(ota=ota):
binary = path / ("with-ota" if ota else "no-ota")
built = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra",
"-Werror", *sanitizer, *(["-DENABLE_OTA=1"] if ota else []),
"-I", str(ROOT / "src"), str(cpp), "-o", str(binary)],
capture_output=True, text=True, timeout=60)
self.assertEqual(built.returncode, 0, built.stdout + built.stderr)
checked = subprocess.run([str(binary)], capture_output=True, text=True, timeout=10)
self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr)
def test_background_paths_and_manual_recovery_use_the_shared_guard(self):
source = MAIN.read_text(encoding="ascii")
for signature in ("static void serviceCompanionWiFiCredentialReload(",
"static void startCompanionWiFi("):
body = extract_braced(source, signature)
self.assertLess(body.index("companionWiFiBackgroundSuspended()"),
body.index("WiFi.mode(WIFI_STA)"))
ntp = extract_braced(source, "static void serviceCompanionWiFiNtp(")
self.assertLess(ntp.index("networkPower().backgroundSuspended()"),
ntp.index("companion_wifi_ntp_operation.tryAcquire()"))
for signature in ("void requestCompanionWiFiSetup(",
"void scheduleCompanionWiFiCredentialReload(",
"bool setCompanionWiFiEnabled("):
self.assertIn("networkPower().requestProbe()", extract_braced(source, signature))
self.assertIn("&& !companionWiFiBackgroundSuspended()\n"
" && !companion_wifi_background_stopped", source)
self.assertIn("if (!wifi_setup_recovery_mode && companionWiFiAutomaticRecoveryAllowed()", source)
start = extract_braced(source, "static void startCompanionWiFi(")
self.assertIn("startCompanionWiFiAutoWebConfig();", start)
self.assertNotIn("startWebConfig(", start)
self.assertNotIn("companion_wifi_webconfig_start_pending = false", start)
if __name__ == "__main__":
unittest.main()
+8 -1
View File
@@ -119,7 +119,14 @@ class CompanionWiFiNtpIntegrationTests(unittest.TestCase):
self.assertIn("_ntp_sync_pending = true", initialize)
self.assertIn("_ntp_reconnect_latch.consumeIfConnected", task_loop)
self.assertIn("_ntp_sync_pending = true", task_loop)
self.assertIn("syncTimeWithNTP();", task_loop)
self.assertIn("serviceBackgroundNtp(static_cast<uint32_t>(now));", task_loop)
self.assertNotIn("syncTimeWithNTP();", task_loop)
background = mqtt[
mqtt.index("void MQTTBridge::serviceBackgroundNtp(") :
mqtt.index("void MQTTBridge::mqttTaskLoop()")
]
self.assertIn("pollNtpRefresh(now);", background)
self.assertIn("refreshNTP();", background)
self.assertIn("_last_ntp_sync = millis();", sync)
self.assertIn(
"fresh_ntp ? 0 : mqttNtpRetryAt(_last_ntp_sync)", sync
+58
View File
@@ -24,6 +24,7 @@ FIXTURE = r'''
#include <cstring>
#include <new>
#include <helpers/WebConfigBatch.h>
#include <helpers/WiFiNetworkPowerPolicy.h>
#include <helpers/WirelessControl.h>
uint32_t now = 0;
uint32_t millis() { return now; }
@@ -143,6 +144,8 @@ struct MQTTNodeInfo {
uint8_t* sf; uint8_t* cr; uint8_t* repeat_flag; bool repeat_when_nonzero; bool canonical_wifi = false;
};
struct MQTTBridge : AbstractBridge {
bool network_suspended = false;
bool isNetworkSuspended() const { return network_suspended; }
template<class... Args> explicit MQTTBridge(Args...) {}
void setDeviceID(const char*) {} void setFirmwareVersion(const char*) {}
void setBoardModel(const char*) {} void setBuildDate(const char*) {}
@@ -156,6 +159,8 @@ struct WebConfigServer {
uint32_t _setup_started_at = 0;
int _mode = 0;
static constexpr int MODE_SETUP = 1;
static constexpr int MODE_LAN = 2;
uint32_t foreground_until = 0;
char _wifi_ssid[33] = {};
template<class... Args> WebConfigServer(void*, void* prefs, bool owner, Args...) : owns(owner) {
if (prefs) strcpy(_wifi_ssid, static_cast<ObserverPrefs*>(prefs)->wifi_ssid);
@@ -165,6 +170,14 @@ struct WebConfigServer {
bool isRunning() const { return running; }
bool isStopping() const { return stopping; }
void updateWiFiOwnership(bool owner) { owns = owner; }
void requestNetworkAccess() {
foreground_until = millis() + 120000;
mesh::wifi::networkPower().requestProbe();
}
bool isNetworkForegroundActive(uint32_t tick) const {
return _mode == MODE_SETUP || stopping
|| (foreground_until && int32_t(tick - foreground_until) < 0);
}
bool startSetupMode(char* reply) {
if (!allow_start) {
if (owns) stopOwnedWiFiRadio();
@@ -406,6 +419,51 @@ class ESP32FullPowerLifecycleTest(unittest.TestCase):
with self.subTest(role=role):
self.compile_and_run(self.fixture(role), *FLAGS)
def test_suspended_mqtt_idle_portal_and_foreground_ownership(self):
checks = r'''
int main() {
MyMesh node; char reply[160] = {};
strcpy(node._cli.prefs.wifi_ssid, "working-network");
auto& power = mesh::wifi::networkPower();
power.setManaged(true);
assert(node.setMqttBridgeState(true));
assert(node.startWebConfig(false, reply));
node._webconfig->_mode = WebConfigServer::MODE_LAN;
now += 120001; // Explicit-start lease has elapsed; listener is idle.
power.consumeProbeRequest();
node.@WORKER@->network_suspended = true;
power.setSuspended(true);
unsigned off_before = WiFi.off_modes;
node.serviceIdleWiFi();
assert(WiFi.off_modes == off_before && sdk_started);
assert(node.hasPendingWork()); // Cannot sleep before worker shutdown completes.
WiFi.mode(WIFI_OFF); // Only the Core 0 network owner performs this shutdown.
node.serviceIdleWiFi();
assert(!sdk_initialized && !node.hasPendingWork());
assert(node.getPowerSaveSleepSeconds(30) == 30);
// An operator can wake the existing listener without creating a second
// server, taking station ownership, or deinitializing the worker's driver.
assert(node.startWebConfig(false, reply));
assert(power.consumeProbeRequest());
assert(node._webconfig->isNetworkForegroundActive(now));
assert(node.hasPendingWork());
now += 120001;
assert(!node.hasPendingWork());
power.setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, true);
assert(node.hasPendingWork());
power.setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, false);
assert(!node.hasPendingWork());
power.setSuspended(false);
node.@WORKER@->network_suspended = false;
node.serviceIdleWiFi();
assert(node.hasPendingWork()); // Resume is owned by the MQTT worker too.
power.setManaged(false);
}
'''
for role in ("simple_repeater", "simple_room_server"):
with self.subTest(role=role):
self.compile_and_run(self.fixture(role, checks), *FLAGS)
def test_primary_radios_and_wifi_disabled_images_do_not_reclaim_wifi(self):
for role in ("simple_repeater", "simple_room_server"):
source = (ROOT / "examples" / role / "MyMesh.cpp").read_text()
+54 -1
View File
@@ -98,6 +98,7 @@ int main() {
#include <cassert>
#include <cstdint>
#include <helpers/UsbHostSleepPolicy.h>
#include <helpers/WiFiNetworkPowerPolicy.h>
static uint32_t now = 1000;
uint32_t millis() { return now; }
using esp_err_t = int;
@@ -135,15 +136,22 @@ struct Interfaces { bool pending = false; bool hasPendingIO() const { return pen
struct Sensors { bool gps_uart = false; bool gpsUsesSerialUart(uint8_t uart) const { return uart == 1 && gps_uart; } } sensors;
struct Button { bool polling = false; bool needsPolling() const { return polling; } } user_btn;
struct Wireless { unsigned services = 0; unsigned enabled() const { return services; } } companion_wireless;
static bool companion_wifi_background_stopped = false;
namespace mesh {
bool logging = false, watchdog = false;
bool isUsbLoggingEnabled() { return logging; }
bool isUsbLoggingWatchdogArmed() { return watchdog; }
namespace wifi {
bool espnow_constrained = false;
bool espNowChannelConstrained() { return espnow_constrained; }
}
namespace wireless {
constexpr unsigned WiFi = 1, Bluetooth = 2, EspNow = 4;
struct Control { bool transition = false; bool pending() const { return transition; } } controller;
Control& control() { return controller; }
}
}
@SUSPENDED@
@READY@
void idle() { @BRANCH@ }
template<class T> void blocks(T& flag) {
@@ -170,6 +178,49 @@ int main() {
blocks(mesh::watchdog); blocks(mesh::wireless::controller.transition);
prefs.powersaving_enabled = false; before = sleeps; idle(); assert(sleeps == before);
prefs.powersaving_enabled = true; idle(); assert(sleeps == before + 1);
// A requested WiFi service and idle WebConfig listener remain logically
// enabled during MQTT cooldown. Sleep needs transport quiescence and the
// station owner's physical stop, rather than a disabled saved preference.
auto& power = mesh::wifi::networkPower();
power.setManaged(true); power.setSuspended(true);
companion_wireless.services = mesh::wireless::WiFi;
companion_wifi_background_stopped = true;
sdk_result = ESP_ERR_WIFI_NOT_INIT;
before = sleeps; idle(); assert(sleeps == before + 1);
for (int stopped_result : {ESP_ERR_WIFI_NOT_STARTED, ESP_OK}) {
sdk_result = stopped_result; sdk_mode = WIFI_MODE_NULL;
before = sleeps; idle(); assert(sleeps == before + 1);
}
sdk_result = ESP_ERR_WIFI_NOT_INIT;
blocks(mesh::wifi::espnow_constrained);
for (wifi_mode_t mode : {WIFI_MODE_STA, WIFI_MODE_AP, WIFI_MODE_APSTA}) {
sdk_mode = mode; sdk_result = ESP_OK;
before = sleeps; idle(); assert(sleeps == before);
}
sdk_mode = WIFI_MODE_NULL; sdk_result = 99;
before = sleeps; idle(); assert(sleeps == before); // Failed physical cleanup/read.
sdk_result = ESP_ERR_WIFI_NOT_INIT;
companion_wifi_background_stopped = false;
before = sleeps; idle(); assert(sleeps == before); // Owner has not stopped STA yet.
companion_wifi_background_stopped = true;
for (unsigned service : {mesh::wireless::Bluetooth, mesh::wireless::EspNow}) {
companion_wireless.services = mesh::wireless::WiFi | service;
before = sleeps; idle(); assert(sleeps == before);
}
companion_wireless.services = mesh::wireless::WiFi;
for (auto hold : {mesh::wifi::NetworkPowerHold::WebConfig,
mesh::wifi::NetworkPowerHold::CompanionClient,
mesh::wifi::NetworkPowerHold::OtaSeeder}) {
power.setHold(hold, true);
before = sleeps; idle(); assert(sleeps == before);
power.setHold(hold, false);
}
blocks(board.ota); blocks(bt_controller_active); blocks(bt_host_active);
power.setSuspended(false);
before = sleeps; idle(); assert(sleeps == before); // Worker has begun its probe.
power.setManaged(false);
companion_wifi_background_stopped = false;
companion_wireless.services = 0; sdk_mode = WIFI_MODE_NULL; sdk_result = ESP_OK;
// The real board policy must observe USB even while pending work/logging
// inhibit this branch. A rebooting host cannot disappear for one SOF gap
// and cause Full to shut down its console immediately.
@@ -179,6 +230,8 @@ int main() {
now += 120000; idle(); assert(sleeps == before + 1);
}
'''.replace("@READY@", ready).replace("@BRANCH@", branch).replace("@USB_HELD@", usb_held)
source = source.replace("@SUSPENDED@", function(main,
"static bool companionWiFiBackgroundSuspended()"))
source = "#include <initializer_list>\n" + source
flags = ("ESP32_PLATFORM", "ESP32", "ENABLE_USB_INTERFACE", "ARDUINO_USB_CDC_ON_BOOT=1",
'WIFI_SSID=""', "BLE_PIN_CODE=123456", "MESH_USB_LOGGING_AVAILABLE=1",
@@ -189,7 +242,7 @@ int main() {
with self.subTest(pm=pm):
result = run_cpp(source, (*flags, f"COMPANION_IDF_PM_AVAILABLE={pm}"))
self.assertEqual(result.returncode, 0, result.stderr)
broken = run_cpp(source.replace("companion_wireless.enabled() != 0", "false"),
broken = run_cpp(source.replace("enabled_services != 0", "false"),
(*flags, "COMPANION_IDF_PM_AVAILABLE=0"))
self.assertNotEqual(broken.returncode, 0)
+323
View File
@@ -0,0 +1,323 @@
#!/usr/bin/env python3
"""Run the real MQTT wrapper stop and outage handoff under SDK failures."""
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import unittest
from test_mqtt_publish_ack import ARDUINO_HEADER, MQTT_HEADER
from test_mqtt_transport_results import method
from test_replay_reset_integration import extract_braced
ROOT = Path(__file__).resolve().parents[1]
ADAPTER = "lib/PsychicMqttClient/src/PsychicMqttClient.cpp"
BRIDGE = "src/helpers/bridges/MQTTBridge.cpp"
HARNESS = r'''
#include <atomic>
#include <cassert>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <functional>
#include <helpers/WiFiNetworkPowerPolicy.h>
#include <helpers/MQTTConnectionPolicy.h>
#define private public
#include "PsychicMqttClient.h"
#undef private
template<typename... Args> void log_stub(Args&&...) {}
static const char* TAG = "test";
#define ESP_LOGI(...) log_stub(__VA_ARGS__)
#define ESP_LOGW(...) log_stub(__VA_ARGS__)
#define ESP_ERROR_CHECK_WITHOUT_ABORT(value) ((void)(value))
#define MQTT_DEBUG_PRINTLN(...) ((void)0)
#define pdMS_TO_TICKS(ms) ((ms) / portTICK_PERIOD_MS)
static uint32_t now_ms = 0, elapsed_ms = 0, tick_ms = 1;
static uint32_t callback_after_ms = UINT32_MAX;
static int disconnect_result = ESP_OK, stop_result = ESP_OK;
static int disconnect_calls = 0, stop_calls = 0, sleeps = 0;
static PsychicMqttClient* active_client = nullptr;
static std::function<void()> on_delay;
unsigned long millis() { return now_ms; }
const char* esp_err_to_name(int) { return "mock"; }
static void deliverDisconnect() {
if (!active_client || elapsed_ms < callback_after_ms) return;
active_client->_connected = false;
active_client->_stopMqttClient = true;
callback_after_ms = UINT32_MAX;
}
void vTaskDelay(unsigned ticks) {
assert(ticks != 0);
++sleeps;
now_ms += ticks * tick_ms;
elapsed_ms += ticks * tick_ms;
deliverDisconnect();
if (on_delay) on_delay();
assert(elapsed_ms <= 2000U + tick_ms);
}
int esp_mqtt_client_disconnect(void*) {
++disconnect_calls;
if (disconnect_result == ESP_OK) deliverDisconnect();
return disconnect_result;
}
int esp_mqtt_client_stop(void*) { ++stop_calls; return stop_result; }
// Destruction is outside this test's stop/handoff seam; failed SDK resources
// remain owned by the surrounding bridge lifecycle in the production code.
PsychicMqttClient::~PsychicMqttClient() {}
@ADAPTER_METHODS@
constexpr int RUNTIME_MQTT_SLOTS = 1, WIFI_MODE_NULL = 0;
constexpr int WIFI_MODE_AP = 2, WIFI_STA = 1, WIFI_OFF = 0;
using wifi_mode_t = int;
static int wifi_mode = WIFI_STA, wifi_stops = 0;
int esp_wifi_get_mode(int* mode) { *mode = wifi_mode; return ESP_OK; }
bool mqttStationWiFiMutationAllowed() { return !(wifi_mode & WIFI_MODE_AP); }
namespace mesh { namespace wifi {
bool espNowChannelConstrained() { return false; }
int restoreEspNowChannel() { assert(false); return ESP_ERR_INVALID_STATE; }
} }
struct {
void setAutoReconnect(bool enabled) { assert(!enabled); }
void disconnect(bool stop, bool erase) { assert(!stop && !erase); }
int getMode() const { return wifi_mode; }
void mode(int mode) {
assert(mode == WIFI_OFF && mesh::wifi::networkPower().backgroundSuspended());
wifi_mode = mode;
++wifi_stops;
}
} WiFi;
struct MQTTBridge {
struct Slot {
PsychicMqttClient* client = nullptr;
bool enabled = true, connected = false, circuit_breaker_tripped = false;
uint32_t connected_at_ms = 0, last_reconnect_attempt = 0;
uint8_t reconnect_backoff = 0;
bool initial_connect_done = true;
char* auth_token = nullptr;
char broker_uri[32] = "mqtt://original";
uint32_t token_expires_at = 1, last_token_renewal = 1;
uint32_t last_log_time = 1, last_deferred_log_ms = 1;
uint8_t max_backoff_failures = 0;
} _slots[RUNTIME_MQTT_SLOTS];
using MQTTSlot = Slot;
mesh::wifi::NetworkPowerPolicy _network_power_policy;
std::atomic<bool> _network_suspended{false}, _fresh_ntp_this_boot{false};
bool _slot_attempt_pending[RUNTIME_MQTT_SLOTS] = {};
bool _status_publish_pending[RUNTIME_MQTT_SLOTS] = {};
bool _slot_force_jwt_mint[RUNTIME_MQTT_SLOTS] = {};
uint32_t _network_publish_acks[RUNTIME_MQTT_SLOTS] = {};
bool _network_espnow_restore_pending = false;
uint32_t _network_espnow_restore_at = 0;
bool _ntp_force_requested = false, _ntp_diag_requested = false;
bool _ntp_sync_pending = false, _manage_wifi = true, _wifi_status_initialized = true;
uint32_t _last_wifi_check = 0, _last_slot_reconnect_ms = 0;
uint32_t _ntp_refresh_retry_at = 0;
std::atomic<bool> _stop_requested{false}, _stop_acked{false};
int destroyed = 0, queues = 0, ntp_starts = 0, slot_starts = 0;
struct Board { bool isOTAUpdateRunning() const { return false; } };
Board* _board = nullptr;
void cancelNtpRefresh() {}
void updateCachedConnectionStatus() {}
void initializeWiFiInTask() {}
void processPacketQueue() { ++queues; }
void teardownSlot(int index, bool force = false);
void destroySlotClients() {
assert(!_stop_acked);
for (auto& slot : _slots) {
assert(!slot.client || !slot.client->isStarted());
slot.client = nullptr;
}
++destroyed;
}
bool serviceNetworkPower(uint32_t now);
void serviceEspNowChannelRestore(uint32_t now);
void workerStop() {
while (true) {
@TASK_STOP@
}
}
void workerOutageOnce() {
for (unsigned iteration = 0; iteration < 1; ++iteration) {
const uint32_t now = now_ms;
@OUTAGE_GUARD@
++ntp_starts;
++slot_starts;
}
}
};
@BRIDGE_METHOD@
int main(int argc, char** argv) {
assert(argc == 4);
const int scenario = std::atoi(argv[1]);
tick_ms = static_cast<uint32_t>(std::atoi(argv[2]));
if (std::atoi(argv[3])) now_ms = UINT32_MAX - 511U;
PsychicMqttClient client;
active_client = &client;
client._client = reinterpret_cast<void*>(1);
client._started = true;
client._connected = true;
if (scenario == 0) {
client.disconnect(); // Lost event: SDK stop is sufficient proof.
assert(elapsed_ms >= 1000U && elapsed_ms < 1000U + tick_ms);
assert(stop_calls == 1 && !client.isStarted() && !client.connected());
const int stopped = stop_calls;
client.disconnect();
assert(stop_calls == stopped); // already stopped is idempotent
} else if (scenario == 1) {
stop_result = ESP_ERR_INVALID_STATE;
auto& control = mesh::wifi::networkPower();
control.setManaged(true);
MQTTBridge bridge;
bridge._slots[0].client = &client;
const uint32_t start = now_ms;
assert(bridge.serviceNetworkPower(start));
now_ms = start + 300000U;
assert(!bridge.serviceNetworkPower(now_ms));
assert(elapsed_ms >= 1000U && stop_calls == 1 && client.isStarted());
assert(!bridge._network_suspended && !control.backgroundSuspended());
assert(wifi_stops == 0 && wifi_mode == WIFI_STA);
// An owning-task retry can now complete teardown without another wait if
// the disconnect request itself fails. Only the successful SDK stop grants
// the bridge permission to turn the station off.
disconnect_result = ESP_ERR_INVALID_STATE;
stop_result = ESP_OK;
assert(!bridge.serviceNetworkPower(now_ms));
assert(!client.isStarted() && control.backgroundSuspended() && wifi_stops == 1);
} else if (scenario == 2) {
disconnect_result = ESP_ERR_INVALID_STATE;
client.disconnect();
assert(sleeps == 0 && disconnect_calls == 1 && stop_calls == 1);
assert(!client.isStarted() && !client.connected());
} else if (scenario == 3) {
callback_after_ms = 0;
stop_result = ESP_ERR_INVALID_STATE;
client.disconnect();
assert(sleeps == 0 && stop_calls == 1 && client.isStarted());
assert(!client.connected()); // Actual event state survives failed stop.
} else if (scenario == 4) {
callback_after_ms = 50U;
client.disconnect();
assert(elapsed_ms >= 50U && elapsed_ms < 50U + tick_ms);
assert(stop_calls == 1 && !client.isStarted() && !client.connected());
} else if (scenario == 5) {
client._started = false;
client.disconnect();
client._client = nullptr;
client.disconnect();
assert(disconnect_calls == 0 && stop_calls == 0 && sleeps == 0);
} else if (scenario == 6) {
client._connected = false;
stop_result = ESP_ERR_INVALID_STATE;
client.disconnect();
assert(disconnect_calls == 0 && stop_calls == 1 && sleeps == 0);
assert(client.isStarted());
} else if (scenario == 7) {
disconnect_result = ESP_ERR_INVALID_STATE;
stop_result = ESP_ERR_INVALID_STATE;
MQTTBridge bridge;
bridge._slots[0].client = &client;
bridge._stop_requested = true;
on_delay = [&]() {
assert(!bridge._stop_acked && bridge.destroyed == 0);
assert(bridge._slots[0].client == &client);
if (elapsed_ms >= 150U) stop_result = ESP_OK;
};
bridge.workerStop();
assert(stop_calls == 2 && !client.isStarted());
assert(bridge._stop_acked && bridge.destroyed == 1);
assert(bridge._slots[0].client == nullptr);
on_delay = nullptr;
} else if (scenario == 8) {
disconnect_result = ESP_ERR_INVALID_STATE;
stop_result = ESP_ERR_INVALID_STATE;
MQTTBridge bridge;
char token[] = "protected-token";
bridge._slots[0].client = &client;
bridge._slots[0].auth_token = token;
bridge.teardownSlot(0);
assert(client.isStarted() && strcmp(token, "protected-token") == 0);
assert(bridge._slots[0].initial_connect_done);
assert(strcmp(bridge._slots[0].broker_uri, "mqtt://original") == 0);
} else if (scenario == 9) {
disconnect_result = ESP_ERR_INVALID_STATE;
stop_result = ESP_ERR_INVALID_STATE;
auto& control = mesh::wifi::networkPower();
control.setManaged(true);
MQTTBridge bridge;
bridge._slots[0].client = &client;
const uint32_t start = now_ms;
assert(bridge.serviceNetworkPower(start));
now_ms = start + 300000U;
bridge.workerOutageOnce();
assert(client.isStarted() && !bridge._network_suspended);
assert(!control.backgroundSuspended() && wifi_stops == 0);
assert(bridge.queues == 1 && bridge.ntp_starts == 0 && bridge.slot_starts == 0);
} else assert(false);
printf("PASS: %d\n", scenario);
}
'''
class MqttDisconnectTests(unittest.TestCase):
def test_real_wrapper_and_wifi_handoff_across_sdk_failures_ticks_and_wrap(self):
compiler = os.environ.get("CXX") or shutil.which("g++")
self.assertIsNotNone(compiler)
adapter = "\n".join(method(ADAPTER, signature) for signature in (
"PsychicMqttClient::PsychicMqttClient()",
"bool PsychicMqttClient::connected()",
"void PsychicMqttClient::disconnect()",
"void PsychicMqttClient::forceStop()",
"unsigned long PsychicMqttClient::getPublishAck()",
))
task = method(BRIDGE, "void MQTTBridge::mqttTaskLoop()")
source = (HARNESS.replace("@ADAPTER_METHODS@", adapter)
.replace("@TASK_STOP@", extract_braced(task, " if (_stop_requested)"))
.replace("@OUTAGE_GUARD@", extract_braced(task, " if (!serviceNetworkPower("))
.replace("@BRIDGE_METHOD@", "\n".join(method(BRIDGE, signature) for signature in (
"bool MQTTBridge::serviceNetworkPower(",
"void MQTTBridge::serviceEspNowChannelRestore(",
"void MQTTBridge::teardownSlot(",
))))
with tempfile.TemporaryDirectory(prefix="meshcore-mqtt-disconnect-") as temporary:
directory = Path(temporary)
(directory / "Arduino.h").write_text(ARDUINO_HEADER, encoding="ascii")
(directory / "mqtt_client.h").write_text(MQTT_HEADER, encoding="ascii")
(directory / "esp_crt_bundle.h").write_text("#pragma once\n", encoding="ascii")
cpp = directory / "disconnect.cpp"
cpp.write_text(source, encoding="ascii")
for idf in (4, 5):
for tick in (1, 10, 50):
with self.subTest(idf=idf, tick=tick):
binary = directory / f"disconnect-{idf}-{tick}"
arduino = ARDUINO_HEADER.replace(
"constexpr unsigned portTICK_PERIOD_MS = 1;",
f"constexpr unsigned portTICK_PERIOD_MS = {tick};")
(directory / "Arduino.h").write_text(arduino, encoding="ascii")
built = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra",
"-Werror", "-DARDUINO_ARCH_ESP32=1", "-DESP_PLATFORM=1",
f"-DESP_IDF_VERSION_MAJOR={idf}",
"-I", str(directory), "-I", str(ROOT / "lib/PsychicMqttClient/src"),
"-I", str(ROOT / "src"), str(cpp), "-o", str(binary)],
capture_output=True, text=True, timeout=60)
self.assertEqual(built.returncode, 0, built.stdout + built.stderr)
for scenario in range(10):
for wrap in (0, 1):
with self.subTest(scenario=scenario, wrap=wrap):
checked = subprocess.run([str(binary), str(scenario), str(tick), str(wrap)],
capture_output=True, text=True, timeout=5)
self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr)
if __name__ == "__main__":
unittest.main()
+13
View File
@@ -150,6 +150,7 @@ struct MQTTBridge {
} _slots[RUNTIME_MQTT_SLOTS];
std::atomic<bool> _stop_requested{false}, _slot_attempt_pending[RUNTIME_MQTT_SLOTS] = {};
bool _slot_force_jwt_mint[RUNTIME_MQTT_SLOTS] = {};
bool _slot_reconfigure_pending[RUNTIME_MQTT_SLOTS] = {};
bool _initialized = true;
int _max_active_slots = 2;
MQTTPrefs prefs;
@@ -189,6 +190,14 @@ int main() {
assert(slot.client->_buffer_capacity == 896);
char token[8] = "old-jwt";
slot.auth_token = token;
stop_result = ESP_ERR_INVALID_STATE;
bridge.applySlotPreset(0, "jwt");
assert(slot.client->isStarted() && slot.initial_connect_done);
assert(slot.preset == &plain_preset && strcmp(token, "old-jwt") == 0);
assert(bridge._slot_reconfigure_pending[0]);
assert(init_calls == 1 && destroy_calls == 0);
stop_result = ESP_OK;
bridge._slot_reconfigure_pending[0] = false; // worker consumes the retry flag
bridge.applySlotPreset(0, "jwt");
assert(token[0] == '\0' && slot.preset == &jwt_preset);
assert(slot.client->_buffer_capacity == 896);
@@ -287,6 +296,10 @@ class MqttNetworkTimeoutTest(unittest.TestCase):
preamble = preamble.replace(
'void vTaskDelay(unsigned) { assert(false && "stub disconnect must deliver its event"); }',
'void vTaskDelay(unsigned ticks) { assert(ticks == 50); }')
preamble = preamble.replace(
'int esp_mqtt_client_stop(void*) { ++stop_calls; return ESP_OK; }',
'static int stop_result = ESP_OK;\n'
'int esp_mqtt_client_stop(void*) { ++stop_calls; return stop_result; }')
source = "#define ESP_PLATFORM 1\n" + preamble + RECONFIGURE_PREAMBLE
source += "\n".join(method(adapter, signature) for signature in (
"PsychicMqttClient::PsychicMqttClient()",
+21
View File
@@ -191,6 +191,7 @@ TASK_PREAMBLE = r'''
#include <cstdlib>
#include <string>
#include <vector>
#include "helpers/WiFiNetworkPowerPolicy.h"
#define PORTABLE_MQTT_OBSERVER 1
#define MQTT_DEBUG_PRINTLN(...) ((void)0)
using TickType_t = uint32_t;
@@ -215,7 +216,12 @@ struct MQTTPrefs {
bool mqtt_status_enabled = true;
};
struct MQTTBridge {
mesh::wifi::NetworkPowerPolicy _network_power_policy;
std::atomic<bool> _network_suspended{false};
bool _network_espnow_restore_pending = false;
struct Slot {
struct Client { bool isStarted() const { return false; } };
Client* client = nullptr;
bool enabled = true, initial_connect_done = false, connected = false;
unsigned long last_reconnect_attempt = 0;
} _slots[RUNTIME_MQTT_SLOTS];
@@ -244,6 +250,14 @@ struct MQTTBridge {
assert(!_stop_acked);
}
bool initializeWiFiInTask() { return true; }
bool serviceNetworkPower(uint32_t) { work(); return true; }
void serviceBackgroundNtp(uint32_t) {
work();
if (scenario == 7) {
_ntp_synced = true;
_stop_requested = true;
}
}
bool handleWiFiConnection(unsigned long) { work(); return false; }
bool syncTimeWithNTP(bool = false, bool = false) {
work();
@@ -296,6 +310,7 @@ struct MQTTBridge {
}
bool waitUnlessStopping(uint32_t delay_ms);
void mqttTaskLoop();
void applyTimezoneRules() {}
};
void vTaskDelay(TickType_t ticks) {
assert(ticks > 0);
@@ -375,6 +390,7 @@ ASYNC_STOP_PREAMBLE = r'''
#include <limits>
#include "helpers/MQTTLifecycle.h"
#include "helpers/MQTTConnectionPolicy.h"
#include "helpers/WiFiNetworkPowerPolicy.h"
#define ESP_PLATFORM 1
#define MQTT_DEBUG_PRINTLN(...) ((void)0)
#define pdMS_TO_TICKS(ms) (ms)
@@ -406,6 +422,7 @@ struct MQTTBridge {
} ops{this};
MQTTLifecycle::Coordinator _lifecycle{ops, 1};
std::atomic<bool> _stop_requested{false}, _stop_acked{false};
std::atomic<bool> _network_suspended{true};
bool _initialized = true, resources_owned = true;
bool _slots_setup_done = true, _staged_raw_valid = true;
bool _ntp_estimate_requested = true, _ntp_estimate_done = true, _ntp_estimate_ok = true;
@@ -416,6 +433,8 @@ struct MQTTBridge {
assert(_lifecycle.requestStart());
assert(_lifecycle.onTaskStarted());
s_mqtt_bridge_instance = this;
mesh::wifi::networkPower().setManaged(true);
mesh::wifi::networkPower().setSuspended(true);
}
void cancelNtpRefresh() { ++cancels; }
void requestStop();
@@ -452,6 +471,7 @@ static void assert_finished(MQTTBridge& bridge, bool clean) {
assert(!bridge._ntp_estimate_requested && !bridge._ntp_estimate_done);
assert(!bridge._ntp_estimate_ok && bridge._ntp_estimate_epoch == 0);
assert(bridge.cancels == 1);
assert(!bridge._network_suspended && !mesh::wifi::networkPower().managed());
}
'''
@@ -547,6 +567,7 @@ class MqttTransportResultsTests(unittest.TestCase):
path.write_text(source, encoding="utf-8")
exe = Path(temp) / ("worker.exe" if os.name == "nt" else "worker")
result = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra",
"-I", str(ROOT / "src"),
str(path), "-o", str(exe)], text=True, capture_output=True)
self.assertEqual(result.returncode, 0, result.stderr)
for scenario in range(12):
+856
View File
@@ -0,0 +1,856 @@
#!/usr/bin/env python3
"""Execute MQTT's production WiFi/SNTP outage handoff with controlled SDK calls.
The actual bridge methods and shared policies run without hardware, network,
Arduino, or PlatformIO. SDK calls observe suspension publication ordering and
can fail independently of MQTT and NTP state.
"""
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import unittest
from test_mqtt_transport_results import method
ROOT = Path(__file__).resolve().parents[1]
BRIDGE = "src/helpers/bridges/MQTTBridge.cpp"
PREAMBLE = r'''
#include <helpers/WiFiNetworkPowerPolicy.h>
#include <helpers/MQTTConnectionPolicy.h>
#include <helpers/esp32/SntpOperationCoordinator.h>
#include <atomic>
#include <cassert>
#include <cstdint>
#include <ctime>
#include <functional>
#include <iostream>
#include <stdexcept>
#include <string>
#include <sys/time.h>
#include <vector>
#define ESP_PLATFORM 1
#define WITH_SNMP 1
#define MQTT_DEBUG_PRINTLN(...) ((void)0)
using wifi_mode_t = int;
constexpr int ESP_OK = 0, ESP_FAIL = -1, WIFI_MODE_NULL = 0;
constexpr int WIFI_STA = 1, WIFI_MODE_AP = 2, WIFI_OFF = 0;
constexpr int WL_CONNECTED = 3, WL_DISCONNECTED = 0, RUNTIME_MQTT_SLOTS = 3;
constexpr int SNTP_SYNC_STATUS_RESET = 0, SNTP_SYNC_STATUS_COMPLETED = 1;
static constexpr uint32_t MQTT_NTP_REFRESH_TIMEOUT_MS = 15000U;
static constexpr uint32_t MQTT_NTP_NON_ESP_SETTLE_MS = 5000U;
static constexpr uint32_t MQTT_NTP_MIN_VALID_EPOCH = 1767225600U;
static uint32_t clock_ms = 100U;
static int64_t system_epoch = 0;
static int sdk_mode = WIFI_STA, sdk_mode_result = ESP_OK;
static bool station_mutation_allowed = true, espnow_constrained = false;
static int ntp_status = SNTP_SYNC_STATUS_RESET, ntp_configs = 0;
static int ntp_stops = 0, channel_restores = 0;
static int channel_restore_result = ESP_OK;
static bool require_quiescent_wifi_stop = false;
static std::vector<std::string> events;
static std::vector<std::string> ntp_primaries;
static std::function<void()> on_client_stop;
static void require(bool value, const char* reason) {
if (!value) throw std::runtime_error(reason);
}
uint32_t millis() { return clock_ms; }
time_t fake_time(void*) { return static_cast<time_t>(system_epoch); }
int fake_settimeofday(const timeval* value, void*) {
system_epoch = value->tv_sec;
return 0;
}
#define time fake_time
#define settimeofday fake_settimeofday
int esp_wifi_get_mode(wifi_mode_t* mode) {
*mode = sdk_mode;
return sdk_mode_result;
}
bool mqttStationWiFiMutationAllowed() {
return station_mutation_allowed && sdk_mode_result == ESP_OK
&& !(sdk_mode & WIFI_MODE_AP);
}
namespace mesh { namespace wifi {
bool espNowChannelConstrained() { return espnow_constrained; }
int restoreEspNowChannel() {
++channel_restores;
events.push_back("espnow");
return channel_restore_result;
}
} }
struct WiFiModel {
int state = WL_CONNECTED;
bool reconnect = true;
int disconnects = 0, offs = 0;
int status() const { return state; }
int getMode() const { return sdk_mode; }
void setAutoReconnect(bool value) {
reconnect = value;
if (!value && require_quiescent_wifi_stop) {
require(mesh::wifi::networkPower().backgroundSuspended(),
"WiFi stopped before worker published transport quiescence");
}
}
void disconnect(bool stop, bool erase) {
require(!stop && !erase, "suspension erased credentials or ESP-NOW driver");
++disconnects;
state = WL_DISCONNECTED;
events.push_back("wifi-disconnect");
}
void mode(int value) {
require(value == WIFI_OFF, "unexpected station mode change");
if (require_quiescent_wifi_stop) {
require(mesh::wifi::networkPower().backgroundSuspended(),
"WiFi driver stopped before the safe handoff");
}
sdk_mode = value;
++offs;
events.push_back("wifi-off");
}
} WiFi;
void sntp_set_sync_status(int value) { ntp_status = value; }
int sntp_get_sync_status() { return ntp_status; }
void esp_sntp_stop() {
if (require_quiescent_wifi_stop) {
require(!mesh::wifi::networkPower().backgroundSuspended(),
"SNTP was stopped after publishing transport quiescence");
}
++ntp_stops;
events.push_back("sntp-stop");
}
void configTime(int, int, const char* primary, const char*, const char*) {
require(!mesh::wifi::networkPower().backgroundSuspended(),
"SNTP restarted during suspension");
++ntp_configs;
ntp_primaries.push_back(primary);
events.push_back("sntp-start");
}
void configTime(int a, int b, const char* primary) {
configTime(a, b, primary, nullptr, nullptr);
}
struct Client {
bool started = true, fail_stop = false;
int stops = 0;
uint32_t publish_acks = 0;
uint32_t getPublishAck() const { return publish_acks; }
bool isStarted() const { return started; }
void disconnect() {
require(!mesh::wifi::networkPower().backgroundSuspended(),
"TLS stopped after handing station off");
++stops;
events.push_back("tls-stop");
if (!fail_stop) started = false;
if (on_client_stop) on_client_stop();
}
};
struct Board {
bool ota = false;
bool isOTAUpdateRunning() const { return ota; }
};
struct Snmp {
bool running = false;
bool isRunning() const { return running; }
};
struct Rtc {
uint32_t epoch = 0;
uint32_t getCurrentTime() const { return epoch; }
void setCurrentTime(uint32_t value) { epoch = value; }
};
struct Prefs {};
const char* effectiveNtpPrimary(const Prefs*) { return "custom"; }
#ifdef ESP_PLATFORM
static void fillNtpServerList(const Prefs*, const char* servers[], int& count) {
static const char* names[] = {"custom", "pool", "google", "cloudflare", "aws", "nist"};
count = 6;
for (int index = 0; index < count; ++index) servers[index] = names[index];
}
#endif
@HELPERS@
struct MQTTBridge {
static constexpr int kMaxNtpServers = 6;
struct Slot {
bool enabled = true, connected = false, circuit_breaker_tripped = true;
uint32_t connected_at_ms = 0, last_reconnect_attempt = 200U;
uint8_t reconnect_backoff = 5;
Client* client = nullptr;
} _slots[RUNTIME_MQTT_SLOTS];
Client clients[RUNTIME_MQTT_SLOTS];
std::atomic<bool> _slot_attempt_pending[RUNTIME_MQTT_SLOTS] = {};
std::atomic<bool> _status_publish_pending[RUNTIME_MQTT_SLOTS] = {};
uint32_t _network_publish_acks[RUNTIME_MQTT_SLOTS] = {};
bool _network_espnow_restore_pending = false;
uint32_t _network_espnow_restore_at = 0;
mesh::wifi::NetworkPowerPolicy _network_power_policy;
std::atomic<bool> _network_suspended{false};
std::atomic<bool> _ntp_synced{false}, _fresh_ntp_this_boot{false};
std::atomic<uint32_t> _backward_clock_reset_epoch{0};
bool _manage_wifi = true, _wifi_status_initialized = true;
bool _ntp_force_requested = false, _ntp_diag_requested = false;
bool _ntp_estimate_requested = false, _ntp_sync_pending = true;
bool _ntp_estimate_done = false, _ntp_estimate_ok = false;
uint32_t _ntp_estimate_epoch = 0;
std::atomic<bool> _stop_requested{false};
bool _ntp_refresh_pending = false;
uint32_t _ntp_refresh_started_at = 0, _ntp_refresh_previous_epoch = 0;
uint32_t _ntp_refresh_retry_at = 0, _last_ntp_sync = 0;
uint32_t _last_wifi_check = 0, _last_slot_reconnect_ms = 0;
uint8_t _background_ntp_server = 0;
Board board;
Board* _board = &board;
Snmp snmp;
Snmp* _snmp_agent = &snmp;
Rtc rtc;
Rtc* _rtc = &rtc;
Prefs prefs;
Prefs* _obs = &prefs;
int initializes = 0, cache_updates = 0;
mesh::sntp_coord::OperationLease _ntp_refresh_operation{
mesh::sntp_coord::processWideCoordinator(), stopMqttSntpOperation};
MQTTBridge() {
for (int index = 0; index < RUNTIME_MQTT_SLOTS; ++index) {
_slots[index].client = &clients[index];
_slot_attempt_pending[index] = true;
_status_publish_pending[index] = true;
}
}
bool initializeWiFiInTask() {
require(!mesh::wifi::networkPower().backgroundSuspended(),
"station resumed before clearing suspension handoff");
++initializes;
sdk_mode = WIFI_STA;
WiFi.reconnect = true;
return true;
}
void updateCachedConnectionStatus() { ++cache_updates; }
bool isRunning() const { return true; }
bool isNetworkSuspended() const { return _network_suspended.load(); }
bool requestNtpTimeEstimate();
bool serviceNetworkPower(uint32_t now);
void serviceEspNowChannelRestore(uint32_t now);
void serviceBackgroundNtp(uint32_t now);
void cancelNtpRefresh();
void refreshNTP();
void pollNtpRefresh(uint32_t now);
};
@METHODS@
static void resetEnvironment() {
auto& power = mesh::wifi::networkPower();
power.setManaged(false);
power.setHold(mesh::wifi::NetworkPowerHold::CompanionClient, false);
power.setHold(mesh::wifi::NetworkPowerHold::WebConfig, false);
power.setHold(mesh::wifi::NetworkPowerHold::OtaSeeder, false);
power.consumeProbeRequest();
power.setManaged(true);
clock_ms = 100U;
system_epoch = 0;
sdk_mode = WIFI_STA;
sdk_mode_result = ESP_OK;
station_mutation_allowed = true;
espnow_constrained = false;
ntp_status = SNTP_SYNC_STATUS_RESET;
ntp_configs = ntp_stops = channel_restores = 0;
channel_restore_result = ESP_OK;
require_quiescent_wifi_stop = false;
events.clear();
ntp_primaries.clear();
on_client_stop = nullptr;
WiFi = {};
}
static constexpr uint32_t grace = 300000U, interval = 1800000U, probe = 120000U;
static void failureStopsBothServicesBeforeWifi() {
resetEnvironment();
MQTTBridge bridge;
require(bridge.serviceNetworkPower(clock_ms), "cold bridge was suspended");
bridge.serviceBackgroundNtp(clock_ms);
require(ntp_configs == 1 && bridge._ntp_refresh_pending,
"initial NTP was not asynchronous");
clock_ms += 15000U;
bridge.serviceBackgroundNtp(clock_ms);
require(!bridge._ntp_refresh_pending && ntp_stops == 1,
"timed-out NTP continued running");
require(bridge._ntp_refresh_retry_at == clock_ms + interval,
"failed NTP was not deferred 30 minutes");
// GOT_IP/NTP retry timers and failing SDK handshakes do not renew the clock.
for (uint32_t offset = 16000U; offset < grace; offset += 1000U) {
clock_ms = 100U + offset;
WiFi.state = WL_CONNECTED;
require(bridge.serviceNetworkPower(clock_ms), "outage grace ended early");
bridge.serviceBackgroundNtp(clock_ms);
}
require(ntp_configs == 1, "failed SNTP retried during its cooldown");
// Keep one live SNTP lease at the cutoff to prove teardown publication order.
bridge._ntp_refresh_retry_at = 0;
bridge.refreshNTP();
require(bridge._ntp_refresh_operation.owns(), "SNTP request lacks ownership");
events.clear();
require_quiescent_wifi_stop = true;
clock_ms = 100U + grace;
require(!bridge.serviceNetworkPower(clock_ms), "failed services kept WiFi on");
require(bridge._network_suspended && mesh::wifi::networkPower().backgroundSuspended(),
"worker did not hand off its transport suspension");
require(events == std::vector<std::string>({"sntp-stop", "tls-stop", "tls-stop",
"tls-stop", "wifi-disconnect", "wifi-off"}),
"WiFi was disabled before SNTP/TLS teardown");
require(!bridge._ntp_refresh_pending && !bridge._ntp_refresh_operation.owns()
&& !bridge._ntp_sync_pending && !WiFi.reconnect && sdk_mode == WIFI_OFF,
"a failed background service retained radio activity");
for (int index = 0; index < RUNTIME_MQTT_SLOTS; ++index) {
require(!bridge.clients[index].started && !bridge._slots[index].connected
&& !bridge._slot_attempt_pending[index] && !bridge._status_publish_pending[index],
"suspension retained a pending transport or publish");
}
}
static void failedSdkStopWithholdsHandoff() {
resetEnvironment();
MQTTBridge bridge;
bridge.serviceNetworkPower(100U);
bridge.clients[1].fail_stop = true;
require(!bridge.serviceNetworkPower(100U + grace),
"failed SDK stop permitted more background TLS or NTP work");
require(!bridge._network_suspended && !mesh::wifi::networkPower().backgroundSuspended()
&& sdk_mode == WIFI_STA && WiFi.disconnects == 0 && WiFi.offs == 0,
"station owner was allowed to stop a still-running TLS client");
require(!bridge.clients[0].started && bridge.clients[1].started
&& bridge.clients[2].started, "teardown did not stop at failed client");
bridge.clients[1].fail_stop = false;
require(!bridge.serviceNetworkPower(101U + grace),
"retry after successful SDK stop failed to suspend");
require(!bridge.clients[1].started && !bridge.clients[2].started && WiFi.offs == 1,
"successful retry leaked a transport or radio");
}
static void recoveryHasOneBoundedWindow() {
for (uint32_t start : {100U, UINT32_MAX - 10000U}) {
resetEnvironment();
MQTTBridge bridge;
require(bridge.serviceNetworkPower(start), "initial network did not run");
const uint32_t suspended = start + grace;
require(!bridge.serviceNetworkPower(suspended), "network did not suspend");
for (uint32_t offset = 1000U; offset < interval; offset += 1000U) {
require(!bridge.serviceNetworkPower(suspended + offset),
"a failed timer woke WiFi during the cooldown");
}
require(bridge.initializes == 0, "cooldown restarted station");
const uint32_t resumed = suspended + interval;
require(bridge.serviceNetworkPower(resumed), "30-minute recovery was not started");
require(bridge.initializes == 1 && !bridge._network_suspended
&& !mesh::wifi::networkPower().backgroundSuspended()
&& sdk_mode == WIFI_STA, "recovery did not release station ownership");
for (int index = 0; index < RUNTIME_MQTT_SLOTS; ++index) {
require(!bridge._slots[index].circuit_breaker_tripped
&& MQTTConnectionPolicy::reconnectDue(resumed,
bridge._slots[index].last_reconnect_attempt,
bridge._slots[index].reconnect_backoff, static_cast<uint8_t>(index)),
"recovery left brokers waiting behind the old breaker");
}
require(bridge._ntp_sync_pending && bridge._ntp_refresh_retry_at == 0,
"recovery did not admit one new NTP attempt");
for (uint32_t offset = 1000U; offset < probe; offset += 1000U) {
require(bridge.serviceNetworkPower(resumed + offset), "probe ended early");
}
require(bridge.initializes == 1, "probe restarted station repeatedly");
require(!bridge.serviceNetworkPower(resumed + probe), "failed probe was unbounded");
require(!bridge.serviceNetworkPower(resumed + probe + interval - 1U),
"failed probe did not start a fresh 30-minute cooldown");
}
}
static void foregroundAndEspNowArePreserved() {
for (int owner = 0; owner < 6; ++owner) {
resetEnvironment();
MQTTBridge bridge;
bridge.serviceNetworkPower(100U);
if (owner == 0) mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, true);
if (owner == 1) bridge.board.ota = true;
if (owner == 2) sdk_mode = WIFI_STA | WIFI_MODE_AP;
if (owner == 3) bridge._ntp_force_requested = true;
if (owner == 4) bridge._ntp_diag_requested = true;
if (owner == 5) bridge.snmp.running = true;
require(bridge.serviceNetworkPower(100U + grace), "foreground owner lost radio");
require(WiFi.disconnects == 0 && WiFi.offs == 0,
"background outage disrupted a foreground owner");
}
resetEnvironment();
{
MQTTBridge bridge;
espnow_constrained = true;
bridge.serviceNetworkPower(100U);
require(!bridge.serviceNetworkPower(100U + grace), "failed STA did not suspend");
require(!WiFi.reconnect && WiFi.disconnects == 1 && WiFi.offs == 0
&& sdk_mode == WIFI_STA && channel_restores == 1,
"suspending failed internet disabled independent ESP-NOW radio");
}
// An AP can take ownership during the SDK's yielding client stop calls.
resetEnvironment();
{
MQTTBridge bridge;
bridge.serviceNetworkPower(100U);
on_client_stop = [] { sdk_mode = WIFI_STA | WIFI_MODE_AP; };
require(!bridge.serviceNetworkPower(100U + grace), "worker did not suspend transports");
require(WiFi.disconnects == 0 && WiFi.offs == 0 && (sdk_mode & WIFI_MODE_AP),
"foreground AP race lost the shared WiFi driver");
}
}
static void automaticClockEstimateCannotWakeSuspendedNetwork() {
resetEnvironment();
MQTTBridge bridge;
bridge.serviceNetworkPower(100U);
bridge._ntp_estimate_requested = true;
require(!bridge.serviceNetworkPower(100U + grace),
"automatic clock estimate bypassed combined outage grace");
require(!bridge.serviceNetworkPower(100U + grace + 1U),
"automatic clock estimate woke failed network immediately");
require(!bridge.serviceNetworkPower(100U + grace + interval - 1U),
"automatic clock estimate bypassed the 30-minute cooldown");
require(bridge.initializes == 0, "background clock request restarted station");
bridge._ntp_estimate_requested = false;
WiFi.state = WL_CONNECTED; // Even a stale connected facade cannot bypass it.
require(!bridge.requestNtpTimeEstimate() && !bridge._ntp_estimate_requested,
"automatic clock bootstrap queued new work during suspension");
require(bridge.serviceNetworkPower(100U + grace + interval),
"scheduled recovery did not admit clock work");
require(bridge.requestNtpTimeEstimate() && bridge._ntp_estimate_requested,
"scheduled recovery still blocked automatic clock bootstrap");
}
static void deferredEspNowChannelRepair() {
for (uint32_t start : {100U, UINT32_MAX - 500U, UINT32_MAX - 999U}) {
resetEnvironment();
MQTTBridge bridge;
espnow_constrained = true;
clock_ms = start - grace;
bridge.serviceNetworkPower(clock_ms);
clock_ms = start;
channel_restore_result = ESP_FAIL;
require(!bridge.serviceNetworkPower(clock_ms), "failed STA did not suspend");
require(bridge._network_espnow_restore_pending && channel_restores == 1
&& WiFi.disconnects == 1 && WiFi.offs == 0,
"failed channel write was not retained independently of WiFi stop");
for (uint32_t offset : {100U, 250U, 500U, 999U}) {
clock_ms = start + offset;
require(!bridge.serviceNetworkPower(clock_ms), "channel repair resumed failed internet");
require(channel_restores == 1 && WiFi.disconnects == 1,
"channel repair was unthrottled or repeated disconnect");
}
clock_ms = start + 1000U;
channel_restore_result = ESP_OK;
bridge.serviceEspNowChannelRestore(clock_ms);
require(channel_restores == 2 && !bridge._network_espnow_restore_pending
&& WiFi.disconnects == 1 && WiFi.offs == 0,
"transient channel failure did not recover without toggling WiFi");
for (uint32_t offset : {1100U, 2000U, 5000U}) {
clock_ms = start + offset;
require(!bridge.serviceNetworkPower(clock_ms), "repair canceled the outage cooldown");
require(channel_restores == 2 && WiFi.disconnects == 1,
"successful channel restoration retained repeated work");
}
}
resetEnvironment();
{
MQTTBridge bridge;
espnow_constrained = true;
bridge.serviceNetworkPower(100U);
clock_ms = 100U + grace;
channel_restore_result = ESP_FAIL;
bridge.serviceNetworkPower(clock_ms);
clock_ms += 1000U;
mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, true);
bridge.serviceEspNowChannelRestore(clock_ms);
mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, false);
sdk_mode = WIFI_STA | WIFI_MODE_AP;
bridge.serviceEspNowChannelRestore(clock_ms);
sdk_mode = WIFI_STA;
bridge.board.ota = true;
bridge.serviceEspNowChannelRestore(clock_ms);
bridge.board.ota = false;
sdk_mode_result = ESP_FAIL;
bridge.serviceEspNowChannelRestore(clock_ms);
require(channel_restores == 1 && bridge._network_espnow_restore_pending,
"deferred restoration changed a foreground or unknown driver channel");
sdk_mode_result = ESP_OK;
channel_restore_result = ESP_OK;
bridge.serviceEspNowChannelRestore(clock_ms);
require(channel_restores == 2 && !bridge._network_espnow_restore_pending,
"released foreground owner stranded channel restoration");
}
resetEnvironment();
{
MQTTBridge bridge;
espnow_constrained = true;
bridge.serviceNetworkPower(100U);
clock_ms = 100U + grace;
channel_restore_result = ESP_FAIL;
bridge.serviceNetworkPower(clock_ms);
espnow_constrained = false;
clock_ms += 1000U;
bridge.serviceEspNowChannelRestore(clock_ms);
require(channel_restores == 1 && !bridge._network_espnow_restore_pending,
"stopped ESP-NOW owner retained a channel mutation");
}
}
static void stableBrokerCountsButFlappingDoesNot() {
resetEnvironment();
{
MQTTBridge bridge;
bridge.serviceNetworkPower(100U);
bridge._slots[0].connected = true;
bridge._slots[0].connected_at_ms = 101U;
require(bridge.serviceNetworkPower(30101U), "30-second quiet broker was not useful");
require(bridge.serviceNetworkPower(100U + 10U * grace), "working broker lost WiFi");
require(WiFi.offs == 0, "NTP failure overrode working MQTT");
const uint32_t failed_at = 100U + 10U * grace;
bridge._slots[0].connected = false;
require(bridge.serviceNetworkPower(failed_at + grace - 1U),
"real broker success did not renew outage grace");
require(!bridge.serviceNetworkPower(failed_at + grace),
"disconnected formerly healthy broker kept WiFi indefinitely");
}
resetEnvironment();
{
MQTTBridge bridge;
bridge.serviceNetworkPower(100U);
for (uint32_t offset = 1000U; offset < grace; offset += 1000U) {
bridge._slots[0].connected = true;
bridge._slots[0].connected_at_ms = 100U + offset - 1U;
require(bridge.serviceNetworkPower(100U + offset), "grace ended early");
}
bridge._slots[0].connected_at_ms = 100U + grace - 1U;
require(!bridge.serviceNetworkPower(100U + grace),
"flapping broker repeatedly renewed the background outage");
}
}
static void publishAcknowledgmentProvesRecoveryImmediately() {
resetEnvironment();
MQTTBridge bridge;
bridge.serviceNetworkPower(100U);
require(!bridge.serviceNetworkPower(100U + grace), "initial outage did not suspend");
const uint32_t resumed = 100U + grace + interval;
require(bridge.serviceNetworkPower(resumed), "recovery probe did not begin");
// A broker reply during the final second proves useful service immediately,
// even though it has not met the quiet 30-second connection threshold.
bridge._slots[0].connected = true;
bridge._slots[0].connected_at_ms = resumed + probe - 1U;
bridge.clients[0].publish_acks = 1U;
require(bridge.serviceNetworkPower(resumed + probe - 1U),
"PUBACK failed to prove useful network");
bridge._slots[0].connected = false;
require(bridge.serviceNetworkPower(resumed + probe),
"successful recovery retained its old short probe deadline");
require(bridge.serviceNetworkPower(resumed + probe - 1U + grace - 1U),
"PUBACK failed to renew a normal outage window");
require(!bridge.serviceNetworkPower(resumed + probe - 1U + grace),
"an old PUBACK repeatedly renewed a disconnected broker");
}
static void quietConnectionHasExactThirtySecondHealthBoundary() {
resetEnvironment();
{
MQTTBridge bridge;
bridge.serviceNetworkPower(100U);
bridge._slots[0].connected = true;
bridge._slots[0].connected_at_ms = 200U;
require(bridge.serviceNetworkPower(30199U), "initial grace ended early");
bridge._slots[0].connected = false;
require(!bridge.serviceNetworkPower(100U + grace),
"29.999-second connection falsely renewed outage grace");
}
resetEnvironment();
{
MQTTBridge bridge;
bridge.serviceNetworkPower(100U);
bridge._slots[0].connected = true;
bridge._slots[0].connected_at_ms = 200U;
require(bridge.serviceNetworkPower(30200U), "30-second connection was not admitted");
bridge._slots[0].connected = false;
require(bridge.serviceNetworkPower(30200U + grace - 1U),
"quiet 30-second broker did not renew useful-service grace");
require(!bridge.serviceNetworkPower(30200U + grace),
"formerly quiet broker kept failed service awake indefinitely");
}
resetEnvironment();
{
MQTTBridge bridge;
bridge.serviceNetworkPower(0U);
bridge._slots[0].connected = true;
bridge._slots[0].connected_at_ms = 0U;
require(bridge.serviceNetworkPower(30000U), "zero-time connection was not admitted");
bridge._slots[0].connected = false;
require(bridge.serviceNetworkPower(30000U + grace - 1U),
"valid zero-time broker connection was mistaken for unset state");
require(!bridge.serviceNetworkPower(30000U + grace),
"disconnected zero-time broker retained WiFi indefinitely");
}
}
static void everyNtpServerGetsTheLeadingTimeoutSlot() {
resetEnvironment();
MQTTBridge bridge;
const char* expected[] = {"custom", "pool", "google", "cloudflare", "aws", "nist", "custom"};
for (const char* primary : expected) {
bridge.refreshNTP();
require(ntp_primaries.back() == primary,
"a blackholed leading server starved configured NTP fallback servers");
clock_ms += 15000U;
bridge.pollNtpRefresh(clock_ms);
require(!bridge._ntp_refresh_pending,
"failed rotating NTP request exceeded its bounded timeout");
clock_ms = bridge._ntp_refresh_retry_at;
}
require(ntp_configs == 7, "NTP rotation skipped or repeated a timeout slot");
}
static void sntpFailureAndBusyLeaseRetries() {
for (uint32_t start : {100U, UINT32_MAX - 4000U, UINT32_MAX - interval + 1U}) {
resetEnvironment();
MQTTBridge bridge;
clock_ms = start;
bridge.serviceBackgroundNtp(clock_ms);
require(ntp_configs == 1 && bridge._ntp_refresh_pending, "boot NTP did not start");
clock_ms += 15000U;
bridge.serviceBackgroundNtp(clock_ms);
const uint32_t failed_at = clock_ms;
const uint32_t retry = mqttNtpFailureRetryAt(failed_at);
require(bridge._ntp_refresh_retry_at == retry && !bridge._ntp_refresh_pending,
"failed SNTP kept running or lost its bounded retry");
clock_ms = failed_at + interval - 1U;
bridge.serviceBackgroundNtp(clock_ms);
require(ntp_configs == 1, "failed SNTP retried before 30 minutes");
clock_ms = retry;
bridge.serviceBackgroundNtp(clock_ms);
require(ntp_configs == 2 && bridge._ntp_refresh_pending, "SNTP retry never became due");
}
resetEnvironment();
{
MQTTBridge bridge;
mesh::sntp_coord::OperationLease foreground_ntp(mesh::sntp_coord::processWideCoordinator());
require(foreground_ntp.tryAcquire(), "foreground SNTP owner did not acquire");
bridge.serviceBackgroundNtp(clock_ms);
require(ntp_configs == 0 && !bridge._ntp_refresh_pending
&& bridge._ntp_refresh_retry_at == clock_ms + 5000U,
"SNTP contention was treated as failed internet or busy-waited");
clock_ms += 4999U;
foreground_ntp.release();
bridge.serviceBackgroundNtp(clock_ms);
require(ntp_configs == 0, "SNTP busy-owner retry was not bounded");
clock_ms += 1U;
bridge.serviceBackgroundNtp(clock_ms);
require(ntp_configs == 1, "released SNTP owner never admitted retry");
}
}
static void retainedClockAndFreshNtpDoNotCreditFailedMqtt() {
resetEnvironment();
MQTTBridge bridge;
bridge.rtc.epoch = MQTT_NTP_MIN_VALID_EPOCH + 100U;
require(bridge.serviceNetworkPower(100U), "initial policy did not run");
bridge.serviceBackgroundNtp(100U);
require(bridge._ntp_synced && !bridge._fresh_ntp_this_boot,
"retained RTC became fake proof of fresh internet");
require(ntp_configs == 1 && bridge._ntp_refresh_pending,
"retained clock suppressed fresh NTP request");
system_epoch = MQTT_NTP_MIN_VALID_EPOCH + 200U;
ntp_status = SNTP_SYNC_STATUS_COMPLETED;
clock_ms = 101U;
bridge.serviceBackgroundNtp(clock_ms);
require(bridge._fresh_ntp_this_boot && bridge._last_ntp_sync == clock_ms
&& !bridge._ntp_refresh_pending && ntp_stops == 1,
"fresh NTP did not update the RTC and release SNTP");
require(bridge.serviceNetworkPower(100U + grace - 1U), "outage grace ended early");
require(!bridge.serviceNetworkPower(100U + grace),
"successful NTP kept failed MQTT's WiFi on indefinitely");
}
int main(int argc, char** argv) {
require(argc == 2, "one case is required");
const std::string name = argv[1];
try {
if (name == "handoff") failureStopsBothServicesBeforeWifi();
else if (name == "failed-stop") failedSdkStopWithholdsHandoff();
else if (name == "recovery") recoveryHasOneBoundedWindow();
else if (name == "owners") foregroundAndEspNowArePreserved();
else if (name == "background-estimate") automaticClockEstimateCannotWakeSuspendedNetwork();
else if (name == "channel-repair") deferredEspNowChannelRepair();
else if (name == "healthy") stableBrokerCountsButFlappingDoesNot();
else if (name == "puback") publishAcknowledgmentProvesRecoveryImmediately();
else if (name == "quiet-boundary") quietConnectionHasExactThirtySecondHealthBoundary();
else if (name == "ntp-retry") sntpFailureAndBusyLeaseRetries();
else if (name == "ntp-rotation") everyNtpServerGetsTheLeadingTimeoutSlot();
else if (name == "ntp-proof") retainedClockAndFreshNtpDoNotCreditFailedMqtt();
else return 2;
} catch (const std::exception& error) {
std::cerr << name << ": " << error.what() << '\n';
return 1;
}
std::cout << "PASS: " << name << '\n';
return 0;
}
'''
PORTABLE_MAIN = r'''
int main() {
try {
resetEnvironment();
MQTTBridge bridge;
clock_ms = UINT32_MAX - 1000U;
bridge.refreshNTP();
require(ntp_configs == 1 && bridge._ntp_refresh_pending,
"portable async SNTP request did not start");
require(!bridge._ntp_refresh_operation.owns(),
"portable stack unexpectedly acquired ESP32 SNTP lease");
clock_ms += 4999U;
bridge.pollNtpRefresh(clock_ms);
require(bridge._ntp_refresh_pending && !bridge._ntp_synced,
"portable stack accepted clock before five-second settlement");
system_epoch = MQTT_NTP_MIN_VALID_EPOCH + 200U;
clock_ms += 1U;
bridge.pollNtpRefresh(clock_ms);
require(!bridge._ntp_refresh_pending && bridge._ntp_synced
&& bridge._fresh_ntp_this_boot && bridge.rtc.epoch == system_epoch
&& bridge._last_ntp_sync == clock_ms,
"portable stack lost its existing settled-clock behavior");
bridge.refreshNTP();
WiFi.state = WL_DISCONNECTED;
bridge.pollNtpRefresh(clock_ms);
require(!bridge._ntp_refresh_pending
&& bridge._ntp_refresh_retry_at == clock_ms + 5000U,
"ESP32 outage policy changed portable WiFi-failure retry");
require(ntp_stops == 0, "portable SNTP called ESP32 SDK cleanup");
require(mqttNtpFailureRetryAt(UINT32_MAX - 4999U) == 1U,
"portable retry lost its wrap-safe zero-sentinel handling");
require(mqttNtpRetryAt(clock_ms) == clock_ms + 5000U,
"portable busy-owner retry unexpectedly changed");
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';
return 1;
}
std::cout << "PASS: portable\n";
return 0;
}
'''
class MqttWiFiNetworkPowerTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++")
if not compiler:
raise RuntimeError("a host C++ compiler is required")
helpers = "\n".join(method(BRIDGE, signature) for signature in (
"static uint32_t mqttNtpFailureRetryAt(",
"static bool mqttNtpRefreshDue(",
"static uint32_t mqttNtpRetryAt(",
"static void stopMqttSntpOperation(",
))
methods = "\n".join(method(BRIDGE, signature) for signature in (
"bool MQTTBridge::serviceNetworkPower(",
"void MQTTBridge::serviceEspNowChannelRestore(",
"void MQTTBridge::serviceBackgroundNtp(",
"void MQTTBridge::cancelNtpRefresh(",
"void MQTTBridge::refreshNTP(",
"void MQTTBridge::pollNtpRefresh(",
"bool MQTTBridge::requestNtpTimeEstimate(",
))
cls.temporary = tempfile.TemporaryDirectory(prefix="meshcore-mqtt-network-power-")
cls.addClassCleanup(cls.temporary.cleanup)
directory = Path(cls.temporary.name)
cpp = directory / "power.cpp"
source = PREAMBLE.replace("@HELPERS@", helpers).replace("@METHODS@", methods)
cpp.write_text(source, encoding="ascii")
cls.binary = directory / "power"
built = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", "-Werror",
"-I", str(ROOT / "src"), str(cpp), "-o", str(cls.binary)],
capture_output=True, text=True, timeout=60)
if built.returncode:
raise RuntimeError(built.stdout + built.stderr)
# The portable stack keeps its previous five-second settle and retry;
# the new 30-minute failed-network policy is ESP32-only.
portable_methods = "\n".join(method(BRIDGE, signature) for signature in (
"void MQTTBridge::cancelNtpRefresh(",
"void MQTTBridge::refreshNTP(",
"void MQTTBridge::pollNtpRefresh(",
))
portable_source = PREAMBLE[:PREAMBLE.index("static constexpr uint32_t grace")]
portable_source = (portable_source.replace("#define ESP_PLATFORM 1", "")
.replace("@HELPERS@", helpers)
.replace("@METHODS@", portable_methods) + PORTABLE_MAIN)
portable_cpp = directory / "portable.cpp"
portable_cpp.write_text(portable_source, encoding="ascii")
cls.portable_binary = directory / "portable"
built = subprocess.run([compiler, "-std=c++17", "-Wall", "-Wextra", "-Werror",
"-I", str(ROOT / "src"), str(portable_cpp),
"-o", str(cls.portable_binary)],
capture_output=True, text=True, timeout=60)
if built.returncode:
raise RuntimeError(built.stdout + built.stderr)
def run_case(self, name):
checked = subprocess.run([str(self.binary), name], capture_output=True,
text=True, timeout=10)
self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr)
self.assertIn("PASS: " + name, checked.stdout)
def test_failed_ntp_and_mqtt_quiesce_before_wifi_shutdown_publication(self):
self.run_case("handoff")
def test_failed_sdk_transport_stop_keeps_station_owned_and_awake(self):
self.run_case("failed-stop")
def test_recovery_restarts_once_and_stops_after_bounded_probe_across_wrap(self):
self.run_case("recovery")
def test_foreground_and_espnow_owners_preserve_driver_even_during_stop_race(self):
self.run_case("owners")
def test_automatic_clock_estimate_cannot_bypass_background_outage_cooldown(self):
self.run_case("background-estimate")
def test_failed_espnow_channel_restore_retries_without_wifi_toggles_or_owner_conflicts(self):
self.run_case("channel-repair")
def test_stable_mqtt_counts_as_useful_and_flapping_mqtt_cannot_renew(self):
self.run_case("healthy")
def test_publish_ack_proves_recovery_immediately_and_is_consumed_once(self):
self.run_case("puback")
def test_quiet_broker_health_requires_exactly_thirty_seconds(self):
self.run_case("quiet-boundary")
def test_sntp_failure_and_busy_lease_have_different_bounded_retries(self):
self.run_case("ntp-retry")
def test_all_six_ntp_servers_lead_a_bounded_request_before_rotation_repeats(self):
self.run_case("ntp-rotation")
def test_retained_clock_or_fresh_ntp_cannot_keep_failed_mqtt_wifi_active(self):
self.run_case("ntp-proof")
def test_portable_non_esp_stack_preserves_settle_and_short_retry_policy(self):
checked = subprocess.run([str(self.portable_binary)], capture_output=True,
text=True, timeout=10)
self.assertEqual(checked.returncode, 0, checked.stdout + checked.stderr)
self.assertIn("PASS: portable", checked.stdout)
if __name__ == "__main__":
unittest.main()
+23 -1
View File
@@ -28,6 +28,7 @@ HARNESS = r'''
#include <helpers/RoomWebMailbox.h>
#include <helpers/CLICommandUtils.h>
#include <helpers/WebConfigBatch.h>
#include <helpers/WiFiNetworkPowerPolicy.h>
using String = std::string;
#define PROGMEM
@HTML@
@@ -153,9 +154,15 @@ struct WebConfigServer {
static constexpr size_t MAX_BODY = @MAX_BODY@;
static constexpr uint32_t STOP_WARN_MS = WebConfigBatch::kStopWarnMs;
enum Mode { MODE_OFF, MODE_SETUP, MODE_LAN };
enum { BATCH_IDLE, BATCH_PENDING };
Mode _mode = MODE_LAN;
unsigned _batch_state = BATCH_IDLE;
bool _setup_wifi_handoff_pending = false;
struct Terminal { bool running = false; struct { bool pending = false; } queue; };
Terminal* _terminal = nullptr;
bool _stopping = false, _stop_warned = false, _board_cmds_probed = true;
uint32_t _handler_refs = 0, _stop_warn_at = 0, _last_activity = 0;
uint32_t _network_activity_until = 0;
std::mutex mutex;
SemaphoreHandle_t _mux = &mutex;
mesh::RoomWebMailbox* _room_mailbox = nullptr;
@@ -170,11 +177,16 @@ struct WebConfigServer {
if (allocate) _room_mailbox = new mesh::RoomWebMailbox;
registerRoutes(); _active = this;
}
~WebConfigServer() { if (_active == this) _active = nullptr; delete _room_mailbox; }
~WebConfigServer() {
if (_active == this) _active = nullptr;
delete _room_mailbox;
mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, false);
}
void finalizeTeardown() { // Dependency stub: production tick decides when reclaim is safe.
assert(_handler_refs == 0); ++finalizations;
delete _room_mailbox; _room_mailbox = nullptr; _server = nullptr;
_stopping = false; _cb->onWebConfigStopped();
_network_activity_until = 0; refreshNetworkPowerHold(millis());
}
void probeBoardCommands() { _board_cmds_probed = true; }
void serviceTerminal(uint32_t) {}
@@ -183,6 +195,9 @@ struct WebConfigServer {
void requestStop();
void detachRoutes();
uint32_t handlerRefCount() const;
void noteNetworkActivity();
bool isNetworkForegroundActive(uint32_t) const;
void refreshNetworkPowerHold(uint32_t);
void registerRoutes();
static void dispatchRequest(AsyncWebServerRequest*, RequestHandler);
static void collectBody(AsyncWebServerRequest*, uint8_t*, size_t, size_t, size_t);
@@ -342,6 +357,8 @@ static void boundsAndFloors() {
static void pagesAndAvailability() {
Callbacks cb; WebConfigServer server(cb); AsyncWebServerRequest page;
server.host.invoke("/room", HTTP_GET, page); assert(page.status == 200);
assert(server.isNetworkForegroundActive(ticks));
assert(mesh::wifi::networkPower().foregroundActive());
assert(page.reply.find("MeshCore Room") != String::npos && page.response_type == "text/html; charset=utf-8");
assert(page.response_headers["Cache-Control"] == "no-store");
assert(page.response_headers["X-Content-Type-Options"] == "nosniff" && page.response_headers["Referrer-Policy"] == "no-referrer");
@@ -360,12 +377,14 @@ static void lifecycle() {
assert(server.handlerRefCount() == 1);
server.requestStop(); server.tick(ticks);
assert(server._stopping && server._room_mailbox && server.finalizations == 0);
assert(mesh::wifi::networkPower().foregroundActive());
assert(cb.calls == 0);
};
submit(server, pending); assert(pending.status == 202 && server.host.ended);
assert(server.handlerRefCount() == 0 && server._stopping && cb.calls == 0);
AsyncWebServerRequest unavailable; identity(unavailable); result(server, unavailable); assert(unavailable.status == 503);
server.tick(ticks); assert(!server._room_mailbox && !server._stopping && cb.stops == 1 && server.finalizations == 1);
assert(!mesh::wifi::networkPower().foregroundActive());
assert(cb.calls == 0); // Pending work is canceled, never executed after stop.
Callbacks off_cb; WebConfigServer off(off_cb); AsyncWebServerRequest post; identity(post); body(post, "{}"); submit(off, post);
off._mode = WebConfigServer::MODE_OFF; off.tick(ticks); assert(off_cb.calls == 0);
@@ -435,6 +454,9 @@ class RoomWebHttpTests(unittest.TestCase):
"void WebConfigServer::handleRoomResult(", "void WebConfigServer::collectBody(",
"void WebConfigServer::dispatchRequest(", "void WebConfigServer::detachRoutes(",
"uint32_t WebConfigServer::handlerRefCount(", "void WebConfigServer::requestStop(",
"void WebConfigServer::noteNetworkActivity(",
"bool WebConfigServer::isNetworkForegroundActive(",
"void WebConfigServer::refreshNetworkPowerHold(",
"void WebConfigServer::registerRoutes(")]
tick = extract_braced(source, "void WebConfigServer::tick(")
# Execute the exact stop/OFF guards and loop service calls; SDK Wi-Fi,
+5
View File
@@ -187,9 +187,14 @@ class RS232MQTTRuntimeTests(unittest.TestCase):
MyMesh mesh; mesh.defaults(); mesh.boot(); CommonCLI cli(mesh);
expect_get(cli, "bridge.type", "> mqtt+rs232");
assert(mesh.isMqttBridgeRunning() && !mesh.isRs232BridgeRunning());
mesh.mqtt_bridge->setNetworkSuspended(true);
assert(mesh.isMqttBridgeRunning() && mesh._prefs.bridge_enabled);
assert(!mesh.bridgesPreventSleep());
char reply[256] = {};
cli.set("rs232.enabled on", reply); assert(strcmp(reply, "OK") == 0);
assert(mesh.isRs232BridgeRunning() && mesh.isMqttBridgeRunning());
assert(mesh.bridgesPreventSleep()); // Paused MQTT cannot mask an active UART.
mesh.mqtt_bridge->setNetworkSuspended(false);
const unsigned starts = mesh.mqtt_bridge->begins;
cli.set("bridge.baud 57600", reply); assert(strcmp(reply, "OK") == 0);
assert(mesh.isRs232BridgeRunning() && mesh.mqtt_bridge->begins == starts);
+12 -5
View File
@@ -190,7 +190,9 @@ class TlsDownloadClockGateTest(unittest.TestCase):
)
self.assertIn("SntpOperationCoordinator.h", implementation)
self.assertEqual(len(re.findall(r"(?m)^\s*configTime\(", implementation)), 2)
# Background refresh has one ESP32 three-server branch and one portable
# single-server branch; the explicit administrator sync is separate.
self.assertEqual(len(re.findall(r"(?m)^\s*configTime\(", implementation)), 3)
self.assertEqual(
len(re.findall(r"(?m)^\s*sntp_set_sync_status\(", implementation)),
2,
@@ -321,10 +323,15 @@ class TlsDownloadClockGateTest(unittest.TestCase):
header,
)
self.assertIn("MQTT_NTP_REFRESH_TIMEOUT_MS = 15000UL", implementation)
self.assertEqual(implementation.count("pollNtpRefresh("), 3)
self.assertGreaterEqual(
implementation.count("&& !_ntp_refresh_pending"), 2
self.assertEqual(implementation.count("pollNtpRefresh("), 4)
background = section(
implementation,
"void MQTTBridge::serviceBackgroundNtp(",
"void MQTTBridge::mqttTaskLoop()",
)
self.assertIn("if (_ntp_refresh_pending) return;", background)
self.assertLess(background.index("if (_ntp_refresh_pending) return;"),
background.index("refreshNTP();"))
reset = refresh.index("sntp_set_sync_status(SNTP_SYNC_STATUS_RESET)")
configure = refresh.index(
@@ -351,7 +358,7 @@ class TlsDownloadClockGateTest(unittest.TestCase):
self.assertIn("_ntp_refresh_operation.owns()", poll)
timeout = poll.index("elapsed >= MQTT_NTP_REFRESH_TIMEOUT_MS")
retry = poll.index("mqttNtpRetryAt(now)", timeout)
retry = poll.index("mqttNtpFailureRetryAt(now)", timeout)
self.assertLess(timeout, retry)
self.assertNotIn("_last_ntp_sync =", poll[timeout:])
+151
View File
@@ -0,0 +1,151 @@
"""Exercise the production portal's bounded foreground ownership policy."""
from pathlib import Path
import unittest
from test_replay_reset_integration import extract_braced
import test_wifi_ota_start as wifi_start
ROOT = Path(__file__).resolve().parents[1]
class WebConfigNetworkPowerTest(unittest.TestCase):
compile_and_run = wifi_start.WiFiOtaStartTest.compile_and_run
def fixture(self):
source = (ROOT / "src/helpers/esp32/WebConfigServer.cpp").read_text()
methods = "\n".join(extract_braced(source, signature) for signature in (
"void WebConfigServer::noteNetworkActivity(",
"void WebConfigServer::requestNetworkAccess(",
"bool WebConfigServer::isNetworkForegroundActive(",
"void WebConfigServer::refreshNetworkPowerHold("))
return r'''
#include <cassert>
#include <cstdint>
#include <helpers/WebConfigBatch.h>
#include <helpers/WiFiNetworkPowerPolicy.h>
#include <helpers/RoomWebMailbox.h>
uint32_t now = 1000;
uint32_t millis() { return now; }
struct WCLock { explicit WCLock(void*) {} };
struct WebConfigServer {
enum { MODE_OFF, MODE_SETUP, MODE_CONNECTING, MODE_LAN };
enum { BATCH_IDLE, BATCH_PENDING, BATCH_DONE };
int _mode = MODE_LAN, _batch_state = BATCH_IDLE;
bool _stopping = false, _setup_wifi_handoff_pending = false;
uint32_t _last_activity = 1000, _network_activity_until = 0;
unsigned _handler_refs = 0;
void* _mux = nullptr;
struct Terminal { bool running = false; struct { bool pending = false; } queue; };
Terminal* _terminal = nullptr;
mesh::RoomWebMailbox* _room_mailbox = nullptr;
unsigned handlerRefCount() const { return _handler_refs; }
void noteNetworkActivity();
void requestNetworkAccess();
bool isNetworkForegroundActive(uint32_t) const;
void refreshNetworkPowerHold(uint32_t);
};
@METHODS@
int main() {
WebConfigServer portal;
auto& power = mesh::wifi::networkPower();
power.setManaged(true); power.setSuspended(true);
// Boot/connection timestamps and a listener alone never hold failed WiFi.
portal._last_activity = now;
assert(!portal.isNetworkForegroundActive(now));
portal.refreshNetworkPowerHold(now);
assert(!power.foregroundActive());
portal.requestNetworkAccess();
assert(power.consumeProbeRequest() && power.foregroundActive());
now += 119999;
assert(portal.isNetworkForegroundActive(now));
++now; portal.refreshNetworkPowerHold(now);
assert(!portal.isNetworkForegroundActive(now) && !power.foregroundActive());
// The bounded lease works across millis() rollover.
now = UINT32_MAX - 100;
portal.noteNetworkActivity();
now += 1000;
assert(portal.isNetworkForegroundActive(now));
now += 119000;
assert(!portal.isNetworkForegroundActive(now));
// Setup, teardown and in-flight handlers/batches survive lease expiry.
portal._mode = WebConfigServer::MODE_SETUP;
assert(portal.isNetworkForegroundActive(now));
portal._mode = WebConfigServer::MODE_LAN;
portal._handler_refs = 1;
assert(portal.isNetworkForegroundActive(now));
portal._handler_refs = 0; portal._batch_state = WebConfigServer::BATCH_PENDING;
assert(portal.isNetworkForegroundActive(now));
portal._batch_state = WebConfigServer::BATCH_DONE;
assert(!portal.isNetworkForegroundActive(now));
portal._setup_wifi_handoff_pending = true;
assert(portal.isNetworkForegroundActive(now));
portal._setup_wifi_handoff_pending = false;
WebConfigServer::Terminal terminal;
portal._terminal = &terminal; terminal.running = true;
assert(portal.isNetworkForegroundActive(now));
terminal.running = false; terminal.queue.pending = true;
assert(portal.isNetworkForegroundActive(now));
terminal.queue.pending = false;
assert(!portal.isNetworkForegroundActive(now));
mesh::RoomWebMailbox mailbox; uint8_t token[32] = {1}, digest[32] = {2};
portal._room_mailbox = &mailbox;
assert(mailbox.submit(token, 1, digest, "{}", 2, now)
== mesh::RoomWebMailbox::Result::Accepted);
assert(portal.isNetworkForegroundActive(now));
assert(mailbox.begin() && portal.isNetworkForegroundActive(now));
mailbox.finish(now);
assert(!portal.isNetworkForegroundActive(now));
portal._mode = WebConfigServer::MODE_OFF; portal._stopping = true;
assert(portal.isNetworkForegroundActive(now));
portal._stopping = false;
portal.refreshNetworkPowerHold(now);
assert(!power.foregroundActive());
power.setManaged(false);
}
'''.replace("@METHODS@", methods)
def test_actual_activity_setup_and_inflight_work_hold_with_bounded_idle(self):
self.compile_and_run(self.fixture())
def test_status_reports_shared_cooldown_before_connecting_or_off(self):
source = (ROOT / "src/helpers/esp32/WebConfigServer.cpp").read_text()
formatter = extract_braced(source, "bool WebConfigServer::formatWiFiStatus(")
block = extract_braced(formatter,
" if (mesh::wifi::networkPower().backgroundSuspended()")
self.assertLess(formatter.index(block),
formatter.index("active->_mode == MODE_CONNECTING"))
fixture = r'''
#include <cassert>
#include <cstdio>
#include <cstring>
#include <helpers/WiFiNetworkPowerPolicy.h>
void appendOtaSeederStatus(char* reply, size_t capacity) {
size_t used = strlen(reply);
snprintf(reply + used, capacity - used, "; seeder status");
}
bool format(char* reply, size_t reply_len, bool configured) {
const char* ssid = "working-network";
@BLOCK@
snprintf(reply, reply_len, "> connecting");
return true;
}
int main() {
char reply[160]; auto& power = mesh::wifi::networkPower();
power.setManaged(true); power.setSuspended(true);
assert(format(reply, sizeof(reply), true));
assert(strcmp(reply, "> suspended; recovery every 30 minutes, SSID: working-network; seeder status") == 0);
assert(format(reply, sizeof(reply), false));
assert(strstr(reply, "SSID: (not configured); seeder status"));
power.setHold(mesh::wifi::NetworkPowerHold::WebConfig, true);
format(reply, sizeof(reply), true); assert(strcmp(reply, "> connecting") == 0);
power.setHold(mesh::wifi::NetworkPowerHold::WebConfig, false);
power.setSuspended(false);
format(reply, sizeof(reply), true); assert(strcmp(reply, "> connecting") == 0);
power.setManaged(false);
}
'''.replace("@BLOCK@", block)
self.compile_and_run(fixture)
if __name__ == "__main__":
unittest.main()
+1 -1
View File
@@ -54,7 +54,7 @@ class WebConfigRecoveryAuthTest(unittest.TestCase):
def test_old_interface_only_fallback_fails_executable_regression(self):
fixture = self.fixture()
old = fixture.replace("if (_initial_setup || _wifi_ssid[0] == 0) return true;", "return true;", 1)
old = fixture.replace("if (_initial_setup || _wifi_ssid[0] == 0) { noteNetworkActivity(); return true; }", "return true;", 1)
self.assertNotEqual(old, fixture)
self.compile_run(old, expect_rejection=True)
+4
View File
@@ -87,6 +87,10 @@ class WebConfigStationOwnershipTest(unittest.TestCase):
def test_non_mqtt_portal_retains_saved_station_recovery(self):
self.compile_and_run(self.station_fixture())
def test_shared_outage_blocks_idle_portal_reconnect_and_allows_foreground(self):
self.compile_and_run(self.station_fixture(), "-DWITH_MQTT_BRIDGE=1",
"-DTEST_BACKGROUND_SUSPEND=1")
def test_manual_companion_start_preserves_legacy_join_and_15s_fallback(self):
self.compile_and_run(self.station_fixture(), "-DWITH_MQTT_BRIDGE=1",
"-DTEST_LEGACY_COMPANION=1")
+405
View File
@@ -0,0 +1,405 @@
#!/usr/bin/env python3
"""Run the shared WiFi outage policy against deterministic clocks and owners.
The production header is compiled directly, without Arduino or PlatformIO.
Independent translation units and concurrent service holds exercise the actual
handoff used between the network worker and foreground service owners.
"""
from pathlib import Path
import os
import shutil
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
HARNESS = r'''
#include <helpers/WiFiNetworkPowerPolicy.h>
#include <atomic>
#include <cstring>
#include <iostream>
#include <stdexcept>
#include <thread>
#include <vector>
using mesh::wifi::NetworkPowerControl;
using mesh::wifi::NetworkPowerHold;
using mesh::wifi::NetworkPowerPolicy;
using mesh::wifi::networkPower;
using Action = NetworkPowerPolicy::Action;
extern NetworkPowerControl* otherControl();
extern void otherSetWebHold(bool value);
extern bool otherSuspended();
static void require(bool value, const char* reason) {
if (!value) throw std::runtime_error(reason);
}
static void expect(NetworkPowerPolicy& policy, uint32_t now,
bool useful, bool foreground, Action expected,
bool probe = false) {
require(policy.update(now, useful, foreground, probe) == expected,
"unexpected network power transition");
require(policy.suspended() == (expected == Action::Suspend),
"transition and suspension state disagree");
}
// Keep expected timings independent of the production constants. Altering a
// timeout must be an intentional policy change, rather than a self-passing test.
static const uint32_t grace = 300000U;
static const uint32_t probe_budget = 120000U;
static const uint32_t cooldown = 1800000U;
static void graceBoundary() {
for (uint32_t start : {0U, 1234U, UINT32_MAX - 10000U}) {
NetworkPowerPolicy policy;
expect(policy, start, false, false, Action::Run);
expect(policy, start + grace - 1U, false, false, Action::Run);
expect(policy, start + grace, false, false, Action::Suspend);
expect(policy, start + grace + 1U, false, false, Action::Suspend);
}
}
static void failedServicesCannotRenew() {
// Serial event stream: associations succeed, SNTP alternates success/failure,
// and MQTT handshakes keep failing. None means the configured broker works.
enum Event { GotIp, NtpTimeout, NtpReply, MqttConnectFailure, MqttRetry };
const Event events[] = {GotIp, NtpTimeout, MqttConnectFailure,
NtpReply, MqttRetry};
for (uint32_t start : {0U, UINT32_MAX - 70000U}) {
NetworkPowerPolicy policy;
expect(policy, start, false, false, Action::Run);
for (uint32_t offset = 1000U; offset < grace; offset += 1000U) {
const Event event = events[(offset / 1000U) % 5U];
require(event == GotIp || event == NtpTimeout || event == NtpReply
|| event == MqttConnectFailure || event == MqttRetry,
"invalid simulated network event");
expect(policy, start + offset, false, false, Action::Run);
}
expect(policy, start + grace, false, false, Action::Suspend);
// Both services keep wanting retries. Their failure notifications cannot
// restart either the grace period or the 30-minute suspension clock.
for (uint32_t offset = 1000U; offset < cooldown; offset += 1000U) {
expect(policy, start + grace + offset, false, false, Action::Suspend);
}
expect(policy, start + grace + cooldown, false, false, Action::Resume);
}
}
static void successfulBrokerRenews() {
for (uint32_t start : {0U, UINT32_MAX - 200000U}) {
NetworkPowerPolicy policy;
expect(policy, start, false, false, Action::Run);
expect(policy, start + grace - 1U, true, false, Action::Run);
const uint32_t last_healthy = start + 2U * grace;
expect(policy, last_healthy, true, false, Action::Run);
expect(policy, last_healthy + grace - 1U, false, false, Action::Run);
expect(policy, last_healthy + grace, false, false, Action::Suspend);
// A genuine healthy connection observed during suspension cancels it and
// receives the normal grace budget, rather than a short recovery probe.
const uint32_t recovered = last_healthy + grace + 20U;
expect(policy, recovered, true, false, Action::Resume);
expect(policy, recovered + probe_budget, false, false, Action::Run);
expect(policy, recovered + grace, false, false, Action::Suspend);
}
}
static void boundedRecoveryCycles() {
for (uint32_t start : {0U, UINT32_MAX - 10000U}) {
NetworkPowerPolicy policy;
expect(policy, start, false, false, Action::Run);
uint32_t suspended_at = start + grace;
expect(policy, suspended_at, false, false, Action::Suspend);
for (unsigned cycle = 0; cycle < 20; ++cycle) {
expect(policy, suspended_at + cooldown - 1U, false, false,
Action::Suspend);
const uint32_t resumed_at = suspended_at + cooldown;
expect(policy, resumed_at, false, false, Action::Resume);
for (uint32_t offset = 1000U; offset < probe_budget; offset += 1000U) {
expect(policy, resumed_at + offset, false, false, Action::Run);
}
expect(policy, resumed_at + probe_budget - 1U, false, false,
Action::Run);
suspended_at = resumed_at + probe_budget;
expect(policy, suspended_at, false, false, Action::Suspend);
}
}
}
static void foregroundPreservesAccess() {
NetworkPowerPolicy policy;
expect(policy, 0, false, true, Action::Run);
expect(policy, grace, false, true, Action::Run);
expect(policy, 3U * grace, false, true, Action::Run);
// Closing the foreground session does not grant a new background grace.
expect(policy, 3U * grace + 1U, false, false, Action::Suspend);
expect(policy, 3U * grace + 2U, false, true, Action::Resume);
expect(policy, 3U * grace + 2U + probe_budget, false, true, Action::Run);
expect(policy, 3U * grace + 3U + probe_budget, false, false,
Action::Suspend);
}
static void manualProbeOnlyOnDemand() {
NetworkPowerPolicy policy;
expect(policy, 100U, false, false, Action::Run);
expect(policy, 100U + grace, false, false, Action::Suspend);
expect(policy, 101U + grace, false, false, Action::Suspend);
const uint32_t requested_at = 102U + grace;
expect(policy, requested_at, false, false, Action::Resume, true);
// Repeated manual requests while running cannot renew the probe deadline.
expect(policy, requested_at + probe_budget - 1U, false, false,
Action::Run, true);
expect(policy, requested_at + probe_budget, false, false,
Action::Suspend, true);
expect(policy, requested_at + probe_budget + 1U, false, false,
Action::Suspend);
expect(policy, requested_at + probe_budget + cooldown - 1U, false, false,
Action::Suspend);
expect(policy, requested_at + probe_budget + cooldown, false, false,
Action::Resume);
}
static void usefulProbeBecomesNormalOperation() {
NetworkPowerPolicy policy;
expect(policy, 0, false, false, Action::Run);
expect(policy, grace, false, false, Action::Suspend);
const uint32_t resumed_at = grace + cooldown;
expect(policy, resumed_at, false, false, Action::Resume);
const uint32_t recovered_at = resumed_at + probe_budget - 1U;
expect(policy, recovered_at, true, false, Action::Run);
expect(policy, recovered_at + probe_budget, false, false, Action::Run);
expect(policy, recovered_at + grace - 1U, false, false, Action::Run);
expect(policy, recovered_at + grace, false, false, Action::Suspend);
}
static void clearControl(NetworkPowerControl& control) {
control.setManaged(false);
control.setHold(NetworkPowerHold::CompanionClient, false);
control.setHold(NetworkPowerHold::WebConfig, false);
control.setHold(NetworkPowerHold::OtaSeeder, false);
control.consumeProbeRequest();
}
static void programWideRegistry() {
NetworkPowerControl& control = networkPower();
clearControl(control);
require(&control == otherControl(), "service translation units split registry");
control.setManaged(true);
control.setSuspended(true);
require(otherSuspended(), "worker suspension invisible to station owner");
otherSetWebHold(true);
require(control.foregroundActive(), "foreground hold invisible to worker");
control.setHold(NetworkPowerHold::OtaSeeder, true);
otherSetWebHold(false);
require(control.foregroundActive(), "closing web session cleared OTA hold");
control.setHold(NetworkPowerHold::OtaSeeder, false);
require(!control.foregroundActive(), "closed services retained WiFi lease");
control.requestProbe();
require(otherControl()->consumeProbeRequest(), "probe invisible across tasks");
require(!control.consumeProbeRequest(), "probe request was not one-shot");
control.setManaged(false);
require(!otherSuspended(), "disabled manager retained suspension");
}
static void controlLifecycle() {
NetworkPowerControl control;
require(!control.managed() && !control.backgroundSuspended()
&& !control.foregroundActive() && !control.consumeProbeRequest(),
"new control has active service state");
control.setSuspended(true);
require(!control.backgroundSuspended(), "unmanaged owner imposed suspension");
control.setManaged(true);
require(control.backgroundSuspended(), "managed suspension was lost");
control.requestProbe();
control.setManaged(false);
require(!control.backgroundSuspended() && !control.consumeProbeRequest(),
"stopping worker preserved stale suspension or probe");
control.setHold(NetworkPowerHold::CompanionClient, true);
control.setManaged(true);
require(!control.backgroundSuspended() && control.foregroundActive(),
"restarting manager lost foreground owner or kept stale suspension");
control.setHold(NetworkPowerHold::CompanionClient, false);
require(!control.foregroundActive(), "last foreground owner failed to release");
}
static void concurrentServiceHolds() {
NetworkPowerControl control;
const NetworkPowerHold owners[] = {NetworkPowerHold::CompanionClient,
NetworkPowerHold::WebConfig,
NetworkPowerHold::OtaSeeder};
for (unsigned round = 0; round < 256; ++round) {
std::atomic<unsigned> ready{0};
std::atomic<bool> go{false};
std::atomic<bool> release[3] = {};
std::atomic<bool> removed[3] = {};
std::vector<std::thread> threads;
for (unsigned index = 0; index < 3; ++index) {
threads.emplace_back([&, index]() {
ready.fetch_add(1, std::memory_order_release);
while (!go.load(std::memory_order_acquire)) std::this_thread::yield();
control.setHold(owners[index], true);
ready.fetch_add(1, std::memory_order_release);
while (!release[index].load(std::memory_order_acquire)) {
std::this_thread::yield();
}
control.setHold(owners[index], false);
removed[index].store(true, std::memory_order_release);
});
}
while (ready.load(std::memory_order_acquire) != 3U) std::this_thread::yield();
go.store(true, std::memory_order_release);
while (ready.load(std::memory_order_acquire) != 6U) std::this_thread::yield();
const bool opens_preserved = control.foregroundActive();
bool remaining_holds_preserved = true;
for (unsigned index = 0; index < 3; ++index) {
release[index].store(true, std::memory_order_release);
while (!removed[index].load(std::memory_order_acquire)) {
std::this_thread::yield();
}
if (index < 2 && !control.foregroundActive()) remaining_holds_preserved = false;
}
for (auto& thread : threads) thread.join();
require(opens_preserved, "simultaneous opens lost all leases");
require(remaining_holds_preserved, "one owner cleared another owner's lease");
require(!control.foregroundActive(), "concurrent closes leaked WiFi hold");
}
}
static void serviceLeaseEventSequence() {
NetworkPowerPolicy policy;
NetworkPowerControl control;
control.setManaged(true);
auto update = [&](uint32_t now, bool useful = false) {
const Action action = policy.update(now, useful, control.foregroundActive(),
control.consumeProbeRequest());
control.setSuspended(action == Action::Suspend);
return action;
};
require(update(0) == Action::Run, "network failed to start");
require(update(grace) == Action::Suspend, "outage left WiFi active");
control.setHold(NetworkPowerHold::WebConfig, true);
require(update(grace + 1U) == Action::Resume, "admin did not wake network");
control.setHold(NetworkPowerHold::OtaSeeder, true);
control.setHold(NetworkPowerHold::WebConfig, false);
require(update(grace + probe_budget + 1U) == Action::Run,
"closing admin page interrupted active OTA");
control.setHold(NetworkPowerHold::OtaSeeder, false);
require(update(grace + probe_budget + 2U) == Action::Suspend,
"completed OTA kept failed background network on");
control.requestProbe();
require(update(grace + probe_budget + 3U) == Action::Resume,
"explicit retry could not recover network");
require(!control.consumeProbeRequest(), "owner probe did not get consumed");
require(update(grace + probe_budget + 4U, true) == Action::Run,
"healthy broker did not resume normal service");
require(!control.backgroundSuspended(), "healthy broker left station suspended");
}
int main(int argc, char** argv) {
if (argc != 2) return 2;
try {
const char* name = argv[1];
if (!strcmp(name, "grace")) graceBoundary();
else if (!strcmp(name, "failures")) failedServicesCannotRenew();
else if (!strcmp(name, "healthy")) successfulBrokerRenews();
else if (!strcmp(name, "cycles")) boundedRecoveryCycles();
else if (!strcmp(name, "foreground")) foregroundPreservesAccess();
else if (!strcmp(name, "manual")) manualProbeOnlyOnDemand();
else if (!strcmp(name, "recovery")) usefulProbeBecomesNormalOperation();
else if (!strcmp(name, "registry")) programWideRegistry();
else if (!strcmp(name, "lifecycle")) controlLifecycle();
else if (!strcmp(name, "threads")) concurrentServiceHolds();
else if (!strcmp(name, "leases")) serviceLeaseEventSequence();
else return 2;
} catch (const std::exception& error) {
std::cerr << argv[1] << ": " << error.what() << '\n';
return 1;
}
std::cout << "PASS: " << argv[1] << '\n';
return 0;
}
'''
OTHER_OWNER = r'''
#include <helpers/WiFiNetworkPowerPolicy.h>
mesh::wifi::NetworkPowerControl* otherControl() {
return &mesh::wifi::networkPower();
}
void otherSetWebHold(bool value) {
mesh::wifi::networkPower().setHold(mesh::wifi::NetworkPowerHold::WebConfig, value);
}
bool otherSuspended() {
return mesh::wifi::networkPower().backgroundSuspended();
}
'''
class WiFiNetworkPowerPolicyTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
compiler = os.environ.get("CXX") or shutil.which("g++") or shutil.which("clang++")
if not compiler:
raise RuntimeError("a host C++ compiler is required")
cls.temporary = tempfile.TemporaryDirectory(prefix="meshcore-wifi-outage-")
cls.addClassCleanup(cls.temporary.cleanup)
work = Path(cls.temporary.name)
source = work / "policy.cpp"
owner = work / "owner.cpp"
source.write_text(HARNESS, encoding="ascii")
owner.write_text(OTHER_OWNER, encoding="ascii")
cls.binary = work / "policy"
built = subprocess.run(
[compiler, "-std=c++11", "-pthread", "-Wall", "-Wextra", "-Werror",
"-I", str(ROOT / "src"), str(source), str(owner), "-o", str(cls.binary)],
capture_output=True, text=True, timeout=60,
)
if built.returncode:
raise RuntimeError(built.stdout + built.stderr)
def run_case(self, name):
result = subprocess.run([str(self.binary), name], capture_output=True,
text=True, timeout=30)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn("PASS: " + name, result.stdout)
def test_initial_five_minute_grace_and_millis_rollover(self):
self.run_case("grace")
def test_ntp_replies_association_and_mqtt_failures_do_not_renew_outage(self):
self.run_case("failures")
def test_healthy_broker_renews_grace_and_cancels_suspension(self):
self.run_case("healthy")
def test_two_minute_probes_and_thirty_minute_cooldowns_repeat_bounded(self):
self.run_case("cycles")
def test_foreground_access_resumes_and_protects_wifi_until_released(self):
self.run_case("foreground")
def test_requested_probe_is_explicit_and_does_not_extend_active_probe(self):
self.run_case("manual")
def test_successful_probe_returns_to_normal_grace(self):
self.run_case("recovery")
def test_two_translation_units_share_owner_holds_and_probe_requests(self):
self.run_case("registry")
def test_worker_lifecycle_clears_stale_suspension_and_probe(self):
self.run_case("lifecycle")
def test_concurrent_foreground_owners_preserve_each_others_holds(self):
self.run_case("threads")
def test_admin_and_ota_lease_sequence_during_combined_background_outage(self):
self.run_case("leases")
if __name__ == "__main__":
unittest.main()
+3 -1
View File
@@ -343,7 +343,9 @@ int main() {
source = (ROOT / "src/helpers/esp32/WebConfigServer.cpp").read_text()
methods = "\n".join(extract_braced(source, signature) for signature in (
"void WebConfigServer::requestStop(", "bool WebConfigServer::stopForOTA(",
"void WebConfigServer::finalizeTeardown("))
"void WebConfigServer::finalizeTeardown(",
"bool WebConfigServer::isNetworkForegroundActive(",
"void WebConfigServer::refreshNetworkPowerHold("))
fixture = (ROOT / "test/fixtures/webconfig_ota_handoff.cpp").read_text()
self.compile_and_run(fixture.replace("@METHODS@", methods))