Reject incomplete WebConfig confirmations under memory pressure

This commit is contained in:
mikecarper committed 2026-10-27 11:58:39 -07:00
1 parent cefda42b53
commit 88bfaebb50
8 files changed
+302 -25

No files matched your search

@@ -313,3 +313,26 @@ stopped broker can attempt recovery, while an additional broker waits when
another slot is connected. The Station G3 Full repeater compiled and passed
capability, dual-OTA partition and linked RAM checks. These are host/build
checks, not a physical five-broker G3 soak.
## Station G3 follow-up: publishing with WebConfig active
The 27 October capture shows `Max=9716`, `DMA=15348/9716`,
`drops=0/3615` and `Outbox=0`. The largest block is below the previous
16,000-byte structured-publication guard. The skipped counter counts attempts,
including retries, rather than distinct lost packets. `rec:1/write` records a
failed synchronous write and recovery; `no errors` describes the current
connected session, not its earlier failures. This capture does not establish
a memory leak or explain the timed-out status command.
The repair prefers PSRAM for WebConfig JSON and each client's receive buffer,
with internal fallbacks charged against the existing DMA reserve. A shared
publish guard covers structured packets, raw packets and status. Its bounded
outgoing workspace is separate from the retained TLS reconnect bounds. CLI
outbox statistics now read a cached value without taking the SDK's network
mutex. Status retries are tracked per broker, and client IDs include the slot
number so two slots at the same broker cannot evict each other.
Allocation-failure checks also reject incomplete MQTT JSON and WebConfig
responses before accepting commands or committing login/result state. Host
tests and a G3 Full build exercise these paths; neither substitutes for a live
five-broker test. The reported G3 was not reachable for flashing or a retest.
+53 -13
View File
@@ -2440,15 +2440,25 @@ void WebConfigServer::handleLogin(AsyncWebServerRequest* req) {
req->send(401, "application/json", "{\"error\":\"password changed; sign in again\"}");
return;
}
static constexpr char response_body[] = "{\"ok\":true}";
AsyncResponseStream* res = req->beginResponseStream("application/json", sizeof(response_body) - 1);
if (!res) { req->send(503); return; }
res->write(reinterpret_cast<const uint8_t*>(response_body), sizeof(response_body) - 1);
char cookie[80];
sprintf(cookie, "wcs=%s; HttpOnly; SameSite=Lax; Path=/", token);
res->addHeader("Set-Cookie", cookie);
// Header insertion can succeed even if its String storage was truncated.
const AsyncWebHeader* retained_cookie = res->getHeader("Set-Cookie");
if (res->available() != sizeof(response_body) - 1
|| !retained_cookie || retained_cookie->value() != cookie) {
delete res;
req->send(503, "application/json", "{\"error\":\"not enough memory\"}");
return;
}
memcpy(_session_token, token, sizeof(_session_token));
_session_last_seen = now;
_session_generation = credential.generation;
noteNetworkActivity();
AsyncWebServerResponse* res = req->beginResponse(200, "application/json", "{\"ok\":true}");
char cookie[80];
sprintf(cookie, "wcs=%s; HttpOnly; SameSite=Lax; Path=/", _session_token);
res->addHeader("Set-Cookie", cookie);
req->send(res);
}
@@ -2816,8 +2826,18 @@ void WebConfigServer::handleConfigPost(AsyncWebServerRequest* req) {
req->send(503, "application/json", "{\"error\":\"not enough memory\"}");
return;
}
String out;
serializeJson(ack, out);
const size_t response_length = measureJson(ack);
// A stream can exist with an incomplete buffer. Build the receipt before
// admitting work, so allocation failure cannot enqueue an unseen change.
AsyncResponseStream* res = req->beginResponseStream("application/json", response_length);
if (!res) { req->send(503); return; }
serializeJson(ack, *res);
if (res->available() != response_length) {
delete res;
req->send(503, "application/json", "{\"error\":\"not enough memory\"}");
return;
}
res->setCode(202);
_batch_kind = BATCH_CONFIG;
_batch_count = count;
@@ -2840,7 +2860,7 @@ void WebConfigServer::handleConfigPost(AsyncWebServerRequest* req) {
"WC: config POST accepted, %d cmds, reboot=%d\n",
count, (int)reboot_after);
req->send(202, "application/json", out);
req->send(res);
}
void WebConfigServer::handleConfigResult(AsyncWebServerRequest* req) {
@@ -2964,6 +2984,13 @@ void WebConfigServer::handleConfigResult(AsyncWebServerRequest* req) {
}
AsyncResponseStream* res = req->beginResponseStream("application/json");
if (!res) { req->send(503); return; }
const size_t response_length = measureJson(doc);
serializeJson(doc, *res);
if (res->available() != response_length) {
delete res;
req->send(503, "application/json", "{\"error\":\"not enough memory\"}");
return;
}
if (_batch_reauth_required && !_batch_reauth_delivered) {
_batch_reauth_delivered = true;
if (_ap_rekey_generation)
@@ -2973,7 +3000,6 @@ void WebConfigServer::handleConfigResult(AsyncWebServerRequest* req) {
_batch_reboot_armed = true;
_reboot_at = confirmed_reboot_at;
}
serializeJson(doc, *res);
req->send(res);
}
@@ -3375,8 +3401,16 @@ void WebConfigServer::handleCliPost(AsyncWebServerRequest* req) {
req->send(503, "application/json", "{\"error\":\"not enough memory\"}");
return;
}
String out;
serializeJson(ack, out);
const size_t response_length = measureJson(ack);
AsyncResponseStream* res = req->beginResponseStream("application/json", response_length);
if (!res) { req->send(503); return; }
serializeJson(ack, *res);
if (res->available() != response_length) {
delete res;
req->send(503, "application/json", "{\"error\":\"not enough memory\"}");
return;
}
res->setCode(202);
_batch_kind = BATCH_CLI;
_batch_count = count;
@@ -3392,7 +3426,7 @@ void WebConfigServer::handleCliPost(AsyncWebServerRequest* req) {
"WC: cli POST accepted, %d cmds, reboot=%d\n",
count, (int)defer_reboot);
req->send(202, "application/json", out);
req->send(res);
}
void WebConfigServer::handleCliResult(AsyncWebServerRequest* req) {
@@ -3500,6 +3534,13 @@ void WebConfigServer::handleCliResult(AsyncWebServerRequest* req) {
}
AsyncResponseStream* res = req->beginResponseStream("application/json");
if (!res) { req->send(503); return; }
const size_t response_length = measureJson(doc);
serializeJson(doc, *res);
if (res->available() != response_length) {
delete res;
req->send(503, "application/json", "{\"error\":\"not enough memory\"}");
return;
}
if (final_read && _batch_reauth_required && !_batch_reauth_delivered) {
_batch_reauth_delivered = true;
if (_ap_rekey_generation)
@@ -3509,7 +3550,6 @@ void WebConfigServer::handleCliResult(AsyncWebServerRequest* req) {
_batch_reboot_armed = true;
_reboot_at = WebConfigBatch::confirmRebootAt(millis());
}
serializeJson(doc, *res);
req->send(res);
}
+22 -2
View File
@@ -23,9 +23,27 @@ struct Client {
int localIP() const { return local_ip; }
uint32_t getRemoteAddress() const { return remote; }
};
struct AsyncWebHeader {
std::string text;
const std::string& value() const { return text; }
};
struct AsyncWebServerResponse {
int status; std::map<std::string, std::string> headers;
void addHeader(const char* key, const char* value) { headers[key]=value; }
int status = 200; std::map<std::string, std::string> headers;
mutable AsyncWebHeader header_view;
bool addHeader(const char* key, const char* value) { headers[key]=value; return true; }
const AsyncWebHeader* getHeader(const char* key) const {
const auto found = headers.find(key);
if (found == headers.end()) return nullptr;
header_view.text = found->second;
return &header_view;
}
};
struct AsyncResponseStream : AsyncWebServerResponse {
std::string body;
size_t available() const { return body.size(); }
void setCode(int value) { status = value; }
size_t write(uint8_t value) { body.push_back(char(value)); return 1; }
size_t write(const uint8_t* bytes, size_t count) { body.append(reinterpret_cast<const char*>(bytes), count); return count; }
};
struct AsyncWebServerRequest {
Client socket; bool has_client=true;
@@ -34,6 +52,8 @@ struct AsyncWebServerRequest {
Client* client() { return has_client ? &socket : nullptr; }
void send(int code, const char* =nullptr, const char* content=nullptr) { status=code; reply=content ? content : ""; }
AsyncWebServerResponse* beginResponse(int code, const char*, const char*) { response.status=code; return &response; }
AsyncResponseStream* beginResponseStream(const char*, size_t = 0) { return new AsyncResponseStream; }
void send(AsyncResponseStream* value) { status=value->status; response=*value; reply=value->body; delete value; }
void send(AsyncWebServerResponse* value) { status=value->status; }
};
struct WebConfigServer {
+11 -1
View File
@@ -60,13 +60,23 @@ struct Callback {
void onConfigBatchStart() { ++starts; }
void onConfigBatchEnd() { ++ends; }
};
struct AsyncResponseStream : std::string {
int code = 200;
size_t available() const { return size(); }
void setCode(int value) { code = value; }
bool addHeader(const char*, const char*) { return true; }
size_t write(uint8_t value) { push_back(char(value)); return 1; }
size_t write(const uint8_t* bytes, size_t count) { append(reinterpret_cast<const char*>(bytes), count); return count; }
};
struct AsyncWebServerRequest {
int status = 0;
std::string body, reply;
void* _tempObject = nullptr;
size_t contentLength() const { return body.size(); }
void send(int code, const char*, const char* text) { status = code; reply = text; }
void send(int code, const char* = nullptr, const char* text = "") { status = code; reply = text; }
void send(int code, const char*, const String& text) { status = code; reply = text; }
AsyncResponseStream* beginResponseStream(const char*, size_t = 0) { return new AsyncResponseStream; }
void send(AsyncResponseStream* response) { status = response->code; reply = *response; delete response; }
void payload(const std::string& text) { body = text; _tempObject = &body[0]; }
};
static const char SECRET_SENTINEL[] = "********";
+23 -3
View File
@@ -52,19 +52,39 @@ struct Client {
uint32_t getRemoteAddress() const { return remote; }
};
struct Header { String cookies; const String& value() const { return cookies; } };
struct AsyncWebHeader {
std::string text;
const std::string& value() const { return text; }
};
struct AsyncWebServerResponse {
int status; std::map<std::string, std::string> headers;
void addHeader(const char* key, const char* value) { headers[key]=value; }
int status = 200; std::map<std::string, std::string> headers;
mutable AsyncWebHeader header_view;
bool addHeader(const char* key, const char* value) { headers[key]=value; return true; }
const AsyncWebHeader* getHeader(const char* key) const {
const auto found = headers.find(key);
if (found == headers.end()) return nullptr;
header_view.text = found->second;
return &header_view;
}
};
struct AsyncResponseStream : AsyncWebServerResponse {
std::string body;
size_t available() const { return body.size(); }
void setCode(int value) { status = value; }
size_t write(uint8_t value) { body.push_back(char(value)); return 1; }
size_t write(const uint8_t* bytes, size_t count) { body.append(reinterpret_cast<const char*>(bytes), count); return count; }
};
struct AsyncWebServerRequest {
Client socket; bool has_client=true;
Header header; std::string body; void* _tempObject=nullptr;
Header header; std::string body, reply; void* _tempObject=nullptr;
int status=0; AsyncWebServerResponse response{};
Client* client() { return has_client ? &socket : nullptr; }
bool hasHeader(const char*) const { return !header.cookies.empty(); }
Header* getHeader(const char*) { return &header; }
void send(int code, const char* =nullptr, const char* =nullptr) { status=code; }
AsyncWebServerResponse* beginResponse(int code, const char*, const char*) { response.status=code; return &response; }
AsyncResponseStream* beginResponseStream(const char*, size_t = 0) { return new AsyncResponseStream; }
void send(AsyncResponseStream* value) { status=value->status; response=*value; reply=value->body; delete value; }
void send(AsyncWebServerResponse* value) { status=value->status; }
void password(const char* value) { body=std::string("{\"password\":\"")+value+"\"}"; _tempObject=body.data(); }
};
+158 -5
View File
@@ -120,17 +120,124 @@ int main() {
end = source.index('\nvoid WebConfigServer::handleConfigPost(', start)
source = source[:start] + extract_braced(SERVER, 'void WebConfigServer::handleConfigGet(') + source[end:]
source = source.replace('struct AsyncWebServerRequest {',
'struct AsyncResponseStream : String { void addHeader(const char*,const char*) {} };\n'
'static void webConfigRegulatorsSnapshot(JsonObject) {}\nstruct AsyncWebServerRequest {')
source = source.replace(' Client socket;', ' AsyncResponseStream response;\n Client socket;')
source = source.replace(' void send(int code,',
' AsyncResponseStream* beginResponseStream(const char*) { return &response; }\n'
' void send(AsyncResponseStream* result) { status=200;output=*result; }\n void send(int code,')
source = source.replace(' Callbacks* _cb = &callbacks;',
' Callbacks* _cb = &callbacks;\n char _wifi_password[65]="saved-secret";\n'
' unsigned _wifi_power_save=0;\n')
return source
def response_failures(self):
source = self.config()
stream = extract_braced(source, 'struct AsyncResponseStream') + ';'
source = source.replace(stream, r'''
struct AsyncResponseStream : String {
static inline size_t limit=SIZE_MAX;
static inline bool cookie_ok=true;
static inline unsigned destroyed=0;
int code=200;
~AsyncResponseStream() { ++destroyed; }
size_t write(const uint8_t* data,size_t count) {
const size_t room=limit>size()?limit-size():0;
const size_t accepted=count<room?count:room;
append(reinterpret_cast<const char*>(data),accepted);return accepted;
}
size_t write(uint8_t byte) { return write(&byte,1); }
size_t available() const { return size(); }
bool addHeader(const char*,const char*) { return cookie_ok; }
void setCode(int value) { code=value; }
};''')
source = source.replace('uint32_t millis() { return 1; }',
'uint32_t fixture_time=1;\nuint32_t millis() { return fixture_time; }')
source = source.replace(' int indexOf(', ' int toInt() const { return atoi(c_str()); }\n int indexOf(')
source = source.replace('void printf(const char*, Args...) {}',
'void printf(const char*, Args...) {} void println(const char*) {}')
source = source.replace('struct AsyncWebServerRequest {',
'struct Parameter { String text="0123456789abcdef"; const String& value() const { return text; } };\n'
'struct AsyncWebServerRequest {\n Parameter parameter;\n'
' bool hasParam(const char* key) const { return !strcmp(key,"reqid"); }\n'
' Parameter* getParam(const char*) { return &parameter; }')
source = source.replace(' struct Callbacks {',
' struct Callbacks {\n bool supportsCliTerminal() const { return true; }')
source = source.replace(' void handleConfigGet(',
' bool _cli_enabled=true, _admin_pwd_set=true;\n'
' uint32_t _reboot_at=0, _ap_rekey_after=0, _ap_rekey_generation=0;\n'
' bool resultReceiptAllowedLocked(AsyncWebServerRequest*,const char*,int,uint32_t) { return false; }\n'
' void handleConfigResult(AsyncWebServerRequest*);\n'
' void handleCliPost(AsyncWebServerRequest*);\n'
' void handleCliResult(AsyncWebServerRequest*);\n'
' void handleConfigGet(')
source += '\n#include <helpers/CLICommandUtils.h>\n' + '\n'.join(
extract_braced(SERVER, signature) for signature in (
'static inline bool wcCliEchoesSecret(', 'static inline bool wcIsDeferredReboot(',
'static const char* wcCliUnavailable(', 'void WebConfigServer::handleCliPost(',
'void WebConfigServer::handleConfigResult(', 'void WebConfigServer::handleCliResult('))
return source
def test_actual_posts_reject_partial_receipts_before_batch_admission(self):
self.compile_run(self.response_failures()+r'''
int main() {
assert(mesh::esp32::managementPassword::publish("password"));
for(bool cli:{false,true}) {
for(size_t limit:{size_t(0),size_t(7),SIZE_MAX}) {
WebConfigServer server;server.callbacks.node.capabilities=WebConfigServer::CAP_LOST_REPLY;
AsyncResponseStream::limit=limit;
const unsigned freed=AsyncResponseStream::destroyed;
AsyncWebServerRequest post(cli
? "{\"reqid\":\"0123456789abcdef\",\"cmds\":[\"get name\"]}"
: writeBody("\"yes\""));
if(cli)server.handleCliPost(&post);else server.handleConfigPost(&post);
assert(AsyncResponseStream::destroyed==freed+1);
if(limit==SIZE_MAX) {
assert(post.status==202&&server._batch_state==WebConfigBatch::State::Pending);
assert(server._batch_count==1);
JsonDocument ack;assert(!deserializeJson(ack,post.output));
assert(!strcmp(ack["reqid"],"0123456789abcdef"));
} else {
assert(post.status==503&&server._batch_state==WebConfigBatch::State::Idle);
assert(server._batch_count==0&&!server._batch_reqid[0]);
assert(!server._batch_auth_generation&&!server._batch_reboot_armed);
}
}
}
}
''')
def test_actual_results_do_not_arm_reboot_or_rekey_until_complete_body(self):
self.compile_run(self.response_failures()+r'''
int main() {
assert(mesh::esp32::managementPassword::publish("password"));
for(bool cli:{false,true})for(size_t limit:{size_t(0),size_t(7)}) {
fixture_time=1;
WebConfigServer server;server._batch_kind=cli?WebConfigServer::BATCH_CLI:WebConfigServer::BATCH_CONFIG;
server._batch_state=WebConfigBatch::State::Done;server._batch_count=server._batch_next=1;
strcpy(server._batch_reqid,"0123456789abcdef");
strcpy(server._batch[0].key,"name");strcpy(server._batch[0].reply,"OK");
server._batch_reboot=server._batch_reauth_required=true;server._ap_rekey_generation=7;
AsyncResponseStream::limit=limit;
const unsigned freed=AsyncResponseStream::destroyed;
AsyncWebServerRequest failed;
if(cli)server.handleCliResult(&failed);else server.handleConfigResult(&failed);
assert(failed.status==503&&AsyncResponseStream::destroyed==freed+1);
assert(!server._batch_reauth_delivered&&!server._batch_reboot_armed);
assert(!server._ap_rekey_after&&!server._reboot_at);
assert(server._batch_state==WebConfigBatch::State::Done);
AsyncResponseStream::limit=SIZE_MAX;
AsyncWebServerRequest retry;
if(cli)server.handleCliResult(&retry);else server.handleConfigResult(&retry);
assert(retry.status==200&&server._batch_reauth_delivered&&server._batch_reboot_armed);
assert(server._ap_rekey_after==WebConfigBatch::scheduleAt(1,3000));
assert(server._reboot_at==WebConfigBatch::confirmRebootAt(1));
JsonDocument result;assert(!deserializeJson(result,retry.output));
assert(!strcmp(result["state"],"done")&&result["results"].size()==1);
const auto reboot=server._reboot_at,rekey=server._ap_rekey_after;
fixture_time=500;
AsyncWebServerRequest reread;
if(cli)server.handleCliResult(&reread);else server.handleConfigResult(&reread);
assert(reread.status==200&&server._reboot_at==reboot&&server._ap_rekey_after==rekey);
}
}
''')
def test_actual_config_handlers_use_psram_and_reject_no_memory_before_queue_admission(self):
self.compile_run(self.config()+r'''
int main() {
@@ -191,6 +298,52 @@ int main() {
request._tempObject=request.body.data();portal.handleLogin(&request);
assert(request.status==200&&portal._session_token[0]);
}
''')
def test_actual_login_keeps_session_when_body_or_cookie_allocation_fails(self):
source = login_fixture.WebConfigLoginThrottleTest.handler_fixture(self).split('static const char* kLocked', 1)[0]
source = source.replace('struct WebConfigServer {', 'struct WebConfigServer {'+ALIASES)
stream = extract_braced(source, 'struct AsyncResponseStream') + ';'
source = source.replace(stream, r'''
struct AsyncResponseStream : AsyncWebServerResponse {
static inline size_t limit=SIZE_MAX;
static inline unsigned cookie_failure=0;
static inline unsigned destroyed=0;
std::string body;
~AsyncResponseStream() { ++destroyed; }
size_t available() const { return body.size(); }
size_t write(const uint8_t* bytes,size_t count) {
const size_t accepted=count<limit?count:limit;
body.append(reinterpret_cast<const char*>(bytes),accepted);return accepted;
}
bool addHeader(const char* key,const char* value) {
if(cookie_failure==1)return true; // library can report success after String OOM
return AsyncWebServerResponse::addHeader(key,cookie_failure==2?"wcs=partial":value);
}
};''')
self.compile_run(source+r'''
int main() {
WebConfigServer portal;
strcpy(portal._session_token,"0123456789abcdef0123456789abcdef");
portal._session_last_seen=123;portal._session_generation=456;
for(int failure=0;failure<4;++failure) {
AsyncResponseStream::limit=failure==0?0:failure==1?7:SIZE_MAX;
AsyncResponseStream::cookie_failure=failure<2?0:failure-1;
const unsigned freed=AsyncResponseStream::destroyed;
AsyncWebServerRequest request;request.body="{\"password\":\"existing-secret\"}";
request._tempObject=request.body.data();portal.handleLogin(&request);
assert(request.status==503&&AsyncResponseStream::destroyed==freed+1);
assert(!strcmp(portal._session_token,"0123456789abcdef0123456789abcdef"));
assert(portal._session_last_seen==123&&portal._session_generation==456);
}
AsyncResponseStream::limit=SIZE_MAX;AsyncResponseStream::cookie_failure=0;
AsyncWebServerRequest retry;retry.body="{\"password\":\"existing-secret\"}";
retry._tempObject=retry.body.data();portal.handleLogin(&retry);
assert(retry.status==200&&retry.reply=="{\"ok\":true}");
assert(strcmp(portal._session_token,"0123456789abcdef0123456789abcdef"));
assert(portal._session_last_seen==now_ms);
assert(retry.response.headers.at("Set-Cookie").find(portal._session_token)!=std::string::npos);
}
''')
def test_non_psram_keeps_original_arduinojson_document_types(self):
+10
View File
@@ -77,6 +77,12 @@ struct String : std::string {
size_t write(uint8_t value) { push_back(char(value)); return 1; }
size_t write(const uint8_t* data, size_t count) { append(reinterpret_cast<const char*>(data), count); return count; }
};
struct AsyncResponseStream : String {
int code = 200;
size_t available() const { return size(); }
void setCode(int value) { code = value; }
bool addHeader(const char*, const char*) { return true; }
};
struct WCLock { explicit WCLock(int) {} };
uint32_t millis() { return 1; }
static constexpr uint32_t WEBCONFIG_SESSION_TTL_MS = 120000;
@@ -102,6 +108,10 @@ struct AsyncWebServerRequest {
Client* client() { return &socket; }
bool hasHeader(const char*) const { return !header.cookies.empty(); }
Header* getHeader(const char*) { return &header; }
AsyncResponseStream* beginResponseStream(const char*, size_t = 0) { return new AsyncResponseStream; }
void send(AsyncResponseStream* response) {
status = response->code; output = static_cast<const String&>(*response); delete response;
}
void send(int code, const char* = nullptr, const String& body = "") { status = code; output = body; }
};
struct WebConfigServer {
+2 -1
View File
@@ -113,7 +113,8 @@ class WebConfigStationOwnershipTest(unittest.TestCase):
fixture = self.station_fixture().split("int main() {", 1)[0]
auth = (ROOT / "test/fixtures/webconfig_login_handler.cpp").read_text()
transport = "\n".join(extract_braced(auth, signature) + ";" for signature in (
"struct Client", "struct AsyncWebServerResponse", "struct AsyncWebServerRequest"))
"struct Client", "struct AsyncWebHeader", "struct AsyncWebServerResponse", "struct AsyncResponseStream",
"struct AsyncWebServerRequest"))
transport = r'''
#include <map>
#include <ArduinoJson.h>