From 004d308ea96f743be7f93506f9329e83720c267c Mon Sep 17 00:00:00 2001 From: agessaman Date: Sun, 21 Jun 2026 11:57:05 -0700 Subject: [PATCH 01/16] feat(ota): implement pull-based OTA from manifest for observer builds Added functionality to support pull-based OTA updates by fetching firmware from a manifest. The new `otaFromManifest` method allows the system to check for available updates and flash the firmware if necessary. This enhancement improves the update process for observer builds using the MQTT bridge, ensuring a more seamless firmware management experience. --- build.sh | 6 +- src/MeshCore.h | 4 + src/helpers/CommonCLI.cpp | 20 ++ src/helpers/ESP32Board.cpp | 194 ++++++++++++++++++ src/helpers/ESP32Board.h | 1 + variants/heltec_t190/platformio.ini | 2 + variants/heltec_v3/platformio.ini | 4 + variants/heltec_v4/platformio.ini | 4 + variants/lilygo_t3s3/platformio.ini | 2 + variants/lilygo_tbeam_1w/platformio.ini | 2 + variants/lilygo_tbeam_SX1262/platformio.ini | 2 + variants/lilygo_tbeam_SX1276/platformio.ini | 2 + .../platformio.ini | 2 + variants/lilygo_tlora_v2_1/platformio.ini | 2 + variants/rak3112/platformio.ini | 2 + variants/station_g2/platformio.ini | 2 + variants/xiao_s3_wio/platformio.ini | 2 + 17 files changed, 251 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index bff89b164..edcabbf0d 100755 --- a/build.sh +++ b/build.sh @@ -151,8 +151,10 @@ build_firmware() { esac EMBEDDED_VERSION_STRING="${FIRMWARE_VERSION}${VARIANT_TAG}-${COMMIT_HASH}" - # add firmware version info to end of existing platformio build flags in environment vars - export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DFIRMWARE_BUILD_DATE='\"${FIRMWARE_BUILD_DATE}\"' -DFIRMWARE_VERSION='\"${EMBEDDED_VERSION_STRING}\"'" + # add firmware version info to end of existing platformio build flags in environment vars. + # OTA_VARIANT is the env name ($1) — it is exactly the asset-filename prefix used above, so the + # observer pull-OTA can match its own build in the web-flasher manifest (config.json). + export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DFIRMWARE_BUILD_DATE='\"${FIRMWARE_BUILD_DATE}\"' -DFIRMWARE_VERSION='\"${EMBEDDED_VERSION_STRING}\"' -DOTA_VARIANT='\"$1\"'" # disable debug flags if requested disable_debug_flags diff --git a/src/MeshCore.h b/src/MeshCore.h index d91c4ea12..6a17a4b68 100644 --- a/src/MeshCore.h +++ b/src/MeshCore.h @@ -64,6 +64,10 @@ public: virtual uint8_t getStartupReason() const = 0; virtual bool getBootloaderVersion(char* version, size_t max_len) { return false; } virtual bool startOTAUpdate(const char* id, char reply[]) { return false; } // not supported + // Pull-based OTA: fetch the firmware build for this variant from a baked-in manifest and flash it. + // current_ver is the running firmware version string (used to skip if already up to date); when + // dry_run is true the build is only reported, not flashed. Observer (ESP32+WiFi) builds only. + virtual bool otaFromManifest(const char* current_ver, bool dry_run, char reply[]) { return false; } // Power management interface (boards with power management override these) virtual bool isExternalPowered() { return false; } diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index e17adf71a..6034507f9 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -793,9 +793,29 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re strcpy(reply, "ERR: unsupported on this platform"); #endif } else if (memcmp(command, "start ota", 9) == 0) { +#if defined(WITH_MQTT_BRIDGE) && defined(OTA_MANIFEST_URL) + // Observer pull-OTA: fetch this variant's build from the baked-in manifest. + // "start ota check" reports the available build without flashing. + const char* arg = command + 9; + while (*arg == ' ') arg++; + bool dry = (memcmp(arg, "check", 5) == 0); + if (WiFi.status() != WL_CONNECTED) { + strcpy(reply, "ERR: WiFi not connected"); + } else { + // Free the MQTT bridge (TLS contexts + task) so the manifest parse and + // OTA download have heap headroom; the WiFi STA link survives end(). + if (!dry) _callbacks->setBridgeState(false); + bool ok = _board->otaFromManifest(_callbacks->getFirmwareVer(), dry, reply); + // On success the board reboots and never returns here; on any abort + // (already up to date, partition change, download error) bring the + // bridge back up so the node resumes uplinking. + if (!ok && !dry) _callbacks->setBridgeState(true); + } +#else if (!_board->startOTAUpdate(_prefs->node_name, reply)) { strcpy(reply, "Error"); } +#endif } else if (memcmp(command, "clock", 5) == 0) { uint32_t now = getRTCClock()->getCurrentTime(); DateTime dt = DateTime(now); diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index e0ca1d0ee..3b4f953b2 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -44,4 +44,198 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[]) { } #endif +// --------------------------------------------------------------------------- +// Manifest-driven pull OTA (observer / MQTT-bridge builds only) +// +// The observer already holds a live WiFi station connection (for the MQTT +// bridge) and embeds a root-CA bundle, so it can fetch its own firmware. The +// caller (CommonCLI) stops the MQTT bridge first to free heap/TLS, then calls +// this. We read the web-flasher manifest (config.json), find the `flash-update` +// (app-only) build for our own variant, refuse partition-change releases (OTA +// can't rewrite the partition table), skip if already up to date, then stream +// the .bin straight into the inactive OTA slot via HTTPUpdate. +// --------------------------------------------------------------------------- +#if defined(WITH_MQTT_BRIDGE) +#include +#include +#include +#include +#include + +// Embedded CA bundle (produced by board_build.embed_files). Weak so non-bundle +// builds still link; we check for presence at runtime. +extern const uint8_t rootca_crt_bundle_start[] asm("_binary_src_certs_x509_crt_bundle_bin_start") __attribute__((weak)); +extern const uint8_t rootca_crt_bundle_end[] asm("_binary_src_certs_x509_crt_bundle_bin_end") __attribute__((weak)); + +// Extract the trailing build hash. For a filename we first drop a ".bin" +// suffix, then take the token after the last '-'. Works for both the manifest +// asset name ("...-v1.16.0-8b084d5.bin" -> "8b084d5") and the embedded +// FIRMWARE_VERSION ("v1.16.0-observer-8b084d5" -> "8b084d5"). +static void ota_extractHash(const char* s, char* out, size_t out_sz) { + if (!s) { if (out_sz) out[0] = 0; return; } + size_t len = strlen(s); + if (len > 4 && strcmp(s + len - 4, ".bin") == 0) len -= 4; + size_t i = len; + while (i > 0 && s[i - 1] != '-') i--; + size_t n = len - i; + if (n >= out_sz) n = out_sz - 1; + memcpy(out, s + i, n); + out[n] = 0; +} + +bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char reply[]) { +#if !defined(OTA_MANIFEST_URL) || !defined(OTA_VARIANT) + strcpy(reply, "ERR: OTA not configured (build via build.sh)"); + return false; +#else + if (WiFi.status() != WL_CONNECTED) { + strcpy(reply, "ERR: WiFi not connected"); + return false; + } + + size_t bundle_len = 0; + if (rootca_crt_bundle_start != nullptr && rootca_crt_bundle_end != nullptr && + rootca_crt_bundle_end > rootca_crt_bundle_start) { + bundle_len = (size_t)(rootca_crt_bundle_end - rootca_crt_bundle_start); + } + if (bundle_len == 0) { + strcpy(reply, "ERR: no embedded cert bundle"); + return false; + } + + // --- Fetch + filter-parse the manifest ----------------------------------- + WiFiClientSecure mclient; +#if ESP_ARDUINO_VERSION_MAJOR >= 3 + mclient.setCACertBundle(rootca_crt_bundle_start, bundle_len); +#else + mclient.setCACertBundle(rootca_crt_bundle_start); +#endif + mclient.setTimeout(15000); + + HTTPClient http; + if (!http.begin(mclient, OTA_MANIFEST_URL)) { + strcpy(reply, "ERR: manifest connect failed"); + return false; + } + int code = http.GET(); + if (code != HTTP_CODE_OK) { + snprintf(reply, 160, "ERR: manifest HTTP %d", code); + http.end(); + return false; + } + + // Filter keeps only staticPath + each firmware entry's notice/version, so the + // parsed document stays a fraction of the full manifest. (The dynamic version + // key forces keeping its whole subtree, incl. release notes.) ArduinoJson v7 + // JsonDocument allocates elastically, so this only grows to the kept subset. + JsonDocument filter; + filter["staticPath"] = true; + filter["device"][0]["firmware"][0]["notice"] = true; + filter["device"][0]["firmware"][0]["version"] = true; + + JsonDocument doc; + DeserializationError err = + deserializeJson(doc, http.getStream(), DeserializationOption::Filter(filter)); + http.end(); + if (err) { + snprintf(reply, 160, "ERR: manifest parse (%s)", err.c_str()); + return false; + } + + // Copy out of the document up front: doc gets cleared before these are used. + char base_url[128] = {0}; + strncpy(base_url, doc["staticPath"] | "", sizeof(base_url) - 1); + if (!base_url[0]) { + strcpy(reply, "ERR: manifest missing staticPath"); + return false; + } + + // --- Locate the flash-update build for our variant ----------------------- + const char* variant = OTA_VARIANT; + size_t vlen = strlen(variant); + char target_name[128] = {0}; + bool partition_change = false; + bool found = false; + + for (JsonObject dev : doc["device"].as()) { + for (JsonObject fw : dev["firmware"].as()) { + const char* notice = fw["notice"].is() ? fw["notice"].as() : nullptr; + for (JsonPair vp : fw["version"].as()) { + for (JsonObject file : vp.value()["files"].as()) { + const char* type = file["type"] | ""; + const char* name = file["name"] | ""; + if (strcmp(type, "flash-update") != 0) continue; + if (strncmp(name, variant, vlen) != 0 || name[vlen] != '-') continue; + strncpy(target_name, name, sizeof(target_name) - 1); + partition_change = (notice != nullptr && strcmp(notice, "partition-change") == 0); + found = true; + break; + } + if (found) break; + } + if (found) break; + } + if (found) break; + } + doc.clear(); + + if (!found) { + snprintf(reply, 160, "ERR: no build for %s in manifest", variant); + return false; + } + + char avail_hash[24], cur_hash[24]; + ota_extractHash(target_name, avail_hash, sizeof(avail_hash)); + ota_extractHash(current_ver, cur_hash, sizeof(cur_hash)); + // Compare by shared prefix: git abbreviates the same commit to 7 chars on a + // shallow CI clone but 8 locally, so an exact match would miss equal builds. + size_t la = strlen(avail_hash), lc = strlen(cur_hash); + size_t m = (la < lc) ? la : lc; + bool up_to_date = (m >= 7 && strncmp(avail_hash, cur_hash, m) == 0); + + if (dry_run) { + snprintf(reply, 160, "%s: %s -> %s%s", up_to_date ? "up to date" : "update available", + cur_hash, avail_hash, partition_change ? " [partition change: cable flash]" : ""); + return true; + } + if (partition_change) { + snprintf(reply, 160, "ERR: %s needs cable flash (partition change)", avail_hash); + return false; + } + if (up_to_date) { + snprintf(reply, 160, "OK: already up to date (%s)", cur_hash); + return false; + } + + // --- Stream the .bin into the inactive OTA slot -------------------------- + char url[256]; + snprintf(url, sizeof(url), "%s/%s", base_url, target_name); + + inhibit_sleep = true; // keep awake through the flash + + WiFiClientSecure uclient; +#if ESP_ARDUINO_VERSION_MAJOR >= 3 + uclient.setCACertBundle(rootca_crt_bundle_start, bundle_len); +#else + uclient.setCACertBundle(rootca_crt_bundle_start); +#endif + uclient.setTimeout(20000); + + httpUpdate.rebootOnUpdate(true); // reboots into the new image on success + t_httpUpdate_return ret = httpUpdate.update(uclient, url); + + // Only reached on failure (success reboots inside update()). + inhibit_sleep = false; + snprintf(reply, 160, "ERR: OTA failed (%d): %s", (int)ret, + httpUpdate.getLastErrorString().c_str()); + return false; +#endif // OTA_MANIFEST_URL && OTA_VARIANT +} +#else +bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char reply[]) { + strcpy(reply, "ERR: not supported"); + return false; +} +#endif // WITH_MQTT_BRIDGE + #endif diff --git a/src/helpers/ESP32Board.h b/src/helpers/ESP32Board.h index a4cbf2a98..93d1bdadf 100644 --- a/src/helpers/ESP32Board.h +++ b/src/helpers/ESP32Board.h @@ -151,6 +151,7 @@ public: } bool startOTAUpdate(const char* id, char reply[]) override; + bool otaFromManifest(const char* current_ver, bool dry_run, char reply[]) override; void setInhibitSleep(bool inhibit) { inhibit_sleep = inhibit; diff --git a/variants/heltec_t190/platformio.ini b/variants/heltec_t190/platformio.ini index 7b8de99f7..71b565564 100644 --- a/variants/heltec_t190/platformio.ini +++ b/variants/heltec_t190/platformio.ini @@ -112,6 +112,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -204,6 +205,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/heltec_v3/platformio.ini b/variants/heltec_v3/platformio.ini index d6cfaaf37..a71cb90f3 100644 --- a/variants/heltec_v3/platformio.ini +++ b/variants/heltec_v3/platformio.ini @@ -123,6 +123,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -192,6 +193,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -480,6 +482,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -528,6 +531,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/heltec_v4/platformio.ini b/variants/heltec_v4/platformio.ini index 7d4d7058f..157ece5dc 100644 --- a/variants/heltec_v4/platformio.ini +++ b/variants/heltec_v4/platformio.ini @@ -162,6 +162,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -213,6 +214,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -315,6 +317,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -354,6 +357,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/lilygo_t3s3/platformio.ini b/variants/lilygo_t3s3/platformio.ini index 08cbb342b..f862a9f69 100644 --- a/variants/lilygo_t3s3/platformio.ini +++ b/variants/lilygo_t3s3/platformio.ini @@ -116,6 +116,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 ; -D MQTT_DEBUG=1 @@ -161,6 +162,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/lilygo_tbeam_1w/platformio.ini b/variants/lilygo_tbeam_1w/platformio.ini index 3c8fc4b83..73723c29f 100644 --- a/variants/lilygo_tbeam_1w/platformio.ini +++ b/variants/lilygo_tbeam_1w/platformio.ini @@ -209,6 +209,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D PERSISTANT_GPS=1 @@ -255,6 +256,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D PERSISTANT_GPS=1 diff --git a/variants/lilygo_tbeam_SX1262/platformio.ini b/variants/lilygo_tbeam_SX1262/platformio.ini index 7ae5671d2..575178576 100644 --- a/variants/lilygo_tbeam_SX1262/platformio.ini +++ b/variants/lilygo_tbeam_SX1262/platformio.ini @@ -161,6 +161,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 ; -D MQTT_DEBUG=1 @@ -203,6 +204,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/lilygo_tbeam_SX1276/platformio.ini b/variants/lilygo_tbeam_SX1276/platformio.ini index e00f8604f..aa3383dbf 100644 --- a/variants/lilygo_tbeam_SX1276/platformio.ini +++ b/variants/lilygo_tbeam_SX1276/platformio.ini @@ -160,6 +160,7 @@ build_flags = -D MAX_NEIGHBOURS=50 -D PERSISTANT_GPS=1 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 ; -D MQTT_DEBUG=1 @@ -202,6 +203,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/lilygo_tbeam_supreme_SX1262/platformio.ini b/variants/lilygo_tbeam_supreme_SX1262/platformio.ini index 6657fa092..a8a847214 100644 --- a/variants/lilygo_tbeam_supreme_SX1262/platformio.ini +++ b/variants/lilygo_tbeam_supreme_SX1262/platformio.ini @@ -120,6 +120,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -162,6 +163,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/lilygo_tlora_v2_1/platformio.ini b/variants/lilygo_tlora_v2_1/platformio.ini index 1f8942c82..9a4a83656 100644 --- a/variants/lilygo_tlora_v2_1/platformio.ini +++ b/variants/lilygo_tlora_v2_1/platformio.ini @@ -157,6 +157,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_TASK_STACK_SIZE=16384 @@ -202,6 +203,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_TASK_STACK_SIZE=16384 diff --git a/variants/rak3112/platformio.ini b/variants/rak3112/platformio.ini index 29cd30c2e..f284a3a17 100644 --- a/variants/rak3112/platformio.ini +++ b/variants/rak3112/platformio.ini @@ -113,6 +113,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -176,6 +177,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/station_g2/platformio.ini b/variants/station_g2/platformio.ini index 856645aeb..aaa74b150 100644 --- a/variants/station_g2/platformio.ini +++ b/variants/station_g2/platformio.ini @@ -187,6 +187,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -305,6 +306,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/xiao_s3_wio/platformio.ini b/variants/xiao_s3_wio/platformio.ini index 01934f2e5..873454c01 100644 --- a/variants/xiao_s3_wio/platformio.ini +++ b/variants/xiao_s3_wio/platformio.ini @@ -103,6 +103,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -146,6 +147,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 + -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 From 454afec91df59178d0d356b5e2c7e6a7e3548d8e Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 22 Jun 2026 13:52:48 -0700 Subject: [PATCH 02/16] feat(mqtt): add flmesh.us preset to MQTT implementation --- MQTT_IMPLEMENTATION.md | 1 + src/helpers/MQTTPresets.h | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 548bef138..910da6b16 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -116,6 +116,7 @@ The MQTT bridge uses a slot-based architecture with up to 6 concurrent connectio | `meshcore-ca-1` | mqtt1.meshcore.ca:443 | JWT (Ed25519) | WSS | | `meshcore-ca-2` | mqtt2.meshcore.ca:443 | JWT (Ed25519) | WSS | | `bostonmesh` | mqttmc01.bostonme.sh:443 | JWT (Ed25519) | WSS | +| `flmesh.us` | mcmqtt.jntconnections.com:443 | JWT (Ed25519) | WSS | | `inwmesh` | scope.inwmesh.org:8883 | Username/password (per slot via `mqttN.username` / `mqttN.password`) | MQTT over TLS | | `custom` | User-configured | Username/Password | MQTT or WSS | | `none` | (disabled) | — | — | diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h index bcc18a68e..23876306c 100644 --- a/src/helpers/MQTTPresets.h +++ b/src/helpers/MQTTPresets.h @@ -105,7 +105,7 @@ static const char ISRG_ROOT_X1[] PROGMEM = "-----END CERTIFICATE-----\n"; // Number of built-in presets -static const int MQTT_PRESET_COUNT = 21; +static const int MQTT_PRESET_COUNT = 22; // Built-in preset definitions (stored in flash) static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { @@ -131,6 +131,7 @@ static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { { "meshcore-ca-2", "wss://mqtt2.meshcore.ca:443/mqtt", "mqtt2.meshcore.ca", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "inwmesh", "mqtts://scope.inwmesh.org:8883", nullptr, ISRG_ROOT_X1, MQTT_AUTH_USERPASS, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "bostonmesh", "wss://mqttmc01.bostonme.sh:443/mqtt", "mqttmc01.bostonme.sh", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "flmesh.us", "wss://mcmqtt.jntconnections.com:443", "mcmqtt.jntconnections.com", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, }; // Find a preset by name, returns nullptr if not found From b679c0d7122ee0c97758f4a4a1592dc3fff3627d Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 22 Jun 2026 19:32:23 -0700 Subject: [PATCH 03/16] fix(ota): handle HTTP response and empty manifest in otaFromManifest Updated the otaFromManifest method to enforce HTTP/1.0 for better compatibility with CDNs and to handle empty manifest responses. This ensures that the JSON parser receives a complete body, preventing errors during firmware update checks. --- src/helpers/ESP32Board.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 3b4f953b2..45f60f833 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -117,12 +117,24 @@ bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char rep strcpy(reply, "ERR: manifest connect failed"); return false; } + // Force HTTP/1.0: a CDN (e.g. Cloudflare) answers HTTP/1.1 with + // Transfer-Encoding: chunked and no Content-Length, and the raw chunked + // stream can't be fed to the JSON parser (chunk-size frames corrupt it). + // HTTP/1.0 yields a Connection: close, unframed body that getString() + // assembles in full before we parse it. + http.useHTTP10(true); int code = http.GET(); if (code != HTTP_CODE_OK) { snprintf(reply, 160, "ERR: manifest HTTP %d", code); http.end(); return false; } + String body = http.getString(); + http.end(); + if (body.length() == 0) { + strcpy(reply, "ERR: empty manifest"); + return false; + } // Filter keeps only staticPath + each firmware entry's notice/version, so the // parsed document stays a fraction of the full manifest. (The dynamic version @@ -135,8 +147,8 @@ bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char rep JsonDocument doc; DeserializationError err = - deserializeJson(doc, http.getStream(), DeserializationOption::Filter(filter)); - http.end(); + deserializeJson(doc, body, DeserializationOption::Filter(filter)); + body = String(); // free the raw manifest before we walk the parsed doc if (err) { snprintf(reply, 160, "ERR: manifest parse (%s)", err.c_str()); return false; From e84c26657600323e8b124511126ddb660f497169 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 22 Jun 2026 19:36:05 -0700 Subject: [PATCH 04/16] feat(ota): add ota check and ota update commands for pull-based updates --- src/helpers/CommonCLI.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 6034507f9..c62e46b6b 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -792,13 +792,15 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re #else strcpy(reply, "ERR: unsupported on this platform"); #endif - } else if (memcmp(command, "start ota", 9) == 0) { + } else if (memcmp(command, "ota check", 9) == 0 || memcmp(command, "ota update", 10) == 0) { + // Observer pull-OTA: fetch this variant's build from the baked-in manifest + // and flash it. Intentionally a separate command from "start ota" (the + // manual ElegantOTA web-upload SoftAP) so a remote/online update is never + // triggered by someone expecting to hand-upload a binary. + // ota check -> report available build, do not flash + // ota update -> download and flash, then reboot #if defined(WITH_MQTT_BRIDGE) && defined(OTA_MANIFEST_URL) - // Observer pull-OTA: fetch this variant's build from the baked-in manifest. - // "start ota check" reports the available build without flashing. - const char* arg = command + 9; - while (*arg == ' ') arg++; - bool dry = (memcmp(arg, "check", 5) == 0); + bool dry = (memcmp(command, "ota check", 9) == 0); if (WiFi.status() != WL_CONNECTED) { strcpy(reply, "ERR: WiFi not connected"); } else { @@ -812,10 +814,13 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re if (!ok && !dry) _callbacks->setBridgeState(true); } #else + strcpy(reply, "ERR: online OTA not supported on this build"); +#endif + } else if (memcmp(command, "start ota", 9) == 0) { + // Manual OTA: bring up the ElegantOTA SoftAP for a hand-uploaded binary. if (!_board->startOTAUpdate(_prefs->node_name, reply)) { strcpy(reply, "Error"); } -#endif } else if (memcmp(command, "clock", 5) == 0) { uint32_t now = getRTCClock()->getCurrentTime(); DateTime dt = DateTime(now); From 3719ebb980b364ea355ffa4c85b95af022127cae Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 22 Jun 2026 19:53:40 -0700 Subject: [PATCH 05/16] fix(ota): improve streaming JSON parsing in otaFromManifest Updated the otaFromManifest method to stream-parse the firmware manifest directly from the network, reducing peak RAM usage during OTA checks. This change enhances compatibility with slow TLS links by implementing a per-read timeout, ensuring a more efficient and reliable update process. --- src/helpers/ESP32Board.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 45f60f833..5eaa06c7a 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -120,26 +120,26 @@ bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char rep // Force HTTP/1.0: a CDN (e.g. Cloudflare) answers HTTP/1.1 with // Transfer-Encoding: chunked and no Content-Length, and the raw chunked // stream can't be fed to the JSON parser (chunk-size frames corrupt it). - // HTTP/1.0 yields a Connection: close, unframed body that getString() - // assembles in full before we parse it. + // HTTP/1.0 yields a Connection: close, unframed body we can stream-parse. http.useHTTP10(true); + http.setTimeout(20000); // per-read timeout while streaming the body int code = http.GET(); if (code != HTTP_CODE_OK) { snprintf(reply, 160, "ERR: manifest HTTP %d", code); http.end(); return false; } - String body = http.getString(); - http.end(); - if (body.length() == 0) { - strcpy(reply, "ERR: empty manifest"); - return false; - } - // Filter keeps only staticPath + each firmware entry's notice/version, so the - // parsed document stays a fraction of the full manifest. (The dynamic version - // key forces keeping its whole subtree, incl. release notes.) ArduinoJson v7 - // JsonDocument allocates elastically, so this only grows to the kept subset. + // Stream-parse straight from the network: the filter discards all but + // staticPath + each firmware entry's notice/version, so peak RAM is just the + // small kept subset (not the ~40 KB manifest). This matters for `ota check`, + // which runs with the MQTT bridge still up and holding heap. (The dynamic + // version key forces keeping its whole subtree, incl. release notes.) + // readBytes() honours the stream timeout, so a slow TLS link won't be + // mistaken for end-of-input. + WiFiClient* stream = http.getStreamPtr(); + stream->setTimeout(20000); + JsonDocument filter; filter["staticPath"] = true; filter["device"][0]["firmware"][0]["notice"] = true; @@ -147,8 +147,8 @@ bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char rep JsonDocument doc; DeserializationError err = - deserializeJson(doc, body, DeserializationOption::Filter(filter)); - body = String(); // free the raw manifest before we walk the parsed doc + deserializeJson(doc, *stream, DeserializationOption::Filter(filter)); + http.end(); if (err) { snprintf(reply, 160, "ERR: manifest parse (%s)", err.c_str()); return false; From dec668386a5a7fdc993f9a9414852af85db1ec14 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 22 Jun 2026 21:28:18 -0700 Subject: [PATCH 06/16] fix(ota): improve OTA update handling for connected WiFi networks Updated the startOTAUpdate method to serve the ElegantOTA on the station IP when connected to a WiFi network, enhancing accessibility for OTA updates. If not connected, it defaults to the MeshCore-OTA SoftAP. This change improves the user experience by allowing easier access to OTA updates without needing to switch networks. --- src/helpers/ESP32Board.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 5eaa06c7a..2306db152 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -12,9 +12,19 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[]) { inhibit_sleep = true; // prevent sleep during OTA - WiFi.softAP("MeshCore-OTA", NULL); - sprintf(reply, "Started: http://%s/update", WiFi.softAPIP().toString().c_str()); + // If the device is already on a WiFi network (e.g. an observer joined in STA + // mode), serve ElegantOTA on the station IP so it's reachable from the LAN + // without joining a separate AP. Otherwise raise the MeshCore-OTA SoftAP. + IPAddress ip; + if (WiFi.status() == WL_CONNECTED) { + ip = WiFi.localIP(); + } else { + WiFi.softAP("MeshCore-OTA", NULL); + ip = WiFi.softAPIP(); + } + + sprintf(reply, "Started: http://%s/update", ip.toString().c_str()); MESH_DEBUG_PRINTLN("startOTAUpdate: %s", reply); static char id_buf[60]; From c1885fcc154fb3c524c2f89b4fd68b269f186c28 Mon Sep 17 00:00:00 2001 From: agessaman Date: Mon, 22 Jun 2026 22:33:57 -0700 Subject: [PATCH 07/16] feat(ota): implement deferred OTA update scheduling and handling Added support for deferred OTA updates in the MyMesh class, allowing the system to schedule firmware updates to occur after a confirmation reply is sent. This change improves the user experience by ensuring that the update process does not block the main application loop, allowing for smoother operation during firmware updates. --- examples/simple_repeater/MyMesh.cpp | 16 ++++++++ examples/simple_repeater/MyMesh.h | 10 +++++ src/helpers/CommonCLI.cpp | 26 ++++++++----- src/helpers/CommonCLI.h | 8 ++++ src/helpers/ESP32Board.cpp | 39 ++++++++++++++++++++ src/helpers/ESP32Board.h | 4 ++ src/helpers/bridges/MQTTBridge.cpp | 57 +++++++++++++++++++---------- src/helpers/bridges/MQTTBridge.h | 5 +++ 8 files changed, 136 insertions(+), 29 deletions(-) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index cc45c2c4f..72363588d 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1445,6 +1445,22 @@ void MyMesh::loop() { MESH_DEBUG_PRINTLN("Radio params restored"); } +#if defined(WITH_MQTT_BRIDGE) && defined(OTA_MANIFEST_URL) + if (_ota_update_at && millisHasNowPassed(_ota_update_at)) { // deferred `ota update` + _ota_update_at = 0; // clear timer + // The "Beginning update..." reply has now gone out. Free the bridge for heap + // headroom, then flash: otaFromManifest reboots into the new image on success + // (so this never returns); on any abort (already up to date, partition change, + // download error) it returns and we resume the bridge. + setBridgeState(false); + char ota_reply[160]; + if (!_cli.getBoard()->otaFromManifest(getFirmwareVer(), false, ota_reply)) { + MESH_DEBUG_PRINTLN("ota update aborted: %s", ota_reply); + setBridgeState(true); + } + } +#endif + // is pending dirty contacts write needed? if (dirty_contacts_expiry && millisHasNowPassed(dirty_contacts_expiry)) { acl.save(_fs); diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 5ea4fcc3c..09ee140e7 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -116,6 +116,7 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks { #endif CayenneLPP telemetry; unsigned long set_radio_at, revert_radio_at; + unsigned long _ota_update_at = 0; // deferred `ota update` fire time (0 = none scheduled) float pending_freq; float pending_bw; uint8_t pending_sf; @@ -306,6 +307,15 @@ public: bridge->setSlotPreset(slot, _prefs.mqtt_slot_preset[slot]); } + // Schedule the pull-OTA flash to run from loop() in ~2.5 s, leaving time for the + // "Beginning update..." CLI reply (CLI_REPLY_DELAY_MILLIS = 600 ms) to transmit + // before the flash blocks the loop and reboots. + bool beginDeferredOtaUpdate() override { + _ota_update_at = millis() + 2500; + if (_ota_update_at == 0) _ota_update_at = 1; // 0 means "none" + return true; + } + int getQueueSize() override { return bridge ? bridge->getQueueSize() : 0; } diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index c62e46b6b..e44269c03 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -800,18 +800,26 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re // ota check -> report available build, do not flash // ota update -> download and flash, then reboot #if defined(WITH_MQTT_BRIDGE) && defined(OTA_MANIFEST_URL) - bool dry = (memcmp(command, "ota check", 9) == 0); if (WiFi.status() != WL_CONNECTED) { strcpy(reply, "ERR: WiFi not connected"); + } else if (memcmp(command, "ota check", 9) == 0) { + // Check is synchronous so its result lands in this reply. Free the MQTT + // bridge first: on a no-PSRAM board only ~70 KB heap is free with the + // bridge up, and a third TLS connection (the manifest fetch) alongside the + // two live MQTT sessions drives free heap to a few hundred bytes, which + // truncates the read. The WiFi STA link survives end(); restore after. + _callbacks->setBridgeState(false); + _board->otaFromManifest(_callbacks->getFirmwareVer(), true, reply); + _callbacks->setBridgeState(true); } else { - // Free the MQTT bridge (TLS contexts + task) so the manifest parse and - // OTA download have heap headroom; the WiFi STA link survives end(). - if (!dry) _callbacks->setBridgeState(false); - bool ok = _board->otaFromManifest(_callbacks->getFirmwareVer(), dry, reply); - // On success the board reboots and never returns here; on any abort - // (already up to date, partition change, download error) bring the - // bridge back up so the node resumes uplinking. - if (!ok && !dry) _callbacks->setBridgeState(true); + // Update is DEFERRED: the flash blocks the loop and then reboots, so it + // must run only AFTER this reply has gone out over the mesh — otherwise + // the requester never gets a confirmation. The app loop runs it shortly. + if (_callbacks->beginDeferredOtaUpdate()) { + strcpy(reply, "Beginning update... (node will reboot if successful)"); + } else { + strcpy(reply, "ERR: online OTA not available"); + } } #else strcpy(reply, "ERR: online OTA not supported on this build"); diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index 9a6502d14..a158324c7 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -288,6 +288,14 @@ public: restartBridge(); }; + // Schedule a pull-OTA firmware update to run shortly (from the app loop), after + // the "Beginning update..." CLI reply has been transmitted. Deferred because the + // flash blocks the loop and then reboots, so it can't run inline with the reply. + // Returns true if scheduled. Default: not supported. + virtual bool beginDeferredOtaUpdate() { + return false; + }; + virtual int getQueueSize() { return 0; // no op by default }; diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 2306db152..139b5f783 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -71,6 +71,8 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[]) { #include #include #include +#include +#include // Embedded CA bundle (produced by board_build.embed_files). Weak so non-bundle // builds still link; we check for presence at runtime. @@ -93,7 +95,44 @@ static void ota_extractHash(const char* s, char* out, size_t out_sz) { out[n] = 0; } +// Parameters handed to the worker task; lives on otaFromManifest()'s stack, +// which stays valid because that function blocks until the worker signals done. +struct OtaTaskArgs { + ESP32Board* self; + const char* current_ver; + bool dry_run; + char* reply; + volatile bool result; + volatile bool done; +}; + +static void ota_task_entry(void* param) { + OtaTaskArgs* a = static_cast(param); + a->result = a->self->otaFromManifestImpl(a->current_ver, a->dry_run, a->reply); + a->done = true; // on a successful `ota update` we reboot before reaching here + vTaskDelete(nullptr); +} + bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char reply[]) { + // The TLS handshake (cert-bundle verify) + JSON parse / HTTPUpdate use far more + // stack than the ~8 KB loop task offers — especially when reached via the deep + // mesh-receive call chain (it overflows the loopTask canary). Run the work in a + // dedicated 24 KB-stack task and block here until it finishes. The big stack is + // freed when the task exits; on a successful update the chip reboots inside it. + OtaTaskArgs args = { this, current_ver, dry_run, reply, false, false }; + TaskHandle_t handle = nullptr; + BaseType_t ok = xTaskCreatePinnedToCore(ota_task_entry, "ota", 24576, &args, 5, &handle, 1); + if (ok != pdPASS) { + strcpy(reply, "ERR: OTA task spawn failed"); + return false; + } + while (!args.done) { + delay(50); // Arduino delay() yields to other tasks + } + return args.result; +} + +bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char reply[]) { #if !defined(OTA_MANIFEST_URL) || !defined(OTA_VARIANT) strcpy(reply, "ERR: OTA not configured (build via build.sh)"); return false; diff --git a/src/helpers/ESP32Board.h b/src/helpers/ESP32Board.h index 93d1bdadf..eae9ab784 100644 --- a/src/helpers/ESP32Board.h +++ b/src/helpers/ESP32Board.h @@ -152,6 +152,10 @@ public: bool startOTAUpdate(const char* id, char reply[]) override; bool otaFromManifest(const char* current_ver, bool dry_run, char reply[]) override; + // Heavy body (TLS + JSON / HTTPUpdate). Runs in a dedicated large-stack task + // spawned by otaFromManifest() — public only so that task entry point can call + // it; not meant to be invoked directly. + bool otaFromManifestImpl(const char* current_ver, bool dry_run, char reply[]); void setInhibitSleep(bool inhibit) { inhibit_sleep = inhibit; diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 677e814bd..0c1ff86b9 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -754,27 +754,44 @@ void MQTTBridge::initializeWiFiInTask() { WiFi.setAutoReconnect(true); WiFi.setAutoConnect(true); - // Set up WiFi event handlers for better diagnostics and immediate disconnection detection - WiFi.onEvent([this](WiFiEvent_t event, WiFiEventInfo_t info) { - switch(event) { - case ARDUINO_EVENT_WIFI_STA_GOT_IP: - MQTT_DEBUG_PRINTLN("WiFi connected: %s", IPAddress(info.got_ip.ip_info.ip.addr).toString().c_str()); - // Set flag to trigger NTP sync from loop() instead of doing it here - if (!_ntp_synced && !_ntp_sync_pending) { - _ntp_sync_pending = true; - } - break; - case ARDUINO_EVENT_WIFI_STA_DISCONNECTED: - s_wifi_disconnect_reason = info.wifi_sta_disconnected.reason; - s_wifi_disconnect_time = millis(); - MQTT_DEBUG_PRINTLN("WiFi disconnected: reason %d", s_wifi_disconnect_reason); - break; - default: - break; - } - }); + // Set up WiFi event handlers for better diagnostics and immediate disconnection + // detection. Register ONCE — the bridge is reused across restarts (e.g. stopped + // for `ota check`/`ota update`, or `set mqtt…` reconfigure) and WiFi.onEvent() + // never removes prior callbacks, so re-registering leaks handlers and duplicates + // every log line. + if (!_wifi_event_registered) { + WiFi.onEvent([this](WiFiEvent_t event, WiFiEventInfo_t info) { + switch(event) { + case ARDUINO_EVENT_WIFI_STA_GOT_IP: + MQTT_DEBUG_PRINTLN("WiFi connected: %s", IPAddress(info.got_ip.ip_info.ip.addr).toString().c_str()); + // Set flag to trigger NTP sync from loop() instead of doing it here + if (!_ntp_synced && !_ntp_sync_pending) { + _ntp_sync_pending = true; + } + break; + case ARDUINO_EVENT_WIFI_STA_DISCONNECTED: + s_wifi_disconnect_reason = info.wifi_sta_disconnected.reason; + s_wifi_disconnect_time = millis(); + MQTT_DEBUG_PRINTLN("WiFi disconnected: reason %d", s_wifi_disconnect_reason); + break; + default: + break; + } + }); + _wifi_event_registered = true; + } - WiFi.begin(_prefs->wifi_ssid, _prefs->wifi_password); + // Only (re)start the WiFi association if it isn't already up. end() leaves the + // STA link connected, so on a restart (e.g. after `ota check`) calling + // WiFi.begin() again forces a needless disconnect/reconnect — which also races + // the MQTT task's first DNS lookup (getaddrinfo fails until WiFi/DNS recovers). + // When already connected, the deferred slot setup still fires in mqttTaskLoop() + // because _ntp_synced persists across end() (only _slots_setup_done is reset). + if (WiFi.status() != WL_CONNECTED) { + WiFi.begin(_prefs->wifi_ssid, _prefs->wifi_password); + } else if (!_ntp_synced && !_ntp_sync_pending) { + _ntp_sync_pending = true; // already connected but never synced — kick NTP now + } // NOTE: Slot setup is deferred until after NTP sync in mqttTaskLoop(). // JWT-auth slots need valid timestamps for token creation, and connecting diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 9eaf66de0..e19e0544a 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -180,6 +180,11 @@ private: bool _ntp_synced; bool _ntp_sync_pending; // Flag to trigger NTP sync from loop() instead of event handler bool _slots_setup_done; // Deferred: slots set up after NTP sync + // WiFi.onEvent() handler registered once and never removed by end(); the bridge + // object is reused across restarts, so re-registering would leak handlers and + // duplicate every connect/disconnect log line. Inline-initialised so it survives + // construction and is NOT reset by end(). + bool _wifi_event_registered = false; int _max_active_slots; // Runtime limit: 5 with PSRAM, 2 without // Pending slot reconfigure: set from CLI (Core 1), processed by MQTT task (Core 0) From addb637b633ffb955167d981f7391207e2bc2d73 Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 23 Jun 2026 09:31:45 -0700 Subject: [PATCH 08/16] feat(mqtt): add meshcore-fi preset to MQTT presets --- src/helpers/MQTTPresets.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h index 23876306c..36ddba1aa 100644 --- a/src/helpers/MQTTPresets.h +++ b/src/helpers/MQTTPresets.h @@ -105,7 +105,7 @@ static const char ISRG_ROOT_X1[] PROGMEM = "-----END CERTIFICATE-----\n"; // Number of built-in presets -static const int MQTT_PRESET_COUNT = 22; +static const int MQTT_PRESET_COUNT = 23; // Built-in preset definitions (stored in flash) static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { @@ -129,6 +129,7 @@ static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { { "dutchmeshcore-2", "wss://collector2.dutchmeshcore.nl:443/mqtt", "collector2.dutchmeshcore.nl", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "meshcore-ca-1", "wss://mqtt1.meshcore.ca:443/mqtt", "mqtt1.meshcore.ca", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "meshcore-ca-2", "wss://mqtt2.meshcore.ca:443/mqtt", "mqtt2.meshcore.ca", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "meshcore-fi", "wss://mc-mqtt.meshcore.fi:443/", "mc-mqtt.meshcore.fi", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "inwmesh", "mqtts://scope.inwmesh.org:8883", nullptr, ISRG_ROOT_X1, MQTT_AUTH_USERPASS, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "bostonmesh", "wss://mqttmc01.bostonme.sh:443/mqtt", "mqttmc01.bostonme.sh", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "flmesh.us", "wss://mcmqtt.jntconnections.com:443", "mcmqtt.jntconnections.com", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, From 730eb37dbd28e85927eb49e6d24341aac9d7af3e Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 23 Jun 2026 12:55:07 -0700 Subject: [PATCH 09/16] feat(ota): implement build number suffix for firmware versioning Enhanced the firmware versioning system by appending a build number suffix when available, allowing for better tracking of published builds. This change improves the OTA update process by providing clearer versioning information in the embedded firmware string. --- .../workflows/build-observer-firmwares.yml | 52 ++++- MQTT_IMPLEMENTATION.md | 2 +- build.sh | 17 +- examples/simple_repeater/MyMesh.cpp | 2 +- src/helpers/CommonCLI.cpp | 13 +- src/helpers/ESP32Board.cpp | 182 ++++++++++-------- src/helpers/MQTTPresets.h | 2 +- variants/heltec_t190/platformio.ini | 4 +- variants/heltec_v3/platformio.ini | 8 +- variants/heltec_v4/platformio.ini | 8 +- variants/lilygo_t3s3/platformio.ini | 4 +- variants/lilygo_tbeam_1w/platformio.ini | 4 +- variants/lilygo_tbeam_SX1262/platformio.ini | 4 +- variants/lilygo_tbeam_SX1276/platformio.ini | 4 +- .../platformio.ini | 4 +- variants/lilygo_tlora_v2_1/platformio.ini | 4 +- variants/rak3112/platformio.ini | 4 +- variants/station_g2/platformio.ini | 4 +- variants/xiao_s3_wio/platformio.ini | 4 +- 19 files changed, 203 insertions(+), 123 deletions(-) diff --git a/.github/workflows/build-observer-firmwares.yml b/.github/workflows/build-observer-firmwares.yml index 728aab148..0db674ae6 100644 --- a/.github/workflows/build-observer-firmwares.yml +++ b/.github/workflows/build-observer-firmwares.yml @@ -47,6 +47,7 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.split.outputs.matrix }} + build_number: ${{ steps.buildnum.outputs.n }} steps: - name: Clone Repo uses: actions/checkout@v4 @@ -66,6 +67,27 @@ jobs: | jq -cs .) echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" + - name: Compute observer build number + id: buildnum + shell: bash + run: | + # Per-base published-build counter. N increments once per release and + # resets to 1 when FIRMWARE_VERSION (the MeshCore base version) changes. + # Read-only here (off the published counter); the release job is the sole + # writer and only writes on a successful build, so a failed build doesn't + # burn a number. First run / 404 -> empty -> N=1. + COUNTER_URL="https://observer.gessaman.com/observer-build-counter.json" + CUR=$(curl -fsSL "$COUNTER_URL" 2>/dev/null || echo '{}') + PREV_BASE=$(echo "$CUR" | jq -r '.baseVersion // ""') + PREV_BUILD=$(echo "$CUR" | jq -r '.build // 0') + if [ "$PREV_BASE" = "$FIRMWARE_VERSION" ]; then + N=$((PREV_BUILD + 1)) + else + N=1 + fi + echo "Base $FIRMWARE_VERSION; previous build $PREV_BUILD (base $PREV_BASE) -> N=$N" + echo "n=$N" >> "$GITHUB_OUTPUT" + # Build one shard (several envs) per runner. build.sh emits both the app # .bin and the ESP32 -merged.bin into out/ for each env. build: @@ -100,6 +122,9 @@ jobs: uses: ./.github/actions/setup-build-environment - name: Build Shard ${{ matrix.shard.idx }} + env: + # Stamp the per-base build number into the embedded version (v1.16.0.N). + FIRMWARE_BUILD_NUMBER: ${{ needs.enumerate.outputs.build_number }} run: /usr/bin/env bash build.sh build-firmware ${{ matrix.shard.envs }} - name: Upload Shard Artifact @@ -112,7 +137,7 @@ jobs: # Collect all shard outputs, publish the rolling release, then point the # flasher at the new build by bumping the hash + notes in its config.json. release: - needs: build + needs: [enumerate, build] runs-on: ubuntu-latest steps: - name: Clone Repo @@ -196,6 +221,23 @@ jobs: "${{ steps.sha.outputs.short }}" \ "$GITHUB_WORKSPACE/firmware-notes.html" + - name: Generate Slim Per-Variant Manifests + Persist Build Counter + env: + BUILD_NUMBER: ${{ needs.enumerate.outputs.build_number }} + run: | + # Derive the slim per-variant manifests (flasher/v/.json) that the + # firmware fetches for `ota check`/`ota update`, from the just-updated + # config.json (single source of truth), stamping this build's number. + # Then persist the counter so the next run increments from here. + python3 flasher/scripts/gen-slim-manifests.py \ + --config flasher/config.json \ + --out-dir flasher/v \ + --base-version "$FIRMWARE_VERSION" \ + --build "$BUILD_NUMBER" + printf '{\n "baseVersion": "%s",\n "build": %s\n}\n' \ + "$FIRMWARE_VERSION" "$BUILD_NUMBER" > flasher/observer-build-counter.json + echo "Build $FIRMWARE_VERSION.$BUILD_NUMBER" + - name: Unshallow for Changelog run: | # gen_changelog.py needs the full branch history; deepen only now, AFTER @@ -233,11 +275,15 @@ jobs: - name: Commit & Push Flasher Config working-directory: flasher run: | - if git diff --quiet; then + # Stage everything: update-firmware.py edits config.json in place, but the + # slim manifests (v/*.json) and the build counter can be NEW files, which + # `commit -am` would miss — so add -A and check the staged diff. + git add -A + if git diff --cached --quiet; then echo "No flasher changes to commit." exit 0 fi git config user.name "meshcore-bot" git config user.email "noreply@gessaman.com" - git commit -am "Update observer firmware to ${{ steps.sha.outputs.short }}" + git commit -m "Update observer firmware to ${{ steps.sha.outputs.short }} (build ${FIRMWARE_VERSION}.${{ needs.enumerate.outputs.build_number }})" git push diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 910da6b16..d218eef9b 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -116,7 +116,7 @@ The MQTT bridge uses a slot-based architecture with up to 6 concurrent connectio | `meshcore-ca-1` | mqtt1.meshcore.ca:443 | JWT (Ed25519) | WSS | | `meshcore-ca-2` | mqtt2.meshcore.ca:443 | JWT (Ed25519) | WSS | | `bostonmesh` | mqttmc01.bostonme.sh:443 | JWT (Ed25519) | WSS | -| `flmesh.us` | mcmqtt.jntconnections.com:443 | JWT (Ed25519) | WSS | +| `flmesh` | mcmqtt.jntconnections.com:443 | JWT (Ed25519) | WSS | | `inwmesh` | scope.inwmesh.org:8883 | Username/password (per slot via `mqttN.username` / `mqttN.password`) | MQTT over TLS | | `custom` | User-configured | Username/Password | MQTT or WSS | | `none` | (disabled) | — | — | diff --git a/build.sh b/build.sh index edcabbf0d..3f58d2a04 100755 --- a/build.sh +++ b/build.sh @@ -149,11 +149,22 @@ build_firmware() { case "$1" in *observer*) VARIANT_TAG="-observer" ;; esac - EMBEDDED_VERSION_STRING="${FIRMWARE_VERSION}${VARIANT_TAG}-${COMMIT_HASH}" + + # Observer build number: when CI provides FIRMWARE_BUILD_NUMBER (the per-base + # published-build counter), append it as a 4th version component so the node + # reports e.g. v1.16.0.5-observer-abcdef and `ota check` can show how many + # builds behind it is. Local dev builds leave it unset → no 4th component. + # The *filename* (FIRMWARE_VERSION_STRING above) is deliberately left without + # the build number so assets stay -v-.bin. + BUILD_NUMBER_SUFFIX="" + if [ -n "$FIRMWARE_BUILD_NUMBER" ]; then + BUILD_NUMBER_SUFFIX=".${FIRMWARE_BUILD_NUMBER}" + fi + EMBEDDED_VERSION_STRING="${FIRMWARE_VERSION}${BUILD_NUMBER_SUFFIX}${VARIANT_TAG}-${COMMIT_HASH}" # add firmware version info to end of existing platformio build flags in environment vars. - # OTA_VARIANT is the env name ($1) — it is exactly the asset-filename prefix used above, so the - # observer pull-OTA can match its own build in the web-flasher manifest (config.json). + # OTA_VARIANT is the env name ($1) — it selects this build's slim per-variant manifest + # (/.json) that the observer pull-OTA fetches. export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DFIRMWARE_BUILD_DATE='\"${FIRMWARE_BUILD_DATE}\"' -DFIRMWARE_VERSION='\"${EMBEDDED_VERSION_STRING}\"' -DOTA_VARIANT='\"$1\"'" # disable debug flags if requested diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 72363588d..d283acbd3 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1445,7 +1445,7 @@ void MyMesh::loop() { MESH_DEBUG_PRINTLN("Radio params restored"); } -#if defined(WITH_MQTT_BRIDGE) && defined(OTA_MANIFEST_URL) +#if defined(WITH_MQTT_BRIDGE) && defined(OTA_MANIFEST_BASE) if (_ota_update_at && millisHasNowPassed(_ota_update_at)) { // deferred `ota update` _ota_update_at = 0; // clear timer // The "Beginning update..." reply has now gone out. Free the bridge for heap diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index e44269c03..367ee47cc 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -799,18 +799,15 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re // triggered by someone expecting to hand-upload a binary. // ota check -> report available build, do not flash // ota update -> download and flash, then reboot -#if defined(WITH_MQTT_BRIDGE) && defined(OTA_MANIFEST_URL) +#if defined(WITH_MQTT_BRIDGE) && defined(OTA_MANIFEST_BASE) if (WiFi.status() != WL_CONNECTED) { strcpy(reply, "ERR: WiFi not connected"); } else if (memcmp(command, "ota check", 9) == 0) { - // Check is synchronous so its result lands in this reply. Free the MQTT - // bridge first: on a no-PSRAM board only ~70 KB heap is free with the - // bridge up, and a third TLS connection (the manifest fetch) alongside the - // two live MQTT sessions drives free heap to a few hundred bytes, which - // truncates the read. The WiFi STA link survives end(); restore after. - _callbacks->setBridgeState(false); + // Check is synchronous so its result lands in this reply, and runs with the + // MQTT bridge UP: the slim per-variant manifest is tiny, so the fetch only + // costs a single TLS handshake (no large JSON doc) — which fits alongside + // the live MQTT sessions even on no-PSRAM boards. No bridge bounce needed. _board->otaFromManifest(_callbacks->getFirmwareVer(), true, reply); - _callbacks->setBridgeState(true); } else { // Update is DEFERRED: the flash blocks the loop and then reboots, so it // must run only AFTER this reply has gone out over the mesh — otherwise diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 139b5f783..e1267a6ff 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -95,6 +95,29 @@ static void ota_extractHash(const char* s, char* out, size_t out_sz) { out[n] = 0; } +// Split a version token "vMAJOR.MINOR.PATCH[.BUILD]" into its base +// ("vMAJOR.MINOR.PATCH") and build number (BUILD, or -1 if there's no 4th +// component). The base has exactly two dots; a third dot introduces the build. +static void ota_parseVersion(const char* ver, char* base_out, size_t base_sz, int* build_out) { + *build_out = -1; + if (base_sz) base_out[0] = 0; + if (!ver) return; + int dots = 0, third_dot = -1; + for (int j = 0; ver[j]; j++) { + if (ver[j] == '.' && ++dots == 3) { third_dot = j; break; } + } + if (third_dot >= 0) { + size_t n = (size_t)third_dot; + if (n >= base_sz) n = base_sz - 1; + memcpy(base_out, ver, n); + base_out[n] = 0; + *build_out = atoi(ver + third_dot + 1); + } else { + strncpy(base_out, ver, base_sz - 1); + base_out[base_sz - 1] = 0; + } +} + // Parameters handed to the worker task; lives on otaFromManifest()'s stack, // which stays valid because that function blocks until the worker signals done. struct OtaTaskArgs { @@ -133,7 +156,7 @@ bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char rep } bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char reply[]) { -#if !defined(OTA_MANIFEST_URL) || !defined(OTA_VARIANT) +#if !defined(OTA_MANIFEST_BASE) || !defined(OTA_VARIANT) strcpy(reply, "ERR: OTA not configured (build via build.sh)"); return false; #else @@ -152,7 +175,14 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char return false; } - // --- Fetch + filter-parse the manifest ----------------------------------- + // --- Fetch this variant's slim manifest ---------------------------------- + // /.json — a ~180 byte per-variant file, NOT + // the full config.json. Keeping the body tiny is what lets `ota check` run with + // the MQTT bridge up on no-PSRAM boards: only the TLS handshake costs heap, with + // no large JSON document layered on top. + char murl[200]; + snprintf(murl, sizeof(murl), "%s/%s.json", OTA_MANIFEST_BASE, OTA_VARIANT); + WiFiClientSecure mclient; #if ESP_ARDUINO_VERSION_MAJOR >= 3 mclient.setCACertBundle(rootca_crt_bundle_start, bundle_len); @@ -162,16 +192,15 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char mclient.setTimeout(15000); HTTPClient http; - if (!http.begin(mclient, OTA_MANIFEST_URL)) { + if (!http.begin(mclient, murl)) { strcpy(reply, "ERR: manifest connect failed"); return false; } - // Force HTTP/1.0: a CDN (e.g. Cloudflare) answers HTTP/1.1 with - // Transfer-Encoding: chunked and no Content-Length, and the raw chunked - // stream can't be fed to the JSON parser (chunk-size frames corrupt it). - // HTTP/1.0 yields a Connection: close, unframed body we can stream-parse. + // Force HTTP/1.0: a CDN (e.g. Cloudflare) answers HTTP/1.1 with chunked encoding + // and no Content-Length; the raw chunked stream corrupts the parse. HTTP/1.0 + // yields a Connection: close, unframed body. http.useHTTP10(true); - http.setTimeout(20000); // per-read timeout while streaming the body + http.setTimeout(20000); int code = http.GET(); if (code != HTTP_CODE_OK) { snprintf(reply, 160, "ERR: manifest HTTP %d", code); @@ -179,99 +208,96 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char return false; } - // Stream-parse straight from the network: the filter discards all but - // staticPath + each firmware entry's notice/version, so peak RAM is just the - // small kept subset (not the ~40 KB manifest). This matters for `ota check`, - // which runs with the MQTT bridge still up and holding heap. (The dynamic - // version key forces keeping its whole subtree, incl. release notes.) - // readBytes() honours the stream timeout, so a slow TLS link won't be - // mistaken for end-of-input. WiFiClient* stream = http.getStreamPtr(); - stream->setTimeout(20000); - - JsonDocument filter; - filter["staticPath"] = true; - filter["device"][0]["firmware"][0]["notice"] = true; - filter["device"][0]["firmware"][0]["version"] = true; + stream->setTimeout(20000); // readBytes honours this, so a slow TLS link != EOF JsonDocument doc; - DeserializationError err = - deserializeJson(doc, *stream, DeserializationOption::Filter(filter)); + DeserializationError err = deserializeJson(doc, *stream); http.end(); if (err) { snprintf(reply, 160, "ERR: manifest parse (%s)", err.c_str()); return false; } - // Copy out of the document up front: doc gets cleared before these are used. - char base_url[128] = {0}; - strncpy(base_url, doc["staticPath"] | "", sizeof(base_url) - 1); - if (!base_url[0]) { - strcpy(reply, "ERR: manifest missing staticPath"); - return false; - } - - // --- Locate the flash-update build for our variant ----------------------- - const char* variant = OTA_VARIANT; - size_t vlen = strlen(variant); - char target_name[128] = {0}; - bool partition_change = false; - bool found = false; - - for (JsonObject dev : doc["device"].as()) { - for (JsonObject fw : dev["firmware"].as()) { - const char* notice = fw["notice"].is() ? fw["notice"].as() : nullptr; - for (JsonPair vp : fw["version"].as()) { - for (JsonObject file : vp.value()["files"].as()) { - const char* type = file["type"] | ""; - const char* name = file["name"] | ""; - if (strcmp(type, "flash-update") != 0) continue; - if (strncmp(name, variant, vlen) != 0 || name[vlen] != '-') continue; - strncpy(target_name, name, sizeof(target_name) - 1); - partition_change = (notice != nullptr && strcmp(notice, "partition-change") == 0); - found = true; - break; - } - if (found) break; - } - if (found) break; - } - if (found) break; - } + // Copy fields out before the document is reused/cleared. + char file_url[200] = {0}, avail_version[40] = {0}, avail_base[40] = {0}, avail_hash[24] = {0}; + strncpy(file_url, doc["file"] | "", sizeof(file_url) - 1); + strncpy(avail_version, doc["version"] | "", sizeof(avail_version) - 1); + strncpy(avail_base, doc["baseVersion"] | "", sizeof(avail_base) - 1); + strncpy(avail_hash, doc["hash"] | "", sizeof(avail_hash) - 1); + int avail_build = doc["build"] | -1; + bool partition_change = doc["partitionChange"] | false; doc.clear(); - if (!found) { - snprintf(reply, 160, "ERR: no build for %s in manifest", variant); + if (!file_url[0]) { + strcpy(reply, "ERR: manifest missing file"); return false; } - char avail_hash[24], cur_hash[24]; - ota_extractHash(target_name, avail_hash, sizeof(avail_hash)); - ota_extractHash(current_ver, cur_hash, sizeof(cur_hash)); - // Compare by shared prefix: git abbreviates the same commit to 7 chars on a - // shallow CI clone but 8 locally, so an exact match would miss equal builds. - size_t la = strlen(avail_hash), lc = strlen(cur_hash); - size_t m = (la < lc) ? la : lc; - bool up_to_date = (m >= 7 && strncmp(avail_hash, cur_hash, m) == 0); + // --- Determine current-vs-available -------------------------------------- + // Our running version token (e.g. "v1.16.0.5"), i.e. current_ver up to the + // first '-' (which precedes "-observer-"). + char own_version[40] = {0}; + for (size_t i = 0; current_ver && current_ver[i] && current_ver[i] != '-' && i < sizeof(own_version) - 1; i++) { + own_version[i] = current_ver[i]; + } + char own_base[40]; + int own_build; + ota_parseVersion(own_version, own_base, sizeof(own_base), &own_build); + // Fallback identity by commit hash (handles pre-build-number / local images that + // carry no 4th version component). Shared-prefix compare absorbs the 7- vs 8-char + // git abbreviation difference. + char cur_hash[24]; + ota_extractHash(current_ver, cur_hash, sizeof(cur_hash)); + size_t lh = strlen(avail_hash), lc = strlen(cur_hash); + size_t m = (lh < lc) ? lh : lc; + bool hash_equal = (m >= 7 && strncmp(avail_hash, cur_hash, m) == 0); + + bool same_base = (own_base[0] && avail_base[0] && strcmp(own_base, avail_base) == 0); + bool have_builds = (own_build >= 0 && avail_build >= 0); + bool diff_base = (own_base[0] && avail_base[0] && !same_base); + + int behind = 0; + bool up_to_date; + if (same_base && have_builds) { + behind = avail_build - own_build; + up_to_date = (behind <= 0); + } else if (diff_base) { + up_to_date = false; // different base version is always an update + } else { + up_to_date = hash_equal; // unknown build numbers -> fall back to hash + } + + const char* avail_disp = avail_version[0] ? avail_version : avail_hash; + const char* own_disp = own_version[0] ? own_version : cur_hash; + const char* pc_note = partition_change ? " [partition change: cable flash]" : ""; + + // --- Report (dry run / `ota check`) -------------------------------------- if (dry_run) { - snprintf(reply, 160, "%s: %s -> %s%s", up_to_date ? "up to date" : "update available", - cur_hash, avail_hash, partition_change ? " [partition change: cable flash]" : ""); + if (up_to_date) { + snprintf(reply, 160, "up to date: %s", avail_disp); + } else if (same_base && have_builds) { + snprintf(reply, 160, "update available: %s -> %s (%d behind)%s", own_disp, avail_disp, behind, pc_note); + } else if (diff_base) { + snprintf(reply, 160, "update available: %s -> %s (new base)%s", own_disp, avail_disp, pc_note); + } else { + snprintf(reply, 160, "update available: %s -> %s%s", own_disp, avail_disp, pc_note); + } return true; } + + // --- Gates (real `ota update`) ------------------------------------------- if (partition_change) { - snprintf(reply, 160, "ERR: %s needs cable flash (partition change)", avail_hash); + snprintf(reply, 160, "ERR: %s needs cable flash (partition change)", avail_disp); return false; } if (up_to_date) { - snprintf(reply, 160, "OK: already up to date (%s)", cur_hash); + snprintf(reply, 160, "OK: already up to date (%s)", avail_disp); return false; } - // --- Stream the .bin into the inactive OTA slot -------------------------- - char url[256]; - snprintf(url, sizeof(url), "%s/%s", base_url, target_name); - + // --- Stream the .bin (the manifest's full URL) into the inactive OTA slot - inhibit_sleep = true; // keep awake through the flash WiFiClientSecure uclient; @@ -283,14 +309,14 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char uclient.setTimeout(20000); httpUpdate.rebootOnUpdate(true); // reboots into the new image on success - t_httpUpdate_return ret = httpUpdate.update(uclient, url); + t_httpUpdate_return ret = httpUpdate.update(uclient, file_url); // Only reached on failure (success reboots inside update()). inhibit_sleep = false; snprintf(reply, 160, "ERR: OTA failed (%d): %s", (int)ret, httpUpdate.getLastErrorString().c_str()); return false; -#endif // OTA_MANIFEST_URL && OTA_VARIANT +#endif // OTA_MANIFEST_BASE && OTA_VARIANT } #else bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char reply[]) { diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h index 36ddba1aa..d6726003a 100644 --- a/src/helpers/MQTTPresets.h +++ b/src/helpers/MQTTPresets.h @@ -132,7 +132,7 @@ static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { { "meshcore-fi", "wss://mc-mqtt.meshcore.fi:443/", "mc-mqtt.meshcore.fi", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "inwmesh", "mqtts://scope.inwmesh.org:8883", nullptr, ISRG_ROOT_X1, MQTT_AUTH_USERPASS, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "bostonmesh", "wss://mqttmc01.bostonme.sh:443/mqtt", "mqttmc01.bostonme.sh", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, - { "flmesh.us", "wss://mcmqtt.jntconnections.com:443", "mcmqtt.jntconnections.com", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "flmesh", "wss://mcmqtt.jntconnections.com:443", "mcmqtt.jntconnections.com", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, }; // Find a preset by name, returns nullptr if not found diff --git a/variants/heltec_t190/platformio.ini b/variants/heltec_t190/platformio.ini index 71b565564..5a9e8db20 100644 --- a/variants/heltec_t190/platformio.ini +++ b/variants/heltec_t190/platformio.ini @@ -112,7 +112,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -205,7 +205,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/heltec_v3/platformio.ini b/variants/heltec_v3/platformio.ini index a71cb90f3..4bd3819a0 100644 --- a/variants/heltec_v3/platformio.ini +++ b/variants/heltec_v3/platformio.ini @@ -123,7 +123,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -193,7 +193,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -482,7 +482,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -531,7 +531,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/heltec_v4/platformio.ini b/variants/heltec_v4/platformio.ini index 157ece5dc..f3622bc56 100644 --- a/variants/heltec_v4/platformio.ini +++ b/variants/heltec_v4/platformio.ini @@ -162,7 +162,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -214,7 +214,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -317,7 +317,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -357,7 +357,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/lilygo_t3s3/platformio.ini b/variants/lilygo_t3s3/platformio.ini index f862a9f69..4a259ee77 100644 --- a/variants/lilygo_t3s3/platformio.ini +++ b/variants/lilygo_t3s3/platformio.ini @@ -116,7 +116,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 ; -D MQTT_DEBUG=1 @@ -162,7 +162,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/lilygo_tbeam_1w/platformio.ini b/variants/lilygo_tbeam_1w/platformio.ini index 73723c29f..827cac385 100644 --- a/variants/lilygo_tbeam_1w/platformio.ini +++ b/variants/lilygo_tbeam_1w/platformio.ini @@ -209,7 +209,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D PERSISTANT_GPS=1 @@ -256,7 +256,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D PERSISTANT_GPS=1 diff --git a/variants/lilygo_tbeam_SX1262/platformio.ini b/variants/lilygo_tbeam_SX1262/platformio.ini index 575178576..2e0d8334a 100644 --- a/variants/lilygo_tbeam_SX1262/platformio.ini +++ b/variants/lilygo_tbeam_SX1262/platformio.ini @@ -161,7 +161,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 ; -D MQTT_DEBUG=1 @@ -204,7 +204,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/lilygo_tbeam_SX1276/platformio.ini b/variants/lilygo_tbeam_SX1276/platformio.ini index aa3383dbf..41f561153 100644 --- a/variants/lilygo_tbeam_SX1276/platformio.ini +++ b/variants/lilygo_tbeam_SX1276/platformio.ini @@ -160,7 +160,7 @@ build_flags = -D MAX_NEIGHBOURS=50 -D PERSISTANT_GPS=1 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 ; -D MQTT_DEBUG=1 @@ -203,7 +203,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/lilygo_tbeam_supreme_SX1262/platformio.ini b/variants/lilygo_tbeam_supreme_SX1262/platformio.ini index a8a847214..7c588e7c5 100644 --- a/variants/lilygo_tbeam_supreme_SX1262/platformio.ini +++ b/variants/lilygo_tbeam_supreme_SX1262/platformio.ini @@ -120,7 +120,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -163,7 +163,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/lilygo_tlora_v2_1/platformio.ini b/variants/lilygo_tlora_v2_1/platformio.ini index 9a4a83656..37a909f84 100644 --- a/variants/lilygo_tlora_v2_1/platformio.ini +++ b/variants/lilygo_tlora_v2_1/platformio.ini @@ -157,7 +157,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_TASK_STACK_SIZE=16384 @@ -203,7 +203,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_TASK_STACK_SIZE=16384 diff --git a/variants/rak3112/platformio.ini b/variants/rak3112/platformio.ini index f284a3a17..130b83444 100644 --- a/variants/rak3112/platformio.ini +++ b/variants/rak3112/platformio.ini @@ -113,7 +113,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -177,7 +177,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/station_g2/platformio.ini b/variants/station_g2/platformio.ini index aaa74b150..338f4c311 100644 --- a/variants/station_g2/platformio.ini +++ b/variants/station_g2/platformio.ini @@ -187,7 +187,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -306,7 +306,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 diff --git a/variants/xiao_s3_wio/platformio.ini b/variants/xiao_s3_wio/platformio.ini index 873454c01..8e4314cf6 100644 --- a/variants/xiao_s3_wio/platformio.ini +++ b/variants/xiao_s3_wio/platformio.ini @@ -103,7 +103,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D MAX_NEIGHBOURS=50 -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 @@ -147,7 +147,7 @@ build_flags = -D ADMIN_PASSWORD='"password"' -D ROOM_PASSWORD='"hello"' -D WITH_MQTT_BRIDGE=1 - -D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"' + -D OTA_MANIFEST_BASE='"https://observer.gessaman.com/v"' -D MAX_MQTT_BROKERS=3 -D MQTT_MAX_PACKET_SIZE=1024 -D MQTT_DEBUG=1 From 5acfdd736a1cdbe0d4fa429b7b12d98c455160a0 Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 23 Jun 2026 14:08:12 -0700 Subject: [PATCH 10/16] fix(ota): enhance manifest fetching logic for OTA updates Improved the otaFromManifestImpl method to handle dry run scenarios by allowing HTTP fetches without TLS, reducing heap usage on no-PSRAM boards. This change ensures better compatibility and reliability during OTA checks while maintaining security for actual updates. --- src/helpers/ESP32Board.cpp | 55 +++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index e1267a6ff..f698e59a0 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -170,32 +170,51 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char rootca_crt_bundle_end > rootca_crt_bundle_start) { bundle_len = (size_t)(rootca_crt_bundle_end - rootca_crt_bundle_start); } - if (bundle_len == 0) { + if (!dry_run && bundle_len == 0) { strcpy(reply, "ERR: no embedded cert bundle"); return false; } // --- Fetch this variant's slim manifest ---------------------------------- - // /.json — a ~180 byte per-variant file, NOT - // the full config.json. Keeping the body tiny is what lets `ota check` run with - // the MQTT bridge up on no-PSRAM boards: only the TLS handshake costs heap, with - // no large JSON document layered on top. + // /.json — a ~180 byte per-variant file, not the + // full config.json. char murl[200]; - snprintf(murl, sizeof(murl), "%s/%s.json", OTA_MANIFEST_BASE, OTA_VARIANT); - - WiFiClientSecure mclient; -#if ESP_ARDUINO_VERSION_MAJOR >= 3 - mclient.setCACertBundle(rootca_crt_bundle_start, bundle_len); -#else - mclient.setCACertBundle(rootca_crt_bundle_start); -#endif - mclient.setTimeout(15000); - HTTPClient http; - if (!http.begin(mclient, murl)) { - strcpy(reply, "ERR: manifest connect failed"); - return false; + WiFiClientSecure mclient; // only used for the HTTPS (update) path + + if (dry_run) { + // `ota check`: fetch over PLAIN HTTP. With no TLS handshake the fetch costs + // negligible heap, so the check runs with the MQTT bridge UP even on no-PSRAM + // — where the cert-bundle TLS verify would otherwise exhaust internal heap + // alongside the two live MQTT TLS sessions (free heap collapses to a few KB + // and the handshake + the bridge both fail). This only reads version info; the + // firmware download below (ota update) is always TLS-verified. Requires the + // manifest host to serve /v over HTTP (no forced HTTPS redirect). + if (strncmp(OTA_MANIFEST_BASE, "https://", 8) == 0) { + snprintf(murl, sizeof(murl), "http://%s/%s.json", OTA_MANIFEST_BASE + 8, OTA_VARIANT); + } else { + snprintf(murl, sizeof(murl), "%s/%s.json", OTA_MANIFEST_BASE, OTA_VARIANT); + } + if (!http.begin(murl)) { + strcpy(reply, "ERR: manifest connect failed"); + return false; + } + } else { + // `ota update`: HTTPS. The bridge is torn down for an update so heap is free, + // and integrity matters because we're about to flash. +#if ESP_ARDUINO_VERSION_MAJOR >= 3 + mclient.setCACertBundle(rootca_crt_bundle_start, bundle_len); +#else + mclient.setCACertBundle(rootca_crt_bundle_start); +#endif + mclient.setTimeout(15000); + snprintf(murl, sizeof(murl), "%s/%s.json", OTA_MANIFEST_BASE, OTA_VARIANT); + if (!http.begin(mclient, murl)) { + strcpy(reply, "ERR: manifest connect failed"); + return false; + } } + // Force HTTP/1.0: a CDN (e.g. Cloudflare) answers HTTP/1.1 with chunked encoding // and no Content-Length; the raw chunked stream corrupts the parse. HTTP/1.0 // yields a Connection: close, unframed body. From a6851813f96586a0fc492129eb6f7d1b0c56e050 Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 23 Jun 2026 14:25:06 -0700 Subject: [PATCH 11/16] fix(ota): improve display of version and hash in OTA status messages Updated the otaFromManifestImpl method to enhance the display of available and current firmware versions. The changes ensure that the short commit hash is included in the status messages, providing clearer versioning information during OTA checks. --- src/helpers/ESP32Board.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index f698e59a0..516c14221 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -288,8 +288,13 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char up_to_date = hash_equal; // unknown build numbers -> fall back to hash } - const char* avail_disp = avail_version[0] ? avail_version : avail_hash; - const char* own_disp = own_version[0] ? own_version : cur_hash; + // Display strings carry the short commit hash in the same form as the asset + // filename, e.g. "v1.16.0.2 (5acfdd7)" (or just the hash if there's no version). + char avail_disp[72], own_disp[72]; + if (avail_version[0]) snprintf(avail_disp, sizeof(avail_disp), "%s (%s)", avail_version, avail_hash); + else snprintf(avail_disp, sizeof(avail_disp), "%s", avail_hash); + if (own_version[0]) snprintf(own_disp, sizeof(own_disp), "%s (%s)", own_version, cur_hash); + else snprintf(own_disp, sizeof(own_disp), "%s", cur_hash); const char* pc_note = partition_change ? " [partition change: cable flash]" : ""; // --- Report (dry run / `ota check`) -------------------------------------- @@ -312,7 +317,7 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char return false; } if (up_to_date) { - snprintf(reply, 160, "OK: already up to date (%s)", avail_disp); + snprintf(reply, 160, "OK: already up to date: %s", avail_disp); return false; } From 125ddac184a07683c5dcc8a648ca13b5f1630d4e Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 23 Jun 2026 15:38:10 -0700 Subject: [PATCH 12/16] feat(ota): add partition-table signature handling for OTA updates Implemented functionality to generate and compare partition-table signatures during OTA updates. This enhancement ensures that the target build's partition layout matches the device's actual layout, improving the reliability of OTA updates and preventing issues related to partition changes. --- .../workflows/build-observer-firmwares.yml | 5 +- build.sh | 7 ++ examples/simple_repeater/MyMesh.cpp | 3 +- examples/simple_repeater/main.cpp | 5 ++ scripts/partition_signature.py | 44 +++++++++++ src/helpers/CommonCLI.cpp | 42 +++++++++-- src/helpers/ESP32Board.cpp | 74 ++++++++++++++++++- 7 files changed, 169 insertions(+), 11 deletions(-) create mode 100644 scripts/partition_signature.py diff --git a/.github/workflows/build-observer-firmwares.yml b/.github/workflows/build-observer-firmwares.yml index 0db674ae6..6a0c70cf8 100644 --- a/.github/workflows/build-observer-firmwares.yml +++ b/.github/workflows/build-observer-firmwares.yml @@ -159,6 +159,8 @@ jobs: run: | mkdir -p out find artifacts -type f -name '*.bin' -exec cp -f {} out/ \; + # Per-env partition-table signatures (for the slim manifest's OTA gate). + find artifacts -type f -name '*.partsig' -exec cp -f {} out/ \; echo "Collected binaries:"; ls -1 out - name: Compute Short SHA @@ -233,7 +235,8 @@ jobs: --config flasher/config.json \ --out-dir flasher/v \ --base-version "$FIRMWARE_VERSION" \ - --build "$BUILD_NUMBER" + --build "$BUILD_NUMBER" \ + --partsig-dir out printf '{\n "baseVersion": "%s",\n "build": %s\n}\n' \ "$FIRMWARE_VERSION" "$BUILD_NUMBER" > flasher/observer-build-counter.json echo "Build $FIRMWARE_VERSION.$BUILD_NUMBER" diff --git a/build.sh b/build.sh index 3f58d2a04..8ea83b40c 100755 --- a/build.sh +++ b/build.sh @@ -188,6 +188,13 @@ build_firmware() { cp .pio/build/$1/firmware.uf2 out/${FIRMWARE_FILENAME}.uf2 2>/dev/null || true cp .pio/build/$1/firmware.zip out/${FIRMWARE_FILENAME}.zip 2>/dev/null || true + # Emit the partition-table signature (ESP32) for OTA partition-compatibility + # checks. Keyed by env name so the slim-manifest generator can find it; the + # firmware computes the same signature at runtime from its flashed table. + if [ -f ".pio/build/$1/partitions.bin" ]; then + python3 scripts/partition_signature.py ".pio/build/$1/partitions.bin" > "out/$1.partsig" 2>/dev/null || true + fi + } # firmwares containing $1 will be built diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index d283acbd3..833c318f4 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1452,10 +1452,11 @@ void MyMesh::loop() { // headroom, then flash: otaFromManifest reboots into the new image on success // (so this never returns); on any abort (already up to date, partition change, // download error) it returns and we resume the bridge. + Serial.println("OTA: starting update"); setBridgeState(false); char ota_reply[160]; if (!_cli.getBoard()->otaFromManifest(getFirmwareVer(), false, ota_reply)) { - MESH_DEBUG_PRINTLN("ota update aborted: %s", ota_reply); + Serial.print("OTA: aborted, resuming bridge - "); Serial.println(ota_reply); setBridgeState(true); } } diff --git a/examples/simple_repeater/main.cpp b/examples/simple_repeater/main.cpp index 2ce056f52..c56c0257a 100644 --- a/examples/simple_repeater/main.cpp +++ b/examples/simple_repeater/main.cpp @@ -82,6 +82,11 @@ void setup() { store.save("_main", the_mesh.self_id); } + // Print the running firmware version at boot so it's visible after an OTA + // reboot without having to issue `ver` manually. + Serial.print("Firmware: "); Serial.print(FIRMWARE_VERSION); + Serial.print(" (built "); Serial.print(FIRMWARE_BUILD_DATE); Serial.println(")"); + Serial.print("Repeater ID: "); mesh::Utils::printHex(Serial, the_mesh.self_id.pub_key, PUB_KEY_SIZE); Serial.println(); diff --git a/scripts/partition_signature.py b/scripts/partition_signature.py new file mode 100644 index 000000000..326b9e514 --- /dev/null +++ b/scripts/partition_signature.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Canonical signature of an ESP32 partition table (from a partitions.bin). + +Used to decide OTA partition compatibility: the observer firmware computes the +same signature at runtime from its *flashed* partition table (via esp_partition), +and `ota update` refuses only when the target build's signature differs from the +running device's — i.e. a real partition-table change, not a blanket flag. + +The signature MUST be computed identically here and in firmware +(src/helpers/ESP32Board.cpp). Definition: + + for each partition-table entry: (type, subtype, offset, size) + sort by offset ascending + format each as "%x:%x:%x:%x" (lowercase hex, no 0x, no padding) + join with "," + +partitions.bin layout: 32-byte records, each starting with magic 0xAA 0x50; the +trailing MD5 record (magic 0xEB 0xEB) and 0xFF padding are ignored. + +Usage: partition_signature.py # prints the signature +""" +import struct +import sys + + +def signature(bin_path: str) -> str: + data = open(bin_path, "rb").read() + entries = [] + for i in range(0, len(data) - 31, 32): + rec = data[i:i + 32] + if rec[0:2] != b"\xaa\x50": # not a partition entry (MD5 record / padding) + continue + ptype, subtype = rec[2], rec[3] + offset, size = struct.unpack("", file=sys.stderr) + sys.exit(2) + print(signature(sys.argv[1])) diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 367ee47cc..99849b910 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -809,13 +809,41 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re // the live MQTT sessions even on no-PSRAM boards. No bridge bounce needed. _board->otaFromManifest(_callbacks->getFirmwareVer(), true, reply); } else { - // Update is DEFERRED: the flash blocks the loop and then reboots, so it - // must run only AFTER this reply has gone out over the mesh — otherwise - // the requester never gets a confirmation. The app loop runs it shortly. - if (_callbacks->beginDeferredOtaUpdate()) { - strcpy(reply, "Beginning update... (node will reboot if successful)"); - } else { - strcpy(reply, "ERR: online OTA not available"); + // `ota update`: cheap pre-check first (plain HTTP, bridge stays up). Only + // schedule the real update — which tears the bridge down, flashes, and + // reboots — when an applicable build actually exists. otaFromManifest(dry) + // returns true iff so; otherwise it leaves the explanation (up to date / + // cable flash / error) in reply, which we send without disturbing the + // bridge or misleading the user with a "Beginning update..." that no-ops. + if (_board->otaFromManifest(_callbacks->getFirmwareVer(), true, reply)) { + // reply now holds "update available: -> (N behind|new base)", + // where is "vX.Y.Z.B (hash)". Pull out for a friendlier + // start message. The "-> " ... trailing " (" framing is produced by + // ESP32Board::otaFromManifestImpl; ends at the LAST " (" (the + // "(N behind)"/"(new base)" suffix), since the version's own hash-paren + // comes before it. + char target[48] = {0}; + const char* arrow = strstr(reply, "-> "); + if (arrow) { + arrow += 3; + const char* suffix = nullptr; + for (const char* p = arrow; (p = strstr(p, " (")) != nullptr; p++) suffix = p; + size_t len = suffix ? (size_t)(suffix - arrow) : strlen(arrow); + if (len >= sizeof(target)) len = sizeof(target) - 1; + memcpy(target, arrow, len); + target[len] = 0; + } + // Update is DEFERRED so this ack goes out over the mesh before the flash + // blocks the loop and reboots (the app loop runs it shortly). + if (_callbacks->beginDeferredOtaUpdate()) { + if (target[0]) { + snprintf(reply, 160, "Updating to %s; reboots when done (~30s offline). Check 'ver' after.", target); + } else { + strcpy(reply, "Beginning update... (node will reboot if successful)"); + } + } else { + strcpy(reply, "ERR: online OTA not available"); + } } } #else diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 516c14221..5a4576388 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -73,6 +73,7 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[]) { #include #include #include +#include // Embedded CA bundle (produced by board_build.embed_files). Weak so non-bundle // builds still link; we check for presence at runtime. @@ -118,6 +119,41 @@ static void ota_parseVersion(const char* ver, char* base_out, size_t base_sz, in } } +// Canonical signature of the FLASHED partition table — MUST match +// scripts/partition_signature.py: each entry "type:subtype:offset:size" in +// lowercase hex, sorted by offset, joined by ','. Lets `ota update` compare the +// target build's partition layout (carried in the manifest as partSig) against +// what's actually on this device, instead of a blanket per-variant flag. +static void ota_partitionSignature(char* out, size_t out_sz) { + struct PE { uint8_t type, subtype; uint32_t off, size; } e[24]; + int n = 0; + esp_partition_iterator_t it = + esp_partition_find(ESP_PARTITION_TYPE_ANY, ESP_PARTITION_SUBTYPE_ANY, nullptr); + while (it != nullptr && n < (int)(sizeof(e) / sizeof(e[0]))) { + const esp_partition_t* p = esp_partition_get(it); + e[n].type = (uint8_t)p->type; + e[n].subtype = (uint8_t)p->subtype; + e[n].off = p->address; + e[n].size = p->size; + n++; + it = esp_partition_next(it); + } + esp_partition_iterator_release(it); // safe on NULL (loop exhausted) + // insertion sort by offset (matches the script's sort key) + for (int i = 1; i < n; i++) { + PE k = e[i]; + int j = i - 1; + while (j >= 0 && e[j].off > k.off) { e[j + 1] = e[j]; j--; } + e[j + 1] = k; + } + size_t pos = 0; + if (out_sz) out[0] = 0; + for (int i = 0; i < n && pos + 1 < out_sz; i++) { + pos += snprintf(out + pos, out_sz - pos, "%s%x:%x:%x:%x", + i ? "," : "", e[i].type, e[i].subtype, (unsigned)e[i].off, (unsigned)e[i].size); + } +} + // Parameters handed to the worker task; lives on otaFromManifest()'s stack, // which stays valid because that function blocks until the worker signals done. struct OtaTaskArgs { @@ -215,6 +251,8 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char } } + if (!dry_run) { Serial.print("OTA: checking manifest "); Serial.println(murl); } + // Force HTTP/1.0: a CDN (e.g. Cloudflare) answers HTTP/1.1 with chunked encoding // and no Content-Length; the raw chunked stream corrupts the parse. HTTP/1.0 // yields a Connection: close, unframed body. @@ -245,7 +283,9 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char strncpy(avail_base, doc["baseVersion"] | "", sizeof(avail_base) - 1); strncpy(avail_hash, doc["hash"] | "", sizeof(avail_hash) - 1); int avail_build = doc["build"] | -1; - bool partition_change = doc["partitionChange"] | false; + bool legacy_partition_change = doc["partitionChange"] | false; + char manifest_partsig[256] = {0}; + strncpy(manifest_partsig, doc["partSig"] | "", sizeof(manifest_partsig) - 1); doc.clear(); if (!file_url[0]) { @@ -253,6 +293,19 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char return false; } + // Partition compatibility: prefer the precise per-build signature — compare the + // target build's partition layout (manifest partSig) to what's actually flashed + // on THIS device. Refuse only on a real mismatch (OTA can't rewrite the table). + // Fall back to the legacy bool for manifests that predate partSig. + bool partition_change; + if (manifest_partsig[0]) { + char dev_partsig[256]; + ota_partitionSignature(dev_partsig, sizeof(dev_partsig)); + partition_change = (strcmp(dev_partsig, manifest_partsig) != 0); + } else { + partition_change = legacy_partition_change; + } + // --- Determine current-vs-available -------------------------------------- // Our running version token (e.g. "v1.16.0.5"), i.e. current_ver up to the // first '-' (which precedes "-observer-"). @@ -298,6 +351,9 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char const char* pc_note = partition_change ? " [partition change: cable flash]" : ""; // --- Report (dry run / `ota check`) -------------------------------------- + // Returns true iff an OTA-applicable update is available (not up-to-date and not + // a partition-change build). `ota check` ignores the return and just shows the + // reply; `ota update` uses it to decide whether to actually schedule the flash. if (dry_run) { if (up_to_date) { snprintf(reply, 160, "up to date: %s", avail_disp); @@ -308,7 +364,7 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char } else { snprintf(reply, 160, "update available: %s -> %s%s", own_disp, avail_disp, pc_note); } - return true; + return (!up_to_date && !partition_change); } // --- Gates (real `ota update`) ------------------------------------------- @@ -322,6 +378,8 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char } // --- Stream the .bin (the manifest's full URL) into the inactive OTA slot - + Serial.printf("OTA: update %s -> %s\n", own_disp, avail_disp); + Serial.print("OTA: downloading "); Serial.println(file_url); inhibit_sleep = true; // keep awake through the flash WiFiClientSecure uclient; @@ -332,6 +390,17 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char #endif uclient.setTimeout(20000); + // Console progress to the USB serial (always on; MESH_DEBUG is off on the default + // observer profile). Non-capturing lambdas + a file-static decile, so the global + // httpUpdate object never holds a dangling reference after this function returns. + static int ota_progress_decile; + ota_progress_decile = -1; + httpUpdate.onProgress([](int cur, int total) { + if (total <= 0) return; + int d = (int)((int64_t)cur * 10 / total); + if (d != ota_progress_decile) { ota_progress_decile = d; Serial.printf("OTA: %d%%\n", d * 10); } + }); + httpUpdate.onEnd([]() { Serial.println("OTA: write complete, rebooting..."); }); httpUpdate.rebootOnUpdate(true); // reboots into the new image on success t_httpUpdate_return ret = httpUpdate.update(uclient, file_url); @@ -339,6 +408,7 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char inhibit_sleep = false; snprintf(reply, 160, "ERR: OTA failed (%d): %s", (int)ret, httpUpdate.getLastErrorString().c_str()); + Serial.print("OTA: FAILED - "); Serial.println(reply); return false; #endif // OTA_MANIFEST_BASE && OTA_VARIANT } From b5797da46244c2338a112854cdd9737eba3f3be5 Mon Sep 17 00:00:00 2001 From: agessaman Date: Tue, 23 Jun 2026 15:59:28 -0700 Subject: [PATCH 13/16] fix(github-actions): exclude .partsig files from release asset uploads Updated the build-observer-firmwares workflow to exclude .partsig files when uploading release assets. This change prevents issues with the slim-manifest generator and ensures that only relevant firmware files are published, improving the integrity of the release process. --- .github/workflows/build-observer-firmwares.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-observer-firmwares.yml b/.github/workflows/build-observer-firmwares.yml index 6a0c70cf8..3c13bf0ca 100644 --- a/.github/workflows/build-observer-firmwares.yml +++ b/.github/workflows/build-observer-firmwares.yml @@ -180,8 +180,10 @@ jobs: --notes "Rolling build of all *_observer_mqtt firmwares. The git short hash is embedded in each asset filename." fi - # Upload this build, overwriting same-named assets. - gh release upload "$RELEASE_TAG" out/* --clobber + # Upload this build, overwriting same-named assets. Exclude the .partsig + # files — those are consumed locally by the slim-manifest generator below, + # not published as release assets (and they'd break the hash-based prune). + gh release upload "$RELEASE_TAG" $(find out -maxdepth 1 -type f ! -name '*.partsig') --clobber # Prune old assets, but RETAIN the most recent KEEP_BUILDS build hashes # (not just the current one). The flasher SPA reads config.json once at @@ -203,7 +205,10 @@ jobs: echo "Retaining build hashes:"; echo "$keep_hashes" gh release view "$RELEASE_TAG" --json assets -q '.assets[].name' \ | while read -r asset; do - ah=$(printf '%s' "$asset" | grep -oiE '[0-9a-f]{7,40}(-merged)?\.bin$' | grep -oiE '^[0-9a-f]{7,40}') + # `|| true`: a non-firmware asset (no embedded hash) makes grep exit + # non-zero, which under `bash -e` would abort the step. Empty ah then + # falls through to delete-asset, cleaning up any stray non-.bin asset. + ah=$(printf '%s' "$asset" | grep -oiE '[0-9a-f]{7,40}(-merged)?\.bin$' | grep -oiE '^[0-9a-f]{7,40}' || true) if [ -n "$ah" ] && grep -qxF "$ah" <<<"$keep_hashes"; then continue fi From 7416d632b907c7432ccace1a7ca392c9f868a627 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 24 Jun 2026 17:14:11 -0700 Subject: [PATCH 14/16] feat(mqtt): add NTP server configuration and diagnostics commands Implemented new commands for configuring and diagnosing NTP server settings in the MQTT bridge. Users can now set a custom NTP server and probe connectivity to configured servers. This enhancement improves time synchronization reliability for JWT authentication and provides better diagnostics for NTP connectivity issues. --- MQTT_IMPLEMENTATION.md | 12 +- docs/cli_commands.md | 27 +++ examples/simple_repeater/MyMesh.h | 15 ++ examples/simple_room_server/MyMesh.h | 15 ++ src/helpers/CommonCLI.cpp | 62 +++++++ src/helpers/CommonCLI.h | 18 ++ src/helpers/bridges/MQTTBridge.cpp | 268 +++++++++++++++++++++++---- src/helpers/bridges/MQTTBridge.h | 50 ++++- 8 files changed, 429 insertions(+), 38 deletions(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index d218eef9b..6695ed193 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -378,6 +378,8 @@ These settings apply across all MQTT slots: - `get mqtt.rx` - Get RX packet uplinking setting (on/off) - `get mqtt.tx` - Get TX packet uplinking setting (on/off/advert) - `get mqtt.interval` - Get status publish interval +- `get mqtt.ntp` - Get effective NTP server hostname +- `get mqtt.ntp.diag` - Probe every configured NTP server for connectivity (does not change the clock; serial console shows each server's reported time, LoRa shows a compact ` ok|fail` list) - `get mqtt.owner` - Get owner public key (serial console only) - `get mqtt.email` - Get owner email address (serial console only) @@ -393,6 +395,7 @@ These settings apply across all MQTT slots: - `advert` - Uplink only this node's own advert packets (self-originated) - `off` - Disable TX packet uplinking - `set mqtt.interval ` - Set status publish interval (1-60 minutes) +- `set mqtt.ntp ` - Set custom NTP server (validated with immediate sync); `none` reverts to default - `set mqtt.owner <64-hex-char-public-key>` - Set owner public key - `set mqtt.email ` - Set owner email address @@ -607,8 +610,13 @@ Minimal raw packet data for map integration. - Automatic reconnection with exponential backoff ### NTP Time Synchronization -- Automatic time synchronization with NTP servers -- Periodic time updates (every hour) +- Automatic time synchronization with NTP servers (required for JWT authentication) +- Default primary: `pool.ntp.org`; built-in fallbacks (tried sequentially on failure): `time.google.com`, `time.cloudflare.com`, `time.aws.com`, `time.nist.gov` +- Custom primary via `set mqtt.ntp `; `set mqtt.ntp none` reverts to default +- `set mqtt.ntp` runs an immediate sync (primary only, so a typo fails fast) when WiFi is connected and the bridge is running +- `get mqtt.ntp` returns the effective primary hostname +- `get mqtt.ntp.diag` probes every configured server (primary + fallbacks) for connectivity and reports each server's time without changing the system clock — a pure diagnostic +- Periodic time updates (every hour) on the effective primary only - Proper UTC system time handling ### Authentication diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 32aeb95c2..b0badbcd3 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -1076,6 +1076,33 @@ region save --- +#### View or change the NTP server (MQTT observer only) +**Usage:** +- `get mqtt.ntp` +- `set mqtt.ntp ` +- `set mqtt.ntp none` + +**Description:** Sets the primary NTP server used for clock sync (required for JWT MQTT auth). On `set`, the device attempts an immediate sync of the just-configured server (primary only, so a typo fails fast) when WiFi is connected and the MQTT bridge is running. + +**Fallbacks:** If the primary fails, the firmware tries `pool.ntp.org`, `time.google.com`, `time.cloudflare.com`, `time.aws.com`, and `time.nist.gov` in order (skipping duplicates). + +**Default:** `pool.ntp.org` (when unset or `none`) + +--- + +#### Diagnose NTP server connectivity (MQTT observer only) +**Usage:** +- `get mqtt.ntp.diag` + +**Description:** Probes every configured NTP server (the custom primary, if set, plus the built-in fallbacks) and reports whether each responds. This is a pure connectivity diagnostic — it does **not** change the system clock. + +- **Serial console:** prints a detailed table with each server's reported UTC time (or `FAIL`). +- **Over LoRa:** returns a compact ` ok|fail` list, one per line. + +Requires WiFi connected and the MQTT bridge running. + +--- + #### View or change the speed of the bridge (RS-232 only) **Usage:** - `get bridge.baud` diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 09ee140e7..9ef04ac8b 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -319,6 +319,21 @@ public: int getQueueSize() override { return bridge ? bridge->getQueueSize() : 0; } + + bool isMqttBridgeRunning() override { + return bridge && bridge->isRunning(); + } + + bool syncMqttNtp() override { + if (!bridge || !bridge->isRunning()) return false; + // Marshal onto the MQTT task (Core 0); this runs on the CLI thread (Core 1). + return bridge->requestForcedNtpSync(); + } + + bool runMqttNtpDiag(char* reply, size_t reply_size, bool verbose) override { + if (!bridge || !bridge->isRunning()) return false; + return bridge->ntpDiag(reply, reply_size, verbose); + } #endif // To check if there is pending work diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index afc4828cf..72e5ff969 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -296,5 +296,20 @@ public: int getQueueSize() override { return bridge ? bridge->getQueueSize() : 0; } + + bool isMqttBridgeRunning() override { + return bridge && bridge->isRunning(); + } + + bool syncMqttNtp() override { + if (!bridge || !bridge->isRunning()) return false; + // Marshal onto the MQTT task (Core 0); this runs on the CLI thread (Core 1). + return bridge->requestForcedNtpSync(); + } + + bool runMqttNtpDiag(char* reply, size_t reply_size, bool verbose) override { + if (!bridge || !bridge->isRunning()) return false; + return bridge->ntpDiag(reply, reply_size, verbose); + } #endif }; diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 99849b910..507736381 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -50,6 +50,21 @@ static int getMQTTPresetNameCount() { return MQTT_PRESET_COUNT + 2; // built-ins + custom + none } +static bool isValidNtpHostname(const char* host) { + if (!host || host[0] == '\0') return false; + size_t len = strlen(host); + if (len > 63) return false; + if (host[0] == '.' || host[len - 1] == '.') return false; + for (size_t i = 0; i < len; i++) { + char c = host[i]; + if (!((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9') || c == '.' || c == '-')) { + return false; + } + } + return true; +} + static const char* getMQTTPresetNameByIndex(int index) { if (index < MQTT_PRESET_COUNT) return MQTT_PRESETS[index].name; if (index == MQTT_PRESET_COUNT) return MQTT_PRESET_CUSTOM; @@ -657,6 +672,7 @@ void CommonCLI::syncMQTTPrefsToNodePrefs() { } StrHelper::strncpy(_prefs->mqtt_owner_public_key, _mqtt_prefs.mqtt_owner_public_key, sizeof(_prefs->mqtt_owner_public_key)); StrHelper::strncpy(_prefs->mqtt_email, _mqtt_prefs.mqtt_email, sizeof(_prefs->mqtt_email)); + StrHelper::strncpy(_prefs->mqtt_ntp_server, _mqtt_prefs.mqtt_ntp_server, sizeof(_prefs->mqtt_ntp_server)); } void CommonCLI::syncNodePrefsToMQTTPrefs() { @@ -688,6 +704,7 @@ void CommonCLI::syncNodePrefsToMQTTPrefs() { } StrHelper::strncpy(_mqtt_prefs.mqtt_owner_public_key, _prefs->mqtt_owner_public_key, sizeof(_mqtt_prefs.mqtt_owner_public_key)); StrHelper::strncpy(_mqtt_prefs.mqtt_email, _prefs->mqtt_email, sizeof(_mqtt_prefs.mqtt_email)); + StrHelper::strncpy(_mqtt_prefs.mqtt_ntp_server, _prefs->mqtt_ntp_server, sizeof(_mqtt_prefs.mqtt_ntp_server)); } #endif @@ -1466,6 +1483,35 @@ void CommonCLI::handleSetCmd(uint32_t sender_timestamp, char* command, char* rep } else { strcpy(reply, "Error: interval must be between 1-60 minutes"); } + } else if (memcmp(config, "mqtt.ntp ", 9) == 0) { + const char* host = &config[9]; + while (*host == ' ') host++; + bool clearing = strcmp(host, "none") == 0; + if (!clearing && !isValidNtpHostname(host)) { + strcpy(reply, "Error: invalid NTP hostname"); + } else { + if (clearing) { + _prefs->mqtt_ntp_server[0] = '\0'; + } else { + StrHelper::strncpy(_prefs->mqtt_ntp_server, host, sizeof(_prefs->mqtt_ntp_server)); + } + savePrefs(); +#ifdef ESP_PLATFORM + // Validate by running an immediate sync. syncMqttNtp() marshals onto the MQTT + // task (Core 0) so no NTP I/O happens on this (Core 1) CLI thread. + if (WiFi.status() != WL_CONNECTED) { + strcpy(reply, "OK - saved (WiFi not connected; NTP sync pending)"); + } else if (!_callbacks->isMqttBridgeRunning()) { + strcpy(reply, "OK - saved (MQTT bridge not running)"); + } else if (_callbacks->syncMqttNtp()) { + strcpy(reply, "OK - time synced"); + } else { + strcpy(reply, "Error: NTP sync failed"); + } +#else + strcpy(reply, "OK - saved"); +#endif + } } else if (memcmp(config, "wifi.ssid ", 10) == 0) { StrHelper::strncpy(_prefs->wifi_ssid, &config[10], sizeof(_prefs->wifi_ssid)); savePrefs(); @@ -2020,6 +2066,22 @@ void CommonCLI::handleGetCmd(uint32_t sender_timestamp, char* command, char* rep } else if (memcmp(config, "mqtt.interval", 13) == 0) { uint32_t minutes = (_prefs->mqtt_status_interval + 29999) / 60000; sprintf(reply, "> %u minutes (%lu ms)", minutes, (unsigned long)_prefs->mqtt_status_interval); + } else if (memcmp(config, "mqtt.ntp.diag", 13) == 0 && (config[13] == '\0' || config[13] == ' ')) { +#ifdef ESP_PLATFORM + // Connectivity probe across all configured NTP servers; never updates the clock. + // Serial console (sender_timestamp == 0) gets a detailed table; LoRa gets a compact list. + if (WiFi.status() != WL_CONNECTED) { + strcpy(reply, "Error: WiFi not connected"); + } else if (!_callbacks->isMqttBridgeRunning()) { + strcpy(reply, "Error: MQTT bridge not running"); + } else if (!_callbacks->runMqttNtpDiag(reply, 160, sender_timestamp == 0)) { + strcpy(reply, "Error: NTP diag unavailable"); + } +#else + strcpy(reply, "Error: not supported on this platform"); +#endif + } else if (memcmp(config, "mqtt.ntp", 8) == 0 && (config[8] == '\0' || config[8] == ' ')) { + sprintf(reply, "> %s", MQTTBridge::effectiveNtpPrimary(_prefs)); } else if (config[0] == 'm' && config[1] == 'q' && config[2] == 't' && config[3] == 't' && config[4] >= '1' && config[4] <= ('0' + MAX_MQTT_SLOTS) && config[5] == '.') { // Slot-based commands: get mqtt1.preset, get mqtt1.server, etc. diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index a158324c7..49f7fe04f 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -125,6 +125,9 @@ struct NodePrefs { // persisted to file // time, so the operator can name a region that doesn't exist yet without // polluting region_map state. Falls back to default_scope on miss. char alert_region[31]; + + // Custom NTP server (MQTT observer); empty = built-in default primary (pool.ntp.org) + char mqtt_ntp_server[64]; }; #ifdef WITH_MQTT_BRIDGE @@ -202,6 +205,7 @@ struct MQTTPrefs { // --- Appended fields (added after initial 6-slot migration) --- uint8_t mqtt_rx_enabled; // Enable RX packet uplinking (default: on) + char mqtt_ntp_server[64]; // Custom NTP server; empty = pool.ntp.org }; // 3-slot MQTTPrefs layout — used for migrating from 3-slot to 6-slot format. @@ -300,6 +304,20 @@ public: return 0; // no op by default }; + virtual bool syncMqttNtp() { + return false; // WITH_MQTT_BRIDGE builds override + }; + + virtual bool isMqttBridgeRunning() { + return false; + }; + + // Probe all configured NTP servers for connectivity (verbose=serial console gets a + // detailed table; otherwise reply gets a compact " ok|fail" list). + virtual bool runMqttNtpDiag(char* reply, size_t reply_size, bool verbose) { + return false; // WITH_MQTT_BRIDGE builds override + }; + virtual void setRxBoostedGain(bool enable) { // no op by default }; diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 0c1ff86b9..3c3ac91f3 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #ifdef WITH_SNMP #include "../SNMPAgent.h" @@ -34,6 +35,50 @@ static void applyEffectiveOrigin(const NodePrefs* prefs, char* dest, size_t dest StrHelper::stripSurroundingQuotes(dest, dest_size); } +static const char* const kNtpBuiltinFallbacks[] = { + "pool.ntp.org", + "time.google.com", + "time.cloudflare.com", + "time.aws.com", + "time.nist.gov", +}; +static constexpr size_t kNtpBuiltinFallbackCount = + sizeof(kNtpBuiltinFallbacks) / sizeof(kNtpBuiltinFallbacks[0]); +static_assert(MQTTBridge::kMaxNtpServers >= 1 + (int)kNtpBuiltinFallbackCount, + "kMaxNtpServers must hold the custom primary plus all built-in fallbacks"); + +static bool ntpHostnameEquals(const char* a, const char* b) { + if (!a || !b) return false; + return strcasecmp(a, b) == 0; +} + +static void fillNtpServerList(const NodePrefs* prefs, const char* servers[], int& count) { + count = 0; + if (prefs && prefs->mqtt_ntp_server[0] != '\0') { + servers[count++] = prefs->mqtt_ntp_server; + } + for (size_t i = 0; i < kNtpBuiltinFallbackCount && count < MQTTBridge::kMaxNtpServers; i++) { + const char* fb = kNtpBuiltinFallbacks[i]; + bool dup = false; + for (int j = 0; j < count; j++) { + if (ntpHostnameEquals(servers[j], fb)) { + dup = true; + break; + } + } + if (!dup) { + servers[count++] = fb; + } + } +} + +const char* MQTTBridge::effectiveNtpPrimary(const NodePrefs* prefs) { + if (prefs && prefs->mqtt_ntp_server[0] != '\0') { + return prefs->mqtt_ntp_server; + } + return kNtpBuiltinFallbacks[0]; +} + void MQTTBridge::refreshOriginFromPrefs() { if (!_prefs) return; applyEffectiveOrigin(_prefs, _origin, sizeof(_origin)); @@ -343,7 +388,9 @@ MQTTBridge::MQTTBridge(NodePrefs *prefs, mesh::PacketManager *mgr, mesh::RTCCloc : BridgeBase(prefs, mgr, rtc), _queue_count(0), _last_status_publish(0), _last_status_retry(0), _status_interval(300000), - _ntp_client(_ntp_udp, "pool.ntp.org", 0, 60000), _last_ntp_sync(0), _ntp_synced(false), _ntp_sync_pending(false), _slots_setup_done(false), _max_active_slots(RUNTIME_MQTT_SLOTS), + _ntp_client(_ntp_udp, effectiveNtpPrimary(prefs), 0, 60000), _last_ntp_sync(0), _ntp_synced(false), _ntp_sync_pending(false), _slots_setup_done(false), _max_active_slots(RUNTIME_MQTT_SLOTS), + _ntp_force_requested(false), _ntp_force_done(false), _ntp_force_result(false), + _ntp_diag_requested(false), _ntp_diag_done(false), _ntp_diag_count(0), // Default to UTC; setRules() will be called from syncTimeWithNTP when a // non-UTC timezone string is configured. Timezone has no default ctor, // so we must pass rules here. @@ -402,6 +449,16 @@ MQTTBridge::MQTTBridge(NodePrefs *prefs, mesh::PacketManager *mgr, mesh::RTCCloc _slot_reconfigure_pending[i] = false; } + // Reset CLI-requested forced NTP sync handshake (bridge object is reused across restarts) + _ntp_force_requested = false; + _ntp_force_done = false; + _ntp_force_result = false; + + // Reset CLI-requested NTP diagnostic handshake + _ntp_diag_requested = false; + _ntp_diag_done = false; + _ntp_diag_count = 0; + // Initialize JWT username _jwt_username[0] = '\0'; @@ -859,6 +916,25 @@ void MQTTBridge::mqttTaskLoop() { } } + // Process a CLI-requested forced NTP sync (queued from Core 1). Running it here + // keeps all NTP I/O on Core 0; requestForcedNtpSync() blocks the CLI thread until + // we publish the outcome below. + if (_ntp_force_requested) { + _ntp_force_requested = false; + // primary_only: validate just the server that was set, so a typo fails fast. + bool ok = syncTimeWithNTP(true, /*primary_only=*/true); + _ntp_force_result = ok; + _ntp_force_done = true; // set last so the waiter sees a consistent result + } + + // Process a CLI-requested NTP connectivity diagnostic (queued from Core 1). + // Probe-only — never touches the system clock. + if (_ntp_diag_requested) { + _ntp_diag_requested = false; + runNtpDiagProbe(); + _ntp_diag_done = true; // set last so the waiter sees populated results + } + // Deferred slot setup: wait until NTP is synced so JWT tokens get valid timestamps. // This avoids wasted TLS handshakes that get rejected due to bad token times. if (_ntp_synced && !_slots_setup_done) { @@ -2827,52 +2903,70 @@ void MQTTBridge::refreshNTP() { // Lightweight periodic refresh: just restart SNTP which runs async in the background. // No blocking DNS, no UDP sockets, no retry loops on the MQTT task loop. // The heavy syncTimeWithNTP() is only used for initial sync and WiFi reconnect recovery. - configTime(0, 0, "pool.ntp.org"); + configTime(0, 0, effectiveNtpPrimary(_prefs)); _last_ntp_sync = millis(); MQTT_DEBUG_PRINTLN("NTP refresh triggered (async SNTP)"); } -void MQTTBridge::syncTimeWithNTP() { +bool MQTTBridge::syncTimeWithNTP(bool force, bool primary_only) { if (!WiFi.isConnected()) { MQTT_DEBUG_PRINTLN("Cannot sync time - WiFi not connected"); - return; + return false; } unsigned long now = millis(); - if (_ntp_synced && (now - _last_ntp_sync) < 5000) { - return; + if (!force && _ntp_synced && (now - _last_ntp_sync) < 5000) { + return false; } static bool sync_in_progress = false; if (sync_in_progress) { - return; + return false; } sync_in_progress = true; MQTT_DEBUG_PRINTLN("Syncing time with NTP..."); - #ifdef ESP_PLATFORM - IPAddress resolved_ip; - if (!WiFi.hostByName("pool.ntp.org", resolved_ip)) { - MQTT_DEBUG_PRINTLN("WARNING: DNS resolution failed for pool.ntp.org - NTP sync may fail"); + const char* servers[kMaxNtpServers]; + int server_count = 0; + if (primary_only) { + // Validation path (e.g. set mqtt.ntp): test only the configured primary so a + // typo fails fast instead of walking the entire fallback list. + servers[0] = effectiveNtpPrimary(_prefs); + server_count = 1; + } else { + fillNtpServerList(_prefs, servers, server_count); } - #endif bool ntp_ok = false; unsigned long epochTime = 0; const unsigned long kMinValidEpoch = 1767225600; // 2026-01-01 00:00:00 UTC + const char* ntp_server_used = nullptr; _ntp_client.begin(); - const int kMaxNtpRetries = 3; - for (int attempt = 1; attempt <= kMaxNtpRetries && !ntp_ok; attempt++) { - if (attempt > 1) { - MQTT_DEBUG_PRINTLN("NTP retry %d/%d...", attempt, kMaxNtpRetries); - delay(1000); + const int kMaxNtpRetriesPerServer = 2; + for (int s = 0; s < server_count && !ntp_ok; s++) { + const char* server = servers[s]; + _ntp_client.setPoolServerName(server); + + #ifdef ESP_PLATFORM + IPAddress resolved_ip; + if (!WiFi.hostByName(server, resolved_ip)) { + MQTT_DEBUG_PRINTLN("WARNING: DNS resolution failed for %s - NTP sync may fail", server); } - if (_ntp_client.forceUpdate()) { - epochTime = _ntp_client.getEpochTime(); - if (epochTime >= kMinValidEpoch) { - ntp_ok = true; + #endif + + for (int attempt = 1; attempt <= kMaxNtpRetriesPerServer && !ntp_ok; attempt++) { + if (attempt > 1) { + MQTT_DEBUG_PRINTLN("NTP retry %d/%d on %s...", attempt, kMaxNtpRetriesPerServer, server); + delay(1000); + } + if (_ntp_client.forceUpdate()) { + epochTime = _ntp_client.getEpochTime(); + if (epochTime >= kMinValidEpoch) { + ntp_ok = true; + ntp_server_used = server; + } } } } @@ -2882,21 +2976,26 @@ void MQTTBridge::syncTimeWithNTP() { #ifdef ESP_PLATFORM if (!ntp_ok) { MQTT_DEBUG_PRINTLN("NTP client failed, trying SNTP fallback..."); - configTime(0, 0, "pool.ntp.org"); - for (int i = 0; i < 20; i++) { - delay(500); - epochTime = (unsigned long)time(nullptr); - if (epochTime >= kMinValidEpoch) { - ntp_ok = true; - MQTT_DEBUG_PRINTLN("SNTP fallback succeeded: %lu", epochTime); - break; + for (int s = 0; s < server_count && !ntp_ok; s++) { + const char* server = servers[s]; + MQTT_DEBUG_PRINTLN("SNTP fallback trying %s...", server); + configTime(0, 0, server); + for (int i = 0; i < 20; i++) { + delay(500); + epochTime = (unsigned long)time(nullptr); + if (epochTime >= kMinValidEpoch) { + ntp_ok = true; + ntp_server_used = server; + MQTT_DEBUG_PRINTLN("SNTP fallback succeeded on %s: %lu", server, epochTime); + break; + } } } } #endif - if (ntp_ok) { - configTime(0, 0, "pool.ntp.org"); + if (ntp_ok && ntp_server_used) { + configTime(0, 0, ntp_server_used); if (_rtc) { _rtc->setCurrentTime(epochTime); @@ -2907,7 +3006,7 @@ void MQTTBridge::syncTimeWithNTP() { _last_ntp_sync = millis(); sync_in_progress = false; - MQTT_DEBUG_PRINTLN("Time synced: %lu", epochTime); + MQTT_DEBUG_PRINTLN("Time synced: %lu (via %s)", epochTime, ntp_server_used); // If slots are already set up and the time jumped significantly (e.g., SNTP // initially returned stale RTC time, then a later sync corrected it), tear down @@ -2949,10 +3048,109 @@ void MQTTBridge::syncTimeWithNTP() { (void)gmtime((time_t*)&epochTime); (void)localtime((time_t*)&epochTime); - } else { - MQTT_DEBUG_PRINTLN("NTP sync failed"); - sync_in_progress = false; + return true; } + + MQTT_DEBUG_PRINTLN("NTP sync failed"); + sync_in_progress = false; + return false; +} + +bool MQTTBridge::requestForcedNtpSync(uint32_t timeout_ms) { + if (!isRunning()) return false; + + // Publish the request to the MQTT task. Clear the completion flags before + // raising _ntp_force_requested so the task can't observe a stale result. + _ntp_force_done = false; + _ntp_force_result = false; + _ntp_force_requested = true; + + unsigned long start = millis(); + while (!_ntp_force_done) { + if (millis() - start >= timeout_ms) { + MQTT_DEBUG_PRINTLN("Forced NTP sync timed out waiting for MQTT task"); + return false; // task still running; result is ignored when it eventually completes + } + vTaskDelay(pdMS_TO_TICKS(50)); + } + return _ntp_force_result; +} + +// Runs on the MQTT task (Core 0). Probes every configured NTP server for connectivity +// and records the time each reports. Deliberately does NOT call configTime() or update +// the RTC — this is a read-only diagnostic and must leave the system clock untouched. +void MQTTBridge::runNtpDiagProbe() { + const char* servers[kMaxNtpServers]; + int count = 0; + fillNtpServerList(_prefs, servers, count); + + _ntp_client.begin(); + for (int i = 0; i < count; i++) { + _ntp_client.setPoolServerName(servers[i]); + bool ok = _ntp_client.forceUpdate(); + NtpDiagResult& r = _ntp_diag_results[i]; + strncpy(r.server, servers[i], sizeof(r.server) - 1); + r.server[sizeof(r.server) - 1] = '\0'; + r.ok = ok; + r.epoch = ok ? (uint32_t)_ntp_client.getEpochTime() : 0; + } + _ntp_diag_count = count; +} + +bool MQTTBridge::ntpDiag(char* reply, size_t reply_size, bool verbose) { + if (!isRunning() || reply == nullptr || reply_size == 0) return false; + + // Marshal the probe onto the MQTT task (Core 0); clear the completion flag first. + _ntp_diag_done = false; + _ntp_diag_requested = true; + + unsigned long start = millis(); + while (!_ntp_diag_done) { + if (millis() - start >= 30000) { + snprintf(reply, reply_size, "Error: NTP diag timed out"); + return true; + } + vTaskDelay(pdMS_TO_TICKS(50)); + } + + int ok_count = 0; + for (int i = 0; i < _ntp_diag_count; i++) { + if (_ntp_diag_results[i].ok) ok_count++; + } + + if (verbose) { + // Detailed table to the serial console; reply carries a short summary (the operator + // sees the table above the "-> " line). Mirrors the dumpLogFile() convention. + Serial.printf("NTP diag - %d server(s):\r\n", _ntp_diag_count); + for (int i = 0; i < _ntp_diag_count; i++) { + const NtpDiagResult& r = _ntp_diag_results[i]; + if (r.ok) { + time_t t = (time_t)r.epoch; + struct tm* tmv = gmtime(&t); + Serial.printf(" %-20s OK %04d-%02d-%02d %02d:%02d:%02d UTC\r\n", + r.server, tmv->tm_year + 1900, tmv->tm_mon + 1, tmv->tm_mday, + tmv->tm_hour, tmv->tm_min, tmv->tm_sec); + } else { + Serial.printf(" %-20s FAIL\r\n", r.server); + } + } + snprintf(reply, reply_size, "> NTP diag: %d/%d OK (see console)", ok_count, _ntp_diag_count); + } else { + // Compact " ok|fail" list for LoRa, bounded to reply_size. + size_t used = 0; + reply[0] = '\0'; + for (int i = 0; i < _ntp_diag_count; i++) { + const NtpDiagResult& r = _ntp_diag_results[i]; + int n = snprintf(reply + used, reply_size - used, "%s%s %s", + used ? "\n" : "", r.server, r.ok ? "ok" : "fail"); + if (n < 0 || (size_t)n >= reply_size - used) { + reply[used] = '\0'; // out of room — truncate cleanly + break; + } + used += (size_t)n; + } + } + return true; } // --------------------------------------------------------------------------- diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index e19e0544a..b9df91ab8 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -56,6 +56,10 @@ class MeshSNMPAgent; // Forward declaration * - Available presets: analyzer-us, analyzer-eu, meshmapper, custom, none */ class MQTTBridge : public BridgeBase { +public: + // Max NTP servers in a try-list: 1 custom primary + the built-in fallbacks. + static const int kMaxNtpServers = 6; + private: static const size_t AUTH_TOKEN_SIZE = 768; @@ -190,6 +194,31 @@ private: // Pending slot reconfigure: set from CLI (Core 1), processed by MQTT task (Core 0) volatile bool _slot_reconfigure_pending[RUNTIME_MQTT_SLOTS]; + // CLI-requested forced NTP sync, marshalled onto the MQTT task (Core 0). + // All NTP I/O (_ntp_client, configTime) must run on Core 0; the CLI thread + // (Core 1) sets _ntp_force_requested and blocks in requestForcedNtpSync() + // until the task publishes the outcome via _ntp_force_result/_ntp_force_done. + // Single-requester assumption: CLI commands are serialized, so at most one + // forced sync is outstanding at a time. + volatile bool _ntp_force_requested; + volatile bool _ntp_force_done; + volatile bool _ntp_force_result; + + // CLI-requested NTP connectivity diagnostic, marshalled onto the MQTT task (Core 0) + // with the same handshake as the forced sync. Probe-only: it queries each server and + // records the reported time but never calls configTime()/setCurrentTime(), so the + // system clock is left untouched. Results are written by the task and read by the CLI + // thread once _ntp_diag_done is set. + volatile bool _ntp_diag_requested; + volatile bool _ntp_diag_done; + struct NtpDiagResult { + char server[64]; + bool ok; + uint32_t epoch; // server-reported UTC epoch when ok + }; + NtpDiagResult _ntp_diag_results[kMaxNtpServers]; + int _ntp_diag_count; + // Timezone handling. // _timezone_storage is inline class storage (zero heap) that is reconfigured // via setRules() whenever the preferred timezone string changes. _timezone @@ -331,8 +360,8 @@ private: void queuePacket(mesh::Packet* packet, bool is_tx); void dequeuePacket(); bool isAnySlotConnected(); - void syncTimeWithNTP(); void refreshNTP(); // Lightweight periodic NTP refresh (non-blocking) + void runNtpDiagProbe(); // Probe every server for connectivity; never sets the clock. Core 0 only. // Populates dst_out/std_out with TimeChangeRules for the given IANA or // abbreviation string. Returns false if the string is not recognized // (callers should fall back to UTC). Zero-allocation. @@ -411,6 +440,25 @@ public: static int getRuntimeSlotCount() { return RUNTIME_MQTT_SLOTS; } /** Resolved origin for MQTT JSON: node_name when mqtt_origin is empty, else mqtt_origin (with quote stripping). */ static void getEffectiveMqttOrigin(const NodePrefs* prefs, char* buf, size_t buf_size); + static const char* effectiveNtpPrimary(const NodePrefs* prefs); + /** Sync system clock via NTP. force=true bypasses the 5s post-sync rate limit. + * primary_only=true tests just the effective primary server (no fallback walk) so a + * mistyped hostname fails fast instead of blocking through the whole fallback list. + * Performs blocking NTP I/O and must only be called from the MQTT task (Core 0). + * Other tasks (e.g. the CLI on Core 1) must use requestForcedNtpSync() instead. */ + bool syncTimeWithNTP(bool force = false, bool primary_only = false); + /** Request a forced NTP sync from another task (e.g. CLI on Core 1). Marshals the + * work onto the MQTT task so all NTP I/O stays on Core 0, then blocks up to + * timeout_ms for the result. Returns true if the sync succeeded, false on failure, + * timeout, or if the bridge is not running. */ + bool requestForcedNtpSync(uint32_t timeout_ms = 30000); + /** Probe every configured NTP server (custom primary + built-in fallbacks) for + * connectivity and report each server's reported time WITHOUT touching the system + * clock. Marshals the probe onto the MQTT task (Core 0), then formats on the caller's + * thread. verbose=true prints a detailed table to the serial console and leaves a short + * summary in reply; verbose=false fills reply with a compact " ok|fail" list + * (for LoRa). Returns false if the bridge is not running. */ + bool ntpDiag(char* reply, size_t reply_size, bool verbose); static void formatMqttStatusReply(char* buf, size_t bufsize, const NodePrefs* prefs); /** True when WiFi is set and at least one MQTT slot can run (preset + custom host if needed). */ static bool isConfigValid(const NodePrefs* prefs); From b458ac42e4f9ed7817b597269d981819ff708b90 Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 24 Jun 2026 20:33:00 -0700 Subject: [PATCH 15/16] perf(mqtt): remove dead code and optimize hex conversion Remove unused MQTTMessageBuilder members (getPacketTypeString, formatTimestamp/Time/Date stubs, JSON_BUFFER_SIZE constant) for a small flash saving with no behavior change. Replace the per-byte snprintf("%02X") in bytesToHex with a nibble lookup table, avoiding a format-string parse up to ~512x per publish on the MQTT task. Output is byte-for-byte identical uppercase hex. --- src/helpers/MQTTMessageBuilder.cpp | 45 ++++-------------------------- src/helpers/MQTTMessageBuilder.h | 23 --------------- 2 files changed, 6 insertions(+), 62 deletions(-) diff --git a/src/helpers/MQTTMessageBuilder.cpp b/src/helpers/MQTTMessageBuilder.cpp index d768cc40a..a29b3d96c 100644 --- a/src/helpers/MQTTMessageBuilder.cpp +++ b/src/helpers/MQTTMessageBuilder.cpp @@ -401,28 +401,6 @@ int MQTTMessageBuilder::buildRawJSON( return buildRawMessage(origin, origin_id, timestamp, raw_hex, buffer, buffer_size); } -const char* MQTTMessageBuilder::getPacketTypeString(int packet_type) { - switch (packet_type) { - case 0: return "0"; // REQ - case 1: return "1"; // RESPONSE - case 2: return "2"; // TXT_MSG - case 3: return "3"; // ACK - case 4: return "4"; // ADVERT - case 5: return "5"; // GRP_TXT - case 6: return "6"; // GRP_DATA - case 7: return "7"; // ANON_REQ - case 8: return "8"; // PATH - case 9: return "9"; // TRACE - case 10: return "10"; // MULTIPART - case 11: return "11"; // Type11 - case 12: return "12"; // Type12 - case 13: return "13"; // Type13 - case 14: return "14"; // Type14 - case 15: return "15"; // RAW_CUSTOM - default: return "0"; - } -} - const char* MQTTMessageBuilder::getRouteTypeString(int route_type) { switch (route_type) { case 0: return "F"; // FLOOD @@ -432,26 +410,15 @@ const char* MQTTMessageBuilder::getRouteTypeString(int route_type) { } } -void MQTTMessageBuilder::formatTimestamp(unsigned long timestamp, char* buffer, size_t buffer_size) { - // Simplified timestamp formatting - in real implementation would use proper time - snprintf(buffer, buffer_size, "2024-01-01T12:00:00.000000"); -} - -void MQTTMessageBuilder::formatTime(unsigned long timestamp, char* buffer, size_t buffer_size) { - // Simplified time formatting - snprintf(buffer, buffer_size, "12:00:00"); -} - -void MQTTMessageBuilder::formatDate(unsigned long timestamp, char* buffer, size_t buffer_size) { - // Simplified date formatting - snprintf(buffer, buffer_size, "01/01/2024"); -} - void MQTTMessageBuilder::bytesToHex(const uint8_t* data, size_t len, char* hex, size_t hex_size) { if (hex_size < len * 2 + 1) return; - + + // Nibble lookup instead of a per-byte snprintf("%02X"): same uppercase hex + // output, but avoids re-parsing the format string up to ~512 times per publish. + static const char HEX_DIGITS[] = "0123456789ABCDEF"; for (size_t i = 0; i < len; i++) { - snprintf(hex + i * 2, 3, "%02X", data[i]); + hex[i * 2] = HEX_DIGITS[data[i] >> 4]; + hex[i * 2 + 1] = HEX_DIGITS[data[i] & 0x0F]; } hex[len * 2] = '\0'; } diff --git a/src/helpers/MQTTMessageBuilder.h b/src/helpers/MQTTMessageBuilder.h index b89117440..8e3a6d45f 100644 --- a/src/helpers/MQTTMessageBuilder.h +++ b/src/helpers/MQTTMessageBuilder.h @@ -19,9 +19,6 @@ * regex fields. */ class MQTTMessageBuilder { -private: - static const int JSON_BUFFER_SIZE = 1024; - public: /** * Format the MQTT JSON `timestamp` field (same rule for status, packet, raw). @@ -214,31 +211,11 @@ public: ); private: - /** - * Convert packet type to string - */ - static const char* getPacketTypeString(int packet_type); - /** * Convert route type to string */ static const char* getRouteTypeString(int route_type); - /** - * Format timestamp to ISO 8601 format - */ - static void formatTimestamp(unsigned long timestamp, char* buffer, size_t buffer_size); - - /** - * Format time to HH:MM:SS format - */ - static void formatTime(unsigned long timestamp, char* buffer, size_t buffer_size); - - /** - * Format date to DD/MM/YYYY format - */ - static void formatDate(unsigned long timestamp, char* buffer, size_t buffer_size); - /** * Convert bytes to hex string (uppercase) */ From c39ab639b080750674c728bf64756203769f9f0f Mon Sep 17 00:00:00 2001 From: agessaman Date: Wed, 24 Jun 2026 21:20:56 -0700 Subject: [PATCH 16/16] feat(mqtt): add ipnt.uk preset to MQTT presets --- MQTT_IMPLEMENTATION.md | 1 + src/helpers/MQTTPresets.h | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/MQTT_IMPLEMENTATION.md b/MQTT_IMPLEMENTATION.md index 6695ed193..e4f534ba4 100644 --- a/MQTT_IMPLEMENTATION.md +++ b/MQTT_IMPLEMENTATION.md @@ -116,6 +116,7 @@ The MQTT bridge uses a slot-based architecture with up to 6 concurrent connectio | `meshcore-ca-1` | mqtt1.meshcore.ca:443 | JWT (Ed25519) | WSS | | `meshcore-ca-2` | mqtt2.meshcore.ca:443 | JWT (Ed25519) | WSS | | `bostonmesh` | mqttmc01.bostonme.sh:443 | JWT (Ed25519) | WSS | +| `ipnt.uk` | mqtt.ipnt.uk:443 | JWT (Ed25519) | WSS | | `flmesh` | mcmqtt.jntconnections.com:443 | JWT (Ed25519) | WSS | | `inwmesh` | scope.inwmesh.org:8883 | Username/password (per slot via `mqttN.username` / `mqttN.password`) | MQTT over TLS | | `custom` | User-configured | Username/Password | MQTT or WSS | diff --git a/src/helpers/MQTTPresets.h b/src/helpers/MQTTPresets.h index d6726003a..d4c620cd5 100644 --- a/src/helpers/MQTTPresets.h +++ b/src/helpers/MQTTPresets.h @@ -105,7 +105,7 @@ static const char ISRG_ROOT_X1[] PROGMEM = "-----END CERTIFICATE-----\n"; // Number of built-in presets -static const int MQTT_PRESET_COUNT = 23; +static const int MQTT_PRESET_COUNT = 24; // Built-in preset definitions (stored in flash) static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { @@ -132,6 +132,7 @@ static const MQTTPresetDef MQTT_PRESETS[MQTT_PRESET_COUNT] = { { "meshcore-fi", "wss://mc-mqtt.meshcore.fi:443/", "mc-mqtt.meshcore.fi", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "inwmesh", "mqtts://scope.inwmesh.org:8883", nullptr, ISRG_ROOT_X1, MQTT_AUTH_USERPASS, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "bostonmesh", "wss://mqttmc01.bostonme.sh:443/mqtt", "mqttmc01.bostonme.sh", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, + { "ipnt.uk", "wss://mqtt.ipnt.uk:443", "mqtt.ipnt.uk", ISRG_ROOT_X1, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, { "flmesh", "wss://mcmqtt.jntconnections.com:443", "mcmqtt.jntconnections.com", GTS_ROOT_R4, MQTT_AUTH_JWT, MQTT_TOPIC_MESHCORE, 0, true, 55, nullptr, nullptr }, };