feat(ota): implement pull-based OTA from manifest for observer builds

Added functionality to support pull-based OTA updates by fetching firmware
from a manifest. The new `otaFromManifest` method allows the system to
check for available updates and flash the firmware if necessary. This
enhancement improves the update process for observer builds using the
MQTT bridge, ensuring a more seamless firmware management experience.
This commit is contained in:
agessaman committed 2026-06-21 11:57:05 -07:00
1 parent 8b084d5ac3
commit 004d308ea9
17 files changed
+251 -2

No files matched your search

+4 -2
View File
@@ -151,8 +151,10 @@ build_firmware() {
esac
EMBEDDED_VERSION_STRING="${FIRMWARE_VERSION}${VARIANT_TAG}-${COMMIT_HASH}"
# add firmware version info to end of existing platformio build flags in environment vars
export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DFIRMWARE_BUILD_DATE='\"${FIRMWARE_BUILD_DATE}\"' -DFIRMWARE_VERSION='\"${EMBEDDED_VERSION_STRING}\"'"
# add firmware version info to end of existing platformio build flags in environment vars.
# OTA_VARIANT is the env name ($1) — it is exactly the asset-filename prefix used above, so the
# observer pull-OTA can match its own build in the web-flasher manifest (config.json).
export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DFIRMWARE_BUILD_DATE='\"${FIRMWARE_BUILD_DATE}\"' -DFIRMWARE_VERSION='\"${EMBEDDED_VERSION_STRING}\"' -DOTA_VARIANT='\"$1\"'"
# disable debug flags if requested
disable_debug_flags
+4
View File
@@ -64,6 +64,10 @@ public:
virtual uint8_t getStartupReason() const = 0;
virtual bool getBootloaderVersion(char* version, size_t max_len) { return false; }
virtual bool startOTAUpdate(const char* id, char reply[]) { return false; } // not supported
// Pull-based OTA: fetch the firmware build for this variant from a baked-in manifest and flash it.
// current_ver is the running firmware version string (used to skip if already up to date); when
// dry_run is true the build is only reported, not flashed. Observer (ESP32+WiFi) builds only.
virtual bool otaFromManifest(const char* current_ver, bool dry_run, char reply[]) { return false; }
// Power management interface (boards with power management override these)
virtual bool isExternalPowered() { return false; }
+20
View File
@@ -793,9 +793,29 @@ void CommonCLI::handleCommand(uint32_t sender_timestamp, char* command, char* re
strcpy(reply, "ERR: unsupported on this platform");
#endif
} else if (memcmp(command, "start ota", 9) == 0) {
#if defined(WITH_MQTT_BRIDGE) && defined(OTA_MANIFEST_URL)
// Observer pull-OTA: fetch this variant's build from the baked-in manifest.
// "start ota check" reports the available build without flashing.
const char* arg = command + 9;
while (*arg == ' ') arg++;
bool dry = (memcmp(arg, "check", 5) == 0);
if (WiFi.status() != WL_CONNECTED) {
strcpy(reply, "ERR: WiFi not connected");
} else {
// Free the MQTT bridge (TLS contexts + task) so the manifest parse and
// OTA download have heap headroom; the WiFi STA link survives end().
if (!dry) _callbacks->setBridgeState(false);
bool ok = _board->otaFromManifest(_callbacks->getFirmwareVer(), dry, reply);
// On success the board reboots and never returns here; on any abort
// (already up to date, partition change, download error) bring the
// bridge back up so the node resumes uplinking.
if (!ok && !dry) _callbacks->setBridgeState(true);
}
#else
if (!_board->startOTAUpdate(_prefs->node_name, reply)) {
strcpy(reply, "Error");
}
#endif
} else if (memcmp(command, "clock", 5) == 0) {
uint32_t now = getRTCClock()->getCurrentTime();
DateTime dt = DateTime(now);
+194
View File
@@ -44,4 +44,198 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[]) {
}
#endif
// ---------------------------------------------------------------------------
// Manifest-driven pull OTA (observer / MQTT-bridge builds only)
//
// The observer already holds a live WiFi station connection (for the MQTT
// bridge) and embeds a root-CA bundle, so it can fetch its own firmware. The
// caller (CommonCLI) stops the MQTT bridge first to free heap/TLS, then calls
// this. We read the web-flasher manifest (config.json), find the `flash-update`
// (app-only) build for our own variant, refuse partition-change releases (OTA
// can't rewrite the partition table), skip if already up to date, then stream
// the .bin straight into the inactive OTA slot via HTTPUpdate.
// ---------------------------------------------------------------------------
#if defined(WITH_MQTT_BRIDGE)
#include <WiFi.h>
#include <WiFiClientSecure.h>
#include <HTTPClient.h>
#include <HTTPUpdate.h>
#include <ArduinoJson.h>
// Embedded CA bundle (produced by board_build.embed_files). Weak so non-bundle
// builds still link; we check for presence at runtime.
extern const uint8_t rootca_crt_bundle_start[] asm("_binary_src_certs_x509_crt_bundle_bin_start") __attribute__((weak));
extern const uint8_t rootca_crt_bundle_end[] asm("_binary_src_certs_x509_crt_bundle_bin_end") __attribute__((weak));
// Extract the trailing build hash. For a filename we first drop a ".bin"
// suffix, then take the token after the last '-'. Works for both the manifest
// asset name ("...-v1.16.0-8b084d5.bin" -> "8b084d5") and the embedded
// FIRMWARE_VERSION ("v1.16.0-observer-8b084d5" -> "8b084d5").
static void ota_extractHash(const char* s, char* out, size_t out_sz) {
if (!s) { if (out_sz) out[0] = 0; return; }
size_t len = strlen(s);
if (len > 4 && strcmp(s + len - 4, ".bin") == 0) len -= 4;
size_t i = len;
while (i > 0 && s[i - 1] != '-') i--;
size_t n = len - i;
if (n >= out_sz) n = out_sz - 1;
memcpy(out, s + i, n);
out[n] = 0;
}
bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char reply[]) {
#if !defined(OTA_MANIFEST_URL) || !defined(OTA_VARIANT)
strcpy(reply, "ERR: OTA not configured (build via build.sh)");
return false;
#else
if (WiFi.status() != WL_CONNECTED) {
strcpy(reply, "ERR: WiFi not connected");
return false;
}
size_t bundle_len = 0;
if (rootca_crt_bundle_start != nullptr && rootca_crt_bundle_end != nullptr &&
rootca_crt_bundle_end > rootca_crt_bundle_start) {
bundle_len = (size_t)(rootca_crt_bundle_end - rootca_crt_bundle_start);
}
if (bundle_len == 0) {
strcpy(reply, "ERR: no embedded cert bundle");
return false;
}
// --- Fetch + filter-parse the manifest -----------------------------------
WiFiClientSecure mclient;
#if ESP_ARDUINO_VERSION_MAJOR >= 3
mclient.setCACertBundle(rootca_crt_bundle_start, bundle_len);
#else
mclient.setCACertBundle(rootca_crt_bundle_start);
#endif
mclient.setTimeout(15000);
HTTPClient http;
if (!http.begin(mclient, OTA_MANIFEST_URL)) {
strcpy(reply, "ERR: manifest connect failed");
return false;
}
int code = http.GET();
if (code != HTTP_CODE_OK) {
snprintf(reply, 160, "ERR: manifest HTTP %d", code);
http.end();
return false;
}
// Filter keeps only staticPath + each firmware entry's notice/version, so the
// parsed document stays a fraction of the full manifest. (The dynamic version
// key forces keeping its whole subtree, incl. release notes.) ArduinoJson v7
// JsonDocument allocates elastically, so this only grows to the kept subset.
JsonDocument filter;
filter["staticPath"] = true;
filter["device"][0]["firmware"][0]["notice"] = true;
filter["device"][0]["firmware"][0]["version"] = true;
JsonDocument doc;
DeserializationError err =
deserializeJson(doc, http.getStream(), DeserializationOption::Filter(filter));
http.end();
if (err) {
snprintf(reply, 160, "ERR: manifest parse (%s)", err.c_str());
return false;
}
// Copy out of the document up front: doc gets cleared before these are used.
char base_url[128] = {0};
strncpy(base_url, doc["staticPath"] | "", sizeof(base_url) - 1);
if (!base_url[0]) {
strcpy(reply, "ERR: manifest missing staticPath");
return false;
}
// --- Locate the flash-update build for our variant -----------------------
const char* variant = OTA_VARIANT;
size_t vlen = strlen(variant);
char target_name[128] = {0};
bool partition_change = false;
bool found = false;
for (JsonObject dev : doc["device"].as<JsonArray>()) {
for (JsonObject fw : dev["firmware"].as<JsonArray>()) {
const char* notice = fw["notice"].is<const char*>() ? fw["notice"].as<const char*>() : nullptr;
for (JsonPair vp : fw["version"].as<JsonObject>()) {
for (JsonObject file : vp.value()["files"].as<JsonArray>()) {
const char* type = file["type"] | "";
const char* name = file["name"] | "";
if (strcmp(type, "flash-update") != 0) continue;
if (strncmp(name, variant, vlen) != 0 || name[vlen] != '-') continue;
strncpy(target_name, name, sizeof(target_name) - 1);
partition_change = (notice != nullptr && strcmp(notice, "partition-change") == 0);
found = true;
break;
}
if (found) break;
}
if (found) break;
}
if (found) break;
}
doc.clear();
if (!found) {
snprintf(reply, 160, "ERR: no build for %s in manifest", variant);
return false;
}
char avail_hash[24], cur_hash[24];
ota_extractHash(target_name, avail_hash, sizeof(avail_hash));
ota_extractHash(current_ver, cur_hash, sizeof(cur_hash));
// Compare by shared prefix: git abbreviates the same commit to 7 chars on a
// shallow CI clone but 8 locally, so an exact match would miss equal builds.
size_t la = strlen(avail_hash), lc = strlen(cur_hash);
size_t m = (la < lc) ? la : lc;
bool up_to_date = (m >= 7 && strncmp(avail_hash, cur_hash, m) == 0);
if (dry_run) {
snprintf(reply, 160, "%s: %s -> %s%s", up_to_date ? "up to date" : "update available",
cur_hash, avail_hash, partition_change ? " [partition change: cable flash]" : "");
return true;
}
if (partition_change) {
snprintf(reply, 160, "ERR: %s needs cable flash (partition change)", avail_hash);
return false;
}
if (up_to_date) {
snprintf(reply, 160, "OK: already up to date (%s)", cur_hash);
return false;
}
// --- Stream the .bin into the inactive OTA slot --------------------------
char url[256];
snprintf(url, sizeof(url), "%s/%s", base_url, target_name);
inhibit_sleep = true; // keep awake through the flash
WiFiClientSecure uclient;
#if ESP_ARDUINO_VERSION_MAJOR >= 3
uclient.setCACertBundle(rootca_crt_bundle_start, bundle_len);
#else
uclient.setCACertBundle(rootca_crt_bundle_start);
#endif
uclient.setTimeout(20000);
httpUpdate.rebootOnUpdate(true); // reboots into the new image on success
t_httpUpdate_return ret = httpUpdate.update(uclient, url);
// Only reached on failure (success reboots inside update()).
inhibit_sleep = false;
snprintf(reply, 160, "ERR: OTA failed (%d): %s", (int)ret,
httpUpdate.getLastErrorString().c_str());
return false;
#endif // OTA_MANIFEST_URL && OTA_VARIANT
}
#else
bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char reply[]) {
strcpy(reply, "ERR: not supported");
return false;
}
#endif // WITH_MQTT_BRIDGE
#endif
+1
View File
@@ -151,6 +151,7 @@ public:
}
bool startOTAUpdate(const char* id, char reply[]) override;
bool otaFromManifest(const char* current_ver, bool dry_run, char reply[]) override;
void setInhibitSleep(bool inhibit) {
inhibit_sleep = inhibit;
+2
View File
@@ -112,6 +112,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D MAX_NEIGHBOURS=50
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -204,6 +205,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
+4
View File
@@ -123,6 +123,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D MAX_NEIGHBOURS=50
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -192,6 +193,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -480,6 +482,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D MAX_NEIGHBOURS=50
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -528,6 +531,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
+4
View File
@@ -162,6 +162,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D MAX_NEIGHBOURS=50
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -213,6 +214,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D MAX_NEIGHBOURS=50
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -315,6 +317,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -354,6 +357,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
+2
View File
@@ -116,6 +116,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D MAX_NEIGHBOURS=50
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
; -D MQTT_DEBUG=1
@@ -161,6 +162,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
+2
View File
@@ -209,6 +209,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D MAX_NEIGHBOURS=50
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D PERSISTANT_GPS=1
@@ -255,6 +256,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D PERSISTANT_GPS=1
@@ -161,6 +161,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D MAX_NEIGHBOURS=50
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
; -D MQTT_DEBUG=1
@@ -203,6 +204,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -160,6 +160,7 @@ build_flags =
-D MAX_NEIGHBOURS=50
-D PERSISTANT_GPS=1
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
; -D MQTT_DEBUG=1
@@ -202,6 +203,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -120,6 +120,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D MAX_NEIGHBOURS=50
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -162,6 +163,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -157,6 +157,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D MAX_NEIGHBOURS=50
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_TASK_STACK_SIZE=16384
@@ -202,6 +203,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_TASK_STACK_SIZE=16384
+2
View File
@@ -113,6 +113,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D MAX_NEIGHBOURS=50
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -176,6 +177,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
+2
View File
@@ -187,6 +187,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -305,6 +306,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D MAX_NEIGHBOURS=50
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
+2
View File
@@ -103,6 +103,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D MAX_NEIGHBOURS=50
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1
@@ -146,6 +147,7 @@ build_flags =
-D ADMIN_PASSWORD='"password"'
-D ROOM_PASSWORD='"hello"'
-D WITH_MQTT_BRIDGE=1
-D OTA_MANIFEST_URL='"https://observer.gessaman.com/config.json"'
-D MAX_MQTT_BROKERS=3
-D MQTT_MAX_PACKET_SIZE=1024
-D MQTT_DEBUG=1