From 33f1b1096dd614a8f082187f396a7662b1c68ae9 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 26 Sep 2026 10:03:48 -0700 Subject: [PATCH 01/23] feat(observer): add dev-preview observer envs for 11 more ESP32-S3 boards Repeater + room-server observer builds for Heltec RC32 (with and without display), Wireless Tracker v1.0, Wireless Paper, Vision Master E213/E290, Meshnology W12, LilyGo T-ETH Elite (WiFi only), T3S3 SX1276, Xiao S3 and Ebyte EoRa-S3. The 4 MB boards (T3S3 SX1276, EoRa-S3) use min_spiffs and omit SNMP; non-PSRAM boards opt into MQTT_NEIGHBORS_WITHOUT_PSRAM. --- variants/ebyte_eora_s3/platformio.ini | 77 +++++++++ variants/heltec_e213/platformio.ini | 85 ++++++++++ variants/heltec_e290/platformio.ini | 85 ++++++++++ variants/heltec_rc32/platformio.ini | 158 ++++++++++++++++++ variants/heltec_tracker/platformio.ini | 89 ++++++++++ variants/heltec_wireless_paper/platformio.ini | 89 ++++++++++ variants/lilygo_t3s3_sx1276/platformio.ini | 77 +++++++++ variants/lilygo_teth_elite/platformio.ini | 77 +++++++++ variants/meshnology_w12/platformio.ini | 83 +++++++++ variants/xiao_s3/platformio.ini | 77 +++++++++ 10 files changed, 897 insertions(+) diff --git a/variants/ebyte_eora_s3/platformio.ini b/variants/ebyte_eora_s3/platformio.ini index 1ab3d3fb..258d7193 100644 --- a/variants/ebyte_eora_s3/platformio.ini +++ b/variants/ebyte_eora_s3/platformio.ini @@ -141,3 +141,80 @@ lib_deps = extends = Ebyte_EoRa-S3 build_src_filter = ${Ebyte_EoRa-S3.build_src_filter} +<../examples/kiss_modem/> + +[env:Ebyte_EoRa-S3_repeater_observer_mqtt] +extends = Ebyte_EoRa-S3 +board_build.partitions = min_spiffs.csv +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Ebyte_EoRa-S3.build_flags} + -D DISPLAY_CLASS=SSD1306Display + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 +build_src_filter = ${Ebyte_EoRa-S3.build_src_filter} + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${Ebyte_EoRa-S3.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + +[env:Ebyte_EoRa-S3_room_server_observer_mqtt] +extends = Ebyte_EoRa-S3 +board_build.partitions = min_spiffs.csv +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Ebyte_EoRa-S3.build_flags} + -D DISPLAY_CLASS=SSD1306Display + -D ADVERT_NAME='"EoRa-S3 Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 +build_src_filter = ${Ebyte_EoRa-S3.build_src_filter} + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${Ebyte_EoRa-S3.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 diff --git a/variants/heltec_e213/platformio.ini b/variants/heltec_e213/platformio.ini index 02193819..7f77cfe9 100644 --- a/variants/heltec_e213/platformio.ini +++ b/variants/heltec_e213/platformio.ini @@ -172,3 +172,88 @@ lib_deps = extends = Heltec_E213_base build_src_filter = ${Heltec_E213_base.build_src_filter} +<../examples/kiss_modem/> + +[env:Heltec_E213_repeater_observer_mqtt] +extends = Heltec_E213_base +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_E213_base.build_flags} + -D DISPLAY_CLASS=E213Display + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${Heltec_E213_base.build_src_filter} + + + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${Heltec_E213_base.lib_deps} + ${esp32_ota.lib_deps} + bakercp/CRC32 @ ^2.0.0 + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + +[env:Heltec_E213_room_server_observer_mqtt] +extends = Heltec_E213_base +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_E213_base.build_flags} + -D DISPLAY_CLASS=E213Display + -D ADVERT_NAME='"Heltec E213 Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${Heltec_E213_base.build_src_filter} + + + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${Heltec_E213_base.lib_deps} + ${esp32_ota.lib_deps} + bakercp/CRC32 @ ^2.0.0 + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent diff --git a/variants/heltec_e290/platformio.ini b/variants/heltec_e290/platformio.ini index c7cd5f21..1a310e34 100644 --- a/variants/heltec_e290/platformio.ini +++ b/variants/heltec_e290/platformio.ini @@ -166,3 +166,88 @@ lib_deps = extends = Heltec_E290_base build_src_filter = ${Heltec_E290_base.build_src_filter} +<../examples/kiss_modem/> + +[env:Heltec_E290_repeater_observer_mqtt] +extends = Heltec_E290_base +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_E290_base.build_flags} + -D DISPLAY_CLASS=E290Display + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${Heltec_E290_base.build_src_filter} + + + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${Heltec_E290_base.lib_deps} + ${esp32_ota.lib_deps} + bakercp/CRC32 @ ^2.0.0 + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + +[env:Heltec_E290_room_server_observer_mqtt] +extends = Heltec_E290_base +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_E290_base.build_flags} + -D DISPLAY_CLASS=E290Display + -D ADVERT_NAME='"Heltec E290 Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${Heltec_E290_base.build_src_filter} + + + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${Heltec_E290_base.lib_deps} + ${esp32_ota.lib_deps} + bakercp/CRC32 @ ^2.0.0 + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent diff --git a/variants/heltec_rc32/platformio.ini b/variants/heltec_rc32/platformio.ini index 2f07c021..0823abd5 100644 --- a/variants/heltec_rc32/platformio.ini +++ b/variants/heltec_rc32/platformio.ini @@ -355,3 +355,161 @@ lib_deps = extends = Heltec_RC32 build_src_filter = ${Heltec_RC32.build_src_filter} +<../examples/kiss_modem/> + +[env:heltec_rc32_repeater_observer_mqtt] +extends = Heltec_RC32_with_display +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_RC32_with_display.build_flags} + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${Heltec_RC32_with_display.build_src_filter} + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${Heltec_RC32_with_display.lib_deps} + ${esp32_ota.lib_deps} + bakercp/CRC32 @ ^2.0.0 + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + +[env:heltec_rc32_room_server_observer_mqtt] +extends = Heltec_RC32_with_display +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_RC32_with_display.build_flags} + -D ADVERT_NAME='"Heltec RC32 Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${Heltec_RC32_with_display.build_src_filter} + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${Heltec_RC32_with_display.lib_deps} + ${esp32_ota.lib_deps} + bakercp/CRC32 @ ^2.0.0 + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + +[env:heltec_rc32_without_display_repeater_observer_mqtt] +extends = Heltec_RC32 +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_RC32.build_flags} + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${Heltec_RC32.build_src_filter} + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${Heltec_RC32.lib_deps} + ${esp32_ota.lib_deps} + bakercp/CRC32 @ ^2.0.0 + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + +[env:heltec_rc32_without_display_room_server_observer_mqtt] +extends = Heltec_RC32 +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_RC32.build_flags} + -D ADVERT_NAME='"Heltec RC32 Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${Heltec_RC32.build_src_filter} + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${Heltec_RC32.lib_deps} + ${esp32_ota.lib_deps} + bakercp/CRC32 @ ^2.0.0 + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent diff --git a/variants/heltec_tracker/platformio.ini b/variants/heltec_tracker/platformio.ini index 75d1cadf..8c807c6e 100644 --- a/variants/heltec_tracker/platformio.ini +++ b/variants/heltec_tracker/platformio.ini @@ -192,3 +192,92 @@ lib_deps = extends = Heltec_tracker_base build_src_filter = ${Heltec_tracker_base.build_src_filter} +<../examples/kiss_modem/> + +[env:Heltec_Wireless_Tracker_repeater_observer_mqtt] +extends = Heltec_tracker_base +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_tracker_base.build_flags} + -D DISPLAY_ROTATION=1 + -D DISPLAY_CLASS=ST7735Display + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 +; Non-PSRAM board: neighbors table costs ~4 KB of internal DRAM (see MQTTBridge.h). + -D MQTT_NEIGHBORS_WITHOUT_PSRAM=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=240 + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${Heltec_tracker_base.build_src_filter} + + + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${Heltec_tracker_base.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + +[env:Heltec_Wireless_Tracker_room_server_observer_mqtt] +extends = Heltec_tracker_base +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_tracker_base.build_flags} + -D DISPLAY_ROTATION=1 + -D DISPLAY_CLASS=ST7735Display + -D ADVERT_NAME='"Heltec Tracker Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 +; Non-PSRAM board: neighbors table costs ~4 KB of internal DRAM (see MQTTBridge.h). + -D MQTT_NEIGHBORS_WITHOUT_PSRAM=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=240 + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${Heltec_tracker_base.build_src_filter} + + + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${Heltec_tracker_base.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent diff --git a/variants/heltec_wireless_paper/platformio.ini b/variants/heltec_wireless_paper/platformio.ini index 55ed62b3..35b94fcb 100644 --- a/variants/heltec_wireless_paper/platformio.ini +++ b/variants/heltec_wireless_paper/platformio.ini @@ -171,3 +171,92 @@ lib_deps = extends = Heltec_Wireless_Paper_base build_src_filter = ${Heltec_Wireless_Paper_base.build_src_filter} +<../examples/kiss_modem/> + +[env:Heltec_Wireless_Paper_repeater_observer_mqtt] +extends = Heltec_Wireless_Paper_base +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_Wireless_Paper_base.build_flags} + -D DISPLAY_CLASS=E213Display + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 +; Non-PSRAM board: neighbors table costs ~4 KB of internal DRAM (see MQTTBridge.h). + -D MQTT_NEIGHBORS_WITHOUT_PSRAM=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${Heltec_Wireless_Paper_base.build_src_filter} + + + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${Heltec_Wireless_Paper_base.lib_deps} + ${esp32_ota.lib_deps} + bakercp/CRC32 @ ^2.0.0 + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + +[env:Heltec_Wireless_Paper_room_server_observer_mqtt] +extends = Heltec_Wireless_Paper_base +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Heltec_Wireless_Paper_base.build_flags} + -D DISPLAY_CLASS=E213Display + -D ADVERT_NAME='"Heltec WP Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 +; Non-PSRAM board: neighbors table costs ~4 KB of internal DRAM (see MQTTBridge.h). + -D MQTT_NEIGHBORS_WITHOUT_PSRAM=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D MQTT_WIFI_TX_POWER=WIFI_POWER_11dBm + -D WITH_SNMP=1 +build_src_filter = ${Heltec_Wireless_Paper_base.build_src_filter} + + + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${Heltec_Wireless_Paper_base.lib_deps} + ${esp32_ota.lib_deps} + bakercp/CRC32 @ ^2.0.0 + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent diff --git a/variants/lilygo_t3s3_sx1276/platformio.ini b/variants/lilygo_t3s3_sx1276/platformio.ini index c6497dcb..5e5513f7 100644 --- a/variants/lilygo_t3s3_sx1276/platformio.ini +++ b/variants/lilygo_t3s3_sx1276/platformio.ini @@ -175,3 +175,80 @@ lib_deps = extends = LilyGo_T3S3_sx1276 build_src_filter = ${LilyGo_T3S3_sx1276.build_src_filter} +<../examples/kiss_modem/> + +[env:LilyGo_T3S3_sx1276_repeater_observer_mqtt] +extends = LilyGo_T3S3_sx1276 +board_build.partitions = min_spiffs.csv +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${LilyGo_T3S3_sx1276.build_flags} + -D DISPLAY_CLASS=SSD1306Display + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 +build_src_filter = ${LilyGo_T3S3_sx1276.build_src_filter} + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${LilyGo_T3S3_sx1276.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + +[env:LilyGo_T3S3_sx1276_room_server_observer_mqtt] +extends = LilyGo_T3S3_sx1276 +board_build.partitions = min_spiffs.csv +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${LilyGo_T3S3_sx1276.build_flags} + -D DISPLAY_CLASS=SSD1306Display + -D ADVERT_NAME='"T3S3-1276 Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 +build_src_filter = ${LilyGo_T3S3_sx1276.build_src_filter} + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${LilyGo_T3S3_sx1276.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 diff --git a/variants/lilygo_teth_elite/platformio.ini b/variants/lilygo_teth_elite/platformio.ini index 2debb039..a428b9ae 100644 --- a/variants/lilygo_teth_elite/platformio.ini +++ b/variants/lilygo_teth_elite/platformio.ini @@ -103,3 +103,80 @@ lib_deps = extends = LilyGo_TETH_Elite_sx1262 build_src_filter = ${LilyGo_TETH_Elite_sx1262.build_src_filter} +<../examples/kiss_modem/> + +[env:LilyGo_TETH_Elite_sx1262_repeater_observer_mqtt] +extends = LilyGo_TETH_Elite_sx1262 +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${LilyGo_TETH_Elite_sx1262.build_flags} + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D WITH_SNMP=1 +build_src_filter = ${LilyGo_TETH_Elite_sx1262.build_src_filter} + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${LilyGo_TETH_Elite_sx1262.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + +[env:LilyGo_TETH_Elite_sx1262_room_server_observer_mqtt] +extends = LilyGo_TETH_Elite_sx1262 +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${LilyGo_TETH_Elite_sx1262.build_flags} + -D ADVERT_NAME='"T-ETH Elite Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D WITH_SNMP=1 +build_src_filter = ${LilyGo_TETH_Elite_sx1262.build_src_filter} + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${LilyGo_TETH_Elite_sx1262.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent diff --git a/variants/meshnology_w12/platformio.ini b/variants/meshnology_w12/platformio.ini index ffab9d57..f319863f 100644 --- a/variants/meshnology_w12/platformio.ini +++ b/variants/meshnology_w12/platformio.ini @@ -216,3 +216,86 @@ lib_deps = extends = meshnology_w12 build_src_filter = ${meshnology_w12.build_src_filter} +<../examples/kiss_modem/> + +[env:meshnology_w12_repeater_observer_mqtt] +extends = meshnology_w12 +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${meshnology_w12.build_flags} + -D DISPLAY_CLASS=SSD1306Display + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D WITH_SNMP=1 +build_src_filter = ${meshnology_w12.build_src_filter} + + + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${meshnology_w12.lib_deps} + ${esp32_ota.lib_deps} + bakercp/CRC32 @ ^2.0.0 + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + +[env:meshnology_w12_room_server_observer_mqtt] +extends = meshnology_w12 +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${meshnology_w12.build_flags} + -D DISPLAY_CLASS=SSD1306Display + -D ADVERT_NAME='"Meshnology W12 Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D WITH_SNMP=1 +build_src_filter = ${meshnology_w12.build_src_filter} + + + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${meshnology_w12.lib_deps} + ${esp32_ota.lib_deps} + bakercp/CRC32 @ ^2.0.0 + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent diff --git a/variants/xiao_s3/platformio.ini b/variants/xiao_s3/platformio.ini index 1632d112..7546d7bb 100644 --- a/variants/xiao_s3/platformio.ini +++ b/variants/xiao_s3/platformio.ini @@ -158,3 +158,80 @@ lib_deps = extends = Xiao_S3 build_src_filter = ${Xiao_S3.build_src_filter} +<../examples/kiss_modem/> + +[env:Xiao_S3_repeater_observer_mqtt] +extends = Xiao_S3 +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Xiao_S3.build_flags} + -D ADVERT_NAME='"MQTT Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D WITH_SNMP=1 +build_src_filter = ${Xiao_S3.build_src_filter} + + + + + + + + + +<../examples/simple_repeater> +lib_deps = + ${Xiao_S3.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent + +[env:Xiao_S3_room_server_observer_mqtt] +extends = Xiao_S3 +extra_scripts = + ${esp32_base.extra_scripts} + pre:scripts/generate_cert_bundle.py +board_ssl_cert_source = adafruit-full +board_build.embed_files = src/certs/x509_crt_bundle.bin +build_flags = + ${Xiao_S3.build_flags} + -D ADVERT_NAME='"Xiao S3 Room Observer"' + -D ADVERT_LAT=0.0 + -D ADVERT_LON=0.0 + -D ADMIN_PASSWORD='"password"' + -D ROOM_PASSWORD='"hello"' + -D MAX_NEIGHBOURS=50 + -D WITH_MQTT_BRIDGE=1 + -D MQTT_MAX_PACKET_SIZE=1024 + -D MQTT_DEBUG=1 +; -D MESH_PACKET_LOGGING=1 +; -D MESH_DEBUG=1 + -D CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y + -D ESP32_CPU_FREQ=160 + -D WITH_SNMP=1 +build_src_filter = ${Xiao_S3.build_src_filter} + + + + + + + + + +<../examples/simple_room_server> +lib_deps = + ${Xiao_S3.lib_deps} + ${esp32_ota.lib_deps} + elims/PsychicMqttClient@^0.2.4 + bblanchon/ArduinoJson @ 7.4.3 + arduino-libraries/NTPClient + JChristensen/Timezone + paulstoffregen/Time@1.6.1 + 0neblock/SNMP_Agent From a91009e3ddfd3ec5cbd1897dbf1d3e96aee7c184 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 26 Sep 2026 10:46:11 -0700 Subject: [PATCH 02/23] fix(mqtt): omit SNR/RSSI instead of publishing placeholder values The reconstruction-less fallback path in buildPacketJSON (queued packet with no captured raw frame that cannot be re-serialized) published fabricated SNR 12.5 / RSSI -65 that consumers could not distinguish from real radio readings. It now passes NAN / MQTT_RSSI_UNKNOWN, and buildPacketMessage omits the SNR/RSSI keys entirely when no measurement exists, mirroring the existing score omission. Ports the fix from ACETyr's PR #13 (0a037040) on mqtt-bridge-implementation. Co-Authored-By: ACETyr <35745290+ACETyr@users.noreply.github.com> --- src/helpers/MQTTMessageBuilder.cpp | 4 +- src/helpers/MQTTPayloadBuilder.cpp | 14 +++- src/helpers/MQTTPayloadBuilder.h | 5 ++ .../test_mqtt_payload_builder.cpp | 76 +++++++++++++++++++ 4 files changed, 93 insertions(+), 6 deletions(-) diff --git a/src/helpers/MQTTMessageBuilder.cpp b/src/helpers/MQTTMessageBuilder.cpp index 062f52ed..afb53ef0 100644 --- a/src/helpers/MQTTMessageBuilder.cpp +++ b/src/helpers/MQTTMessageBuilder.cpp @@ -209,8 +209,8 @@ int MQTTMessageBuilder::buildPacketJSON( packet_type, route_str, packet->payload_len, raw_hex, - 12.5f, // SNR - using reasonable default - -65, // RSSI - using reasonable default + NAN, // SNR - unknown on this reconstruction-less fallback path + MQTT_RSSI_UNKNOWN, // RSSI - unknown on this reconstruction-less fallback path NAN, // score - unknown on this reconstruction-less fallback path hash_str, has_path ? packet->path : nullptr, diff --git a/src/helpers/MQTTPayloadBuilder.cpp b/src/helpers/MQTTPayloadBuilder.cpp index 265969ae..47a7b2e8 100644 --- a/src/helpers/MQTTPayloadBuilder.cpp +++ b/src/helpers/MQTTPayloadBuilder.cpp @@ -120,8 +120,6 @@ int MQTTPayloadBuilder::buildPacketMessage( snprintf(len_str, sizeof(len_str), "%d", len); snprintf(packet_type_str, sizeof(packet_type_str), "%d", packet_type); snprintf(payload_len_str, sizeof(payload_len_str), "%d", payload_len); - snprintf(snr_str, sizeof(snr_str), "%.1f", snr); - snprintf(rssi_str, sizeof(rssi_str), "%d", rssi); root["timestamp"] = timestamp; root["hash"] = hash; @@ -138,8 +136,16 @@ int MQTTPayloadBuilder::buildPacketMessage( root["origin_id"] = origin_id; if (direction && strcmp(direction, "rx") == 0) { - root["SNR"] = snr_str; - root["RSSI"] = rssi_str; + // Omit SNR/RSSI entirely when no measurement exists rather than publish + // a placeholder a consumer could mistake for a real reading. + if (!isnan(snr)) { + snprintf(snr_str, sizeof(snr_str), "%.1f", snr); + root["SNR"] = snr_str; + } + if (rssi != MQTT_RSSI_UNKNOWN) { + snprintf(rssi_str, sizeof(rssi_str), "%d", rssi); + root["RSSI"] = rssi_str; + } if (!isnan(score)) { snprintf(score_str, sizeof(score_str), "%d", static_cast(score * 1000)); root["score"] = score_str; diff --git a/src/helpers/MQTTPayloadBuilder.h b/src/helpers/MQTTPayloadBuilder.h index 7bca170d..ee2a7cc9 100644 --- a/src/helpers/MQTTPayloadBuilder.h +++ b/src/helpers/MQTTPayloadBuilder.h @@ -7,6 +7,11 @@ // Mesh-independent JSON serialization core for MQTT publication payloads. // MQTTMessageBuilder keeps the firmware-facing API and delegates these three // deterministic contracts here so they can be exercised by native tests. + +// RSSI sentinel for "no radio measurement": far outside the small negative ints +// a radio actually reports, so a real reading can never collide with it. +static const int MQTT_RSSI_UNKNOWN = INT16_MIN; + class MQTTPayloadBuilder { public: static int buildStatusMessage( diff --git a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp index 37486fd2..181f0a86 100644 --- a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp +++ b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp @@ -36,6 +36,16 @@ static int buildRepresentativePacket(JsonDocument& scratch, const char* directio path_hash_size, 64, buffer, buffer_size); } +static int buildPacketWithMetrics(JsonDocument& scratch, const char* direction, + float snr, int rssi, float score, + char* buffer, size_t buffer_size) { + return MQTTPayloadBuilder::buildPacketMessage( + scratch, "DEN Repeater", "0123456789ABCDEF", kTimestamp, direction, + "12:34:56", "18/07/2026", 42, 4, "D", 20, "A0B1C2D3", + snr, rssi, score, "89ABCDEF01234567", nullptr, 0, 0, 64, + buffer, buffer_size); +} + TEST(MQTTPayloadBuilder, MinimalStatusHasExactRequiredContract) { JsonDocument scratch; char buffer[768]; @@ -169,6 +179,72 @@ TEST(MQTTPayloadBuilder, RxPacketOmitsUnknownNanScore) { EXPECT_FALSE(parsed["score"].is()); } +TEST(MQTTPayloadBuilder, RxPacketOmitsUnknownSnrAndRssi) { + JsonDocument scratch; + char buffer[2048]; + int len = buildPacketWithMetrics( + scratch, "rx", NAN, MQTT_RSSI_UNKNOWN, NAN, buffer, sizeof(buffer)); + + ASSERT_GT(len, 0); + JsonDocument parsed; + ASSERT_FALSE(deserializeJson(parsed, buffer)); + EXPECT_FALSE(parsed["SNR"].is()); + EXPECT_FALSE(parsed["RSSI"].is()); + EXPECT_FALSE(parsed["score"].is()); +} + +TEST(MQTTPayloadBuilder, RxPacketFormatsRealSnrAndRssi) { + JsonDocument scratch; + char buffer[2048]; + int len = buildPacketWithMetrics( + scratch, "rx", 7.25f, -93, NAN, buffer, sizeof(buffer)); + + ASSERT_GT(len, 0); + JsonDocument parsed; + ASSERT_FALSE(deserializeJson(parsed, buffer)); + EXPECT_STREQ("7.2", parsed["SNR"].as()); + EXPECT_STREQ("-93", parsed["RSSI"].as()); + EXPECT_FALSE(parsed["score"].is()); +} + +TEST(MQTTPayloadBuilder, RxPacketOmitsOnlyUnknownMetric) { + // SNR unknown, RSSI real: only the SNR key is omitted. + JsonDocument scratch; + char buffer[2048]; + int len = buildPacketWithMetrics( + scratch, "rx", NAN, -93, NAN, buffer, sizeof(buffer)); + + ASSERT_GT(len, 0); + JsonDocument parsed; + ASSERT_FALSE(deserializeJson(parsed, buffer)); + EXPECT_FALSE(parsed["SNR"].is()); + EXPECT_STREQ("-93", parsed["RSSI"].as()); + + // RSSI unknown, SNR real: only the RSSI key is omitted. + len = buildPacketWithMetrics( + scratch, "rx", 7.25f, MQTT_RSSI_UNKNOWN, NAN, buffer, sizeof(buffer)); + + ASSERT_GT(len, 0); + parsed.clear(); + ASSERT_FALSE(deserializeJson(parsed, buffer)); + EXPECT_STREQ("7.2", parsed["SNR"].as()); + EXPECT_FALSE(parsed["RSSI"].is()); +} + +TEST(MQTTPayloadBuilder, TxPacketOmitsSnrRssiAndScoreWithRealValues) { + JsonDocument scratch; + char buffer[2048]; + int len = buildPacketWithMetrics( + scratch, "tx", 7.25f, -93, 0.5f, buffer, sizeof(buffer)); + + ASSERT_GT(len, 0); + JsonDocument parsed; + ASSERT_FALSE(deserializeJson(parsed, buffer)); + EXPECT_FALSE(parsed["SNR"].is()); + EXPECT_FALSE(parsed["RSSI"].is()); + EXPECT_FALSE(parsed["score"].is()); +} + TEST(MQTTPayloadBuilder, RawMessageHasExactContractAndEscapesData) { char buffer[512]; JsonDocument doc; From f7ed2f176d604a2326b4847136a0b94209bc6dd4 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 8 Aug 2026 19:12:10 -0700 Subject: [PATCH 03/23] feat(mqtt): report slot connection health so an outage is visible A slot that cannot establish TLS reported nothing. The per-slot counters track publish successes and failures, and a slot that never connects never attempts a publish -- so its ok count freezes and its error count stays at zero for the entire outage. Observed on hardware: a slot sat dead for over five hours while `sN=` showed 0 errors and the published status payload carried no indication at all. Two surfaces gain the missing signal: - `get mqtt.stats` gains `down=` (omitted when zero, like `filt=`) and marks each disconnected slot with a trailing `!`. The marker goes after the counts so existing "s(\d)=(\d+)/(\d+)" parsers keep matching. - The published status payload gains `mqtt_slots_up`, `mqtt_slots_total`, `heap_largest`, and `mqtt_outage_secs`. These ride on every healthy slot's status message, because the slot that is down cannot report itself. `mqtt_slots_up < mqtt_slots_total` is the condition to alert on. `mqtt_outage_secs` is detail: it is present only when the outage has a known start, so its absence does not imply health -- a slot that has never connected since boot has no outage start time. `heap_largest` is included because free heap alone does not predict a failed connection. A slot needs one contiguous 16,384-byte block for its TLS record buffers; measured immediately before an attempt this value predicted the outcome in every observed case, while total free heap was 85 KB throughout a multi-hour outage. A low value with every slot up is normal rather than a fault -- established sessions already hold their buffers. Publish-error semantics are unchanged: connect failures are not folded into them. Adds host coverage for the new fields, for the outage key's presence rule, and for worst-case payload size. That last one matters because serializeComplete() returns 0 and clears the buffer on overflow and the bridge publishes only a positive length, so exceeding the 768-byte status buffer would silently stop status publication -- the same blindness this change exists to remove. --- src/helpers/MQTTMessageBuilder.cpp | 5 +- src/helpers/MQTTMessageBuilder.h | 4 +- src/helpers/MQTTPayloadBuilder.cpp | 18 ++++- src/helpers/MQTTPayloadBuilder.h | 23 +++++- src/helpers/bridges/MQTTBridge.cpp | 60 +++++++++++++- src/helpers/bridges/MQTTBridge.h | 5 ++ .../test_mqtt_payload_builder.cpp | 79 +++++++++++++++++++ 7 files changed, 184 insertions(+), 10 deletions(-) diff --git a/src/helpers/MQTTMessageBuilder.cpp b/src/helpers/MQTTMessageBuilder.cpp index afb53ef0..fcce321e 100644 --- a/src/helpers/MQTTMessageBuilder.cpp +++ b/src/helpers/MQTTMessageBuilder.cpp @@ -57,13 +57,14 @@ int MQTTMessageBuilder::buildStatusMessage( int internal_heap, int packets_sent, int packets_received, - const char* repeat + const char* repeat, + const MQTTConnHealth& conn_health ) { return MQTTPayloadBuilder::buildStatusMessage( doc, origin, origin_id, model, firmware_version, radio, client_version, status, timestamp, buffer, buffer_size, battery_mv, uptime_secs, errors, queue_len, noise_floor, tx_air_secs, rx_air_secs, recv_errors, internal_heap, - packets_sent, packets_received, repeat); + packets_sent, packets_received, repeat, conn_health); } int MQTTMessageBuilder::buildPacketMessage( diff --git a/src/helpers/MQTTMessageBuilder.h b/src/helpers/MQTTMessageBuilder.h index 91a47b6e..b3d82032 100644 --- a/src/helpers/MQTTMessageBuilder.h +++ b/src/helpers/MQTTMessageBuilder.h @@ -68,6 +68,7 @@ public: * @param recv_errors Radio receive/CRC errors (optional, -1 to omit) * @param internal_heap Internal heap free bytes (optional, -1 to omit) * @param repeat Repeat/forwarding status ("on" or "off"); nullptr omits the field + * @param conn_health MQTT slot connection health; defaults omit every field * @return Length of JSON string, or 0 on error */ static int buildStatusMessage( @@ -93,7 +94,8 @@ public: int internal_heap = -1, int packets_sent = -1, int packets_received = -1, - const char* repeat = nullptr + const char* repeat = nullptr, + const MQTTConnHealth& conn_health = MQTTConnHealth() ); /** diff --git a/src/helpers/MQTTPayloadBuilder.cpp b/src/helpers/MQTTPayloadBuilder.cpp index 47a7b2e8..536bb5e7 100644 --- a/src/helpers/MQTTPayloadBuilder.cpp +++ b/src/helpers/MQTTPayloadBuilder.cpp @@ -45,7 +45,8 @@ int MQTTPayloadBuilder::buildStatusMessage( int internal_heap, int packets_sent, int packets_received, - const char* repeat + const char* repeat, + const MQTTConnHealth& conn_health ) { doc.clear(); JsonObject root = doc.to(); @@ -62,9 +63,12 @@ int MQTTPayloadBuilder::buildStatusMessage( root["repeat"] = repeat; } + const bool has_conn_health = conn_health.slots_up >= 0 || conn_health.slots_total >= 0 || + conn_health.worst_outage_secs >= 0 || conn_health.heap_largest >= 0; + if (battery_mv >= 0 || uptime_secs >= 0 || errors >= 0 || queue_len >= 0 || noise_floor > -999 || tx_air_secs >= 0 || rx_air_secs >= 0 || recv_errors >= 0 || - internal_heap >= 0 || packets_sent >= 0 || packets_received >= 0) { + internal_heap >= 0 || packets_sent >= 0 || packets_received >= 0 || has_conn_health) { JsonObject stats = root["stats"].to(); if (battery_mv >= 0) stats["battery_mv"] = battery_mv; @@ -78,6 +82,16 @@ int MQTTPayloadBuilder::buildStatusMessage( if (rx_air_secs >= 0) stats["rx_air_secs"] = rx_air_secs; if (recv_errors >= 0) stats["recv_errors"] = recv_errors; if (internal_heap >= 0) stats["internal_heap"] = internal_heap; + if (conn_health.heap_largest >= 0) stats["heap_largest"] = conn_health.heap_largest; + if (conn_health.slots_up >= 0) stats["mqtt_slots_up"] = conn_health.slots_up; + if (conn_health.slots_total >= 0) stats["mqtt_slots_total"] = conn_health.slots_total; + // Emitted only during an outage of known duration, so the steady-state payload + // does not grow. Presence implies a slot is down, but absence does NOT imply + // health: a slot that has never connected since boot has no outage start time. + // `mqtt_slots_up < mqtt_slots_total` is the signal to alert on; this is detail. + if (conn_health.worst_outage_secs >= 0) { + stats["mqtt_outage_secs"] = conn_health.worst_outage_secs; + } } return serializeComplete(root, buffer, buffer_size); diff --git a/src/helpers/MQTTPayloadBuilder.h b/src/helpers/MQTTPayloadBuilder.h index ee2a7cc9..b675f397 100644 --- a/src/helpers/MQTTPayloadBuilder.h +++ b/src/helpers/MQTTPayloadBuilder.h @@ -4,6 +4,26 @@ #include #include +// Connection health of the MQTT slots themselves, reported inside a status +// publication. A slot that cannot reach its broker cannot report its own outage, +// so this travels on every *healthy* slot's status message and is how a fleet +// learns that a sibling slot is down. Publish-success/error counters cannot show +// this: a slot that never connects never attempts a publish, so its error count +// stays at zero for the entire outage. +// +// `heap_largest` is the leading indicator. Observed on non-PSRAM hardware: a slot +// needs one contiguous 16,384-byte block for its TLS record buffers, and measured +// immediately before an attempt this value predicted the outcome every time. Free +// heap alone does not — an 85 KB-free board sat unable to connect for hours. +// A low value with all slots up is normal, not a fault: established sessions +// already hold their buffers. It means only that the next slot to ask will fail. +struct MQTTConnHealth { + int slots_up = -1; // -1 = not supplied, field omitted + int slots_total = -1; // enabled slots, whether or not they connect + int worst_outage_secs = -1; // longest current outage; -1 = nothing is down + int heap_largest = -1; // largest free internal block, bytes +}; + // Mesh-independent JSON serialization core for MQTT publication payloads. // MQTTMessageBuilder keeps the firmware-facing API and delegates these three // deterministic contracts here so they can be exercised by native tests. @@ -37,7 +57,8 @@ public: int internal_heap = -1, int packets_sent = -1, int packets_received = -1, - const char* repeat = nullptr + const char* repeat = nullptr, + const MQTTConnHealth& conn_health = MQTTConnHealth() ); static int buildPacketMessage( diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 29edc89e..bde48149 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -373,12 +373,26 @@ void MQTTBridge::formatMqttStatsReply(char* buf, size_t bufsize) { if (b->_filtered_packets > 0) { replyAppendf(buf, bufsize, &pos, " filt=%lu", b->_filtered_packets); } + // down=: enabled slots that are not connected. Without this the reply cannot + // show an outage at all — sN= counts publishes, and a slot that never connects + // never publishes, so its ok freezes and its err stays 0 for the whole outage. + // Omitted while zero, like filt=, so a healthy node's reply keeps its length. + int down = 0; + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + if (b->_slots[i].enabled && !b->_slots[i].connected) down++; + } + if (down > 0) { + replyAppendf(buf, bufsize, &pos, " down=%d", down); + } replyAppendf(buf, bufsize, &pos, " |"); for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { if (!b->_slots[i].enabled || !b->_slots[i].client) continue; - replyAppendf(buf, bufsize, &pos, " s%d=%lu/%lu", i + 1, + // Trailing '!' marks a disconnected slot. Appended after the counts so the + // established "s(\d)=(\d+)/(\d+)" parsers keep matching unchanged. + replyAppendf(buf, bufsize, &pos, " s%d=%lu/%lu%s", i + 1, b->_slots[i].client->getPublishOk(), - b->_slots[i].client->getPublishErr()); + b->_slots[i].client->getPublishErr(), + b->_slots[i].connected ? "" : "!"); } } @@ -3135,7 +3149,8 @@ void MQTTBridge::publishStatusToSlot(int index) { battery_mv, uptime_secs, errors, _queue_count, noise_floor, tx_air_secs, rx_air_secs, recv_errors, internal_heap_free, packets_sent, packets_received, - _prefs->disable_fwd ? "off" : "on" + _prefs->disable_fwd ? "off" : "on", + collectConnHealth() ); if (len > 0) { @@ -3151,6 +3166,42 @@ void MQTTBridge::publishStatusToSlot(int index) { } } +MQTTConnHealth MQTTBridge::collectConnHealth() const { + MQTTConnHealth health; + health.slots_up = 0; + health.slots_total = 0; + + const unsigned long now = millis(); + unsigned long worst_outage_ms = 0; + bool any_outage_timed = false; + + for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { + const MQTTSlot& slot = _slots[i]; + if (!slot.enabled) continue; // "none" slots are not an outage, they are unconfigured + health.slots_total++; + if (slot.connected) { + health.slots_up++; + continue; + } + // A slot that has never connected since boot has no outage start time. It still + // counts against slots_up, which is why that pair — not this duration — is the + // signal to alert on. + if (slot.current_outage_started_ms != 0) { + const unsigned long outage_ms = now - slot.current_outage_started_ms; // wrap-safe + if (!any_outage_timed || outage_ms > worst_outage_ms) { + worst_outage_ms = outage_ms; + any_outage_timed = true; + } + } + } + + if (any_outage_timed) { + health.worst_outage_secs = static_cast(worst_outage_ms / 1000UL); + } + health.heap_largest = static_cast(heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL)); + return health; +} + void MQTTBridge::updateCachedConnectionStatus() { bool any_connected = false; for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { @@ -3993,7 +4044,8 @@ bool MQTTBridge::publishStatus() { battery_mv, uptime_secs, errors, _queue_count, noise_floor, tx_air_secs, rx_air_secs, recv_errors, internal_heap_free, packets_sent, packets_received, - _prefs->disable_fwd ? "off" : "on" + _prefs->disable_fwd ? "off" : "on", + collectConnHealth() ); if (len > 0) { diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 1126da0b..ca7e061a 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -9,6 +9,7 @@ #include #include "helpers/JWTHelper.h" #include "helpers/MQTTPacketFilter.h" +#include "helpers/MQTTPayloadBuilder.h" // MQTTConnHealth #include "helpers/MQTTPresets.h" #include "helpers/MQTTPresetPolicy.h" #include "helpers/MQTTLifecycle.h" @@ -542,6 +543,10 @@ private: bool publishToSlot(int index, const char* topic, const char* payload, size_t payload_len, bool retained = false, uint8_t qos = 0); bool publishToAllSlots(const char* topic, const char* payload, size_t payload_len, bool retained = false, uint8_t qos = 0); void publishStatusToSlot(int index); + // Snapshot of slot connection health for the status payload. A slot that cannot + // reach its broker cannot report its own outage, so every healthy slot carries + // this on its status publication. + MQTTConnHealth collectConnHealth() const; void updateCachedConnectionStatus(); void processPacketQueue(); diff --git a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp index 181f0a86..e682f8b4 100644 --- a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp +++ b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp @@ -93,6 +93,85 @@ TEST(MQTTPayloadBuilder, StatusIncludesRepeatAndEveryRequestedStatistic) { EXPECT_EQ(47, stats["packets_received"].as()); } +TEST(MQTTPayloadBuilder, StatusReportsConnectionHealthWhenSupplied) { + JsonDocument scratch; + char buffer[1024]; + MQTTConnHealth health; + health.slots_up = 2; + health.slots_total = 3; + health.worst_outage_secs = 18432; + health.heap_largest = 16116; + + int len = MQTTPayloadBuilder::buildStatusMessage( + scratch, "node", "id", "model", "firmware", "radio", "client", "online", + kTimestamp, buffer, sizeof(buffer), -1, -1, -1, -1, -999, + -1, -1, -1, -1, -1, -1, nullptr, health); + + ASSERT_GT(len, 0); + JsonDocument parsed; + ASSERT_FALSE(deserializeJson(parsed, buffer)); + JsonObject stats = parsed["stats"].as(); + ASSERT_FALSE(stats.isNull()); + EXPECT_EQ(2, stats["mqtt_slots_up"].as()); + EXPECT_EQ(3, stats["mqtt_slots_total"].as()); + EXPECT_EQ(18432, stats["mqtt_outage_secs"].as()); + EXPECT_EQ(16116, stats["heap_largest"].as()); +} + +// The outage key doubles as the "something is down" flag, so its absence has to be +// meaningful: a fully healthy board must not emit it. +TEST(MQTTPayloadBuilder, StatusOmitsOutageKeyWhenEverySlotIsUp) { + JsonDocument scratch; + char buffer[768]; + MQTTConnHealth health; + health.slots_up = 3; + health.slots_total = 3; + health.heap_largest = 54260; // worst_outage_secs left at -1 + + int len = MQTTPayloadBuilder::buildStatusMessage( + scratch, "node", "id", "model", "firmware", "radio", "client", "online", + kTimestamp, buffer, sizeof(buffer), -1, -1, -1, -1, -999, + -1, -1, -1, -1, -1, -1, nullptr, health); + + ASSERT_GT(len, 0); + JsonDocument parsed; + ASSERT_FALSE(deserializeJson(parsed, buffer)); + JsonObject stats = parsed["stats"].as(); + ASSERT_FALSE(stats.isNull()); + EXPECT_EQ(3, stats["mqtt_slots_up"].as()); + EXPECT_FALSE(stats["mqtt_outage_secs"].is()); +} + +// serializeComplete() returns 0 and clears the buffer when the payload does not fit, +// and MQTTBridge publishes only a positive length — so overflowing STATUS_JSON_BUFFER_SIZE +// silently stops status publication altogether. That is the same class of blindness this +// field set exists to fix, so the worst realistic payload must fit with margin. +TEST(MQTTPayloadBuilder, WorstCaseStatusWithConnectionHealthFitsProductionBuffer) { + constexpr size_t kStatusJsonBufferSize = 768; // MQTTBridge::STATUS_JSON_BUFFER_SIZE + JsonDocument scratch; + char buffer[kStatusJsonBufferSize]; + MQTTConnHealth health; + health.slots_up = 0; + health.slots_total = 5; // PSRAM slot cap, the widest configuration + health.worst_outage_secs = 999999; + health.heap_largest = 4194304; + + int len = MQTTPayloadBuilder::buildStatusMessage( + scratch, + "A Repeater With A Deliberately Long Node Name", + "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF", // 64-char device id + "Heltec Wireless Tracker V1.1", "v1.17.0-observer-dev.12", + "915.000000,62.5,7,5", "MeshCore 1.17.0", "online", kTimestamp, + buffer, sizeof(buffer), + 4200, 8640000, 65535, 6, -128, 999999, 999999, 65535, 4194304, 999999, 999999, + "off", health); + + ASSERT_GT(len, 0) << "worst-case status payload overflowed the production buffer; " + "status publication would stop silently"; + EXPECT_LT(static_cast(len), kStatusJsonBufferSize - 32) + << "less than 32 bytes of headroom left in the status buffer"; +} + TEST(MQTTPayloadBuilder, StatusOmissionSentinelsRemainOmitted) { JsonDocument scratch; char buffer[768]; From 383339ecccf611bd83458b189dfd4ff20d5d8fc1 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 26 Sep 2026 11:09:21 -0700 Subject: [PATCH 04/23] fix(mqtt): size the status buffer for real field maxima The worst-case test added with the connection-health fields did not model production maxima -- it used a 28-byte model and a 24-byte firmware string, while MQTTBridge permits _origin[32], _device_id[65], _board_model[64], _firmware_version[64] and derives a 63-byte client version. isValidName() rejects [ ] / \ : , ? * but permits '"', so a 31-character node name of quotes is legal and every quote escapes to two bytes. Measured at those real maxima: without connection-health fields 762 bytes with connection-health fields 851 bytes buffer 768 bytes So the 768-byte buffer already had only six bytes of headroom before this work, and the four new fields overflow it. Overflow is silent: serializeComplete() returns 0 and clears the buffer, and the bridge publishes only a positive length, so a device would simply stop publishing status with no error anywhere -- the same blindness the connection-health fields exist to remove. Raises STATUS_JSON_BUFFER_SIZE to 1024 and corrects the test to use the actual maxima. Status serializes into the shared 2048-byte publish scratch buffer, so this costs no extra RAM; only the PSRAM-allocation-failed stack fallback grows by 256 bytes. A static_assert keeps the status ceiling within the shared buffer. Found by adversarial review of the preceding commit. --- src/helpers/bridges/MQTTBridge.h | 12 ++++-- .../test_mqtt_payload_builder.cpp | 37 +++++++++++++------ 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index ca7e061a..b755ce2f 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -366,9 +366,15 @@ private: // internal heap. On non-PSRAM: inline in the class object so the allocation doesn't // interleave with large TLS buffers at startup. static const size_t PUBLISH_JSON_BUFFER_SIZE = 2048; - // Status keeps its own smaller ceiling: raising it would change which oversized - // status documents get published instead of dropped. - static const size_t STATUS_JSON_BUFFER_SIZE = 768; + // Status keeps its own smaller ceiling: raising it changes which oversized status + // documents get published instead of dropped. 768 left only 6 bytes spare at the real + // field maxima (_origin[32] of quotes, which isValidName permits and JSON doubles, plus + // _device_id[65], _board_model[64], _firmware_version[64] and a 63-byte client version): + // measured 762/768 before the connection-health fields existed. Overflow is silent — + // serializeComplete() returns 0 and the bridge publishes only len>0, so status stops + // entirely. Raised to 1024, which costs no extra RAM here because status serializes into + // the shared scratch buffer. See the worst-case test in test_mqtt_payload_builder. + static const size_t STATUS_JSON_BUFFER_SIZE = 1024; static_assert(STATUS_JSON_BUFFER_SIZE <= PUBLISH_JSON_BUFFER_SIZE, "status payloads serialize into the shared publish buffer"); #if defined(BOARD_HAS_PSRAM) diff --git a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp index e682f8b4..b0c2d438 100644 --- a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp +++ b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp @@ -146,30 +146,43 @@ TEST(MQTTPayloadBuilder, StatusOmitsOutageKeyWhenEverySlotIsUp) { // and MQTTBridge publishes only a positive length — so overflowing STATUS_JSON_BUFFER_SIZE // silently stops status publication altogether. That is the same class of blindness this // field set exists to fix, so the worst realistic payload must fit with margin. +// Field widths are the real MQTTBridge maxima, not plausible-looking strings: +// _origin[32] _device_id[65] _firmware_version[64] _board_model[64] +// and client_version is snprintf("meshcore/%s", _firmware_version) into char[64]. +// isValidName() rejects [ ] / \ : , ? * but ALLOWS '"', so a node name of 31 quotes +// is legal and every one escapes to two bytes — that is the true worst case for the +// only free-text field an operator controls. TEST(MQTTPayloadBuilder, WorstCaseStatusWithConnectionHealthFitsProductionBuffer) { - constexpr size_t kStatusJsonBufferSize = 768; // MQTTBridge::STATUS_JSON_BUFFER_SIZE + constexpr size_t kStatusJsonBufferSize = 1024; // MQTTBridge::STATUS_JSON_BUFFER_SIZE + const std::string origin(31, '"'); // _origin[32], worst-case escaping + const std::string origin_id(64, 'A'); // _device_id[65] + const std::string model(63, 'M'); // _board_model[64] + const std::string firmware(63, 'F'); // _firmware_version[64] + const std::string client_version(63, 'C'); // char client_version[64] + const std::string radio("9999.999999,1000.0,12,8"); + JsonDocument scratch; char buffer[kStatusJsonBufferSize]; MQTTConnHealth health; health.slots_up = 0; health.slots_total = 5; // PSRAM slot cap, the widest configuration - health.worst_outage_secs = 999999; - health.heap_largest = 4194304; + health.worst_outage_secs = 4294967; // millis() rollover / 1000, the widest value + health.heap_largest = 8388608; int len = MQTTPayloadBuilder::buildStatusMessage( - scratch, - "A Repeater With A Deliberately Long Node Name", - "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF", // 64-char device id - "Heltec Wireless Tracker V1.1", "v1.17.0-observer-dev.12", - "915.000000,62.5,7,5", "MeshCore 1.17.0", "online", kTimestamp, + scratch, origin.c_str(), origin_id.c_str(), model.c_str(), firmware.c_str(), + radio.c_str(), client_version.c_str(), "online", kTimestamp, buffer, sizeof(buffer), - 4200, 8640000, 65535, 6, -128, 999999, 999999, 65535, 4194304, 999999, 999999, + 4200, 2147483, 2147483647, 6, -128, 2147483, 2147483, 2147483647, + 8388608, 2147483647, 2147483647, "off", health); - ASSERT_GT(len, 0) << "worst-case status payload overflowed the production buffer; " - "status publication would stop silently"; + ASSERT_GT(len, 0) << "worst-case status payload overflowed the " << kStatusJsonBufferSize + << "-byte production buffer; serializeComplete() returns 0 and the " + "bridge publishes only len>0, so ALL status publication would " + "stop silently"; EXPECT_LT(static_cast(len), kStatusJsonBufferSize - 32) - << "less than 32 bytes of headroom left in the status buffer"; + << "less than 32 bytes of headroom left in the status buffer (len=" << len << ")"; } TEST(MQTTPayloadBuilder, StatusOmissionSentinelsRemainOmitted) { From 7004b81c0c4773b642663cd7218bf96456d19283 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 26 Sep 2026 11:10:13 -0700 Subject: [PATCH 05/23] feat(mqtt): count connect failures and report tripped breakers Two gaps remained after the first connection-health commit. A slot retrying into a heap that cannot satisfy its TLS allocation moved no counter at all. disconnect_count rose, but it also rises for dropped live sessions, so it could not separate "actively failing to connect" from "was connected, dropped once". Adds a per-slot connect_failures, incremented in onDisconnect when the event arrives while the client is Starting -- meaning it ended an attempt that never reached onConnect. Late events from a stopped or quarantined client, and soft-disconnects of a live session, are in other states and are not miscounted. A slot parked by the circuit breaker also looked identical to one still retrying, despite being materially worse: the breaker only probes every 30 minutes, so recovery is far slower and an operator may need to intervene. Surfaces both: - `mqttN.diag` gains `cf:` beside the existing `dc:`. - the status payload gains `mqtt_connect_failures` (summed across slots) and `mqtt_slots_breaker` (count of tripped slots), each emitted only when non-zero so a healthy board's payload does not grow. The counter is what distinguishes a retrying slot from a quiet one: while a slot is down its publish counters cannot move by definition, so this is the only value that shows the device is still trying. Both suggested by adversarial review. --- src/helpers/MQTTPayloadBuilder.cpp | 9 +++- src/helpers/MQTTPayloadBuilder.h | 7 ++++ src/helpers/bridges/MQTTBridge.cpp | 20 +++++++++ src/helpers/bridges/MQTTBridge.h | 4 ++ .../test_mqtt_payload_builder.cpp | 42 +++++++++++++++++++ 5 files changed, 81 insertions(+), 1 deletion(-) diff --git a/src/helpers/MQTTPayloadBuilder.cpp b/src/helpers/MQTTPayloadBuilder.cpp index 536bb5e7..9811b98f 100644 --- a/src/helpers/MQTTPayloadBuilder.cpp +++ b/src/helpers/MQTTPayloadBuilder.cpp @@ -64,7 +64,8 @@ int MQTTPayloadBuilder::buildStatusMessage( } const bool has_conn_health = conn_health.slots_up >= 0 || conn_health.slots_total >= 0 || - conn_health.worst_outage_secs >= 0 || conn_health.heap_largest >= 0; + conn_health.worst_outage_secs >= 0 || conn_health.heap_largest >= 0 || + conn_health.connect_failures >= 0 || conn_health.slots_breaker >= 0; if (battery_mv >= 0 || uptime_secs >= 0 || errors >= 0 || queue_len >= 0 || noise_floor > -999 || tx_air_secs >= 0 || rx_air_secs >= 0 || recv_errors >= 0 || @@ -92,6 +93,12 @@ int MQTTPayloadBuilder::buildStatusMessage( if (conn_health.worst_outage_secs >= 0) { stats["mqtt_outage_secs"] = conn_health.worst_outage_secs; } + if (conn_health.connect_failures > 0) { + stats["mqtt_connect_failures"] = conn_health.connect_failures; + } + if (conn_health.slots_breaker > 0) { + stats["mqtt_slots_breaker"] = conn_health.slots_breaker; + } } return serializeComplete(root, buffer, buffer_size); diff --git a/src/helpers/MQTTPayloadBuilder.h b/src/helpers/MQTTPayloadBuilder.h index b675f397..42dd3ff8 100644 --- a/src/helpers/MQTTPayloadBuilder.h +++ b/src/helpers/MQTTPayloadBuilder.h @@ -22,6 +22,13 @@ struct MQTTConnHealth { int slots_total = -1; // enabled slots, whether or not they connect int worst_outage_secs = -1; // longest current outage; -1 = nothing is down int heap_largest = -1; // largest free internal block, bytes + // Attempts that ended without reaching onConnect, summed across slots. A slot + // stuck retrying moves this while its publish counters stay frozen, which is + // what separates "actively failing" from "idle and quiet". + int connect_failures = -1; + // Slots parked by the circuit breaker. Materially worse than a retrying slot: + // the breaker only probes every 30 minutes, so recovery is far slower. + int slots_breaker = -1; }; // Mesh-independent JSON serialization core for MQTT publication payloads. diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index bde48149..bcff6917 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -584,6 +584,12 @@ void MQTTBridge::formatSlotDiagReply(char* buf, size_t bufsize, int slot_index) replyAppendf(buf, bufsize, &pos, "> mqtt%d: %s", slot_index + 1, state); if (slot.disconnect_count > 0) { replyAppendf(buf, bufsize, &pos, ", dc:%lu", (unsigned long)slot.disconnect_count); + // cf: attempts that never reached onConnect. A slot retrying into a heap that + // cannot satisfy the TLS allocation moves only this counter — dc rises with it, + // but the publish counters cannot move at all while the slot is down. + if (slot.connect_failures > 0) { + replyAppendf(buf, bufsize, &pos, ", cf:%lu", (unsigned long)slot.connect_failures); + } if (slot.first_disconnect_time > 0) { unsigned long first_disc_age_sec = (millis() - slot.first_disconnect_time) / 1000; replyAppendf(buf, bufsize, &pos, ", first_disc:%lus", first_disc_age_sec); @@ -1757,6 +1763,11 @@ bool MQTTBridge::ensureSlotClient(int index) { }); slot.client->onDisconnect([this, index](bool sessionPresent) { MQTT_DEBUG_PRINTLN("MQTT%d disconnected", index + 1); + // An event while an attempt is in flight ended one that never reached onConnect. + // Stopped/quarantined late events and soft-disconnects are not Starting. + if (mqttClientStateHasAttemptInFlight(_slots[index].client_state)) { + _slots[index].connect_failures++; + } // Only a live client's disconnect is news. One arriving for a client we // already stopped (or quarantined) must not resurrect its state. if (clientStateIsLive(_slots[index].client_state)) { @@ -3174,11 +3185,15 @@ MQTTConnHealth MQTTBridge::collectConnHealth() const { const unsigned long now = millis(); unsigned long worst_outage_ms = 0; bool any_outage_timed = false; + uint32_t connect_failures = 0; + int breaker_slots = 0; for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { const MQTTSlot& slot = _slots[i]; if (!slot.enabled) continue; // "none" slots are not an outage, they are unconfigured health.slots_total++; + connect_failures += slot.connect_failures; + if (slot.circuit_breaker_tripped) breaker_slots++; if (slot.connected) { health.slots_up++; continue; @@ -3198,6 +3213,11 @@ MQTTConnHealth MQTTBridge::collectConnHealth() const { if (any_outage_timed) { health.worst_outage_secs = static_cast(worst_outage_ms / 1000UL); } + // Clamp: the payload field is int, and these are lifetime counters. + health.connect_failures = connect_failures > static_cast(INT32_MAX) + ? INT32_MAX + : static_cast(connect_failures); + health.slots_breaker = breaker_slots; health.heap_largest = static_cast(heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL)); return health; } diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index b755ce2f..bbf04472 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -183,6 +183,10 @@ private: uint8_t last_connack_code; unsigned long last_error_time; // millis() of last error uint32_t disconnect_count; // Number of disconnect callbacks since boot + // Attempts that ended without ever reaching onConnect. Distinct from + // disconnect_count (which also counts dropped live sessions) and from the + // publish error counter, which cannot move at all while a slot is down. + uint32_t connect_failures; unsigned long first_disconnect_time; // millis() of first disconnect after boot // Current-outage timer (used by AlertReporter to fire faults after a sustained diff --git a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp index b0c2d438..f539ee8d 100644 --- a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp +++ b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp @@ -118,6 +118,46 @@ TEST(MQTTPayloadBuilder, StatusReportsConnectionHealthWhenSupplied) { EXPECT_EQ(16116, stats["heap_largest"].as()); } +TEST(MQTTPayloadBuilder, StatusReportsConnectFailuresAndBreakerOnlyWhenNonZero) { + JsonDocument scratch; + char buffer[1024]; + MQTTConnHealth health; + health.slots_up = 1; + health.slots_total = 3; + health.connect_failures = 57; + health.slots_breaker = 1; + + int len = MQTTPayloadBuilder::buildStatusMessage( + scratch, "node", "id", "model", "firmware", "radio", "client", "online", + kTimestamp, buffer, sizeof(buffer), -1, -1, -1, -1, -999, + -1, -1, -1, -1, -1, -1, nullptr, health); + + ASSERT_GT(len, 0); + JsonDocument parsed; + ASSERT_FALSE(deserializeJson(parsed, buffer)); + JsonObject stats = parsed["stats"].as(); + EXPECT_EQ(57, stats["mqtt_connect_failures"].as()); + EXPECT_EQ(1, stats["mqtt_slots_breaker"].as()); + + // A healthy board must not carry either key: zero connect failures and zero + // tripped breakers are the normal case, and the steady-state payload is + // already close to the buffer bound. + MQTTConnHealth healthy; + healthy.slots_up = 3; + healthy.slots_total = 3; + healthy.connect_failures = 0; + healthy.slots_breaker = 0; + ASSERT_GT(MQTTPayloadBuilder::buildStatusMessage( + scratch, "node", "id", "model", "firmware", "radio", "client", "online", + kTimestamp, buffer, sizeof(buffer), -1, -1, -1, -1, -999, + -1, -1, -1, -1, -1, -1, nullptr, healthy), 0); + JsonDocument healthy_parsed; + ASSERT_FALSE(deserializeJson(healthy_parsed, buffer)); + JsonObject healthy_stats = healthy_parsed["stats"].as(); + EXPECT_FALSE(healthy_stats["mqtt_connect_failures"].is()); + EXPECT_FALSE(healthy_stats["mqtt_slots_breaker"].is()); +} + // The outage key doubles as the "something is down" flag, so its absence has to be // meaningful: a fully healthy board must not emit it. TEST(MQTTPayloadBuilder, StatusOmitsOutageKeyWhenEverySlotIsUp) { @@ -168,6 +208,8 @@ TEST(MQTTPayloadBuilder, WorstCaseStatusWithConnectionHealthFitsProductionBuffer health.slots_total = 5; // PSRAM slot cap, the widest configuration health.worst_outage_secs = 4294967; // millis() rollover / 1000, the widest value health.heap_largest = 8388608; + health.connect_failures = 2147483647; + health.slots_breaker = 5; int len = MQTTPayloadBuilder::buildStatusMessage( scratch, origin.c_str(), origin_id.c_str(), model.c_str(), firmware.c_str(), From 73098a44b6d958a54851334cbc02bf7b1f85df8a Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 26 Sep 2026 11:53:49 -0700 Subject: [PATCH 06/23] fix(mqtt): count connect failures from a flag set before the attempt The Starting-state gate raced the event task: client_state becomes Starting only after connect()/reconnect() returns, so a fast failure could deliver DISCONNECTED first and go uncounted, while a deliberate stop of an attempt in flight was counted as a failure. A per-slot flag is now raised before the call and cleared by onConnect, a failed start, or stopSlotClient(). Also stops zero-valued connect_failures/slots_breaker from creating an empty stats object. --- src/helpers/MQTTPayloadBuilder.cpp | 2 +- src/helpers/bridges/MQTTBridge.cpp | 13 ++++++++++--- src/helpers/bridges/MQTTBridge.h | 5 +++++ .../test_mqtt_payload_builder.cpp | 17 +++++++++++++++++ 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/helpers/MQTTPayloadBuilder.cpp b/src/helpers/MQTTPayloadBuilder.cpp index 9811b98f..8cbb03f4 100644 --- a/src/helpers/MQTTPayloadBuilder.cpp +++ b/src/helpers/MQTTPayloadBuilder.cpp @@ -65,7 +65,7 @@ int MQTTPayloadBuilder::buildStatusMessage( const bool has_conn_health = conn_health.slots_up >= 0 || conn_health.slots_total >= 0 || conn_health.worst_outage_secs >= 0 || conn_health.heap_largest >= 0 || - conn_health.connect_failures >= 0 || conn_health.slots_breaker >= 0; + conn_health.connect_failures > 0 || conn_health.slots_breaker > 0; if (battery_mv >= 0 || uptime_secs >= 0 || errors >= 0 || queue_len >= 0 || noise_floor > -999 || tx_air_secs >= 0 || rx_air_secs >= 0 || recv_errors >= 0 || diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index bcff6917..134b12e7 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -772,6 +772,7 @@ MQTTBridge::MQTTBridge(NodePrefs *prefs, MQTTPrefs *obs, mesh::PacketManager *mg _slot_reconfigure_pending[i] = false; _slot_force_jwt_mint[i] = false; _status_publish_pending[i] = false; + _slot_attempt_pending[i] = false; } // Reset CLI-requested forced NTP sync handshake (bridge object is reused across restarts) @@ -1730,6 +1731,7 @@ bool MQTTBridge::ensureSlotClient(int index) { return; } MQTT_DEBUG_PRINTLN("MQTT%d connected", index + 1); + _slot_attempt_pending[index] = false; _slots[index].client_state = ClientState::Connected; _slots[index].connected = true; _slot_force_jwt_mint[index] = false; @@ -1763,9 +1765,10 @@ bool MQTTBridge::ensureSlotClient(int index) { }); slot.client->onDisconnect([this, index](bool sessionPresent) { MQTT_DEBUG_PRINTLN("MQTT%d disconnected", index + 1); - // An event while an attempt is in flight ended one that never reached onConnect. - // Stopped/quarantined late events and soft-disconnects are not Starting. - if (mqttClientStateHasAttemptInFlight(_slots[index].client_state)) { + // Ended an attempt that never reached onConnect. A deliberate stop clears the flag + // first, so cancelling an attempt is not counted as a failure. + if (_slot_attempt_pending[index]) { + _slot_attempt_pending[index] = false; _slots[index].connect_failures++; } // Only a live client's disconnect is news. One arriving for a client we @@ -2411,6 +2414,7 @@ void MQTTBridge::closeLiveClientForLinkTransition(int index) { // the rest of the boot. void MQTTBridge::stopSlotClient(int index) { MQTTSlot& slot = _slots[index]; + _slot_attempt_pending[index] = false; const esp_err_t r = slot.client->disconnect(); // What the SDK's results actually mean here (mqtt_client.h documents @@ -2505,6 +2509,7 @@ esp_err_t MQTTBridge::reconnectSlotClient(int index) { } esp_err_t r; + _slot_attempt_pending[index] = true; if (!slot.client->isStarted()) { MQTT_DEBUG_PRINTLN("MQTT%d start (client was stopped)", index + 1); r = slot.client->connect(); @@ -2517,6 +2522,8 @@ esp_err_t MQTTBridge::reconnectSlotClient(int index) { // The attempt carries whatever credential is configured right now, so this // is the point at which a freshly minted token becomes the one in use. slot.applied_token_expires_at = slot.token_expires_at; + } else { + _slot_attempt_pending[index] = false; } return r; } diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index bbf04472..87b75b05 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -286,6 +286,11 @@ private: // _slot_reconfigure_pending; a single-byte volatile store/load is atomic. volatile bool _status_publish_pending[RUNTIME_MQTT_SLOTS]; + // Set by the bridge task before connect()/reconnect(), so a failure the event task + // delivers before that call returns is still seen; cleared by the attempt's outcome + // or a deliberate stop. A disconnect that finds it set is a connect failure. + volatile bool _slot_attempt_pending[RUNTIME_MQTT_SLOTS]; + // CLI-requested forced NTP sync, marshalled onto the MQTT task (Core 0). // All NTP I/O must run on Core 0; the CLI thread // (Core 1) sets _ntp_force_requested and blocks in requestForcedNtpSync() diff --git a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp index f539ee8d..e8ae51c8 100644 --- a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp +++ b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp @@ -158,6 +158,23 @@ TEST(MQTTPayloadBuilder, StatusReportsConnectFailuresAndBreakerOnlyWhenNonZero) EXPECT_FALSE(healthy_stats["mqtt_slots_breaker"].is()); } +// Zero counters with nothing else supplied must not create an empty stats object. +TEST(MQTTPayloadBuilder, StatusOmitsStatsWhenOnlyZeroCountersSupplied) { + JsonDocument scratch; + char buffer[768]; + MQTTConnHealth health; + health.connect_failures = 0; + health.slots_breaker = 0; + + ASSERT_GT(MQTTPayloadBuilder::buildStatusMessage( + scratch, "node", "id", "model", "firmware", "radio", "client", "online", + kTimestamp, buffer, sizeof(buffer), -1, -1, -1, -1, -999, + -1, -1, -1, -1, -1, -1, nullptr, health), 0); + JsonDocument parsed; + ASSERT_FALSE(deserializeJson(parsed, buffer)); + EXPECT_FALSE(parsed["stats"].is()); +} + // The outage key doubles as the "something is down" flag, so its absence has to be // meaningful: a fully healthy board must not emit it. TEST(MQTTPayloadBuilder, StatusOmitsOutageKeyWhenEverySlotIsUp) { From 6390763c719998ed7000820aaac97a1c87e9a227 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 26 Sep 2026 12:25:53 -0700 Subject: [PATCH 07/23] fix(mqtt): count failed starts and ignore bounced sessions in cf connect()/reconnect() returning an error started no attempt, so no event arrived and the failure went uncounted -- the low-heap case the counter exists for. Counted now in a bridge-task-only start_failures, summed into cf: and mqtt_connect_failures. A renewal or corrected-clock bounce whose softDisconnect() times out gets its old session's DISCONNECTED after the new attempt raised the flag. That session is still marked connected until the handler runs, so the gate now also requires !connected. An ignored late CONNECTED clears the flag too. cf: no longer depends on dc being non-zero. Comments corrected for the PSRAM-fallback stack cost and heap_largest's sampling point; the worst-case test now models 6 slots. --- src/helpers/MQTTPayloadBuilder.h | 2 ++ src/helpers/bridges/MQTTBridge.cpp | 22 +++++++++++-------- src/helpers/bridges/MQTTBridge.h | 11 +++++++--- .../test_mqtt_payload_builder.cpp | 4 ++-- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/helpers/MQTTPayloadBuilder.h b/src/helpers/MQTTPayloadBuilder.h index 42dd3ff8..7f0cd85b 100644 --- a/src/helpers/MQTTPayloadBuilder.h +++ b/src/helpers/MQTTPayloadBuilder.h @@ -17,6 +17,8 @@ // heap alone does not — an 85 KB-free board sat unable to connect for hours. // A low value with all slots up is normal, not a fault: established sessions // already hold their buffers. It means only that the next slot to ask will fail. +// Sampled at status-publish time, not at the attempt, so it is a trend, not a +// per-attempt reading. struct MQTTConnHealth { int slots_up = -1; // -1 = not supplied, field omitted int slots_total = -1; // enabled slots, whether or not they connect diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 134b12e7..2bee5bb2 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -584,17 +584,18 @@ void MQTTBridge::formatSlotDiagReply(char* buf, size_t bufsize, int slot_index) replyAppendf(buf, bufsize, &pos, "> mqtt%d: %s", slot_index + 1, state); if (slot.disconnect_count > 0) { replyAppendf(buf, bufsize, &pos, ", dc:%lu", (unsigned long)slot.disconnect_count); - // cf: attempts that never reached onConnect. A slot retrying into a heap that - // cannot satisfy the TLS allocation moves only this counter — dc rises with it, - // but the publish counters cannot move at all while the slot is down. - if (slot.connect_failures > 0) { - replyAppendf(buf, bufsize, &pos, ", cf:%lu", (unsigned long)slot.connect_failures); - } if (slot.first_disconnect_time > 0) { unsigned long first_disc_age_sec = (millis() - slot.first_disconnect_time) / 1000; replyAppendf(buf, bufsize, &pos, ", first_disc:%lus", first_disc_age_sec); } } + // cf: attempts that never reached onConnect, plus starts that failed outright. A + // slot retrying into a heap that cannot satisfy the TLS allocation moves only this; + // the publish counters cannot move at all while the slot is down. + const uint32_t cf = slot.connect_failures + slot.start_failures; + if (cf > 0) { + replyAppendf(buf, bufsize, &pos, ", cf:%lu", (unsigned long)cf); + } // Connected with no errors: nothing more to say about the connection. if (slot.connected && slot.last_error_time == 0) { @@ -1728,6 +1729,7 @@ bool MQTTBridge::ensureSlotClient(int index) { MQTT_DEBUG_PRINTLN("MQTT%d ignoring late CONNECTED (state=%s, gen=%lu, enabled=%d)", index + 1, clientStateName(st), (unsigned long)_slots[index].generation, (int)_slots[index].enabled); + _slot_attempt_pending[index] = false; // still the end of that attempt return; } MQTT_DEBUG_PRINTLN("MQTT%d connected", index + 1); @@ -1766,8 +1768,9 @@ bool MQTTBridge::ensureSlotClient(int index) { slot.client->onDisconnect([this, index](bool sessionPresent) { MQTT_DEBUG_PRINTLN("MQTT%d disconnected", index + 1); // Ended an attempt that never reached onConnect. A deliberate stop clears the flag - // first, so cancelling an attempt is not counted as a failure. - if (_slot_attempt_pending[index]) { + // first; a bounced live session is still marked connected until this handler runs, + // so its late DISCONNECTED after a timed-out softDisconnect() is not counted either. + if (_slot_attempt_pending[index] && !_slots[index].connected) { _slot_attempt_pending[index] = false; _slots[index].connect_failures++; } @@ -2524,6 +2527,7 @@ esp_err_t MQTTBridge::reconnectSlotClient(int index) { slot.applied_token_expires_at = slot.token_expires_at; } else { _slot_attempt_pending[index] = false; + slot.start_failures++; } return r; } @@ -3199,7 +3203,7 @@ MQTTConnHealth MQTTBridge::collectConnHealth() const { const MQTTSlot& slot = _slots[i]; if (!slot.enabled) continue; // "none" slots are not an outage, they are unconfigured health.slots_total++; - connect_failures += slot.connect_failures; + connect_failures += slot.connect_failures + slot.start_failures; if (slot.circuit_breaker_tripped) breaker_slots++; if (slot.connected) { health.slots_up++; diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 87b75b05..87e585b1 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -187,6 +187,10 @@ private: // disconnect_count (which also counts dropped live sessions) and from the // publish error counter, which cannot move at all while a slot is down. uint32_t connect_failures; + // connect()/reconnect() calls that returned an error, so no attempt started and + // no event will arrive. Written only by the bridge task; reported summed with + // connect_failures. + uint32_t start_failures; unsigned long first_disconnect_time; // millis() of first disconnect after boot // Current-outage timer (used by AlertReporter to fire faults after a sustained @@ -288,7 +292,7 @@ private: // Set by the bridge task before connect()/reconnect(), so a failure the event task // delivers before that call returns is still seen; cleared by the attempt's outcome - // or a deliberate stop. A disconnect that finds it set is a connect failure. + // (including an ignored late CONNECTED) or a deliberate stop. volatile bool _slot_attempt_pending[RUNTIME_MQTT_SLOTS]; // CLI-requested forced NTP sync, marshalled onto the MQTT task (Core 0). @@ -381,8 +385,9 @@ private: // _device_id[65], _board_model[64], _firmware_version[64] and a 63-byte client version): // measured 762/768 before the connection-health fields existed. Overflow is silent — // serializeComplete() returns 0 and the bridge publishes only len>0, so status stops - // entirely. Raised to 1024, which costs no extra RAM here because status serializes into - // the shared scratch buffer. See the worst-case test in test_mqtt_payload_builder. + // entirely. Raised to 1024: no extra RAM while status serializes into the shared scratch + // buffer, but +256 B of bridge-task stack in the fallback used if the PSRAM allocation + // failed. See the worst-case test in test_mqtt_payload_builder. static const size_t STATUS_JSON_BUFFER_SIZE = 1024; static_assert(STATUS_JSON_BUFFER_SIZE <= PUBLISH_JSON_BUFFER_SIZE, "status payloads serialize into the shared publish buffer"); diff --git a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp index e8ae51c8..eedf6979 100644 --- a/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp +++ b/test/test_mqtt_payload_builder/test_mqtt_payload_builder.cpp @@ -222,11 +222,11 @@ TEST(MQTTPayloadBuilder, WorstCaseStatusWithConnectionHealthFitsProductionBuffer char buffer[kStatusJsonBufferSize]; MQTTConnHealth health; health.slots_up = 0; - health.slots_total = 5; // PSRAM slot cap, the widest configuration + health.slots_total = 6; // RUNTIME_MQTT_SLOTS on PSRAM, the widest health.worst_outage_secs = 4294967; // millis() rollover / 1000, the widest value health.heap_largest = 8388608; health.connect_failures = 2147483647; - health.slots_breaker = 5; + health.slots_breaker = 6; int len = MQTTPayloadBuilder::buildStatusMessage( scratch, origin.c_str(), origin_id.c_str(), model.c_str(), firmware.c_str(), From f37445f8a7d14f7d75e1ec848398f591d793b6f6 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 26 Sep 2026 12:48:44 -0700 Subject: [PATCH 08/23] fix(mqtt): count only failed starts, not rejected reconnects, in cf reconnect() on a started client returns ESP_FAIL while an attempt is already in progress. On a Heltec V3, a live reconfigure hit exactly that and the attempt then connected, yet cf read 1. Count start_failures only when connect() on a stopped client fails, and restore the attempt flag to its prior value on any failed call so the in-progress attempt is still judged by its own outcome. --- src/helpers/bridges/MQTTBridge.cpp | 11 ++++++++--- src/helpers/bridges/MQTTBridge.h | 6 +++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 2bee5bb2..8e5eec70 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -2512,8 +2512,10 @@ esp_err_t MQTTBridge::reconnectSlotClient(int index) { } esp_err_t r; + const bool was_pending = _slot_attempt_pending[index]; + const bool starting = !slot.client->isStarted(); _slot_attempt_pending[index] = true; - if (!slot.client->isStarted()) { + if (starting) { MQTT_DEBUG_PRINTLN("MQTT%d start (client was stopped)", index + 1); r = slot.client->connect(); } else { @@ -2526,8 +2528,11 @@ esp_err_t MQTTBridge::reconnectSlotClient(int index) { // is the point at which a freshly minted token becomes the one in use. slot.applied_token_expires_at = slot.token_expires_at; } else { - _slot_attempt_pending[index] = false; - slot.start_failures++; + _slot_attempt_pending[index] = was_pending; + // A failed start (buffer/task allocation, config) means no attempt could begin. + // reconnect() on a started client returns ESP_FAIL while one is already in + // progress, which is not a failure (seen on hardware after a reconfigure). + if (starting) slot.start_failures++; } return r; } diff --git a/src/helpers/bridges/MQTTBridge.h b/src/helpers/bridges/MQTTBridge.h index 87e585b1..767a49e1 100644 --- a/src/helpers/bridges/MQTTBridge.h +++ b/src/helpers/bridges/MQTTBridge.h @@ -187,9 +187,9 @@ private: // disconnect_count (which also counts dropped live sessions) and from the // publish error counter, which cannot move at all while a slot is down. uint32_t connect_failures; - // connect()/reconnect() calls that returned an error, so no attempt started and - // no event will arrive. Written only by the bridge task; reported summed with - // connect_failures. + // connect() calls on a stopped client that returned an error, so no attempt + // started and no event will arrive. Written only by the bridge task; reported + // summed with connect_failures. uint32_t start_failures; unsigned long first_disconnect_time; // millis() of first disconnect after boot From 7d0a602b20cd61419005566bce5b5da7cb644160 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 26 Sep 2026 13:54:43 -0700 Subject: [PATCH 09/23] fix(mqtt): exclude unready slots from slot health and keep cf monotonic A slot waiting on a token/IATA/credential stays enabled but never attempts a connection, so it held mqtt_slots_up below mqtt_slots_total (and down= at 1) permanently -- a standing false alert on a node that is merely unconfigured. slots_total and down= now count only ready slots, matching the CLI's "wait". mqtt_connect_failures summed only enabled slots, so disabling a failing slot made it drop. It now sums every slot, monotonic within a boot. cf: moves after the error tail in mqttN.diag so a clamped 160-byte reply loses the count rather than the refusal/TLS detail. A failed start now clears the attempt flag instead of restoring a possibly stale one. Outage comments corrected: the clock starts at a slot's first DISCONNECTED, not first connect. --- src/helpers/MQTTPayloadBuilder.cpp | 2 +- src/helpers/MQTTPayloadBuilder.h | 10 ++++----- src/helpers/bridges/MQTTBridge.cpp | 36 +++++++++++++++++------------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/helpers/MQTTPayloadBuilder.cpp b/src/helpers/MQTTPayloadBuilder.cpp index 8cbb03f4..56e2a259 100644 --- a/src/helpers/MQTTPayloadBuilder.cpp +++ b/src/helpers/MQTTPayloadBuilder.cpp @@ -88,7 +88,7 @@ int MQTTPayloadBuilder::buildStatusMessage( if (conn_health.slots_total >= 0) stats["mqtt_slots_total"] = conn_health.slots_total; // Emitted only during an outage of known duration, so the steady-state payload // does not grow. Presence implies a slot is down, but absence does NOT imply - // health: a slot that has never connected since boot has no outage start time. + // health: a slot has no outage start time until its first attempt ends. // `mqtt_slots_up < mqtt_slots_total` is the signal to alert on; this is detail. if (conn_health.worst_outage_secs >= 0) { stats["mqtt_outage_secs"] = conn_health.worst_outage_secs; diff --git a/src/helpers/MQTTPayloadBuilder.h b/src/helpers/MQTTPayloadBuilder.h index 7f0cd85b..fa46e741 100644 --- a/src/helpers/MQTTPayloadBuilder.h +++ b/src/helpers/MQTTPayloadBuilder.h @@ -21,12 +21,12 @@ // per-attempt reading. struct MQTTConnHealth { int slots_up = -1; // -1 = not supplied, field omitted - int slots_total = -1; // enabled slots, whether or not they connect - int worst_outage_secs = -1; // longest current outage; -1 = nothing is down + int slots_total = -1; // enabled, fully configured slots, connected or not + int worst_outage_secs = -1; // longest timed outage; -1 = none timed (see below) int heap_largest = -1; // largest free internal block, bytes - // Attempts that ended without reaching onConnect, summed across slots. A slot - // stuck retrying moves this while its publish counters stay frozen, which is - // what separates "actively failing" from "idle and quiet". + // Attempts that ended without reaching onConnect, summed across all slots since + // boot. A slot stuck retrying moves this while its publish counters stay frozen, + // which is what separates "actively failing" from "idle and quiet". int connect_failures = -1; // Slots parked by the circuit breaker. Materially worse than a retrying slot: // the breaker only probes every 30 minutes, so recovery is far slower. diff --git a/src/helpers/bridges/MQTTBridge.cpp b/src/helpers/bridges/MQTTBridge.cpp index 8e5eec70..8c24fbf2 100644 --- a/src/helpers/bridges/MQTTBridge.cpp +++ b/src/helpers/bridges/MQTTBridge.cpp @@ -373,13 +373,13 @@ void MQTTBridge::formatMqttStatsReply(char* buf, size_t bufsize) { if (b->_filtered_packets > 0) { replyAppendf(buf, bufsize, &pos, " filt=%lu", b->_filtered_packets); } - // down=: enabled slots that are not connected. Without this the reply cannot + // down=: ready slots that are not connected. Without this the reply cannot // show an outage at all — sN= counts publishes, and a slot that never connects // never publishes, so its ok freezes and its err stays 0 for the whole outage. // Omitted while zero, like filt=, so a healthy node's reply keeps its length. int down = 0; for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { - if (b->_slots[i].enabled && !b->_slots[i].connected) down++; + if (b->_slots[i].enabled && !b->_slots[i].connected && b->isSlotReady(i)) down++; } if (down > 0) { replyAppendf(buf, bufsize, &pos, " down=%d", down); @@ -589,13 +589,6 @@ void MQTTBridge::formatSlotDiagReply(char* buf, size_t bufsize, int slot_index) replyAppendf(buf, bufsize, &pos, ", first_disc:%lus", first_disc_age_sec); } } - // cf: attempts that never reached onConnect, plus starts that failed outright. A - // slot retrying into a heap that cannot satisfy the TLS allocation moves only this; - // the publish counters cannot move at all while the slot is down. - const uint32_t cf = slot.connect_failures + slot.start_failures; - if (cf > 0) { - replyAppendf(buf, bufsize, &pos, ", cf:%lu", (unsigned long)cf); - } // Connected with no errors: nothing more to say about the connection. if (slot.connected && slot.last_error_time == 0) { @@ -642,6 +635,14 @@ void MQTTBridge::formatSlotDiagReply(char* buf, size_t bufsize, int slot_index) } else if (!slot.connected) { replyAppendf(buf, bufsize, &pos, ", no error info"); } + // cf: attempts that never reached onConnect, plus starts that failed outright. A + // slot retrying into a heap that cannot satisfy the TLS allocation moves only this; + // the publish counters cannot move at all while the slot is down. + // After the error tail so a clamped reply loses this count, not the error detail. + const uint32_t cf = slot.connect_failures + slot.start_failures; + if (cf > 0) { + replyAppendf(buf, bufsize, &pos, ", cf:%lu", (unsigned long)cf); + } // Appended last so it never displaces connection diagnostics. replyAppendf // clamps rather than overflows, but a clipped type list is worse than no @@ -2528,7 +2529,8 @@ esp_err_t MQTTBridge::reconnectSlotClient(int index) { // is the point at which a freshly minted token becomes the one in use. slot.applied_token_expires_at = slot.token_expires_at; } else { - _slot_attempt_pending[index] = was_pending; + // A failed start leaves nothing in flight; a rejected reconnect leaves the prior one. + _slot_attempt_pending[index] = starting ? false : was_pending; // A failed start (buffer/task allocation, config) means no attempt could begin. // reconnect() on a started client returns ESP_FAIL while one is already in // progress, which is not a failure (seen on hardware after a reconfigure). @@ -3206,19 +3208,21 @@ MQTTConnHealth MQTTBridge::collectConnHealth() const { for (int i = 0; i < RUNTIME_MQTT_SLOTS; i++) { const MQTTSlot& slot = _slots[i]; - if (!slot.enabled) continue; // "none" slots are not an outage, they are unconfigured - health.slots_total++; + // Summed before the skip so disabling a slot never makes the total go backwards. connect_failures += slot.connect_failures + slot.start_failures; + // Unconfigured slots and slots waiting on a token/IATA/credential are not outages. + if (!slot.enabled || !isSlotReady(i)) continue; + health.slots_total++; if (slot.circuit_breaker_tripped) breaker_slots++; if (slot.connected) { health.slots_up++; continue; } - // A slot that has never connected since boot has no outage start time. It still - // counts against slots_up, which is why that pair — not this duration — is the - // signal to alert on. + // The outage clock starts at a slot's first DISCONNECTED, so a slot that has not yet + // finished its first attempt has no start time. It still counts against slots_up, + // which is why that pair — not this duration — is the signal to alert on. if (slot.current_outage_started_ms != 0) { - const unsigned long outage_ms = now - slot.current_outage_started_ms; // wrap-safe + const unsigned long outage_ms = now - slot.current_outage_started_ms; // wrap-safe under 49.7 days if (!any_outage_timed || outage_ms > worst_outage_ms) { worst_outage_ms = outage_ms; any_outage_timed = true; From f04d163bea10c6a3d4641fc6f76189d334f7768b Mon Sep 17 00:00:00 2001 From: Elektr0Vodka <211697683+Elektr0Vodka@users.noreply.github.com> Date: Mon, 14 Sep 2026 04:05:58 +0200 Subject: [PATCH 10/23] feat(ota): channel-base resolver + arg parser with native tests (cherry picked from commit 52a4ccc39093891b64e3752316fe7c191fd9b9ca) --- platformio.ini | 18 ++++++- src/helpers/OtaChannel.h | 62 ++++++++++++++++++++++ test/test_ota_channel/test_ota_channel.cpp | 34 ++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 src/helpers/OtaChannel.h create mode 100644 test/test_ota_channel/test_ota_channel.cpp diff --git a/platformio.ini b/platformio.ini index 6c012f1e..d4786908 100644 --- a/platformio.ini +++ b/platformio.ini @@ -177,7 +177,7 @@ build_flags = -std=c++17 -I src -I test/mocks test_build_src = yes -test_ignore = test_kiss_modem +test_ignore = test_kiss_modem, test_ota_channel build_src_filter = -<*> +<../src/Utils.cpp> @@ -203,3 +203,19 @@ build_src_filter = lib_deps = google/googletest @ 1.17.0 bblanchon/ArduinoJson @ 7.4.3 + +[env:native_ota_channel] +platform = native +test_framework = googletest +build_flags = -std=c++17 + -I test/mocks + -I src + -DOTA_MANIFEST_BASE="\"https://stable.example/mqtt/v\"" + -DOTA_MANIFEST_BASE_STABLE="\"https://stable.example/mqtt/v\"" + -DOTA_MANIFEST_BASE_DEV="\"https://dev.example/mqtt/dev/v\"" +test_build_src = yes +test_filter = test_ota_channel +build_src_filter = + -<*> +lib_deps = + google/googletest @ 1.17.0 diff --git a/src/helpers/OtaChannel.h b/src/helpers/OtaChannel.h new file mode 100644 index 00000000..d589854b --- /dev/null +++ b/src/helpers/OtaChannel.h @@ -0,0 +1,62 @@ +#pragma once +#include +#include + +// OTA release-channel selector, persisted in NodePrefs::ota_channel. +enum OtaChannel : uint8_t { + OTA_CH_NATIVE = 0, // follow the channel this build was made for + OTA_CH_STABLE = 1, + OTA_CH_DEV = 2, +}; + +// Resolve the effective manifest base URL for a channel selector. +// build.sh injects the three bases as compile-time macros: +// OTA_MANIFEST_BASE = this build's native channel (defined on every OTA build) +// OTA_MANIFEST_BASE_STABLE = stable channel +// OTA_MANIFEST_BASE_DEV = dev channel +// stable/dev fall back to the native base when their macro is undefined (legacy/local +// builds that only define OTA_MANIFEST_BASE), so this never returns nullptr on an +// OTA-capable build. On a non-OTA build it returns nullptr. +static inline const char* ota_resolve_base(uint8_t channel) { +#if defined(OTA_MANIFEST_BASE) + switch (channel) { + case OTA_CH_STABLE: +#if defined(OTA_MANIFEST_BASE_STABLE) + return OTA_MANIFEST_BASE_STABLE; +#else + return OTA_MANIFEST_BASE; +#endif + case OTA_CH_DEV: +#if defined(OTA_MANIFEST_BASE_DEV) + return OTA_MANIFEST_BASE_DEV; +#else + return OTA_MANIFEST_BASE; +#endif + case OTA_CH_NATIVE: + default: + return OTA_MANIFEST_BASE; + } +#else + (void)channel; + return nullptr; +#endif +} + +// Human label for a selector (for the `ota branch` report). +static inline const char* ota_channel_name(uint8_t channel) { + switch (channel) { + case OTA_CH_STABLE: return "stable"; + case OTA_CH_DEV: return "dev"; + default: return "native"; + } +} + +// Parse an `ota branch` argument. Returns true and sets *out on a known keyword +// (stable|dev|default; "default" -> native); returns false and leaves *out untouched +// otherwise. +static inline bool ota_parse_channel(const char* arg, uint8_t* out) { + if (strcmp(arg, "stable") == 0) { *out = OTA_CH_STABLE; return true; } + if (strcmp(arg, "dev") == 0) { *out = OTA_CH_DEV; return true; } + if (strcmp(arg, "default") == 0) { *out = OTA_CH_NATIVE; return true; } + return false; +} diff --git a/test/test_ota_channel/test_ota_channel.cpp b/test/test_ota_channel/test_ota_channel.cpp new file mode 100644 index 00000000..699ffb70 --- /dev/null +++ b/test/test_ota_channel/test_ota_channel.cpp @@ -0,0 +1,34 @@ +#include +#include "helpers/OtaChannel.h" + +// The three base URLs are provided as -D macros by the test env (see platformio.ini). +TEST(OtaChannel, ResolvesNativeToBaseMacro) { + EXPECT_STREQ(ota_resolve_base(OTA_CH_NATIVE), "https://stable.example/mqtt/v"); +} +TEST(OtaChannel, ResolvesStable) { + EXPECT_STREQ(ota_resolve_base(OTA_CH_STABLE), "https://stable.example/mqtt/v"); +} +TEST(OtaChannel, ResolvesDev) { + EXPECT_STREQ(ota_resolve_base(OTA_CH_DEV), "https://dev.example/mqtt/dev/v"); +} +TEST(OtaChannel, ParseKnownKeywords) { + uint8_t ch = 99; + EXPECT_TRUE(ota_parse_channel("stable", &ch)); EXPECT_EQ(ch, OTA_CH_STABLE); + EXPECT_TRUE(ota_parse_channel("dev", &ch)); EXPECT_EQ(ch, OTA_CH_DEV); + EXPECT_TRUE(ota_parse_channel("default", &ch)); EXPECT_EQ(ch, OTA_CH_NATIVE); +} +TEST(OtaChannel, ParseRejectsUnknownAndLeavesOutputUntouched) { + uint8_t ch = 7; + EXPECT_FALSE(ota_parse_channel("beta", &ch)); + EXPECT_EQ(ch, 7); +} +TEST(OtaChannel, NameLabels) { + EXPECT_STREQ(ota_channel_name(OTA_CH_NATIVE), "native"); + EXPECT_STREQ(ota_channel_name(OTA_CH_STABLE), "stable"); + EXPECT_STREQ(ota_channel_name(OTA_CH_DEV), "dev"); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 3910489414f00d16f377c5051cae47051f2e1194 Mon Sep 17 00:00:00 2001 From: Elektr0Vodka <211697683+Elektr0Vodka@users.noreply.github.com> Date: Mon, 14 Sep 2026 04:07:12 +0200 Subject: [PATCH 11/23] feat(ota): persist ota_channel selector in NodePrefs (cherry picked from commit db1db5b57ecce408c7441cb6aa70f29608b09603) --- src/helpers/CommonCLI.h | 2 ++ .../test_config_serializer.cpp | 29 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index e0a0a71b..db77ac8d 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -70,6 +70,7 @@ public: uint8_t path_hash_mode = 0; // which path mode to use when sending uint8_t loop_detect = 0; uint8_t cad_enabled = 0; // hardware Channel Activity Detection before TX (boolean) + uint8_t ota_channel = 0; // OTA release channel selector: 0=native, 1=stable, 2=dev uint8_t extra_sf[4]; // NOTE: observer settings (MQTT/WiFi/timezone/SNMP/alert) are not in NodePrefs. @@ -182,6 +183,7 @@ protected: def("lat", node_lat); def("lon", node_lon); def("disc_mod", discovery_mod_timestamp); // gates 'since'-filtered DISCOVER replies + def("ota_ch", ota_channel); // OTA release channel: 0=native, 1=stable, 2=dev def("radio", radio); def("bridge", bridge); def("gps", gps); diff --git a/test/test_config_serializer/test_config_serializer.cpp b/test/test_config_serializer/test_config_serializer.cpp index 27c3c811..1fbe1c8a 100644 --- a/test/test_config_serializer/test_config_serializer.cpp +++ b/test/test_config_serializer/test_config_serializer.cpp @@ -214,6 +214,35 @@ TEST(NodePrefs, FemGainSettingsRoundTrip) { EXPECT_EQ(1, loaded.radio_fem_txgain); } +TEST(NodePrefs, OtaChannelDefaultsToNative) { + NodePrefs prefs; + EXPECT_EQ(0, prefs.ota_channel); // 0 == native +} + +TEST(NodePrefs, OtaChannelRoundTrip) { + NodePrefs saved; + saved.ota_channel = 2; // dev + + MockPrintStream output; + ASSERT_TRUE(saved.saveSerial(output)); + std::string serialised(reinterpret_cast(output.getBytes()), output.getLength()); + EXPECT_NE(std::string::npos, serialised.find("ota_ch:2")); + + MockInputStream input(serialised.c_str()); + NodePrefs loaded; + loaded.ota_channel = 1; // start different + ASSERT_TRUE(loaded.loadSerial(input)) << serialised; + EXPECT_EQ(2, loaded.ota_channel); +} + +TEST(NodePrefs, OtaChannelMissingKeyKeepsDefault) { + // A /prefs.json written before this field existed has no ota_ch key. + MockInputStream input("{name:\"n\"}"); + NodePrefs loaded; // ota_channel default 0 (native) + ASSERT_TRUE(loaded.loadSerial(input)); + EXPECT_EQ(0, loaded.ota_channel); +} + // ── main ─────────────────────────────────────────────────────── From 942a19eb63947ec71a7f1769ad74374ae3e2e939 Mon Sep 17 00:00:00 2001 From: Elektr0Vodka <211697683+Elektr0Vodka@users.noreply.github.com> Date: Mon, 14 Sep 2026 04:15:20 +0200 Subject: [PATCH 12/23] feat(ota): pass runtime manifest base through otaFromManifest (cherry picked from commit 0bee84c0546780b2e7147148c32087e92d894ac0) --- examples/simple_repeater/MyMesh.cpp | 3 ++- examples/simple_room_server/MyMesh.cpp | 3 ++- src/MeshCore.h | 2 +- src/helpers/CommonCLI_Observer.cpp | 5 +++-- src/helpers/ESP32Board.cpp | 19 ++++++++++--------- src/helpers/ESP32Board.h | 4 ++-- 6 files changed, 20 insertions(+), 16 deletions(-) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index 7e209cfb..b3789657 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -2,6 +2,7 @@ #include #include // for qsort() #include +#include #if defined(WITH_MQTT_NEIGHBORS) #include // kSyncedClockEpoch #endif @@ -1740,7 +1741,7 @@ void MyMesh::loop() { Serial.println("OTA: aborted, MQTT stop did not complete cleanly - resuming bridge"); otaAlert("OTA aborted: MQTT stop unclean, bridge resumed"); setBridgeState(true); - } else if (!_cli.getBoard()->otaFromManifest(getFirmwareVer(), false, ota_reply)) { + } else if (!_cli.getBoard()->otaFromManifest(ota_resolve_base(_prefs.ota_channel), getFirmwareVer(), false, ota_reply)) { Serial.print("OTA: aborted, resuming bridge - "); Serial.println(ota_reply); char ota_alert_msg[160]; snprintf(ota_alert_msg, sizeof(ota_alert_msg), "OTA aborted: %s", ota_reply); diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 9fe5a9c8..2e97d491 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -1,6 +1,7 @@ #include "MyMesh.h" #include #include +#include #if defined(WITH_MQTT_NEIGHBORS) #include // kSyncedClockEpoch #endif @@ -1616,7 +1617,7 @@ void MyMesh::loop() { } char ota_reply[160]; - if (may_flash && !_cli.getBoard()->otaFromManifest(getFirmwareVer(), false, ota_reply)) { + if (may_flash && !_cli.getBoard()->otaFromManifest(ota_resolve_base(_prefs.ota_channel), getFirmwareVer(), false, ota_reply)) { Serial.print("OTA: aborted - "); Serial.println(ota_reply); may_flash = false; } diff --git a/src/MeshCore.h b/src/MeshCore.h index 15ccdc15..32d66973 100644 --- a/src/MeshCore.h +++ b/src/MeshCore.h @@ -67,7 +67,7 @@ public: // Pull-based OTA: fetch the firmware build for this variant from a baked-in manifest and flash it. // current_ver is the running firmware version string (used to skip if already up to date); when // dry_run is true the build is only reported, not flashed. Observer (ESP32+WiFi) builds only. - virtual bool otaFromManifest(const char* current_ver, bool dry_run, char reply[]) { return false; } + virtual bool otaFromManifest(const char* manifest_base, const char* current_ver, bool dry_run, char reply[]) { return false; } // LoRa front-end-module LNA (RX gain) control. Only FEM-equipped boards override // these; others report they can't control it. Driven by NodePrefs.radio_fem_rxgain. diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index c1b7e74a..c6d44f49 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -12,6 +12,7 @@ #include #include "CommonCLI.h" +#include "OtaChannel.h" #include "TxtDataHelpers.h" #include "AlertReporter.h" // for alertReporterBannedChannelMatch[Hex]() #include "MQTTObserverValidation.h" // pure input validators (host-testable) @@ -1088,7 +1089,7 @@ bool CommonCLI::handleObserverCommand(uint32_t sender_timestamp, char* command, // MQTT bridge UP: the slim per-variant manifest is tiny, so the fetch only // costs a single TLS handshake (no large JSON doc) — which fits alongside // the live MQTT sessions even on no-PSRAM boards. No bridge bounce needed. - _board->otaFromManifest(_callbacks->getFirmwareVer(), true, reply); + _board->otaFromManifest(ota_resolve_base(_prefs->ota_channel), _callbacks->getFirmwareVer(), true, reply); } else { // `ota update`: cheap pre-check first (plain HTTP, bridge stays up). Only // schedule the real update — which tears the bridge down, flashes, and @@ -1096,7 +1097,7 @@ bool CommonCLI::handleObserverCommand(uint32_t sender_timestamp, char* command, // returns true iff so; otherwise it leaves the explanation (up to date / // cable flash / error) in reply, which we send without disturbing the // bridge or misleading the user with a "Beginning update..." that no-ops. - if (_board->otaFromManifest(_callbacks->getFirmwareVer(), true, reply)) { + if (_board->otaFromManifest(ota_resolve_base(_prefs->ota_channel), _callbacks->getFirmwareVer(), true, reply)) { // reply now holds "update available: -> (N behind|new base)", // where is "vX.Y.Z.B (hash)". Pull out for a friendlier // start message. The "-> " ... trailing " (" framing is produced by diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 120203ab..36db7fa9 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -162,6 +162,7 @@ static void ota_partitionSignature(char* out, size_t out_sz) { // which stays valid because that function blocks until the worker signals done. struct OtaTaskArgs { ESP32Board* self; + const char* manifest_base; const char* current_ver; bool dry_run; char* reply; @@ -171,18 +172,18 @@ struct OtaTaskArgs { static void ota_task_entry(void* param) { OtaTaskArgs* a = static_cast(param); - a->result = a->self->otaFromManifestImpl(a->current_ver, a->dry_run, a->reply); + a->result = a->self->otaFromManifestImpl(a->manifest_base, a->current_ver, a->dry_run, a->reply); a->done = true; // on a successful `ota update` we reboot before reaching here vTaskDelete(nullptr); } -bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char reply[]) { +bool ESP32Board::otaFromManifest(const char* manifest_base, const char* current_ver, bool dry_run, char reply[]) { // The TLS handshake (cert-bundle verify) + JSON parse / HTTPUpdate use far more // stack than the ~8 KB loop task offers — especially when reached via the deep // mesh-receive call chain (it overflows the loopTask canary). Run the work in a // dedicated 24 KB-stack task and block here until it finishes. The big stack is // freed when the task exits; on a successful update the chip reboots inside it. - OtaTaskArgs args = { this, current_ver, dry_run, reply, false, false }; + OtaTaskArgs args = { this, manifest_base, current_ver, dry_run, reply, false, false }; TaskHandle_t handle = nullptr; BaseType_t ok = xTaskCreatePinnedToCore(ota_task_entry, "ota", 24576, &args, 5, &handle, 1); if (ok != pdPASS) { @@ -195,7 +196,7 @@ bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char rep return args.result; } -bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char reply[]) { +bool ESP32Board::otaFromManifestImpl(const char* manifest_base, const char* current_ver, bool dry_run, char reply[]) { #if !defined(OTA_MANIFEST_BASE) || !defined(OTA_VARIANT) strcpy(reply, "ERR: OTA not configured (build via build.sh)"); return false; @@ -230,10 +231,10 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char // and the handshake + the bridge both fail). This only reads version info; the // firmware download below (ota update) is always TLS-verified. Requires the // manifest host to serve /v over HTTP (no forced HTTPS redirect). - if (strncmp(OTA_MANIFEST_BASE, "https://", 8) == 0) { - snprintf(murl, sizeof(murl), "http://%s/%s.json", OTA_MANIFEST_BASE + 8, OTA_VARIANT); + if (strncmp(manifest_base, "https://", 8) == 0) { + snprintf(murl, sizeof(murl), "http://%s/%s.json", manifest_base + 8, OTA_VARIANT); } else { - snprintf(murl, sizeof(murl), "%s/%s.json", OTA_MANIFEST_BASE, OTA_VARIANT); + snprintf(murl, sizeof(murl), "%s/%s.json", manifest_base, OTA_VARIANT); } if (!http.begin(murl)) { strcpy(reply, "ERR: manifest connect failed"); @@ -248,7 +249,7 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char mclient.setCACertBundle(rootca_crt_bundle_start); #endif mclient.setTimeout(15000); - snprintf(murl, sizeof(murl), "%s/%s.json", OTA_MANIFEST_BASE, OTA_VARIANT); + snprintf(murl, sizeof(murl), "%s/%s.json", manifest_base, OTA_VARIANT); if (!http.begin(mclient, murl)) { strcpy(reply, "ERR: manifest connect failed"); return false; @@ -417,7 +418,7 @@ bool ESP32Board::otaFromManifestImpl(const char* current_ver, bool dry_run, char #endif // OTA_MANIFEST_BASE && OTA_VARIANT } #else -bool ESP32Board::otaFromManifest(const char* current_ver, bool dry_run, char reply[]) { +bool ESP32Board::otaFromManifest(const char* manifest_base, const char* current_ver, bool dry_run, char reply[]) { strcpy(reply, "ERR: not supported"); return false; } diff --git a/src/helpers/ESP32Board.h b/src/helpers/ESP32Board.h index 3ecf6735..af819719 100644 --- a/src/helpers/ESP32Board.h +++ b/src/helpers/ESP32Board.h @@ -155,11 +155,11 @@ public: } bool startOTAUpdate(const char* id, char reply[], bool force_ap = false) override; - bool otaFromManifest(const char* current_ver, bool dry_run, char reply[]) override; + bool otaFromManifest(const char* manifest_base, const char* current_ver, bool dry_run, char reply[]) override; // Heavy body (TLS + JSON / HTTPUpdate). Runs in a dedicated large-stack task // spawned by otaFromManifest() — public only so that task entry point can call // it; not meant to be invoked directly. - bool otaFromManifestImpl(const char* current_ver, bool dry_run, char reply[]); + bool otaFromManifestImpl(const char* manifest_base, const char* current_ver, bool dry_run, char reply[]); void setInhibitSleep(bool inhibit) { inhibit_sleep = inhibit; From ea541e182a20ee89c977f02e375e0ba77d2caae0 Mon Sep 17 00:00:00 2001 From: Elektr0Vodka <211697683+Elektr0Vodka@users.noreply.github.com> Date: Mon, 14 Sep 2026 04:29:28 +0200 Subject: [PATCH 13/23] feat(ota): add 'ota branch [stable|dev|default]' command (cherry picked from commit 240804999edb284fd276489e6898ccad8a19a8e7) --- src/helpers/CommonCLI_Observer.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index c6d44f49..e4d71182 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -1130,6 +1130,35 @@ bool CommonCLI::handleObserverCommand(uint32_t sender_timestamp, char* command, } #else strcpy(reply, "ERR: online OTA not supported on this build"); +#endif + return true; + } else if (memcmp(command, "ota branch", 10) == 0) { + // Switch (or report) the OTA release channel this device pulls from. The + // selection is persisted (NodePrefs::ota_channel) and resolved to a baked-in + // base URL by ota_resolve_base(); it changes only WHERE updates are fetched, + // never the running image's reported version. Reachable from any admin path, + // same as `ota update`. +#if defined(WITH_MQTT_BRIDGE) && defined(OTA_MANIFEST_BASE) + const char* arg = command + 10; + while (*arg == ' ') arg++; + if (*arg == 0) { + snprintf(reply, 160, "channel: %s (%s), base %s", + ota_channel_name(_prefs->ota_channel), + _prefs->ota_channel == OTA_CH_NATIVE ? "native" : "override", + ota_resolve_base(_prefs->ota_channel)); + } else { + uint8_t ch; + if (!ota_parse_channel(arg, &ch)) { + strcpy(reply, "ERR: usage ota branch [stable|dev|default]"); + } else { + _prefs->ota_channel = ch; + savePrefs(); + snprintf(reply, 160, "channel set to %s, base %s", + ota_channel_name(ch), ota_resolve_base(ch)); + } + } +#else + strcpy(reply, "ERR: online OTA not supported on this build"); #endif return true; } else if (memcmp(command, "start webconfig", 15) == 0 && (command[15] == 0 || command[15] == ' ')) { From 763ba835c18379784ce37f1dfba9d3941b384310 Mon Sep 17 00:00:00 2001 From: Elektr0Vodka <211697683+Elektr0Vodka@users.noreply.github.com> Date: Mon, 14 Sep 2026 04:29:28 +0200 Subject: [PATCH 14/23] build(ota): bake stable + dev manifest bases into observer builds (cherry picked from commit 29011959170db757ec5364de88f6beeaae82b995) --- build.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 5d6ae443..a6ad41c6 100755 --- a/build.sh +++ b/build.sh @@ -203,10 +203,18 @@ build_firmware() { # the .ini declarations were removed rather than overridden. OTA_MANIFEST_BASE_URL="${OTA_MANIFEST_BASE_URL:-https://observer.gessaman.com/v}" + # Both named channel bases are baked into EVERY observer build so `ota branch` + # can re-point a device at either channel at runtime. OTA_MANIFEST_BASE above stays + # the build's NATIVE channel (= stable base for stable builds, dev base for dev + # builds), so `ota branch default` resolves correctly. These two must match the + # paths DutchMeshCore-OTA serves (feat/dev-stable-ota-channels): /mqtt/v + /mqtt/dev/v. + OTA_MANIFEST_BASE_STABLE_URL="${OTA_MANIFEST_BASE_STABLE_URL:-https://ota.dutchmeshcore.nl/mqtt/v}" + OTA_MANIFEST_BASE_DEV_URL="${OTA_MANIFEST_BASE_DEV_URL:-https://ota.dutchmeshcore.nl/mqtt/dev/v}" + # add firmware version info to end of existing platformio build flags in environment vars. # OTA_VARIANT is the env name ($1) — it selects this build's slim per-variant manifest # (/.json) that the observer pull-OTA fetches. - export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DFIRMWARE_BUILD_DATE='\"${FIRMWARE_BUILD_DATE}\"' -DFIRMWARE_VERSION='\"${EMBEDDED_VERSION_STRING}\"' -DOTA_VARIANT='\"$1\"' -DOTA_MANIFEST_BASE='\"${OTA_MANIFEST_BASE_URL}\"'" + export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DFIRMWARE_BUILD_DATE='\"${FIRMWARE_BUILD_DATE}\"' -DFIRMWARE_VERSION='\"${EMBEDDED_VERSION_STRING}\"' -DOTA_VARIANT='\"$1\"' -DOTA_MANIFEST_BASE='\"${OTA_MANIFEST_BASE_URL}\"' -DOTA_MANIFEST_BASE_STABLE='\"${OTA_MANIFEST_BASE_STABLE_URL}\"' -DOTA_MANIFEST_BASE_DEV='\"${OTA_MANIFEST_BASE_DEV_URL}\"'" # disable debug flags if requested disable_debug_flags From 2633b4267cdabfc351ce8b3aa9c59ab020497cec Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 3 Oct 2026 15:17:46 -0700 Subject: [PATCH 15/23] fix(ota): compare by commit hash when targeting another channel Build counters are per channel, so a cross-channel build number is not comparable: a switch to a channel with a lower counter reported up to date and never happened. Off the native channel, update unless the hash matches. --- src/helpers/ESP32Board.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 36db7fa9..206a3a7c 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -334,10 +334,14 @@ bool ESP32Board::otaFromManifestImpl(const char* manifest_base, const char* curr bool same_base = (own_base[0] && avail_base[0] && strcmp(own_base, avail_base) == 0); bool have_builds = (own_build >= 0 && avail_build >= 0); bool diff_base = (own_base[0] && avail_base[0] && !same_base); + // Build counters are per channel, so build numbers only compare within the native channel. + bool cross_channel = (strcmp(manifest_base, OTA_MANIFEST_BASE) != 0); int behind = 0; bool up_to_date; - if (same_base && have_builds) { + if (cross_channel) { + up_to_date = hash_equal; + } else if (same_base && have_builds) { behind = avail_build - own_build; up_to_date = (behind <= 0); } else if (diff_base) { @@ -362,6 +366,8 @@ bool ESP32Board::otaFromManifestImpl(const char* manifest_base, const char* curr if (dry_run) { if (up_to_date) { snprintf(reply, 160, "up to date: %s", avail_disp); + } else if (cross_channel) { + snprintf(reply, 160, "update available: %s -> %s (channel switch)%s", own_disp, avail_disp, pc_note); } else if (same_base && have_builds) { snprintf(reply, 160, "update available: %s -> %s (%d behind)%s", own_disp, avail_disp, behind, pc_note); } else if (diff_base) { From 4afb3f7e3f0a03b58b770a6403eba5a25bbf6f87 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 3 Oct 2026 15:17:46 -0700 Subject: [PATCH 16/23] build(ota): point the stable/dev channel bases at observer.gessaman.com --- build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.sh b/build.sh index a6ad41c6..1d6783fb 100755 --- a/build.sh +++ b/build.sh @@ -206,10 +206,10 @@ build_firmware() { # Both named channel bases are baked into EVERY observer build so `ota branch` # can re-point a device at either channel at runtime. OTA_MANIFEST_BASE above stays # the build's NATIVE channel (= stable base for stable builds, dev base for dev - # builds), so `ota branch default` resolves correctly. These two must match the - # paths DutchMeshCore-OTA serves (feat/dev-stable-ota-channels): /mqtt/v + /mqtt/dev/v. - OTA_MANIFEST_BASE_STABLE_URL="${OTA_MANIFEST_BASE_STABLE_URL:-https://ota.dutchmeshcore.nl/mqtt/v}" - OTA_MANIFEST_BASE_DEV_URL="${OTA_MANIFEST_BASE_DEV_URL:-https://ota.dutchmeshcore.nl/mqtt/dev/v}" + # builds), so `ota branch default` resolves correctly. These must match the + # production (/v) and beta (/beta/v) manifest paths. + OTA_MANIFEST_BASE_STABLE_URL="${OTA_MANIFEST_BASE_STABLE_URL:-https://observer.gessaman.com/v}" + OTA_MANIFEST_BASE_DEV_URL="${OTA_MANIFEST_BASE_DEV_URL:-https://observer.gessaman.com/beta/v}" # add firmware version info to end of existing platformio build flags in environment vars. # OTA_VARIANT is the env name ($1) — it selects this build's slim per-variant manifest From 9c65e198f1ab1b44f608fdb5df25956eb513e9f4 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 3 Oct 2026 15:19:06 -0700 Subject: [PATCH 17/23] feat(ota): prod/beta channel names for ota branch, with stable/dev aliases Report the selected channel and the build's own channel, tag each baked-in base for CI, and list ota branch in the CLI docs and portal autocomplete. --- docs/cli_commands.md | 6 ++ src/helpers/CommonCLI_Observer.cpp | 11 ++-- src/helpers/OtaChannel.h | 68 +++++++++++++--------- test/test_ota_channel/test_ota_channel.cpp | 22 +++++-- webui/index.html | 2 + 5 files changed, 72 insertions(+), 37 deletions(-) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 057fc0e2..5267f530 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -88,6 +88,12 @@ This document provides an overview of CLI commands that can be sent to MeshCore - `start ota` — serves the ElegantOTA web upload page on the station IP if joined to a Wi-Fi network, otherwise raises the `MeshCore-OTA` Wi-Fi hotspot. - `start ota ap` — always raises the `MeshCore-OTA` Wi-Fi hotspot, even when joined to a network. Use this when the network applies client isolation and the station IP isn't reachable. +### Switch the OTA release channel (observer builds) + +- `ota branch` — shows the selected channel, the channel this build was made for, and the manifest base `ota check`/`ota update` will use. +- `ota branch prod` (alias `stable`) / `ota branch beta` (alias `dev`) — pull future `ota update`s from that channel. The selection is saved; run `ota update` to switch. +- `ota branch default` — follow the channel this build was made for. + --- ### Erase/Factory Reset diff --git a/src/helpers/CommonCLI_Observer.cpp b/src/helpers/CommonCLI_Observer.cpp index e4d71182..5ef7287a 100644 --- a/src/helpers/CommonCLI_Observer.cpp +++ b/src/helpers/CommonCLI_Observer.cpp @@ -1132,7 +1132,7 @@ bool CommonCLI::handleObserverCommand(uint32_t sender_timestamp, char* command, strcpy(reply, "ERR: online OTA not supported on this build"); #endif return true; - } else if (memcmp(command, "ota branch", 10) == 0) { + } else if (memcmp(command, "ota branch", 10) == 0 && (command[10] == 0 || command[10] == ' ')) { // Switch (or report) the OTA release channel this device pulls from. The // selection is persisted (NodePrefs::ota_channel) and resolved to a baked-in // base URL by ota_resolve_base(); it changes only WHERE updates are fetched, @@ -1142,18 +1142,17 @@ bool CommonCLI::handleObserverCommand(uint32_t sender_timestamp, char* command, const char* arg = command + 10; while (*arg == ' ') arg++; if (*arg == 0) { - snprintf(reply, 160, "channel: %s (%s), base %s", - ota_channel_name(_prefs->ota_channel), - _prefs->ota_channel == OTA_CH_NATIVE ? "native" : "override", + snprintf(reply, 160, "channel: %s (build: %s), base %s", + ota_channel_name(_prefs->ota_channel), ota_native_channel_name(), ota_resolve_base(_prefs->ota_channel)); } else { uint8_t ch; if (!ota_parse_channel(arg, &ch)) { - strcpy(reply, "ERR: usage ota branch [stable|dev|default]"); + strcpy(reply, "ERR: usage ota branch [prod|beta|default] (aliases: stable, dev)"); } else { _prefs->ota_channel = ch; savePrefs(); - snprintf(reply, 160, "channel set to %s, base %s", + snprintf(reply, 160, "channel set to %s, base %s; run ota update to switch", ota_channel_name(ch), ota_resolve_base(ch)); } } diff --git a/src/helpers/OtaChannel.h b/src/helpers/OtaChannel.h index d589854b..2918d83d 100644 --- a/src/helpers/OtaChannel.h +++ b/src/helpers/OtaChannel.h @@ -5,36 +5,44 @@ // OTA release-channel selector, persisted in NodePrefs::ota_channel. enum OtaChannel : uint8_t { OTA_CH_NATIVE = 0, // follow the channel this build was made for - OTA_CH_STABLE = 1, - OTA_CH_DEV = 2, + OTA_CH_STABLE = 1, // production + OTA_CH_DEV = 2, // beta }; // Resolve the effective manifest base URL for a channel selector. // build.sh injects the three bases as compile-time macros: // OTA_MANIFEST_BASE = this build's native channel (defined on every OTA build) -// OTA_MANIFEST_BASE_STABLE = stable channel -// OTA_MANIFEST_BASE_DEV = dev channel +// OTA_MANIFEST_BASE_STABLE = stable (production) channel +// OTA_MANIFEST_BASE_DEV = dev (beta) channel // stable/dev fall back to the native base when their macro is undefined (legacy/local // builds that only define OTA_MANIFEST_BASE), so this never returns nullptr on an // OTA-capable build. On a non-OTA build it returns nullptr. +// +// Each base is stored behind an "ota-base-:" tag so CI can read a binary's +// channels back with `strings` (scripts/verify_ota_channel.sh). Returning a pointer +// into the tagged array keeps the tag referenced, so the linker cannot drop it. +#if defined(OTA_MANIFEST_BASE) +#define OTA_BASE_TAG_NATIVE "ota-base-native:" +#define OTA_BASE_TAG_STABLE "ota-base-stable:" +#define OTA_BASE_TAG_DEV "ota-base-dev:" +#if !defined(OTA_MANIFEST_BASE_STABLE) +#define OTA_MANIFEST_BASE_STABLE OTA_MANIFEST_BASE +#endif +#if !defined(OTA_MANIFEST_BASE_DEV) +#define OTA_MANIFEST_BASE_DEV OTA_MANIFEST_BASE +#endif +static const char ota_tagged_native[] = OTA_BASE_TAG_NATIVE OTA_MANIFEST_BASE; +static const char ota_tagged_stable[] = OTA_BASE_TAG_STABLE OTA_MANIFEST_BASE_STABLE; +static const char ota_tagged_dev[] = OTA_BASE_TAG_DEV OTA_MANIFEST_BASE_DEV; +#endif + static inline const char* ota_resolve_base(uint8_t channel) { #if defined(OTA_MANIFEST_BASE) switch (channel) { - case OTA_CH_STABLE: -#if defined(OTA_MANIFEST_BASE_STABLE) - return OTA_MANIFEST_BASE_STABLE; -#else - return OTA_MANIFEST_BASE; -#endif - case OTA_CH_DEV: -#if defined(OTA_MANIFEST_BASE_DEV) - return OTA_MANIFEST_BASE_DEV; -#else - return OTA_MANIFEST_BASE; -#endif + case OTA_CH_STABLE: return ota_tagged_stable + sizeof(OTA_BASE_TAG_STABLE) - 1; + case OTA_CH_DEV: return ota_tagged_dev + sizeof(OTA_BASE_TAG_DEV) - 1; case OTA_CH_NATIVE: - default: - return OTA_MANIFEST_BASE; + default: return ota_tagged_native + sizeof(OTA_BASE_TAG_NATIVE) - 1; } #else (void)channel; @@ -45,18 +53,26 @@ static inline const char* ota_resolve_base(uint8_t channel) { // Human label for a selector (for the `ota branch` report). static inline const char* ota_channel_name(uint8_t channel) { switch (channel) { - case OTA_CH_STABLE: return "stable"; - case OTA_CH_DEV: return "dev"; - default: return "native"; + case OTA_CH_STABLE: return "prod"; + case OTA_CH_DEV: return "beta"; + default: return "default"; } } +// Label for the channel this build was made for, by matching its native base. +static inline const char* ota_native_channel_name() { + const char* native = ota_resolve_base(OTA_CH_NATIVE); + if (native == nullptr) return "none"; + if (strcmp(native, ota_resolve_base(OTA_CH_STABLE)) == 0) return "prod"; + if (strcmp(native, ota_resolve_base(OTA_CH_DEV)) == 0) return "beta"; + return "custom"; +} + // Parse an `ota branch` argument. Returns true and sets *out on a known keyword -// (stable|dev|default; "default" -> native); returns false and leaves *out untouched -// otherwise. +// (prod|stable, beta|dev, default); returns false and leaves *out untouched otherwise. static inline bool ota_parse_channel(const char* arg, uint8_t* out) { - if (strcmp(arg, "stable") == 0) { *out = OTA_CH_STABLE; return true; } - if (strcmp(arg, "dev") == 0) { *out = OTA_CH_DEV; return true; } - if (strcmp(arg, "default") == 0) { *out = OTA_CH_NATIVE; return true; } + if (strcmp(arg, "prod") == 0 || strcmp(arg, "stable") == 0) { *out = OTA_CH_STABLE; return true; } + if (strcmp(arg, "beta") == 0 || strcmp(arg, "dev") == 0) { *out = OTA_CH_DEV; return true; } + if (strcmp(arg, "default") == 0) { *out = OTA_CH_NATIVE; return true; } return false; } diff --git a/test/test_ota_channel/test_ota_channel.cpp b/test/test_ota_channel/test_ota_channel.cpp index 699ffb70..d2a4e482 100644 --- a/test/test_ota_channel/test_ota_channel.cpp +++ b/test/test_ota_channel/test_ota_channel.cpp @@ -11,21 +11,33 @@ TEST(OtaChannel, ResolvesStable) { TEST(OtaChannel, ResolvesDev) { EXPECT_STREQ(ota_resolve_base(OTA_CH_DEV), "https://dev.example/mqtt/dev/v"); } +TEST(OtaChannel, BasesAreStoredBehindCiTags) { + EXPECT_STREQ(ota_tagged_native, "ota-base-native:https://stable.example/mqtt/v"); + EXPECT_STREQ(ota_tagged_stable, "ota-base-stable:https://stable.example/mqtt/v"); + EXPECT_STREQ(ota_tagged_dev, "ota-base-dev:https://dev.example/mqtt/dev/v"); +} TEST(OtaChannel, ParseKnownKeywords) { uint8_t ch = 99; + EXPECT_TRUE(ota_parse_channel("prod", &ch)); EXPECT_EQ(ch, OTA_CH_STABLE); EXPECT_TRUE(ota_parse_channel("stable", &ch)); EXPECT_EQ(ch, OTA_CH_STABLE); - EXPECT_TRUE(ota_parse_channel("dev", &ch)); EXPECT_EQ(ch, OTA_CH_DEV); + EXPECT_TRUE(ota_parse_channel("beta", &ch)); EXPECT_EQ(ch, OTA_CH_DEV); EXPECT_TRUE(ota_parse_channel("default", &ch)); EXPECT_EQ(ch, OTA_CH_NATIVE); + EXPECT_TRUE(ota_parse_channel("dev", &ch)); EXPECT_EQ(ch, OTA_CH_DEV); } TEST(OtaChannel, ParseRejectsUnknownAndLeavesOutputUntouched) { uint8_t ch = 7; - EXPECT_FALSE(ota_parse_channel("beta", &ch)); + EXPECT_FALSE(ota_parse_channel("production", &ch)); + EXPECT_FALSE(ota_parse_channel("Beta", &ch)); + EXPECT_FALSE(ota_parse_channel("", &ch)); EXPECT_EQ(ch, 7); } TEST(OtaChannel, NameLabels) { - EXPECT_STREQ(ota_channel_name(OTA_CH_NATIVE), "native"); - EXPECT_STREQ(ota_channel_name(OTA_CH_STABLE), "stable"); - EXPECT_STREQ(ota_channel_name(OTA_CH_DEV), "dev"); + EXPECT_STREQ(ota_channel_name(OTA_CH_NATIVE), "default"); + EXPECT_STREQ(ota_channel_name(OTA_CH_STABLE), "prod"); + EXPECT_STREQ(ota_channel_name(OTA_CH_DEV), "beta"); +} +TEST(OtaChannel, NativeChannelNameMatchesBase) { + EXPECT_STREQ(ota_native_channel_name(), "prod"); } int main(int argc, char** argv) { diff --git a/webui/index.html b/webui/index.html index ab7b43d6..30cf4cea 100644 --- a/webui/index.html +++ b/webui/index.html @@ -1495,6 +1495,8 @@ var CLI_VERBS=[ ["alert test","Send a test alert on the configured channel"], ["ota check","Check for a newer build (does not flash)"], ["ota update","Download and flash the newer build, then reboot"], + ["ota branch","Show the OTA release channel"], + ["ota branch ","Set the OTA release channel {prod|beta|default}"], // `start ota` is absent: it binds port 80, which this portal is already using. ["start webconfig","Start this portal on the LAN"], ["start webconfig ap","Start this portal on its own setup AP"], From 65d80bc908a721e76ffdd60499c305aa47205328 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 3 Oct 2026 15:24:11 -0700 Subject: [PATCH 18/23] ci(ota): verify each published binary's native channel by tag Every observer build now carries both channel URLs, so checking for a URL's presence or absence can no longer tell prod from beta (and the old checks would reject every build). Read the ota-base-* tags from every .bin in out/ instead of a single ELF, with URLs shared by build.sh and both workflows. --- .../build-observer-firmwares-beta.yml | 21 +-- .../workflows/build-observer-firmwares.yml | 15 +++ scripts/verify_ota_channel.py | 125 ++++++++++++++++++ 3 files changed, 147 insertions(+), 14 deletions(-) create mode 100644 scripts/verify_ota_channel.py diff --git a/.github/workflows/build-observer-firmwares-beta.yml b/.github/workflows/build-observer-firmwares-beta.yml index 8618da4b..0fa2c9fd 100644 --- a/.github/workflows/build-observer-firmwares-beta.yml +++ b/.github/workflows/build-observer-firmwares-beta.yml @@ -67,6 +67,10 @@ env: # The channel itself. Firmware fetches /.json, # so this URL is what keeps beta nodes on beta. OTA_MANIFEST_BASE_URL: https://observer.gessaman.com/beta/v + # Both channel bases are baked into every build so `ota branch prod|beta` can + # move a node between channels. Identical in both observer workflows. + OTA_MANIFEST_BASE_STABLE_URL: https://observer.gessaman.com/v + OTA_MANIFEST_BASE_DEV_URL: https://observer.gessaman.com/beta/v # Marks the embedded version, e.g. v1.16.0.3-observer-beta-dev-abc1234, so `ver` # (and the MQTT firmware_version / SNMP) identify BOTH the channel and its # provenance: this channel is built from the upstream-dev-merged line, so "dev" @@ -168,20 +172,9 @@ jobs: run: /usr/bin/env bash build.sh build-firmware ${{ matrix.shard.envs }} - name: Verify beta channel is baked in - shell: bash - run: | - # Fail fast rather than publish firmware that would OTA itself onto the - # production channel. Checks one built binary actually carries the beta - # manifest URL and does NOT carry the production one. - BIN=$(find .pio/build -name firmware.elf | head -1) - if [ -z "$BIN" ]; then echo "no ELF found to verify" >&2; exit 1; fi - if ! strings "$BIN" | grep -qF "$OTA_MANIFEST_BASE_URL"; then - echo "ERROR: beta manifest base missing from $BIN" >&2; exit 1 - fi - if strings "$BIN" | grep -qE 'https://observer\.gessaman\.com/v"?$'; then - echo "ERROR: production manifest base present in a beta build" >&2; exit 1 - fi - echo "OK: $BIN carries $OTA_MANIFEST_BASE_URL" + # Fail fast rather than publish firmware that would OTA itself onto the + # production channel: checks every .bin the shard publishes. + run: python3 scripts/verify_ota_channel.py --expect beta - name: Upload Shard Artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/build-observer-firmwares.yml b/.github/workflows/build-observer-firmwares.yml index 13d9cda0..9aebe03f 100644 --- a/.github/workflows/build-observer-firmwares.yml +++ b/.github/workflows/build-observer-firmwares.yml @@ -48,6 +48,16 @@ env: # repo). Baked into the slim OTA manifests' file URLs; must stay consistent # with config.json's staticPath. STATIC_PATH: https://observer-fw.gessaman.com + # Where the firmware fetches .json from — this URL *is* the release + # channel. Stated explicitly (it equals build.sh's default) rather than left + # unset, so "Verify production channel is baked in" below can assert against the + # same value the build was handed instead of a second hardcoded copy. Must match + # build.sh's OTA_MANIFEST_BASE_URL default and the Pages path serving flasher/v. + OTA_MANIFEST_BASE_URL: https://observer.gessaman.com/v + # Both channel bases are baked into every build so `ota branch prod|beta` can + # move a node between channels. Identical in both observer workflows. + OTA_MANIFEST_BASE_STABLE_URL: https://observer.gessaman.com/v + OTA_MANIFEST_BASE_DEV_URL: https://observer.gessaman.com/beta/v jobs: @@ -137,6 +147,11 @@ jobs: FIRMWARE_BUILD_NUMBER: ${{ needs.enumerate.outputs.build_number }} run: /usr/bin/env bash build.sh build-firmware ${{ matrix.shard.envs }} + - name: Verify production channel is baked in + # Fail fast rather than publish firmware that would OTA itself onto the + # wrong channel: checks every .bin the shard publishes. + run: python3 scripts/verify_ota_channel.py --expect prod + - name: Upload Shard Artifact uses: actions/upload-artifact@v4 with: diff --git a/scripts/verify_ota_channel.py b/scripts/verify_ota_channel.py new file mode 100644 index 00000000..15d91c26 --- /dev/null +++ b/scripts/verify_ota_channel.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Verify every built observer binary is baked for the expected OTA channel. + +Observer builds carry three manifest bases (src/helpers/OtaChannel.h), each stored +behind a tag so it can be read back from the binary: + + ota-base-native: the channel the build OTAs from by default + ota-base-stable: production, the target of `ota branch prod` + ota-base-dev: beta, the target of `ota branch beta` + +Every build contains both channel URLs, so checking for the presence or absence +of a URL can no longer tell production from beta. This reads the tags instead and +fails unless, in every .bin in out/ (the files that get published), each tag appears with exactly one value, the stable +and dev tags match the workflow's URLs, and the native tag is the URL of the +channel named by --expect. + + python3 scripts/verify_ota_channel.py --expect prod + python3 scripts/verify_ota_channel.py --self-test + +URLs default to OTA_MANIFEST_BASE_URL, OTA_MANIFEST_BASE_STABLE_URL and +OTA_MANIFEST_BASE_DEV_URL from the environment. Stdlib only. +""" +import argparse +import os +import re +import sys + +TAG_RE = re.compile(rb"ota-base-(native|stable|dev):([\x21-\x7e]*)\x00") + + +def read_tags(data): + tags = {"native": set(), "stable": set(), "dev": set()} + for m in TAG_RE.finditer(data): + tags[m.group(1).decode()].add(m.group(2).decode()) + return tags + + +def check(data, expect, native_url, stable_url, dev_url): + """Return a list of problems with one binary's tags (empty when it passes).""" + problems = [] + if stable_url == dev_url: + problems.append("stable and dev URLs are identical (%s)" % stable_url) + expected_native = stable_url if expect == "prod" else dev_url + if native_url != expected_native: + problems.append("OTA_MANIFEST_BASE_URL %s is not the %s URL %s" + % (native_url, expect, expected_native)) + tags = read_tags(data) + for name, want in (("native", expected_native), ("stable", stable_url), ("dev", dev_url)): + found = sorted(tags[name]) + if found != [want]: + problems.append("ota-base-%s: expected [%s], found %s" % (name, want, found or "nothing")) + return problems + + +def find_bins(root): + for name in os.listdir(root): + if name.endswith(".bin"): + yield os.path.join(root, name) + + +def self_test(): + P, B = "https://h/v", "https://h/beta/v" + + def blob(native, stable=P, dev=B): + return b"\x00junk\x00ota-base-native:%s\x00ota-base-stable:%s\x00ota-base-dev:%s\x00" % ( + native.encode(), stable.encode(), dev.encode()) + + cases = [ + ("prod build passes", blob(P), "prod", P, True), + ("beta build passes", blob(B), "beta", B, True), + ("beta build fails prod check", blob(B), "prod", P, False), + ("prod build fails beta check", blob(P), "beta", B, False), + ("missing tags fail", b"\x00https://h/v\x00https://h/beta/v\x00", "prod", P, False), + ("wrong dev URL fails", blob(P, dev="https://other/v"), "prod", P, False), + ("conflicting native tags fail", blob(P) + blob(B), "prod", P, False), + ("workflow URL off-channel fails", blob(P), "prod", B, False), + ] + failed = 0 + for name, data, expect, native, ok in cases: + got = not check(data, expect, native, P, B) + if got != ok: + failed += 1 + print("FAIL: %s" % name) + print("self-test: %d/%d passed" % (len(cases) - failed, len(cases))) + return 1 if failed else 0 + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--expect", choices=("prod", "beta")) + ap.add_argument("--bin-dir", default="out") + ap.add_argument("--native-url", default=os.environ.get("OTA_MANIFEST_BASE_URL")) + ap.add_argument("--stable-url", default=os.environ.get("OTA_MANIFEST_BASE_STABLE_URL")) + ap.add_argument("--dev-url", default=os.environ.get("OTA_MANIFEST_BASE_DEV_URL")) + ap.add_argument("--self-test", action="store_true") + args = ap.parse_args() + + if args.self_test: + return self_test() + if not args.expect: + ap.error("--expect is required") + for opt in ("native_url", "stable_url", "dev_url"): + if not getattr(args, opt): + ap.error("--%s (or its environment variable) is required" % opt.replace("_", "-")) + + bins = sorted(find_bins(args.bin_dir)) if os.path.isdir(args.bin_dir) else [] + if not bins: + print("ERROR: no .bin files in %s" % args.bin_dir, file=sys.stderr) + return 1 + bad = 0 + for path in bins: + with open(path, "rb") as f: + problems = check(f.read(), args.expect, args.native_url, args.stable_url, args.dev_url) + for p in problems: + print("ERROR: %s: %s" % (path, p), file=sys.stderr) + bad += bool(problems) + if bad: + print("ERROR: %d of %d builds are not baked for %s" % (bad, len(bins), args.expect), file=sys.stderr) + return 1 + print("OK: %d builds default to %s (%s) and carry both channels" % (len(bins), args.expect, args.native_url)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From a1943a8eac75ec7b29b486889bd404861de5356a Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 3 Oct 2026 15:37:49 -0700 Subject: [PATCH 19/23] fix(ota): always offer another channel's image, even from the same commit Prod and beta are often built from one commit (e.g. after a dev->prod merge), so a hash match does not mean the images are the same: the native base differs. After the flash the target channel is native, so this cannot loop. --- src/helpers/ESP32Board.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index 206a3a7c..d36661d0 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -334,13 +334,14 @@ bool ESP32Board::otaFromManifestImpl(const char* manifest_base, const char* curr bool same_base = (own_base[0] && avail_base[0] && strcmp(own_base, avail_base) == 0); bool have_builds = (own_build >= 0 && avail_build >= 0); bool diff_base = (own_base[0] && avail_base[0] && !same_base); - // Build counters are per channel, so build numbers only compare within the native channel. + // Another channel's image always differs (its native base does), even from the same + // commit; build counters are per channel, so build numbers only compare natively. bool cross_channel = (strcmp(manifest_base, OTA_MANIFEST_BASE) != 0); int behind = 0; bool up_to_date; if (cross_channel) { - up_to_date = hash_equal; + up_to_date = false; } else if (same_base && have_builds) { behind = avail_build - own_build; up_to_date = (behind <= 0); From f652e7d185f8752bb1f8b8a73fb97a8031a01a65 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 3 Oct 2026 15:37:49 -0700 Subject: [PATCH 20/23] fix(ota): flash the channel ota update checked, not a later ota branch The deferred flash re-read the channel preference ~2.5 s after ota update acknowledged its target; capture it when the update is scheduled. --- examples/simple_repeater/MyMesh.cpp | 2 +- examples/simple_repeater/MyMesh.h | 2 ++ examples/simple_room_server/MyMesh.cpp | 2 +- examples/simple_room_server/MyMesh.h | 2 ++ 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/examples/simple_repeater/MyMesh.cpp b/examples/simple_repeater/MyMesh.cpp index b3789657..be201fb4 100644 --- a/examples/simple_repeater/MyMesh.cpp +++ b/examples/simple_repeater/MyMesh.cpp @@ -1741,7 +1741,7 @@ void MyMesh::loop() { Serial.println("OTA: aborted, MQTT stop did not complete cleanly - resuming bridge"); otaAlert("OTA aborted: MQTT stop unclean, bridge resumed"); setBridgeState(true); - } else if (!_cli.getBoard()->otaFromManifest(ota_resolve_base(_prefs.ota_channel), getFirmwareVer(), false, ota_reply)) { + } else if (!_cli.getBoard()->otaFromManifest(ota_resolve_base(_ota_update_channel), getFirmwareVer(), false, ota_reply)) { Serial.print("OTA: aborted, resuming bridge - "); Serial.println(ota_reply); char ota_alert_msg[160]; snprintf(ota_alert_msg, sizeof(ota_alert_msg), "OTA aborted: %s", ota_reply); diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index a232ef52..c1e03a8e 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -123,6 +123,7 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks CayenneLPP telemetry; unsigned long set_radio_at, revert_radio_at; unsigned long _ota_update_at = 0; // deferred `ota update` fire time (0 = none scheduled) + uint8_t _ota_update_channel = 0; // channel `ota update` checked; a later `ota branch` cannot retarget it float pending_freq; float pending_bw; uint8_t pending_sf; @@ -441,6 +442,7 @@ public: bool beginDeferredOtaUpdate() override { _ota_update_at = millis() + 2500; if (_ota_update_at == 0) _ota_update_at = 1; // 0 means "none" + _ota_update_channel = _prefs.ota_channel; #if defined(WITH_MQTT_BRIDGE) // Broadcast START now, while the loop still runs (the 2.5 s reply window): // the deferred flash blocks the loop and, on success, reboots — so a start diff --git a/examples/simple_room_server/MyMesh.cpp b/examples/simple_room_server/MyMesh.cpp index 2e97d491..079e43fa 100644 --- a/examples/simple_room_server/MyMesh.cpp +++ b/examples/simple_room_server/MyMesh.cpp @@ -1617,7 +1617,7 @@ void MyMesh::loop() { } char ota_reply[160]; - if (may_flash && !_cli.getBoard()->otaFromManifest(ota_resolve_base(_prefs.ota_channel), getFirmwareVer(), false, ota_reply)) { + if (may_flash && !_cli.getBoard()->otaFromManifest(ota_resolve_base(_ota_update_channel), getFirmwareVer(), false, ota_reply)) { Serial.print("OTA: aborted - "); Serial.println(ota_reply); may_flash = false; } diff --git a/examples/simple_room_server/MyMesh.h b/examples/simple_room_server/MyMesh.h index b0eb4e8f..ef7fb515 100644 --- a/examples/simple_room_server/MyMesh.h +++ b/examples/simple_room_server/MyMesh.h @@ -136,6 +136,7 @@ class MyMesh : public mesh::Mesh, public CommonCLICallbacks TransportKey default_scope; unsigned long set_radio_at, revert_radio_at; unsigned long _ota_update_at = 0; // deferred `ota update` fire time (0 = none scheduled) + uint8_t _ota_update_channel = 0; // channel `ota update` checked; a later `ota branch` cannot retarget it float pending_freq; float pending_bw; uint8_t pending_sf; @@ -433,6 +434,7 @@ public: bool beginDeferredOtaUpdate() override { _ota_update_at = millis() + 2500; if (_ota_update_at == 0) _ota_update_at = 1; // 0 means "none" + _ota_update_channel = _prefs.ota_channel; return true; } From ed8af1b7ca317785cecb164b09368b309122081d Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 3 Oct 2026 15:46:46 -0700 Subject: [PATCH 21/23] fix(ota): refuse a channel switch to a build that cannot carry this node A switch can land on an older build: e.g. prod cannot read dev's /mqtt.json, and prod's M7 image has no Ethernet. Each image carries an ota-compat tag (state generation + transports). A switch is flashed without rebooting, the new image's tag is read (bounded by its length), and boot is pointed back at the running image unless the target keeps this node's state and transports. CI now requires the tag in every published binary. --- scripts/verify_ota_channel.py | 12 ++++- src/helpers/ESP32Board.cpp | 58 +++++++++++++++++++++- src/helpers/OtaChannel.h | 54 ++++++++++++++++++++ test/test_ota_channel/test_ota_channel.cpp | 33 ++++++++++++ 4 files changed, 153 insertions(+), 4 deletions(-) diff --git a/scripts/verify_ota_channel.py b/scripts/verify_ota_channel.py index 15d91c26..6b74030b 100644 --- a/scripts/verify_ota_channel.py +++ b/scripts/verify_ota_channel.py @@ -2,7 +2,8 @@ """Verify every built observer binary is baked for the expected OTA channel. Observer builds carry three manifest bases (src/helpers/OtaChannel.h), each stored -behind a tag so it can be read back from the binary: +behind a tag so it can be read back from the binary (plus an ota-compat tag, which a +channel switch reads from the downloaded image before booting it): ota-base-native: the channel the build OTAs from by default ota-base-stable: production, the target of `ota branch prod` @@ -26,6 +27,7 @@ import re import sys TAG_RE = re.compile(rb"ota-base-(native|stable|dev):([\x21-\x7e]*)\x00") +COMPAT_RE = re.compile(rb"ota-compat:([0-9]+(?:\+[a-z]+)*)\x00") def read_tags(data): @@ -49,6 +51,10 @@ def check(data, expect, native_url, stable_url, dev_url): found = sorted(tags[name]) if found != [want]: problems.append("ota-base-%s: expected [%s], found %s" % (name, want, found or "nothing")) + # Without its compat tag a build cannot be the target of a channel switch. + compat = sorted({m.group(1).decode() for m in COMPAT_RE.finditer(data)}) + if len(compat) != 1: + problems.append("ota-compat: expected one value, found %s" % (compat or "nothing")) return problems @@ -62,7 +68,7 @@ def self_test(): P, B = "https://h/v", "https://h/beta/v" def blob(native, stable=P, dev=B): - return b"\x00junk\x00ota-base-native:%s\x00ota-base-stable:%s\x00ota-base-dev:%s\x00" % ( + return b"\x00junk\x00ota-base-native:%s\x00ota-base-stable:%s\x00ota-base-dev:%s\x00ota-compat:1\x00" % ( native.encode(), stable.encode(), dev.encode()) cases = [ @@ -74,6 +80,8 @@ def self_test(): ("wrong dev URL fails", blob(P, dev="https://other/v"), "prod", P, False), ("conflicting native tags fail", blob(P) + blob(B), "prod", P, False), ("workflow URL off-channel fails", blob(P), "prod", B, False), + ("missing compat tag fails", blob(P).replace(b"ota-compat:1", b"ota-compat:"), "prod", P, False), + ("conflicting compat tags fail", blob(P) + b"ota-compat:2+eth\x00", "prod", P, False), ] failed = 0 for name, data, expect, native, ok in cases: diff --git a/src/helpers/ESP32Board.cpp b/src/helpers/ESP32Board.cpp index d36661d0..7c013c23 100644 --- a/src/helpers/ESP32Board.cpp +++ b/src/helpers/ESP32Board.cpp @@ -78,6 +78,9 @@ bool ESP32Board::startOTAUpdate(const char* id, char reply[], bool force_ap) { #include #include #include +#include +#include +#include "OtaChannel.h" // Embedded CA bundle (produced by board_build.embed_files). Weak so non-bundle // builds still link; we check for presence at runtime. @@ -123,6 +126,31 @@ static void ota_parseVersion(const char* ver, char* base_out, size_t base_sz, in } } +// Read the compat tag of the image just written to `part`, bounded by the image's own +// length: bytes past it can be a stale, tagged image from an earlier flash. +static bool ota_readImageCompat(const esp_partition_t* part, OtaCompat* out) { + const esp_partition_pos_t pos = { part->address, part->size }; + esp_image_metadata_t meta; + if (esp_image_verify(ESP_IMAGE_VERIFY_SILENT, &pos, &meta) != ESP_OK) return false; + static const size_t kChunk = 2048, kTail = 64; // tail > tag + value, so a split tag is seen whole + static uint8_t buf[kTail + kChunk]; + size_t have = 0; + for (uint32_t off = 0; off < meta.image_len;) { + size_t n = meta.image_len - off; + if (n > kChunk) n = kChunk; + if (esp_partition_read(part, off, buf + have, n) != ESP_OK) return false; + have += n; + off += n; + const char* v = ota_compat_find(buf, have); + if (v) return ota_compat_parse(v, out); + if (have > kTail) { + memmove(buf, buf + have - kTail, kTail); + have = kTail; + } + } + return false; +} + // Canonical signature of the FLASHED partition table — MUST match // scripts/partition_signature.py: each entry "type:subtype:offset:size" in // lowercase hex, sorted by offset, joined by ','. Lets `ota update` compare the @@ -412,9 +440,35 @@ bool ESP32Board::otaFromManifestImpl(const char* manifest_base, const char* curr int d = (int)((int64_t)cur * 10 / total); if (d != ota_progress_decile) { ota_progress_decile = d; Serial.printf("OTA: %d%%\n", d * 10); } }); - httpUpdate.onEnd([]() { Serial.println("OTA: write complete, rebooting..."); }); - httpUpdate.rebootOnUpdate(true); // reboots into the new image on success + httpUpdate.onEnd([]() { Serial.println("OTA: write complete"); }); + // A channel switch is checked before it may boot (below); native updates reboot here. + httpUpdate.rebootOnUpdate(!cross_channel); t_httpUpdate_return ret = httpUpdate.update(uclient, file_url); + if (ret == HTTP_UPDATE_OK && cross_channel) { + const esp_partition_t* running = esp_ota_get_running_partition(); + const esp_partition_t* target = esp_ota_get_boot_partition(); + OtaCompat own = {0, 0}, img = {0, 0}; + const char* volatile own_tag = ota_compat_tag; // volatile: the tag must stay in the image + ota_compat_parse(own_tag + sizeof(OTA_COMPAT_TAG) - 1, &own); + bool tagged = target && target != running && ota_readImageCompat(target, &img); + if (tagged && ota_compat_ok(own, img)) { + Serial.println("OTA: channel switch compatible, rebooting..."); + delay(100); + ESP.restart(); + } + // Point boot back at the running image so the refused build never starts. + bool reverted = (esp_ota_set_boot_partition(running) == ESP_OK); + inhibit_sleep = false; + if (!tagged) { + snprintf(reply, 160, "ERR: channel switch refused: %s has no compat tag; cable flash%s", + avail_disp, reverted ? "" : " [boot revert FAILED]"); + } else { + snprintf(reply, 160, "ERR: channel switch refused: target compat %d/%x < this node %d/%x; cable flash%s", + img.gen, img.caps, own.gen, own.caps, reverted ? "" : " [boot revert FAILED]"); + } + Serial.print("OTA: "); Serial.println(reply); + return false; + } // Only reached on failure (success reboots inside update()). inhibit_sleep = false; diff --git a/src/helpers/OtaChannel.h b/src/helpers/OtaChannel.h index 2918d83d..974cc4ba 100644 --- a/src/helpers/OtaChannel.h +++ b/src/helpers/OtaChannel.h @@ -76,3 +76,57 @@ static inline bool ota_parse_channel(const char* arg, uint8_t* out) { if (strcmp(arg, "default") == 0) { *out = OTA_CH_NATIVE; return true; } return false; } + +// Compatibility tag, read from a downloaded image before a channel switch boots it. +// A switch can land on an older build, which would boot without state it cannot read +// or without a transport this node depends on. Bump OTA_STATE_GEN when a build starts +// storing state that older builds cannot read. +// 1: /prefs.json + /mqtt_prefs +#ifndef OTA_STATE_GEN +#define OTA_STATE_GEN 1 +#endif +#define OTA_CAP_ETH 0x01 // carries MQTT over Ethernet +#if defined(NETWORK_PREFER_ETHERNET) +#define OTA_CAPS_STR "+eth" +#else +#define OTA_CAPS_STR "" +#endif +#define OTA_STR_(x) #x +#define OTA_STR(x) OTA_STR_(x) +#define OTA_COMPAT_TAG "ota-compat:" +static const char ota_compat_tag[] = OTA_COMPAT_TAG OTA_STR(OTA_STATE_GEN) OTA_CAPS_STR; + +struct OtaCompat { + int gen; + uint8_t caps; +}; + +// Parse the tag value "[+cap...]"; unknown caps are ignored. +static inline bool ota_compat_parse(const char* s, OtaCompat* out) { + if (*s < '0' || *s > '9') return false; + out->gen = 0; + out->caps = 0; + while (*s >= '0' && *s <= '9') out->gen = out->gen * 10 + (*s++ - '0'); + while (*s == '+') { + const char* cap = ++s; + while (*s && *s != '+') s++; + if ((size_t)(s - cap) == 3 && memcmp(cap, "eth", 3) == 0) out->caps |= OTA_CAP_ETH; + } + return *s == 0; +} + +// Find the tag in an image chunk; returns the NUL-terminated value text, or nullptr when +// the chunk holds no complete tag. +static inline const char* ota_compat_find(const uint8_t* buf, size_t len) { + const size_t tag_len = sizeof(OTA_COMPAT_TAG) - 1; + for (size_t i = 0; i + tag_len < len; i++) { + if (memcmp(buf + i, OTA_COMPAT_TAG, tag_len) != 0) continue; + if (memchr(buf + i + tag_len, 0, len - i - tag_len)) return (const char*)buf + i + tag_len; + } + return nullptr; +} + +// A target must read this node's state and keep every transport this node has. +static inline bool ota_compat_ok(const OtaCompat& own, const OtaCompat& target) { + return target.gen >= own.gen && (target.caps & own.caps) == own.caps; +} diff --git a/test/test_ota_channel/test_ota_channel.cpp b/test/test_ota_channel/test_ota_channel.cpp index d2a4e482..75ce673d 100644 --- a/test/test_ota_channel/test_ota_channel.cpp +++ b/test/test_ota_channel/test_ota_channel.cpp @@ -40,6 +40,39 @@ TEST(OtaChannel, NativeChannelNameMatchesBase) { EXPECT_STREQ(ota_native_channel_name(), "prod"); } +TEST(OtaCompat, OwnTagParses) { + OtaCompat own; + ASSERT_TRUE(ota_compat_parse(ota_compat_tag + sizeof(OTA_COMPAT_TAG) - 1, &own)); + EXPECT_EQ(own.gen, OTA_STATE_GEN); + EXPECT_EQ(own.caps, 0); +} +TEST(OtaCompat, ParsesCapsAndRejectsJunk) { + OtaCompat c; + ASSERT_TRUE(ota_compat_parse("12+eth+future", &c)); + EXPECT_EQ(c.gen, 12); + EXPECT_EQ(c.caps, OTA_CAP_ETH); + EXPECT_FALSE(ota_compat_parse("", &c)); + EXPECT_FALSE(ota_compat_parse("x1", &c)); + EXPECT_FALSE(ota_compat_parse("2eth", &c)); +} +TEST(OtaCompat, FindsCompleteTagOnly) { + const char img[] = "\xe9junk\0ota-compat:2+eth\0tail"; + const char* v = ota_compat_find((const uint8_t*)img, sizeof(img)); + ASSERT_NE(v, nullptr); + EXPECT_STREQ(v, "2+eth"); + const char cut[] = "junk ota-compat:2+e"; // value runs past the chunk end + EXPECT_EQ(ota_compat_find((const uint8_t*)cut, sizeof(cut) - 1), nullptr); + const char none[] = "ota-compat"; + EXPECT_EQ(ota_compat_find((const uint8_t*)none, sizeof(none)), nullptr); +} +TEST(OtaCompat, TargetMustKeepStateAndTransports) { + EXPECT_TRUE(ota_compat_ok({2, 0}, {2, 0})); + EXPECT_TRUE(ota_compat_ok({1, 0}, {2, OTA_CAP_ETH})); + EXPECT_FALSE(ota_compat_ok({2, 0}, {1, 0})); // cannot read /mqtt.json + EXPECT_FALSE(ota_compat_ok({2, OTA_CAP_ETH}, {2, 0})); // drops Ethernet + EXPECT_TRUE(ota_compat_ok({2, OTA_CAP_ETH}, {3, OTA_CAP_ETH})); +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); From a0856bed728d769114c34f13de34f0454c5dacf1 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 3 Oct 2026 15:47:01 -0700 Subject: [PATCH 22/23] feat(ota): dev stores /mqtt.json, so its state generation is 2 Builds that only read /mqtt_prefs (gen 1) are refused as channel-switch targets until they can read /mqtt.json. --- src/helpers/OtaChannel.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/helpers/OtaChannel.h b/src/helpers/OtaChannel.h index 974cc4ba..cd2524c3 100644 --- a/src/helpers/OtaChannel.h +++ b/src/helpers/OtaChannel.h @@ -82,8 +82,9 @@ static inline bool ota_parse_channel(const char* arg, uint8_t* out) { // or without a transport this node depends on. Bump OTA_STATE_GEN when a build starts // storing state that older builds cannot read. // 1: /prefs.json + /mqtt_prefs +// 2: /mqtt.json #ifndef OTA_STATE_GEN -#define OTA_STATE_GEN 1 +#define OTA_STATE_GEN 2 #endif #define OTA_CAP_ETH 0x01 // carries MQTT over Ethernet #if defined(NETWORK_PREFER_ETHERNET) From 8ec31dda71f867566a09c04704050626f243e629 Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 3 Oct 2026 15:52:21 -0700 Subject: [PATCH 23/23] fix(ota): skip the bare compat search literal when scanning an image Every image also holds "ota-compat:" with no value (the search string); finding it first made every channel switch look untagged. --- src/helpers/OtaChannel.h | 4 +++- test/test_ota_channel/test_ota_channel.cpp | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/helpers/OtaChannel.h b/src/helpers/OtaChannel.h index 974cc4ba..4c3b81b5 100644 --- a/src/helpers/OtaChannel.h +++ b/src/helpers/OtaChannel.h @@ -116,11 +116,13 @@ static inline bool ota_compat_parse(const char* s, OtaCompat* out) { } // Find the tag in an image chunk; returns the NUL-terminated value text, or nullptr when -// the chunk holds no complete tag. +// the chunk holds no complete tag. Skips the bare OTA_COMPAT_TAG search literal, which +// every image also contains. static inline const char* ota_compat_find(const uint8_t* buf, size_t len) { const size_t tag_len = sizeof(OTA_COMPAT_TAG) - 1; for (size_t i = 0; i + tag_len < len; i++) { if (memcmp(buf + i, OTA_COMPAT_TAG, tag_len) != 0) continue; + if (buf[i + tag_len] < '0' || buf[i + tag_len] > '9') continue; if (memchr(buf + i + tag_len, 0, len - i - tag_len)) return (const char*)buf + i + tag_len; } return nullptr; diff --git a/test/test_ota_channel/test_ota_channel.cpp b/test/test_ota_channel/test_ota_channel.cpp index 75ce673d..ce6d0251 100644 --- a/test/test_ota_channel/test_ota_channel.cpp +++ b/test/test_ota_channel/test_ota_channel.cpp @@ -62,6 +62,10 @@ TEST(OtaCompat, FindsCompleteTagOnly) { EXPECT_STREQ(v, "2+eth"); const char cut[] = "junk ota-compat:2+e"; // value runs past the chunk end EXPECT_EQ(ota_compat_find((const uint8_t*)cut, sizeof(cut) - 1), nullptr); + const char literal_first[] = "ota-compat:\0code\0ota-compat:1\0"; // search literal precedes the tag + v = ota_compat_find((const uint8_t*)literal_first, sizeof(literal_first)); + ASSERT_NE(v, nullptr); + EXPECT_STREQ(v, "1"); const char none[] = "ota-compat"; EXPECT_EQ(ota_compat_find((const uint8_t*)none, sizeof(none)), nullptr); }