diff --git a/MQTT_INTERNALS.md b/MQTT_INTERNALS.md index 40a0d300..1adb9713 100644 --- a/MQTT_INTERNALS.md +++ b/MQTT_INTERNALS.md @@ -39,16 +39,36 @@ Remaining integration points in upstream files: - `src/helpers/CommonCLI.{h,cpp}` - the three CLI hooks, `MQTTPrefs` load/save/migration - `src/Dispatcher.{h,cpp}` - radio watchdog block, guarded by `#ifdef WITH_MQTT_BRIDGE` +### `/mqtt_prefs` file format + +`/mqtt_prefs` is written with an 8-byte `MQTTPrefsHeader` (`magic`, `version`, +`payload_len`) followed by the raw `MQTTPrefs` payload. The magic is +`{0xF5, 'M', 'Q', 'P'}` — its leading non-ASCII byte can never collide with the first +bytes of a legacy (headerless) file, whose payload begins with the `mqtt_origin` +string. Bump `MQTT_PREFS_VERSION` when the payload layout changes incompatibly; a file +whose version this firmware doesn't recognize is left untouched and the in-memory prefs +fall back to defaults (no downgrade, no misread). + +Adding a field to the current version stays backward compatible: append it to the end +of `MQTTPrefs`. An older, shorter payload still loads and the missing tail keeps its +default; a newer, longer one is truncated harmlessly. + ### Settings upgrade / migration -`loadPrefs()` handles all historical on-device formats one-time at boot: -- `/mqtt_prefs` written in the pre-slot or 3-slot layouts is field-copied into the - current 6-slot `MQTTPrefs` layout (size-based detection). -- A `/com_prefs` written by fork firmware that predates the `MQTTPrefs` split (which - contained a zero-filled MQTT gap plus a trailing observer block) is detected by its - size; the trailing SNMP / radio-watchdog / fault-alert settings and the - `rx_boosted_gain` / `flood_max_*` fields are recovered, carried into `/mqtt_prefs`, - and both files are rewritten in the current formats. +`loadPrefs()` handles every historical on-device format one-time at boot: +- **`/mqtt_prefs`** — if the file has the version header it is read directly. Otherwise + it is a legacy headerless file and its layout is detected by size: pre-slot + (`OldMQTTPrefs`), 3-slot (`ThreeSlotMQTTPrefs`), or the 6-slot layout shipped on + `mqtt-bridge-implementation-flex` (`Legacy6SlotMQTTPrefs`). Each is field-copied into + the current compact `MQTTPrefs` and re-saved with the version header — which also + drops the vestigial `_legacy_*` fields the flex layout carried mid-struct. This is a + one-time rewrite; every deployed device performs it on its first boot of versioned + firmware, after which all reads take the header path. +- **`/com_prefs`** — a file written by fork firmware that predates the `MQTTPrefs` split + (a zero-filled MQTT gap plus a trailing observer block) is detected by size; the + trailing SNMP / radio-watchdog / fault-alert settings and the `rx_boosted_gain` / + `flood_max_*` fields are recovered, carried into `/mqtt_prefs`, and both files are + rewritten in the current formats. - Settings the pre-split firmware stored *inside* the `/com_prefs` MQTT gap (the MQTT slot/WiFi config itself) are **not** recovered — users upgrading from firmware that old must re-enter their MQTT and WiFi configuration. diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index 5f823707..e11c17b5 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -367,6 +367,14 @@ static void setMQTTPrefsDefaults(MQTTPrefs* prefs) { applyMQTTDefaults(prefs); } +static File openMqttPrefsRead(FILESYSTEM* fs) { +#if defined(RP2040_PLATFORM) + return fs->open("/mqtt_prefs", "r"); +#else + return fs->open("/mqtt_prefs"); +#endif +} + void CommonCLI::loadMQTTPrefs(FILESYSTEM* fs) { // Initialize with defaults first setMQTTPrefsDefaults(&_mqtt_prefs); @@ -378,137 +386,200 @@ void CommonCLI::loadMQTTPrefs(FILESYSTEM* fs) { bool file_existed = fs->exists("/mqtt_prefs"); if (file_existed) { - // Load from separate MQTT prefs file -#if defined(RP2040_PLATFORM) - File file = fs->open("/mqtt_prefs", "r"); -#else - File file = fs->open("/mqtt_prefs"); -#endif + // First, peek the header to see if this is a versioned file. + File file = openMqttPrefsRead(fs); + bool versioned = false; if (file) { size_t file_size = file.size(); - - // Detect old (pre-slot) format by file size. - // Old MQTTPrefs was ~472 bytes (no slot fields). New is ~1464 bytes. - // If the file is smaller than the new struct but close to OldMQTTPrefs size, - // read it with the old layout and migrate. - if (file_size > 0 && file_size <= sizeof(OldMQTTPrefs)) { - OldMQTTPrefs old_prefs; - memset(&old_prefs, 0, sizeof(old_prefs)); - size_t bytes_read = file.read((uint8_t *)&old_prefs, file_size < sizeof(old_prefs) ? file_size : sizeof(old_prefs)); - file.close(); - - if (bytes_read > 0) { - MESH_DEBUG_PRINTLN("MQTT: Migrating old-format prefs to slot-based layout"); - - // Copy common fields (identical layout at start of both structs) - memcpy(_mqtt_prefs.mqtt_origin, old_prefs.mqtt_origin, sizeof(_mqtt_prefs.mqtt_origin)); - memcpy(_mqtt_prefs.mqtt_iata, old_prefs.mqtt_iata, sizeof(_mqtt_prefs.mqtt_iata)); - _mqtt_prefs.mqtt_status_enabled = old_prefs.mqtt_status_enabled; - _mqtt_prefs.mqtt_packets_enabled = old_prefs.mqtt_packets_enabled; - _mqtt_prefs.mqtt_raw_enabled = old_prefs.mqtt_raw_enabled; - _mqtt_prefs.mqtt_tx_enabled = old_prefs.mqtt_tx_enabled; - _mqtt_prefs.mqtt_status_interval = old_prefs.mqtt_status_interval; - memcpy(_mqtt_prefs.wifi_ssid, old_prefs.wifi_ssid, sizeof(_mqtt_prefs.wifi_ssid)); - memcpy(_mqtt_prefs.wifi_password, old_prefs.wifi_password, sizeof(_mqtt_prefs.wifi_password)); - _mqtt_prefs.wifi_power_save = old_prefs.wifi_power_save; - memcpy(_mqtt_prefs.timezone_string, old_prefs.timezone_string, sizeof(_mqtt_prefs.timezone_string)); - _mqtt_prefs.timezone_offset = old_prefs.timezone_offset; - - // Migrate shared auth fields - memcpy(_mqtt_prefs.mqtt_owner_public_key, old_prefs.mqtt_owner_public_key, sizeof(_mqtt_prefs.mqtt_owner_public_key)); - memcpy(_mqtt_prefs.mqtt_email, old_prefs.mqtt_email, sizeof(_mqtt_prefs.mqtt_email)); - - // Migrate analyzer presets to slots - if (old_prefs.mqtt_analyzer_us_enabled == 1) { - strncpy(_mqtt_prefs.mqtt_slot_preset[0], "analyzer-us", sizeof(_mqtt_prefs.mqtt_slot_preset[0]) - 1); + if (file_size >= sizeof(MQTTPrefsHeader)) { + MQTTPrefsHeader hdr; + if (file.read((uint8_t *)&hdr, sizeof(hdr)) == sizeof(hdr) + && memcmp(hdr.magic, MQTT_PREFS_MAGIC, sizeof(hdr.magic)) == 0) { + versioned = true; + if (hdr.version == MQTT_PREFS_VERSION) { + // Current version: the payload follows the header. Read up to + // sizeof(MQTTPrefs); a shorter payload (an earlier v1 firmware that + // hadn't appended a tail field) leaves the trailing fields at their + // defaults, and a longer one (a future append) is truncated harmlessly. + size_t payload_avail = file_size - sizeof(hdr); + size_t to_read = payload_avail < sizeof(_mqtt_prefs) ? payload_avail : sizeof(_mqtt_prefs); + size_t got = file.read((uint8_t *)&_mqtt_prefs, to_read); + if (got != to_read) { + setMQTTPrefsDefaults(&_mqtt_prefs); + } else { + has_observer_fields = true; // observer tail is part of the v1 payload + } } else { - strncpy(_mqtt_prefs.mqtt_slot_preset[0], "none", sizeof(_mqtt_prefs.mqtt_slot_preset[0]) - 1); + // Unknown (newer) version: don't risk misreading a layout we don't know. + // Keep defaults for this boot and leave the file untouched (no downgrade). + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs version unsupported, using defaults"); } - if (old_prefs.mqtt_analyzer_eu_enabled == 1) { - strncpy(_mqtt_prefs.mqtt_slot_preset[1], "analyzer-eu", sizeof(_mqtt_prefs.mqtt_slot_preset[1]) - 1); - } else { - strncpy(_mqtt_prefs.mqtt_slot_preset[1], "none", sizeof(_mqtt_prefs.mqtt_slot_preset[1]) - 1); - } - - // Migrate custom server to slot 3 - if (old_prefs.mqtt_server[0] != '\0' && old_prefs.mqtt_port > 0) { - strncpy(_mqtt_prefs.mqtt_slot_preset[2], "custom", sizeof(_mqtt_prefs.mqtt_slot_preset[2]) - 1); - strncpy(_mqtt_prefs.mqtt_slot_host[2], old_prefs.mqtt_server, sizeof(_mqtt_prefs.mqtt_slot_host[2]) - 1); - _mqtt_prefs.mqtt_slot_port[2] = old_prefs.mqtt_port; - strncpy(_mqtt_prefs.mqtt_slot_username[2], old_prefs.mqtt_username, sizeof(_mqtt_prefs.mqtt_slot_username[2]) - 1); - strncpy(_mqtt_prefs.mqtt_slot_password[2], old_prefs.mqtt_password, sizeof(_mqtt_prefs.mqtt_slot_password[2]) - 1); - } else { - strncpy(_mqtt_prefs.mqtt_slot_preset[2], "none", sizeof(_mqtt_prefs.mqtt_slot_preset[2]) - 1); - } - - // Save migrated prefs in new format - saveMQTTPrefs(fs); } - } else if (file_size > 0 && file_size <= sizeof(ThreeSlotMQTTPrefs)) { - // 3-slot format → 6-slot migration - // Array sizes changed from [3] to [6], shifting all field offsets. - // Read into old layout struct and field-copy to new layout. - ThreeSlotMQTTPrefs old3; - memset(&old3, 0, sizeof(old3)); - size_t bytes_to_read = file_size < sizeof(old3) ? file_size : sizeof(old3); - size_t bytes_read = file.read((uint8_t *)&old3, bytes_to_read); - file.close(); + } + file.close(); + } - if (bytes_read > 0) { - MESH_DEBUG_PRINTLN("MQTT: Migrating 3-slot prefs to 6-slot layout"); + if (!versioned) { + // Headerless (legacy) file. Detect the historical on-disk layout by size, + // migrate it into the compact versioned struct, and re-save — which adds the + // header and drops the vestigial `_legacy_*` fields. Reopen because the peek + // above advanced the read cursor past the (non-matching) leading bytes. + File file = openMqttPrefsRead(fs); + if (file) { + size_t file_size = file.size(); - // Copy non-slot fields (identical layout) - memcpy(_mqtt_prefs.mqtt_origin, old3.mqtt_origin, sizeof(_mqtt_prefs.mqtt_origin)); - memcpy(_mqtt_prefs.mqtt_iata, old3.mqtt_iata, sizeof(_mqtt_prefs.mqtt_iata)); - _mqtt_prefs.mqtt_status_enabled = old3.mqtt_status_enabled; - _mqtt_prefs.mqtt_packets_enabled = old3.mqtt_packets_enabled; - _mqtt_prefs.mqtt_raw_enabled = old3.mqtt_raw_enabled; - _mqtt_prefs.mqtt_tx_enabled = old3.mqtt_tx_enabled; - _mqtt_prefs.mqtt_status_interval = old3.mqtt_status_interval; - memcpy(_mqtt_prefs.wifi_ssid, old3.wifi_ssid, sizeof(_mqtt_prefs.wifi_ssid)); - memcpy(_mqtt_prefs.wifi_password, old3.wifi_password, sizeof(_mqtt_prefs.wifi_password)); - _mqtt_prefs.wifi_power_save = old3.wifi_power_save; - memcpy(_mqtt_prefs.timezone_string, old3.timezone_string, sizeof(_mqtt_prefs.timezone_string)); - _mqtt_prefs.timezone_offset = old3.timezone_offset; + // Detect old (pre-slot) format by file size. + // Old MQTTPrefs was ~472 bytes (no slot fields). + // If the file is smaller than the new struct but close to OldMQTTPrefs size, + // read it with the old layout and migrate. + if (file_size > 0 && file_size <= sizeof(OldMQTTPrefs)) { + OldMQTTPrefs old_prefs; + memset(&old_prefs, 0, sizeof(old_prefs)); + size_t bytes_read = file.read((uint8_t *)&old_prefs, file_size < sizeof(old_prefs) ? file_size : sizeof(old_prefs)); + file.close(); - // Copy slot fields for indices 0-2 from old layout - for (int i = 0; i < 3; i++) { - memcpy(_mqtt_prefs.mqtt_slot_preset[i], old3.mqtt_slot_preset[i], sizeof(_mqtt_prefs.mqtt_slot_preset[i])); - memcpy(_mqtt_prefs.mqtt_slot_host[i], old3.mqtt_slot_host[i], sizeof(_mqtt_prefs.mqtt_slot_host[i])); - _mqtt_prefs.mqtt_slot_port[i] = old3.mqtt_slot_port[i]; - memcpy(_mqtt_prefs.mqtt_slot_username[i], old3.mqtt_slot_username[i], sizeof(_mqtt_prefs.mqtt_slot_username[i])); - memcpy(_mqtt_prefs.mqtt_slot_password[i], old3.mqtt_slot_password[i], sizeof(_mqtt_prefs.mqtt_slot_password[i])); - memcpy(_mqtt_prefs.mqtt_slot_token[i], old3.mqtt_slot_token[i], sizeof(_mqtt_prefs.mqtt_slot_token[i])); - memcpy(_mqtt_prefs.mqtt_slot_topic[i], old3.mqtt_slot_topic[i], sizeof(_mqtt_prefs.mqtt_slot_topic[i])); + if (bytes_read > 0) { + MESH_DEBUG_PRINTLN("MQTT: Migrating old-format prefs to versioned layout"); + + // Copy common fields (identical layout at start of both structs) + memcpy(_mqtt_prefs.mqtt_origin, old_prefs.mqtt_origin, sizeof(_mqtt_prefs.mqtt_origin)); + memcpy(_mqtt_prefs.mqtt_iata, old_prefs.mqtt_iata, sizeof(_mqtt_prefs.mqtt_iata)); + _mqtt_prefs.mqtt_status_enabled = old_prefs.mqtt_status_enabled; + _mqtt_prefs.mqtt_packets_enabled = old_prefs.mqtt_packets_enabled; + _mqtt_prefs.mqtt_raw_enabled = old_prefs.mqtt_raw_enabled; + _mqtt_prefs.mqtt_tx_enabled = old_prefs.mqtt_tx_enabled; + _mqtt_prefs.mqtt_status_interval = old_prefs.mqtt_status_interval; + memcpy(_mqtt_prefs.wifi_ssid, old_prefs.wifi_ssid, sizeof(_mqtt_prefs.wifi_ssid)); + memcpy(_mqtt_prefs.wifi_password, old_prefs.wifi_password, sizeof(_mqtt_prefs.wifi_password)); + _mqtt_prefs.wifi_power_save = old_prefs.wifi_power_save; + memcpy(_mqtt_prefs.timezone_string, old_prefs.timezone_string, sizeof(_mqtt_prefs.timezone_string)); + _mqtt_prefs.timezone_offset = old_prefs.timezone_offset; + + // Migrate shared auth fields + memcpy(_mqtt_prefs.mqtt_owner_public_key, old_prefs.mqtt_owner_public_key, sizeof(_mqtt_prefs.mqtt_owner_public_key)); + memcpy(_mqtt_prefs.mqtt_email, old_prefs.mqtt_email, sizeof(_mqtt_prefs.mqtt_email)); + + // Migrate analyzer presets to slots + if (old_prefs.mqtt_analyzer_us_enabled == 1) { + strncpy(_mqtt_prefs.mqtt_slot_preset[0], "analyzer-us", sizeof(_mqtt_prefs.mqtt_slot_preset[0]) - 1); + } else { + strncpy(_mqtt_prefs.mqtt_slot_preset[0], "none", sizeof(_mqtt_prefs.mqtt_slot_preset[0]) - 1); + } + if (old_prefs.mqtt_analyzer_eu_enabled == 1) { + strncpy(_mqtt_prefs.mqtt_slot_preset[1], "analyzer-eu", sizeof(_mqtt_prefs.mqtt_slot_preset[1]) - 1); + } else { + strncpy(_mqtt_prefs.mqtt_slot_preset[1], "none", sizeof(_mqtt_prefs.mqtt_slot_preset[1]) - 1); + } + + // Migrate custom server to slot 3 + if (old_prefs.mqtt_server[0] != '\0' && old_prefs.mqtt_port > 0) { + strncpy(_mqtt_prefs.mqtt_slot_preset[2], "custom", sizeof(_mqtt_prefs.mqtt_slot_preset[2]) - 1); + strncpy(_mqtt_prefs.mqtt_slot_host[2], old_prefs.mqtt_server, sizeof(_mqtt_prefs.mqtt_slot_host[2]) - 1); + _mqtt_prefs.mqtt_slot_port[2] = old_prefs.mqtt_port; + strncpy(_mqtt_prefs.mqtt_slot_username[2], old_prefs.mqtt_username, sizeof(_mqtt_prefs.mqtt_slot_username[2]) - 1); + strncpy(_mqtt_prefs.mqtt_slot_password[2], old_prefs.mqtt_password, sizeof(_mqtt_prefs.mqtt_slot_password[2]) - 1); + } else { + strncpy(_mqtt_prefs.mqtt_slot_preset[2], "none", sizeof(_mqtt_prefs.mqtt_slot_preset[2]) - 1); + } + + // Save migrated prefs in the versioned format + saveMQTTPrefs(fs); } - // Slots 3-5 keep defaults ("none") from setMQTTPrefsDefaults() + } else if (file_size > 0 && file_size <= sizeof(ThreeSlotMQTTPrefs)) { + // 3-slot format → compact 6-slot migration + // Array sizes changed from [3] to [6], shifting all field offsets. + // Read into old layout struct and field-copy to new layout. + ThreeSlotMQTTPrefs old3; + memset(&old3, 0, sizeof(old3)); + size_t bytes_to_read = file_size < sizeof(old3) ? file_size : sizeof(old3); + size_t bytes_read = file.read((uint8_t *)&old3, bytes_to_read); + file.close(); - // Copy shared auth fields - memcpy(_mqtt_prefs.mqtt_owner_public_key, old3.mqtt_owner_public_key, sizeof(_mqtt_prefs.mqtt_owner_public_key)); - memcpy(_mqtt_prefs.mqtt_email, old3.mqtt_email, sizeof(_mqtt_prefs.mqtt_email)); + if (bytes_read > 0) { + MESH_DEBUG_PRINTLN("MQTT: Migrating 3-slot prefs to versioned layout"); - // Save migrated prefs in new 6-slot format - saveMQTTPrefs(fs); - } - } else if (file_size > 0) { - // 6-slot format: read directly. The observer fields (snmp/watchdog/alert) - // count as present only when the file holds the full struct; shorter (older) - // files are read only up to the observer boundary, so trailing struct - // padding in an old-firmware file can't bleed into the new fields. - bool full_struct = file_size >= sizeof(_mqtt_prefs); - size_t obs_boundary = offsetof(MQTTPrefs, snmp_enabled); - size_t bytes_to_read = full_struct ? sizeof(_mqtt_prefs) - : (file_size < obs_boundary ? file_size : obs_boundary); - size_t bytes_read = file.read((uint8_t *)&_mqtt_prefs, bytes_to_read); - file.close(); - if (bytes_read != bytes_to_read) { + // Copy non-slot fields (identical layout) + memcpy(_mqtt_prefs.mqtt_origin, old3.mqtt_origin, sizeof(_mqtt_prefs.mqtt_origin)); + memcpy(_mqtt_prefs.mqtt_iata, old3.mqtt_iata, sizeof(_mqtt_prefs.mqtt_iata)); + _mqtt_prefs.mqtt_status_enabled = old3.mqtt_status_enabled; + _mqtt_prefs.mqtt_packets_enabled = old3.mqtt_packets_enabled; + _mqtt_prefs.mqtt_raw_enabled = old3.mqtt_raw_enabled; + _mqtt_prefs.mqtt_tx_enabled = old3.mqtt_tx_enabled; + _mqtt_prefs.mqtt_status_interval = old3.mqtt_status_interval; + memcpy(_mqtt_prefs.wifi_ssid, old3.wifi_ssid, sizeof(_mqtt_prefs.wifi_ssid)); + memcpy(_mqtt_prefs.wifi_password, old3.wifi_password, sizeof(_mqtt_prefs.wifi_password)); + _mqtt_prefs.wifi_power_save = old3.wifi_power_save; + memcpy(_mqtt_prefs.timezone_string, old3.timezone_string, sizeof(_mqtt_prefs.timezone_string)); + _mqtt_prefs.timezone_offset = old3.timezone_offset; + + // Copy slot fields for indices 0-2 from old layout + for (int i = 0; i < 3; i++) { + memcpy(_mqtt_prefs.mqtt_slot_preset[i], old3.mqtt_slot_preset[i], sizeof(_mqtt_prefs.mqtt_slot_preset[i])); + memcpy(_mqtt_prefs.mqtt_slot_host[i], old3.mqtt_slot_host[i], sizeof(_mqtt_prefs.mqtt_slot_host[i])); + _mqtt_prefs.mqtt_slot_port[i] = old3.mqtt_slot_port[i]; + memcpy(_mqtt_prefs.mqtt_slot_username[i], old3.mqtt_slot_username[i], sizeof(_mqtt_prefs.mqtt_slot_username[i])); + memcpy(_mqtt_prefs.mqtt_slot_password[i], old3.mqtt_slot_password[i], sizeof(_mqtt_prefs.mqtt_slot_password[i])); + memcpy(_mqtt_prefs.mqtt_slot_token[i], old3.mqtt_slot_token[i], sizeof(_mqtt_prefs.mqtt_slot_token[i])); + memcpy(_mqtt_prefs.mqtt_slot_topic[i], old3.mqtt_slot_topic[i], sizeof(_mqtt_prefs.mqtt_slot_topic[i])); + } + // Slots 3-5 keep defaults ("none") from setMQTTPrefsDefaults() + + // Copy shared auth fields + memcpy(_mqtt_prefs.mqtt_owner_public_key, old3.mqtt_owner_public_key, sizeof(_mqtt_prefs.mqtt_owner_public_key)); + memcpy(_mqtt_prefs.mqtt_email, old3.mqtt_email, sizeof(_mqtt_prefs.mqtt_email)); + + // Save migrated prefs in the versioned format + saveMQTTPrefs(fs); + } + } else if (file_size > 0) { + // Headerless 6-slot layout as shipped on mqtt-bridge-implementation-flex + // (the deployed fleet). Same field order as the compact struct but with the + // vestigial `_legacy_*` block mid-struct and no observer tail — so read it + // into Legacy6SlotMQTTPrefs and field-copy across, dropping `_legacy_*`. + Legacy6SlotMQTTPrefs old6; + memset(&old6, 0, sizeof(old6)); + size_t bytes_to_read = file_size < sizeof(old6) ? file_size : sizeof(old6); + size_t bytes_read = file.read((uint8_t *)&old6, bytes_to_read); + file.close(); + + if (bytes_read > 0) { + MESH_DEBUG_PRINTLN("MQTT: Migrating headerless 6-slot prefs to versioned layout"); + + memcpy(_mqtt_prefs.mqtt_origin, old6.mqtt_origin, sizeof(_mqtt_prefs.mqtt_origin)); + memcpy(_mqtt_prefs.mqtt_iata, old6.mqtt_iata, sizeof(_mqtt_prefs.mqtt_iata)); + _mqtt_prefs.mqtt_status_enabled = old6.mqtt_status_enabled; + _mqtt_prefs.mqtt_packets_enabled = old6.mqtt_packets_enabled; + _mqtt_prefs.mqtt_raw_enabled = old6.mqtt_raw_enabled; + _mqtt_prefs.mqtt_tx_enabled = old6.mqtt_tx_enabled; + _mqtt_prefs.mqtt_status_interval = old6.mqtt_status_interval; + memcpy(_mqtt_prefs.wifi_ssid, old6.wifi_ssid, sizeof(_mqtt_prefs.wifi_ssid)); + memcpy(_mqtt_prefs.wifi_password, old6.wifi_password, sizeof(_mqtt_prefs.wifi_password)); + _mqtt_prefs.wifi_power_save = old6.wifi_power_save; + memcpy(_mqtt_prefs.timezone_string, old6.timezone_string, sizeof(_mqtt_prefs.timezone_string)); + _mqtt_prefs.timezone_offset = old6.timezone_offset; + memcpy(_mqtt_prefs.mqtt_slot_preset, old6.mqtt_slot_preset, sizeof(_mqtt_prefs.mqtt_slot_preset)); + memcpy(_mqtt_prefs.mqtt_slot_host, old6.mqtt_slot_host, sizeof(_mqtt_prefs.mqtt_slot_host)); + memcpy(_mqtt_prefs.mqtt_slot_port, old6.mqtt_slot_port, sizeof(_mqtt_prefs.mqtt_slot_port)); + memcpy(_mqtt_prefs.mqtt_slot_username, old6.mqtt_slot_username, sizeof(_mqtt_prefs.mqtt_slot_username)); + memcpy(_mqtt_prefs.mqtt_slot_password, old6.mqtt_slot_password, sizeof(_mqtt_prefs.mqtt_slot_password)); + memcpy(_mqtt_prefs.mqtt_owner_public_key, old6.mqtt_owner_public_key, sizeof(_mqtt_prefs.mqtt_owner_public_key)); + memcpy(_mqtt_prefs.mqtt_email, old6.mqtt_email, sizeof(_mqtt_prefs.mqtt_email)); + // `_legacy_*` fields are intentionally dropped here. + memcpy(_mqtt_prefs.mqtt_slot_token, old6.mqtt_slot_token, sizeof(_mqtt_prefs.mqtt_slot_token)); + memcpy(_mqtt_prefs.mqtt_slot_topic, old6.mqtt_slot_topic, sizeof(_mqtt_prefs.mqtt_slot_topic)); + memcpy(_mqtt_prefs.mqtt_slot_audience, old6.mqtt_slot_audience, sizeof(_mqtt_prefs.mqtt_slot_audience)); + _mqtt_prefs.mqtt_rx_enabled = old6.mqtt_rx_enabled; + memcpy(_mqtt_prefs.mqtt_ntp_server, old6.mqtt_ntp_server, sizeof(_mqtt_prefs.mqtt_ntp_server)); + // Observer tail (snmp/watchdog/alert) keeps defaults; if this device is + // also upgrading across the NodePrefs split, loadPrefsInt captured those + // values from /com_prefs and they are applied below. + + saveMQTTPrefs(fs); + } + } else { + file.close(); setMQTTPrefsDefaults(&_mqtt_prefs); - } else if (full_struct) { - has_observer_fields = true; } - } else { - file.close(); - setMQTTPrefsDefaults(&_mqtt_prefs); } } } else { @@ -549,6 +620,12 @@ void CommonCLI::saveMQTTPrefs(FILESYSTEM* fs) { File file = fs->open("/mqtt_prefs", "w", true); #endif if (file) { + // Versioned format: 8-byte header followed by the raw MQTTPrefs payload. + MQTTPrefsHeader hdr; + memcpy(hdr.magic, MQTT_PREFS_MAGIC, sizeof(hdr.magic)); + hdr.version = MQTT_PREFS_VERSION; + hdr.payload_len = (uint16_t)sizeof(_mqtt_prefs); + file.write((uint8_t *)&hdr, sizeof(hdr)); file.write((uint8_t *)&_mqtt_prefs, sizeof(_mqtt_prefs)); file.close(); } diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index 944df3a1..b5240571 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -129,29 +129,17 @@ struct MQTTPrefs { char mqtt_owner_public_key[65]; // Owner public key (hex string) char mqtt_email[64]; // Owner email address - // --- Legacy fields (vestigial, kept for binary compatibility) --- - // Migration now uses OldMQTTPrefs/ThreeSlotMQTTPrefs structs. These fields are unused - // but must remain to preserve byte offsets for devices that already saved a new-format /mqtt_prefs file. - uint8_t _legacy_analyzer_us_enabled; - uint8_t _legacy_analyzer_eu_enabled; - char _legacy_mqtt_server[64]; - uint16_t _legacy_mqtt_port; - char _legacy_mqtt_username[32]; - char _legacy_mqtt_password[64]; - - // --- New fields (appended at end for migration safety) --- + // Per-slot extended fields char mqtt_slot_token[MAX_MQTT_SLOTS][48]; // Per-slot token (e.g., MeshRank account token) char mqtt_slot_topic[MAX_MQTT_SLOTS][96]; // Per-slot custom topic template (custom preset only) char mqtt_slot_audience[MAX_MQTT_SLOTS][64]; // JWT audience (non-empty enables JWT auth for custom slots) - // --- Appended fields (added after initial 6-slot migration) --- uint8_t mqtt_rx_enabled; // Enable RX packet uplinking (default: on) char mqtt_ntp_server[64]; // Custom NTP server; empty = pool.ntp.org - // --- Observer non-MQTT settings, moved out of NodePrefs (Phase 2). Appended at - // the end so existing /mqtt_prefs files (which lack them) still load. On first - // boot after upgrade, loadPrefsInt() captures the old /com_prefs trailing block - // into LegacyObserverTail and loadMQTTPrefs() copies the values here. --- + // Observer non-MQTT settings (moved out of NodePrefs so this file stays aligned + // with upstream). New fields are appended here so a shorter /mqtt_prefs payload + // from an earlier v1 firmware still loads; the missing tail keeps its default. uint8_t snmp_enabled; // boolean char snmp_community[24]; // community string (default "public") uint8_t radio_watchdog_minutes; // 0=disabled, 1-120 minutes (observer-only radio recovery) @@ -164,6 +152,19 @@ struct MQTTPrefs { char alert_region[31]; // optional region override; empty = default_scope }; +// /mqtt_prefs is written with an 8-byte header so the format is self-describing. +// Files with no header are legacy (versionless) and detected by size in loadMQTTPrefs. +// The magic leads with a non-ASCII byte so it can never collide with the first +// bytes of a legacy file, whose payload starts with the mqtt_origin string. +static const uint8_t MQTT_PREFS_MAGIC[4] = {0xF5, 'M', 'Q', 'P'}; +static const uint16_t MQTT_PREFS_VERSION = 1; // bump when the MQTTPrefs payload layout changes incompatibly + +struct MQTTPrefsHeader { + uint8_t magic[4]; // MQTT_PREFS_MAGIC + uint16_t version; // MQTT_PREFS_VERSION + uint16_t payload_len; // sizeof(MQTTPrefs) at write time (sanity / forward-compat) +}; + // 3-slot MQTTPrefs layout — used for migrating from 3-slot to 6-slot format. // Changing array sizes from [3] to [6] shifts all field offsets, so raw file.read() // into the new struct would corrupt data. This struct preserves the old binary layout. @@ -197,6 +198,44 @@ struct ThreeSlotMQTTPrefs { char mqtt_slot_topic[3][96]; }; +// Versionless 6-slot layout as shipped on mqtt-bridge-implementation-flex (the +// several-thousand-device deployed fleet). This is the current MQTTPrefs minus the +// observer tail, and it still carries the now-removed `_legacy_*` fields mid-struct. +// loadMQTTPrefs reads a headerless file of this size into this struct, then +// field-copies (dropping `_legacy_*`) into the compact versioned MQTTPrefs. +struct Legacy6SlotMQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + char wifi_ssid[32]; + char wifi_password[64]; + uint8_t wifi_power_save; + char timezone_string[32]; + int8_t timezone_offset; + char mqtt_slot_preset[MAX_MQTT_SLOTS][24]; + char mqtt_slot_host[MAX_MQTT_SLOTS][64]; + uint16_t mqtt_slot_port[MAX_MQTT_SLOTS]; + char mqtt_slot_username[MAX_MQTT_SLOTS][32]; + char mqtt_slot_password[MAX_MQTT_SLOTS][64]; + char mqtt_owner_public_key[65]; + char mqtt_email[64]; + uint8_t _legacy_analyzer_us_enabled; + uint8_t _legacy_analyzer_eu_enabled; + char _legacy_mqtt_server[64]; + uint16_t _legacy_mqtt_port; + char _legacy_mqtt_username[32]; + char _legacy_mqtt_password[64]; + char mqtt_slot_token[MAX_MQTT_SLOTS][48]; + char mqtt_slot_topic[MAX_MQTT_SLOTS][96]; + char mqtt_slot_audience[MAX_MQTT_SLOTS][64]; + uint8_t mqtt_rx_enabled; + char mqtt_ntp_server[64]; +}; + // Observer settings captured from the trailing block of an old-format /com_prefs // (fork firmware that predates the NodePrefs -> MQTTPrefs split). loadPrefsInt() // fills this in when it detects the old file layout; loadMQTTPrefs() then applies