diff --git a/src/helpers/CommonCLI.cpp b/src/helpers/CommonCLI.cpp index b9ad2eea..b4c2ed55 100644 --- a/src/helpers/CommonCLI.cpp +++ b/src/helpers/CommonCLI.cpp @@ -3,6 +3,7 @@ #include "TxtDataHelpers.h" #include "AdvertDataHelpers.h" #include "AlertReporter.h" // for alertReporterBannedChannelMatch() +#include "MQTTPrefsAtomicStore.h" #include #include @@ -18,6 +19,7 @@ #ifdef WITH_MQTT_BRIDGE #include "bridges/MQTTBridge.h" #include "MQTTDefaults.h" +#include "MQTTPrefsCodec.h" #endif // Believe it or not, this std C function is busted on some platforms! @@ -59,14 +61,30 @@ static const size_t COM_PREFS_TAIL_BYTES = 5; void CommonCLI::loadPrefs(FILESYSTEM* fs) { bool is_fresh_install = false; bool is_upgrade = false; +#ifdef WITH_MQTT_BRIDGE + bool node_prefs_needs_migration = false; +#endif if (fs->exists("/com_prefs")) { loadPrefsInt(fs, "/com_prefs"); // new filename } else if (fs->exists("/node_prefs")) { loadPrefsInt(fs, "/node_prefs"); is_upgrade = true; // Migrating from old filename - savePrefs(fs); // save to new filename - fs->remove("/node_prefs"); // remove old +#ifdef WITH_MQTT_BRIDGE + // Wait for loadMQTTPrefs() to persist any observer tail captured from this + // old file before replacing or removing its only on-flash copy. + node_prefs_needs_migration = true; +#else + if (saveCommonPrefsImageAtomically(fs)) { + fs->remove("/node_prefs"); // remove old only after the rename commits + } else { + MESH_DEBUG_PRINTLN("Prefs: preserving legacy /node_prefs until /com_prefs migration commits"); + } + // This boot has either completed the filename handoff or deliberately + // preserved /node_prefs for a retry. Do not follow it with the ordinary + // non-atomic legacy compaction path below. + _com_prefs_needs_upgrade = false; +#endif } else { // File doesn't exist - set default bridge settings for fresh installs is_fresh_install = true; @@ -76,28 +94,70 @@ void CommonCLI::loadPrefs(FILESYSTEM* fs) { // Load observer preferences (MQTT/WiFi/timezone/SNMP/alert) from /mqtt_prefs. // Readers (MQTTBridge, AlertReporter, observer CLI) use _mqtt_prefs directly — // these fields no longer exist in NodePrefs, so there is nothing to sync. - loadMQTTPrefs(fs); + MQTTPrefsAtomicStore::LegacyUpgradeGate legacy_upgrade( + _com_prefs_needs_upgrade || node_prefs_needs_migration); + loadMQTTPrefs(fs, &legacy_upgrade); + if (_mqtt_prefs_hold) legacy_upgrade.holdMqttSource(); // For MQTT bridge, migrate bridge.source to RX (logRx) only on fresh installs or upgrades // so legacy "tx" is not the default. mqtt.rx / mqtt.tx are separate (fresh default: advert for TX) if ((is_fresh_install || is_upgrade) && _prefs->bridge_pkt_src == 0) { - MESH_DEBUG_PRINTLN("MQTT Bridge: Migrating bridge.source from tx to rx (MQTT bridge default)"); - _prefs->bridge_pkt_src = 1; // Set to RX (logRx) - savePrefs(fs); // Save the updated preference + if (legacy_upgrade.blocksComPrefsRewrite()) { + MESH_DEBUG_PRINTLN("MQTT Bridge: deferring bridge.source migration until legacy prefs are preserved"); + } else { + MESH_DEBUG_PRINTLN("MQTT Bridge: Migrating bridge.source from tx to rx (MQTT bridge default)"); + _prefs->bridge_pkt_src = 1; // Set to RX (logRx) + if (node_prefs_needs_migration) { + // The atomic /node_prefs -> /com_prefs handoff below persists this + // in-memory change. Do not publish /com_prefs before that transaction. + MESH_DEBUG_PRINTLN("MQTT Bridge: bridge.source will be saved with node prefs migration"); + } else { + savePrefs(fs); // Save the updated preference + } + } } // mqtt_rx_enabled: new field appended to end of MQTTPrefs. On upgrade from older firmware, // the shorter /mqtt_prefs file won't contain it, so it keeps the default value (1 = on) // set by setMQTTPrefsDefaults(). No explicit migration needed. #endif - if (_com_prefs_needs_upgrade) { +#ifdef WITH_MQTT_BRIDGE + if (node_prefs_needs_migration) { + if (legacy_upgrade.mayRewriteComPrefs()) { + // The MQTT image (and any tail from /node_prefs) is committed, so it is + // now safe to publish the replacement name. Keep /node_prefs until the + // complete /com_prefs image is closed and atomically renamed into place. + if (saveCommonPrefsImageAtomically(fs)) { + fs->remove("/node_prefs"); + legacy_upgrade.recordComPrefsRewrite(); + _com_prefs_needs_upgrade = false; + } else { + MESH_DEBUG_PRINTLN("MQTT: preserving legacy /node_prefs until /com_prefs migration commits"); + } + } else { + MESH_DEBUG_PRINTLN("MQTT: preserving legacy /node_prefs until /mqtt_prefs migration commits"); + } + } else if (_com_prefs_needs_upgrade) { // Old-format /com_prefs (legacy MQTT gap + trailing observer block) was detected: // rewrite the prefs files in the current layout, one time. This persists the // recovered rx_boosted_gain/flood_max_* values and (on MQTT builds) the observer // settings that loadMQTTPrefs carried over into /mqtt_prefs. + if (legacy_upgrade.mayRewriteComPrefs()) { + // loadMQTTPrefs has already committed the full MQTT payload (including + // any recovered observer tail), so compact only /com_prefs now. + savePrefs(fs, false); + legacy_upgrade.recordComPrefsRewrite(); + _com_prefs_needs_upgrade = false; + } else { + MESH_DEBUG_PRINTLN("MQTT: preserving legacy /com_prefs until /mqtt_prefs migration commits"); + } + } +#else + if (_com_prefs_needs_upgrade) { savePrefs(fs); _com_prefs_needs_upgrade = false; } +#endif } void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) { @@ -311,7 +371,81 @@ void CommonCLI::loadPrefsInt(FILESYSTEM* fs, const char* filename) { } } -void CommonCLI::savePrefs(FILESYSTEM* fs) { +// Keep the byte layout in one place so ordinary saves and the atomic legacy +// rename path write exactly the same /com_prefs image. The Writer interface is +// deliberately just write(bytes, size), which lets the transaction helper test +// every short-write boundary without an Arduino filesystem. +template +static bool writeCommonPrefsImage(Writer& writer, const NodePrefs* prefs) { + uint8_t pad[8]; + memset(pad, 0, sizeof(pad)); + +#define WRITE_COMMON_PREFS(value) \ + do { \ + if (writer.write((const uint8_t *)(value), sizeof(*(value))) != sizeof(*(value))) return false; \ + } while (0) +#define WRITE_COMMON_PREFS_BYTES(value, size) \ + do { \ + if (writer.write((const uint8_t *)(value), (size)) != (size)) return false; \ + } while (0) + + WRITE_COMMON_PREFS(&prefs->airtime_factor); // 0 + WRITE_COMMON_PREFS(&prefs->node_name); // 4 + WRITE_COMMON_PREFS_BYTES(pad, 4); // 36 + WRITE_COMMON_PREFS(&prefs->node_lat); // 40 + WRITE_COMMON_PREFS(&prefs->node_lon); // 48 + WRITE_COMMON_PREFS_BYTES(prefs->password, sizeof(prefs->password)); // 56 + WRITE_COMMON_PREFS(&prefs->freq); // 72 + WRITE_COMMON_PREFS(&prefs->tx_power_dbm); // 76 + WRITE_COMMON_PREFS(&prefs->disable_fwd); // 77 + WRITE_COMMON_PREFS(&prefs->advert_interval); // 78 + WRITE_COMMON_PREFS_BYTES(pad, 1); // 79 + WRITE_COMMON_PREFS(&prefs->rx_delay_base); // 80 + WRITE_COMMON_PREFS(&prefs->tx_delay_factor); // 84 + WRITE_COMMON_PREFS_BYTES(prefs->guest_password, sizeof(prefs->guest_password)); // 88 + WRITE_COMMON_PREFS(&prefs->direct_tx_delay_factor); // 104 + WRITE_COMMON_PREFS_BYTES(pad, 4); // 108 + WRITE_COMMON_PREFS(&prefs->sf); // 112 + WRITE_COMMON_PREFS(&prefs->cr); // 113 + WRITE_COMMON_PREFS(&prefs->allow_read_only); // 114 + WRITE_COMMON_PREFS(&prefs->multi_acks); // 115 + WRITE_COMMON_PREFS(&prefs->bw); // 116 + WRITE_COMMON_PREFS(&prefs->agc_reset_interval); // 120 + WRITE_COMMON_PREFS(&prefs->path_hash_mode); // 121 + WRITE_COMMON_PREFS(&prefs->loop_detect); // 122 + WRITE_COMMON_PREFS_BYTES(pad, 1); // 123 + WRITE_COMMON_PREFS(&prefs->flood_max); // 124 + WRITE_COMMON_PREFS(&prefs->flood_advert_interval); // 125 + WRITE_COMMON_PREFS(&prefs->interference_threshold); // 126 + WRITE_COMMON_PREFS(&prefs->bridge_enabled); // 127 + WRITE_COMMON_PREFS(&prefs->bridge_delay); // 128 + WRITE_COMMON_PREFS(&prefs->bridge_pkt_src); // 130 + WRITE_COMMON_PREFS(&prefs->bridge_baud); // 131 + WRITE_COMMON_PREFS(&prefs->bridge_channel); // 135 + WRITE_COMMON_PREFS_BYTES(prefs->bridge_secret, sizeof(prefs->bridge_secret)); // 136 + WRITE_COMMON_PREFS(&prefs->powersaving_enabled); // 152 + WRITE_COMMON_PREFS_BYTES(pad, 3); // 153 + WRITE_COMMON_PREFS(&prefs->gps_enabled); // 156 + WRITE_COMMON_PREFS(&prefs->gps_interval); // 157 + WRITE_COMMON_PREFS(&prefs->advert_loc_policy); // 161 + WRITE_COMMON_PREFS(&prefs->discovery_mod_timestamp); // 162 + WRITE_COMMON_PREFS(&prefs->adc_multiplier); // 166 + WRITE_COMMON_PREFS_BYTES(prefs->owner_info, sizeof(prefs->owner_info)); // 170 + // MQTT/observer settings are stored in /mqtt_prefs, not here. No zero-gap is + // written anymore — /com_prefs holds only the (non-observer) fields below. + // These trailing writes are COM_PREFS_TAIL_BYTES; keep the two in sync. + WRITE_COMMON_PREFS(&prefs->rx_boosted_gain); // 290 + WRITE_COMMON_PREFS(&prefs->flood_max_unscoped); // 291 + WRITE_COMMON_PREFS(&prefs->flood_max_advert); // 292 + WRITE_COMMON_PREFS(&prefs->radio_fem_rxgain); // 293 + WRITE_COMMON_PREFS(&prefs->cad_enabled); // 294 + +#undef WRITE_COMMON_PREFS_BYTES +#undef WRITE_COMMON_PREFS + return true; +} + +void CommonCLI::savePrefs(FILESYSTEM* fs, bool save_mqtt) { #if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) fs->remove("/com_prefs"); File file = fs->open("/com_prefs", FILE_O_WRITE); @@ -321,66 +455,13 @@ void CommonCLI::savePrefs(FILESYSTEM* fs) { File file = fs->open("/com_prefs", "w", true); #endif if (file) { - uint8_t pad[8]; - memset(pad, 0, sizeof(pad)); - - file.write((uint8_t *)&_prefs->airtime_factor, sizeof(_prefs->airtime_factor)); // 0 - file.write((uint8_t *)&_prefs->node_name, sizeof(_prefs->node_name)); // 4 - file.write(pad, 4); // 36 - file.write((uint8_t *)&_prefs->node_lat, sizeof(_prefs->node_lat)); // 40 - file.write((uint8_t *)&_prefs->node_lon, sizeof(_prefs->node_lon)); // 48 - file.write((uint8_t *)&_prefs->password[0], sizeof(_prefs->password)); // 56 - file.write((uint8_t *)&_prefs->freq, sizeof(_prefs->freq)); // 72 - file.write((uint8_t *)&_prefs->tx_power_dbm, sizeof(_prefs->tx_power_dbm)); // 76 - file.write((uint8_t *)&_prefs->disable_fwd, sizeof(_prefs->disable_fwd)); // 77 - file.write((uint8_t *)&_prefs->advert_interval, sizeof(_prefs->advert_interval)); // 78 - file.write(pad, 1); // 79 : 1 byte unused (rx_boosted_gain moved to end) - file.write((uint8_t *)&_prefs->rx_delay_base, sizeof(_prefs->rx_delay_base)); // 80 - file.write((uint8_t *)&_prefs->tx_delay_factor, sizeof(_prefs->tx_delay_factor)); // 84 - file.write((uint8_t *)&_prefs->guest_password[0], sizeof(_prefs->guest_password)); // 88 - file.write((uint8_t *)&_prefs->direct_tx_delay_factor, sizeof(_prefs->direct_tx_delay_factor)); // 104 - file.write(pad, 4); // 108 - file.write((uint8_t *)&_prefs->sf, sizeof(_prefs->sf)); // 112 - file.write((uint8_t *)&_prefs->cr, sizeof(_prefs->cr)); // 113 - file.write((uint8_t *)&_prefs->allow_read_only, sizeof(_prefs->allow_read_only)); // 114 - file.write((uint8_t *)&_prefs->multi_acks, sizeof(_prefs->multi_acks)); // 115 - file.write((uint8_t *)&_prefs->bw, sizeof(_prefs->bw)); // 116 - file.write((uint8_t *)&_prefs->agc_reset_interval, sizeof(_prefs->agc_reset_interval)); // 120 - file.write((uint8_t *)&_prefs->path_hash_mode, sizeof(_prefs->path_hash_mode)); // 121 - file.write((uint8_t *)&_prefs->loop_detect, sizeof(_prefs->loop_detect)); // 122 - file.write(pad, 1); // 123 - file.write((uint8_t *)&_prefs->flood_max, sizeof(_prefs->flood_max)); // 124 - file.write((uint8_t *)&_prefs->flood_advert_interval, sizeof(_prefs->flood_advert_interval)); // 125 - file.write((uint8_t *)&_prefs->interference_threshold, sizeof(_prefs->interference_threshold)); // 126 - file.write((uint8_t *)&_prefs->bridge_enabled, sizeof(_prefs->bridge_enabled)); // 127 - file.write((uint8_t *)&_prefs->bridge_delay, sizeof(_prefs->bridge_delay)); // 128 - file.write((uint8_t *)&_prefs->bridge_pkt_src, sizeof(_prefs->bridge_pkt_src)); // 130 - file.write((uint8_t *)&_prefs->bridge_baud, sizeof(_prefs->bridge_baud)); // 131 - file.write((uint8_t *)&_prefs->bridge_channel, sizeof(_prefs->bridge_channel)); // 135 - file.write((uint8_t *)&_prefs->bridge_secret, sizeof(_prefs->bridge_secret)); // 136 - file.write((uint8_t *)&_prefs->powersaving_enabled, sizeof(_prefs->powersaving_enabled)); // 152 - file.write(pad, 3); // 153 - file.write((uint8_t *)&_prefs->gps_enabled, sizeof(_prefs->gps_enabled)); // 156 - file.write((uint8_t *)&_prefs->gps_interval, sizeof(_prefs->gps_interval)); // 157 - file.write((uint8_t *)&_prefs->advert_loc_policy, sizeof(_prefs->advert_loc_policy)); // 161 - file.write((uint8_t *)&_prefs->discovery_mod_timestamp, sizeof(_prefs->discovery_mod_timestamp)); // 162 - file.write((uint8_t *)&_prefs->adc_multiplier, sizeof(_prefs->adc_multiplier)); // 166 - file.write((uint8_t *)_prefs->owner_info, sizeof(_prefs->owner_info)); // 170 - // MQTT/observer settings are stored in /mqtt_prefs, not here. No zero-gap is - // written anymore — /com_prefs holds only the (non-observer) fields below. - // These trailing writes are COM_PREFS_TAIL_BYTES; keep the two in sync. - file.write((uint8_t *)&_prefs->rx_boosted_gain, sizeof(_prefs->rx_boosted_gain)); // 290 - file.write((uint8_t *)&_prefs->flood_max_unscoped, sizeof(_prefs->flood_max_unscoped)); // 291 - file.write((uint8_t *)&_prefs->flood_max_advert, sizeof(_prefs->flood_max_advert)); // 292 - file.write((uint8_t *)&_prefs->radio_fem_rxgain, sizeof(_prefs->radio_fem_rxgain)); // 293 - file.write((uint8_t *)&_prefs->cad_enabled, sizeof(_prefs->cad_enabled)); // 294 - + writeCommonPrefsImage(file, _prefs); file.close(); } #ifdef WITH_MQTT_BRIDGE // Observer config (MQTT/WiFi/timezone/SNMP/alert) is persisted separately. The // observer CLI writes _mqtt_prefs directly, so no NodePrefs->MQTTPrefs sync runs. - saveMQTTPrefs(fs); + if (save_mqtt) saveMQTTPrefs(fs); #endif } @@ -398,228 +479,307 @@ static File openMqttPrefsRead(FILESYSTEM* fs) { #endif } -void CommonCLI::loadMQTTPrefs(FILESYSTEM* fs) { - // Initialize with defaults first - setMQTTPrefsDefaults(&_mqtt_prefs); - _mqtt_prefs_hold = false; +// Filesystem adapter for MQTTPrefsAtomicStore. It writes the new image only to +// /mqtt_prefs.tmp, then relies on the filesystem's atomic rename to publish it +// over /mqtt_prefs. It never removes the current source file itself. +class MQTTPrefsFileStore { +public: + explicit MQTTPrefsFileStore(FILESYSTEM* fs) : _fs(fs) {} - // Whether the loaded /mqtt_prefs already contained the observer fields (snmp/ - // watchdog/alert) appended in Phase 2 — if not, they may be carried over from an - // old-format /com_prefs trailing block below. - bool has_observer_fields = false; - - bool file_existed = fs->exists("/mqtt_prefs"); - if (file_existed) { - // First, peek the header to see if this is a versioned file. - File file = openMqttPrefsRead(fs); - bool versioned = false; - if (file) { - size_t file_size = file.size(); - if (file_size >= sizeof(MQTTPrefsHeader)) { - MQTTPrefsHeader hdr; - if (file.read((uint8_t *)&hdr, sizeof(hdr)) == sizeof(hdr) - && memcmp(hdr.magic, MQTT_PREFS_MAGIC, sizeof(hdr.magic)) == 0) { - versioned = true; - if (hdr.version == MQTT_PREFS_VERSION) { - // Current version: the payload follows the header. Read up to - // sizeof(MQTTPrefs); a shorter payload (an earlier v1 firmware that - // hadn't appended a tail field) leaves the trailing fields at their - // defaults, and a longer one (a future append) is truncated harmlessly. - size_t payload_avail = file_size - sizeof(hdr); - size_t to_read = payload_avail < sizeof(_mqtt_prefs) ? payload_avail : sizeof(_mqtt_prefs); - size_t got = file.read((uint8_t *)&_mqtt_prefs, to_read); - if (got != to_read) { - setMQTTPrefsDefaults(&_mqtt_prefs); - } else { - has_observer_fields = true; // observer tail is part of the v1 payload - } - } else { - // Unknown (newer) version: don't risk misreading a layout we don't know. - // Keep defaults for this boot and hold the file so later savePrefs() - // calls can't overwrite the newer config (no downgrade). - _mqtt_prefs_hold = true; - MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs version unsupported, using defaults (file preserved)"); - } - } - } - file.close(); - } - - if (!versioned) { - // Headerless (legacy) file. Detect the historical on-disk layout by size, - // migrate it into the compact versioned struct, and re-save — which adds the - // header and drops the vestigial `_legacy_*` fields. Reopen because the peek - // above advanced the read cursor past the (non-matching) leading bytes. - File file = openMqttPrefsRead(fs); - if (file) { - size_t file_size = file.size(); - - // Detect old (pre-slot) format by file size. - // Old MQTTPrefs was ~472 bytes (no slot fields). - // If the file is smaller than the new struct but close to OldMQTTPrefs size, - // read it with the old layout and migrate. - if (file_size > 0 && file_size <= sizeof(OldMQTTPrefs)) { - OldMQTTPrefs old_prefs; - memset(&old_prefs, 0, sizeof(old_prefs)); - size_t bytes_read = file.read((uint8_t *)&old_prefs, file_size < sizeof(old_prefs) ? file_size : sizeof(old_prefs)); - file.close(); - - if (bytes_read > 0) { - MESH_DEBUG_PRINTLN("MQTT: Migrating old-format prefs to versioned layout"); - - // Copy common fields (identical layout at start of both structs) - memcpy(_mqtt_prefs.mqtt_origin, old_prefs.mqtt_origin, sizeof(_mqtt_prefs.mqtt_origin)); - memcpy(_mqtt_prefs.mqtt_iata, old_prefs.mqtt_iata, sizeof(_mqtt_prefs.mqtt_iata)); - _mqtt_prefs.mqtt_status_enabled = old_prefs.mqtt_status_enabled; - _mqtt_prefs.mqtt_packets_enabled = old_prefs.mqtt_packets_enabled; - _mqtt_prefs.mqtt_raw_enabled = old_prefs.mqtt_raw_enabled; - _mqtt_prefs.mqtt_tx_enabled = old_prefs.mqtt_tx_enabled; - _mqtt_prefs.mqtt_status_interval = old_prefs.mqtt_status_interval; - memcpy(_mqtt_prefs.wifi_ssid, old_prefs.wifi_ssid, sizeof(_mqtt_prefs.wifi_ssid)); - memcpy(_mqtt_prefs.wifi_password, old_prefs.wifi_password, sizeof(_mqtt_prefs.wifi_password)); - _mqtt_prefs.wifi_power_save = old_prefs.wifi_power_save; - memcpy(_mqtt_prefs.timezone_string, old_prefs.timezone_string, sizeof(_mqtt_prefs.timezone_string)); - _mqtt_prefs.timezone_offset = old_prefs.timezone_offset; - - // Migrate shared auth fields - memcpy(_mqtt_prefs.mqtt_owner_public_key, old_prefs.mqtt_owner_public_key, sizeof(_mqtt_prefs.mqtt_owner_public_key)); - memcpy(_mqtt_prefs.mqtt_email, old_prefs.mqtt_email, sizeof(_mqtt_prefs.mqtt_email)); - - // Migrate analyzer presets to slots - if (old_prefs.mqtt_analyzer_us_enabled == 1) { - strncpy(_mqtt_prefs.mqtt_slot_preset[0], "analyzer-us", sizeof(_mqtt_prefs.mqtt_slot_preset[0]) - 1); - } else { - strncpy(_mqtt_prefs.mqtt_slot_preset[0], "none", sizeof(_mqtt_prefs.mqtt_slot_preset[0]) - 1); - } - if (old_prefs.mqtt_analyzer_eu_enabled == 1) { - strncpy(_mqtt_prefs.mqtt_slot_preset[1], "analyzer-eu", sizeof(_mqtt_prefs.mqtt_slot_preset[1]) - 1); - } else { - strncpy(_mqtt_prefs.mqtt_slot_preset[1], "none", sizeof(_mqtt_prefs.mqtt_slot_preset[1]) - 1); - } - - // Migrate custom server to slot 3 - if (old_prefs.mqtt_server[0] != '\0' && old_prefs.mqtt_port > 0) { - strncpy(_mqtt_prefs.mqtt_slot_preset[2], "custom", sizeof(_mqtt_prefs.mqtt_slot_preset[2]) - 1); - strncpy(_mqtt_prefs.mqtt_slot_host[2], old_prefs.mqtt_server, sizeof(_mqtt_prefs.mqtt_slot_host[2]) - 1); - _mqtt_prefs.mqtt_slot_port[2] = old_prefs.mqtt_port; - strncpy(_mqtt_prefs.mqtt_slot_username[2], old_prefs.mqtt_username, sizeof(_mqtt_prefs.mqtt_slot_username[2]) - 1); - strncpy(_mqtt_prefs.mqtt_slot_password[2], old_prefs.mqtt_password, sizeof(_mqtt_prefs.mqtt_slot_password[2]) - 1); - } else { - strncpy(_mqtt_prefs.mqtt_slot_preset[2], "none", sizeof(_mqtt_prefs.mqtt_slot_preset[2]) - 1); - } - - // Save migrated prefs in the versioned format - saveMQTTPrefs(fs); - } - } else if (file_size > 0 && file_size <= sizeof(ThreeSlotMQTTPrefs)) { - // 3-slot format → compact 6-slot migration - // Array sizes changed from [3] to [6], shifting all field offsets. - // Read into old layout struct and field-copy to new layout. - ThreeSlotMQTTPrefs old3; - memset(&old3, 0, sizeof(old3)); - size_t bytes_to_read = file_size < sizeof(old3) ? file_size : sizeof(old3); - size_t bytes_read = file.read((uint8_t *)&old3, bytes_to_read); - file.close(); - - if (bytes_read > 0) { - MESH_DEBUG_PRINTLN("MQTT: Migrating 3-slot prefs to versioned layout"); - - // Copy non-slot fields (identical layout) - memcpy(_mqtt_prefs.mqtt_origin, old3.mqtt_origin, sizeof(_mqtt_prefs.mqtt_origin)); - memcpy(_mqtt_prefs.mqtt_iata, old3.mqtt_iata, sizeof(_mqtt_prefs.mqtt_iata)); - _mqtt_prefs.mqtt_status_enabled = old3.mqtt_status_enabled; - _mqtt_prefs.mqtt_packets_enabled = old3.mqtt_packets_enabled; - _mqtt_prefs.mqtt_raw_enabled = old3.mqtt_raw_enabled; - _mqtt_prefs.mqtt_tx_enabled = old3.mqtt_tx_enabled; - _mqtt_prefs.mqtt_status_interval = old3.mqtt_status_interval; - memcpy(_mqtt_prefs.wifi_ssid, old3.wifi_ssid, sizeof(_mqtt_prefs.wifi_ssid)); - memcpy(_mqtt_prefs.wifi_password, old3.wifi_password, sizeof(_mqtt_prefs.wifi_password)); - _mqtt_prefs.wifi_power_save = old3.wifi_power_save; - memcpy(_mqtt_prefs.timezone_string, old3.timezone_string, sizeof(_mqtt_prefs.timezone_string)); - _mqtt_prefs.timezone_offset = old3.timezone_offset; - - // Copy slot fields for indices 0-2 from old layout - for (int i = 0; i < 3; i++) { - memcpy(_mqtt_prefs.mqtt_slot_preset[i], old3.mqtt_slot_preset[i], sizeof(_mqtt_prefs.mqtt_slot_preset[i])); - memcpy(_mqtt_prefs.mqtt_slot_host[i], old3.mqtt_slot_host[i], sizeof(_mqtt_prefs.mqtt_slot_host[i])); - _mqtt_prefs.mqtt_slot_port[i] = old3.mqtt_slot_port[i]; - memcpy(_mqtt_prefs.mqtt_slot_username[i], old3.mqtt_slot_username[i], sizeof(_mqtt_prefs.mqtt_slot_username[i])); - memcpy(_mqtt_prefs.mqtt_slot_password[i], old3.mqtt_slot_password[i], sizeof(_mqtt_prefs.mqtt_slot_password[i])); - memcpy(_mqtt_prefs.mqtt_slot_token[i], old3.mqtt_slot_token[i], sizeof(_mqtt_prefs.mqtt_slot_token[i])); - memcpy(_mqtt_prefs.mqtt_slot_topic[i], old3.mqtt_slot_topic[i], sizeof(_mqtt_prefs.mqtt_slot_topic[i])); - } - // Slots 3-5 keep defaults ("none") from setMQTTPrefsDefaults() - - // Copy shared auth fields - memcpy(_mqtt_prefs.mqtt_owner_public_key, old3.mqtt_owner_public_key, sizeof(_mqtt_prefs.mqtt_owner_public_key)); - memcpy(_mqtt_prefs.mqtt_email, old3.mqtt_email, sizeof(_mqtt_prefs.mqtt_email)); - - // Save migrated prefs in the versioned format - saveMQTTPrefs(fs); - } - } else if (file_size > 0) { - // Headerless 6-slot layout as shipped on mqtt-bridge-implementation-flex - // (the deployed fleet). Same field order as the compact struct but with the - // vestigial `_legacy_*` block mid-struct and no observer tail — so read it - // into Legacy6SlotMQTTPrefs and field-copy across, dropping `_legacy_*`. - Legacy6SlotMQTTPrefs old6; - memset(&old6, 0, sizeof(old6)); - size_t bytes_to_read = file_size < sizeof(old6) ? file_size : sizeof(old6); - size_t bytes_read = file.read((uint8_t *)&old6, bytes_to_read); - file.close(); - - if (bytes_read > 0) { - MESH_DEBUG_PRINTLN("MQTT: Migrating headerless 6-slot prefs to versioned layout"); - - memcpy(_mqtt_prefs.mqtt_origin, old6.mqtt_origin, sizeof(_mqtt_prefs.mqtt_origin)); - memcpy(_mqtt_prefs.mqtt_iata, old6.mqtt_iata, sizeof(_mqtt_prefs.mqtt_iata)); - _mqtt_prefs.mqtt_status_enabled = old6.mqtt_status_enabled; - _mqtt_prefs.mqtt_packets_enabled = old6.mqtt_packets_enabled; - _mqtt_prefs.mqtt_raw_enabled = old6.mqtt_raw_enabled; - _mqtt_prefs.mqtt_tx_enabled = old6.mqtt_tx_enabled; - _mqtt_prefs.mqtt_status_interval = old6.mqtt_status_interval; - memcpy(_mqtt_prefs.wifi_ssid, old6.wifi_ssid, sizeof(_mqtt_prefs.wifi_ssid)); - memcpy(_mqtt_prefs.wifi_password, old6.wifi_password, sizeof(_mqtt_prefs.wifi_password)); - _mqtt_prefs.wifi_power_save = old6.wifi_power_save; - memcpy(_mqtt_prefs.timezone_string, old6.timezone_string, sizeof(_mqtt_prefs.timezone_string)); - _mqtt_prefs.timezone_offset = old6.timezone_offset; - memcpy(_mqtt_prefs.mqtt_slot_preset, old6.mqtt_slot_preset, sizeof(_mqtt_prefs.mqtt_slot_preset)); - memcpy(_mqtt_prefs.mqtt_slot_host, old6.mqtt_slot_host, sizeof(_mqtt_prefs.mqtt_slot_host)); - memcpy(_mqtt_prefs.mqtt_slot_port, old6.mqtt_slot_port, sizeof(_mqtt_prefs.mqtt_slot_port)); - memcpy(_mqtt_prefs.mqtt_slot_username, old6.mqtt_slot_username, sizeof(_mqtt_prefs.mqtt_slot_username)); - memcpy(_mqtt_prefs.mqtt_slot_password, old6.mqtt_slot_password, sizeof(_mqtt_prefs.mqtt_slot_password)); - memcpy(_mqtt_prefs.mqtt_owner_public_key, old6.mqtt_owner_public_key, sizeof(_mqtt_prefs.mqtt_owner_public_key)); - memcpy(_mqtt_prefs.mqtt_email, old6.mqtt_email, sizeof(_mqtt_prefs.mqtt_email)); - // `_legacy_*` fields are intentionally dropped here. - memcpy(_mqtt_prefs.mqtt_slot_token, old6.mqtt_slot_token, sizeof(_mqtt_prefs.mqtt_slot_token)); - memcpy(_mqtt_prefs.mqtt_slot_topic, old6.mqtt_slot_topic, sizeof(_mqtt_prefs.mqtt_slot_topic)); - memcpy(_mqtt_prefs.mqtt_slot_audience, old6.mqtt_slot_audience, sizeof(_mqtt_prefs.mqtt_slot_audience)); - _mqtt_prefs.mqtt_rx_enabled = old6.mqtt_rx_enabled; - memcpy(_mqtt_prefs.mqtt_ntp_server, old6.mqtt_ntp_server, sizeof(_mqtt_prefs.mqtt_ntp_server)); - // Observer tail (snmp/watchdog/alert) keeps defaults; if this device is - // also upgrading across the NodePrefs split, loadPrefsInt captured those - // values from /com_prefs and they are applied below. - - saveMQTTPrefs(fs); - } - } else { - file.close(); - setMQTTPrefsDefaults(&_mqtt_prefs); - } - } - } - } else { - // No /mqtt_prefs file — defaults already set. (MQTT slot/WiFi settings from - // pre-/mqtt_prefs-split fork firmware are NOT recovered from /com_prefs — that - // offset-based migration was fragile and was removed; those users re-enter - // their MQTT config. The observer trailing block IS recovered, below.) + bool begin() { + _finished = false; + _open = false; + _bytes_written = 0; + _fs->remove("/mqtt_prefs.tmp"); // clear only a stale, unpublished transaction + if (_fs->exists("/mqtt_prefs.tmp")) return false; +#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) + _file = _fs->open("/mqtt_prefs.tmp", FILE_O_WRITE); +#elif defined(RP2040_PLATFORM) + _file = _fs->open("/mqtt_prefs.tmp", "w"); +#else + _file = _fs->open("/mqtt_prefs.tmp", "w", true); +#endif + _open = _file; + return _open; + } + + size_t write(const uint8_t* bytes, size_t size) { + if (!_open) return 0; + const size_t written = _file.write(bytes, size); + _bytes_written += written; + return written; + } + + bool finish() { + if (!_open) return false; + _file.close(); + _open = false; +#if defined(RP2040_PLATFORM) + File verify = _fs->open("/mqtt_prefs.tmp", "r"); +#else + File verify = _fs->open("/mqtt_prefs.tmp"); +#endif + if (!verify) return false; + const bool complete = verify.size() == _bytes_written; + verify.close(); + if (!complete) return false; + _finished = true; + return true; + } + + bool commit() { + return _finished && _fs->rename("/mqtt_prefs.tmp", "/mqtt_prefs"); + } + + void abort() { + if (_open) _file.close(); + _open = false; + _finished = false; + _fs->remove("/mqtt_prefs.tmp"); + } + +private: + FILESYSTEM* _fs; + File _file; + bool _open = false; + bool _finished = false; + size_t _bytes_written = 0; +}; + +#endif // WITH_MQTT_BRIDGE + +// The old /node_prefs name is only removed after this transaction has published +// a complete /com_prefs image. At this migration point /com_prefs is absent, so +// rename never needs a platform-specific replace-existing implementation. +class CommonPrefsFileStore { +public: + explicit CommonPrefsFileStore(FILESYSTEM* fs) : _fs(fs) {} + + bool begin() { + _finished = false; + _open = false; + _bytes_written = 0; + // The old-name migration only starts when /com_prefs is absent. Refuse to + // overwrite a destination that appeared unexpectedly before this handoff. + if (_fs->exists("/com_prefs")) return false; + _fs->remove("/com_prefs.tmp"); // clear only stale, unpublished output + if (_fs->exists("/com_prefs.tmp")) return false; +#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) + _file = _fs->open("/com_prefs.tmp", FILE_O_WRITE); +#elif defined(RP2040_PLATFORM) + _file = _fs->open("/com_prefs.tmp", "w"); +#else + _file = _fs->open("/com_prefs.tmp", "w", true); +#endif + _open = _file; + return _open; + } + + size_t write(const uint8_t* bytes, size_t size) { + if (!_open) return 0; + const size_t written = _file.write(bytes, size); + _bytes_written += written; + return written; + } + + bool finish() { + if (!_open) return false; + _file.close(); + _open = false; +#if defined(RP2040_PLATFORM) + File verify = _fs->open("/com_prefs.tmp", "r"); +#else + File verify = _fs->open("/com_prefs.tmp"); +#endif + if (!verify) return false; + const bool complete = verify.size() == _bytes_written; + verify.close(); + if (!complete) return false; + _finished = true; + return true; + } + + bool commit() { + return _finished && _fs->rename("/com_prefs.tmp", "/com_prefs"); + } + + void abort() { + if (_open) _file.close(); + _open = false; + _finished = false; + _fs->remove("/com_prefs.tmp"); + } + +private: + FILESYSTEM* _fs; + File _file; + bool _open = false; + bool _finished = false; + size_t _bytes_written = 0; +}; + +static const char* commonPrefsSaveResultName(MQTTPrefsAtomicStore::ImageResult result) { + switch (result) { + case MQTTPrefsAtomicStore::ImageResult::BeginFailed: return "begin"; + case MQTTPrefsAtomicStore::ImageResult::WriteFailed: return "write"; + case MQTTPrefsAtomicStore::ImageResult::FinishFailed: return "close"; + case MQTTPrefsAtomicStore::ImageResult::CommitFailed: return "rename"; + case MQTTPrefsAtomicStore::ImageResult::Committed: return "committed"; + } + return "unknown"; +} + +bool CommonCLI::saveCommonPrefsImageAtomically(FILESYSTEM* fs) { + CommonPrefsFileStore store(fs); + const MQTTPrefsAtomicStore::ImageResult result = MQTTPrefsAtomicStore::writeImage( + store, [this](CommonPrefsFileStore& target) { + return writeCommonPrefsImage(target, _prefs); + }); + if (!MQTTPrefsAtomicStore::imageCommitted(result)) { + MESH_DEBUG_PRINTLN("Prefs: atomic /com_prefs migration save failed at %s; /node_prefs preserved", + commonPrefsSaveResultName(result)); + return false; + } + return true; +} + +#ifdef WITH_MQTT_BRIDGE + +static const char* mqttPrefsSaveResultName(MQTTPrefsAtomicStore::Result result) { + switch (result) { + case MQTTPrefsAtomicStore::Result::BeginFailed: return "begin"; + case MQTTPrefsAtomicStore::Result::HeaderWriteFailed: return "header write"; + case MQTTPrefsAtomicStore::Result::PayloadWriteFailed: return "payload write"; + case MQTTPrefsAtomicStore::Result::FinishFailed: return "close"; + case MQTTPrefsAtomicStore::Result::CommitFailed: return "rename"; + case MQTTPrefsAtomicStore::Result::Committed: return "committed"; + } + return "unknown"; +} + +void CommonCLI::loadMQTTPrefs( + FILESYSTEM* fs, MQTTPrefsAtomicStore::LegacyUpgradeGate* legacy_upgrade) { + setMQTTPrefsDefaults(&_mqtt_prefs); + _mqtt_prefs_hold = false; + bool has_observer_fields = false; + bool mqtt_rewrite_pending = false; + bool migrated_legacy_mqtt = false; + + if (fs->exists("/mqtt_prefs")) { + File file = openMqttPrefsRead(fs); + if (file) { + const size_t file_size = file.size(); + uint8_t prefix[sizeof(MQTTPrefsHeader)] = {}; + const size_t prefix_size = file_size < sizeof(prefix) ? file_size : sizeof(prefix); + const size_t prefix_read = file.read(prefix, prefix_size); + file.close(); + + const MQTTPrefsCodec::DecodePlan plan = + MQTTPrefsCodec::classify(prefix, prefix_read, file_size); + if (plan.preserve_file) { + _mqtt_prefs_hold = true; + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs is unsupported or corrupt, using defaults (file preserved)"); + } else if (plan.source == MQTTPrefsCodec::Source::Current) { + file = openMqttPrefsRead(fs); + MQTTPrefsHeader header; + if (!file || file.read((uint8_t *)&header, sizeof(header)) != sizeof(header) || + file.read((uint8_t *)&_mqtt_prefs, plan.payload_len) != plan.payload_len) { + setMQTTPrefsDefaults(&_mqtt_prefs); + _mqtt_prefs_hold = true; + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs read failed, using defaults (file preserved)"); + } else { + has_observer_fields = plan.observer_fields_present; + } + if (file) file.close(); + } else if (plan.rewrite_legacy) { + bool migrated = false; + file = openMqttPrefsRead(fs); + if (file) { + switch (plan.source) { + case MQTTPrefsCodec::Source::LegacyPreSlot: { + union { + OldMQTTPrefs post_wifi_power; + PreWifiPowerOldMQTTPrefs pre_wifi_power; + } old_prefs = {}; + if (file.read((uint8_t *)&old_prefs, sizeof(old_prefs)) == sizeof(old_prefs)) { + if (MQTTPrefsCodec::isPlausibleLegacy(plan.source, + (const uint8_t *)&old_prefs, sizeof(old_prefs))) { + if (MQTTPrefsCodec::looksLikePreWifiPower((uint8_t *)&old_prefs, sizeof(old_prefs))) { + MQTTPrefsCodec::migratePreWifiPower(old_prefs.pre_wifi_power, &_mqtt_prefs); + } else { + MQTTPrefsCodec::migratePreSlot(old_prefs.post_wifi_power, &_mqtt_prefs); + } + migrated = true; + } else { + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs legacy content failed plausibility checks"); + } + } + break; + } + case MQTTPrefsCodec::Source::LegacyThreeSlotBase: { + ThreeSlotBaseMQTTPrefs old_prefs = {}; + if (file.read((uint8_t *)&old_prefs, sizeof(old_prefs)) == sizeof(old_prefs)) { + if (MQTTPrefsCodec::isPlausibleLegacy(plan.source, + (const uint8_t *)&old_prefs, sizeof(old_prefs))) { + MQTTPrefsCodec::migrateThreeSlot(old_prefs, &_mqtt_prefs); + migrated = true; + } else { + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs legacy content failed plausibility checks"); + } + } + break; + } + case MQTTPrefsCodec::Source::LegacyThreeSlot: { + ThreeSlotMQTTPrefs old_prefs = {}; + if (file.read((uint8_t *)&old_prefs, sizeof(old_prefs)) == sizeof(old_prefs)) { + if (MQTTPrefsCodec::isPlausibleLegacy(plan.source, + (const uint8_t *)&old_prefs, sizeof(old_prefs))) { + MQTTPrefsCodec::migrateThreeSlot(old_prefs, &_mqtt_prefs); + migrated = true; + } else { + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs legacy content failed plausibility checks"); + } + } + break; + } + case MQTTPrefsCodec::Source::LegacySixSlotBase: + case MQTTPrefsCodec::Source::LegacySixSlotAudience: + case MQTTPrefsCodec::Source::LegacySixSlotAudienceRx: + case MQTTPrefsCodec::Source::LegacySixSlot: { + Legacy6SlotMQTTPrefs old_prefs = {}; + if (file.read((uint8_t *)&old_prefs, plan.payload_len) == plan.payload_len) { + if (MQTTPrefsCodec::isPlausibleLegacy(plan.source, + (const uint8_t *)&old_prefs, plan.payload_len)) { + MQTTPrefsCodec::migrateLegacySixSlot(old_prefs, plan.source, &_mqtt_prefs); + migrated = true; + } else { + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs legacy content failed plausibility checks"); + } + } + break; + } + default: + break; + } + file.close(); + } + if (migrated) { + // Do not save yet: a legacy /com_prefs observer tail may still need + // to be overlaid below. Publish the complete v1 image once, after it. + mqtt_rewrite_pending = true; + migrated_legacy_mqtt = true; + } else { + setMQTTPrefsDefaults(&_mqtt_prefs); + _mqtt_prefs_hold = true; + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs legacy read failed, using defaults (file preserved)"); + } + } + } else { + _mqtt_prefs_hold = true; + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs could not be opened, using defaults (file preserved)"); + } } - // One-time upgrade path: if loadPrefsInt captured the trailing observer block of - // an old-format /com_prefs and this /mqtt_prefs predates the appended observer - // fields (or doesn't exist), carry the settings over so SNMP, radio-watchdog and - // fault-alert config survive the firmware upgrade. loadPrefs() persists both - // files in the new layout right after this. if (_legacy_tail.valid && !has_observer_fields) { _mqtt_prefs.snmp_enabled = _legacy_tail.snmp_enabled; memcpy(_mqtt_prefs.snmp_community, _legacy_tail.snmp_community, sizeof(_mqtt_prefs.snmp_community)); @@ -631,37 +791,52 @@ void CommonCLI::loadMQTTPrefs(FILESYSTEM* fs) { _mqtt_prefs.alert_min_interval_min = _legacy_tail.alert_min_interval_min; memcpy(_mqtt_prefs.alert_hashtag, _legacy_tail.alert_hashtag, sizeof(_mqtt_prefs.alert_hashtag)); memcpy(_mqtt_prefs.alert_region, _legacy_tail.alert_region, sizeof(_mqtt_prefs.alert_region)); + mqtt_rewrite_pending = true; MESH_DEBUG_PRINTLN("MQTT: Migrated observer settings from legacy /com_prefs trailing block"); } _legacy_tail.valid = false; + + if (mqtt_rewrite_pending) { + legacy_upgrade->requireMqttRewrite(); + if (migrated_legacy_mqtt) { + MESH_DEBUG_PRINTLN("MQTT: Migrating headerless /mqtt_prefs to versioned layout"); + } else { + MESH_DEBUG_PRINTLN("MQTT: Persisting observer tail into /mqtt_prefs before /com_prefs compaction"); + } + if (saveMQTTPrefs(fs)) { + legacy_upgrade->recordMqttSave(true); + } else { + // The legacy source(s) remain intact because the failed transaction never + // published its temp file. Hold this boot so loadPrefs leaves /com_prefs + // untouched; the next boot can recover the tail and retry the transaction. + _mqtt_prefs_hold = true; + legacy_upgrade->recordMqttSave(false); + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs migration save failed; legacy files preserved and held"); + } + } } -void CommonCLI::saveMQTTPrefs(FILESYSTEM* fs) { +bool CommonCLI::saveMQTTPrefs(FILESYSTEM* fs) { if (_mqtt_prefs_hold) { - // /mqtt_prefs was written by newer firmware; overwriting it here (v1 header + - // this boot's defaults) would destroy that config. Observer settings changed - // this boot are not persisted until current-or-older firmware is flashed. - MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs from newer firmware, not overwriting"); - return; + // Loading deliberately preserved the source file. Do not replace it with this + // boot's defaults after an unsupported, corrupt, or temporarily failed read. + MESH_DEBUG_PRINTLN("MQTT: /mqtt_prefs held, not overwriting"); + return false; } -#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) - fs->remove("/mqtt_prefs"); - File file = fs->open("/mqtt_prefs", FILE_O_WRITE); -#elif defined(RP2040_PLATFORM) - File file = fs->open("/mqtt_prefs", "w"); -#else - File file = fs->open("/mqtt_prefs", "w", true); -#endif - if (file) { - // Versioned format: 8-byte header followed by the raw MQTTPrefs payload. - MQTTPrefsHeader hdr; - memcpy(hdr.magic, MQTT_PREFS_MAGIC, sizeof(hdr.magic)); - hdr.version = MQTT_PREFS_VERSION; - hdr.payload_len = (uint16_t)sizeof(_mqtt_prefs); - file.write((uint8_t *)&hdr, sizeof(hdr)); - file.write((uint8_t *)&_mqtt_prefs, sizeof(_mqtt_prefs)); - file.close(); + + // Write header and payload sequentially so the transaction needs no second + // full-size (2.8 KiB) staging buffer on constrained targets. + const MQTTPrefsHeader header = MQTTPrefsCodec::makeHeader(); + MQTTPrefsFileStore store(fs); + const MQTTPrefsAtomicStore::Result result = MQTTPrefsAtomicStore::write( + store, (const uint8_t *)&header, sizeof(header), + (const uint8_t *)&_mqtt_prefs, sizeof(_mqtt_prefs)); + if (!MQTTPrefsAtomicStore::committed(result)) { + MESH_DEBUG_PRINTLN("MQTT: atomic /mqtt_prefs save failed at %s; source preserved", + mqttPrefsSaveResultName(result)); + return false; } + return true; } #endif diff --git a/src/helpers/CommonCLI.h b/src/helpers/CommonCLI.h index f0c0f479..e36dfa2d 100644 --- a/src/helpers/CommonCLI.h +++ b/src/helpers/CommonCLI.h @@ -78,180 +78,9 @@ struct NodePrefs { // persisted to file }; #ifdef WITH_MQTT_BRIDGE -// Old MQTT preferences layout (pre-slot firmware) — used only for migration detection -struct OldMQTTPrefs { - char mqtt_origin[32]; - char mqtt_iata[8]; - uint8_t mqtt_status_enabled; - uint8_t mqtt_packets_enabled; - uint8_t mqtt_raw_enabled; - uint8_t mqtt_tx_enabled; - uint32_t mqtt_status_interval; - char wifi_ssid[32]; - char wifi_password[64]; - uint8_t wifi_power_save; - char timezone_string[32]; - int8_t timezone_offset; - char mqtt_server[64]; - uint16_t mqtt_port; - char mqtt_username[32]; - char mqtt_password[64]; - uint8_t mqtt_analyzer_us_enabled; - uint8_t mqtt_analyzer_eu_enabled; - char mqtt_owner_public_key[65]; - char mqtt_email[64]; -}; - -// MQTT preferences stored in separate file to avoid conflicts with upstream NodePrefs changes -struct MQTTPrefs { - // MQTT settings - char mqtt_origin[32]; // Device name for MQTT topics - char mqtt_iata[8]; // IATA code for MQTT topics - uint8_t mqtt_status_enabled; // Enable status messages - uint8_t mqtt_packets_enabled; // Enable packet messages - uint8_t mqtt_raw_enabled; // Enable raw messages - uint8_t mqtt_tx_enabled; // Enable TX packet uplinking - uint32_t mqtt_status_interval; // Status publish interval (ms) - - // WiFi settings - char wifi_ssid[32]; // WiFi SSID - char wifi_password[64]; // WiFi password - uint8_t wifi_power_save; // WiFi power save mode: 0=min, 1=none, 2=max (default: 1=none) - - // Timezone settings - char timezone_string[32]; // Timezone string (e.g., "America/Los_Angeles") - int8_t timezone_offset; // Timezone offset in hours (-12 to +14) - fallback - - // Slot presets (up to MAX_MQTT_SLOTS) - char mqtt_slot_preset[MAX_MQTT_SLOTS][24]; // e.g. "analyzer-us", "meshmapper", "custom", "none" - - // Per-slot custom broker settings (only used when preset is "custom") - char mqtt_slot_host[MAX_MQTT_SLOTS][64]; - uint16_t mqtt_slot_port[MAX_MQTT_SLOTS]; - char mqtt_slot_username[MAX_MQTT_SLOTS][32]; - char mqtt_slot_password[MAX_MQTT_SLOTS][64]; - - // Shared authentication - char mqtt_owner_public_key[65]; // Owner public key (hex string) - char mqtt_email[64]; // Owner email address - - // Per-slot extended fields - char mqtt_slot_token[MAX_MQTT_SLOTS][48]; // Per-slot token (e.g., MeshRank account token) - char mqtt_slot_topic[MAX_MQTT_SLOTS][96]; // Per-slot custom topic template (custom preset only) - char mqtt_slot_audience[MAX_MQTT_SLOTS][64]; // JWT audience (non-empty enables JWT auth for custom slots) - - uint8_t mqtt_rx_enabled; // Enable RX packet uplinking (default: on) - char mqtt_ntp_server[64]; // Custom NTP server; empty = pool.ntp.org - - // Observer non-MQTT settings (moved out of NodePrefs so this file stays aligned - // with upstream). New fields are appended here so a shorter /mqtt_prefs payload - // from an earlier v1 firmware still loads; the missing tail keeps its default. - uint8_t snmp_enabled; // boolean - char snmp_community[24]; // community string (default "public") - uint8_t radio_watchdog_minutes; // 0=disabled, 1-120 minutes (observer-only radio recovery) - uint8_t alert_enabled; // 0 = off (default) - char alert_psk_hex[33]; // 32 hex chars + null; empty = alerts disabled - uint16_t alert_wifi_minutes; // WiFi-down threshold (0 = disabled), default 30 - uint16_t alert_mqtt_minutes; // MQTT-down threshold (0 = disabled), default 240 - uint16_t alert_min_interval_min; // min minutes between same-fault alerts, default 60 - char alert_hashtag[24]; // readback for `get alert.hashtag` - char alert_region[31]; // optional region override; empty = default_scope -}; - -// /mqtt_prefs is written with an 8-byte header so the format is self-describing. -// Files with no header are legacy (versionless) and detected by size in loadMQTTPrefs. -// The magic leads with a non-ASCII byte so it can never collide with the first -// bytes of a legacy file, whose payload starts with the mqtt_origin string. -static const uint8_t MQTT_PREFS_MAGIC[4] = {0xF5, 'M', 'Q', 'P'}; -static const uint16_t MQTT_PREFS_VERSION = 1; // bump when the MQTTPrefs payload layout changes incompatibly - -struct MQTTPrefsHeader { - uint8_t magic[4]; // MQTT_PREFS_MAGIC - uint16_t version; // MQTT_PREFS_VERSION - uint16_t payload_len; // sizeof(MQTTPrefs) at write time (sanity / forward-compat) -}; - -// 3-slot MQTTPrefs layout — used for migrating from 3-slot to 6-slot format. -// Changing array sizes from [3] to [6] shifts all field offsets, so raw file.read() -// into the new struct would corrupt data. This struct preserves the old binary layout. -struct ThreeSlotMQTTPrefs { - char mqtt_origin[32]; - char mqtt_iata[8]; - uint8_t mqtt_status_enabled; - uint8_t mqtt_packets_enabled; - uint8_t mqtt_raw_enabled; - uint8_t mqtt_tx_enabled; - uint32_t mqtt_status_interval; - char wifi_ssid[32]; - char wifi_password[64]; - uint8_t wifi_power_save; - char timezone_string[32]; - int8_t timezone_offset; - char mqtt_slot_preset[3][24]; - char mqtt_slot_host[3][64]; - uint16_t mqtt_slot_port[3]; - char mqtt_slot_username[3][32]; - char mqtt_slot_password[3][64]; - char mqtt_owner_public_key[65]; - char mqtt_email[64]; - uint8_t _legacy_analyzer_us_enabled; - uint8_t _legacy_analyzer_eu_enabled; - char _legacy_mqtt_server[64]; - uint16_t _legacy_mqtt_port; - char _legacy_mqtt_username[32]; - char _legacy_mqtt_password[64]; - char mqtt_slot_token[3][48]; - char mqtt_slot_topic[3][96]; -}; - -// Versionless 6-slot layout as shipped on mqtt-bridge-implementation-flex (the -// several-thousand-device deployed fleet). This is the current MQTTPrefs minus the -// observer tail, and it still carries the now-removed `_legacy_*` fields mid-struct. -// loadMQTTPrefs reads a headerless file of this size into this struct, then -// field-copies (dropping `_legacy_*`) into the compact versioned MQTTPrefs. -struct Legacy6SlotMQTTPrefs { - char mqtt_origin[32]; - char mqtt_iata[8]; - uint8_t mqtt_status_enabled; - uint8_t mqtt_packets_enabled; - uint8_t mqtt_raw_enabled; - uint8_t mqtt_tx_enabled; - uint32_t mqtt_status_interval; - char wifi_ssid[32]; - char wifi_password[64]; - uint8_t wifi_power_save; - char timezone_string[32]; - int8_t timezone_offset; - char mqtt_slot_preset[MAX_MQTT_SLOTS][24]; - char mqtt_slot_host[MAX_MQTT_SLOTS][64]; - uint16_t mqtt_slot_port[MAX_MQTT_SLOTS]; - char mqtt_slot_username[MAX_MQTT_SLOTS][32]; - char mqtt_slot_password[MAX_MQTT_SLOTS][64]; - char mqtt_owner_public_key[65]; - char mqtt_email[64]; - uint8_t _legacy_analyzer_us_enabled; - uint8_t _legacy_analyzer_eu_enabled; - char _legacy_mqtt_server[64]; - uint16_t _legacy_mqtt_port; - char _legacy_mqtt_username[32]; - char _legacy_mqtt_password[64]; - char mqtt_slot_token[MAX_MQTT_SLOTS][48]; - char mqtt_slot_topic[MAX_MQTT_SLOTS][96]; - char mqtt_slot_audience[MAX_MQTT_SLOTS][64]; - uint8_t mqtt_rx_enabled; - char mqtt_ntp_server[64]; -}; - -// The legacy layouts above describe files already written to the deployed fleet's -// flash, so their sizes are frozen forever — loadMQTTPrefs() tells the eras apart -// by file size and reads each file as a raw struct dump. These asserts pin the -// layouts on every target toolchain; if one fires, the compiler (or an edit to a -// legacy struct or MAX_MQTT_SLOTS) has changed a layout and fleet files would be -// read at wrong offsets. -static_assert(sizeof(MQTTPrefsHeader) == 8, "versioned /mqtt_prefs header must stay 8 bytes"); -static_assert(sizeof(OldMQTTPrefs) == 472, "frozen pre-slot /mqtt_prefs layout changed"); -static_assert(sizeof(ThreeSlotMQTTPrefs) == 1464, "frozen 3-slot /mqtt_prefs layout changed"); -static_assert(sizeof(Legacy6SlotMQTTPrefs) == 2904, "frozen deployed-fleet /mqtt_prefs layout changed"); +#include +static_assert(MQTT_PREFS_SLOT_COUNT == MAX_MQTT_SLOTS, + "MQTT prefs layout and slot count must change together"); // Observer settings captured from the trailing block of an old-format /com_prefs // (fork firmware that predates the NodePrefs -> MQTTPrefs split). loadPrefsInt() @@ -384,6 +213,12 @@ public: } }; +#ifdef WITH_MQTT_BRIDGE +namespace MQTTPrefsAtomicStore { +class LegacyUpgradeGate; +} +#endif + class CommonCLI { mesh::RTCClock* _rtc; NodePrefs* _prefs; @@ -396,9 +231,8 @@ class CommonCLI { #ifdef WITH_MQTT_BRIDGE MQTTPrefs _mqtt_prefs; LegacyObserverTail _legacy_tail; - // /mqtt_prefs carries a version newer than this firmware understands (a downgrade). - // The in-memory prefs run on defaults and saveMQTTPrefs() must not overwrite the - // file, or the first `set` command would destroy the newer config. + // /mqtt_prefs is newer, corrupt, or temporarily unreadable. The in-memory prefs + // run on defaults and saveMQTTPrefs() must not overwrite the source file. bool _mqtt_prefs_hold = false; #endif bool _com_prefs_needs_upgrade = false; // old-format /com_prefs detected; rewrite once after load @@ -406,9 +240,10 @@ class CommonCLI { mesh::RTCClock* getRTCClock() { return _rtc; } void savePrefs(); void loadPrefsInt(FILESYSTEM* _fs, const char* filename); + bool saveCommonPrefsImageAtomically(FILESYSTEM* fs); #ifdef WITH_MQTT_BRIDGE - void loadMQTTPrefs(FILESYSTEM* fs); - void saveMQTTPrefs(FILESYSTEM* fs); + void loadMQTTPrefs(FILESYSTEM* fs, MQTTPrefsAtomicStore::LegacyUpgradeGate* legacy_upgrade); + bool saveMQTTPrefs(FILESYSTEM* fs); #endif void handleRegionCmd(char* command, char* reply); @@ -430,7 +265,7 @@ public: : _board(&board), _rtc(&rtc), _sensors(&sensors), _region_map(®ion_map), _acl(&acl), _prefs(prefs), _callbacks(callbacks) { } void loadPrefs(FILESYSTEM* _fs); - void savePrefs(FILESYSTEM* _fs); + void savePrefs(FILESYSTEM* _fs, bool save_mqtt = true); void handleCommand(uint32_t sender_timestamp, char* command, char* reply); mesh::MainBoard* getBoard() { return _board; } uint8_t buildAdvertData(uint8_t node_type, uint8_t* app_data); diff --git a/src/helpers/MQTTPrefsAtomicStore.h b/src/helpers/MQTTPrefsAtomicStore.h new file mode 100644 index 00000000..be77bf99 --- /dev/null +++ b/src/helpers/MQTTPrefsAtomicStore.h @@ -0,0 +1,127 @@ +#pragma once + +#include +#include + +// Transactional writer for /mqtt_prefs. The Store interface is intentionally +// narrow so host tests can exercise every failure boundary without an Arduino +// filesystem: begin(), write(), finish(), commit(), and abort(). The caller +// supplies header and payload separately, avoiding a second full-size buffer. +namespace MQTTPrefsAtomicStore { + +enum class Result : uint8_t { + Committed, + BeginFailed, + HeaderWriteFailed, + PayloadWriteFailed, + FinishFailed, + CommitFailed, +}; + +inline bool committed(Result result) { + return result == Result::Committed; +} + +// Generic streaming transaction for structured images such as /com_prefs. +// ImageWriter writes its fields directly to Store and returns false on any +// short write, so no contiguous staging allocation is required. +enum class ImageResult : uint8_t { + Committed, + BeginFailed, + WriteFailed, + FinishFailed, + CommitFailed, +}; + +inline bool imageCommitted(ImageResult result) { + return result == ImageResult::Committed; +} + +template +inline ImageResult writeImage(Store& store, ImageWriter write_image) { + if (!store.begin()) { + store.abort(); + return ImageResult::BeginFailed; + } + if (!write_image(store)) { + store.abort(); + return ImageResult::WriteFailed; + } + if (!store.finish()) { + store.abort(); + return ImageResult::FinishFailed; + } + if (!store.commit()) { + store.abort(); + return ImageResult::CommitFailed; + } + return ImageResult::Committed; +} + +// Coordinates a two-file legacy upgrade. /com_prefs must not be compacted +// until the observer tail it carries has been published into /mqtt_prefs. +// Keeping this state in a tiny pure helper lets host tests cover power-cut +// boundaries without an Arduino filesystem. +class LegacyUpgradeGate { +public: + explicit LegacyUpgradeGate(bool com_prefs_rewrite_pending) + : _com_prefs_rewrite_pending(com_prefs_rewrite_pending) {} + + void requireMqttRewrite() { _mqtt_rewrite_pending = true; } + + void recordMqttSave(bool did_commit) { + if (did_commit) { + _mqtt_rewrite_pending = false; + _mqtt_source_held = false; + } else { + _mqtt_source_held = true; + } + } + + void holdMqttSource() { _mqtt_source_held = true; } + + bool mqttRewritePending() const { return _mqtt_rewrite_pending; } + bool blocksComPrefsRewrite() const { + return _com_prefs_rewrite_pending && (_mqtt_rewrite_pending || _mqtt_source_held); + } + bool mayRewriteComPrefs() const { + return _com_prefs_rewrite_pending && !blocksComPrefsRewrite(); + } + + void recordComPrefsRewrite() { + if (mayRewriteComPrefs()) _com_prefs_rewrite_pending = false; + } + +private: + bool _com_prefs_rewrite_pending; + bool _mqtt_rewrite_pending = false; + bool _mqtt_source_held = false; +}; + +template +inline Result write(Store& store, const uint8_t* header, size_t header_size, + const uint8_t* payload, size_t payload_size) { + if (!store.begin()) { + store.abort(); + return Result::BeginFailed; + } + if (store.write(header, header_size) != header_size) { + store.abort(); + return Result::HeaderWriteFailed; + } + if (store.write(payload, payload_size) != payload_size) { + store.abort(); + return Result::PayloadWriteFailed; + } + if (!store.finish()) { + store.abort(); + return Result::FinishFailed; + } + if (!store.commit()) { + store.abort(); + return Result::CommitFailed; + } + return Result::Committed; +} + +} // namespace MQTTPrefsAtomicStore diff --git a/src/helpers/MQTTPrefsCodec.h b/src/helpers/MQTTPrefsCodec.h new file mode 100644 index 00000000..85629dff --- /dev/null +++ b/src/helpers/MQTTPrefsCodec.h @@ -0,0 +1,439 @@ +#pragma once + +#include + +#include "MQTTPrefsStorage.h" + +#ifdef WITH_MQTT_BRIDGE + +// Pure /mqtt_prefs format classification and field-copy migration. Production +// reads directly into the selected layout; this header never requires a second +// large staging buffer. +namespace MQTTPrefsCodec { + +enum class Source : uint8_t { + Defaults, + Current, + LegacyPreSlot, + LegacyThreeSlotBase, + LegacyThreeSlot, + LegacySixSlotBase, + LegacySixSlotAudience, + LegacySixSlotAudienceRx, + LegacySixSlot, + UnsupportedVersion, + Corrupt, +}; + +struct DecodePlan { + Source source; + bool rewrite_legacy; + bool preserve_file; + // False means the decoded payload stops before snmp_enabled, so production + // may apply a captured observer tail from legacy /com_prefs. + bool observer_fields_present; + size_t payload_len; +}; + +static const size_t kV1PreObserverPayloadSize = MQTT_PREFS_V1_PRE_OBSERVER_PAYLOAD_SIZE; +static const size_t kV1BaselinePayloadSize = MQTT_PREFS_V1_FULL_PAYLOAD_SIZE; +static const size_t kEncodedSize = sizeof(MQTTPrefsHeader) + kV1BaselinePayloadSize; + +inline MQTTPrefsHeader makeHeader() { + MQTTPrefsHeader header; + memcpy(header.magic, MQTT_PREFS_MAGIC, sizeof(header.magic)); + header.version = MQTT_PREFS_VERSION; + header.payload_len = static_cast(kV1BaselinePayloadSize); + return header; +} + +inline size_t encode(const MQTTPrefs& prefs, uint8_t* output, size_t output_size) { + if (output == nullptr || output_size < kEncodedSize) return 0; + const MQTTPrefsHeader header = makeHeader(); + memcpy(output, &header, sizeof(header)); + memcpy(output + sizeof(header), &prefs, sizeof(prefs)); + return kEncodedSize; +} + +inline bool isMagicPrefix(const uint8_t* input, size_t available) { + if (input == nullptr || available == 0) return false; + const size_t compare_len = available < sizeof(MQTT_PREFS_MAGIC) + ? available : sizeof(MQTT_PREFS_MAGIC); + return memcmp(input, MQTT_PREFS_MAGIC, compare_len) == 0; +} + +inline DecodePlan corruptPlan() { + return {Source::Corrupt, false, true, false, 0}; +} + +// Classify from the first eight bytes and the filesystem-reported file size. +// Headerless layouts are an explicit, audited whitelist. Call +// isPlausibleLegacy() after reading the selected layout and before rewriting: +// size alone cannot distinguish a valid legacy payload from arbitrary bytes. +inline DecodePlan classify(const uint8_t* prefix, size_t prefix_read, size_t file_size) { + if (file_size == 0) return corruptPlan(); + if (prefix == nullptr || prefix_read == 0) return corruptPlan(); + const size_t expected_prefix = file_size < sizeof(MQTTPrefsHeader) + ? file_size : sizeof(MQTTPrefsHeader); + if (prefix_read < expected_prefix) return corruptPlan(); + + if (file_size < sizeof(MQTTPrefsHeader) && isMagicPrefix(prefix, prefix_read)) { + return corruptPlan(); + } + if (file_size >= sizeof(MQTTPrefsHeader)) { + MQTTPrefsHeader header; + memcpy(&header, prefix, sizeof(header)); + if (memcmp(header.magic, MQTT_PREFS_MAGIC, sizeof(header.magic)) == 0) { + if (header.version != MQTT_PREFS_VERSION) { + return {Source::UnsupportedVersion, false, true, false, 0}; + } + const size_t payload_available = file_size - sizeof(header); + if (header.payload_len != payload_available) { + return corruptPlan(); + } + // A same-version append is still unknown to this binary. Holding the + // file prevents a downgrade from discarding it on the next CLI save. + if (header.payload_len > kV1BaselinePayloadSize) { + return {Source::UnsupportedVersion, false, true, false, 0}; + } + if (header.payload_len == kV1BaselinePayloadSize) { + return {Source::Current, false, false, true, kV1BaselinePayloadSize}; + } + if (header.payload_len == kV1PreObserverPayloadSize) { + return {Source::Current, false, false, false, kV1PreObserverPayloadSize}; + } + return corruptPlan(); + } + } + + switch (file_size) { + case sizeof(OldMQTTPrefs): + return {Source::LegacyPreSlot, true, false, false, file_size}; + case sizeof(ThreeSlotBaseMQTTPrefs): + return {Source::LegacyThreeSlotBase, true, false, false, file_size}; + case sizeof(ThreeSlotMQTTPrefs): + return {Source::LegacyThreeSlot, true, false, false, file_size}; + case LEGACY6_BASE_SIZE: + return {Source::LegacySixSlotBase, true, false, false, file_size}; + case LEGACY6_AUDIENCE_SIZE: + return {Source::LegacySixSlotAudience, true, false, false, file_size}; + case LEGACY6_AUDIENCE_RX_SIZE: + return {Source::LegacySixSlotAudienceRx, true, false, false, file_size}; + case sizeof(Legacy6SlotMQTTPrefs): + return {Source::LegacySixSlot, true, false, false, file_size}; + default: + return corruptPlan(); + } +} + +inline bool looksLikePreWifiPower(const uint8_t* input, size_t size) { + if (input == nullptr || size != sizeof(OldMQTTPrefs)) return false; + // At byte 144 the newer layout has wifi_power_save (0..2); the older + // layout has timezone_string[0]. A non-empty timezone is unambiguous. + if (input[144] > 2) return true; + // With an empty timezone, byte 177 is the older mqtt_server[0] but the + // newer timezone_offset. If it cannot be an offset, it is also unambiguous. + const int8_t newer_offset = static_cast(input[177]); + // Byte 176 is the older timezone_offset but the final byte of the newer + // timezone string (always NUL for values saved through the CLI). + return input[144] == 0 && + (input[176] != 0 || newer_offset < -12 || newer_offset > 14); +} + +// Headerless files have no checksum or magic, so their integrity cannot be +// proven. These checks intentionally reject obvious random data (unterminated +// strings and impossible flag/range values) without demanding application-level +// values that a real but sparsely configured device may not have set. +inline bool hasTerminatedText(const uint8_t* input, size_t size, size_t offset, size_t field_size) { + if (input == nullptr || offset > size || field_size > size - offset) return false; + for (size_t i = 0; i < field_size; ++i) { + if (input[offset + i] == '\0') return true; + } + return false; +} + +inline bool hasPlausibleCommonFields(const uint8_t* input, size_t size, bool pre_wifi_power) { + if (input == nullptr || size < sizeof(OldMQTTPrefs)) return false; + const size_t timezone_offset = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, timezone_string) + : offsetof(OldMQTTPrefs, timezone_string); + const size_t utc_offset = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, timezone_offset) + : offsetof(OldMQTTPrefs, timezone_offset); + const size_t timezone_size = 32; + const int8_t timezone_hours = static_cast(input[utc_offset]); + + return input[offsetof(OldMQTTPrefs, mqtt_status_enabled)] <= 1 && + input[offsetof(OldMQTTPrefs, mqtt_packets_enabled)] <= 1 && + input[offsetof(OldMQTTPrefs, mqtt_raw_enabled)] <= 1 && + input[offsetof(OldMQTTPrefs, mqtt_tx_enabled)] <= 2 && + (pre_wifi_power || input[offsetof(OldMQTTPrefs, wifi_power_save)] <= 2) && + timezone_hours >= -12 && timezone_hours <= 14 && + hasTerminatedText(input, size, offsetof(OldMQTTPrefs, mqtt_origin), + 32) && + hasTerminatedText(input, size, offsetof(OldMQTTPrefs, mqtt_iata), + 8) && + hasTerminatedText(input, size, offsetof(OldMQTTPrefs, wifi_ssid), + 32) && + hasTerminatedText(input, size, offsetof(OldMQTTPrefs, wifi_password), + 64) && + hasTerminatedText(input, size, timezone_offset, timezone_size); +} + +inline bool hasPlausibleSlotText(const uint8_t* input, size_t size, size_t slot_count, + size_t preset_offset, size_t host_offset, size_t username_offset, + size_t password_offset, size_t token_offset, size_t topic_offset, + size_t audience_offset) { + const size_t no_field = static_cast(-1); + for (size_t i = 0; i < slot_count; ++i) { + if (!hasTerminatedText(input, size, preset_offset + i * 24, 24) || + !hasTerminatedText(input, size, host_offset + i * 64, 64) || + !hasTerminatedText(input, size, username_offset + i * 32, 32) || + !hasTerminatedText(input, size, password_offset + i * 64, 64) || + (token_offset != no_field && !hasTerminatedText(input, size, token_offset + i * 48, 48)) || + (topic_offset != no_field && !hasTerminatedText(input, size, topic_offset + i * 96, 96)) || + (audience_offset != no_field && !hasTerminatedText(input, size, audience_offset + i * 64, 64))) { + return false; + } + } + return true; +} + +inline bool hasPlausibleSharedAuth(const uint8_t* input, size_t size, size_t owner_offset, + size_t email_offset) { + return hasTerminatedText(input, size, owner_offset, 65) && + hasTerminatedText(input, size, email_offset, 64); +} + +inline bool isPlausibleLegacy(Source source, const uint8_t* input, size_t size) { + const size_t no_field = static_cast(-1); + switch (source) { + case Source::LegacyPreSlot: { + if (size != sizeof(OldMQTTPrefs)) return false; + const bool pre_wifi_power = looksLikePreWifiPower(input, size); + const size_t server_offset = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_server) + : offsetof(OldMQTTPrefs, mqtt_server); + const size_t username_offset = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_username) + : offsetof(OldMQTTPrefs, mqtt_username); + const size_t password_offset = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_password) + : offsetof(OldMQTTPrefs, mqtt_password); + const size_t us_enabled = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_analyzer_us_enabled) + : offsetof(OldMQTTPrefs, mqtt_analyzer_us_enabled); + const size_t eu_enabled = pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_analyzer_eu_enabled) + : offsetof(OldMQTTPrefs, mqtt_analyzer_eu_enabled); + return hasPlausibleCommonFields(input, size, pre_wifi_power) && + input[us_enabled] <= 1 && input[eu_enabled] <= 1 && + hasTerminatedText(input, size, server_offset, 64) && + hasTerminatedText(input, size, username_offset, 32) && + hasTerminatedText(input, size, password_offset, 64) && + hasTerminatedText(input, size, pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_owner_public_key) + : offsetof(OldMQTTPrefs, mqtt_owner_public_key), 65) && + hasTerminatedText(input, size, pre_wifi_power + ? offsetof(PreWifiPowerOldMQTTPrefs, mqtt_email) + : offsetof(OldMQTTPrefs, mqtt_email), 64); + } + case Source::LegacyThreeSlotBase: + return size == sizeof(ThreeSlotBaseMQTTPrefs) && + hasPlausibleCommonFields(input, size, false) && + hasPlausibleSharedAuth(input, size, + offsetof(ThreeSlotBaseMQTTPrefs, mqtt_owner_public_key), + offsetof(ThreeSlotBaseMQTTPrefs, mqtt_email)) && + hasPlausibleSlotText(input, size, 3, + offsetof(ThreeSlotBaseMQTTPrefs, mqtt_slot_preset), + offsetof(ThreeSlotBaseMQTTPrefs, mqtt_slot_host), + offsetof(ThreeSlotBaseMQTTPrefs, mqtt_slot_username), + offsetof(ThreeSlotBaseMQTTPrefs, mqtt_slot_password), + no_field, no_field, no_field); + case Source::LegacyThreeSlot: + return size == sizeof(ThreeSlotMQTTPrefs) && + hasPlausibleCommonFields(input, size, false) && + hasPlausibleSharedAuth(input, size, + offsetof(ThreeSlotMQTTPrefs, mqtt_owner_public_key), + offsetof(ThreeSlotMQTTPrefs, mqtt_email)) && + hasPlausibleSlotText(input, size, 3, + offsetof(ThreeSlotMQTTPrefs, mqtt_slot_preset), + offsetof(ThreeSlotMQTTPrefs, mqtt_slot_host), + offsetof(ThreeSlotMQTTPrefs, mqtt_slot_username), + offsetof(ThreeSlotMQTTPrefs, mqtt_slot_password), + offsetof(ThreeSlotMQTTPrefs, mqtt_slot_token), + offsetof(ThreeSlotMQTTPrefs, mqtt_slot_topic), no_field); + case Source::LegacySixSlotBase: + case Source::LegacySixSlotAudience: + case Source::LegacySixSlotAudienceRx: + case Source::LegacySixSlot: { + const size_t expected_size = source == Source::LegacySixSlotBase ? LEGACY6_BASE_SIZE + : source == Source::LegacySixSlotAudience ? LEGACY6_AUDIENCE_SIZE + : source == Source::LegacySixSlotAudienceRx ? LEGACY6_AUDIENCE_RX_SIZE + : sizeof(Legacy6SlotMQTTPrefs); + const size_t audience_offset = source == Source::LegacySixSlotBase + ? no_field : offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_audience); + const bool has_rx = source == Source::LegacySixSlotAudienceRx || + source == Source::LegacySixSlot; + const bool has_ntp = source == Source::LegacySixSlot; + return size == expected_size && hasPlausibleCommonFields(input, size, false) && + hasPlausibleSharedAuth(input, size, + offsetof(Legacy6SlotMQTTPrefs, mqtt_owner_public_key), + offsetof(Legacy6SlotMQTTPrefs, mqtt_email)) && + (!has_rx || input[offsetof(Legacy6SlotMQTTPrefs, mqtt_rx_enabled)] <= 1) && + (!has_ntp || hasTerminatedText(input, size, + offsetof(Legacy6SlotMQTTPrefs, mqtt_ntp_server), 64)) && + hasPlausibleSlotText(input, size, MQTT_PREFS_SLOT_COUNT, + offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_preset), + offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_host), + offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_username), + offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_password), + offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_token), + offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_topic), audience_offset); + } + default: + return false; + } +} + +inline void migratePreSlot(const OldMQTTPrefs& old_prefs, MQTTPrefs* prefs) { + memcpy(prefs->mqtt_origin, old_prefs.mqtt_origin, sizeof(prefs->mqtt_origin)); + memcpy(prefs->mqtt_iata, old_prefs.mqtt_iata, sizeof(prefs->mqtt_iata)); + prefs->mqtt_status_enabled = old_prefs.mqtt_status_enabled; + prefs->mqtt_packets_enabled = old_prefs.mqtt_packets_enabled; + prefs->mqtt_raw_enabled = old_prefs.mqtt_raw_enabled; + prefs->mqtt_tx_enabled = old_prefs.mqtt_tx_enabled; + prefs->mqtt_status_interval = old_prefs.mqtt_status_interval; + memcpy(prefs->wifi_ssid, old_prefs.wifi_ssid, sizeof(prefs->wifi_ssid)); + memcpy(prefs->wifi_password, old_prefs.wifi_password, sizeof(prefs->wifi_password)); + prefs->wifi_power_save = old_prefs.wifi_power_save; + memcpy(prefs->timezone_string, old_prefs.timezone_string, sizeof(prefs->timezone_string)); + prefs->timezone_offset = old_prefs.timezone_offset; + memcpy(prefs->mqtt_owner_public_key, old_prefs.mqtt_owner_public_key, + sizeof(prefs->mqtt_owner_public_key)); + memcpy(prefs->mqtt_email, old_prefs.mqtt_email, sizeof(prefs->mqtt_email)); + strncpy(prefs->mqtt_slot_preset[0], old_prefs.mqtt_analyzer_us_enabled == 1 + ? "analyzer-us" : "none", sizeof(prefs->mqtt_slot_preset[0]) - 1); + strncpy(prefs->mqtt_slot_preset[1], old_prefs.mqtt_analyzer_eu_enabled == 1 + ? "analyzer-eu" : "none", sizeof(prefs->mqtt_slot_preset[1]) - 1); + if (old_prefs.mqtt_server[0] != '\0' && old_prefs.mqtt_port > 0) { + strncpy(prefs->mqtt_slot_preset[2], "custom", sizeof(prefs->mqtt_slot_preset[2]) - 1); + strncpy(prefs->mqtt_slot_host[2], old_prefs.mqtt_server, + sizeof(prefs->mqtt_slot_host[2]) - 1); + prefs->mqtt_slot_port[2] = old_prefs.mqtt_port; + strncpy(prefs->mqtt_slot_username[2], old_prefs.mqtt_username, + sizeof(prefs->mqtt_slot_username[2]) - 1); + strncpy(prefs->mqtt_slot_password[2], old_prefs.mqtt_password, + sizeof(prefs->mqtt_slot_password[2]) - 1); + } else { + strncpy(prefs->mqtt_slot_preset[2], "none", sizeof(prefs->mqtt_slot_preset[2]) - 1); + } +} + +inline void migratePreWifiPower(const PreWifiPowerOldMQTTPrefs& old_prefs, MQTTPrefs* prefs) { + OldMQTTPrefs normalized = {}; + memcpy(normalized.mqtt_origin, old_prefs.mqtt_origin, sizeof(normalized.mqtt_origin)); + memcpy(normalized.mqtt_iata, old_prefs.mqtt_iata, sizeof(normalized.mqtt_iata)); + normalized.mqtt_status_enabled = old_prefs.mqtt_status_enabled; + normalized.mqtt_packets_enabled = old_prefs.mqtt_packets_enabled; + normalized.mqtt_raw_enabled = old_prefs.mqtt_raw_enabled; + normalized.mqtt_tx_enabled = old_prefs.mqtt_tx_enabled; + normalized.mqtt_status_interval = old_prefs.mqtt_status_interval; + memcpy(normalized.wifi_ssid, old_prefs.wifi_ssid, sizeof(normalized.wifi_ssid)); + memcpy(normalized.wifi_password, old_prefs.wifi_password, sizeof(normalized.wifi_password)); + normalized.wifi_power_save = prefs->wifi_power_save; // field did not exist yet + memcpy(normalized.timezone_string, old_prefs.timezone_string, sizeof(normalized.timezone_string)); + normalized.timezone_offset = old_prefs.timezone_offset; + memcpy(normalized.mqtt_server, old_prefs.mqtt_server, sizeof(normalized.mqtt_server)); + normalized.mqtt_port = old_prefs.mqtt_port; + memcpy(normalized.mqtt_username, old_prefs.mqtt_username, sizeof(normalized.mqtt_username)); + memcpy(normalized.mqtt_password, old_prefs.mqtt_password, sizeof(normalized.mqtt_password)); + normalized.mqtt_analyzer_us_enabled = old_prefs.mqtt_analyzer_us_enabled; + normalized.mqtt_analyzer_eu_enabled = old_prefs.mqtt_analyzer_eu_enabled; + memcpy(normalized.mqtt_owner_public_key, old_prefs.mqtt_owner_public_key, + sizeof(normalized.mqtt_owner_public_key)); + memcpy(normalized.mqtt_email, old_prefs.mqtt_email, sizeof(normalized.mqtt_email)); + migratePreSlot(normalized, prefs); +} + +template +inline void migrateThreeSlotCommon(const T& old_prefs, MQTTPrefs* prefs) { + memcpy(prefs->mqtt_origin, old_prefs.mqtt_origin, sizeof(prefs->mqtt_origin)); + memcpy(prefs->mqtt_iata, old_prefs.mqtt_iata, sizeof(prefs->mqtt_iata)); + prefs->mqtt_status_enabled = old_prefs.mqtt_status_enabled; + prefs->mqtt_packets_enabled = old_prefs.mqtt_packets_enabled; + prefs->mqtt_raw_enabled = old_prefs.mqtt_raw_enabled; + prefs->mqtt_tx_enabled = old_prefs.mqtt_tx_enabled; + prefs->mqtt_status_interval = old_prefs.mqtt_status_interval; + memcpy(prefs->wifi_ssid, old_prefs.wifi_ssid, sizeof(prefs->wifi_ssid)); + memcpy(prefs->wifi_password, old_prefs.wifi_password, sizeof(prefs->wifi_password)); + prefs->wifi_power_save = old_prefs.wifi_power_save; + memcpy(prefs->timezone_string, old_prefs.timezone_string, sizeof(prefs->timezone_string)); + prefs->timezone_offset = old_prefs.timezone_offset; + for (int i = 0; i < 3; i++) { + memcpy(prefs->mqtt_slot_preset[i], old_prefs.mqtt_slot_preset[i], sizeof(prefs->mqtt_slot_preset[i])); + memcpy(prefs->mqtt_slot_host[i], old_prefs.mqtt_slot_host[i], sizeof(prefs->mqtt_slot_host[i])); + prefs->mqtt_slot_port[i] = old_prefs.mqtt_slot_port[i]; + memcpy(prefs->mqtt_slot_username[i], old_prefs.mqtt_slot_username[i], sizeof(prefs->mqtt_slot_username[i])); + memcpy(prefs->mqtt_slot_password[i], old_prefs.mqtt_slot_password[i], sizeof(prefs->mqtt_slot_password[i])); + } + memcpy(prefs->mqtt_owner_public_key, old_prefs.mqtt_owner_public_key, sizeof(prefs->mqtt_owner_public_key)); + memcpy(prefs->mqtt_email, old_prefs.mqtt_email, sizeof(prefs->mqtt_email)); +} + +inline void migrateThreeSlot(const ThreeSlotBaseMQTTPrefs& old_prefs, MQTTPrefs* prefs) { + migrateThreeSlotCommon(old_prefs, prefs); +} + +inline void migrateThreeSlot(const ThreeSlotMQTTPrefs& old_prefs, MQTTPrefs* prefs) { + migrateThreeSlotCommon(old_prefs, prefs); + for (int i = 0; i < 3; i++) { + memcpy(prefs->mqtt_slot_token[i], old_prefs.mqtt_slot_token[i], sizeof(prefs->mqtt_slot_token[i])); + memcpy(prefs->mqtt_slot_topic[i], old_prefs.mqtt_slot_topic[i], sizeof(prefs->mqtt_slot_topic[i])); + } +} + +inline void migrateLegacySixSlotCommon(const Legacy6SlotMQTTPrefs& old_prefs, MQTTPrefs* prefs) { + memcpy(prefs->mqtt_origin, old_prefs.mqtt_origin, sizeof(prefs->mqtt_origin)); + memcpy(prefs->mqtt_iata, old_prefs.mqtt_iata, sizeof(prefs->mqtt_iata)); + prefs->mqtt_status_enabled = old_prefs.mqtt_status_enabled; + prefs->mqtt_packets_enabled = old_prefs.mqtt_packets_enabled; + prefs->mqtt_raw_enabled = old_prefs.mqtt_raw_enabled; + prefs->mqtt_tx_enabled = old_prefs.mqtt_tx_enabled; + prefs->mqtt_status_interval = old_prefs.mqtt_status_interval; + memcpy(prefs->wifi_ssid, old_prefs.wifi_ssid, sizeof(prefs->wifi_ssid)); + memcpy(prefs->wifi_password, old_prefs.wifi_password, sizeof(prefs->wifi_password)); + prefs->wifi_power_save = old_prefs.wifi_power_save; + memcpy(prefs->timezone_string, old_prefs.timezone_string, sizeof(prefs->timezone_string)); + prefs->timezone_offset = old_prefs.timezone_offset; + memcpy(prefs->mqtt_slot_preset, old_prefs.mqtt_slot_preset, sizeof(prefs->mqtt_slot_preset)); + memcpy(prefs->mqtt_slot_host, old_prefs.mqtt_slot_host, sizeof(prefs->mqtt_slot_host)); + memcpy(prefs->mqtt_slot_port, old_prefs.mqtt_slot_port, sizeof(prefs->mqtt_slot_port)); + memcpy(prefs->mqtt_slot_username, old_prefs.mqtt_slot_username, sizeof(prefs->mqtt_slot_username)); + memcpy(prefs->mqtt_slot_password, old_prefs.mqtt_slot_password, sizeof(prefs->mqtt_slot_password)); + memcpy(prefs->mqtt_owner_public_key, old_prefs.mqtt_owner_public_key, sizeof(prefs->mqtt_owner_public_key)); + memcpy(prefs->mqtt_email, old_prefs.mqtt_email, sizeof(prefs->mqtt_email)); + memcpy(prefs->mqtt_slot_token, old_prefs.mqtt_slot_token, sizeof(prefs->mqtt_slot_token)); + memcpy(prefs->mqtt_slot_topic, old_prefs.mqtt_slot_topic, sizeof(prefs->mqtt_slot_topic)); +} + +inline void migrateLegacySixSlot(const Legacy6SlotMQTTPrefs& old_prefs, Source source, + MQTTPrefs* prefs) { + migrateLegacySixSlotCommon(old_prefs, prefs); + if (source == Source::LegacySixSlotAudience || source == Source::LegacySixSlotAudienceRx || + source == Source::LegacySixSlot) { + memcpy(prefs->mqtt_slot_audience, old_prefs.mqtt_slot_audience, + sizeof(prefs->mqtt_slot_audience)); + } + if (source == Source::LegacySixSlotAudienceRx || source == Source::LegacySixSlot) { + prefs->mqtt_rx_enabled = old_prefs.mqtt_rx_enabled; + } + if (source == Source::LegacySixSlot) { + memcpy(prefs->mqtt_ntp_server, old_prefs.mqtt_ntp_server, + sizeof(prefs->mqtt_ntp_server)); + } +} + +} // namespace MQTTPrefsCodec + +#endif // WITH_MQTT_BRIDGE diff --git a/src/helpers/MQTTPrefsStorage.h b/src/helpers/MQTTPrefsStorage.h new file mode 100644 index 00000000..fd898694 --- /dev/null +++ b/src/helpers/MQTTPrefsStorage.h @@ -0,0 +1,267 @@ +#pragma once + +#include +#include + +// /mqtt_prefs is a raw binary persistence format. Keep the layout-only types +// independent from CommonCLI so the migration decoder can be tested on the host +// without pulling in Arduino, filesystem, or radio dependencies. +#ifdef WITH_MQTT_BRIDGE + +// Must match MAX_MQTT_SLOTS in MQTTPresets.h. CommonCLI.h enforces that link on +// firmware builds; keeping this header standalone avoids importing preset data +// into host migration tests. +static const int MQTT_PREFS_SLOT_COUNT = 6; + +// Old MQTT preferences layout (pre-slot firmware) — used only for migration detection. +struct OldMQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + char wifi_ssid[32]; + char wifi_password[64]; + uint8_t wifi_power_save; + char timezone_string[32]; + int8_t timezone_offset; + char mqtt_server[64]; + uint16_t mqtt_port; + char mqtt_username[32]; + char mqtt_password[64]; + uint8_t mqtt_analyzer_us_enabled; + uint8_t mqtt_analyzer_eu_enabled; + char mqtt_owner_public_key[65]; + char mqtt_email[64]; +}; + +// The pre-WiFi-power pre-slot layout has the same frozen size as +// OldMQTTPrefs, but timezone/server start one byte earlier. A conservative +// classifier distinguishes meaningful configurations before migration. +struct PreWifiPowerOldMQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + char wifi_ssid[32]; + char wifi_password[64]; + char timezone_string[32]; + int8_t timezone_offset; + char mqtt_server[64]; + uint16_t mqtt_port; + char mqtt_username[32]; + char mqtt_password[64]; + uint8_t mqtt_analyzer_us_enabled; + uint8_t mqtt_analyzer_eu_enabled; + char mqtt_owner_public_key[65]; + char mqtt_email[64]; +}; + +// MQTT preferences stored separately from NodePrefs to avoid upstream layout +// conflicts. The full layout is the frozen v1 payload baseline. The prefix +// before observer settings is also an explicitly supported v1 payload: it was +// used before the observer fields were appended. +struct MQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + + char wifi_ssid[32]; + char wifi_password[64]; + uint8_t wifi_power_save; + + char timezone_string[32]; + int8_t timezone_offset; + + char mqtt_slot_preset[MQTT_PREFS_SLOT_COUNT][24]; + char mqtt_slot_host[MQTT_PREFS_SLOT_COUNT][64]; + uint16_t mqtt_slot_port[MQTT_PREFS_SLOT_COUNT]; + char mqtt_slot_username[MQTT_PREFS_SLOT_COUNT][32]; + char mqtt_slot_password[MQTT_PREFS_SLOT_COUNT][64]; + + char mqtt_owner_public_key[65]; + char mqtt_email[64]; + + char mqtt_slot_token[MQTT_PREFS_SLOT_COUNT][48]; + char mqtt_slot_topic[MQTT_PREFS_SLOT_COUNT][96]; + char mqtt_slot_audience[MQTT_PREFS_SLOT_COUNT][64]; + + uint8_t mqtt_rx_enabled; + char mqtt_ntp_server[64]; + + uint8_t snmp_enabled; + char snmp_community[24]; + uint8_t radio_watchdog_minutes; + uint8_t alert_enabled; + char alert_psk_hex[33]; + uint16_t alert_wifi_minutes; + uint16_t alert_mqtt_minutes; + uint16_t alert_min_interval_min; + char alert_hashtag[24]; + char alert_region[31]; +}; + +// Version-1 has exactly two layouts this firmware can decode. Never infer a +// compatible payload from an arbitrary shorter size: raw prefs have no checksum. +static const size_t MQTT_PREFS_V1_PRE_OBSERVER_PAYLOAD_SIZE = 2736; +static const size_t MQTT_PREFS_V1_FULL_PAYLOAD_SIZE = 2860; + +// /mqtt_prefs starts with a self-describing 8-byte header. Headerless files +// are deployed legacy layouts and continue to be distinguished by size. +static const uint8_t MQTT_PREFS_MAGIC[4] = {0xF5, 'M', 'Q', 'P'}; +static const uint16_t MQTT_PREFS_VERSION = 1; + +struct MQTTPrefsHeader { + uint8_t magic[4]; + uint16_t version; + uint16_t payload_len; +}; + +// 3-slot MQTTPrefs layout. Array dimensions changed in the current format, so +// it must be field-copied rather than read into MQTTPrefs directly. +struct ThreeSlotMQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + char wifi_ssid[32]; + char wifi_password[64]; + uint8_t wifi_power_save; + char timezone_string[32]; + int8_t timezone_offset; + char mqtt_slot_preset[3][24]; + char mqtt_slot_host[3][64]; + uint16_t mqtt_slot_port[3]; + char mqtt_slot_username[3][32]; + char mqtt_slot_password[3][64]; + char mqtt_owner_public_key[65]; + char mqtt_email[64]; + uint8_t _legacy_analyzer_us_enabled; + uint8_t _legacy_analyzer_eu_enabled; + char _legacy_mqtt_server[64]; + uint16_t _legacy_mqtt_port; + char _legacy_mqtt_username[32]; + char _legacy_mqtt_password[64]; + char mqtt_slot_token[3][48]; + char mqtt_slot_topic[3][96]; +}; + +// The earlier 3-slot format preceded token/topic fields. +struct ThreeSlotBaseMQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + char wifi_ssid[32]; + char wifi_password[64]; + uint8_t wifi_power_save; + char timezone_string[32]; + int8_t timezone_offset; + char mqtt_slot_preset[3][24]; + char mqtt_slot_host[3][64]; + uint16_t mqtt_slot_port[3]; + char mqtt_slot_username[3][32]; + char mqtt_slot_password[3][64]; + char mqtt_owner_public_key[65]; + char mqtt_email[64]; + uint8_t _legacy_analyzer_us_enabled; + uint8_t _legacy_analyzer_eu_enabled; + char _legacy_mqtt_server[64]; + uint16_t _legacy_mqtt_port; + char _legacy_mqtt_username[32]; + char _legacy_mqtt_password[64]; +}; + +// Headerless 6-slot layout shipped to the deployed flex fleet. It retains the +// removed `_legacy_*` block, so it too is field-copied into MQTTPrefs. +struct Legacy6SlotMQTTPrefs { + char mqtt_origin[32]; + char mqtt_iata[8]; + uint8_t mqtt_status_enabled; + uint8_t mqtt_packets_enabled; + uint8_t mqtt_raw_enabled; + uint8_t mqtt_tx_enabled; + uint32_t mqtt_status_interval; + char wifi_ssid[32]; + char wifi_password[64]; + uint8_t wifi_power_save; + char timezone_string[32]; + int8_t timezone_offset; + char mqtt_slot_preset[MQTT_PREFS_SLOT_COUNT][24]; + char mqtt_slot_host[MQTT_PREFS_SLOT_COUNT][64]; + uint16_t mqtt_slot_port[MQTT_PREFS_SLOT_COUNT]; + char mqtt_slot_username[MQTT_PREFS_SLOT_COUNT][32]; + char mqtt_slot_password[MQTT_PREFS_SLOT_COUNT][64]; + char mqtt_owner_public_key[65]; + char mqtt_email[64]; + uint8_t _legacy_analyzer_us_enabled; + uint8_t _legacy_analyzer_eu_enabled; + char _legacy_mqtt_server[64]; + uint16_t _legacy_mqtt_port; + char _legacy_mqtt_username[32]; + char _legacy_mqtt_password[64]; + char mqtt_slot_token[MQTT_PREFS_SLOT_COUNT][48]; + char mqtt_slot_topic[MQTT_PREFS_SLOT_COUNT][96]; + char mqtt_slot_audience[MQTT_PREFS_SLOT_COUNT][64]; + uint8_t mqtt_rx_enabled; + char mqtt_ntp_server[64]; +}; + +// Historical headerless 6-slot variants. They share a common prefix but only +// later files contain the appended audience, RX, and NTP fields. +static const size_t LEGACY6_BASE_SIZE = 2452; +static const size_t LEGACY6_AUDIENCE_SIZE = 2836; +static const size_t LEGACY6_AUDIENCE_RX_SIZE = 2840; + +// Frozen on-flash layouts; every firmware and native fixture build checks them. +static_assert(sizeof(MQTTPrefsHeader) == 8, "versioned /mqtt_prefs header must stay 8 bytes"); +static_assert(offsetof(MQTTPrefs, snmp_enabled) == MQTT_PREFS_V1_PRE_OBSERVER_PAYLOAD_SIZE, + "v1 pre-observer /mqtt_prefs boundary changed"); +static_assert(sizeof(MQTTPrefs) == MQTT_PREFS_V1_FULL_PAYLOAD_SIZE, + "v1 /mqtt_prefs payload layout changed"); +static_assert(sizeof(OldMQTTPrefs) == 472, "frozen pre-slot /mqtt_prefs layout changed"); +static_assert(sizeof(PreWifiPowerOldMQTTPrefs) == 472, "frozen pre-WiFi-power /mqtt_prefs layout changed"); +static_assert(offsetof(OldMQTTPrefs, wifi_power_save) == 144, + "frozen post-WiFi-power discriminator offset changed"); +static_assert(offsetof(OldMQTTPrefs, timezone_string) == 145, + "frozen post-WiFi-power timezone offset changed"); +static_assert(offsetof(OldMQTTPrefs, timezone_offset) == 177, + "frozen post-WiFi-power UTC offset changed"); +static_assert(offsetof(OldMQTTPrefs, mqtt_server) == 178, + "frozen post-WiFi-power server offset changed"); +static_assert(offsetof(PreWifiPowerOldMQTTPrefs, timezone_string) == 144, + "frozen pre-WiFi-power timezone offset changed"); +static_assert(offsetof(PreWifiPowerOldMQTTPrefs, timezone_offset) == 176, + "frozen pre-WiFi-power UTC offset changed"); +static_assert(offsetof(PreWifiPowerOldMQTTPrefs, mqtt_server) == 177, + "frozen pre-WiFi-power server offset changed"); +static_assert(sizeof(ThreeSlotBaseMQTTPrefs) == 1032, "frozen early 3-slot /mqtt_prefs layout changed"); +static_assert(sizeof(ThreeSlotMQTTPrefs) == 1464, "frozen 3-slot /mqtt_prefs layout changed"); +static_assert(offsetof(ThreeSlotMQTTPrefs, mqtt_slot_token) == 1030, + "frozen 3-slot token offset changed"); +static_assert(offsetof(ThreeSlotMQTTPrefs, mqtt_slot_topic) == 1174, + "frozen 3-slot topic offset changed"); +static_assert(offsetof(Legacy6SlotMQTTPrefs, mqtt_slot_audience) == LEGACY6_BASE_SIZE, + "frozen early 6-slot /mqtt_prefs prefix changed"); +static_assert(offsetof(Legacy6SlotMQTTPrefs, mqtt_rx_enabled) == LEGACY6_AUDIENCE_SIZE, + "frozen audience 6-slot /mqtt_prefs prefix changed"); +static_assert(offsetof(Legacy6SlotMQTTPrefs, mqtt_ntp_server) == 2837, + "frozen RX 6-slot /mqtt_prefs prefix changed"); +static_assert(sizeof(Legacy6SlotMQTTPrefs) == 2904, "frozen deployed-fleet /mqtt_prefs layout changed"); + +#endif // WITH_MQTT_BRIDGE diff --git a/test/README.md b/test/README.md index e60a3879..15a19a54 100644 --- a/test/README.md +++ b/test/README.md @@ -31,6 +31,8 @@ does not reflect the GoogleTest count — run the built binary directly | `test_mqtt_connection_policy` | `src/helpers/MQTTConnectionPolicy.h` | reconnect guard/backoff/stagger and breaker transitions; stable reset; JWT lifetime/renewal policy; exact timing boundaries and 32-bit `millis()` rollover | | `test_mqtt_packet_queue_policy` | `src/helpers/MQTTPacketQueuePolicy.h` | queue-full eviction; stale-disconnect flush; adaptive drain limits; bounded QoS0 retries; exact timing boundaries and 32-bit `millis()` rollover | | `test_mqtt_runtime_buffer_lifecycle` | `src/helpers/MQTTRuntimeBufferLifecycle.h` | idempotent allocation/release; partial-allocation degradation; retry of only missing buffers | +| `test_mqtt_prefs_codec` | `src/helpers/MQTTPrefsStorage.h`, `src/helpers/MQTTPrefsCodec.h` | binary pre-slot/3-slot/6-slot migration fixtures; v1 header integrity; downgrade preservation | +| `test_mqtt_prefs_atomic_store` | `src/helpers/MQTTPrefsAtomicStore.h` | transactional MQTT writes and legacy `/node_prefs` handoff; exact short-write detection; begin/finish/rename failure cleanup; original-file preservation | | `test_mqtt_payload_builder` | `src/helpers/MQTTPayloadBuilder.cpp` | status/packet/raw JSON contracts; optional fields; escaping; RX metrics and path; score handling; exact buffer bounds; maximum representative payloads | | `test_utils` | `src/Utils.cpp` | `Utils::toHex` (upstream) | diff --git a/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp b/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp new file mode 100644 index 00000000..3d7d6f02 --- /dev/null +++ b/test/test_mqtt_prefs_atomic_store/test_mqtt_prefs_atomic_store.cpp @@ -0,0 +1,399 @@ +#include + +#include +#include +#include +#include + +#include "helpers/MQTTPrefsAtomicStore.h" + +namespace AtomicStore = MQTTPrefsAtomicStore; + +namespace { + +enum class FailurePoint { + None, + Begin, + HeaderWrite, + PayloadWrite, + ImageWrite, + Finish, + Commit, +}; + +class InMemoryStore { +public: + explicit InMemoryStore(FailurePoint failure) : _failure(failure) { + _files["/mqtt_prefs"] = {'o', 'l', 'd', '-', 'p', 'r', 'e', 'f', 's'}; + } + + bool begin() { + ++begin_calls; + _files.erase("/mqtt_prefs.tmp"); + _open = _failure != FailurePoint::Begin; + return _open; + } + + size_t write(const uint8_t* bytes, size_t size) { + ++write_calls; + if (!_open) return 0; + const bool should_fail = (write_calls == 1 && _failure == FailurePoint::HeaderWrite) || + (write_calls == 2 && _failure == FailurePoint::PayloadWrite); + const size_t written = should_fail && size > 0 ? size - 1 : size; + _staging.insert(_staging.end(), bytes, bytes + written); + return written; + } + + bool finish() { + ++finish_calls; + _open = false; + if (_failure == FailurePoint::Finish) return false; + _files["/mqtt_prefs.tmp"] = _staging; + return true; + } + + bool commit() { + ++commit_calls; + if (_failure == FailurePoint::Commit) return false; + _files["/mqtt_prefs"] = _files["/mqtt_prefs.tmp"]; + _files.erase("/mqtt_prefs.tmp"); + return true; + } + + void abort() { + ++abort_calls; + _open = false; + _staging.clear(); + _files.erase("/mqtt_prefs.tmp"); + } + + const std::vector& source() const { return _files.at("/mqtt_prefs"); } + bool tempExists() const { return _files.count("/mqtt_prefs.tmp") != 0; } + + int begin_calls = 0; + int write_calls = 0; + int finish_calls = 0; + int commit_calls = 0; + int abort_calls = 0; + +private: + FailurePoint _failure; + bool _open = false; + std::vector _staging; + std::map> _files; +}; + +AtomicStore::Result run(InMemoryStore* store) { + const uint8_t header[] = {0xf5, 'M', 'Q', 'P', 1, 0, 0x09, 0x00}; + const uint8_t payload[] = {'n', 'e', 'w', '-', 'p', 'r', 'e', 'f', 's'}; + return AtomicStore::write(*store, header, sizeof(header), payload, sizeof(payload)); +} + +AtomicStore::Result runWithObserverTail(InMemoryStore* store) { + const uint8_t header[] = {0xf5, 'M', 'Q', 'P', 1, 0, 0x0a, 0x00}; + // The final three bytes stand in for the observer tail transferred from a + // legacy /com_prefs file. They must be committed before that source is compacted. + const uint8_t payload[] = {'m', 'i', 'g', 'r', 'a', 't', 'e', 0x91, 0x7e, 0xa5}; + return AtomicStore::write(*store, header, sizeof(header), payload, sizeof(payload)); +} + +class LegacyComPrefs { +public: + LegacyComPrefs() : bytes({'l', 'e', 'g', 'a', 'c', 'y', '-', 'c', 'o', 'm'}) {} + + void compactAfterMqttCommit(const std::vector& mqtt_bytes) { + const std::vector observer_tail = {0x91, 0x7e, 0xa5}; + mqtt_tail_present_before_compaction = mqtt_bytes.size() >= observer_tail.size() && + std::equal(observer_tail.rbegin(), observer_tail.rend(), mqtt_bytes.rbegin()); + bytes = {'c', 'o', 'm', 'p', 'a', 'c', 't'}; + ++compact_calls; + } + + std::vector bytes; + bool mqtt_tail_present_before_compaction = false; + int compact_calls = 0; +}; + +class LegacyNodePrefs { +public: + LegacyNodePrefs() : bytes({'l', 'e', 'g', 'a', 'c', 'y', '-', 'n', 'o', 'd', 'e'}) {} + + void migrateAfterMqttCommit(const std::vector& mqtt_bytes) { + const std::vector observer_tail = {0x91, 0x7e, 0xa5}; + mqtt_tail_present_before_removal = mqtt_bytes.size() >= observer_tail.size() && + std::equal(observer_tail.rbegin(), observer_tail.rend(), mqtt_bytes.rbegin()); + bytes.clear(); // model removal after current-layout /com_prefs is written + ++migration_calls; + } + + std::vector bytes; + bool mqtt_tail_present_before_removal = false; + int migration_calls = 0; +}; + +// Models the final old-name migration separately from the MQTT transaction: +// /com_prefs is absent while /node_prefs is authoritative. A failed temp write +// or rename must leave that source as the only usable preference image. +class InMemoryCommonPrefsStore { +public: + explicit InMemoryCommonPrefsStore(FailurePoint failure) : _failure(failure) { + _files["/node_prefs"] = {'l', 'e', 'g', 'a', 'c', 'y', '-', 'n', 'o', 'd', 'e'}; + } + + bool begin() { + ++begin_calls; + _files.erase("/com_prefs.tmp"); + _staging.clear(); + _open = _failure != FailurePoint::Begin; + return _open; + } + + size_t write(const uint8_t* bytes, size_t size) { + ++write_calls; + if (!_open) return 0; + const bool should_fail = _failure == FailurePoint::ImageWrite && write_calls == 2; + const size_t written = should_fail && size > 0 ? size - 1 : size; + _staging.insert(_staging.end(), bytes, bytes + written); + return written; + } + + bool finish() { + ++finish_calls; + _open = false; + if (_failure == FailurePoint::Finish) return false; + _files["/com_prefs.tmp"] = _staging; + return true; + } + + bool commit() { + ++commit_calls; + if (_failure == FailurePoint::Commit) return false; + _files["/com_prefs"] = _files["/com_prefs.tmp"]; + _files.erase("/com_prefs.tmp"); + return true; + } + + void abort() { + ++abort_calls; + _open = false; + _staging.clear(); + _files.erase("/com_prefs.tmp"); + } + + void removeNodeSource() { _files.erase("/node_prefs"); } + const std::vector& nodeSource() const { return _files.at("/node_prefs"); } + const std::vector& destination() const { return _files.at("/com_prefs"); } + bool destinationExists() const { return _files.count("/com_prefs") != 0; } + bool tempExists() const { return _files.count("/com_prefs.tmp") != 0; } + bool nodeSourceIsPreferred() const { + return _files.count("/node_prefs") != 0 && _files.count("/com_prefs") == 0; + } + bool nodeSourceExists() const { return _files.count("/node_prefs") != 0; } + + int begin_calls = 0; + int write_calls = 0; + int finish_calls = 0; + int commit_calls = 0; + int abort_calls = 0; + +private: + FailurePoint _failure; + bool _open = false; + std::vector _staging; + std::map> _files; +}; + +AtomicStore::ImageResult runCommonPrefsImage(InMemoryCommonPrefsStore* store) { + const uint8_t core[] = {'c', 'o', 'm', '-', 'p', 'r', 'e', 'f', 's'}; + const uint8_t tail[] = {0x19, 0xa4, 0x7e}; + return AtomicStore::writeImage(*store, [&core, &tail](InMemoryCommonPrefsStore& target) { + return target.write(core, sizeof(core)) == sizeof(core) && + target.write(tail, sizeof(tail)) == sizeof(tail); + }); +} + +} // namespace + +TEST(MQTTPrefsAtomicStore, CommitPublishesExactHeaderThenPayload) { + InMemoryStore store(FailurePoint::None); + + EXPECT_EQ(AtomicStore::Result::Committed, run(&store)); + EXPECT_EQ((std::vector{0xf5, 'M', 'Q', 'P', 1, 0, 0x09, 0x00, + 'n', 'e', 'w', '-', 'p', 'r', 'e', 'f', 's'}), + store.source()); + EXPECT_FALSE(store.tempExists()); + EXPECT_EQ(1, store.begin_calls); + EXPECT_EQ(2, store.write_calls); + EXPECT_EQ(1, store.finish_calls); + EXPECT_EQ(1, store.commit_calls); + EXPECT_EQ(0, store.abort_calls); +} + +TEST(MQTTPrefsAtomicStore, AnyFailureAbortsAndPreservesExistingSource) { + const std::vector source = {'o', 'l', 'd', '-', 'p', 'r', 'e', 'f', 's'}; + const struct { + FailurePoint point; + AtomicStore::Result expected; + int writes; + int finishes; + int commits; + } cases[] = { + {FailurePoint::Begin, AtomicStore::Result::BeginFailed, 0, 0, 0}, + {FailurePoint::HeaderWrite, AtomicStore::Result::HeaderWriteFailed, 1, 0, 0}, + {FailurePoint::PayloadWrite, AtomicStore::Result::PayloadWriteFailed, 2, 0, 0}, + {FailurePoint::Finish, AtomicStore::Result::FinishFailed, 2, 1, 0}, + {FailurePoint::Commit, AtomicStore::Result::CommitFailed, 2, 1, 1}, + }; + + for (const auto& test_case : cases) { + InMemoryStore store(test_case.point); + EXPECT_EQ(test_case.expected, run(&store)); + EXPECT_EQ(source, store.source()); + EXPECT_FALSE(store.tempExists()); + EXPECT_EQ(1, store.begin_calls); + EXPECT_EQ(test_case.writes, store.write_calls); + EXPECT_EQ(test_case.finishes, store.finish_calls); + EXPECT_EQ(test_case.commits, store.commit_calls); + EXPECT_EQ(1, store.abort_calls); + } +} + +TEST(MQTTPrefsAtomicStore, LegacyCrossFileUpgradeCommitsTailBeforeCompactingComPrefs) { + InMemoryStore mqtt_store(FailurePoint::None); + LegacyComPrefs com_prefs; + AtomicStore::LegacyUpgradeGate gate(true); + gate.requireMqttRewrite(); + + const AtomicStore::Result result = runWithObserverTail(&mqtt_store); + gate.recordMqttSave(AtomicStore::committed(result)); + ASSERT_TRUE(gate.mayRewriteComPrefs()); + + com_prefs.compactAfterMqttCommit(mqtt_store.source()); + gate.recordComPrefsRewrite(); + + EXPECT_TRUE(com_prefs.mqtt_tail_present_before_compaction); + EXPECT_EQ(1, com_prefs.compact_calls); + EXPECT_EQ((std::vector{'c', 'o', 'm', 'p', 'a', 'c', 't'}), com_prefs.bytes); + EXPECT_FALSE(gate.mayRewriteComPrefs()); +} + +TEST(MQTTPrefsAtomicStore, LegacyCrossFilePowerCutPreservesBothSources) { + const std::vector legacy_mqtt = {'o', 'l', 'd', '-', 'p', 'r', 'e', 'f', 's'}; + const std::vector legacy_com = {'l', 'e', 'g', 'a', 'c', 'y', '-', 'c', 'o', 'm'}; + for (const FailurePoint point : {FailurePoint::Begin, FailurePoint::HeaderWrite, + FailurePoint::PayloadWrite, FailurePoint::Finish, + FailurePoint::Commit}) { + InMemoryStore mqtt_store(point); + LegacyComPrefs com_prefs; + AtomicStore::LegacyUpgradeGate gate(true); + gate.requireMqttRewrite(); + + const AtomicStore::Result result = runWithObserverTail(&mqtt_store); + gate.recordMqttSave(AtomicStore::committed(result)); + if (gate.mayRewriteComPrefs()) { + com_prefs.compactAfterMqttCommit(mqtt_store.source()); + gate.recordComPrefsRewrite(); + } + + EXPECT_FALSE(AtomicStore::committed(result)); + EXPECT_EQ(legacy_mqtt, mqtt_store.source()); + EXPECT_EQ(legacy_com, com_prefs.bytes); + EXPECT_EQ(0, com_prefs.compact_calls); + EXPECT_TRUE(gate.blocksComPrefsRewrite()); + } +} + +TEST(MQTTPrefsAtomicStore, LegacyNodePrefsMigrationWaitsForObserverTailCommit) { + InMemoryStore mqtt_store(FailurePoint::None); + LegacyNodePrefs node_prefs; + AtomicStore::LegacyUpgradeGate gate(true); + gate.requireMqttRewrite(); + + const AtomicStore::Result result = runWithObserverTail(&mqtt_store); + gate.recordMqttSave(AtomicStore::committed(result)); + ASSERT_TRUE(gate.mayRewriteComPrefs()); + + node_prefs.migrateAfterMqttCommit(mqtt_store.source()); + gate.recordComPrefsRewrite(); + + EXPECT_TRUE(node_prefs.mqtt_tail_present_before_removal); + EXPECT_EQ(1, node_prefs.migration_calls); + EXPECT_TRUE(node_prefs.bytes.empty()); +} + +TEST(MQTTPrefsAtomicStore, LegacyNodePrefsPowerCutPreservesSource) { + const std::vector legacy_mqtt = {'o', 'l', 'd', '-', 'p', 'r', 'e', 'f', 's'}; + const std::vector legacy_node = { + 'l', 'e', 'g', 'a', 'c', 'y', '-', 'n', 'o', 'd', 'e'}; + for (const FailurePoint point : {FailurePoint::Begin, FailurePoint::HeaderWrite, + FailurePoint::PayloadWrite, FailurePoint::Finish, + FailurePoint::Commit}) { + InMemoryStore mqtt_store(point); + LegacyNodePrefs node_prefs; + AtomicStore::LegacyUpgradeGate gate(true); + gate.requireMqttRewrite(); + + const AtomicStore::Result result = runWithObserverTail(&mqtt_store); + gate.recordMqttSave(AtomicStore::committed(result)); + if (gate.mayRewriteComPrefs()) { + node_prefs.migrateAfterMqttCommit(mqtt_store.source()); + gate.recordComPrefsRewrite(); + } + + EXPECT_FALSE(AtomicStore::committed(result)); + EXPECT_EQ(legacy_mqtt, mqtt_store.source()); + EXPECT_EQ(legacy_node, node_prefs.bytes); + EXPECT_EQ(0, node_prefs.migration_calls); + EXPECT_TRUE(gate.blocksComPrefsRewrite()); + } +} + +TEST(MQTTPrefsAtomicStore, NodePrefsMigrationPublishesComPrefsBeforeRemovingSource) { + InMemoryCommonPrefsStore store(FailurePoint::None); + + ASSERT_EQ(AtomicStore::ImageResult::Committed, runCommonPrefsImage(&store)); + EXPECT_TRUE(store.nodeSourceExists()); // caller removes it only after commit + EXPECT_EQ((std::vector{'c', 'o', 'm', '-', 'p', 'r', 'e', 'f', 's', 0x19, 0xa4, 0x7e}), + store.destination()); + EXPECT_FALSE(store.tempExists()); + + store.removeNodeSource(); + EXPECT_FALSE(store.nodeSourceExists()); + EXPECT_TRUE(store.destinationExists()); +} + +TEST(MQTTPrefsAtomicStore, NodePrefsMigrationFailurePreservesSourceAndNeverPrefersPartialDestination) { + const std::vector legacy_node = { + 'l', 'e', 'g', 'a', 'c', 'y', '-', 'n', 'o', 'd', 'e'}; + const struct { + FailurePoint point; + AtomicStore::ImageResult expected; + int writes; + int finishes; + int commits; + } cases[] = { + {FailurePoint::Begin, AtomicStore::ImageResult::BeginFailed, 0, 0, 0}, + {FailurePoint::ImageWrite, AtomicStore::ImageResult::WriteFailed, 2, 0, 0}, + {FailurePoint::Finish, AtomicStore::ImageResult::FinishFailed, 2, 1, 0}, + {FailurePoint::Commit, AtomicStore::ImageResult::CommitFailed, 2, 1, 1}, + }; + + for (const auto& test_case : cases) { + InMemoryCommonPrefsStore store(test_case.point); + EXPECT_EQ(test_case.expected, runCommonPrefsImage(&store)); + EXPECT_EQ(legacy_node, store.nodeSource()); + EXPECT_TRUE(store.nodeSourceIsPreferred()); + EXPECT_FALSE(store.destinationExists()); + EXPECT_FALSE(store.tempExists()); + EXPECT_EQ(1, store.begin_calls); + EXPECT_EQ(test_case.writes, store.write_calls); + EXPECT_EQ(test_case.finishes, store.finish_calls); + EXPECT_EQ(test_case.commits, store.commit_calls); + EXPECT_EQ(1, store.abort_calls); + } +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/test/test_mqtt_prefs_codec/FIXTURES.md b/test/test_mqtt_prefs_codec/FIXTURES.md new file mode 100644 index 00000000..0068e1a5 --- /dev/null +++ b/test/test_mqtt_prefs_codec/FIXTURES.md @@ -0,0 +1,24 @@ +# MQTT preferences fixture provenance + +The codec tests construct synthetic, non-secret byte vectors at hard-coded +offsets. They intentionally do not serialize the production structs: a layout +change must disagree with the frozen fixture bytes and fail the field checks. + +| Bytes | Historical layout | Source history | +|---:|---|---| +| 472 | Pre-slot, before and after `wifi_power_save` | Initial `/mqtt_prefs` layout; `34c8bea7` inserted WiFi power without changing the padded total size | +| 1032 | Initial three-slot layout | `b43e9618` | +| 1464 | Three slots with token/topic tails | `95874f0c` | +| 2452 | Six slots with token/topic tails | `1b5884bd` | +| 2836 | Six slots with audience tail | `1263e71d` | +| 2840 | Six slots with RX flag | `47b632aa` | +| 2904 | Six slots with NTP server | `7416d632` | +| 2736 / 2860 | Version-1 payload before observer tail / complete payload | `58b9cb66` introduced the eight-byte versioned header; the shorter form exercises its append-compatible prefix contract | + +The 3024-byte raw observer-tail form is not accepted as deployed fleet data: +repository history indicates it existed briefly before versioning but was not +shipped. Tests require it to be preserved rather than guessed and rewritten. + +Headerless formats have no checksum. Plausibility checks reject obvious random +or malformed content, but cannot authenticate a byte sequence that happens to +look like a valid historical struct. diff --git a/test/test_mqtt_prefs_codec/test_mqtt_prefs_codec.cpp b/test/test_mqtt_prefs_codec/test_mqtt_prefs_codec.cpp new file mode 100644 index 00000000..2909fa5f --- /dev/null +++ b/test/test_mqtt_prefs_codec/test_mqtt_prefs_codec.cpp @@ -0,0 +1,359 @@ +#include + +#include +#include + +#define WITH_MQTT_BRIDGE 1 +#include "helpers/MQTTPrefsCodec.h" + +namespace Codec = MQTTPrefsCodec; + +namespace { + +MQTTPrefs defaults() { + MQTTPrefs prefs = {}; + prefs.mqtt_status_enabled = 1; + prefs.mqtt_packets_enabled = 1; + prefs.mqtt_tx_enabled = 2; + prefs.mqtt_rx_enabled = 1; + prefs.mqtt_status_interval = 300000; + prefs.wifi_power_save = 1; + for (int i = 0; i < MQTT_PREFS_SLOT_COUNT; ++i) { + strncpy(prefs.mqtt_slot_preset[i], "none", sizeof(prefs.mqtt_slot_preset[i]) - 1); + } + strncpy(prefs.snmp_community, "public", sizeof(prefs.snmp_community) - 1); + prefs.radio_watchdog_minutes = 5; + prefs.alert_wifi_minutes = 30; + prefs.alert_mqtt_minutes = 240; + prefs.alert_min_interval_min = 60; + return prefs; +} + +void writeText(std::vector* bytes, size_t offset, const char* value) { + const size_t length = strlen(value); + ASSERT_LE(offset + length, bytes->size()); + memcpy(bytes->data() + offset, value, length); +} + +void writeLe16(std::vector* bytes, size_t offset, uint16_t value) { + ASSERT_LE(offset + 2, bytes->size()); + (*bytes)[offset] = static_cast(value & 0xff); + (*bytes)[offset + 1] = static_cast(value >> 8); +} + +void writeHeader(std::vector* bytes, uint16_t version, uint16_t payload_len) { + ASSERT_GE(bytes->size(), sizeof(MQTTPrefsHeader)); + (*bytes)[0] = MQTT_PREFS_MAGIC[0]; + (*bytes)[1] = MQTT_PREFS_MAGIC[1]; + (*bytes)[2] = MQTT_PREFS_MAGIC[2]; + (*bytes)[3] = MQTT_PREFS_MAGIC[3]; + writeLe16(bytes, 4, version); + writeLe16(bytes, 6, payload_len); +} + +Codec::DecodePlan classify(const std::vector& bytes) { + const size_t prefix_size = bytes.size() < sizeof(MQTTPrefsHeader) + ? bytes.size() : sizeof(MQTTPrefsHeader); + return Codec::classify(bytes.data(), prefix_size, bytes.size()); +} + +void fillHighEntropy(std::vector* bytes) { + uint32_t state = 0x89abcdef; + for (size_t i = 0; i < bytes->size(); ++i) { + state = state * 1664525u + 1013904223u; + // Keep every byte non-NUL so this is a useful corruption fixture rather + // than a sparse/default-like legacy file. + (*bytes)[i] = static_cast((state >> 24) | 0x80); + } +} + +} // namespace + +TEST(MQTTPrefsCodec, MigratesPostWifiPowerPreSlotFixture) { + // Frozen post-WiFi-power pre-slot offsets: status=44, ssid=48, power=144, + // timezone=145, server=178, port=242. Do not derive this fixture from structs. + std::vector bytes(472, 0); + writeText(&bytes, 0, "legacy-node"); + writeText(&bytes, 32, "YYZ"); + bytes[40] = 1; + bytes[41] = 1; + bytes[43] = 2; + bytes[144] = 2; + writeText(&bytes, 145, "UTC"); + writeText(&bytes, 178, "broker.example"); + writeLe16(&bytes, 242, 1883); + writeText(&bytes, 244, "alice"); + writeText(&bytes, 276, "secret"); + bytes[340] = 1; + + const Codec::DecodePlan plan = classify(bytes); + ASSERT_EQ(Codec::Source::LegacyPreSlot, plan.source); + ASSERT_TRUE(plan.rewrite_legacy); + ASSERT_FALSE(Codec::looksLikePreWifiPower(bytes.data(), bytes.size())); + ASSERT_TRUE(Codec::isPlausibleLegacy(plan.source, bytes.data(), bytes.size())); + + OldMQTTPrefs old_prefs = {}; + memcpy(&old_prefs, bytes.data(), sizeof(old_prefs)); + MQTTPrefs prefs = defaults(); + Codec::migratePreSlot(old_prefs, &prefs); + + EXPECT_STREQ("legacy-node", prefs.mqtt_origin); + EXPECT_STREQ("analyzer-us", prefs.mqtt_slot_preset[0]); + EXPECT_STREQ("custom", prefs.mqtt_slot_preset[2]); + EXPECT_STREQ("broker.example", prefs.mqtt_slot_host[2]); + EXPECT_EQ(1883, prefs.mqtt_slot_port[2]); + EXPECT_STREQ("secret", prefs.mqtt_slot_password[2]); +} + +TEST(MQTTPrefsCodec, MigratesPreWifiPowerFixtureWithConservativeHeuristic) { + // In the pre-WiFi-power variant timezone starts at 144 and server at 177; + // port still aligns at 242. A non-empty timezone makes the layout unambiguous. + std::vector bytes(472, 0); + writeText(&bytes, 0, "pre-power"); + writeText(&bytes, 144, "PST8PDT"); + writeText(&bytes, 177, "old-broker"); + writeLe16(&bytes, 242, 8883); + + ASSERT_TRUE(Codec::looksLikePreWifiPower(bytes.data(), bytes.size())); + ASSERT_TRUE(Codec::isPlausibleLegacy(Codec::Source::LegacyPreSlot, + bytes.data(), bytes.size())); + PreWifiPowerOldMQTTPrefs old_prefs = {}; + memcpy(&old_prefs, bytes.data(), sizeof(old_prefs)); + MQTTPrefs prefs = defaults(); + Codec::migratePreWifiPower(old_prefs, &prefs); + + EXPECT_STREQ("PST8PDT", prefs.timezone_string); + EXPECT_STREQ("old-broker", prefs.mqtt_slot_host[2]); + EXPECT_EQ(8883, prefs.mqtt_slot_port[2]); + EXPECT_EQ(1, prefs.wifi_power_save); // retained current default; old layout had none +} + +TEST(MQTTPrefsCodec, DetectsPreWifiPowerFixtureWithOnlyUtcOffsetConfigured) { + std::vector bytes(472, 0); + bytes[176] = static_cast(-8); + + EXPECT_TRUE(Codec::looksLikePreWifiPower(bytes.data(), bytes.size())); +} + +TEST(MQTTPrefsCodec, MigratesThreeSlotBaseAndExtendedFixtures) { + // Frozen 3-slot offsets: presets=178, hosts=250, ports=442, users=448, + // passwords=544, owner=736, tokens=1030, topics=1174. The base file has + // two tail-padding bytes, so its frozen size is 1032. + std::vector base(1032, 0); + writeText(&base, 0, "three-base"); + writeText(&base, 178, "meshmapper"); + writeText(&base, 250, "three.example"); + writeLe16(&base, 442, 1884); + const Codec::DecodePlan base_plan = classify(base); + ASSERT_EQ(Codec::Source::LegacyThreeSlotBase, base_plan.source); + ASSERT_TRUE(Codec::isPlausibleLegacy(base_plan.source, base.data(), base.size())); + ThreeSlotBaseMQTTPrefs old_base = {}; + memcpy(&old_base, base.data(), sizeof(old_base)); + MQTTPrefs base_prefs = defaults(); + Codec::migrateThreeSlot(old_base, &base_prefs); + EXPECT_STREQ("three-base", base_prefs.mqtt_origin); + EXPECT_STREQ("meshmapper", base_prefs.mqtt_slot_preset[0]); + EXPECT_EQ(1884, base_prefs.mqtt_slot_port[0]); + EXPECT_STREQ("none", base_prefs.mqtt_slot_preset[3]); + + std::vector extended(1464, 0); + writeText(&extended, 0, "three-extended"); + writeText(&extended, 178 + 24, "custom"); + writeText(&extended, 250 + 64, "slot-two.example"); + writeText(&extended, 1030 + 48, "token-two"); + writeText(&extended, 1174 + 96, "custom/{type}"); + const Codec::DecodePlan extended_plan = classify(extended); + ASSERT_EQ(Codec::Source::LegacyThreeSlot, extended_plan.source); + ASSERT_TRUE(Codec::isPlausibleLegacy(extended_plan.source, extended.data(), extended.size())); + ThreeSlotMQTTPrefs old_extended = {}; + memcpy(&old_extended, extended.data(), sizeof(old_extended)); + MQTTPrefs extended_prefs = defaults(); + Codec::migrateThreeSlot(old_extended, &extended_prefs); + EXPECT_STREQ("custom", extended_prefs.mqtt_slot_preset[1]); + EXPECT_STREQ("token-two", extended_prefs.mqtt_slot_token[1]); + EXPECT_STREQ("custom/{type}", extended_prefs.mqtt_slot_topic[1]); +} + +TEST(MQTTPrefsCodec, MigratesAllLegacySixSlotPrefixesWithoutClobberingDefaults) { + // Frozen 6-slot offsets: presets=178, hosts=322, ports=706, tokens=1588, + // topics=1876, audience=2452, rx=2836, ntp=2837. + for (const size_t size : {size_t(2452), size_t(2836), size_t(2840), size_t(2904)}) { + std::vector bytes(size, 0); + writeText(&bytes, 0, "six-slot"); + writeText(&bytes, 178 + 5 * 24, "custom"); + writeText(&bytes, 322 + 5 * 64, "six.example"); + writeText(&bytes, 1588 + 5 * 48, "token-six"); + writeText(&bytes, 1876 + 5 * 96, "six/{type}"); + if (size >= 2836) writeText(&bytes, 2452 + 5 * 64, "audience-six"); + if (size >= 2840) bytes[2836] = 0; + if (size >= 2904) writeText(&bytes, 2837, "time.example"); + + const Codec::DecodePlan plan = classify(bytes); + const Codec::Source expected_source = size == 2452 ? Codec::Source::LegacySixSlotBase + : size == 2836 ? Codec::Source::LegacySixSlotAudience + : size == 2840 ? Codec::Source::LegacySixSlotAudienceRx + : Codec::Source::LegacySixSlot; + EXPECT_EQ(expected_source, plan.source) << size; + ASSERT_TRUE(plan.rewrite_legacy); + ASSERT_TRUE(Codec::isPlausibleLegacy(plan.source, bytes.data(), bytes.size())) << size; + Legacy6SlotMQTTPrefs old_prefs = {}; + memcpy(&old_prefs, bytes.data(), bytes.size()); + MQTTPrefs prefs = defaults(); + Codec::migrateLegacySixSlot(old_prefs, plan.source, &prefs); + + EXPECT_STREQ("custom", prefs.mqtt_slot_preset[5]); + EXPECT_STREQ("token-six", prefs.mqtt_slot_token[5]); + if (size < 2836) { + EXPECT_EQ('\0', prefs.mqtt_slot_audience[5][0]); + } else { + EXPECT_STREQ("audience-six", prefs.mqtt_slot_audience[5]); + } + if (size < 2840) { + EXPECT_EQ(1, prefs.mqtt_rx_enabled) << size; + } else { + EXPECT_EQ(0, prefs.mqtt_rx_enabled); + } + if (size < 2904) { + EXPECT_EQ('\0', prefs.mqtt_ntp_server[0]); + } else { + EXPECT_STREQ("time.example", prefs.mqtt_ntp_server); + } + } +} + +TEST(MQTTPrefsCodec, CurrentVersionedPayloadRoundTripsExactly) { + MQTTPrefs source = defaults(); + strncpy(source.mqtt_origin, "current-node", sizeof(source.mqtt_origin) - 1); + strncpy(source.mqtt_slot_password[2], "preserve-me", sizeof(source.mqtt_slot_password[2]) - 1); + strncpy(source.alert_region, "PNW", sizeof(source.alert_region) - 1); + std::vector bytes(Codec::kEncodedSize); + ASSERT_EQ(Codec::kEncodedSize, Codec::encode(source, bytes.data(), bytes.size())); + + const Codec::DecodePlan plan = classify(bytes); + ASSERT_EQ(Codec::Source::Current, plan.source); + ASSERT_FALSE(plan.preserve_file); + ASSERT_TRUE(plan.observer_fields_present); + MQTTPrefs loaded = defaults(); + memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + EXPECT_EQ(0, memcmp(&source, &loaded, sizeof(source))); +} + +TEST(MQTTPrefsCodec, CompatibleShortV1PayloadPreservesDefaultsBeyondObserverBoundary) { + MQTTPrefs source = defaults(); + strncpy(source.mqtt_origin, "short-v1-node", sizeof(source.mqtt_origin) - 1); + strncpy(source.mqtt_ntp_server, "ntp.short.example", sizeof(source.mqtt_ntp_server) - 1); + source.snmp_enabled = 1; + strncpy(source.snmp_community, "do-not-copy", sizeof(source.snmp_community) - 1); + source.radio_watchdog_minutes = 99; + + std::vector bytes(sizeof(MQTTPrefsHeader) + Codec::kV1PreObserverPayloadSize, 0); + writeHeader(&bytes, MQTT_PREFS_VERSION, + static_cast(Codec::kV1PreObserverPayloadSize)); + memcpy(bytes.data() + sizeof(MQTTPrefsHeader), &source, Codec::kV1PreObserverPayloadSize); + + const Codec::DecodePlan plan = classify(bytes); + ASSERT_EQ(Codec::Source::Current, plan.source); + ASSERT_EQ(Codec::kV1PreObserverPayloadSize, plan.payload_len); + ASSERT_FALSE(plan.preserve_file); + ASSERT_FALSE(plan.observer_fields_present); + + MQTTPrefs loaded = defaults(); + memcpy(&loaded, bytes.data() + sizeof(MQTTPrefsHeader), plan.payload_len); + EXPECT_STREQ("short-v1-node", loaded.mqtt_origin); + EXPECT_STREQ("ntp.short.example", loaded.mqtt_ntp_server); + EXPECT_EQ(0, loaded.snmp_enabled); + EXPECT_STREQ("public", loaded.snmp_community); + EXPECT_EQ(5, loaded.radio_watchdog_minutes); +} + +TEST(MQTTPrefsCodec, CorruptOrShortVersionedInputsArePreserved) { + Codec::DecodePlan plan = Codec::classify(nullptr, 0, 0); + EXPECT_EQ(Codec::Source::Corrupt, plan.source); + EXPECT_TRUE(plan.preserve_file); + + std::vector partial_magic = {MQTT_PREFS_MAGIC[0], MQTT_PREFS_MAGIC[1], MQTT_PREFS_MAGIC[2]}; + plan = classify(partial_magic); + EXPECT_EQ(Codec::Source::Corrupt, plan.source); + EXPECT_TRUE(plan.preserve_file); + + std::vector short_payload(sizeof(MQTTPrefsHeader) + 4, 0); + writeHeader(&short_payload, MQTT_PREFS_VERSION, + static_cast(Codec::kV1BaselinePayloadSize - 1)); + plan = classify(short_payload); + EXPECT_EQ(Codec::Source::Corrupt, plan.source); + EXPECT_TRUE(plan.preserve_file); + + std::vector declared_short_with_full_body( + sizeof(MQTTPrefsHeader) + Codec::kV1BaselinePayloadSize, 0); + writeHeader(&declared_short_with_full_body, MQTT_PREFS_VERSION, + static_cast(Codec::kV1PreObserverPayloadSize)); + plan = classify(declared_short_with_full_body); + EXPECT_EQ(Codec::Source::Corrupt, plan.source); + EXPECT_TRUE(plan.preserve_file); + + std::vector declared_full_with_short_body( + sizeof(MQTTPrefsHeader) + Codec::kV1PreObserverPayloadSize, 0); + writeHeader(&declared_full_with_short_body, MQTT_PREFS_VERSION, + static_cast(Codec::kV1BaselinePayloadSize)); + plan = classify(declared_full_with_short_body); + EXPECT_EQ(Codec::Source::Corrupt, plan.source); + EXPECT_TRUE(plan.preserve_file); + + std::vector truncated(sizeof(MQTTPrefsHeader) + Codec::kV1BaselinePayloadSize - 1, 0); + writeHeader(&truncated, MQTT_PREFS_VERSION, + static_cast(Codec::kV1BaselinePayloadSize)); + plan = classify(truncated); + EXPECT_EQ(Codec::Source::Corrupt, plan.source); + EXPECT_TRUE(plan.preserve_file); + + std::vector trailing(sizeof(MQTTPrefsHeader) + Codec::kV1BaselinePayloadSize + 1, 0); + writeHeader(&trailing, MQTT_PREFS_VERSION, + static_cast(Codec::kV1BaselinePayloadSize)); + plan = classify(trailing); + EXPECT_EQ(Codec::Source::Corrupt, plan.source); + EXPECT_TRUE(plan.preserve_file); +} + +TEST(MQTTPrefsCodec, LegacyPlausibilityRejectsHighEntropyBytesAtEveryWhitelistedSize) { + // A headerless raw struct has no checksum, so this only reduces false + // migrations; it cannot prove that a plausible-looking file is authentic. + for (const size_t size : {size_t(472), size_t(1032), size_t(1464), size_t(2452), + size_t(2836), size_t(2840), size_t(2904)}) { + std::vector bytes(size, 0); + fillHighEntropy(&bytes); + const Codec::DecodePlan plan = classify(bytes); + ASSERT_TRUE(plan.rewrite_legacy) << size; + EXPECT_FALSE(Codec::isPlausibleLegacy(plan.source, bytes.data(), bytes.size())) << size; + } +} + +TEST(MQTTPrefsCodec, UnsupportedHeaderlessSizesArePreserved) { + // 3024 was produced briefly before versioning, but repository history says + // that raw observer-tail form was not shipped. Preserve it rather than guess. + for (const size_t size : {size_t(471), size_t(473), size_t(1465), size_t(2905), size_t(3024)}) { + std::vector bytes(size, 0); + const Codec::DecodePlan plan = classify(bytes); + EXPECT_EQ(Codec::Source::Corrupt, plan.source) << size; + EXPECT_TRUE(plan.preserve_file) << size; + } +} + +TEST(MQTTPrefsCodec, NewerAndSameVersionExtendedPayloadsAreHeld) { + std::vector newer(sizeof(MQTTPrefsHeader), 0); + writeHeader(&newer, MQTT_PREFS_VERSION + 1, 0); + Codec::DecodePlan plan = classify(newer); + EXPECT_EQ(Codec::Source::UnsupportedVersion, plan.source); + EXPECT_TRUE(plan.preserve_file); + + std::vector extended(sizeof(MQTTPrefsHeader) + Codec::kV1BaselinePayloadSize + 1, 0); + writeHeader(&extended, MQTT_PREFS_VERSION, + static_cast(Codec::kV1BaselinePayloadSize + 1)); + plan = classify(extended); + EXPECT_EQ(Codec::Source::UnsupportedVersion, plan.source); + EXPECT_TRUE(plan.preserve_file); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}