From cdfd1ad6910a2560434fe3f5fceb3b0672d5449c Mon Sep 17 00:00:00 2001 From: mikecarper Date: Sat, 17 Oct 2026 15:50:19 -0700 Subject: [PATCH] Retain USB packet logging in canonical nRF sensor OTA pairs --- build_legacy.sh | 64 ++++++++-- docs/cli_commands.md | 10 +- docs/ota_protocol.md | 8 +- test/test_nrf52_ota_profiles.sh | 2 + test/test_nrf52_paired_logging.py | 205 ++++++++++++++++++++++++++++++ 5 files changed, 278 insertions(+), 11 deletions(-) create mode 100644 test/test_nrf52_paired_logging.py diff --git a/build_legacy.sh b/build_legacy.sh index a0774142f..4e4242896 100755 --- a/build_legacy.sh +++ b/build_legacy.sh @@ -216,8 +216,9 @@ Environment Variables: A single custom version suffix found in existing OUTPUT_DIR artifacts is carried forward after the new numeric version. DISABLE_DEBUG=1: Disables build-time debug logging. Full Companion retains its - runtime-gated diagnostics; canonical XIAO QSPI repeaters retain - runtime-gated USB packet logging while MESH_DEBUG stays disabled. + runtime-gated diagnostics; canonical XIAO QSPI repeaters and + qualified RAK/Tower sensor-OTA pairs retain runtime-gated USB + packet logging while MESH_DEBUG stays disabled. If not set, debug flags from variant platformio.ini files are used. RESUME_BUILD_OUTPUT=1: Preserves out/ and skips targets whose expected output artifacts already exist. Option 3 resumes by default. @@ -2721,7 +2722,9 @@ disable_debug_flags() { # apparent order in PLATFORMIO_BUILD_FLAGS. if [ -n "$env_name" ] && is_companion_radio_full_target "$env_name"; then usb_logging_undefs="" - elif [ -n "$env_name" ] && is_xiao_qspi_canonical_repeater_build "$env_name"; then + elif [ -n "$env_name" ] \ + && { is_xiao_qspi_canonical_repeater_build "$env_name" \ + || is_nrf52_paired_runtime_logging_build "$env_name"; }; then # Its combined artifact promises runtime USB packet logging. Keep the # logger linked without retaining optional MESH_DEBUG output. usb_logging_undefs="-UMESH_DEBUG" @@ -2761,7 +2764,9 @@ apply_debug_overrides() { if [ -n "$env_name" ] && is_companion_radio_full_target "$env_name"; then preserve_full_companion_logging=1 preserve_runtime_packet_logging=1 - elif [ -n "$env_name" ] && is_xiao_qspi_canonical_repeater_build "$env_name"; then + elif [ -n "$env_name" ] \ + && { is_xiao_qspi_canonical_repeater_build "$env_name" \ + || is_nrf52_paired_runtime_logging_build "$env_name"; }; then preserve_runtime_packet_logging=1 fi @@ -2794,6 +2799,37 @@ apply_debug_overrides() { esac } +is_nrf52_paired_runtime_logging_build() { + local env_name=$1 + + # Required sensor pairs normalize normal nodes onto historical OTA aliases. + # Those aliases must not silently discard the ordinary USB logger. Qualify + # these exact canonical board/role recipes first; do not broaden the old + # size-limited OTA or specialized transport policy for unrelated targets. + [ "${PIO_ENV_PLATFORM_BY_NAME[$env_name]:-}" = "NRF52_PLATFORM" ] || return 1 + case "${NRF52_OTA_SENSOR_PROFILE:-}" in + full|reduced) ;; + *) return 1 ;; + esac + case "${env_name,,}" in + rak_3401_repeater|rak_3401_room_server|rak_3401_sensor|\ + rak_3401_repeater_lora_ota_no_external_sensors|\ + rak_3401_room_server_lora_ota_no_external_sensors|\ + rak_3401_sensor_lora_ota_no_external_sensors|\ + rak_3401_repeater_unified_lora_ota|\ + rak_4631_repeater|rak_4631_room_server|rak_4631_sensor|\ + rak_4631_repeater_lora_ota_no_external_sensors|\ + rak_4631_room_server_lora_ota_no_external_sensors|\ + rak_4631_sensor_lora_ota_no_external_sensors|\ + rak_4631_repeater_unified_lora_ota|\ + heltec_tower_v2_sdcard_repeater_lora_ota_no_external_sensors|\ + heltec_tower_v2_room_server|\ + heltec_tower_v2_room_server_lora_ota_no_external_sensors) + return 0 ;; + *) return 1 ;; + esac +} + uses_merged_standard_usb_logging() { local env_name=$1 @@ -2807,6 +2843,8 @@ uses_merged_standard_usb_logging() { ;; esac + is_nrf52_paired_runtime_logging_build "$env_name" && return 0 + # These profiles either own Serial for framed traffic, deliberately trade # logging for OTA space, or provide their own logging contract. Keep those # contracts unchanged. @@ -2834,13 +2872,18 @@ apply_merged_standard_usb_logging_profile() { uses_merged_standard_usb_logging "$env_name" || return 0 - # Canonical XIAO QSPI repeaters include packet logging in the immutable - # combined artifact. The saved usb.logging switch controls its live stream. + # Canonical combined recipes include packet logging in the immutable + # artifact. The saved usb.logging switch controls its live stream. local preserve_runtime_packet_logging=0 - if is_xiao_qspi_canonical_repeater_build "$env_name"; then + if is_xiao_qspi_canonical_repeater_build "$env_name" \ + || is_nrf52_paired_runtime_logging_build "$env_name"; then preserve_runtime_packet_logging=1 export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DMESH_PACKET_LOGGING=1 -DMESH_USB_LOGGING_MERGED=1" fi + if is_nrf52_paired_runtime_logging_build "$env_name" \ + && [ "${MESHDEBUG_OVERRIDE,,}" != "on" ]; then + export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -UMESH_DEBUG" + fi # Explicit diagnostic overrides retain their documented meaning. Canonical # builds otherwise compile packet/debug output into the ordinary artifact; @@ -2853,7 +2896,9 @@ apply_merged_standard_usb_logging_profile() { export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DMESH_PACKET_LOGGING=1 -DMESH_USB_LOGGING_MERGED=1" fi if [ "${MESHDEBUG_OVERRIDE,,}" != "off" ] \ - && ! is_logging_size_constrained_target "$env_name"; then + && ! is_logging_size_constrained_target "$env_name" \ + && { ! is_nrf52_paired_runtime_logging_build "$env_name" \ + || [ "${MESHDEBUG_OVERRIDE,,}" = "on" ]; }; then export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DMESH_DEBUG=1" fi } @@ -3424,7 +3469,8 @@ declare_full_logging_application_contract() { local env_name=$1 BUILD_APPLICATION_EXPECTATIONS=() if requires_full_usb_packet_logging "$env_name" \ - || is_xiao_qspi_canonical_repeater_build "$env_name"; then + || is_xiao_qspi_canonical_repeater_build "$env_name" \ + || is_nrf52_paired_runtime_logging_build "$env_name"; then # Inspect the packaged application, not an ELF debug/symbol string. The # settings alone can exist in debug-only builds with packet logging absent. record_application_expectation "logging.usb.packets" \ diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 8eb2ca7c1..fa46306b8 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -1249,7 +1249,15 @@ expected by ordinary USB-to-MQTT parsers. Ordinary non-OTA artifacts compile packet logging into the canonical image and control its live USB output at runtime; no separate `-logging-` artifact is -emitted. Use the separately named `-ota-` artifact when LoRa OTA is required. A +emitted. Canonical RAK3401/RAK4631 repeater, room and sensor OTA pairs, plus +Tower SD repeater and Tower room pairs, also retain packet logging in both +Full-sensor and Reduced-sensor artifacts. Verbose debug stays off by default; +`get/set usb.logging` controls packet output. Their capability manifests must +verify both `logging.usb.packets` and `logging.usb.control` in the packaged +application. Specialized storage/transport aliases and other space-constrained +OTA-only targets retain their target-specific logging policy. + +For other roles, use the separately named `-ota-` artifact when required. A `-full-usb-wifi-ota-` artifact combines USB packet logging, direct WiFi MQTT, LoRa OTA, and the expanded FULL feature set. A `-full-logging-ota-` artifact is emitted only when that hardware/role has no matching WiFi MQTT environment. diff --git a/docs/ota_protocol.md b/docs/ota_protocol.md index 6e68fd37a..8f324d1c4 100644 --- a/docs/ota_protocol.md +++ b/docs/ota_protocol.md @@ -189,7 +189,13 @@ FULL-capable non-companion hardware/role that has a matching MQTT environment. It compiles USB packet logging and direct WiFi MQTT together. A `*-full-logging-ota-*` fallback is emitted only when there is no MQTT sibling; ordinary non-OTA roles compile runtime USB logging into their canonical image, -so separate standard-logging artifacts are not emitted. Non-MQTT FULL twins +so separate standard-logging artifacts are not emitted. Canonical RAK3401/RAK4631 +repeater, room and sensor OTA pairs, and Tower SD repeater/room pairs, retain +runtime USB packet logging in both sensor variants too. Verbose debug is off +by default; packaged `logging.usb.packets` and `logging.usb.control` capability +checks prevent a stale no-logger image from satisfying a resumed build. +Specialized storage/transport aliases and other constrained OTA targets keep +their existing logging policy. Non-MQTT FULL twins are also skipped for covered ESP32 roles. MQTT observers and ESP-NOW bridges are emitted only with expanded FULL partitions. Menu option 8, or `build-full-esp32-firmwares`, builds the unified profiles plus necessary fallbacks. Menu option 9, or diff --git a/test/test_nrf52_ota_profiles.sh b/test/test_nrf52_ota_profiles.sh index e86c8d61f..e87e41f56 100644 --- a/test/test_nrf52_ota_profiles.sh +++ b/test/test_nrf52_ota_profiles.sh @@ -300,4 +300,6 @@ PY done done +python3 -B test/test_nrf52_paired_logging.py -v + echo "test_nrf52_ota_profiles: OK" diff --git a/test/test_nrf52_paired_logging.py b/test/test_nrf52_paired_logging.py new file mode 100644 index 000000000..cd18275ee --- /dev/null +++ b/test/test_nrf52_paired_logging.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Normal nRF52 sensor/OTA pairs retain their runtime USB packet logger.""" + +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +NORMAL_TARGETS = ( + "RAK_3401_repeater", "RAK_3401_room_server", "RAK_3401_sensor", + "RAK_4631_repeater", "RAK_4631_room_server", "RAK_4631_sensor", + "Heltec_tower_v2_repeater", "Heltec_tower_v2_room_server", +) +CANONICAL_TARGETS = tuple( + target + "_lora_ota_no_external_sensors" for target in NORMAL_TARGETS + if target != "Heltec_tower_v2_repeater" +) + ( + "Heltec_tower_v2_sdcard_repeater_lora_ota_no_external_sensors", + "RAK_3401_repeater_unified_lora_ota", "RAK_4631_repeater_unified_lora_ota", +) +PACKET_MARKER = "logging.usb.packets=%s: %s, len=%d (type=%d, route=%s, payload_len=%d)" +CONTROL_MARKER = "logging.usb.control=OK - USB logging %s (saved)" + + +def build_policy(target, *, profile="full", disable_debug=0, debug="", packet="", + normalize=False, platform="NRF52_PLATFORM"): + # Source production helpers without asking PlatformIO for inventory. The + # normal-to-alias mappings below are the inputs used by the real resolver. + command = r''' +set -euo pipefail +source build.sh +for base in RAK_3401_repeater RAK_3401_room_server RAK_3401_sensor \ + RAK_4631_repeater RAK_4631_room_server RAK_4631_sensor \ + Heltec_tower_v2_room_server; do + PIO_ENV_PLATFORM_BY_NAME["$base"]=NRF52_PLATFORM + PIO_ENV_PLATFORM_BY_NAME["${base}_lora_ota_no_external_sensors"]=NRF52_PLATFORM +done +PIO_ENV_PLATFORM_BY_NAME[Heltec_tower_v2_repeater]=NRF52_PLATFORM +PIO_ENV_PLATFORM_BY_NAME[Heltec_tower_v2_sdcard_repeater_lora_ota_no_external_sensors]=NRF52_PLATFORM +env_name="$1" +if [ "$6" = yes ]; then + env_name=$(get_nrf52_sensor_ota_pair_target "$env_name") +fi +PIO_ENV_PLATFORM_BY_NAME["$env_name"]="$7" +NRF52_OTA_SENSOR_PROFILE="$2" +DISABLE_DEBUG="$3" +MESHDEBUG_OVERRIDE="$4" +PACKET_LOGGING_OVERRIDE="$5" +BUILD_PROFILE_FOR_TARGET=auto +PLATFORMIO_BUILD_FLAGS="" +BUILD_APPLICATION_EXPECTATIONS=() +disable_debug_flags "$env_name" +apply_debug_overrides "$env_name" +apply_mqtt_bridge_override "$env_name" +disable_usb_logging_for_mqtt "$env_name" +apply_merged_standard_usb_logging_profile "$env_name" +declare_full_logging_application_contract "$env_name" +printf '%s\n' "$env_name" +printf '%s\n' "$PLATFORMIO_BUILD_FLAGS" +printf '%s\n' "${BUILD_APPLICATION_EXPECTATIONS[@]}" +''' + environment = os.environ.copy() + for name in ("MQTT_BRIDGE_OVERRIDE", "MQTT_DEBUG_OVERRIDE", + "PLATFORMIO_BUILD_FLAGS", "MESHDEBUG_OVERRIDE", + "PACKET_LOGGING_OVERRIDE", "DISABLE_DEBUG"): + environment.pop(name, None) + result = subprocess.run( + ["bash", "-c", command, "nrf52-paired-logging", target, profile, + str(disable_debug), debug, packet, "yes" if normalize else "no", + platform], cwd=ROOT, env=environment, text=True, capture_output=True, + check=True, timeout=10, + ) + lines = result.stdout.splitlines() + return lines[0], shlex.split(lines[1]), lines[2:] + + +class Nrf52PairedLoggingTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.compiler = shutil.which("g++") or shutil.which("c++") + if cls.compiler is None: + raise unittest.SkipTest("C++ preprocessor required") + cls.temporary = tempfile.TemporaryDirectory(prefix="meshcore-nrf-paired-logging-") + cls.includes = Path(cls.temporary.name) + (cls.includes / "Arduino.h").write_text("// No peripherals needed.\n", encoding="ascii") + cls.macros_cache = {} + + @classmethod + def tearDownClass(cls): + cls.temporary.cleanup() + + def macros(self, flags): + key = tuple(flags) + if key in self.macros_cache: + return self.macros_cache[key] + # Reproduce PlatformIO's -D-before--U ordering. Seed only verbose + # diagnostics, not packet logging: the normal policy must supply it. + defines = [flag for flag in flags if flag.startswith("-D")] + undefines = [flag for flag in flags if flag.startswith("-U")] + result = subprocess.run( + [self.compiler, "-x", "c++", "-E", "-dM", "-", + "-DARDUINO=10819", "-DNRF52_PLATFORM=1", "-DUSE_TINYUSB=1", + "-DMESH_DEBUG=1", "-I", str(self.includes), "-I", str(ROOT / "src"), + *defines, *undefines], input='#include \n', + text=True, capture_output=True, check=True, timeout=10, + ) + macros = {} + for line in result.stdout.splitlines(): + if line.startswith("#define "): + parts = line.split(maxsplit=2) + macros[parts[1]] = parts[2] if len(parts) == 3 else "" + self.macros_cache[key] = macros + return macros + + def assert_contract(self, flags, expectations): + macros = self.macros(flags) + self.assertEqual(macros.get("MESH_PACKET_LOGGING"), "1") + self.assertEqual(macros.get("MESH_USB_LOGGING_MERGED"), "1") + self.assertEqual(macros.get("MESH_USB_LOGGING_AVAILABLE"), "1") + self.assertIn(PACKET_MARKER, expectations) + self.assertIn(CONTROL_MARKER, expectations) + return macros + + def test_normal_targets_keep_logger_after_actual_alias_normalization(self): + for target in NORMAL_TARGETS: + for profile in ("full", "reduced"): + with self.subTest(target=target, profile=profile): + selected, flags, expectations = build_policy( + target, profile=profile, normalize=True) + self.assertIn(selected, CANONICAL_TARGETS) + macros = self.assert_contract(flags, expectations) + self.assertNotIn("MESH_DEBUG", macros) + + def test_both_canonical_pairs_preserve_packet_logging_against_build_overrides(self): + for target in CANONICAL_TARGETS: + for profile in ("full", "reduced"): + for disable_debug, debug, packet in ((0, "off", ""), (1, "", ""), + (0, "off", "off"), (1, "off", "off")): + with self.subTest(target=target, profile=profile, + disable_debug=disable_debug, packet=packet): + _, flags, expectations = build_policy( + target, profile=profile, disable_debug=disable_debug, + debug=debug, packet=packet) + macros = self.assert_contract(flags, expectations) + self.assertNotIn("MESH_DEBUG", macros) + + def test_verbose_debug_is_explicit_and_disable_debug_still_wins(self): + for disable_debug in (0, 1): + with self.subTest(disable_debug=disable_debug): + _, flags, expectations = build_policy( + CANONICAL_TARGETS[0], debug="on", disable_debug=disable_debug) + macros = self.assert_contract(flags, expectations) + if disable_debug: + self.assertNotIn("MESH_DEBUG", macros) + else: + self.assertEqual(macros.get("MESH_DEBUG"), "1") + + def test_old_ota_aliases_outside_the_required_pair_keep_their_policy(self): + for profile in ("", "custom"): + for target in CANONICAL_TARGETS: + with self.subTest(target=target, profile=profile): + _, flags, expectations = build_policy( + target, profile=profile, debug="off", packet="off") + macros = self.macros(flags) + self.assertNotIn("MESH_PACKET_LOGGING", macros) + self.assertEqual(macros.get("MESH_USB_LOGGING_AVAILABLE"), "0") + self.assertNotIn(PACKET_MARKER, expectations) + self.assertNotIn(CONTROL_MARKER, expectations) + + def test_specialized_unqualified_and_stm32_targets_are_not_broadened(self): + targets = ( + "RAK_3401_repeater_rak13302_w25q16_lora_ota", + "RAK_4631_repeater_rak15001_slot_c_lora_ota", + "RAK_4631_repeater_w25q16_lora_ota", + "RAK_4631_repeater_bridge_rs232_serial1_lora_ota_no_external_sensors", + "RAK_4631_repeater_bridge_rs232_serial2_lora_ota_no_external_sensors", + "RAK_4631_repeater_ethernet", "RAK_4631_room_server_ethernet", + "RAK_4631_repeater_rak15002_sd_seeder_only", + "RAK_4631_companion_radio_ble", "RAK_4631_kiss_modem", + "nrf_repeater_lora_ota_no_external_sensors", "Heltec_t096_repeater_lora_ota_no_external_sensors", + "RAK_3x72_companion_radio_usb", "Tiny_Relay_companion_radio_usb", + "wio-e5_companion_radio_usb", "wio-e5-mini_companion_radio_usb", + ) + for target in targets: + with self.subTest(target=target): + _, flags, expectations = build_policy( + target, debug="off", packet="off", disable_debug=1) + macros = self.macros(flags) + self.assertNotIn("MESH_PACKET_LOGGING", macros) + self.assertEqual(macros.get("MESH_USB_LOGGING_AVAILABLE"), "0") + self.assertNotIn(PACKET_MARKER, expectations) + _, flags, expectations = build_policy( + CANONICAL_TARGETS[0], debug="off", packet="off", disable_debug=1, + platform="STM32_PLATFORM") + self.assertNotIn("MESH_PACKET_LOGGING", self.macros(flags)) + self.assertNotIn(PACKET_MARKER, expectations) + + +if __name__ == "__main__": + unittest.main()