Files
MeshCore/test/test_esp32_hwcdc_recipes.py
T

407 lines
20 KiB
Python

#!/usr/bin/env python3
"""Resolve ESP32 USB recipes with PlatformIO without starting a build.
The board manifests select Serial's SDK class and the build-local SDK patch.
Exercise both selectors after PlatformIO's real flag/unflag processing. This
also checks UART and non-ESP32 recipes, whose transports must not migrate.
"""
from collections import Counter
import json
import os
from pathlib import Path
import re
import runpy
import sys
import unittest
try:
import SCons.Script
except ImportError:
core = Path(os.environ.get("PLATFORMIO_CORE_DIR", Path.home() / ".platformio"))
for path in sorted((core / "packages" / "tool-scons").glob("scons-local-*"), reverse=True):
sys.path.insert(0, str(path))
import SCons.Script
from platformio.builder.tools.piobuild import ParseFlagsExtended, ProcessFlags, ProcessUnFlags
from platformio.project.config import ProjectConfig
ROOT = Path(__file__).resolve().parents[1]
CORE = Path(os.environ.get("PLATFORMIO_CORE_DIR", Path.home() / ".platformio"))
USB_MODE = "ARDUINO_USB_MODE"
USB_CDC = "ARDUINO_USB_CDC_ON_BOOT"
USB_MACROS = (USB_MODE, USB_CDC)
# These are the eight native-USB families that used TinyUSB before migration.
# G2 already used HWCDC for its observer repeater and KISS roles. New
# Sensor roles inherit the same reviewed backend rather than migrating it.
MIGRATED_BOARD_COUNTS = {
"heltec_e213": 8,
"heltec_e290": 9,
"heltec_t190": 9,
"heltec_tracker_v1_1": 2,
"heltec_tracker_v2": 15,
"station-g2": 12,
"station-g3-esp32": 16,
"t_beam_1w": 11,
}
PREVIOUS_G2_HWCDC = {"Station_G2_repeater_observer_mqtt", "Station_G2_kiss_modem"}
LEGACY_DIAGNOSTIC_CDC_OVERRIDES = {
"heltec_v4_partition_migrator_test_usb_diagnostic",
"xiao_s3_partition_migrator_test_usb_diagnostic",
}
# Host-managed partition utilities keep application CDC disabled while flash
# tables change. Review them by name, independently of normal firmware roles:
# S3/C3 recover the ROM USB interface, and classic ESP32 uses external UART.
USB_PARTITION_BRIDGES = {
"esp32_s3_4mb_partition_migrator_usb": ("esp32-s3-devkitc-1", "esp32s3", "uart"),
"esp32_s3_8mb_partition_migrator_usb": ("esp32-s3-devkitc-1", "esp32s3", "uart"),
"esp32_s3_16mb_partition_migrator_usb": ("esp32-s3-devkitc-1", "esp32s3", "uart"),
"esp32_c3_4mb_partition_migrator_usb": ("esp32-c3-devkitm-1", "esp32c3", "uart"),
"esp32_4mb_partition_migrator_usb": ("esp32dev", "esp32", "classic_uart"),
"esp32_8mb_partition_migrator_usb": ("esp32dev", "esp32", "classic_uart"),
}
# Pin reviewed family/transport counts rather than copying a machine's
# environment inventory. USB partition bridges are tallied separately above;
# mixed firmware families have explicit role rules below.
UNCHANGED_COUNTS = {
("ESP32-S3-WROOM-1-N4", "uart"): 21,
("ebyte_eora-s3", "hwcdc"): 8,
("esp32-c3-devkitm-1", "uart"): 19,
("esp32-c6-devkitm-1", "hwcdc"): 16,
("esp32-s3-devkitc-1", "hwcdc"): 22,
("esp32-s3-devkitc-1", "uart"): 55,
("esp32-s3-zero", "hwcdc"): 17,
("esp32dev", "classic_uart"): 5,
("esp32doit-devkit-v1", "classic_uart"): 19,
("esp32s3box", "hwcdc"): 7,
("heltec-rc32", "hwcdc"): 22,
("heltec_v4", "hwcdc"): 45,
("heltec_v4_migrator", "hwcdc"): 1,
("heltec_v4_migrator", "uart"): 4,
("heltec_v4_r8", "hwcdc"): 25,
("heltec_wifi_lora_32_V2", "classic_uart"): 9,
("meshnology_w12", "hwcdc"): 12,
("seeed_xiao_esp32c3", "hwcdc"): 7,
("seeed_xiao_esp32s3", "hwcdc"): 26,
("seeed_xiao_esp32s3_migrator", "hwcdc"): 1,
("seeed_xiao_esp32s3_migrator", "uart"): 3,
("t-deck", "hwcdc"): 4,
("t3_s3_v1_x", "hwcdc"): 19,
("t_beams3_supreme", "hwcdc"): 9,
("thinknode_m7", "uart"): 10,
("thinknode_m9", "uart"): 6,
("ttgo-lora32-v1", "classic_uart"): 12,
("ttgo-t-beam", "classic_uart"): 16,
}
# Native-only CI does not install the ESP32 platform's standard manifests.
# Use installed pinned manifests when present, and retain their 6.11.0 USB
# defaults for a test requiring no downloads. Project-owned manifests,
# including every migrated board, are always read from the checkout.
EXTERNAL_BOARD_DEFAULTS = {
"esp32-c3-devkitm-1": ("esp32c3", []),
"esp32-c6-devkitm-1": ("esp32c6", []),
"esp32-s3-devkitc-1": ("esp32s3", ["-DARDUINO_USB_MODE=1"]),
"esp32dev": ("esp32", []),
"esp32doit-devkit-v1": ("esp32", []),
"esp32s3box": ("esp32s3", ["-DARDUINO_USB_MODE=1", "-DARDUINO_USB_CDC_ON_BOOT=1"]),
"heltec_wifi_lora_32_V2": ("esp32", []),
"seeed_xiao_esp32c3": ("esp32c3", ["-DARDUINO_USB_MODE=1", "-DARDUINO_USB_CDC_ON_BOOT=1"]),
"seeed_xiao_esp32s3": ("esp32s3", ["-DARDUINO_USB_MODE=1", "-DARDUINO_USB_CDC_ON_BOOT=1"]),
"ttgo-lora32-v1": ("esp32", []),
"ttgo-t-beam": ("esp32", []),
}
def flag_environment(board_flags=(), build_flags=(), build_unflags=()):
env = SCons.Script.Environment(tools=["gcc"])
for function in (ParseFlagsExtended, ProcessFlags, ProcessUnFlags):
env.AddMethod(function)
# Match ProcessProgramDeps: board, user, then build_unflags.
env.ProcessFlags(list(board_flags))
env.ProcessFlags(list(build_flags))
env.ProcessUnFlags(list(build_unflags))
return env
def usb_values(env, *, legacy_diagnostic_cdc_override=False):
values = {name: [] for name in USB_MACROS}
for definition in env.get("CPPDEFINES", []):
if isinstance(definition, (tuple, list)):
name, value = definition[:2]
else:
name, value = definition, "1"
if name in values:
values[name].append(str(value))
for name, history in values.items():
definitions = set(history)
permitted_override = legacy_diagnostic_cdc_override and name == USB_CDC
if (len(definitions) > 1 and not permitted_override) or definitions - {"0", "1"}:
raise ValueError(f"ambiguous USB selector {name}: {sorted(definitions)}")
# ProcessFlags moves -U options to the end of the compiler command.
# The SDK patch selector reads CPPDEFINES, so such an override could
# compile a different class from the one whose source was patched.
if re.search(r"-U\s*" + re.escape(name) + r"\b", str(env.get("_CPPDEFFLAGS", ""))):
raise ValueError(f"USB selector undefinition bypasses SDK selection: {name}")
result = {name: history[-1] if history else "0" for name, history in values.items()}
if result[USB_CDC] == "1" and not values[USB_MODE]:
raise ValueError("native CDC requires an explicit USB backend for SDK selection")
return result
def bool_option(value):
if isinstance(value, bool):
return value
if str(value).lower() not in ("true", "false", "1", "0", "yes", "no"):
raise ValueError(f"invalid upload boolean: {value}")
return str(value).lower() in ("true", "1", "yes")
def expected_other_transport(board, name):
if name in USB_PARTITION_BRIDGES:
expected_board, _, kind = USB_PARTITION_BRIDGES[name]
if board != expected_board:
raise AssertionError(f"USB partition bridge {name} changed board: {board}")
return kind
choices = {kind for family, kind in UNCHANGED_COUNTS if family == board}
if len(choices) == 1:
return next(iter(choices))
if board == "esp32-s3-devkitc-1":
return "hwcdc" if name.startswith(("Heltec_Wireless_Tracker_", "RAK_3112_")) else "uart"
if board in ("heltec_v4_migrator", "seeed_xiao_esp32s3_migrator"):
return "hwcdc" if name.endswith("_test_usb_diagnostic") else "uart"
raise AssertionError(f"unreviewed ESP32 board: {board}")
class Esp32HwcdcRecipeTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
# Load tracked recipes explicitly. A developer's platformio.local.ini
# must not create extra matrix rows or change this regression contract.
cls.config = ProjectConfig(str(ROOT / "platformio.ini"), parse_extra=False)
for path in sorted((ROOT / "variants").glob("*/platformio.ini")):
cls.config.read(str(path), parse_extra=False)
cls.sdk = runpy.run_path(str(ROOT / "scripts/esp32_usb_session_fix.py"))
cls.rows = {}
for section in cls.config.sections():
if not section.startswith("env:"):
continue
build_flags = cls.config.get(section, "build_flags", [])
board = cls.config.get(section, "board", "")
manifest_path = ROOT / "boards" / f"{board}.json"
manifest = json.loads(manifest_path.read_text()) if manifest_path.is_file() else {}
esp32 = any(re.search(r"\bESP32_PLATFORM\b", flag) for flag in build_flags)
if manifest:
mcu = manifest["build"].get("mcu", "")
board_flags = manifest["build"].get("extra_flags", [])
elif esp32:
mcu, board_flags = EXTERNAL_BOARD_DEFAULTS[board]
pinned_path = CORE / "platforms" / "espressif32@6.11.0" / "boards" / f"{board}.json"
if pinned_path.is_file():
pinned = json.loads(pinned_path.read_text())
mcu = pinned["build"]["mcu"]
board_flags = pinned["build"].get("extra_flags", [])
else:
mcu, board_flags = "", []
if isinstance(board_flags, str):
board_flags = [board_flags]
env = flag_environment(board_flags, build_flags,
cls.config.get(section, "build_unflags", []))
try:
values = usb_values(env, legacy_diagnostic_cdc_override=(
section[4:] in LEGACY_DIAGNOSTIC_CDC_OVERRIDES))
except ValueError as error:
raise ValueError(f"{section}: {error}") from error
if not esp32:
kind = "non_esp32"
elif mcu == "esp32":
kind = "classic_uart"
elif values[USB_CDC] == "0":
kind = "uart"
else:
kind = "hwcdc" if values[USB_MODE] == "1" else "tinyusb"
cls.rows[section[4:]] = {
"board": board, "mcu": mcu, "env": env, "values": values,
"esp32": esp32, "kind": kind, "manifest": manifest,
}
def test_all_reviewed_family_roles_select_hwcdc_and_its_sdk_patch(self):
counts = Counter()
migrated = set()
for name, row in self.rows.items():
if row["board"] not in MIGRATED_BOARD_COUNTS:
continue
with self.subTest(environment=name):
counts[row["board"]] += 1
self.assertEqual(row["mcu"], "esp32s3")
self.assertEqual(row["values"], {USB_MODE: "1", USB_CDC: "1"})
self.assertEqual(row["kind"], "hwcdc")
self.assertTrue(self.sdk["hwcdc_primary_enabled"](row["env"]))
self.assertFalse(self.sdk["native_primary_enabled"](row["env"]))
self.assertEqual(self.config.get(f"env:{name}", "platform"),
"platformio/espressif32@6.11.0")
self.assertFalse(self.config.get(f"env:{name}", "platform_packages", []))
self.assertIn("pre:scripts/esp32_usb_session_fix.py",
self.config.get(f"env:{name}", "extra_scripts", []))
if name not in PREVIOUS_G2_HWCDC:
migrated.add(name)
self.assertEqual(counts, MIGRATED_BOARD_COUNTS)
self.assertTrue(PREVIOUS_G2_HWCDC <= self.rows.keys())
self.assertEqual(len(migrated), 80)
def test_migrated_uploads_use_hardware_reset_without_touch_or_port_wait(self):
for name, row in self.rows.items():
if row["board"] not in MIGRATED_BOARD_COUNTS:
continue
with self.subTest(environment=name):
upload = row["manifest"]["upload"]
for option in ("use_1200bps_touch", "wait_for_upload_port"):
self.assertIn(option, upload, "native HWCDC upload defaults must be explicit")
value = self.config.get(f"env:{name}", f"board_upload.{option}", upload[option])
self.assertFalse(bool_option(value), option)
before_reset = self.config.get(f"env:{name}", "board_upload.before_reset",
upload.get("before_reset", "default_reset"))
self.assertIn(before_reset, ("default_reset", "usb_reset"))
self.assertIn(["0x303A", "0x1001"], row["manifest"]["build"]["hwids"])
def test_no_native_tinyusb_esp32_role_remains_and_sdk_selectors_agree(self):
counts = Counter()
for name, row in self.rows.items():
if not row["esp32"]:
continue
with self.subTest(environment=name):
counts[row["kind"]] += 1
self.assertNotEqual(row["kind"], "tinyusb")
self.assertEqual(self.sdk["hwcdc_primary_enabled"](row["env"]),
row["kind"] == "hwcdc")
self.assertFalse(self.sdk["native_primary_enabled"](row["env"]))
expected = Counter({"hwcdc": 323, "uart": 118, "classic_uart": 61})
expected.update(kind for _, _, kind in USB_PARTITION_BRIDGES.values())
self.assertEqual(counts, expected)
def test_usb_partition_bridges_keep_application_usb_disabled(self):
names = {name for name in self.rows if name.startswith("esp32_")
and name.endswith("_partition_migrator_usb")}
self.assertEqual(names, USB_PARTITION_BRIDGES.keys())
for name, (board, mcu, kind) in USB_PARTITION_BRIDGES.items():
with self.subTest(environment=name):
row = self.rows[name]
self.assertEqual((row["board"], row["mcu"], row["kind"]), (board, mcu, kind))
self.assertEqual(row["values"], {USB_MODE: "0", USB_CDC: "0"})
definitions = dict(definition if isinstance(definition, (tuple, list))
else (definition, 1)
for definition in row["env"].get("CPPDEFINES", []))
self.assertEqual(str(definitions.get("MESHCORE_MIGRATION_USB")), "1")
for option in ("ARDUINO_USB_MSC_ON_BOOT", "ARDUINO_USB_DFU_ON_BOOT",
"ARDUINO_USB_ON_BOOT"):
self.assertEqual(str(definitions.get(option, 0)), "0", option)
self.assertFalse(self.sdk["hwcdc_primary_enabled"](row["env"]))
self.assertFalse(self.sdk["native_primary_enabled"](row["env"]))
def test_all_m5stack_native_usb_roles_select_hwcdc_and_its_startup_patch(self):
# M5's documented USB-C wiring exposes C6 Serial/JTAG, including when
# the Companion uses BLE. Resolve real SDK/compiler flags so a UART
# console regression cannot hide in the unchanged-transport counts.
names = {name for name in self.rows if name.startswith("M5Stack_Unit_C6L_")}
self.assertEqual(names, {
"M5Stack_Unit_C6L_repeater", "M5Stack_Unit_C6L_room_server",
"M5Stack_Unit_C6L_companion_radio_ble",
"M5Stack_Unit_C6L_companion_radio_usb", "M5Stack_Unit_C6L_kiss_modem",
})
for name in names:
with self.subTest(environment=name):
row = self.rows[name]
self.assertEqual(row["mcu"], "esp32c6")
self.assertEqual(row["kind"], "hwcdc")
self.assertEqual(row["values"], {USB_MODE: "1", USB_CDC: "1"})
self.assertTrue(self.sdk["hwcdc_primary_enabled"](row["env"]))
self.assertFalse(self.sdk["native_primary_enabled"](row["env"]))
self.assertEqual(self.config.get(f"env:{name}", "extra_scripts", []).count(
"pre:scripts/esp32_usb_session_fix.py"), 1)
self.assertEqual(self.config.get(f"env:{name}", "platform"),
"https://github.com/pioarduino/platform-espressif32/releases/"
"download/53.03.13-1/platform-espressif32.zip")
def test_m5stack_selector_removal_reproduces_the_old_uart_backend(self):
for name in ("M5Stack_Unit_C6L_repeater", "M5Stack_Unit_C6L_room_server",
"M5Stack_Unit_C6L_companion_radio_ble", "M5Stack_Unit_C6L_kiss_modem"):
with self.subTest(environment=name):
env = flag_environment(build_flags=self.config.get(f"env:{name}", "build_flags"),
build_unflags=["-DARDUINO_USB_MODE=1",
"-DARDUINO_USB_CDC_ON_BOOT=1"])
self.assertEqual(usb_values(env), {USB_MODE: "0", USB_CDC: "0"})
self.assertFalse(self.sdk["hwcdc_primary_enabled"](env))
def test_uart_existing_hwcdc_and_newer_mcu_recipes_match_reviewed_transports(self):
counts = Counter()
for name, row in self.rows.items():
if not row["esp32"] or row["board"] in MIGRATED_BOARD_COUNTS:
continue
with self.subTest(environment=name):
expected = expected_other_transport(row["board"], name)
self.assertEqual(row["kind"], expected)
counts[row["board"], row["kind"]] += 1
expected = Counter(UNCHANGED_COUNTS)
expected.update((board, kind) for board, _, kind in USB_PARTITION_BRIDGES.values())
self.assertEqual(counts, expected)
def test_non_esp32_recipes_do_not_acquire_esp32_usb_selectors(self):
count = 0
for name, row in self.rows.items():
if row["esp32"]:
continue
with self.subTest(environment=name):
count += 1
self.assertEqual(row["values"], {USB_MODE: "0", USB_CDC: "0"})
self.assertFalse(self.sdk["hwcdc_primary_enabled"](row["env"]))
self.assertFalse(self.sdk["native_primary_enabled"](row["env"]))
self.assertEqual(count, 321)
def test_only_existing_partition_diagnostics_keep_their_cdc_override(self):
# These two pre-existing diagnostic recipes intentionally select CDC1
# over a CDC0 recovery board. Keep their compiler/SDK last-definition
# behavior while rejecting conflicting selectors in normal images.
for name in LEGACY_DIAGNOSTIC_CDC_OVERRIDES:
with self.subTest(environment=name):
row = self.rows[name]
with self.assertRaises(ValueError):
usb_values(row["env"])
self.assertEqual(row["kind"], "hwcdc")
self.assertEqual(row["values"], {USB_MODE: "1", USB_CDC: "1"})
def test_real_unflags_remove_the_old_selector_and_identical_duplicates_are_safe(self):
env = flag_environment(["-DARDUINO_USB_MODE=0", "-DARDUINO_USB_CDC_ON_BOOT=1"],
["-D ARDUINO_USB_MODE=1", "-DARDUINO_USB_MODE=1"],
["-DARDUINO_USB_MODE=0"])
self.assertEqual(usb_values(env), {USB_MODE: "1", USB_CDC: "1"})
self.assertTrue(self.sdk["hwcdc_primary_enabled"](env))
self.assertFalse(self.sdk["native_primary_enabled"](env))
env = flag_environment(["-DARDUINO_USB_MODE=1", "-DARDUINO_USB_CDC_ON_BOOT=0"])
self.assertEqual(usb_values(env), {USB_MODE: "1", USB_CDC: "0"})
self.assertFalse(self.sdk["hwcdc_primary_enabled"](env))
def test_conflicting_missing_and_undefined_backend_flags_are_rejected(self):
cases = (
(["-DARDUINO_USB_MODE=0", "-DARDUINO_USB_CDC_ON_BOOT=1"],
["-DARDUINO_USB_MODE=1"], []),
(["-DARDUINO_USB_MODE=1", "-DARDUINO_USB_CDC_ON_BOOT=1"],
["-DARDUINO_USB_CDC_ON_BOOT=0"], []),
(["-DARDUINO_USB_MODE=1", "-DARDUINO_USB_CDC_ON_BOOT=1"],
["-UARDUINO_USB_MODE"], []),
(["-DARDUINO_USB_MODE=1", "-DARDUINO_USB_CDC_ON_BOOT=1"],
[], ["-DARDUINO_USB_MODE"]),
(["-DARDUINO_USB_MODE=2", "-DARDUINO_USB_CDC_ON_BOOT=1"], [], []),
)
for board_flags, flags, unflags in cases:
with self.subTest(board=board_flags, flags=flags, unflags=unflags):
with self.assertRaises(ValueError):
usb_values(flag_environment(board_flags, flags, unflags))
if __name__ == "__main__":
unittest.main()