Resolve inbound channel identity from the live slot, not a stale cache
Channel messages carry only a slot index on the wire, which the firmware assigns from the channel's current position. The old idx->identity cache was trusted on a hit, so a slot reorder or re-key (in-app, via the CLI/ config tool, or on another client) left the cache stale and misfiled later messages under the wrong channel. channel_id_for now reads the occupying slot fresh from the device on every call, so identity always reflects who is actually in the slot right now; _channel_ids is demoted to a priming/backfill warm map and a read-failure fallback. Inbound recording moves onto a single serialized worker draining an asyncio.Queue, so channel messages still persist in arrival order despite each needing an async device read, and a slow read can no longer stall the event pump. Second root cause, same symptom: a Public message could still land in another channel (e.g. #bots) even after the fresh-read fix. The meshcore library's get_channel waits for "the next CHANNEL_INFO event" with no correlation to the slot it asked for, and the dispatcher fans that event to every in-flight waiter. Two concurrent get_channel calls therefore both resolve on whichever response arrives first, and one caller silently receives the other's channel — which fired whenever the chat worker's read raced any other channel read (conversation-list builder, channels screen, refresh_channels). Fixed in the device wrapper: a per-device asyncio.Lock serializes channel reads so at most one is outstanding and the response is unambiguously ours, plus get_channel now validates the response's channel_idx matches the requested slot and raises rather than handing back foreign channel data (the chat layer then falls back to the last-known identity instead of misrouting). Adds tests for filing by the current slot occupant after an out-of-band reorder, for arrival-order preservation across a burst, and for the serialized/index-validated channel read (no cross-talk, mismatch rejected). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
1 parent
8c1c9959e3
commit
18368d6ca1
5 files changed
+274
-52
No files matched your search
@@ -408,6 +408,12 @@ class MeshCoreDevice(Device):
|
||||
self._port = port
|
||||
self._baudrate = baudrate
|
||||
self._mc = None # type: ignore[var-annotated] # meshcore.MeshCore
|
||||
# Serializes channel reads. The meshcore library's get_channel waits for "the next
|
||||
# CHANNEL_INFO event" with no correlation to the index it asked for, and the dispatcher
|
||||
# fans that event to *every* in-flight waiter — so two concurrent reads both resolve on
|
||||
# the first response and one caller silently gets the other's channel. Holding this lock
|
||||
# keeps at most one channel read outstanding, so the response is unambiguously ours.
|
||||
self._channel_read_lock = asyncio.Lock()
|
||||
|
||||
async def connect(self) -> None: # noqa: D102 - inherited docstring
|
||||
if self._mc is not None:
|
||||
@@ -906,8 +912,19 @@ class MeshCoreDevice(Device):
|
||||
return dict(getattr(event, "payload", {}) or {})
|
||||
|
||||
async def get_channel(self, index: int) -> Optional[dict]: # noqa: D102
|
||||
event = self._ok(await self._require().commands.get_channel(index))
|
||||
# The read is serialized (see self._channel_read_lock) so the uncorrelated
|
||||
# CHANNEL_INFO response can't be stolen by a concurrent read. We still verify the
|
||||
# response is for the slot we asked about: a stray CHANNEL_INFO (from another source,
|
||||
# or a slow response arriving after a timeout) would otherwise misidentify the slot and
|
||||
# misfile a channel's messages. On mismatch we raise rather than return foreign data.
|
||||
async with self._channel_read_lock:
|
||||
event = self._ok(await self._require().commands.get_channel(index))
|
||||
payload = getattr(event, "payload", {}) or {}
|
||||
got = payload.get("channel_idx")
|
||||
if got is not None and int(got) != index:
|
||||
raise DeviceCommandError(
|
||||
f"channel read for slot {index} returned slot {got}"
|
||||
)
|
||||
if not payload.get("channel_name"):
|
||||
return None
|
||||
return payload
|
||||
|
||||
@@ -60,9 +60,15 @@ class ChatService:
|
||||
self._unread: dict[str, int] = {}
|
||||
self._active: Optional[str] = None
|
||||
self._session_count = 0
|
||||
# Cache of channel slot index -> the channel's intrinsic identity. The wire only
|
||||
# tells us a message's slot index, so this bridges it to the identity history is
|
||||
# keyed by. Rebuilt from the device on demand (see :meth:`refresh_channels`).
|
||||
# Inbound messages are recorded off a single serialized worker (see
|
||||
# :meth:`_process_inbound`) so channel messages — whose identity is resolved with an
|
||||
# async device read — are still stored in the order they arrived. The queue is the
|
||||
# hand-off from the (synchronous) hub callback to that worker.
|
||||
self._queue: Optional["asyncio.Queue[Message]"] = None
|
||||
self._worker: Optional["asyncio.Task[None]"] = None
|
||||
# Last-known channel slot index -> intrinsic identity. Inbound resolution reads the
|
||||
# slot fresh every time (see :meth:`channel_id_for`), so this is not the source of
|
||||
# truth — only a warm map for priming/backfill and a fallback when a read fails.
|
||||
self._channel_ids: dict[int, str] = {}
|
||||
|
||||
@property
|
||||
@@ -118,11 +124,16 @@ class ChatService:
|
||||
self._channel_ids = ids
|
||||
|
||||
async def channel_id_for(self, idx: int) -> str:
|
||||
"""Resolve a channel slot index to its intrinsic identity (reading the device once).
|
||||
"""Resolve a channel slot index to its intrinsic identity, reading the slot fresh.
|
||||
|
||||
Refreshes the cache on a miss, then falls back to a slot-derived identity only if the
|
||||
device has no channel there. The fallback is cached so a stream of messages on an
|
||||
unconfigured slot doesn't re-read the device on every one.
|
||||
The slot's *current* occupant is read from the device every call, so the identity
|
||||
always reflects the channel that is actually in that slot right now — even if the
|
||||
slots were reordered or re-keyed (in this app, via the CLI/config tool, or on another
|
||||
client such as the phone app) since the cache was last built. Trusting a session-long
|
||||
slot→identity cache instead is exactly what let a reorder misfile a channel's
|
||||
messages. The resolved identity refreshes the cache; a transient read failure falls
|
||||
back to the last identity we saw for the slot (so a blip doesn't split a channel's
|
||||
history), and only a genuinely empty slot yields the slot-derived identity.
|
||||
|
||||
Args:
|
||||
idx: The channel slot index the wire reported.
|
||||
@@ -130,15 +141,19 @@ class ChatService:
|
||||
Returns:
|
||||
The channel's identity, suitable for keying its history.
|
||||
"""
|
||||
cid = self._channel_ids.get(idx)
|
||||
if cid is not None:
|
||||
return cid
|
||||
await self.refresh_channels()
|
||||
cid = self._channel_ids.get(idx)
|
||||
if cid is None:
|
||||
cid = _fallback_channel_id(idx)
|
||||
try:
|
||||
device = await self._ctx.device()
|
||||
payload = await device.get_channel(idx)
|
||||
except Exception as exc: # noqa: BLE001 - fall back rather than misroute or drop
|
||||
self._ctx.log.debug("chat: channel read for slot %s failed: %s", idx, exc)
|
||||
return self._channel_ids.get(idx) or _fallback_channel_id(idx)
|
||||
if payload:
|
||||
name = str(payload.get("channel_name") or "")
|
||||
secret = bytes(payload.get("channel_secret") or b"\x00" * 16)
|
||||
cid = channel_identity(name, secret)
|
||||
self._channel_ids[idx] = cid
|
||||
return cid
|
||||
return cid
|
||||
return _fallback_channel_id(idx)
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Begin recording inbound messages to history. Idempotent.
|
||||
@@ -157,13 +172,17 @@ class ChatService:
|
||||
run_id = self._ctx.repo.start_run(
|
||||
"chat", {"mode": "background"}, self._ctx.profile_name
|
||||
)
|
||||
self._queue = asyncio.Queue()
|
||||
self._worker = asyncio.ensure_future(self._process_inbound(run_id))
|
||||
|
||||
def on_event(event: MeshEvent) -> None:
|
||||
# Runs on the event loop as messages arrive; keep it cheap and defensive so a
|
||||
# single bad write can never take down the subscription.
|
||||
# Runs on the event loop as messages arrive; keep it cheap and non-blocking. It
|
||||
# only hands the message to the worker queue — resolution and the database write
|
||||
# happen off the worker so a slow device read can never stall the event pump.
|
||||
message = event.message
|
||||
if message is not None:
|
||||
self._record_inbound(run_id, message)
|
||||
queue = self._queue
|
||||
if message is not None and queue is not None:
|
||||
queue.put_nowait(message)
|
||||
|
||||
self._unsubscribe = self._ctx.events.subscribe(on_event, EventKind.MESSAGE)
|
||||
self._run_id = run_id
|
||||
@@ -171,12 +190,12 @@ class ChatService:
|
||||
await self._prime_channels()
|
||||
|
||||
async def _prime_channels(self) -> None:
|
||||
"""Warm the channel-identity cache and backfill legacy (index-keyed) history.
|
||||
"""Warm the fallback channel-identity map and backfill legacy (index-keyed) history.
|
||||
|
||||
Reading the channels up front means most inbound messages resolve their identity
|
||||
from the cache without a device round-trip. It also backfills any pre-identity
|
||||
messages using the channels currently in each slot — best-effort, since the old
|
||||
slot-to-channel mapping wasn't recorded.
|
||||
Reading the channels up front seeds the map that :meth:`channel_id_for` falls back to
|
||||
when a per-message read fails. It also backfills any pre-identity messages using the
|
||||
channels currently in each slot — best-effort, since the old slot-to-channel mapping
|
||||
wasn't recorded.
|
||||
"""
|
||||
try:
|
||||
await self.refresh_channels()
|
||||
@@ -189,7 +208,7 @@ class ChatService:
|
||||
"""Stop recording and close the run record. Idempotent.
|
||||
|
||||
A no-op if not recording. The event hub keeps listening; only this service's
|
||||
recording subscription is removed.
|
||||
recording subscription and its inbound worker are removed.
|
||||
"""
|
||||
if not self.active:
|
||||
return
|
||||
@@ -198,6 +217,14 @@ class ChatService:
|
||||
self._unsubscribe()
|
||||
finally:
|
||||
self._unsubscribe = None
|
||||
if self._worker is not None:
|
||||
self._worker.cancel()
|
||||
try:
|
||||
await self._worker
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._worker = None
|
||||
self._queue = None
|
||||
if self._run_id is not None:
|
||||
self._ctx.repo.finish_run(
|
||||
self._run_id, "ok", {"messages": self._session_count}
|
||||
@@ -208,35 +235,32 @@ class ChatService:
|
||||
"""Stop recording at session end."""
|
||||
await self.stop()
|
||||
|
||||
def _record_inbound(self, run_id: int, message: Message) -> None:
|
||||
"""Persist one inbound message and bump its conversation's unread count.
|
||||
async def _process_inbound(self, run_id: int) -> None:
|
||||
"""Serially resolve and record queued inbound messages, preserving arrival order.
|
||||
|
||||
Channel messages are keyed by identity, but the wire carries only a slot index. When
|
||||
the slot is already cached this resolves synchronously; on a cache miss (a channel
|
||||
added since the last device read) resolution is deferred to the event loop so the
|
||||
message is still recorded against its intrinsic identity.
|
||||
A single worker drains the queue so that channel messages — whose identity is
|
||||
resolved with an async device read (see :meth:`channel_id_for`) — are still stored in
|
||||
the order they arrived. History is ordered by insertion, so recording concurrently
|
||||
could interleave two messages by their differing read latencies; the serial worker
|
||||
makes that impossible while still resolving each against the live device.
|
||||
|
||||
Args:
|
||||
run_id: The owning background ``chat`` run.
|
||||
message: The received message to record.
|
||||
run_id: The owning background ``chat`` run to link recorded messages to.
|
||||
"""
|
||||
if message.is_channel:
|
||||
channel_id = self._channel_ids.get(message.channel)
|
||||
if channel_id is None:
|
||||
asyncio.ensure_future(self._resolve_then_record(run_id, message))
|
||||
return
|
||||
else:
|
||||
channel_id = None
|
||||
self._store_inbound(run_id, message, channel_id)
|
||||
|
||||
async def _resolve_then_record(self, run_id: int, message: Message) -> None:
|
||||
"""Resolve an uncached channel's identity off the event loop, then record it."""
|
||||
try:
|
||||
channel_id = await self.channel_id_for(message.channel)
|
||||
except Exception as exc: # noqa: BLE001 - fall back rather than drop the message
|
||||
self._ctx.log.debug("chat: channel resolve failed: %s", exc)
|
||||
channel_id = _fallback_channel_id(message.channel)
|
||||
self._store_inbound(run_id, message, channel_id)
|
||||
assert self._queue is not None
|
||||
while True:
|
||||
message = await self._queue.get()
|
||||
try:
|
||||
channel_id = (
|
||||
await self.channel_id_for(message.channel)
|
||||
if message.is_channel
|
||||
else None
|
||||
)
|
||||
self._store_inbound(run_id, message, channel_id)
|
||||
except Exception as exc: # noqa: BLE001 - one bad message must not kill the worker
|
||||
self._ctx.log.debug("chat: failed to record message: %s", exc)
|
||||
finally:
|
||||
self._queue.task_done()
|
||||
|
||||
def _store_inbound(
|
||||
self, run_id: int, message: Message, channel_id: Optional[str]
|
||||
|
||||
@@ -260,10 +260,13 @@ async def test_recreating_a_slot_refiles_messages_to_the_new_channel(ctx: AppCon
|
||||
|
||||
await ctx.chat.start() # prime the slot→identity cache (slot 0 == Public)
|
||||
try:
|
||||
# A message on Public arrives before we touch anything — files under Public.
|
||||
# A message on Public arrives before we touch anything — files under Public. The
|
||||
# inbound worker resolves it against the slot as it stands now (Public), which is
|
||||
# exactly what happens live: messages are recorded on arrival, ahead of any later edit.
|
||||
ctx.events.publish(
|
||||
MeshEvent.message_event(Message(text="hi public", channel=0, is_channel=True))
|
||||
)
|
||||
await ctx.chat._queue.join()
|
||||
|
||||
# Drive the manager: open Public's detail → clear it, then create a new private
|
||||
# channel (which reuses freed slot 0), then back out.
|
||||
@@ -281,6 +284,7 @@ async def test_recreating_a_slot_refiles_messages_to_the_new_channel(ctx: AppCon
|
||||
ctx.events.publish(
|
||||
MeshEvent.message_event(Message(text="ops secret", channel=0, is_channel=True))
|
||||
)
|
||||
await ctx.chat._queue.join()
|
||||
|
||||
public_msgs = ctx.repo.recent_chat_messages(is_channel=True, channel_id=public_id)
|
||||
ops_msgs = ctx.repo.recent_chat_messages(is_channel=True, channel_id=ops.identity)
|
||||
|
||||
@@ -337,6 +337,7 @@ async def test_service_records_inbound_and_tracks_unread(repo: Repository) -> No
|
||||
ctx.events.publish(
|
||||
MeshEvent.message_event(Message(text="ping", sender="ffeeddcc", is_channel=False))
|
||||
)
|
||||
await chat._queue.join() # let the inbound worker resolve and record the message
|
||||
assert chat.unread("dm:ffeeddcc") == 1
|
||||
assert chat.unread_total() >= 1
|
||||
stored = repo.recent_chat_messages(is_channel=False, peer="ffeeddcc")
|
||||
@@ -357,6 +358,7 @@ async def test_service_active_conversation_suppresses_unread(repo: Repository) -
|
||||
ctx.events.publish(
|
||||
MeshEvent.message_event(Message(text="ping", sender="ffeeddcc", is_channel=False))
|
||||
)
|
||||
await chat._queue.join() # let the inbound worker resolve and record the message
|
||||
assert chat.unread("dm:ffeeddcc") == 0 # active thread doesn't accrue unread
|
||||
assert repo.recent_chat_messages(is_channel=False, peer="ffeeddcc") # still recorded
|
||||
finally:
|
||||
@@ -364,6 +366,81 @@ async def test_service_active_conversation_suppresses_unread(repo: Repository) -
|
||||
await device.disconnect()
|
||||
|
||||
|
||||
async def test_service_files_channel_message_by_current_slot_occupant(repo: Repository) -> None:
|
||||
"""A channel message is filed under whatever channel is in its slot *now*.
|
||||
|
||||
The wire reports only a slot index, which the firmware assigns from the channel's current
|
||||
position. Reordering the slots (here simulated out of band, without notifying the service)
|
||||
must not misroute later messages: resolution reads the slot fresh, so a message on slot 0
|
||||
lands in whatever channel occupies slot 0 at that moment — never a stale cached identity.
|
||||
"""
|
||||
from meshtools.core.channels import channel_identity
|
||||
|
||||
device = MockDevice()
|
||||
ctx = _StubContext(device, repo)
|
||||
await device.connect()
|
||||
secret_a, secret_b = bytes(range(16)), bytes(range(16, 32))
|
||||
await device.set_channel(0, "Alpha", secret_a)
|
||||
await device.set_channel(1, "Bravo", secret_b)
|
||||
id_a = channel_identity("Alpha", secret_a)
|
||||
id_b = channel_identity("Bravo", secret_b)
|
||||
|
||||
chat = ChatService(ctx)
|
||||
await chat.start() # primes slot 0 -> Alpha, slot 1 -> Bravo
|
||||
try:
|
||||
ctx.events.publish(
|
||||
MeshEvent.message_event(Message(text="from alpha", is_channel=True, channel=0))
|
||||
)
|
||||
await chat._queue.join()
|
||||
|
||||
# Swap the occupants of slots 0 and 1 out of band — the service is never told.
|
||||
await device.set_channel(0, "Bravo", secret_b)
|
||||
await device.set_channel(1, "Alpha", secret_a)
|
||||
|
||||
ctx.events.publish(
|
||||
MeshEvent.message_event(Message(text="from bravo", is_channel=True, channel=0))
|
||||
)
|
||||
await chat._queue.join()
|
||||
|
||||
alpha = repo.recent_chat_messages(is_channel=True, channel_id=id_a)
|
||||
bravo = repo.recent_chat_messages(is_channel=True, channel_id=id_b)
|
||||
assert [m.text for m in alpha] == ["from alpha"] # unaffected by the reorder
|
||||
assert [m.text for m in bravo] == ["from bravo"] # not misfiled under Alpha's identity
|
||||
finally:
|
||||
await chat.stop()
|
||||
await device.disconnect()
|
||||
|
||||
|
||||
async def test_service_records_channel_messages_in_arrival_order(repo: Repository) -> None:
|
||||
"""A burst of channel messages is recorded in arrival order despite async resolution.
|
||||
|
||||
Each channel message resolves its identity with a device read; the serial inbound worker
|
||||
guarantees they still land in the transcript (ordered by insertion) in the order received.
|
||||
"""
|
||||
from meshtools.core.channels import channel_identity
|
||||
|
||||
device = MockDevice()
|
||||
ctx = _StubContext(device, repo)
|
||||
await device.connect()
|
||||
secret = bytes(range(16))
|
||||
await device.set_channel(0, "Alpha", secret)
|
||||
channel_id = channel_identity("Alpha", secret)
|
||||
|
||||
chat = ChatService(ctx)
|
||||
await chat.start()
|
||||
try:
|
||||
for i in range(5):
|
||||
ctx.events.publish(
|
||||
MeshEvent.message_event(Message(text=f"m{i}", is_channel=True, channel=0))
|
||||
)
|
||||
await chat._queue.join()
|
||||
stored = repo.recent_chat_messages(is_channel=True, channel_id=channel_id)
|
||||
assert [m.text for m in stored] == ["m0", "m1", "m2", "m3", "m4"]
|
||||
finally:
|
||||
await chat.stop()
|
||||
await device.disconnect()
|
||||
|
||||
|
||||
# -- chat screen --------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Regression tests for serialized, index-validated channel reads.
|
||||
|
||||
The meshcore library's ``get_channel`` waits for "the next CHANNEL_INFO event" with no
|
||||
correlation to the slot it asked for, and the dispatcher fans that event to every in-flight
|
||||
waiter. Two concurrent reads therefore both resolve on the first response and one caller
|
||||
silently gets the other's channel — which misfiles that channel's messages. :class:`MeshCoreDevice`
|
||||
guards against this by serializing reads and verifying the response is for the slot requested.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from meshtools.core.connection import DeviceCommandError, MeshCoreDevice
|
||||
|
||||
|
||||
class _Event:
|
||||
"""Minimal stand-in for a meshcore CHANNEL_INFO event."""
|
||||
|
||||
def __init__(self, payload: dict) -> None:
|
||||
self.payload = payload
|
||||
|
||||
def is_error(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
class _CrosstalkCommands:
|
||||
"""Fake ``commands`` that reproduces the uncorrelated-response race if reads overlap.
|
||||
|
||||
Each call records itself as in-flight and yields control. If a second read starts while
|
||||
the first is still outstanding, both return the payload of whichever finishes first —
|
||||
exactly the library behavior that misroutes messages. Serialized callers never overlap, so
|
||||
each gets its own channel back.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.in_flight: list[int] = []
|
||||
self.peak_concurrency = 0
|
||||
|
||||
async def get_channel(self, index: int) -> _Event:
|
||||
self.in_flight.append(index)
|
||||
self.peak_concurrency = max(self.peak_concurrency, len(self.in_flight))
|
||||
try:
|
||||
await asyncio.sleep(0) # a real await point where interleaving would happen
|
||||
# Model "the first response resolves everyone": the crossed answer is the
|
||||
# oldest in-flight request, not necessarily this one.
|
||||
served = self.in_flight[0]
|
||||
return _Event(
|
||||
{
|
||||
"channel_idx": served,
|
||||
"channel_name": f"chan{served}",
|
||||
"channel_secret": bytes([served]) * 16,
|
||||
}
|
||||
)
|
||||
finally:
|
||||
self.in_flight.remove(index)
|
||||
|
||||
|
||||
class _MismatchCommands:
|
||||
"""Fake ``commands`` that always answers with the wrong slot index."""
|
||||
|
||||
async def get_channel(self, index: int) -> _Event:
|
||||
wrong = index + 1
|
||||
return _Event(
|
||||
{
|
||||
"channel_idx": wrong,
|
||||
"channel_name": f"chan{wrong}",
|
||||
"channel_secret": bytes([wrong]) * 16,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _device_with(commands: object) -> MeshCoreDevice:
|
||||
device = MeshCoreDevice("COM-TEST")
|
||||
device._mc = type("_MC", (), {"commands": commands})()
|
||||
return device
|
||||
|
||||
|
||||
async def test_get_channel_serializes_concurrent_reads() -> None:
|
||||
"""Concurrent reads never overlap, so each returns its own slot (no cross-talk)."""
|
||||
commands = _CrosstalkCommands()
|
||||
device = _device_with(commands)
|
||||
|
||||
results = await asyncio.gather(*(device.get_channel(i) for i in range(8)))
|
||||
|
||||
assert commands.peak_concurrency == 1
|
||||
for i, payload in enumerate(results):
|
||||
assert payload is not None
|
||||
assert payload["channel_idx"] == i
|
||||
assert payload["channel_name"] == f"chan{i}"
|
||||
|
||||
|
||||
async def test_get_channel_rejects_response_for_a_different_slot() -> None:
|
||||
"""A response whose index doesn't match the request is refused, never returned as-is."""
|
||||
device = _device_with(_MismatchCommands())
|
||||
|
||||
with pytest.raises(DeviceCommandError, match="returned slot"):
|
||||
await device.get_channel(0)
|
||||
Reference in new issue
Block a user