From ca99a5ac3275e90699f8f83376c327059e406cd3 Mon Sep 17 00:00:00 2001 From: San Date: Sat, 7 Nov 2026 14:29:04 -0800 Subject: [PATCH 1/7] conformance(v1): the served gate question records reply after the send tray.sample.json carries the server's gate question as of agentchat-platform/agentchat#152, and gate-prompt.expected.txt is its rendering. Two lines changed on the server: defer states the server-owned ladder (15 min, 24 h, stop) instead of "say when to look again", and the verdict line records reply after the answer is sent instead of "the send is the verdict" (a send marks nothing read). The tool-less triage preamble is unchanged: it already tells the model not to call agentchat_decide and that the host records the verdicts. Byte-identical to agentchat-typescript's conformance/v1. --- conformance/v1/gate-prompt.expected.txt | 4 ++-- conformance/v1/tray.sample.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conformance/v1/gate-prompt.expected.txt b/conformance/v1/gate-prompt.expected.txt index 4fef890..a1896d5 100644 --- a/conformance/v1/gate-prompt.expected.txt +++ b/conformance/v1/gate-prompt.expected.txt @@ -6,11 +6,11 @@ Choose dismiss when the exchange is finished or nothing actually needs you: the Choose reply only when there is a real reason: an open question, request, or task directed at you is unanswered; new information genuinely calls for your input; you started this and the peer’s answer needs a substantive follow-up to reach the goal. -Choose defer when it needs you but not now (waiting on something, batching), and say when to look again. +Choose defer when it needs you but not now (waiting on something, batching). The server owns the reminder: your first defer brings the conversation back in 15 minutes, a second in 24 hours, a third stops the reminders until something new arrives. Decisive bias: once a conversation is winding down, prefer dismiss. A reply must earn its place. Two agents trading acknowledgements forever is the exact failure this question exists to prevent; if the only thing you could add is another acknowledgement, choose dismiss. A high 60-second pace where each message only restates or acknowledges the last IS the loop. On this network silence is a valid reply. -Record dismiss and defer with agentchat_decide (category only, never your reasoning). To reply, open the conversation and send; the send is the verdict. +Record dismiss and defer with agentchat_decide right away (category only, never your reasoning). For reply: open the conversation, send your answer, then record reply with agentchat_decide; a send alone marks nothing read. If after reading you choose not to answer, record dismiss instead. Categories: open_request, new_info, goal_followup, closing, acknowledgement, not_addressed, no_action_needed, spam, other diff --git a/conformance/v1/tray.sample.json b/conformance/v1/tray.sample.json index 43eb9db..99ae6d5 100644 --- a/conformance/v1/tray.sample.json +++ b/conformance/v1/tray.sample.json @@ -139,7 +139,7 @@ } ], "deferred": 1, - "question": "For each conversation on this tray decide ONE thing: reply, dismiss, or defer. You are not writing the reply yet.\n\nChoose dismiss when the exchange is finished or nothing actually needs you: the other side is acknowledging or closing out (thanks, ok, got it, πŸ‘, bye); the last message is a pleasantry or reaction with no question, request, or new information for you; you already answered what was asked and nothing new is on the table; in a group, the message is not addressed to you and does not need you.\n\nChoose reply only when there is a real reason: an open question, request, or task directed at you is unanswered; new information genuinely calls for your input; you started this and the peer’s answer needs a substantive follow-up to reach the goal.\n\nChoose defer when it needs you but not now (waiting on something, batching), and say when to look again.\n\nDecisive bias: once a conversation is winding down, prefer dismiss. A reply must earn its place. Two agents trading acknowledgements forever is the exact failure this question exists to prevent; if the only thing you could add is another acknowledgement, choose dismiss. A high 60-second pace where each message only restates or acknowledges the last IS the loop. On this network silence is a valid reply.\n\nRecord dismiss and defer with agentchat_decide (category only, never your reasoning). To reply, open the conversation and send; the send is the verdict.", + "question": "For each conversation on this tray decide ONE thing: reply, dismiss, or defer. You are not writing the reply yet.\n\nChoose dismiss when the exchange is finished or nothing actually needs you: the other side is acknowledging or closing out (thanks, ok, got it, πŸ‘, bye); the last message is a pleasantry or reaction with no question, request, or new information for you; you already answered what was asked and nothing new is on the table; in a group, the message is not addressed to you and does not need you.\n\nChoose reply only when there is a real reason: an open question, request, or task directed at you is unanswered; new information genuinely calls for your input; you started this and the peer’s answer needs a substantive follow-up to reach the goal.\n\nChoose defer when it needs you but not now (waiting on something, batching). The server owns the reminder: your first defer brings the conversation back in 15 minutes, a second in 24 hours, a third stops the reminders until something new arrives.\n\nDecisive bias: once a conversation is winding down, prefer dismiss. A reply must earn its place. Two agents trading acknowledgements forever is the exact failure this question exists to prevent; if the only thing you could add is another acknowledgement, choose dismiss. A high 60-second pace where each message only restates or acknowledges the last IS the loop. On this network silence is a valid reply.\n\nRecord dismiss and defer with agentchat_decide right away (category only, never your reasoning). For reply: open the conversation, send your answer, then record reply with agentchat_decide; a send alone marks nothing read. If after reading you choose not to answer, record dismiss instead.", "truncated": false, "waited_ms": null, "you": "@relay-bot" From bedbe9b14a69558c257c74262bb2fc16d8ee0905 Mon Sep 17 00:00:00 2001 From: San Date: Sat, 7 Nov 2026 14:30:18 -0800 Subject: [PATCH 2/7] fix(realtime): ack an already-handled repeat below the seq anchor; engine conformance scenarios Default mode dropped a live message.new below the conversation's seq anchor without an ack, even when this session had already dispatched that message id. The delivery-ack protocol acknowledges every dedup hit (the earlier dispatch is the proof), and the TypeScript SDK does. It is now acked again, still without re-dispatch. A below-anchor frame no handler has seen is unchanged: dropped unacked, the server re-offers it. Adds conformance/v1/engine-scenarios.json, run in both modes by a new harness (identical bytes ship in the TypeScript SDK): drain gap + close during an earlier handler or inside the gap window, a raising handler during a drain, and an already-handled repeat below the live anchor as a frame and as a drain row. Also pins the drain-cursor rule for a socket drop during an earlier handler with a regression test. --- CHANGELOG.md | 21 ++ conformance/v1/engine-scenarios.json | 438 +++++++++++++++++++++++++++ src/agentchatme/_realtime.py | 13 +- tests/test_conformance.py | 12 +- tests/test_conformance_engine.py | 229 ++++++++++++++ tests/test_realtime.py | 48 +++ tests/test_realtime_engine.py | 45 +++ 7 files changed, 801 insertions(+), 5 deletions(-) create mode 100644 conformance/v1/engine-scenarios.json create mode 100644 tests/test_conformance_engine.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d9c10dd..ecdb057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,27 @@ major. The on-the-wire API is versioned separately under `/v1/...`. ## [Unreleased] +### Fixed + +- In the default mode, a live `message.new` below the conversation's seq + anchor that this session had already handled (the server sent it again: + a drain/live overlap or a lost ack) was dropped without an ack. The + delivery-ack protocol acknowledges every dedup hit, because the earlier + dispatch is the proof; the frame is now acked again, still without + re-dispatching it. A below-anchor frame no handler has seen is unchanged: + dropped unacked, so the server re-offers it. Matches the TypeScript SDK. + +### Added + +- `conformance/v1/engine-scenarios.json` β€” the realtime engine's delivery + rules as shared scenarios, run in both modes by both SDKs: a drain page + with a gap and a socket drop during an earlier handler or inside the gap + window, a handler that raises during a drain, and a repeat of an + already-handled message below the live anchor (as a frame and as a drain + row). Invariants checked on every scenario: nothing is acknowledged that a + handler did not finish, no message is dispatched twice, seqs stay in order + per conversation. Identical bytes ship in the TypeScript SDK. + ### Changed - Docs: a `reply` verdict's `record()` is called at the **end of a turn that diff --git a/conformance/v1/engine-scenarios.json b/conformance/v1/engine-scenarios.json new file mode 100644 index 0000000..3888756 --- /dev/null +++ b/conformance/v1/engine-scenarios.json @@ -0,0 +1,438 @@ +{ + "about": "Realtime engine scenarios every AgentChat SDK runs against its own RealtimeClient: a mock socket, a fake REST client and one recording message.new handler. Each scenario runs twice, once per mode: 'default' (ack after every handler finished) and 'ack_on_receipt'. The REST client serves sync_pages in order, then empty pages; sync/ack records the cursor; gap-fill returns nothing. The automatic drain on hello.ok is off and reconnect is off, so only the steps below act. A message is 'handled' when its handler finished without raising; a sync/ack cursor 'covers' every row served so far up to and including the row carrying that delivery_id. Results are read after the last step and any started drain finished, before the client is disconnected (disconnect flushes parked frames).", + "expectations": { + "cursor_acks": "Exactly these delivery_ids were passed to sync/ack, in order.", + "dispatched": "Exactly these message ids reached the handler, in this order.", + "dispatched_includes": "These message ids reached the handler (others may too).", + "ws_acks": "Exactly these message ids were sent in ack frames, in order." + }, + "invariants": { + "acked_was_handled": "At the end, every message the client acknowledged (an ack frame, or a row a sync/ack cursor covers) was handled. An ack is a promise the server never re-offers the message.", + "acks_follow_handling": "When each sync/ack cursor is sent (both modes), and each ack frame in the default mode, every message it acknowledges was already handled. Under ack_on_receipt an ack frame is sent at receipt, before the handler.", + "dispatched_once": "No message id reaches the handler twice in these scenarios.", + "seq_order": "Within one conversation the handler sees seqs in ascending order." + }, + "ops": { + "close": "The server side ends the socket with close code 1006 (an abnormal drop).", + "connect": "Create the client for the mode, connect, open the socket and answer HELLO with {\"type\": \"hello.ok\", \"capabilities\": [\"ack\"]}.", + "drain": "Start the client's offline drain (drainOfflineEnvelopes / drain_offline_envelopes) without awaiting it.", + "frame": "The server sends {\"type\": \"message.new\", \"payload\": } on the socket.", + "release": "Let the held handler for message_id finish.", + "wait_handler": "Wait until the handler has been entered for message_id." + }, + "rules": "Every op is followed by letting the client run until it is idle (pending handlers that are held stay pending). The handler blocks on the ids in 'hold' until a 'release' step, and raises for the ids in 'fail'. A harness rejects an op, expectation or invariant name it does not implement.", + "scenarios": [ + { + "expect": { + "ack_on_receipt": { + "cursor_acks": [ + "del_000000000000000000000000000c0003" + ], + "dispatched": [ + "msg_crew0seq0001", + "msg_crew0seq0003" + ], + "ws_acks": [] + }, + "default": { + "dispatched_includes": [ + "msg_crew0seq0001" + ], + "ws_acks": [] + } + }, + "hold": [ + "msg_crew0seq0001" + ], + "name": "drain-gap-close-during-earlier-handler", + "steps": [ + { + "op": "connect" + }, + { + "op": "drain" + }, + { + "message_id": "msg_crew0seq0001", + "op": "wait_handler" + }, + { + "op": "close" + }, + { + "message_id": "msg_crew0seq0001", + "op": "release" + } + ], + "sync_pages": [ + [ + { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0crew0001", + "created_at": "2026-11-04T23:01:00.000Z", + "delivery_id": "del_000000000000000000000000000c0001", + "id": "msg_crew0seq0001", + "sender": "nova", + "seq": 1, + "type": "text" + }, + { + "content": { + "text": "message 3" + }, + "conversation_id": "conv_group0crew0001", + "created_at": "2026-11-04T23:03:00.000Z", + "delivery_id": "del_000000000000000000000000000c0003", + "id": "msg_crew0seq0003", + "sender": "nova", + "seq": 3, + "type": "text" + } + ] + ], + "why": "A group where the agent's own message is seq 2, so the drain page is [1, 3] and seq 3 waits on the gap. The handler for seq 1 is slow (it calls a model) and the socket drops while it runs. Whatever the engine does with seq 3, the cursor may cover it only if a handler finished it: a close that clears a parked row is not a dispatch." + }, + { + "expect": { + "ack_on_receipt": { + "cursor_acks": [ + "del_000000000000000000000000000a0001" + ], + "dispatched": [ + "msg_ops0seq00001", + "msg_ops0seq00003" + ], + "ws_acks": [] + }, + "default": { + "cursor_acks": [ + "del_000000000000000000000000000a0001" + ], + "dispatched": [ + "msg_ops0seq00001" + ], + "ws_acks": [] + } + }, + "name": "drain-gap-close-while-parked", + "steps": [ + { + "op": "connect" + }, + { + "op": "drain" + }, + { + "op": "close" + } + ], + "sync_pages": [ + [ + { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:01:00.000Z", + "delivery_id": "del_000000000000000000000000000a0001", + "id": "msg_ops0seq00001", + "sender": "nova", + "seq": 1, + "type": "text" + }, + { + "content": { + "text": "message 3" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:03:00.000Z", + "delivery_id": "del_000000000000000000000000000a0003", + "id": "msg_ops0seq00003", + "sender": "nova", + "seq": 3, + "type": "text" + } + ] + ], + "why": "Drain page [1, 3] with seq 2 absent; seq 3 parks on the gap and the socket drops inside the gap window. The cursor stops at seq 1. The default mode drops the parked row unacked (the server re-offers it); ack_on_receipt hands parked frames to the handler at close." + }, + { + "expect": { + "ack_on_receipt": { + "cursor_acks": [ + "del_000000000000000000000000000e0001" + ], + "dispatched": [ + "msg_gone0seq0001", + "msg_gone0seq0002", + "msg_gone0seq0003" + ], + "ws_acks": [] + }, + "default": { + "cursor_acks": [ + "del_000000000000000000000000000e0001" + ], + "dispatched": [ + "msg_gone0seq0001", + "msg_gone0seq0002", + "msg_gone0seq0003" + ], + "ws_acks": [] + } + }, + "fail": [ + "msg_gone0seq0002" + ], + "name": "drain-handler-raises", + "steps": [ + { + "op": "connect" + }, + { + "op": "drain" + } + ], + "sync_pages": [ + [ + { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0gone0001", + "created_at": "2026-11-04T23:01:00.000Z", + "delivery_id": "del_000000000000000000000000000e0001", + "id": "msg_gone0seq0001", + "sender": "nova", + "seq": 1, + "type": "text" + }, + { + "content": { + "text": "message 2" + }, + "conversation_id": "conv_group0gone0001", + "created_at": "2026-11-04T23:02:00.000Z", + "delivery_id": "del_000000000000000000000000000e0002", + "id": "msg_gone0seq0002", + "sender": "nova", + "seq": 2, + "type": "text" + }, + { + "content": { + "text": "message 3" + }, + "conversation_id": "conv_group0gone0001", + "created_at": "2026-11-04T23:03:00.000Z", + "delivery_id": "del_000000000000000000000000000e0003", + "id": "msg_gone0seq0003", + "sender": "nova", + "seq": 3, + "type": "text" + } + ] + ], + "why": "The handler raises on the second drain row. The cursor stops before it in both modes, so the server re-offers that row and everything after it." + }, + { + "expect": { + "ack_on_receipt": { + "cursor_acks": [], + "dispatched": [ + "msg_first0seq001", + "msg_first0seq002", + "msg_first0seq003" + ], + "ws_acks": [ + "msg_first0seq001", + "msg_first0seq003", + "msg_first0seq001", + "msg_first0seq002" + ] + }, + "default": { + "cursor_acks": [], + "dispatched": [ + "msg_first0seq001", + "msg_first0seq002", + "msg_first0seq003" + ], + "ws_acks": [ + "msg_first0seq001", + "msg_first0seq001", + "msg_first0seq002", + "msg_first0seq003" + ] + } + }, + "name": "below-anchor-duplicate-frame", + "steps": [ + { + "op": "connect" + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_direct0first01", + "created_at": "2026-11-04T23:01:00.000Z", + "id": "msg_first0seq001", + "sender": "nova", + "seq": 1, + "type": "text" + } + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 3" + }, + "conversation_id": "conv_direct0first01", + "created_at": "2026-11-04T23:03:00.000Z", + "id": "msg_first0seq003", + "sender": "nova", + "seq": 3, + "type": "text" + } + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_direct0first01", + "created_at": "2026-11-04T23:01:00.000Z", + "id": "msg_first0seq001", + "sender": "nova", + "seq": 1, + "type": "text" + } + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 2" + }, + "conversation_id": "conv_direct0first01", + "created_at": "2026-11-04T23:02:00.000Z", + "id": "msg_first0seq002", + "sender": "nova", + "seq": 2, + "type": "text" + } + } + ], + "why": "Seq 1 anchors the conversation and seq 3 parks on the gap, so the ordering state is live. The server then sends seq 1 again (a drain/live overlap or a lost ack): it is below the anchor and this client already handled it, so it is not handed over again, and it is acknowledged again, because the earlier dispatch is the proof. Seq 2 then releases 2 and 3 in order." + }, + { + "expect": { + "ack_on_receipt": { + "cursor_acks": [ + "del_000000000000000000000000000b0001" + ], + "dispatched": [ + "msg_ops0seq00001", + "msg_ops0seq00002", + "msg_ops0seq00003" + ], + "ws_acks": [ + "msg_ops0seq00001", + "msg_ops0seq00003", + "msg_ops0seq00002" + ] + }, + "default": { + "cursor_acks": [ + "del_000000000000000000000000000b0001" + ], + "dispatched": [ + "msg_ops0seq00001", + "msg_ops0seq00002", + "msg_ops0seq00003" + ], + "ws_acks": [ + "msg_ops0seq00001", + "msg_ops0seq00002", + "msg_ops0seq00003" + ] + } + }, + "name": "below-anchor-duplicate-drain-row", + "steps": [ + { + "op": "connect" + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:01:00.000Z", + "id": "msg_ops0seq00001", + "sender": "nova", + "seq": 1, + "type": "text" + } + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 3" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:03:00.000Z", + "id": "msg_ops0seq00003", + "sender": "nova", + "seq": 3, + "type": "text" + } + }, + { + "op": "drain" + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 2" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:02:00.000Z", + "id": "msg_ops0seq00002", + "sender": "nova", + "seq": 2, + "type": "text" + } + } + ], + "sync_pages": [ + [ + { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:01:00.000Z", + "delivery_id": "del_000000000000000000000000000b0001", + "id": "msg_ops0seq00001", + "sender": "nova", + "seq": 1, + "type": "text" + } + ] + ], + "why": "As above, but the repeat of seq 1 is a REST drain row. It is not handed over again and the cursor covers it: this client already handled that message id. Drain rows are acknowledged by the cursor, never by an ack frame." + } + ] +} diff --git a/src/agentchatme/_realtime.py b/src/agentchatme/_realtime.py index 48d49ea..7f8d549 100644 --- a/src/agentchatme/_realtime.py +++ b/src/agentchatme/_realtime.py @@ -1672,7 +1672,8 @@ async def _process_ordered_message( Returns where the envelope went: ``"delivered"`` (handed to :meth:`_deliver_message_new` now), ``"duplicate"`` (below this - connection's seq anchor, dropped), or ``"buffered"`` (parked on a + connection's seq anchor, dropped; a live frame this session already + dispatched is acknowledged again), or ``"buffered"`` (parked on a seq gap; dispatched later by the gap machinery or the shutdown flush). The REST drain uses this to decide what its ack cursor may cover. @@ -1710,10 +1711,16 @@ async def _process_ordered_message( # rows (seq 3, 4). Unacked, those come back in order later and # are safe to drop here; acked at parse, dropping them would lose # them β€” deliver on a dedup miss, out of order but not lost. + message_id = _extract_message_id(message) + seen = message_id is not None and self._dedup.hit(message_id) if not ack_spent: + if seen and ws_ack and message_id is not None: + # A repeat of a message this session already handled: + # the earlier dispatch is the proof, so acknowledge it + # again, as the dedup path does for every other repeat. + await self._send_delivery_ack(message_id) return "duplicate" - message_id = _extract_message_id(message) - if message_id is not None and self._dedup.hit(message_id): + if seen: return "duplicate" await self._deliver_message_new(message, ws_ack=False) return "delivered" diff --git a/tests/test_conformance.py b/tests/test_conformance.py index e580edf..99bfd4b 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -21,6 +21,7 @@ FIXTURES = Path(__file__).resolve().parent.parent / "conformance" / "v1" NAMES = [ "ack-frame.json", + "engine-scenarios.json", "gate-prompt.expected.txt", "tray.sample.json", "verdict-parse.cases.json", @@ -45,12 +46,19 @@ def tray() -> dict[str, Any]: # ─────────────── Canonical form ─────────────── -def test_ships_exactly_the_five_named_files() -> None: +def test_ships_exactly_the_six_named_files() -> None: assert sorted(p.name for p in FIXTURES.iterdir()) == NAMES @pytest.mark.parametrize( - "name", ["tray.sample.json", "verdict-parse.cases.json", "wake-frame.json", "ack-frame.json"] + "name", + [ + "tray.sample.json", + "verdict-parse.cases.json", + "wake-frame.json", + "ack-frame.json", + "engine-scenarios.json", + ], ) def test_json_fixture_has_sorted_keys_two_space_indent_and_one_trailing_newline( name: str, diff --git a/tests/test_conformance_engine.py b/tests/test_conformance_engine.py new file mode 100644 index 0000000..798c167 --- /dev/null +++ b/tests/test_conformance_engine.py @@ -0,0 +1,229 @@ +"""``conformance/v1/engine-scenarios.json``: the realtime engine's delivery +rules (drain cursor, parked frames across a close, below-anchor repeats) as +scenarios both SDKs run byte for byte. + +The fixture holds the steps and the expected outcomes; this file only +interprets them. See the fixture's ``about``, ``ops`` and ``invariants``. +""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import pytest + +from agentchatme import RealtimeClient + +from .test_realtime import MockWebSocket, _make_client + +FIXTURE = Path(__file__).resolve().parent.parent / "conformance" / "v1" / "engine-scenarios.json" +DOC: dict[str, Any] = json.loads(FIXTURE.read_text(encoding="utf-8")) + +MODES = ["default", "ack_on_receipt"] +OPS = ["close", "connect", "drain", "frame", "release", "wait_handler"] +EXPECTATIONS = ["cursor_acks", "dispatched", "dispatched_includes", "ws_acks"] +INVARIANTS = ["acked_was_handled", "acks_follow_handling", "dispatched_once", "seq_order"] + + +async def _idle() -> None: + """Let the client run until nothing is ready to run (held handlers stay + pending): the drain task, the receive loop and their acks.""" + for _ in range(50): + await asyncio.sleep(0) + + +class _RecordingSocket(MockWebSocket): + def __init__(self, events: list[tuple[str, str]]) -> None: + super().__init__() + self._events = events + + async def send(self, data: str) -> None: + frame = json.loads(data) + if frame.get("type") == "ack": + self._events.append(("ws_ack", str(frame.get("message_id")))) + await super().send(data) + + +class _ScenarioApi: + """The REST surface the engine uses: ``sync`` serves the scenario's pages + in order and then empty pages, ``sync_ack`` records the cursor, gap-fill + returns nothing.""" + + def __init__(self, pages: list[list[dict[str, Any]]], events: list[tuple[str, str]]) -> None: + self._pages = [list(page) for page in pages] + self._events = events + self.served: list[dict[str, Any]] = [] + + async def sync(self, **_kwargs: Any) -> list[dict[str, Any]]: + page = self._pages.pop(0) if self._pages else [] + self.served.extend(page) + return page + + async def sync_ack(self, last_delivery_id: str, **_kwargs: Any) -> dict[str, Any]: + self._events.append(("cursor", last_delivery_id)) + return {"acked": 1} + + async def get_messages(self, _conversation_id: str, **_kwargs: Any) -> list[dict[str, Any]]: + return [] + + +@dataclass +class _Outcome: + dispatched: list[dict[str, Any]] = field(default_factory=list) + # ("handled", message_id) | ("ws_ack", message_id) | ("cursor", delivery_id) + events: list[tuple[str, str]] = field(default_factory=list) + served: list[dict[str, Any]] = field(default_factory=list) + + +async def _run(scenario: dict[str, Any], mode: str) -> _Outcome: + out = _Outcome() + api = _ScenarioApi(scenario.get("sync_pages", []), out.events) + holds = {mid: asyncio.Event() for mid in scenario.get("hold", [])} + failing = set(scenario.get("fail", [])) + + async def handler(message: dict[str, Any]) -> None: + payload = message["payload"] + out.dispatched.append(payload) + hold = holds.get(payload["id"]) + if hold is not None: + await hold.wait() + if payload["id"] in failing: + raise RuntimeError(f"scenario handler fails {payload['id']}") + out.events.append(("handled", payload["id"])) + + rt: RealtimeClient | None = None + ws: _RecordingSocket | None = None + drains: list[asyncio.Task[None]] = [] + try: + for step in scenario["steps"]: + op = step["op"] + if op == "connect": + ws = _RecordingSocket(out.events) + rt, _ = _make_client( + ws=ws, + client=api, # type: ignore[arg-type] + auto_drain_on_connect=False, + ack_on_receipt=mode == "ack_on_receipt", + ) + rt.on("message.new", handler) + await rt.connect() + await _idle() + await ws.push({"type": "hello.ok", "capabilities": ["ack"]}) + elif op == "frame": + assert ws is not None + await ws.push({"type": "message.new", "payload": step["payload"]}) + elif op == "drain": + assert rt is not None + drains.append(asyncio.create_task(rt.drain_offline_envelopes())) + elif op == "wait_handler": + for _ in range(100): + if any(p["id"] == step["message_id"] for p in out.dispatched): + break + await asyncio.sleep(0.001) + assert any(p["id"] == step["message_id"] for p in out.dispatched), ( + f"handler entered for {step['message_id']}" + ) + elif op == "close": + assert ws is not None + await ws.close(code=1006, reason="abnormal") + elif op == "release": + holds[step["message_id"]].set() + else: + raise AssertionError(f"unknown op {op}") + await _idle() + if drains: + await asyncio.wait_for(asyncio.gather(*drains), timeout=2.0) + await _idle() + out.served = list(api.served) + return _Outcome(list(out.dispatched), list(out.events), out.served) + finally: + for hold in holds.values(): + hold.set() + if rt is not None: + await rt.disconnect() + + +def _acknowledged(event: tuple[str, str], served: list[dict[str, Any]]) -> list[str]: + """Message ids an event acknowledges: an ack frame names one; a cursor + covers every row served so far up to and including the row carrying it.""" + kind, value = event + if kind == "ws_ack": + return [value] + if kind != "cursor": + return [] + index = next((i for i, row in enumerate(served) if row.get("delivery_id") == value), None) + assert index is not None, f"cursor {value} names a served row" + return [row["id"] for row in served[: index + 1]] + + +def _check_invariants(out: _Outcome, mode: str) -> None: + handled_at: dict[str, int] = {} + for at, (kind, value) in enumerate(out.events): + if kind == "handled": + handled_at.setdefault(value, at) + + for at, event in enumerate(out.events): + for mid in _acknowledged(event, out.served): + # acked_was_handled + assert mid in handled_at, f"{mid} acknowledged but never handled" + # acks_follow_handling + if event[0] == "cursor" or mode == "default": + assert handled_at[mid] < at, f"{mid} acknowledged before it was handled" + + # dispatched_once + ids = [p["id"] for p in out.dispatched] + assert len(set(ids)) == len(ids), f"a message reached the handler twice: {ids}" + + # seq_order + last: dict[str, int] = {} + for p in out.dispatched: + previous = last.get(p["conversation_id"]) + if previous is not None: + assert p["seq"] > previous, f"{p['conversation_id']}: seq {p['seq']} after {previous}" + last[p["conversation_id"]] = p["seq"] + + +def _check_expectations(out: _Outcome, expected: dict[str, Any]) -> None: + ids = [p["id"] for p in out.dispatched] + if "dispatched" in expected: + assert ids == expected["dispatched"] + for mid in expected.get("dispatched_includes", []): + assert mid in ids + if "ws_acks" in expected: + assert [v for k, v in out.events if k == "ws_ack"] == expected["ws_acks"] + if "cursor_acks" in expected: + assert [v for k, v in out.events if k == "cursor"] == expected["cursor_acks"] + + +def test_names_only_ops_expectations_and_invariants_this_harness_implements() -> None: + assert sorted(DOC["ops"]) == OPS + assert sorted(DOC["expectations"]) == EXPECTATIONS + assert sorted(DOC["invariants"]) == INVARIANTS + for scenario in DOC["scenarios"]: + for step in scenario["steps"]: + assert step["op"] in OPS + assert sorted(scenario["expect"]) == sorted(MODES) + for mode in MODES: + for key in scenario["expect"][mode]: + assert key in EXPECTATIONS + + +def test_scenario_names_are_unique() -> None: + names = [s["name"] for s in DOC["scenarios"]] + assert len(set(names)) == len(names) + + +@pytest.mark.parametrize( + ("scenario", "mode"), + [(s, m) for s in DOC["scenarios"] for m in MODES], + ids=[f"{s['name']}-{m}" for s in DOC["scenarios"] for m in MODES], +) +@pytest.mark.asyncio +async def test_engine_scenario(scenario: dict[str, Any], mode: str) -> None: + out = await _run(scenario, mode) + _check_invariants(out, mode) + _check_expectations(out, scenario["expect"][mode]) diff --git a/tests/test_realtime.py b/tests/test_realtime.py index 95f835c..47f4b35 100644 --- a/tests/test_realtime.py +++ b/tests/test_realtime.py @@ -1299,6 +1299,54 @@ async def test_drain_never_acks_a_below_anchor_row_nobody_dispatched() -> None: await rt.disconnect() +@pytest.mark.parametrize("ack_on_receipt", [False, True]) +@pytest.mark.asyncio +async def test_drain_cursor_covers_only_handled_rows_when_the_socket_drops_mid_handler( + ack_on_receipt: bool, +) -> None: + # A group where the agent's own message is seq 2: the page is [1, 3]. + # The handler for seq 1 is slow and the socket drops while it runs. The + # close resets the ordering state; whatever happens to seq 3 afterwards, + # the cursor may cover it only once a handler finished it. + page = [ + _sync_row("m_1", conversation_id="g", delivery_id="del_1", seq=1), + _sync_row("m_3", conversation_id="g", delivery_id="del_3", seq=3), + ] + mock_api = MockAsyncClient(sync_pages=[page]) + rt, ws = _make_client( + client=mock_api, auto_drain_on_connect=False, ack_on_receipt=ack_on_receipt + ) + entered: list[str] = [] + finished: list[str] = [] + release = asyncio.Event() + + async def slow(m: dict[str, Any]) -> None: + entered.append(m["payload"]["id"]) + if m["payload"]["id"] == "m_1": + await release.wait() + finished.append(m["payload"]["id"]) + + rt.on("message.new", slow) + try: + await rt.connect() + await _settle() + await ws.push({"type": "hello.ok", "capabilities": ["ack"]}) + await _settle() + drain = asyncio.create_task(rt.drain_offline_envelopes()) + await _settle() + assert entered == ["m_1"] + await ws.close(code=1006, reason="abnormal") + await _settle() + release.set() + await asyncio.wait_for(drain, timeout=1.0) + + assert mock_api.sync_ack_calls, "the cursor covers at least the handled seq 1" + covered = {"del_1": ["m_1"], "del_3": ["m_1", "m_3"]}[mock_api.sync_ack_calls[-1]] + assert all(mid in finished for mid in covered) + finally: + await rt.disconnect() + + @pytest.mark.asyncio async def test_drain_dispatches_and_acks_nothing_after_disconnect() -> None: gate = asyncio.Event() diff --git a/tests/test_realtime_engine.py b/tests/test_realtime_engine.py index 0ec00ca..492aec0 100644 --- a/tests/test_realtime_engine.py +++ b/tests/test_realtime_engine.py @@ -295,6 +295,51 @@ async def test_ack_on_receipt_flushes_acked_parked_frames_when_the_socket_closes await rt.disconnect() +@pytest.mark.asyncio +async def test_default_mode_acks_a_below_anchor_repeat_it_already_handled() -> None: + # Seq 3 parks, so the ordering state stays live and the repeat of seq 1 + # really is below the anchor. It is not handed over again, and it is + # acknowledged again: the earlier dispatch is the proof (the dedup rule + # of the delivery-ack protocol). + rt, ws = await connected(capabilities=["ack"]) + handled: list[str] = [] + rt.on("message.new", lambda m: handled.append(m["payload"]["id"])) + try: + await ws.push(message_new("c", 1)) + await ws.push(message_new("c", 3)) + await _settle() + await ws.push(message_new("c", 1)) + await _settle() + assert handled == ["msg_1"] + assert _ack_frames(ws) == ["msg_1", "msg_1"] + + await ws.push(message_new("c", 2)) + await _settle() + assert handled == ["msg_1", "msg_2", "msg_3"] + assert _ack_frames(ws) == ["msg_1", "msg_1", "msg_2", "msg_3"] + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_default_mode_never_acks_a_below_anchor_frame_nobody_handled() -> None: + # Live seq 5 anchors before the server's reconnect backlog brings the + # stored seq 3. Unseen and unacked, it stays `stored` for the next drain. + rt, ws = await connected(capabilities=["ack"]) + handled: list[str] = [] + rt.on("message.new", lambda m: handled.append(m["payload"]["id"])) + try: + await ws.push(message_new("c", 5)) + await ws.push(message_new("c", 7)) + await _settle() + await ws.push(message_new("c", 3)) + await _settle() + assert handled == ["msg_5"] + assert _ack_frames(ws) == ["msg_5"] + finally: + await rt.disconnect() + + @pytest.mark.asyncio async def test_default_mode_leaves_unacked_parked_frames_to_the_next_drain() -> None: gaps: list[Any] = [] From bf24ad306402494717fef0902dbfac19d486d925 Mon Sep 17 00:00:00 2001 From: San Date: Sat, 7 Nov 2026 14:52:22 -0800 Subject: [PATCH 3/7] test(conformance): drain gap with a socket drop during a later row's handler Shared scenario (identical bytes in agentchat-typescript): page [g1, g3, h1], h1's handler held, the socket drops, h1 released; the default-mode cursor stops at g1. This drain awaits each handler while routing, so the drop lands during a LATER row's handler and clears the parked g3 before the cursor is computed. Copying a cursor that stops only while a row is still buffered into _drain_row_ackable now fails this scenario and the reworked unit test (the earlier-handler shape did not catch it). CHANGELOG: say where the SDKs still differ on a below-anchor frame no handler has seen. --- CHANGELOG.md | 10 ++-- conformance/v1/engine-scenarios.json | 86 ++++++++++++++++++++++++++++ tests/test_realtime.py | 29 +++++----- 3 files changed, 108 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecdb057..6815039 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,15 +14,17 @@ major. The on-the-wire API is versioned separately under `/v1/...`. a drain/live overlap or a lost ack) was dropped without an ack. The delivery-ack protocol acknowledges every dedup hit, because the earlier dispatch is the proof; the frame is now acked again, still without - re-dispatching it. A below-anchor frame no handler has seen is unchanged: - dropped unacked, so the server re-offers it. Matches the TypeScript SDK. + re-dispatching it, as the TypeScript SDK does. A below-anchor frame no + handler has seen is unchanged, and there the SDKs still differ: Python + drops it unacked, so it waits for the server to re-offer it on the next + reconnect; TypeScript dispatches it late (out of order) and acks it. ### Added - `conformance/v1/engine-scenarios.json` β€” the realtime engine's delivery rules as shared scenarios, run in both modes by both SDKs: a drain page - with a gap and a socket drop during an earlier handler or inside the gap - window, a handler that raises during a drain, and a repeat of an + with a gap and a socket drop during an earlier or a later row's handler + or inside the gap window, a handler that raises during a drain, and a repeat of an already-handled message below the live anchor (as a frame and as a drain row). Invariants checked on every scenario: nothing is acknowledged that a handler did not finish, no message is dispatched twice, seqs stay in order diff --git a/conformance/v1/engine-scenarios.json b/conformance/v1/engine-scenarios.json index 3888756..fda9539 100644 --- a/conformance/v1/engine-scenarios.json +++ b/conformance/v1/engine-scenarios.json @@ -94,6 +94,92 @@ ], "why": "A group where the agent's own message is seq 2, so the drain page is [1, 3] and seq 3 waits on the gap. The handler for seq 1 is slow (it calls a model) and the socket drops while it runs. Whatever the engine does with seq 3, the cursor may cover it only if a handler finished it: a close that clears a parked row is not a dispatch." }, + { + "expect": { + "ack_on_receipt": { + "dispatched": [ + "msg_crew0late0001", + "msg_first0late001", + "msg_crew0late0003" + ], + "ws_acks": [] + }, + "default": { + "cursor_acks": [ + "del_000000000000000000000000000f0001" + ], + "dispatched": [ + "msg_crew0late0001", + "msg_first0late001" + ], + "ws_acks": [] + } + }, + "hold": [ + "msg_first0late001" + ], + "name": "drain-gap-close-during-later-handler", + "steps": [ + { + "op": "connect" + }, + { + "op": "drain" + }, + { + "message_id": "msg_first0late001", + "op": "wait_handler" + }, + { + "op": "close" + }, + { + "message_id": "msg_first0late001", + "op": "release" + } + ], + "sync_pages": [ + [ + { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0crew0001", + "created_at": "2026-11-04T23:01:00.000Z", + "delivery_id": "del_000000000000000000000000000f0001", + "id": "msg_crew0late0001", + "sender": "nova", + "seq": 1, + "type": "text" + }, + { + "content": { + "text": "message 3" + }, + "conversation_id": "conv_group0crew0001", + "created_at": "2026-11-04T23:03:00.000Z", + "delivery_id": "del_000000000000000000000000000f0003", + "id": "msg_crew0late0003", + "sender": "nova", + "seq": 3, + "type": "text" + }, + { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_direct0first01", + "created_at": "2026-11-04T23:01:00.000Z", + "delivery_id": "del_000000000000000000000000000f0004", + "id": "msg_first0late001", + "sender": "nova", + "seq": 1, + "type": "text" + } + ] + ], + "why": "Drain page [g1, g3, h1]: seq 3 of the group waits on the gap (seq 2 is the agent's own message) and the handler for the direct message h1, a later row, is slow. The socket drops while it runs, which clears the parked g3 without dispatching it. The cursor stops at g1: an SDK that routes rows one at a time sees g3 cleared, not parked, by the time it computes the cursor." + }, { "expect": { "ack_on_receipt": { diff --git a/tests/test_realtime.py b/tests/test_realtime.py index 47f4b35..c231149 100644 --- a/tests/test_realtime.py +++ b/tests/test_realtime.py @@ -1304,25 +1304,25 @@ async def test_drain_never_acks_a_below_anchor_row_nobody_dispatched() -> None: async def test_drain_cursor_covers_only_handled_rows_when_the_socket_drops_mid_handler( ack_on_receipt: bool, ) -> None: - # A group where the agent's own message is seq 2: the page is [1, 3]. - # The handler for seq 1 is slow and the socket drops while it runs. The - # close resets the ordering state; whatever happens to seq 3 afterwards, - # the cursor may cover it only once a handler finished it. + # Page [g1, g3, h1]: g3 parks on the gap (the agent's own message is g's + # seq 2) and h1's handler, a LATER row, is slow. This drain routes rows + # one at a time, so the socket drop during h1 clears the parked g3 before + # the cursor is computed: g3 is then neither parked nor dispatched, and + # the cursor must stop before it unless a handler finished it. page = [ - _sync_row("m_1", conversation_id="g", delivery_id="del_1", seq=1), - _sync_row("m_3", conversation_id="g", delivery_id="del_3", seq=3), + _sync_row("g_1", conversation_id="g", delivery_id="del_g1", seq=1), + _sync_row("g_3", conversation_id="g", delivery_id="del_g3", seq=3), + _sync_row("h_1", conversation_id="h", delivery_id="del_h1", seq=1), ] mock_api = MockAsyncClient(sync_pages=[page]) rt, ws = _make_client( client=mock_api, auto_drain_on_connect=False, ack_on_receipt=ack_on_receipt ) - entered: list[str] = [] finished: list[str] = [] release = asyncio.Event() async def slow(m: dict[str, Any]) -> None: - entered.append(m["payload"]["id"]) - if m["payload"]["id"] == "m_1": + if m["payload"]["id"] == "h_1": await release.wait() finished.append(m["payload"]["id"]) @@ -1334,15 +1334,18 @@ async def slow(m: dict[str, Any]) -> None: await _settle() drain = asyncio.create_task(rt.drain_offline_envelopes()) await _settle() - assert entered == ["m_1"] await ws.close(code=1006, reason="abnormal") await _settle() release.set() await asyncio.wait_for(drain, timeout=1.0) - assert mock_api.sync_ack_calls, "the cursor covers at least the handled seq 1" - covered = {"del_1": ["m_1"], "del_3": ["m_1", "m_3"]}[mock_api.sync_ack_calls[-1]] - assert all(mid in finished for mid in covered) + if ack_on_receipt: + # The close handed the parked g3 to the handler first. + assert sorted(finished) == ["g_1", "g_3", "h_1"] + assert mock_api.sync_ack_calls == ["del_h1"] + else: + assert finished == ["g_1", "h_1"] + assert mock_api.sync_ack_calls == ["del_g1"] finally: await rt.disconnect() From 3be8f044b28f07acce88d65a7f3d68059fa3d833 Mon Sep 17 00:00:00 2001 From: San Date: Sat, 7 Nov 2026 15:00:52 -0800 Subject: [PATCH 4/7] fix(realtime): never hand a message to the handlers twice while its first copy is running After a reconnect the server pushes its stored backlog on the socket and the REST drain brings the same rows. The dedup cache is filled only once a handler succeeds, so with a slow handler the second copy was dispatched again (e.g. an in-order copy whose cursor advances only after the first copy's handler returned). _deliver_message_new now keeps a future per message id in flight; a copy that finds one is not dispatched and takes the outcome (shielded; a cancelled first copy counts as not handled). A copy waits only where it waited on its own handler call before; a live repeat below the anchor, which used to be dropped, never holds up the receive loop and is acked once the first copy finished. Off with dedup_cache_size=0. Also closes two races between concurrent copies: the per-conversation cursor never moves backwards (_advance_past), and _cleanup_if_idle removes only the state it was handed, never a newer connection's. Shared scenarios repeat-while-handler-runs-in-order / -below-anchor (identical bytes in agentchat-typescript). --- CHANGELOG.md | 19 +++ conformance/v1/engine-scenarios.json | 164 +++++++++++++++++++++ src/agentchatme/_realtime.py | 93 +++++++++++- tests/test_realtime_engine.py | 209 ++++++++++++++++++++++++++- 4 files changed, 476 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6815039..a2588ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,25 @@ major. The on-the-wire API is versioned separately under `/v1/...`. handler has seen is unchanged, and there the SDKs still differ: Python drops it unacked, so it waits for the server to re-offer it on the next reconnect; TypeScript dispatches it late (out of order) and acks it. +- A message could reach the `message.new` handlers twice while its first + copy's handler was still running. After a reconnect the server pushes its + stored backlog on the socket and the REST drain brings the same rows; the + dedup cache is filled only once a handler succeeds, so with a slow handler + (a model call) the second copy was dispatched again, and a script that + replies per message replied twice. A copy whose first copy is still with + the handlers is now not dispatched: it takes that outcome. Success acks it + (an ack frame for a live copy, the cursor for a drain row); a failure or a + cancelled first copy leaves it unacked so the server re-offers it. A copy + waits on the first copy only where it waited on its own handler call + before; a live repeat below the anchor, which used to be dropped at once, + still never holds up the socket and is acked when the first copy + finished. With `dedup_cache_size=0` every copy is dispatched, as before. + Matches the TypeScript SDK. +- Two ordering-state races between concurrent copies are closed: the + per-conversation cursor never moves backwards when a slower copy finishes + after a faster one moved it on, and a caller that resumes after a socket + close no longer removes the new connection's ordering state (which could + drop frames parked in it). ### Added diff --git a/conformance/v1/engine-scenarios.json b/conformance/v1/engine-scenarios.json index fda9539..8916b37 100644 --- a/conformance/v1/engine-scenarios.json +++ b/conformance/v1/engine-scenarios.json @@ -519,6 +519,170 @@ ] ], "why": "As above, but the repeat of seq 1 is a REST drain row. It is not handed over again and the cursor covers it: this client already handled that message id. Drain rows are acknowledged by the cursor, never by an ack frame." + }, + { + "expect": { + "ack_on_receipt": { + "cursor_acks": [ + "del_000000000000000000000000000d0002" + ], + "dispatched": [ + "msg_ops0slow00001", + "msg_ops0slow00002" + ], + "ws_acks": [ + "msg_ops0slow00001", + "msg_ops0slow00002" + ] + }, + "default": { + "cursor_acks": [ + "del_000000000000000000000000000d0002" + ], + "dispatched": [ + "msg_ops0slow00001", + "msg_ops0slow00002" + ], + "ws_acks": [ + "msg_ops0slow00001", + "msg_ops0slow00002" + ] + } + }, + "hold": [ + "msg_ops0slow00002" + ], + "name": "repeat-while-handler-runs-in-order", + "steps": [ + { + "op": "connect" + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:01:00.000Z", + "id": "msg_ops0slow00001", + "sender": "nova", + "seq": 1, + "type": "text" + } + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 2" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:02:00.000Z", + "id": "msg_ops0slow00002", + "sender": "nova", + "seq": 2, + "type": "text" + } + }, + { + "op": "drain" + }, + { + "message_id": "msg_ops0slow00002", + "op": "release" + } + ], + "sync_pages": [ + [ + { + "content": { + "text": "message 2" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:02:00.000Z", + "delivery_id": "del_000000000000000000000000000d0002", + "id": "msg_ops0slow00002", + "sender": "nova", + "seq": 2, + "type": "text" + } + ] + ], + "why": "Seq 2 arrives live and its handler is slow (it calls a model). Meanwhile the same stored row comes back through the REST drain, as it does after every reconnect because the server also pushes its backlog on the socket. The second copy is not handed to the handler: it takes the first copy's outcome, so the cursor covers it once that handler finished." + }, + { + "expect": { + "ack_on_receipt": { + "cursor_acks": [ + "del_00000000000000000000000000090001" + ], + "dispatched": [ + "msg_crew0slow0001" + ], + "ws_acks": [ + "msg_crew0slow0001" + ] + }, + "default": { + "cursor_acks": [ + "del_00000000000000000000000000090001" + ], + "dispatched": [ + "msg_crew0slow0001" + ], + "ws_acks": [ + "msg_crew0slow0001" + ] + } + }, + "hold": [ + "msg_crew0slow0001" + ], + "name": "repeat-while-handler-runs-below-anchor", + "steps": [ + { + "op": "connect" + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0crew0001", + "created_at": "2026-11-04T23:01:00.000Z", + "id": "msg_crew0slow0001", + "sender": "nova", + "seq": 1, + "type": "text" + } + }, + { + "op": "drain" + }, + { + "message_id": "msg_crew0slow0001", + "op": "release" + } + ], + "sync_pages": [ + [ + { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0crew0001", + "created_at": "2026-11-04T23:01:00.000Z", + "delivery_id": "del_00000000000000000000000000090001", + "id": "msg_crew0slow0001", + "sender": "nova", + "seq": 1, + "type": "text" + } + ] + ], + "why": "The conversation's first live frame has a slow handler, and the REST drain brings the same row while it runs: below the anchor, not yet in the dedup cache. It is not handed over again; the cursor covers it once the first copy's handler finished." } ] } diff --git a/src/agentchatme/_realtime.py b/src/agentchatme/_realtime.py index 7f8d549..aa66b07 100644 --- a/src/agentchatme/_realtime.py +++ b/src/agentchatme/_realtime.py @@ -437,6 +437,10 @@ def __init__(self, capacity: int) -> None: self._capacity = capacity self._entries: OrderedDict[str, None] = OrderedDict() + @property + def enabled(self) -> bool: + return self._capacity > 0 + def hit(self, key: str) -> bool: """Return ``True`` (and refresh recency) if ``key`` was seen before.""" if key not in self._entries: @@ -602,6 +606,14 @@ def __init__( # reconnects deliberately: redelivery across connections is exactly # the duplicate source it exists to absorb. self._dedup = _BoundedLruSet(options.dedup_cache_size) + # Dispatches still running, by message id: resolves to the first + # copy's outcome. A second copy that arrives meanwhile (the server's + # reconnect backlog and the REST drain bring the same stored rows; a + # slow handler is still on the first) is not handed to the handlers + # again. An entry lives exactly as long as its handlers run, so the + # map is bounded by the dispatches in flight. Off with the dedup + # cache (``dedup_cache_size=0``). + self._inflight: dict[str, asyncio.Future[bool]] = {} # One REST drain at a time: the server-side ack pointer only moves # forward, so a second concurrent drain adds nothing but races. self._drain_in_flight = False @@ -1608,15 +1620,56 @@ async def _deliver_message_new( await self._send_delivery_ack(message_id) return self._settle_drain_row(message, True) - dispatched_cleanly = await self._dispatch(message) + first = self._inflight.get(message_id) if message_id is not None else None + if first is not None and message_id is not None: + # The same message is still with the handlers. Take that + # outcome instead of dispatching it twice: success covers this + # copy too (acked like a dedup hit), a failure leaves it unacked + # so the server re-offers it. The caller waits exactly where it + # would otherwise have waited for its own handler call; shielded + # so that cancelling this caller never cancels the first copy. + ok = await asyncio.shield(first) + if ok and ws_ack: + await self._send_delivery_ack(message_id) + return self._settle_drain_row(message, ok) + + own: asyncio.Future[bool] | None = None + if message_id is not None and self._dedup.enabled: + own = asyncio.get_running_loop().create_future() + self._inflight[message_id] = own + dispatched_cleanly = False + try: + dispatched_cleanly = await self._dispatch(message) + if dispatched_cleanly and message_id is not None: + self._dedup.add(message_id) + finally: + # In the same step the outcome is known (and, on success, the id + # is in the dedup cache): a later copy finds one or the other, + # never neither. A cancelled dispatch counts as not handled. + if own is not None and message_id is not None: + if self._inflight.get(message_id) is own: + del self._inflight[message_id] + if not own.done(): + own.set_result(dispatched_cleanly) if not dispatched_cleanly: return self._settle_drain_row(message, False) - if message_id is not None: - self._dedup.add(message_id) - if ws_ack: - await self._send_delivery_ack(message_id) + if message_id is not None and ws_ack: + await self._send_delivery_ack(message_id) return self._settle_drain_row(message, True) + def _ack_when_handled(self, message_id: str) -> None: + """Acknowledge ``message_id`` once its in-flight first copy finishes + cleanly, without waiting for it here.""" + first = self._inflight.get(message_id) + if first is None: + return + + def done(future: asyncio.Future[bool]) -> None: + if not future.cancelled() and future.result() and not self._disposed: + self._spawn(self._send_delivery_ack(message_id)) + + first.add_done_callback(done) + def _settle_drain_row(self, message: dict[str, Any], ok: bool) -> bool: key = id(message) if key in self._drain_settlements: @@ -1719,6 +1772,17 @@ async def _process_ordered_message( # the earlier dispatch is the proof, so acknowledge it # again, as the dedup path does for every other repeat. await self._send_delivery_ack(message_id) + elif message_id is not None and message_id in self._inflight: + # A repeat of a message whose first copy is still with + # the handlers. A drain row takes that outcome for its + # cursor (the drain waits on handlers anyway); a live + # frame never holds up the socket for it and is acked + # once the first copy finished cleanly. + if id(message) in self._drain_settlements: + await self._deliver_message_new(message, ws_ack=False) + return "delivered" + if ws_ack: + self._ack_when_handled(message_id) return "duplicate" if seen: return "duplicate" @@ -1727,7 +1791,9 @@ async def _process_ordered_message( if seq == state.next_expected_seq: await self._deliver_message_new(message, ws_ack=ws_ack) - state.next_expected_seq = seq + 1 + # Never backwards: another copy of this message (or a drain of + # the buffer) may have moved the cursor on during the await. + _advance_past(state, seq) await self._drain_consecutive(conversation_id, state) self._maybe_clear_gap_timer(state) self._cleanup_if_idle(conversation_id, state) @@ -1870,9 +1936,10 @@ async def _drain_consecutive( return False drained = False while state.next_expected_seq in state.buffer: - pending = state.buffer.pop(state.next_expected_seq) + seq = state.next_expected_seq + pending = state.buffer.pop(seq) await self._deliver_message_new(pending.message, ws_ack=pending.ws_ack) - state.next_expected_seq += 1 + _advance_past(state, seq) drained = True if drained: self._cleanup_if_idle(conversation_id, state) @@ -1944,6 +2011,9 @@ def _cleanup_if_idle(self, conversation_id: str, state: _OrderState) -> None: not state.buffer and state.gap_task is None and not state.gap_fill_in_flight + # A state a socket close replaced while this caller was awaiting + # a handler is not the conversation's state any more. + and self._order_states.get(conversation_id) is state ): self._order_states.pop(conversation_id, None) @@ -2094,6 +2164,13 @@ def _is_valid_sync_row(row: object) -> bool: return "content" not in row or isinstance(row["content"], dict) +def _advance_past(state: _OrderState, seq: int) -> None: + """Move the conversation's cursor past ``seq``, never backwards.""" + current = state.next_expected_seq + if current is None or current <= seq: + state.next_expected_seq = seq + 1 + + def _min_buffered_seq(state: _OrderState) -> int | None: if not state.buffer: return None diff --git a/tests/test_realtime_engine.py b/tests/test_realtime_engine.py index 492aec0..db08e24 100644 --- a/tests/test_realtime_engine.py +++ b/tests/test_realtime_engine.py @@ -12,7 +12,7 @@ import pytest -from agentchatme import RealtimeClient +from agentchatme import RealtimeClient, _realtime from agentchatme.types.ws import WakePayload from .test_realtime import MockAsyncClient, MockWebSocket, _ack_frames, _make_client, _settle @@ -407,6 +407,213 @@ async def test_default_mode_acks_gap_fill_rows_after_dispatch( await rt.disconnect() +# ─────────────── a repeat while the first copy is still with the handlers ─────────────── +# +# After a reconnect the server pushes its stored backlog on the socket and the +# REST drain brings the same rows. With a slow handler the second copy comes +# before the first one finished, so the dedup cache (filled on success) cannot +# catch it. + + +def _row(seq: int, *, mid: str | None = None) -> dict[str, Any]: + return { + "id": mid or f"msg_{seq}", + "conversation_id": "c", + "seq": seq, + "delivery_id": f"del_{seq}", + "sender": "nova", + "type": "text", + "content": {"text": "hi"}, + "created_at": "2026-11-04T23:00:00.000Z", + } + + +@pytest.mark.parametrize("ack_on_receipt", [False, True]) +@pytest.mark.asyncio +async def test_a_drain_row_repeat_of_a_running_live_message_is_not_dispatched_again( + ack_on_receipt: bool, +) -> None: + api = MockAsyncClient(sync_pages=[[_row(2)]]) + rt, ws = await connected( + capabilities=["ack"], client=api, auto_drain_on_connect=False, ack_on_receipt=ack_on_receipt + ) + release = asyncio.Event() + calls: list[str] = [] + + async def slow(m: dict[str, Any]) -> None: + calls.append(m["payload"]["id"]) + if m["payload"]["id"] == "msg_2": + await release.wait() + + rt.on("message.new", slow) + try: + await ws.push(message_new("c", 1)) + await ws.push(message_new("c", 2)) # in order; its handler is slow + await _settle() + drain = asyncio.create_task(rt.drain_offline_envelopes()) + await _settle() + assert calls == ["msg_1", "msg_2"] + assert api.sync_ack_calls == [] + + release.set() + await asyncio.wait_for(drain, timeout=1.0) + await _settle() + assert calls == ["msg_1", "msg_2"] + assert api.sync_ack_calls == ["del_2"] + assert _ack_frames(ws) == ["msg_1", "msg_2"] + finally: + release.set() + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_a_live_repeat_below_the_anchor_never_holds_up_the_socket() -> None: + # The first copy is a drain row whose handler is slow. The server's + # backlog push brings the same message live, below the anchor. It is not + # dispatched, the socket keeps flowing (another conversation's frame is + # handled meanwhile), and it is acked once the first copy finished. + api = MockAsyncClient(sync_pages=[[_row(1)]]) + rt, ws = await connected(capabilities=["ack"], client=api, auto_drain_on_connect=False) + release = asyncio.Event() + calls: list[str] = [] + + async def slow(m: dict[str, Any]) -> None: + calls.append(m["payload"]["id"]) + if m["payload"]["id"] == "msg_1": + await release.wait() + + rt.on("message.new", slow) + try: + drain = asyncio.create_task(rt.drain_offline_envelopes()) + await _settle() + assert calls == ["msg_1"] + await ws.push(message_new("c", 1)) # the same message, live + await ws.push(message_new("other", 9, id="msg_other")) + await _settle() + assert calls == ["msg_1", "msg_other"] + assert _ack_frames(ws) == ["msg_other"] + + release.set() + await asyncio.wait_for(drain, timeout=1.0) + await _settle() + assert calls == ["msg_1", "msg_other"] + assert _ack_frames(ws) == ["msg_other", "msg_1"] + assert api.sync_ack_calls == ["del_1"] + finally: + release.set() + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_a_failed_first_copy_leaves_the_repeat_unacked_and_a_later_copy_is_dispatched() -> None: + api = MockAsyncClient(sync_pages=[[_row(2)]]) + rt, ws = await connected(capabilities=["ack"], client=api, auto_drain_on_connect=False) + release = asyncio.Event() + calls: list[str] = [] + + async def flaky(m: dict[str, Any]) -> None: + calls.append(m["payload"]["id"]) + if m["payload"]["id"] == "msg_2" and calls.count("msg_2") == 1: + await release.wait() + raise RuntimeError("model call failed") + + rt.on("message.new", flaky) + try: + await ws.push(message_new("c", 1)) + await ws.push(message_new("c", 2)) + await _settle() + drain = asyncio.create_task(rt.drain_offline_envelopes()) + await _settle() + release.set() + await asyncio.wait_for(drain, timeout=1.0) + await _settle() + # Neither copy is acknowledged: the server re-offers the message. + assert calls == ["msg_1", "msg_2"] + assert api.sync_ack_calls == [] + assert _ack_frames(ws) == ["msg_1"] + + # The re-offer is a fresh attempt. + await ws.push(message_new("c", 2)) + await _settle() + assert calls == ["msg_1", "msg_2", "msg_2"] + assert _ack_frames(ws) == ["msg_1", "msg_2"] + finally: + release.set() + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_a_cancelled_first_copy_releases_the_repeat_as_not_handled() -> None: + rt, _ws = await connected(capabilities=["ack"]) + entered = asyncio.Event() + + async def hangs(_m: dict[str, Any]) -> None: + entered.set() + await asyncio.Event().wait() + + rt.on("message.new", hangs) + try: + frame = message_new("c", 1) + first = asyncio.create_task(rt._deliver_message_new(frame, ws_ack=False)) + await entered.wait() + repeat = asyncio.create_task(rt._deliver_message_new(dict(frame), ws_ack=False)) + await _settle() + assert not repeat.done() + first.cancel() + assert await asyncio.wait_for(repeat, timeout=1.0) is False + assert rt._inflight == {} + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_with_the_dedup_cache_off_every_copy_is_dispatched() -> None: + rt, _ws = await connected(capabilities=["ack"], dedup_cache_size=0) + release = asyncio.Event() + calls: list[str] = [] + + async def slow(m: dict[str, Any]) -> None: + calls.append(m["payload"]["id"]) + await release.wait() + + rt.on("message.new", slow) + try: + frame = message_new("c", 1) + tasks = [ + asyncio.create_task(rt._deliver_message_new(dict(frame), ws_ack=False)) + for _ in range(2) + ] + await _settle() + assert calls == ["msg_1", "msg_1"] + release.set() + await asyncio.gather(*tasks) + finally: + release.set() + await rt.disconnect() + + +def test_the_ordering_cursor_never_moves_backwards() -> None: + state = _realtime._OrderState(next_expected_seq=8) + _realtime._advance_past(state, 5) # a late copy of seq 5 finishing + assert state.next_expected_seq == 8 + _realtime._advance_past(state, 8) + assert state.next_expected_seq == 9 + + +@pytest.mark.asyncio +async def test_cleanup_never_drops_a_state_a_close_replaced() -> None: + rt, _ws = await connected(capabilities=["ack"]) + try: + stale = _realtime._OrderState(next_expected_seq=3) + current = _realtime._OrderState(next_expected_seq=1) + current.buffer[2] = _realtime._PendingEnvelope(message=message_new("c", 2), ws_ack=True) + rt._order_states["c"] = current + rt._cleanup_if_idle("c", stale) # an idle orphan from before the close + assert rt._order_states["c"] is current + finally: + await rt.disconnect() + + # ─────────────── a guest in a host process ─────────────── From b140d1eb02275a72faafbc252b2bf256b894a9dd Mon Sep 17 00:00:00 2001 From: San Date: Sat, 7 Nov 2026 15:10:41 -0800 Subject: [PATCH 5/7] fix(realtime): a frame that parks while a gap is resolved is handed over, never cleared unseen _resolve_gap and _flush_parked_frames snapshotted the buffer, awaited each handler, then cleared it. The receive loop keeps running during those awaits, so a frame that parked meanwhile was cleared without dispatch: under ack_on_receipt (acked at receipt) it was never handled; in the default mode it waited unacked for the next reconnect. A concurrent in-order arrival could also pop entries the snapshot still expected (KeyError in the gap task) or hand them over twice. Both now pop each frame before handing it over, lowest seq first, and keep going until nothing is parked (the flush across every conversation, since disconnect() flushes before it closes the socket). The cursor advances past each frame after its handler, never backwards. Shared scenario frame-arrives-while-a-lapsed-gap-is-handled, with a new op gap_window_lapses (identical bytes in agentchat-typescript). --- CHANGELOG.md | 15 ++++ conformance/v1/engine-scenarios.json | 89 ++++++++++++++++++++ src/agentchatme/_realtime.py | 102 ++++++++++++---------- tests/test_conformance_engine.py | 7 +- tests/test_realtime_engine.py | 121 +++++++++++++++++++++++++++ 5 files changed, 290 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2588ee..6008187 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,18 @@ major. The on-the-wire API is versioned separately under `/v1/...`. after a faster one moved it on, and a caller that resumes after a socket close no longer removes the new connection's ordering state (which could drop frames parked in it). +- A frame could be acked and never handed to a handler under + `ack_on_receipt` (the Hermes setting). When a gap is resolved without a + fill (the window lapsed, the fill failed, or the buffer overflowed), or + parked frames are flushed at close or `disconnect()`, the client awaits + each handler in turn, and the receive loop keeps running meanwhile. A frame + that parked during those awaits was then cleared with the buffer: acked at + receipt, it was never handled. In the default mode it was left unacked and + waited for the next reconnect. Each frame is now taken out of the buffer + before it is handed over, and the client keeps going until no frame is + parked, so such frames are handed over too. This also ends a crash: when + the missing seq arrived meanwhile, the receive loop could take buffered + frames the gap task still expected (`KeyError`), or hand them over twice. ### Added @@ -48,6 +60,9 @@ major. The on-the-wire API is versioned separately under `/v1/...`. row). Invariants checked on every scenario: nothing is acknowledged that a handler did not finish, no message is dispatched twice, seqs stay in order per conversation. Identical bytes ship in the TypeScript SDK. +- Engine scenarios `repeat-while-handler-runs-in-order` / `-below-anchor` + and `frame-arrives-while-a-lapsed-gap-is-handled` (new op + `gap_window_lapses`, a real 2 s wait). ### Changed diff --git a/conformance/v1/engine-scenarios.json b/conformance/v1/engine-scenarios.json index 8916b37..56a2c71 100644 --- a/conformance/v1/engine-scenarios.json +++ b/conformance/v1/engine-scenarios.json @@ -17,6 +17,7 @@ "connect": "Create the client for the mode, connect, open the socket and answer HELLO with {\"type\": \"hello.ok\", \"capabilities\": [\"ack\"]}.", "drain": "Start the client's offline drain (drainOfflineEnvelopes / drain_offline_envelopes) without awaiting it.", "frame": "The server sends {\"type\": \"message.new\", \"payload\": } on the socket.", + "gap_window_lapses": "Wait until the 2 s gap window has passed, so the client resolves the open gap (the REST gap fill returns nothing) and hands the parked frames over.", "release": "Let the held handler for message_id finish.", "wait_handler": "Wait until the handler has been entered for message_id." }, @@ -683,6 +684,94 @@ ] ], "why": "The conversation's first live frame has a slow handler, and the REST drain brings the same row while it runs: below the anchor, not yet in the dedup cache. It is not handed over again; the cursor covers it once the first copy's handler finished." + }, + { + "expect": { + "ack_on_receipt": { + "cursor_acks": [], + "dispatched": [ + "msg_gone0wait0001", + "msg_gone0wait0003", + "msg_gone0wait0005" + ], + "ws_acks": [ + "msg_gone0wait0001", + "msg_gone0wait0003", + "msg_gone0wait0005" + ] + }, + "default": { + "cursor_acks": [], + "dispatched": [ + "msg_gone0wait0001", + "msg_gone0wait0003", + "msg_gone0wait0005" + ] + } + }, + "hold": [ + "msg_gone0wait0003" + ], + "name": "frame-arrives-while-a-lapsed-gap-is-handled", + "steps": [ + { + "op": "connect" + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0gone0001", + "created_at": "2026-11-04T23:01:00.000Z", + "id": "msg_gone0wait0001", + "sender": "nova", + "seq": 1, + "type": "text" + } + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 3" + }, + "conversation_id": "conv_group0gone0001", + "created_at": "2026-11-04T23:03:00.000Z", + "id": "msg_gone0wait0003", + "sender": "nova", + "seq": 3, + "type": "text" + } + }, + { + "op": "gap_window_lapses" + }, + { + "message_id": "msg_gone0wait0003", + "op": "wait_handler" + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 5" + }, + "conversation_id": "conv_group0gone0001", + "created_at": "2026-11-04T23:05:00.000Z", + "id": "msg_gone0wait0005", + "sender": "nova", + "seq": 5, + "type": "text" + } + }, + { + "message_id": "msg_gone0wait0003", + "op": "release" + } + ], + "why": "Seq 3 waits on a gap that is never filled. When the gap window lapses, seq 3 is handed over and its handler is slow. Seq 5 arrives while it runs. It must reach the handler: under ack_on_receipt it was acked at receipt, so a client that clears it with the buffer loses it for good." } ] } diff --git a/src/agentchatme/_realtime.py b/src/agentchatme/_realtime.py index aa66b07..1682a08 100644 --- a/src/agentchatme/_realtime.py +++ b/src/agentchatme/_realtime.py @@ -1966,19 +1966,17 @@ async def _resolve_gap( else 0 ) - seqs = sorted(state.buffer.keys()) - if state.next_expected_seq is not None: - highest_dispatched = state.next_expected_seq - 1 - else: - highest_dispatched = -1 - for s in seqs: - pending = state.buffer[s] + # Take each frame out of the buffer before handing it over, lowest + # seq first, until the buffer is empty. The receive loop keeps + # running while a handler is awaited here: a frame that parks + # meanwhile is handed over too, never cleared unseen (under + # ack_on_receipt it is already acked and would be lost), and an entry + # another caller took in the meantime is simply not here any more. + while state.buffer: + seq = min(state.buffer) + pending = state.buffer.pop(seq) await self._deliver_message_new(pending.message, ws_ack=pending.ws_ack) - if s > highest_dispatched: - highest_dispatched = s - state.buffer.clear() - if highest_dispatched >= 0: - state.next_expected_seq = highest_dispatched + 1 + _advance_past(state, seq) if state.gap_task is not None: state.gap_task.cancel() @@ -2032,41 +2030,61 @@ async def _flush_parked_frames(self, reason: GapReason) -> None: close (the frames are already acked). No gap-fill: an in-flight HTTP request would race the close. """ - for conversation_id, state in list(self._order_states.items()): + # Keep going until no conversation has a parked frame: handlers are + # awaited here and the receive loop may still be running (disconnect + # flushes before it closes the socket), so a frame can park in any + # conversation meanwhile. Clearing it unseen would lose it once it is + # acked at receipt. + for state in self._order_states.values(): + if state.gap_task is not None: + state.gap_task.cancel() + state.gap_task = None + while True: + parked = [(cid, st) for cid, st in self._order_states.items() if st.buffer] + if not parked: + break + for conversation_id, state in parked: + await self._flush_conversation(conversation_id, state, reason) + for state in self._order_states.values(): if state.gap_task is not None: state.gap_task.cancel() state.gap_task = None - if not state.buffer: - continue - seqs = sorted(state.buffer.keys()) - for s in seqs: - pending = state.buffer[s] - await self._deliver_message_new( - pending.message, ws_ack=pending.ws_ack - ) - state.buffer.clear() - if self._opts.on_sequence_gap is not None: - if state.gap_started_expected_seq is not None: - expected_seq = state.gap_started_expected_seq - elif state.next_expected_seq is not None: - expected_seq = state.next_expected_seq - else: - expected_seq = 0 - info = SequenceGapInfo( - conversation_id=conversation_id, - expected_seq=expected_seq, - buffered_seq=seqs[0] if seqs else None, - gap_ms=( - int((time.monotonic() - state.gap_started_at) * 1000) - if state.gap_started_at is not None - else 0 - ), - recovered=False, - reason=reason, - ) - await _invoke(self._opts.on_sequence_gap, info) self._order_states.clear() + async def _flush_conversation( + self, conversation_id: str, state: _OrderState, reason: GapReason + ) -> None: + if state.gap_task is not None: + state.gap_task.cancel() + state.gap_task = None + seqs: list[int] = [] + while state.buffer: + seq = min(state.buffer) + pending = state.buffer.pop(seq) + seqs.append(seq) + await self._deliver_message_new(pending.message, ws_ack=pending.ws_ack) + if not seqs or self._opts.on_sequence_gap is None: + return + if state.gap_started_expected_seq is not None: + expected_seq = state.gap_started_expected_seq + elif state.next_expected_seq is not None: + expected_seq = state.next_expected_seq + else: + expected_seq = 0 + info = SequenceGapInfo( + conversation_id=conversation_id, + expected_seq=expected_seq, + buffered_seq=seqs[0], + gap_ms=( + int((time.monotonic() - state.gap_started_at) * 1000) + if state.gap_started_at is not None + else 0 + ), + recovered=False, + reason=reason, + ) + await _invoke(self._opts.on_sequence_gap, info) + # ───────────────────────── Module helpers ───────────────────────── diff --git a/tests/test_conformance_engine.py b/tests/test_conformance_engine.py index 798c167..2228f27 100644 --- a/tests/test_conformance_engine.py +++ b/tests/test_conformance_engine.py @@ -16,7 +16,7 @@ import pytest -from agentchatme import RealtimeClient +from agentchatme import RealtimeClient, _realtime from .test_realtime import MockWebSocket, _make_client @@ -24,7 +24,7 @@ DOC: dict[str, Any] = json.loads(FIXTURE.read_text(encoding="utf-8")) MODES = ["default", "ack_on_receipt"] -OPS = ["close", "connect", "drain", "frame", "release", "wait_handler"] +OPS = ["close", "connect", "drain", "frame", "gap_window_lapses", "release", "wait_handler"] EXPECTATIONS = ["cursor_acks", "dispatched", "dispatched_includes", "ws_acks"] INVARIANTS = ["acked_was_handled", "acks_follow_handling", "dispatched_once", "seq_order"] @@ -119,6 +119,9 @@ async def handler(message: dict[str, Any]) -> None: elif op == "drain": assert rt is not None drains.append(asyncio.create_task(rt.drain_offline_envelopes())) + elif op == "gap_window_lapses": + # The client's gap window plus slack, in real time. + await asyncio.sleep(_realtime._GAP_FILL_WINDOW_S + 0.15) elif op == "wait_handler": for _ in range(100): if any(p["id"] == step["message_id"] for p in out.dispatched): diff --git a/tests/test_realtime_engine.py b/tests/test_realtime_engine.py index db08e24..a1c2196 100644 --- a/tests/test_realtime_engine.py +++ b/tests/test_realtime_engine.py @@ -614,6 +614,127 @@ async def test_cleanup_never_drops_a_state_a_close_replaced() -> None: await rt.disconnect() +# ─────────────── resolving a gap while a handler runs ─────────────── +# +# When the gap window lapses, the gap task hands the parked frames over and +# awaits each handler. The receive loop keeps running meanwhile, so frames +# arrive while the gap is being resolved. + + +def _gap_hook(errors: list[BaseException]) -> None: + asyncio.get_running_loop().set_exception_handler( + lambda _loop, context: errors.append(context.get("exception") or RuntimeError(context["message"])) + ) + + +@pytest.mark.parametrize("ack_on_receipt", [False, True]) +@pytest.mark.asyncio +async def test_a_frame_that_parks_while_a_resolved_gap_is_handled_is_dispatched( + monkeypatch: pytest.MonkeyPatch, ack_on_receipt: bool +) -> None: + # No REST client, so the lapsed gap is resolved without a fill. Seq 5 + # parks while the handler for seq 3 runs. It was cleared with the buffer + # without ever being dispatched; under ack_on_receipt it was already + # acked, so no handler would ever see it. + monkeypatch.setattr("agentchatme._realtime._GAP_FILL_WINDOW_S", 0.05) + rt, ws = await connected(capabilities=["ack"], ack_on_receipt=ack_on_receipt) + release = asyncio.Event() + entered = asyncio.Event() + seen: list[int] = [] + + async def slow(m: dict[str, Any]) -> None: + seen.append(m["payload"]["seq"]) + if m["payload"]["seq"] == 3: + entered.set() + await release.wait() + + rt.on("message.new", slow) + try: + await ws.push(message_new("c", 1)) + await ws.push(message_new("c", 3)) + await asyncio.wait_for(entered.wait(), timeout=1.0) # gap lapsed; 3 handed over + await ws.push(message_new("c", 5)) + await _settle() + release.set() + await asyncio.sleep(0.2) + assert seen == [1, 3, 5] + assert sorted(_ack_frames(ws)) == ["msg_1", "msg_3", "msg_5"] + finally: + release.set() + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_the_missing_seq_arriving_while_a_resolved_gap_is_handled_is_not_a_crash( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Seq 2 arrives while the lapsed gap's handler for seq 3 runs. The + # receive loop dispatches it in order and drains the buffer; the gap + # task must not then look up an entry that is gone (KeyError) or hand + # seq 4 over a second time. + monkeypatch.setattr("agentchatme._realtime._GAP_FILL_WINDOW_S", 0.05) + errors: list[BaseException] = [] + _gap_hook(errors) + rt, ws = await connected(capabilities=["ack"], ack_on_receipt=True) + release = asyncio.Event() + entered = asyncio.Event() + seen: list[int] = [] + + async def slow(m: dict[str, Any]) -> None: + seen.append(m["payload"]["seq"]) + if m["payload"]["seq"] == 3: + entered.set() + await release.wait() + + rt.on("message.new", slow) + try: + await ws.push(message_new("c", 1)) + await ws.push(message_new("c", 3)) + await ws.push(message_new("c", 4)) + await asyncio.wait_for(entered.wait(), timeout=1.0) + await ws.push(message_new("c", 2)) + await _settle() + release.set() + await asyncio.sleep(0.2) + assert sorted(seen) == [1, 2, 3, 4] + assert len(seen) == 4 + assert errors == [] + assert rt._order_states.get("c") is None or not rt._order_states["c"].buffer + finally: + release.set() + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_disconnect_dispatches_frames_that_park_during_its_own_flush() -> None: + # disconnect() flushes parked frames while the socket is still open; a + # frame that parks during that flush (the handler for an earlier one is + # awaited) was cleared with the ordering state. Acked at receipt, it + # must reach the handler. + rt, ws = await connected(capabilities=["ack"], ack_on_receipt=True) + release = asyncio.Event() + entered = asyncio.Event() + seen: list[int] = [] + + async def slow(m: dict[str, Any]) -> None: + seen.append(m["payload"]["seq"]) + if m["payload"]["seq"] == 3: + entered.set() + await release.wait() + + rt.on("message.new", slow) + await ws.push(message_new("c", 1)) + await ws.push(message_new("c", 3)) + await _settle() + stopping = asyncio.create_task(rt.disconnect()) + await asyncio.wait_for(entered.wait(), timeout=1.0) + await ws.push(message_new("c", 6)) + await _settle() + release.set() + await asyncio.wait_for(stopping, timeout=3.0) + assert seen == [1, 3, 6] + + # ─────────────── a guest in a host process ─────────────── From 801ce7d44de4603dbccf596dfe1ca72b917103b6 Mon Sep 17 00:00:00 2001 From: San Date: Sat, 7 Nov 2026 16:10:20 -0800 Subject: [PATCH 6/7] fix(realtime): progress beats dedup: nothing waits on another copy, gaps resolve on a snapshot Review of #11. An in-flight repeat no longer waits for the first copy anywhere: _adopt_in_flight records the outcome and returns. A live repeat is acked when the first copy finished cleanly, never if it failed; a drain row's cursor reads the outcome when computed and stops if the first copy is not done (the next drain covers it as a dedup hit), so a slow or hung live handler no longer holds up later drain rows, and a handler that awaits disconnect() while a repeat of its own message arrives no longer deadlocks. Adoption is per connection generation: a re-offer on a later connection is dispatched afresh even if the first copy still runs. _resolve_gap and the shutdown flush work on the frames parked when they start; each frame leaves the buffer and the cursor moves past it before its handler is awaited, so in-order arrivals go to the receive loop, a frame behind a new hole gets its own gap window and report, and nothing is cleared unseen (a state being resolved is never dropped as idle). disconnect() stops taking frames in before it flushes, so it returns promptly while frames keep arriving; a frame acked at receipt is never dropped by the flush. --- CHANGELOG.md | 41 +++-- conformance/v1/engine-scenarios.json | 64 ++++--- src/agentchatme/_realtime.py | 260 ++++++++++++++++++--------- tests/test_realtime_engine.py | 200 ++++++++++++++++++--- 4 files changed, 422 insertions(+), 143 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6008187..50bea90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,15 +23,20 @@ major. The on-the-wire API is versioned separately under `/v1/...`. stored backlog on the socket and the REST drain brings the same rows; the dedup cache is filled only once a handler succeeds, so with a slow handler (a model call) the second copy was dispatched again, and a script that - replies per message replied twice. A copy whose first copy is still with - the handlers is now not dispatched: it takes that outcome. Success acks it - (an ack frame for a live copy, the cursor for a drain row); a failure or a - cancelled first copy leaves it unacked so the server re-offers it. A copy - waits on the first copy only where it waited on its own handler call - before; a live repeat below the anchor, which used to be dropped at once, - still never holds up the socket and is acked when the first copy - finished. With `dedup_cache_size=0` every copy is dispatched, as before. - Matches the TypeScript SDK. + replies per message replied twice. A copy that arrives on the same + connection while its first copy is still with the handlers is now not + dispatched, and nothing waits for the first copy: progress beats dedup. + A live copy is acked once the first copy finished cleanly, never if it + failed. A drain row's cursor stops at the row unless the first copy + already finished cleanly (the next drain covers it as a dedup hit). If the + first copy fails, the repeat is not acked either: in the default mode the + server re-offers the message; under `ack_on_receipt` the live copy was + already acked at receipt, so it is not re-offered, as with any failed + handler in that mode. A copy on a later connection is + dispatched afresh even if the first copy is still running, so a handler + that never finishes cannot swallow every re-offer. In-flight entries live + only while handlers run; with `dedup_cache_size=0` every copy is + dispatched, as before. Matches the TypeScript SDK. - Two ordering-state races between concurrent copies are closed: the per-conversation cursor never moves backwards when a slower copy finishes after a faster one moved it on, and a caller that resumes after a socket @@ -44,11 +49,17 @@ major. The on-the-wire API is versioned separately under `/v1/...`. each handler in turn, and the receive loop keeps running meanwhile. A frame that parked during those awaits was then cleared with the buffer: acked at receipt, it was never handled. In the default mode it was left unacked and - waited for the next reconnect. Each frame is now taken out of the buffer - before it is handed over, and the client keeps going until no frame is - parked, so such frames are handed over too. This also ends a crash: when - the missing seq arrived meanwhile, the receive loop could take buffered - frames the gap task still expected (`KeyError`), or hand them over twice. + waited for the next reconnect. Now the frames parked when the gap resolved + are each taken out of the buffer, and the cursor moves past each, before + its handler is awaited. A frame that arrives in order meanwhile is + dispatched by the receive loop at once; one behind a new hole parks with a + gap window and report of its own; nothing is cleared. `disconnect()` stops + taking frames in before it flushes (an arriving frame is neither acked nor + dispatched; the server keeps it), so it returns promptly while frames keep + arriving, and a frame acked at receipt is never dropped by the flush. This + also ends a crash: when the missing seq arrived meanwhile, the receive loop + could take buffered frames the gap task still expected (`KeyError`), or + hand them over twice. ### Added @@ -61,7 +72,7 @@ major. The on-the-wire API is versioned separately under `/v1/...`. handler did not finish, no message is dispatched twice, seqs stay in order per conversation. Identical bytes ship in the TypeScript SDK. - Engine scenarios `repeat-while-handler-runs-in-order` / `-below-anchor` - and `frame-arrives-while-a-lapsed-gap-is-handled` (new op + and `frames-arrive-while-a-lapsed-gap-is-handled` (new op `gap_window_lapses`, a real 2 s wait). ### Changed diff --git a/conformance/v1/engine-scenarios.json b/conformance/v1/engine-scenarios.json index 56a2c71..9c74ccc 100644 --- a/conformance/v1/engine-scenarios.json +++ b/conformance/v1/engine-scenarios.json @@ -524,9 +524,7 @@ { "expect": { "ack_on_receipt": { - "cursor_acks": [ - "del_000000000000000000000000000d0002" - ], + "cursor_acks": [], "dispatched": [ "msg_ops0slow00001", "msg_ops0slow00002" @@ -537,9 +535,7 @@ ] }, "default": { - "cursor_acks": [ - "del_000000000000000000000000000d0002" - ], + "cursor_acks": [], "dispatched": [ "msg_ops0slow00001", "msg_ops0slow00002" @@ -610,14 +606,12 @@ } ] ], - "why": "Seq 2 arrives live and its handler is slow (it calls a model). Meanwhile the same stored row comes back through the REST drain, as it does after every reconnect because the server also pushes its backlog on the socket. The second copy is not handed to the handler: it takes the first copy's outcome, so the cursor covers it once that handler finished." + "why": "Seq 2 arrives live and its handler is slow (it calls a model). Meanwhile the same stored row comes back through the REST drain, as it does after every reconnect because the server also pushes its backlog on the socket. The second copy is not handed to the handler again, and nothing waits for the first: the cursor stops at the row (the next drain finds it in the dedup cache), and the live copy is acknowledged when its handler finished." }, { "expect": { "ack_on_receipt": { - "cursor_acks": [ - "del_00000000000000000000000000090001" - ], + "cursor_acks": [], "dispatched": [ "msg_crew0slow0001" ], @@ -626,9 +620,7 @@ ] }, "default": { - "cursor_acks": [ - "del_00000000000000000000000000090001" - ], + "cursor_acks": [], "dispatched": [ "msg_crew0slow0001" ], @@ -683,7 +675,7 @@ } ] ], - "why": "The conversation's first live frame has a slow handler, and the REST drain brings the same row while it runs: below the anchor, not yet in the dedup cache. It is not handed over again; the cursor covers it once the first copy's handler finished." + "why": "The conversation's first live frame has a slow handler, and the REST drain brings the same row while it runs: below the anchor, not yet in the dedup cache. It is not handed over again and the drain does not wait for the first copy: the cursor stops at the row." }, { "expect": { @@ -692,12 +684,14 @@ "dispatched": [ "msg_gone0wait0001", "msg_gone0wait0003", - "msg_gone0wait0005" + "msg_gone0wait0004", + "msg_gone0wait0006" ], "ws_acks": [ "msg_gone0wait0001", "msg_gone0wait0003", - "msg_gone0wait0005" + "msg_gone0wait0004", + "msg_gone0wait0006" ] }, "default": { @@ -705,14 +699,21 @@ "dispatched": [ "msg_gone0wait0001", "msg_gone0wait0003", - "msg_gone0wait0005" + "msg_gone0wait0004", + "msg_gone0wait0006" + ], + "ws_acks": [ + "msg_gone0wait0001", + "msg_gone0wait0004", + "msg_gone0wait0003", + "msg_gone0wait0006" ] } }, "hold": [ "msg_gone0wait0003" ], - "name": "frame-arrives-while-a-lapsed-gap-is-handled", + "name": "frames-arrive-while-a-lapsed-gap-is-handled", "steps": [ { "op": "connect" @@ -756,22 +757,39 @@ "op": "frame", "payload": { "content": { - "text": "message 5" + "text": "message 4" + }, + "conversation_id": "conv_group0gone0001", + "created_at": "2026-11-04T23:04:00.000Z", + "id": "msg_gone0wait0004", + "sender": "nova", + "seq": 4, + "type": "text" + } + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 6" }, "conversation_id": "conv_group0gone0001", - "created_at": "2026-11-04T23:05:00.000Z", - "id": "msg_gone0wait0005", + "created_at": "2026-11-04T23:06:00.000Z", + "id": "msg_gone0wait0006", "sender": "nova", - "seq": 5, + "seq": 6, "type": "text" } }, { "message_id": "msg_gone0wait0003", "op": "release" + }, + { + "op": "gap_window_lapses" } ], - "why": "Seq 3 waits on a gap that is never filled. When the gap window lapses, seq 3 is handed over and its handler is slow. Seq 5 arrives while it runs. It must reach the handler: under ack_on_receipt it was acked at receipt, so a client that clears it with the buffer loses it for good." + "why": "Seq 3 waits on a gap that is never filled. When the gap window lapses, seq 3 is handed over and its handler is slow. Meanwhile seq 4 arrives in order and seq 6 behind a new hole. Both must reach the handler (under ack_on_receipt they were acked at receipt, so a client that clears them with the buffer loses them): seq 4 at once, seq 6 after a gap window of its own." } ] } diff --git a/src/agentchatme/_realtime.py b/src/agentchatme/_realtime.py index 1682a08..1b93488 100644 --- a/src/agentchatme/_realtime.py +++ b/src/agentchatme/_realtime.py @@ -420,6 +420,9 @@ class _PendingEnvelope: message: dict[str, Any] ws_ack: bool + # Acknowledged when it was received (ack_on_receipt): the server will + # never re-offer it, so it must reach a handler, whatever happens. + acked: bool = False class _BoundedLruSet: @@ -457,6 +460,15 @@ def add(self, key: str) -> None: self._entries.popitem(last=False) +@dataclass +class _InFlight: + """A ``message.new`` whose handlers are running: its outcome, and the + connection generation it was dispatched on.""" + + outcome: asyncio.Future[bool] + generation: int + + @dataclass class _OrderState: next_expected_seq: int | None = None @@ -465,6 +477,10 @@ class _OrderState: gap_started_at: float | None = None gap_started_expected_seq: int | None = None gap_fill_in_flight: bool = False + # Callers handing a resolved gap's frames over right now. The receive + # loop keeps running meanwhile and must not drop this state as idle: + # the next frame would anchor afresh and skip a hole unseen. + resolving: int = 0 # ───────────────────────── Client ───────────────────────── @@ -613,7 +629,12 @@ def __init__( # again. An entry lives exactly as long as its handlers run, so the # map is bounded by the dispatches in flight. Off with the dedup # cache (``dedup_cache_size=0``). - self._inflight: dict[str, asyncio.Future[bool]] = {} + self._inflight: dict[str, _InFlight] = {} + # Bumped on every socket close. A copy adopts an in-flight first copy + # only on the same connection; a re-offer on a later one is + # dispatched afresh even if the first copy is still running (a + # handler that never finishes must not swallow every re-offer). + self._generation = 0 # One REST drain at a time: the server-side ack pointer only moves # forward, so a second concurrent drain adds nothing but races. self._drain_in_flight = False @@ -623,6 +644,10 @@ def __init__( # or a dedup hit) or ``False`` (a handler raised). The drain reads it # before moving the ack cursor β€” see :meth:`drain_offline_envelopes`. self._drain_settlements: dict[int, bool | None] = {} + # Drain rows of the current page that adopted an in-flight first copy + # instead of being dispatched, by envelope identity: the cursor reads + # that copy's outcome without ever waiting for it. + self._drain_adopted: dict[int, asyncio.Future[bool]] = {} self._handlers: dict[str, set[MessageHandler]] = {} self._error_handlers: set[ErrorHandler] = set() @@ -990,6 +1015,14 @@ async def _recv_loop(self) -> None: self._spawn(self.drain_offline_envelopes()) continue + if self._disposed: + # disconnect() has begun: take nothing new in. A frame + # is neither acked nor dispatched, so the server keeps it + # for the next connection, and the shutdown flush works + # on a fixed set of parked frames instead of chasing an + # open socket. + continue + frame_type = message.get("type") if frame_type == "message.new": if self._opts.ack_on_receipt: @@ -1079,6 +1112,7 @@ async def _recv_loop(self) -> None: if self._opts.ack_on_receipt: await self._flush_parked_frames("connection_closed") self._reset_order_states() + self._generation += 1 if not self._disposed: _log.debug( "realtime socket closed (code=%s reason=%r ours=%s handshake_done=%s)", @@ -1134,6 +1168,11 @@ async def drain_offline_envelopes(self) -> None: - A row the ordering layer dropped as a below-anchor duplicate is covered only when this session dispatched it (dedup cache hit); otherwise nothing ever showed it to the application. + - A row whose message another copy is still handling on this + connection (the server's reconnect backlog brings the same rows) is + not dispatched again, and the drain never waits for that copy: the + row is covered only if the copy already finished cleanly; otherwise + the cursor stops there and the next drain covers it as a dedup hit. - A row that fails minimal validation stops the drain: the clean prefix before it is processed; the cursor never crosses it. @@ -1149,6 +1188,7 @@ async def drain_offline_envelopes(self) -> None: finally: self._drain_in_flight = False self._drain_settlements = {} + self._drain_adopted = {} async def _run_drain(self, client: AsyncAgentChatClient) -> None: after: str | None = None @@ -1185,6 +1225,7 @@ async def _run_drain(self, client: AsyncAgentChatClient) -> None: {"type": "message.new", "payload": row} for row in prefix ] self._drain_settlements = {id(envelope): None for envelope in envelopes} + self._drain_adopted = {} routed: list[_Routed] = [] for envelope in envelopes: if self._disposed: @@ -1217,6 +1258,7 @@ async def _run_drain(self, client: AsyncAgentChatClient) -> None: ) return self._drain_settlements = {} + self._drain_adopted = {} if invalid_index is not None: await self._emit_error( @@ -1248,6 +1290,13 @@ async def _run_drain(self, client: AsyncAgentChatClient) -> None: def _drain_row_ackable(self, envelope: dict[str, Any], outcome: _Routed) -> bool: """May the sync/ack cursor cover this drain row? See :meth:`drain_offline_envelopes`.""" + adopted = self._drain_adopted.get(id(envelope)) + if adopted is not None: + # Another copy of this message is (or was) with the handlers. + # Covered only if it already finished cleanly; never waited for. + # Otherwise the cursor stops and the next drain re-offers the row, + # a dedup hit by then. + return adopted.done() and not adopted.cancelled() and adopted.result() settled = self._drain_settlements.get(id(envelope)) if settled is not None: # Handed to handlers during this page (possibly after a stint in @@ -1620,22 +1669,12 @@ async def _deliver_message_new( await self._send_delivery_ack(message_id) return self._settle_drain_row(message, True) - first = self._inflight.get(message_id) if message_id is not None else None - if first is not None and message_id is not None: - # The same message is still with the handlers. Take that - # outcome instead of dispatching it twice: success covers this - # copy too (acked like a dedup hit), a failure leaves it unacked - # so the server re-offers it. The caller waits exactly where it - # would otherwise have waited for its own handler call; shielded - # so that cancelling this caller never cancels the first copy. - ok = await asyncio.shield(first) - if ok and ws_ack: - await self._send_delivery_ack(message_id) - return self._settle_drain_row(message, ok) + if self._adopt_in_flight(message, ws_ack=ws_ack): + return False - own: asyncio.Future[bool] | None = None + own: _InFlight | None = None if message_id is not None and self._dedup.enabled: - own = asyncio.get_running_loop().create_future() + own = _InFlight(asyncio.get_running_loop().create_future(), self._generation) self._inflight[message_id] = own dispatched_cleanly = False try: @@ -1649,26 +1688,40 @@ async def _deliver_message_new( if own is not None and message_id is not None: if self._inflight.get(message_id) is own: del self._inflight[message_id] - if not own.done(): - own.set_result(dispatched_cleanly) + if not own.outcome.done(): + own.outcome.set_result(dispatched_cleanly) if not dispatched_cleanly: return self._settle_drain_row(message, False) if message_id is not None and ws_ack: await self._send_delivery_ack(message_id) return self._settle_drain_row(message, True) - def _ack_when_handled(self, message_id: str) -> None: - """Acknowledge ``message_id`` once its in-flight first copy finishes - cleanly, without waiting for it here.""" + def _adopt_in_flight(self, message: dict[str, Any], *, ws_ack: bool) -> bool: + """Take the outcome of a copy of this message whose handlers are still + running on this connection, instead of dispatching it again. + + Never waits: progress beats dedup. A live copy is acknowledged once + the first copy finished cleanly (``ws_ack``), never if it failed; a + drain row's cursor reads that outcome when it is computed and stops + if the first copy has not finished. Returns ``True`` when adopted. + """ + message_id = _extract_message_id(message) + if message_id is None: + return False first = self._inflight.get(message_id) - if first is None: - return + if first is None or first.generation != self._generation: + return False + if id(message) in self._drain_settlements: + self._drain_adopted[id(message)] = first.outcome + if ws_ack: - def done(future: asyncio.Future[bool]) -> None: - if not future.cancelled() and future.result() and not self._disposed: + def ack_if_handled(outcome: asyncio.Future[bool]) -> None: + if outcome.cancelled() or not outcome.result() or self._disposed: + return self._spawn(self._send_delivery_ack(message_id)) - first.add_done_callback(done) + first.outcome.add_done_callback(ack_if_handled) + return True def _settle_drain_row(self, message: dict[str, Any], ok: bool) -> bool: key = id(message) @@ -1772,17 +1825,10 @@ async def _process_ordered_message( # the earlier dispatch is the proof, so acknowledge it # again, as the dedup path does for every other repeat. await self._send_delivery_ack(message_id) - elif message_id is not None and message_id in self._inflight: + else: # A repeat of a message whose first copy is still with - # the handlers. A drain row takes that outcome for its - # cursor (the drain waits on handlers anyway); a live - # frame never holds up the socket for it and is acked - # once the first copy finished cleanly. - if id(message) in self._drain_settlements: - await self._deliver_message_new(message, ws_ack=False) - return "delivered" - if ws_ack: - self._ack_when_handled(message_id) + # the handlers takes that outcome, without waiting. + self._adopt_in_flight(message, ws_ack=ws_ack) return "duplicate" if seen: return "duplicate" @@ -1790,6 +1836,10 @@ async def _process_ordered_message( return "delivered" if seq == state.next_expected_seq: + if self._adopt_in_flight(message, ws_ack=ws_ack): + # The first copy is this same seq on this connection; its + # caller moves the cursor on once its handler finished. + return "delivered" await self._deliver_message_new(message, ws_ack=ws_ack) # Never backwards: another copy of this message (or a drain of # the buffer) may have moved the cursor on during the await. @@ -1800,7 +1850,7 @@ async def _process_ordered_message( return "delivered" # seq > next_expected_seq β€” out of order. Buffer + arm gap timer. - state.buffer[seq] = _PendingEnvelope(message=message, ws_ack=ws_ack) + state.buffer[seq] = _PendingEnvelope(message=message, ws_ack=ws_ack, acked=ack_spent) if len(state.buffer) > _MAX_BUFFERED_PER_CONVERSATION: await self._resolve_gap( @@ -1813,15 +1863,18 @@ async def _process_ordered_message( return "buffered" if state.gap_task is None: - state.gap_started_at = time.monotonic() - state.gap_started_expected_seq = state.next_expected_seq - # Owned through _bg_tasks as well: the handler clears - # `state.gap_task` once the timer has fired, but the same task - # then runs the gap-fill fetch, and disconnect() must still be - # able to cancel and await it mid-fetch. - state.gap_task = self._spawn(self._gap_timer(conversation_id)) + self._open_gap(conversation_id, state) return "buffered" + def _open_gap(self, conversation_id: str, state: _OrderState) -> None: + state.gap_started_at = time.monotonic() + state.gap_started_expected_seq = state.next_expected_seq + # Owned through _bg_tasks as well: the handler clears + # `state.gap_task` once the timer has fired, but the same task + # then runs the gap-fill fetch, and disconnect() must still be + # able to cancel and await it mid-fetch. + state.gap_task = self._spawn(self._gap_timer(conversation_id)) + async def _gap_timer(self, conversation_id: str) -> None: try: await asyncio.sleep(_GAP_FILL_WINDOW_S) @@ -1908,7 +1961,9 @@ async def _handle_gap_timer(self, conversation_id: str) -> None: if fetched_id is not None: await self._send_delivery_ack(fetched_id) state.buffer[row_seq] = _PendingEnvelope( - message=envelope, ws_ack=not self._opts.ack_on_receipt + message=envelope, + ws_ack=not self._opts.ack_on_receipt, + acked=self._opts.ack_on_receipt, ) drained = await self._drain_consecutive(conversation_id, state) @@ -1966,23 +2021,40 @@ async def _resolve_gap( else 0 ) - # Take each frame out of the buffer before handing it over, lowest - # seq first, until the buffer is empty. The receive loop keeps - # running while a handler is awaited here: a frame that parks - # meanwhile is handed over too, never cleared unseen (under - # ack_on_receipt it is already acked and would be lost), and an entry - # another caller took in the meantime is simply not here any more. - while state.buffer: - seq = min(state.buffer) - pending = state.buffer.pop(seq) - await self._deliver_message_new(pending.message, ws_ack=pending.ws_ack) - _advance_past(state, seq) + # Hand over the frames parked when the gap resolved. Each leaves the + # buffer, and the cursor moves past it, before its handler is + # awaited: the receive loop keeps running meanwhile, and a frame it + # gets in order is dispatched there directly, while one behind a new + # hole parks with a gap window of its own (below). Nothing is + # cleared: under ack_on_receipt a parked frame is already acked. An + # entry another caller took meanwhile is skipped. + entry_gap_task = state.gap_task + state.resolving += 1 + try: + for seq in sorted(state.buffer): + await self._hand_over(state, seq) + # Frames that parked behind those and are now next in line. + await self._hand_over_in_order(state) + finally: + state.resolving -= 1 - if state.gap_task is not None: - state.gap_task.cancel() - state.gap_task = None - state.gap_started_at = None - state.gap_started_expected_seq = None + if ( + state.buffer + and state.gap_task is not None + and state.gap_task is not entry_gap_task + ): + # A newer gap opened while handlers ran; its window, fill and + # report are its own. + pass + else: + if state.gap_task is not None: + state.gap_task.cancel() + state.gap_task = None + state.gap_started_at = None + state.gap_started_expected_seq = None + if state.buffer: + # Still parked behind a hole that opened meanwhile. + self._open_gap(conversation_id, state) if self._opts.on_sequence_gap is not None: info = SequenceGapInfo( @@ -1997,6 +2069,26 @@ async def _resolve_gap( self._cleanup_if_idle(conversation_id, state) + async def _hand_over(self, state: _OrderState, seq: int) -> None: + """Take the frame parked at ``seq`` (if still there), move the cursor + past it, then hand it over.""" + pending = state.buffer.pop(seq, None) + if pending is None: + return + _advance_past(state, seq) + await self._deliver_message_new(pending.message, ws_ack=pending.ws_ack) + + async def _hand_over_in_order(self, state: _OrderState) -> None: + """Hand over the parked frames that are next in line. The cursor + moves before each handler is awaited, so a frame arriving in order + meanwhile goes to the receive loop rather than lengthening this + loop.""" + while True: + seq = state.next_expected_seq + if seq is None or seq not in state.buffer: + return + await self._hand_over(state, seq) + def _maybe_clear_gap_timer(self, state: _OrderState) -> None: if state.gap_task is not None and not state.buffer: state.gap_task.cancel() @@ -2009,6 +2101,7 @@ def _cleanup_if_idle(self, conversation_id: str, state: _OrderState) -> None: not state.buffer and state.gap_task is None and not state.gap_fill_in_flight + and state.resolving == 0 # A state a socket close replaced while this caller was awaiting # a handler is not the conversation's state any more. and self._order_states.get(conversation_id) is state @@ -2030,40 +2123,39 @@ async def _flush_parked_frames(self, reason: GapReason) -> None: close (the frames are already acked). No gap-fill: an in-flight HTTP request would race the close. """ - # Keep going until no conversation has a parked frame: handlers are - # awaited here and the receive loop may still be running (disconnect - # flushes before it closes the socket), so a frame can park in any - # conversation meanwhile. Clearing it unseen would lose it once it is - # acked at receipt. + # Works on the frames parked now. On a close the receive loop is the + # caller, so no frame arrives; disconnect() stops taking frames in + # before it flushes. Each frame leaves the buffer before its handler + # is awaited, so one another caller took meanwhile is skipped. for state in self._order_states.values(): if state.gap_task is not None: state.gap_task.cancel() state.gap_task = None - while True: - parked = [(cid, st) for cid, st in self._order_states.items() if st.buffer] - if not parked: - break - for conversation_id, state in parked: - await self._flush_conversation(conversation_id, state, reason) + for conversation_id, state in list(self._order_states.items()): + seqs = sorted(state.buffer) + for seq in seqs: + await self._hand_over(state, seq) + if seqs: + await self._report_flush(conversation_id, state, seqs[0], reason) + # A frame acked at receipt is never dropped: if one parked during the + # awaits above (it cannot keep happening: nothing new is acked now), + # hand it over too. The rest are unacked REST rows the drain routed + # meanwhile; the server re-offers them. + for state in list(self._order_states.values()): + for seq in sorted(state.buffer): + pending = state.buffer.get(seq) + if pending is not None and pending.acked: + await self._hand_over(state, seq) for state in self._order_states.values(): if state.gap_task is not None: state.gap_task.cancel() state.gap_task = None self._order_states.clear() - async def _flush_conversation( - self, conversation_id: str, state: _OrderState, reason: GapReason + async def _report_flush( + self, conversation_id: str, state: _OrderState, first_seq: int, reason: GapReason ) -> None: - if state.gap_task is not None: - state.gap_task.cancel() - state.gap_task = None - seqs: list[int] = [] - while state.buffer: - seq = min(state.buffer) - pending = state.buffer.pop(seq) - seqs.append(seq) - await self._deliver_message_new(pending.message, ws_ack=pending.ws_ack) - if not seqs or self._opts.on_sequence_gap is None: + if self._opts.on_sequence_gap is None: return if state.gap_started_expected_seq is not None: expected_seq = state.gap_started_expected_seq @@ -2074,7 +2166,7 @@ async def _flush_conversation( info = SequenceGapInfo( conversation_id=conversation_id, expected_seq=expected_seq, - buffered_seq=seqs[0], + buffered_seq=first_seq, gap_ms=( int((time.monotonic() - state.gap_started_at) * 1000) if state.gap_started_at is not None diff --git a/tests/test_realtime_engine.py b/tests/test_realtime_engine.py index a1c2196..55dc497 100644 --- a/tests/test_realtime_engine.py +++ b/tests/test_realtime_engine.py @@ -8,6 +8,7 @@ import asyncio import json +import time from typing import Any import pytest @@ -430,10 +431,16 @@ def _row(seq: int, *, mid: str | None = None) -> dict[str, Any]: @pytest.mark.parametrize("ack_on_receipt", [False, True]) @pytest.mark.asyncio -async def test_a_drain_row_repeat_of_a_running_live_message_is_not_dispatched_again( +async def test_a_drain_repeat_of_a_running_message_is_not_dispatched_and_never_waited_for( ack_on_receipt: bool, ) -> None: - api = MockAsyncClient(sync_pages=[[_row(2)]]) + # The live copy of seq 2 has a slow handler. The drain brings the same + # row and, after it, a row of another conversation. The repeat is not + # dispatched, the drain does not wait for the live handler (the other + # conversation's row is handled meanwhile and the drain finishes), and + # the cursor stops at the repeat; the next drain covers it as a dedup hit. + other = {**_row(1, mid="msg_d1"), "conversation_id": "d", "delivery_id": "del_d1"} + api = MockAsyncClient(sync_pages=[[_row(2), other], [_row(2), other]]) rt, ws = await connected( capabilities=["ack"], client=api, auto_drain_on_connect=False, ack_on_receipt=ack_on_receipt ) @@ -450,22 +457,80 @@ async def slow(m: dict[str, Any]) -> None: await ws.push(message_new("c", 1)) await ws.push(message_new("c", 2)) # in order; its handler is slow await _settle() - drain = asyncio.create_task(rt.drain_offline_envelopes()) - await _settle() - assert calls == ["msg_1", "msg_2"] + await asyncio.wait_for(rt.drain_offline_envelopes(), timeout=1.0) + assert calls == ["msg_1", "msg_2", "msg_d1"] assert api.sync_ack_calls == [] release.set() - await asyncio.wait_for(drain, timeout=1.0) await _settle() - assert calls == ["msg_1", "msg_2"] - assert api.sync_ack_calls == ["del_2"] assert _ack_frames(ws) == ["msg_1", "msg_2"] + await asyncio.wait_for(rt.drain_offline_envelopes(), timeout=1.0) + assert calls == ["msg_1", "msg_2", "msg_d1"] + assert api.sync_ack_calls == ["del_d1"] + finally: + release.set() + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_a_live_repeat_of_a_first_copy_that_failed_is_never_acked() -> None: + # The first copy is a drain row whose handler fails; the server's + # backlog push brings the same message live, below the anchor, while it + # runs. The repeat adopts the first copy, so it must not be acked. + api = MockAsyncClient(sync_pages=[[_row(1)]]) + rt, ws = await connected(capabilities=["ack"], client=api, auto_drain_on_connect=False) + release = asyncio.Event() + + async def fails(_m: dict[str, Any]) -> None: + await release.wait() + raise RuntimeError("model call failed") + + rt.on("message.new", fails) + try: + drain = asyncio.create_task(rt.drain_offline_envelopes()) + await _settle() + await ws.push(message_new("c", 1)) + await _settle() + release.set() + await asyncio.wait_for(drain, timeout=1.0) + await _settle() + assert _ack_frames(ws) == [] + assert api.sync_ack_calls == [] finally: release.set() await rt.disconnect() +@pytest.mark.asyncio +async def test_a_copy_on_a_later_connection_is_dispatched_afresh_while_the_first_runs() -> None: + # Adoption is per connection: a handler that never finishes must not + # swallow every re-offer of its message. + rt, _ws = await connected(capabilities=["ack"]) + entered = asyncio.Event() + calls: list[str] = [] + + async def first_hangs(m: dict[str, Any]) -> None: + calls.append(m["payload"]["id"]) + if len(calls) == 1: + entered.set() + await asyncio.Event().wait() + + rt.on("message.new", first_hangs) + first = asyncio.create_task(rt._deliver_message_new(message_new("c", 1), ws_ack=False)) + try: + await entered.wait() + repeat = rt._deliver_message_new(message_new("c", 1), ws_ack=False) + assert await asyncio.wait_for(repeat, timeout=1.0) is False + assert calls == ["msg_1"] # same connection: adopted, not waited for + rt._generation += 1 # what a socket close does + later = rt._deliver_message_new(message_new("c", 1), ws_ack=False) + assert await asyncio.wait_for(later, timeout=1.0) is True + assert calls == ["msg_1", "msg_1"] + finally: + first.cancel() + await rt.disconnect() + + @pytest.mark.asyncio async def test_a_live_repeat_below_the_anchor_never_holds_up_the_socket() -> None: # The first copy is a drain row whose handler is slow. The server's @@ -543,8 +608,8 @@ async def flaky(m: dict[str, Any]) -> None: @pytest.mark.asyncio -async def test_a_cancelled_first_copy_releases_the_repeat_as_not_handled() -> None: - rt, _ws = await connected(capabilities=["ack"]) +async def test_a_cancelled_first_copy_counts_as_not_handled() -> None: + rt, ws = await connected(capabilities=["ack"]) entered = asyncio.Event() async def hangs(_m: dict[str, Any]) -> None: @@ -554,13 +619,12 @@ async def hangs(_m: dict[str, Any]) -> None: rt.on("message.new", hangs) try: frame = message_new("c", 1) - first = asyncio.create_task(rt._deliver_message_new(frame, ws_ack=False)) + first = asyncio.create_task(rt._deliver_message_new(frame, ws_ack=True)) await entered.wait() - repeat = asyncio.create_task(rt._deliver_message_new(dict(frame), ws_ack=False)) - await _settle() - assert not repeat.done() + assert rt._adopt_in_flight(dict(frame), ws_ack=True) is True first.cancel() - assert await asyncio.wait_for(repeat, timeout=1.0) is False + await _settle() + assert _ack_frames(ws) == [] assert rt._inflight == {} finally: await rt.disconnect() @@ -706,11 +770,10 @@ async def slow(m: dict[str, Any]) -> None: @pytest.mark.asyncio -async def test_disconnect_dispatches_frames_that_park_during_its_own_flush() -> None: - # disconnect() flushes parked frames while the socket is still open; a - # frame that parks during that flush (the handler for an earlier one is - # awaited) was cleared with the ordering state. Acked at receipt, it - # must reach the handler. +async def test_disconnect_takes_no_frame_in_once_it_began() -> None: + # disconnect() flushes parked frames while the socket is still open. A + # frame that arrives meanwhile is neither acked nor dispatched (the + # server keeps it), so the flush ends on a fixed set. rt, ws = await connected(capabilities=["ack"], ack_on_receipt=True) release = asyncio.Event() entered = asyncio.Event() @@ -732,7 +795,102 @@ async def slow(m: dict[str, Any]) -> None: await _settle() release.set() await asyncio.wait_for(stopping, timeout=3.0) - assert seen == [1, 3, 6] + assert seen == [1, 3] + assert _ack_frames(ws) == ["msg_1", "msg_3"] + + +@pytest.mark.parametrize("ack_on_receipt", [False, True]) +@pytest.mark.asyncio +async def test_disconnect_returns_promptly_while_frames_keep_arriving(ack_on_receipt: bool) -> None: + rt, ws = await connected(capabilities=["ack"], ack_on_receipt=ack_on_receipt) + + async def slow(_m: dict[str, Any]) -> None: + await asyncio.sleep(0.03) + + rt.on("message.new", slow) + await ws.push(message_new("c", 1)) + await asyncio.sleep(0.05) + await ws.push(message_new("c", 3)) # parks on the hole at 2 + await _settle() + started = time.monotonic() + stopping = asyncio.create_task(rt.disconnect()) + seq = 4 + while not stopping.done() and time.monotonic() - started < 2.0: + await ws.push(message_new("c", seq)) + seq += 1 + await asyncio.sleep(0.01) + await asyncio.wait_for(stopping, timeout=3.0) + assert time.monotonic() - started < 0.5 + + +@pytest.mark.asyncio +async def test_a_handler_that_disconnects_while_a_repeat_of_its_message_arrives_returns( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("agentchatme._realtime._GAP_FILL_WINDOW_S", 0.05) + rt, ws = await connected(capabilities=["ack"]) + entered = asyncio.Event() + go = asyncio.Event() + returned = asyncio.Event() + + async def h(m: dict[str, Any]) -> None: + if m["payload"]["seq"] == 3: + entered.set() + await go.wait() + await rt.disconnect() + returned.set() + + rt.on("message.new", h) + await ws.push(message_new("c", 1)) + await ws.push(message_new("c", 3)) + await asyncio.wait_for(entered.wait(), timeout=1.0) # the lapsed gap handed 3 over + await ws.push(message_new("c", 3)) # the server repeats it while its handler runs + await _settle() + go.set() + await asyncio.wait_for(returned.wait(), timeout=2.0) + assert rt.state == "stopped" + assert rt._inflight == {} + + +@pytest.mark.asyncio +async def test_a_frame_behind_a_new_hole_gets_its_own_gap_window( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # While a lapsed gap's handler for seq 3 runs, seq 4 arrives in order and + # seq 6 behind a new hole. Seq 4 is dispatched at once by the receive + # loop; seq 6 waits for its own gap window and its own report. + monkeypatch.setattr("agentchatme._realtime._GAP_FILL_WINDOW_S", 0.1) + gaps: list[Any] = [] + rt, ws = await connected(capabilities=["ack"], on_sequence_gap=gaps.append) + release = asyncio.Event() + entered = asyncio.Event() + seen: list[int] = [] + + async def slow(m: dict[str, Any]) -> None: + seen.append(m["payload"]["seq"]) + if m["payload"]["seq"] == 3: + entered.set() + await release.wait() + + rt.on("message.new", slow) + try: + await ws.push(message_new("c", 1)) + await ws.push(message_new("c", 3)) + await asyncio.wait_for(entered.wait(), timeout=1.0) + await ws.push(message_new("c", 4)) + await ws.push(message_new("c", 6)) + await _settle() + assert seen == [1, 3, 4] + release.set() + await _settle() + assert seen == [1, 3, 4] + assert [g.expected_seq for g in gaps] == [2] + await asyncio.sleep(0.3) + assert seen == [1, 3, 4, 6] + assert [g.expected_seq for g in gaps] == [2, 5] + finally: + release.set() + await rt.disconnect() # ─────────────── a guest in a host process ─────────────── From 5e7b5f0d0ef37ac953f4351b5f62a022d134d8a9 Mon Sep 17 00:00:00 2001 From: San Date: Sat, 7 Nov 2026 16:27:26 -0800 Subject: [PATCH 7/7] fix(realtime): a gap fill acks nothing once disconnect() began; an adopted in-order repeat moves the cursor on A gap fill whose fetch returned while disconnect() was flushing acked its rows at receipt (ack_on_receipt), and the shutdown then cancelled the gap task, possibly mid-handover: acked rows never handled. Once disconnect() has begun, the fill now returns before acking (a gap fill marks nothing on the server, which keeps the rows). A live repeat that adopted a drain row still running its handler left the cursor where it was, so the next live frame parked and opened a spurious gap (a REST gap fill and a report 2 s later, though nothing was missing). The in-order branch now moves the cursor on for an adopted copy exactly as for a dispatch. Docs: handlers in one conversation can overlap (they start in seq order), as in the TypeScript SDK. --- CHANGELOG.md | 9 ++++ README.md | 2 + src/agentchatme/_realtime.py | 16 ++++--- tests/test_realtime_engine.py | 87 +++++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50bea90..ad5524d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,15 @@ major. The on-the-wire API is versioned separately under `/v1/...`. ### Changed +- Handlers in one conversation can now overlap, as they already do in the + TypeScript SDK: they still *start* in seq order, but while a resolved gap's + frames are handed over, a frame that arrives in order is dispatched at once + (seq 4 can start while seq 3's handler runs), and a repeat that adopts a + running first copy moves the conversation on without waiting for it. Before, + the next frame waited (or, behind a gap resolution, could be lost). A + handler that needs one-at-a-time processing per conversation should queue + its own work. A gap fill that returns after `disconnect()` began acks + nothing (its rows stay with the server). - Docs: a `reply` verdict's `record()` is called at the **end of a turn that completed** β€” never before the turn, never at its start. A completed turn that looked counts as read even if it chose not to send; a crashed turn, diff --git a/README.md b/README.md index b81eeda..2c37cb5 100644 --- a/README.md +++ b/README.md @@ -474,6 +474,8 @@ When the realtime feed sees a per-conversation seq gap (e.g. `seq=8` arrives, th Without a `client` option, gap recovery is disabled and `recovered=False` is reported whenever a gap is detected. +Handlers are *started* in seq order per conversation, but two of them in one conversation can run at the same time, as in the TypeScript SDK: while a resolved gap's frames are being handed over, a frame that arrives in order is dispatched by the receive loop at once (seq 4 can start while seq 3's handler still runs), and a repeat of a message whose handler is still running moves the conversation on without waiting for it. A handler that needs one-at-a-time processing per conversation should queue its own work. + ### Offline drain After every `hello.ok`, the client pages `/v1/messages/sync` (a bare array of rows), dispatches each row through the same `message.new` handlers, then acknowledges with `/v1/messages/sync/ack` using the last non-empty `delivery_id` string cursor of the rows that were actually handed to your handlers. The cursor commits everything at-or-before it, so it only ever covers rows whose fate is known: a row that fails minimal validation stops the drain at the clean prefix; a row whose handler raised freezes the cursor before it; a row parked in the out-of-order buffer (waiting on a seq gap) is not acked until it is dispatched; a row the ordering layer dropped as a duplicate is covered only when this session already dispatched it. Anything left unacked is re-offered on the next drain and absorbed by the dedup cache. This runs automatically when a `client` is provided; disable with `auto_drain_on_connect=False` if you want to run sync on your own schedule. diff --git a/src/agentchatme/_realtime.py b/src/agentchatme/_realtime.py index 1b93488..cc9606d 100644 --- a/src/agentchatme/_realtime.py +++ b/src/agentchatme/_realtime.py @@ -1836,10 +1836,10 @@ async def _process_ordered_message( return "delivered" if seq == state.next_expected_seq: - if self._adopt_in_flight(message, ws_ack=ws_ack): - # The first copy is this same seq on this connection; its - # caller moves the cursor on once its handler finished. - return "delivered" + # Dispatched, or adopted by a copy of this message whose handler + # is still running (never waited for). Either way the cursor moves + # on now, so the next frame is not taken for a gap; that frame's + # handler may then start while the first copy's still runs. await self._deliver_message_new(message, ws_ack=ws_ack) # Never backwards: another copy of this message (or a drain of # the buffer) may have moved the cursor on during the await. @@ -1925,9 +1925,13 @@ async def _handle_gap_timer(self, conversation_id: str) -> None: finally: state.gap_fill_in_flight = False - # State may have been reset (disconnect during the await). + # State may have been reset (a close during the await), or + # disconnect() may have begun: then take nothing in. The fetched rows + # are neither acked nor handed over (a gap fill marks nothing on the + # server, which keeps them); acking them now would lose them when the + # shutdown cancels this task mid-handover. state_now = self._order_states.get(conversation_id) - if state_now is not state: + if self._disposed or state_now is not state: return if fill_error: diff --git a/tests/test_realtime_engine.py b/tests/test_realtime_engine.py index 55dc497..4d2afe7 100644 --- a/tests/test_realtime_engine.py +++ b/tests/test_realtime_engine.py @@ -852,6 +852,93 @@ async def h(m: dict[str, Any]) -> None: assert rt._inflight == {} +@pytest.mark.asyncio +async def test_a_gap_fill_that_returns_during_disconnect_acks_nothing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Under ack_on_receipt a gap fill acks its rows as it receives them. If + # the fetch returns while disconnect() is flushing, the shutdown then + # cancels the gap task, possibly mid-handover: an acked row would never + # be handled. Once disconnect() has begun the fill takes nothing in. + monkeypatch.setattr("agentchatme._realtime._GAP_FILL_WINDOW_S", 0.05) + fetch_gate = asyncio.Event() + fetching = asyncio.Event() + + class _SlowFill(MockAsyncClient): + async def get_messages(self, conversation_id: str, **kwargs: Any) -> list[dict[str, Any]]: + fetching.set() + await fetch_gate.wait() + return [message_new("c", 2)["payload"]] + + rt, ws = await connected( + capabilities=["ack"], client=_SlowFill(), auto_drain_on_connect=False, ack_on_receipt=True + ) + release = asyncio.Event() + entered = asyncio.Event() + + async def slow(m: dict[str, Any]) -> None: + if m["payload"]["seq"] == 3: + entered.set() + await release.wait() + + rt.on("message.new", slow) + await ws.push(message_new("c", 1)) + await ws.push(message_new("c", 3)) + await asyncio.wait_for(fetching.wait(), timeout=1.0) # the gap fill is in flight + stopping = asyncio.create_task(rt.disconnect()) + await asyncio.wait_for(entered.wait(), timeout=1.0) # the flush hands 3 over + fetch_gate.set() # the fetch returns mid-shutdown + await _settle() + release.set() + await asyncio.wait_for(stopping, timeout=3.0) + assert "msg_2" not in _ack_frames(ws) + + +@pytest.mark.asyncio +async def test_an_adopted_in_order_repeat_moves_the_cursor_on( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The drain hands seq 2 over first and its handler is slow; the live + # repeat of seq 2 adopts it. The next live frame (seq 3) must be taken in + # order, not parked behind a gap that does not exist (2 s later: a REST + # gap fill and a gap report although nothing was missing). + monkeypatch.setattr("agentchatme._realtime._GAP_FILL_WINDOW_S", 0.05) + gaps: list[Any] = [] + api = MockAsyncClient(sync_pages=[[_row(2)]]) + rt, ws = await connected( + capabilities=["ack"], client=api, auto_drain_on_connect=False, on_sequence_gap=gaps.append + ) + release = asyncio.Event() + seen: list[int] = [] + + async def slow(m: dict[str, Any]) -> None: + seen.append(m["payload"]["seq"]) + if m["payload"]["seq"] == 2 and seen.count(2) == 1: + await release.wait() + + rt.on("message.new", slow) + try: + await ws.push(message_new("c", 1)) + await _settle() + drain = asyncio.create_task(rt.drain_offline_envelopes()) + await _settle() + assert seen == [1, 2] # the drain row's handler is running + await ws.push(message_new("c", 2)) # live repeat: adopted + await ws.push(message_new("c", 3)) + await _settle() + assert seen == [1, 2, 3] + await asyncio.sleep(0.2) # well past the gap window + assert gaps == [] + assert api.get_messages_calls == [] + release.set() + await asyncio.wait_for(drain, timeout=1.0) + await _settle() + assert sorted(_ack_frames(ws)) == ["msg_1", "msg_2", "msg_3"] + finally: + release.set() + await rt.disconnect() + + @pytest.mark.asyncio async def test_a_frame_behind_a_new_hole_gets_its_own_gap_window( monkeypatch: pytest.MonkeyPatch,