From bedbe9b14a69558c257c74262bb2fc16d8ee0905 Mon Sep 17 00:00:00 2001 From: San Date: Sat, 7 Nov 2026 14:30:18 -0800 Subject: [PATCH 1/2] fix(realtime): ack an already-handled repeat below the seq anchor; engine conformance scenarios Default mode dropped a live message.new below the conversation's seq anchor without an ack, even when this session had already dispatched that message id. The delivery-ack protocol acknowledges every dedup hit (the earlier dispatch is the proof), and the TypeScript SDK does. It is now acked again, still without re-dispatch. A below-anchor frame no handler has seen is unchanged: dropped unacked, the server re-offers it. Adds conformance/v1/engine-scenarios.json, run in both modes by a new harness (identical bytes ship in the TypeScript SDK): drain gap + close during an earlier handler or inside the gap window, a raising handler during a drain, and an already-handled repeat below the live anchor as a frame and as a drain row. Also pins the drain-cursor rule for a socket drop during an earlier handler with a regression test. --- CHANGELOG.md | 21 ++ conformance/v1/engine-scenarios.json | 438 +++++++++++++++++++++++++++ src/agentchatme/_realtime.py | 13 +- tests/test_conformance.py | 12 +- tests/test_conformance_engine.py | 229 ++++++++++++++ tests/test_realtime.py | 48 +++ tests/test_realtime_engine.py | 45 +++ 7 files changed, 801 insertions(+), 5 deletions(-) create mode 100644 conformance/v1/engine-scenarios.json create mode 100644 tests/test_conformance_engine.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d9c10dd..ecdb057 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,27 @@ major. The on-the-wire API is versioned separately under `/v1/...`. ## [Unreleased] +### Fixed + +- In the default mode, a live `message.new` below the conversation's seq + anchor that this session had already handled (the server sent it again: + a drain/live overlap or a lost ack) was dropped without an ack. The + delivery-ack protocol acknowledges every dedup hit, because the earlier + dispatch is the proof; the frame is now acked again, still without + re-dispatching it. A below-anchor frame no handler has seen is unchanged: + dropped unacked, so the server re-offers it. Matches the TypeScript SDK. + +### Added + +- `conformance/v1/engine-scenarios.json` — the realtime engine's delivery + rules as shared scenarios, run in both modes by both SDKs: a drain page + with a gap and a socket drop during an earlier handler or inside the gap + window, a handler that raises during a drain, and a repeat of an + already-handled message below the live anchor (as a frame and as a drain + row). Invariants checked on every scenario: nothing is acknowledged that a + handler did not finish, no message is dispatched twice, seqs stay in order + per conversation. Identical bytes ship in the TypeScript SDK. + ### Changed - Docs: a `reply` verdict's `record()` is called at the **end of a turn that diff --git a/conformance/v1/engine-scenarios.json b/conformance/v1/engine-scenarios.json new file mode 100644 index 0000000..3888756 --- /dev/null +++ b/conformance/v1/engine-scenarios.json @@ -0,0 +1,438 @@ +{ + "about": "Realtime engine scenarios every AgentChat SDK runs against its own RealtimeClient: a mock socket, a fake REST client and one recording message.new handler. Each scenario runs twice, once per mode: 'default' (ack after every handler finished) and 'ack_on_receipt'. The REST client serves sync_pages in order, then empty pages; sync/ack records the cursor; gap-fill returns nothing. The automatic drain on hello.ok is off and reconnect is off, so only the steps below act. A message is 'handled' when its handler finished without raising; a sync/ack cursor 'covers' every row served so far up to and including the row carrying that delivery_id. Results are read after the last step and any started drain finished, before the client is disconnected (disconnect flushes parked frames).", + "expectations": { + "cursor_acks": "Exactly these delivery_ids were passed to sync/ack, in order.", + "dispatched": "Exactly these message ids reached the handler, in this order.", + "dispatched_includes": "These message ids reached the handler (others may too).", + "ws_acks": "Exactly these message ids were sent in ack frames, in order." + }, + "invariants": { + "acked_was_handled": "At the end, every message the client acknowledged (an ack frame, or a row a sync/ack cursor covers) was handled. An ack is a promise the server never re-offers the message.", + "acks_follow_handling": "When each sync/ack cursor is sent (both modes), and each ack frame in the default mode, every message it acknowledges was already handled. Under ack_on_receipt an ack frame is sent at receipt, before the handler.", + "dispatched_once": "No message id reaches the handler twice in these scenarios.", + "seq_order": "Within one conversation the handler sees seqs in ascending order." + }, + "ops": { + "close": "The server side ends the socket with close code 1006 (an abnormal drop).", + "connect": "Create the client for the mode, connect, open the socket and answer HELLO with {\"type\": \"hello.ok\", \"capabilities\": [\"ack\"]}.", + "drain": "Start the client's offline drain (drainOfflineEnvelopes / drain_offline_envelopes) without awaiting it.", + "frame": "The server sends {\"type\": \"message.new\", \"payload\": } on the socket.", + "release": "Let the held handler for message_id finish.", + "wait_handler": "Wait until the handler has been entered for message_id." + }, + "rules": "Every op is followed by letting the client run until it is idle (pending handlers that are held stay pending). The handler blocks on the ids in 'hold' until a 'release' step, and raises for the ids in 'fail'. A harness rejects an op, expectation or invariant name it does not implement.", + "scenarios": [ + { + "expect": { + "ack_on_receipt": { + "cursor_acks": [ + "del_000000000000000000000000000c0003" + ], + "dispatched": [ + "msg_crew0seq0001", + "msg_crew0seq0003" + ], + "ws_acks": [] + }, + "default": { + "dispatched_includes": [ + "msg_crew0seq0001" + ], + "ws_acks": [] + } + }, + "hold": [ + "msg_crew0seq0001" + ], + "name": "drain-gap-close-during-earlier-handler", + "steps": [ + { + "op": "connect" + }, + { + "op": "drain" + }, + { + "message_id": "msg_crew0seq0001", + "op": "wait_handler" + }, + { + "op": "close" + }, + { + "message_id": "msg_crew0seq0001", + "op": "release" + } + ], + "sync_pages": [ + [ + { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0crew0001", + "created_at": "2026-11-04T23:01:00.000Z", + "delivery_id": "del_000000000000000000000000000c0001", + "id": "msg_crew0seq0001", + "sender": "nova", + "seq": 1, + "type": "text" + }, + { + "content": { + "text": "message 3" + }, + "conversation_id": "conv_group0crew0001", + "created_at": "2026-11-04T23:03:00.000Z", + "delivery_id": "del_000000000000000000000000000c0003", + "id": "msg_crew0seq0003", + "sender": "nova", + "seq": 3, + "type": "text" + } + ] + ], + "why": "A group where the agent's own message is seq 2, so the drain page is [1, 3] and seq 3 waits on the gap. The handler for seq 1 is slow (it calls a model) and the socket drops while it runs. Whatever the engine does with seq 3, the cursor may cover it only if a handler finished it: a close that clears a parked row is not a dispatch." + }, + { + "expect": { + "ack_on_receipt": { + "cursor_acks": [ + "del_000000000000000000000000000a0001" + ], + "dispatched": [ + "msg_ops0seq00001", + "msg_ops0seq00003" + ], + "ws_acks": [] + }, + "default": { + "cursor_acks": [ + "del_000000000000000000000000000a0001" + ], + "dispatched": [ + "msg_ops0seq00001" + ], + "ws_acks": [] + } + }, + "name": "drain-gap-close-while-parked", + "steps": [ + { + "op": "connect" + }, + { + "op": "drain" + }, + { + "op": "close" + } + ], + "sync_pages": [ + [ + { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:01:00.000Z", + "delivery_id": "del_000000000000000000000000000a0001", + "id": "msg_ops0seq00001", + "sender": "nova", + "seq": 1, + "type": "text" + }, + { + "content": { + "text": "message 3" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:03:00.000Z", + "delivery_id": "del_000000000000000000000000000a0003", + "id": "msg_ops0seq00003", + "sender": "nova", + "seq": 3, + "type": "text" + } + ] + ], + "why": "Drain page [1, 3] with seq 2 absent; seq 3 parks on the gap and the socket drops inside the gap window. The cursor stops at seq 1. The default mode drops the parked row unacked (the server re-offers it); ack_on_receipt hands parked frames to the handler at close." + }, + { + "expect": { + "ack_on_receipt": { + "cursor_acks": [ + "del_000000000000000000000000000e0001" + ], + "dispatched": [ + "msg_gone0seq0001", + "msg_gone0seq0002", + "msg_gone0seq0003" + ], + "ws_acks": [] + }, + "default": { + "cursor_acks": [ + "del_000000000000000000000000000e0001" + ], + "dispatched": [ + "msg_gone0seq0001", + "msg_gone0seq0002", + "msg_gone0seq0003" + ], + "ws_acks": [] + } + }, + "fail": [ + "msg_gone0seq0002" + ], + "name": "drain-handler-raises", + "steps": [ + { + "op": "connect" + }, + { + "op": "drain" + } + ], + "sync_pages": [ + [ + { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0gone0001", + "created_at": "2026-11-04T23:01:00.000Z", + "delivery_id": "del_000000000000000000000000000e0001", + "id": "msg_gone0seq0001", + "sender": "nova", + "seq": 1, + "type": "text" + }, + { + "content": { + "text": "message 2" + }, + "conversation_id": "conv_group0gone0001", + "created_at": "2026-11-04T23:02:00.000Z", + "delivery_id": "del_000000000000000000000000000e0002", + "id": "msg_gone0seq0002", + "sender": "nova", + "seq": 2, + "type": "text" + }, + { + "content": { + "text": "message 3" + }, + "conversation_id": "conv_group0gone0001", + "created_at": "2026-11-04T23:03:00.000Z", + "delivery_id": "del_000000000000000000000000000e0003", + "id": "msg_gone0seq0003", + "sender": "nova", + "seq": 3, + "type": "text" + } + ] + ], + "why": "The handler raises on the second drain row. The cursor stops before it in both modes, so the server re-offers that row and everything after it." + }, + { + "expect": { + "ack_on_receipt": { + "cursor_acks": [], + "dispatched": [ + "msg_first0seq001", + "msg_first0seq002", + "msg_first0seq003" + ], + "ws_acks": [ + "msg_first0seq001", + "msg_first0seq003", + "msg_first0seq001", + "msg_first0seq002" + ] + }, + "default": { + "cursor_acks": [], + "dispatched": [ + "msg_first0seq001", + "msg_first0seq002", + "msg_first0seq003" + ], + "ws_acks": [ + "msg_first0seq001", + "msg_first0seq001", + "msg_first0seq002", + "msg_first0seq003" + ] + } + }, + "name": "below-anchor-duplicate-frame", + "steps": [ + { + "op": "connect" + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_direct0first01", + "created_at": "2026-11-04T23:01:00.000Z", + "id": "msg_first0seq001", + "sender": "nova", + "seq": 1, + "type": "text" + } + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 3" + }, + "conversation_id": "conv_direct0first01", + "created_at": "2026-11-04T23:03:00.000Z", + "id": "msg_first0seq003", + "sender": "nova", + "seq": 3, + "type": "text" + } + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_direct0first01", + "created_at": "2026-11-04T23:01:00.000Z", + "id": "msg_first0seq001", + "sender": "nova", + "seq": 1, + "type": "text" + } + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 2" + }, + "conversation_id": "conv_direct0first01", + "created_at": "2026-11-04T23:02:00.000Z", + "id": "msg_first0seq002", + "sender": "nova", + "seq": 2, + "type": "text" + } + } + ], + "why": "Seq 1 anchors the conversation and seq 3 parks on the gap, so the ordering state is live. The server then sends seq 1 again (a drain/live overlap or a lost ack): it is below the anchor and this client already handled it, so it is not handed over again, and it is acknowledged again, because the earlier dispatch is the proof. Seq 2 then releases 2 and 3 in order." + }, + { + "expect": { + "ack_on_receipt": { + "cursor_acks": [ + "del_000000000000000000000000000b0001" + ], + "dispatched": [ + "msg_ops0seq00001", + "msg_ops0seq00002", + "msg_ops0seq00003" + ], + "ws_acks": [ + "msg_ops0seq00001", + "msg_ops0seq00003", + "msg_ops0seq00002" + ] + }, + "default": { + "cursor_acks": [ + "del_000000000000000000000000000b0001" + ], + "dispatched": [ + "msg_ops0seq00001", + "msg_ops0seq00002", + "msg_ops0seq00003" + ], + "ws_acks": [ + "msg_ops0seq00001", + "msg_ops0seq00002", + "msg_ops0seq00003" + ] + } + }, + "name": "below-anchor-duplicate-drain-row", + "steps": [ + { + "op": "connect" + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:01:00.000Z", + "id": "msg_ops0seq00001", + "sender": "nova", + "seq": 1, + "type": "text" + } + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 3" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:03:00.000Z", + "id": "msg_ops0seq00003", + "sender": "nova", + "seq": 3, + "type": "text" + } + }, + { + "op": "drain" + }, + { + "op": "frame", + "payload": { + "content": { + "text": "message 2" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:02:00.000Z", + "id": "msg_ops0seq00002", + "sender": "nova", + "seq": 2, + "type": "text" + } + } + ], + "sync_pages": [ + [ + { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0ops00001", + "created_at": "2026-11-04T23:01:00.000Z", + "delivery_id": "del_000000000000000000000000000b0001", + "id": "msg_ops0seq00001", + "sender": "nova", + "seq": 1, + "type": "text" + } + ] + ], + "why": "As above, but the repeat of seq 1 is a REST drain row. It is not handed over again and the cursor covers it: this client already handled that message id. Drain rows are acknowledged by the cursor, never by an ack frame." + } + ] +} diff --git a/src/agentchatme/_realtime.py b/src/agentchatme/_realtime.py index 48d49ea..7f8d549 100644 --- a/src/agentchatme/_realtime.py +++ b/src/agentchatme/_realtime.py @@ -1672,7 +1672,8 @@ async def _process_ordered_message( Returns where the envelope went: ``"delivered"`` (handed to :meth:`_deliver_message_new` now), ``"duplicate"`` (below this - connection's seq anchor, dropped), or ``"buffered"`` (parked on a + connection's seq anchor, dropped; a live frame this session already + dispatched is acknowledged again), or ``"buffered"`` (parked on a seq gap; dispatched later by the gap machinery or the shutdown flush). The REST drain uses this to decide what its ack cursor may cover. @@ -1710,10 +1711,16 @@ async def _process_ordered_message( # rows (seq 3, 4). Unacked, those come back in order later and # are safe to drop here; acked at parse, dropping them would lose # them — deliver on a dedup miss, out of order but not lost. + message_id = _extract_message_id(message) + seen = message_id is not None and self._dedup.hit(message_id) if not ack_spent: + if seen and ws_ack and message_id is not None: + # A repeat of a message this session already handled: + # the earlier dispatch is the proof, so acknowledge it + # again, as the dedup path does for every other repeat. + await self._send_delivery_ack(message_id) return "duplicate" - message_id = _extract_message_id(message) - if message_id is not None and self._dedup.hit(message_id): + if seen: return "duplicate" await self._deliver_message_new(message, ws_ack=False) return "delivered" diff --git a/tests/test_conformance.py b/tests/test_conformance.py index e580edf..99bfd4b 100644 --- a/tests/test_conformance.py +++ b/tests/test_conformance.py @@ -21,6 +21,7 @@ FIXTURES = Path(__file__).resolve().parent.parent / "conformance" / "v1" NAMES = [ "ack-frame.json", + "engine-scenarios.json", "gate-prompt.expected.txt", "tray.sample.json", "verdict-parse.cases.json", @@ -45,12 +46,19 @@ def tray() -> dict[str, Any]: # ─────────────── Canonical form ─────────────── -def test_ships_exactly_the_five_named_files() -> None: +def test_ships_exactly_the_six_named_files() -> None: assert sorted(p.name for p in FIXTURES.iterdir()) == NAMES @pytest.mark.parametrize( - "name", ["tray.sample.json", "verdict-parse.cases.json", "wake-frame.json", "ack-frame.json"] + "name", + [ + "tray.sample.json", + "verdict-parse.cases.json", + "wake-frame.json", + "ack-frame.json", + "engine-scenarios.json", + ], ) def test_json_fixture_has_sorted_keys_two_space_indent_and_one_trailing_newline( name: str, diff --git a/tests/test_conformance_engine.py b/tests/test_conformance_engine.py new file mode 100644 index 0000000..798c167 --- /dev/null +++ b/tests/test_conformance_engine.py @@ -0,0 +1,229 @@ +"""``conformance/v1/engine-scenarios.json``: the realtime engine's delivery +rules (drain cursor, parked frames across a close, below-anchor repeats) as +scenarios both SDKs run byte for byte. + +The fixture holds the steps and the expected outcomes; this file only +interprets them. See the fixture's ``about``, ``ops`` and ``invariants``. +""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import pytest + +from agentchatme import RealtimeClient + +from .test_realtime import MockWebSocket, _make_client + +FIXTURE = Path(__file__).resolve().parent.parent / "conformance" / "v1" / "engine-scenarios.json" +DOC: dict[str, Any] = json.loads(FIXTURE.read_text(encoding="utf-8")) + +MODES = ["default", "ack_on_receipt"] +OPS = ["close", "connect", "drain", "frame", "release", "wait_handler"] +EXPECTATIONS = ["cursor_acks", "dispatched", "dispatched_includes", "ws_acks"] +INVARIANTS = ["acked_was_handled", "acks_follow_handling", "dispatched_once", "seq_order"] + + +async def _idle() -> None: + """Let the client run until nothing is ready to run (held handlers stay + pending): the drain task, the receive loop and their acks.""" + for _ in range(50): + await asyncio.sleep(0) + + +class _RecordingSocket(MockWebSocket): + def __init__(self, events: list[tuple[str, str]]) -> None: + super().__init__() + self._events = events + + async def send(self, data: str) -> None: + frame = json.loads(data) + if frame.get("type") == "ack": + self._events.append(("ws_ack", str(frame.get("message_id")))) + await super().send(data) + + +class _ScenarioApi: + """The REST surface the engine uses: ``sync`` serves the scenario's pages + in order and then empty pages, ``sync_ack`` records the cursor, gap-fill + returns nothing.""" + + def __init__(self, pages: list[list[dict[str, Any]]], events: list[tuple[str, str]]) -> None: + self._pages = [list(page) for page in pages] + self._events = events + self.served: list[dict[str, Any]] = [] + + async def sync(self, **_kwargs: Any) -> list[dict[str, Any]]: + page = self._pages.pop(0) if self._pages else [] + self.served.extend(page) + return page + + async def sync_ack(self, last_delivery_id: str, **_kwargs: Any) -> dict[str, Any]: + self._events.append(("cursor", last_delivery_id)) + return {"acked": 1} + + async def get_messages(self, _conversation_id: str, **_kwargs: Any) -> list[dict[str, Any]]: + return [] + + +@dataclass +class _Outcome: + dispatched: list[dict[str, Any]] = field(default_factory=list) + # ("handled", message_id) | ("ws_ack", message_id) | ("cursor", delivery_id) + events: list[tuple[str, str]] = field(default_factory=list) + served: list[dict[str, Any]] = field(default_factory=list) + + +async def _run(scenario: dict[str, Any], mode: str) -> _Outcome: + out = _Outcome() + api = _ScenarioApi(scenario.get("sync_pages", []), out.events) + holds = {mid: asyncio.Event() for mid in scenario.get("hold", [])} + failing = set(scenario.get("fail", [])) + + async def handler(message: dict[str, Any]) -> None: + payload = message["payload"] + out.dispatched.append(payload) + hold = holds.get(payload["id"]) + if hold is not None: + await hold.wait() + if payload["id"] in failing: + raise RuntimeError(f"scenario handler fails {payload['id']}") + out.events.append(("handled", payload["id"])) + + rt: RealtimeClient | None = None + ws: _RecordingSocket | None = None + drains: list[asyncio.Task[None]] = [] + try: + for step in scenario["steps"]: + op = step["op"] + if op == "connect": + ws = _RecordingSocket(out.events) + rt, _ = _make_client( + ws=ws, + client=api, # type: ignore[arg-type] + auto_drain_on_connect=False, + ack_on_receipt=mode == "ack_on_receipt", + ) + rt.on("message.new", handler) + await rt.connect() + await _idle() + await ws.push({"type": "hello.ok", "capabilities": ["ack"]}) + elif op == "frame": + assert ws is not None + await ws.push({"type": "message.new", "payload": step["payload"]}) + elif op == "drain": + assert rt is not None + drains.append(asyncio.create_task(rt.drain_offline_envelopes())) + elif op == "wait_handler": + for _ in range(100): + if any(p["id"] == step["message_id"] for p in out.dispatched): + break + await asyncio.sleep(0.001) + assert any(p["id"] == step["message_id"] for p in out.dispatched), ( + f"handler entered for {step['message_id']}" + ) + elif op == "close": + assert ws is not None + await ws.close(code=1006, reason="abnormal") + elif op == "release": + holds[step["message_id"]].set() + else: + raise AssertionError(f"unknown op {op}") + await _idle() + if drains: + await asyncio.wait_for(asyncio.gather(*drains), timeout=2.0) + await _idle() + out.served = list(api.served) + return _Outcome(list(out.dispatched), list(out.events), out.served) + finally: + for hold in holds.values(): + hold.set() + if rt is not None: + await rt.disconnect() + + +def _acknowledged(event: tuple[str, str], served: list[dict[str, Any]]) -> list[str]: + """Message ids an event acknowledges: an ack frame names one; a cursor + covers every row served so far up to and including the row carrying it.""" + kind, value = event + if kind == "ws_ack": + return [value] + if kind != "cursor": + return [] + index = next((i for i, row in enumerate(served) if row.get("delivery_id") == value), None) + assert index is not None, f"cursor {value} names a served row" + return [row["id"] for row in served[: index + 1]] + + +def _check_invariants(out: _Outcome, mode: str) -> None: + handled_at: dict[str, int] = {} + for at, (kind, value) in enumerate(out.events): + if kind == "handled": + handled_at.setdefault(value, at) + + for at, event in enumerate(out.events): + for mid in _acknowledged(event, out.served): + # acked_was_handled + assert mid in handled_at, f"{mid} acknowledged but never handled" + # acks_follow_handling + if event[0] == "cursor" or mode == "default": + assert handled_at[mid] < at, f"{mid} acknowledged before it was handled" + + # dispatched_once + ids = [p["id"] for p in out.dispatched] + assert len(set(ids)) == len(ids), f"a message reached the handler twice: {ids}" + + # seq_order + last: dict[str, int] = {} + for p in out.dispatched: + previous = last.get(p["conversation_id"]) + if previous is not None: + assert p["seq"] > previous, f"{p['conversation_id']}: seq {p['seq']} after {previous}" + last[p["conversation_id"]] = p["seq"] + + +def _check_expectations(out: _Outcome, expected: dict[str, Any]) -> None: + ids = [p["id"] for p in out.dispatched] + if "dispatched" in expected: + assert ids == expected["dispatched"] + for mid in expected.get("dispatched_includes", []): + assert mid in ids + if "ws_acks" in expected: + assert [v for k, v in out.events if k == "ws_ack"] == expected["ws_acks"] + if "cursor_acks" in expected: + assert [v for k, v in out.events if k == "cursor"] == expected["cursor_acks"] + + +def test_names_only_ops_expectations_and_invariants_this_harness_implements() -> None: + assert sorted(DOC["ops"]) == OPS + assert sorted(DOC["expectations"]) == EXPECTATIONS + assert sorted(DOC["invariants"]) == INVARIANTS + for scenario in DOC["scenarios"]: + for step in scenario["steps"]: + assert step["op"] in OPS + assert sorted(scenario["expect"]) == sorted(MODES) + for mode in MODES: + for key in scenario["expect"][mode]: + assert key in EXPECTATIONS + + +def test_scenario_names_are_unique() -> None: + names = [s["name"] for s in DOC["scenarios"]] + assert len(set(names)) == len(names) + + +@pytest.mark.parametrize( + ("scenario", "mode"), + [(s, m) for s in DOC["scenarios"] for m in MODES], + ids=[f"{s['name']}-{m}" for s in DOC["scenarios"] for m in MODES], +) +@pytest.mark.asyncio +async def test_engine_scenario(scenario: dict[str, Any], mode: str) -> None: + out = await _run(scenario, mode) + _check_invariants(out, mode) + _check_expectations(out, scenario["expect"][mode]) diff --git a/tests/test_realtime.py b/tests/test_realtime.py index 95f835c..47f4b35 100644 --- a/tests/test_realtime.py +++ b/tests/test_realtime.py @@ -1299,6 +1299,54 @@ async def test_drain_never_acks_a_below_anchor_row_nobody_dispatched() -> None: await rt.disconnect() +@pytest.mark.parametrize("ack_on_receipt", [False, True]) +@pytest.mark.asyncio +async def test_drain_cursor_covers_only_handled_rows_when_the_socket_drops_mid_handler( + ack_on_receipt: bool, +) -> None: + # A group where the agent's own message is seq 2: the page is [1, 3]. + # The handler for seq 1 is slow and the socket drops while it runs. The + # close resets the ordering state; whatever happens to seq 3 afterwards, + # the cursor may cover it only once a handler finished it. + page = [ + _sync_row("m_1", conversation_id="g", delivery_id="del_1", seq=1), + _sync_row("m_3", conversation_id="g", delivery_id="del_3", seq=3), + ] + mock_api = MockAsyncClient(sync_pages=[page]) + rt, ws = _make_client( + client=mock_api, auto_drain_on_connect=False, ack_on_receipt=ack_on_receipt + ) + entered: list[str] = [] + finished: list[str] = [] + release = asyncio.Event() + + async def slow(m: dict[str, Any]) -> None: + entered.append(m["payload"]["id"]) + if m["payload"]["id"] == "m_1": + await release.wait() + finished.append(m["payload"]["id"]) + + rt.on("message.new", slow) + try: + await rt.connect() + await _settle() + await ws.push({"type": "hello.ok", "capabilities": ["ack"]}) + await _settle() + drain = asyncio.create_task(rt.drain_offline_envelopes()) + await _settle() + assert entered == ["m_1"] + await ws.close(code=1006, reason="abnormal") + await _settle() + release.set() + await asyncio.wait_for(drain, timeout=1.0) + + assert mock_api.sync_ack_calls, "the cursor covers at least the handled seq 1" + covered = {"del_1": ["m_1"], "del_3": ["m_1", "m_3"]}[mock_api.sync_ack_calls[-1]] + assert all(mid in finished for mid in covered) + finally: + await rt.disconnect() + + @pytest.mark.asyncio async def test_drain_dispatches_and_acks_nothing_after_disconnect() -> None: gate = asyncio.Event() diff --git a/tests/test_realtime_engine.py b/tests/test_realtime_engine.py index 0ec00ca..492aec0 100644 --- a/tests/test_realtime_engine.py +++ b/tests/test_realtime_engine.py @@ -295,6 +295,51 @@ async def test_ack_on_receipt_flushes_acked_parked_frames_when_the_socket_closes await rt.disconnect() +@pytest.mark.asyncio +async def test_default_mode_acks_a_below_anchor_repeat_it_already_handled() -> None: + # Seq 3 parks, so the ordering state stays live and the repeat of seq 1 + # really is below the anchor. It is not handed over again, and it is + # acknowledged again: the earlier dispatch is the proof (the dedup rule + # of the delivery-ack protocol). + rt, ws = await connected(capabilities=["ack"]) + handled: list[str] = [] + rt.on("message.new", lambda m: handled.append(m["payload"]["id"])) + try: + await ws.push(message_new("c", 1)) + await ws.push(message_new("c", 3)) + await _settle() + await ws.push(message_new("c", 1)) + await _settle() + assert handled == ["msg_1"] + assert _ack_frames(ws) == ["msg_1", "msg_1"] + + await ws.push(message_new("c", 2)) + await _settle() + assert handled == ["msg_1", "msg_2", "msg_3"] + assert _ack_frames(ws) == ["msg_1", "msg_1", "msg_2", "msg_3"] + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_default_mode_never_acks_a_below_anchor_frame_nobody_handled() -> None: + # Live seq 5 anchors before the server's reconnect backlog brings the + # stored seq 3. Unseen and unacked, it stays `stored` for the next drain. + rt, ws = await connected(capabilities=["ack"]) + handled: list[str] = [] + rt.on("message.new", lambda m: handled.append(m["payload"]["id"])) + try: + await ws.push(message_new("c", 5)) + await ws.push(message_new("c", 7)) + await _settle() + await ws.push(message_new("c", 3)) + await _settle() + assert handled == ["msg_5"] + assert _ack_frames(ws) == ["msg_5"] + finally: + await rt.disconnect() + + @pytest.mark.asyncio async def test_default_mode_leaves_unacked_parked_frames_to_the_next_drain() -> None: gaps: list[Any] = [] From bf24ad306402494717fef0902dbfac19d486d925 Mon Sep 17 00:00:00 2001 From: San Date: Sat, 7 Nov 2026 14:52:22 -0800 Subject: [PATCH 2/2] test(conformance): drain gap with a socket drop during a later row's handler Shared scenario (identical bytes in agentchat-typescript): page [g1, g3, h1], h1's handler held, the socket drops, h1 released; the default-mode cursor stops at g1. This drain awaits each handler while routing, so the drop lands during a LATER row's handler and clears the parked g3 before the cursor is computed. Copying a cursor that stops only while a row is still buffered into _drain_row_ackable now fails this scenario and the reworked unit test (the earlier-handler shape did not catch it). CHANGELOG: say where the SDKs still differ on a below-anchor frame no handler has seen. --- CHANGELOG.md | 10 ++-- conformance/v1/engine-scenarios.json | 86 ++++++++++++++++++++++++++++ tests/test_realtime.py | 29 +++++----- 3 files changed, 108 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ecdb057..6815039 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,15 +14,17 @@ major. The on-the-wire API is versioned separately under `/v1/...`. a drain/live overlap or a lost ack) was dropped without an ack. The delivery-ack protocol acknowledges every dedup hit, because the earlier dispatch is the proof; the frame is now acked again, still without - re-dispatching it. A below-anchor frame no handler has seen is unchanged: - dropped unacked, so the server re-offers it. Matches the TypeScript SDK. + re-dispatching it, as the TypeScript SDK does. A below-anchor frame no + handler has seen is unchanged, and there the SDKs still differ: Python + drops it unacked, so it waits for the server to re-offer it on the next + reconnect; TypeScript dispatches it late (out of order) and acks it. ### Added - `conformance/v1/engine-scenarios.json` — the realtime engine's delivery rules as shared scenarios, run in both modes by both SDKs: a drain page - with a gap and a socket drop during an earlier handler or inside the gap - window, a handler that raises during a drain, and a repeat of an + with a gap and a socket drop during an earlier or a later row's handler + or inside the gap window, a handler that raises during a drain, and a repeat of an already-handled message below the live anchor (as a frame and as a drain row). Invariants checked on every scenario: nothing is acknowledged that a handler did not finish, no message is dispatched twice, seqs stay in order diff --git a/conformance/v1/engine-scenarios.json b/conformance/v1/engine-scenarios.json index 3888756..fda9539 100644 --- a/conformance/v1/engine-scenarios.json +++ b/conformance/v1/engine-scenarios.json @@ -94,6 +94,92 @@ ], "why": "A group where the agent's own message is seq 2, so the drain page is [1, 3] and seq 3 waits on the gap. The handler for seq 1 is slow (it calls a model) and the socket drops while it runs. Whatever the engine does with seq 3, the cursor may cover it only if a handler finished it: a close that clears a parked row is not a dispatch." }, + { + "expect": { + "ack_on_receipt": { + "dispatched": [ + "msg_crew0late0001", + "msg_first0late001", + "msg_crew0late0003" + ], + "ws_acks": [] + }, + "default": { + "cursor_acks": [ + "del_000000000000000000000000000f0001" + ], + "dispatched": [ + "msg_crew0late0001", + "msg_first0late001" + ], + "ws_acks": [] + } + }, + "hold": [ + "msg_first0late001" + ], + "name": "drain-gap-close-during-later-handler", + "steps": [ + { + "op": "connect" + }, + { + "op": "drain" + }, + { + "message_id": "msg_first0late001", + "op": "wait_handler" + }, + { + "op": "close" + }, + { + "message_id": "msg_first0late001", + "op": "release" + } + ], + "sync_pages": [ + [ + { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_group0crew0001", + "created_at": "2026-11-04T23:01:00.000Z", + "delivery_id": "del_000000000000000000000000000f0001", + "id": "msg_crew0late0001", + "sender": "nova", + "seq": 1, + "type": "text" + }, + { + "content": { + "text": "message 3" + }, + "conversation_id": "conv_group0crew0001", + "created_at": "2026-11-04T23:03:00.000Z", + "delivery_id": "del_000000000000000000000000000f0003", + "id": "msg_crew0late0003", + "sender": "nova", + "seq": 3, + "type": "text" + }, + { + "content": { + "text": "message 1" + }, + "conversation_id": "conv_direct0first01", + "created_at": "2026-11-04T23:01:00.000Z", + "delivery_id": "del_000000000000000000000000000f0004", + "id": "msg_first0late001", + "sender": "nova", + "seq": 1, + "type": "text" + } + ] + ], + "why": "Drain page [g1, g3, h1]: seq 3 of the group waits on the gap (seq 2 is the agent's own message) and the handler for the direct message h1, a later row, is slow. The socket drops while it runs, which clears the parked g3 without dispatching it. The cursor stops at g1: an SDK that routes rows one at a time sees g3 cleared, not parked, by the time it computes the cursor." + }, { "expect": { "ack_on_receipt": { diff --git a/tests/test_realtime.py b/tests/test_realtime.py index 47f4b35..c231149 100644 --- a/tests/test_realtime.py +++ b/tests/test_realtime.py @@ -1304,25 +1304,25 @@ async def test_drain_never_acks_a_below_anchor_row_nobody_dispatched() -> None: async def test_drain_cursor_covers_only_handled_rows_when_the_socket_drops_mid_handler( ack_on_receipt: bool, ) -> None: - # A group where the agent's own message is seq 2: the page is [1, 3]. - # The handler for seq 1 is slow and the socket drops while it runs. The - # close resets the ordering state; whatever happens to seq 3 afterwards, - # the cursor may cover it only once a handler finished it. + # Page [g1, g3, h1]: g3 parks on the gap (the agent's own message is g's + # seq 2) and h1's handler, a LATER row, is slow. This drain routes rows + # one at a time, so the socket drop during h1 clears the parked g3 before + # the cursor is computed: g3 is then neither parked nor dispatched, and + # the cursor must stop before it unless a handler finished it. page = [ - _sync_row("m_1", conversation_id="g", delivery_id="del_1", seq=1), - _sync_row("m_3", conversation_id="g", delivery_id="del_3", seq=3), + _sync_row("g_1", conversation_id="g", delivery_id="del_g1", seq=1), + _sync_row("g_3", conversation_id="g", delivery_id="del_g3", seq=3), + _sync_row("h_1", conversation_id="h", delivery_id="del_h1", seq=1), ] mock_api = MockAsyncClient(sync_pages=[page]) rt, ws = _make_client( client=mock_api, auto_drain_on_connect=False, ack_on_receipt=ack_on_receipt ) - entered: list[str] = [] finished: list[str] = [] release = asyncio.Event() async def slow(m: dict[str, Any]) -> None: - entered.append(m["payload"]["id"]) - if m["payload"]["id"] == "m_1": + if m["payload"]["id"] == "h_1": await release.wait() finished.append(m["payload"]["id"]) @@ -1334,15 +1334,18 @@ async def slow(m: dict[str, Any]) -> None: await _settle() drain = asyncio.create_task(rt.drain_offline_envelopes()) await _settle() - assert entered == ["m_1"] await ws.close(code=1006, reason="abnormal") await _settle() release.set() await asyncio.wait_for(drain, timeout=1.0) - assert mock_api.sync_ack_calls, "the cursor covers at least the handled seq 1" - covered = {"del_1": ["m_1"], "del_3": ["m_1", "m_3"]}[mock_api.sync_ack_calls[-1]] - assert all(mid in finished for mid in covered) + if ack_on_receipt: + # The close handed the parked g3 to the handler first. + assert sorted(finished) == ["g_1", "g_3", "h_1"] + assert mock_api.sync_ack_calls == ["del_h1"] + else: + assert finished == ["g_1", "h_1"] + assert mock_api.sync_ack_calls == ["del_g1"] finally: await rt.disconnect()