From 6d62db927fb7a5863d9b58e667cdab94237ab246 Mon Sep 17 00:00:00 2001 From: San Date: Wed, 4 Nov 2026 18:34:06 -0800 Subject: [PATCH 1/2] =?UTF-8?q?fix(realtime):=20the=20socket=20never=20die?= =?UTF-8?q?s=20on=20its=20own=20=E2=80=94=20own=20closes=20are=20not=20aut?= =?UTF-8?q?h,=20no=20reconnect=20cap,=20blocked=20retries=20slowly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carries the OpenClaw plugin's connection rules (agentchat-openclaw #14) into RealtimeClient, identically to the TypeScript SDK, because Hermes and OpenClaw now receive through the SDKs. - A close is an auth rejection only when the server sent it before hello.ok and it is 4401, 4403, or 1008 with the server's auth reason (Invalid token). Any other 1008 (HELLO frame required from the server's 5 s timer, an eviction after key rotation/suspension) is transient. - A close the client initiated (handshake watchdog, liveness watchdog, HELLO send failure, disconnect) is never read as a verdict: code 4000 with a `client ...` reason; on_disconnect reports initiator. - No reconnect cap. Jittered exponential backoff up to max_reconnect_interval_ms; after 60 consecutive failures the cap widens to 5 min and one ERROR line is logged. max_reconnect_attempts is deprecated and ignored (warns). - A rejection enters `blocked` (state + blocked_reason, on_error, on("blocked")) and retries a fresh HELLO every 10-15 min until a hello.ok clears it; first rejection logs at ERROR, later ones at DEBUG. An explicit connect() retries at once. - Watchdogs: handshake_timeout_ms (15 s, open -> hello.ok) and liveness_timeout_ms (100 s without an inbound frame, then a ping probe with a 10 s pong deadline; default armed only on sockets with ping()). - Typed state: RealtimeState = idle | connecting | ready | backoff | blocked | stopped (same names as the TypeScript SDK), on("state"), on("stopped"), ack_negotiated. - Fixes a dead state on HELLO send failure and a socket leak when disconnect() raced the transport open. Tests: tests/test_realtime_resilience.py (controllable sleep, pinging socket fake); README and CHANGELOG (Unreleased) updated; no version bump. --- CHANGELOG.md | 73 ++++ README.md | 26 +- src/agentchatme/__init__.py | 2 + src/agentchatme/_realtime.py | 631 +++++++++++++++++++++++---- tests/test_realtime.py | 34 +- tests/test_realtime_engine.py | 4 +- tests/test_realtime_resilience.py | 698 ++++++++++++++++++++++++++++++ 7 files changed, 1360 insertions(+), 108 deletions(-) create mode 100644 tests/test_realtime_resilience.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 88214bb..cb38a0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,79 @@ file. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/ and the SDK uses [SemVer](https://semver.org/) — breaking changes bump the major. The on-the-wire API is versioned separately under `/v1/...`. +## [Unreleased] + +**The socket never dies on its own.** `RealtimeClient` now carries the +connection rules the OpenClaw plugin's socket had (its PR #14), identically +to the TypeScript SDK, because Hermes and OpenClaw now receive through the +SDKs: no dead state except `disconnect()`, our own closes are never +evidence about the server, only a real rejection blocks, and blocked retries +slowly. No wire change; no version bump here. + +### Fixed + +- A server close with code `1008` ended the client for good, whatever the + reason. The server also closes a slow handshake with `1008 'HELLO frame + required'` from its own 5 s timer (one slow key lookup or network leg + with a perfectly good key) and evicts live sockets with `1008` after key + rotation, suspension or sign-out — so one slow handshake could silence an + integration until a restart. A close is now an auth rejection only when + the server sent it before `hello.ok` and it is `4401`, `4403`, or `1008` + with the server's auth reason (`Invalid token`, case-insensitive); every + other close takes the normal backoff, and an eviction after `hello.ok` + lets the next HELLO decide. +- A close the client initiated itself (handshake watchdog, now the liveness + watchdog, `disconnect()`) is never read as a verdict from the server. The + client's own closes use code `4000` with a `client ...` reason (the + handshake watchdog used `1008`, the server's policy code). +- A `HELLO` send failure left the socket open with nothing reading it and + no reconnect scheduled; it now closes the socket and takes the backoff. +- `disconnect()` while the transport was still opening leaked the socket + that opened afterwards; it is now closed. + +### Changed + +- **No reconnect cap.** Backoff is jittered exponential up to + `max_reconnect_interval_ms` for the first 60 consecutive failures, then + capped at no less than five minutes, forever; the 61st failure logs one + line at `ERROR` on `agentchat.realtime` and retrying continues. +- **An auth rejection blocks instead of ending the client.** The client + enters `state == "blocked"` with `blocked_reason` set, fires `on_error` + and `on("blocked")`, and retries a fresh HELLO every 10–15 minutes + (jittered) until a `hello.ok` clears it. The first rejection in a run + logs at `ERROR`, later ones at `DEBUG`. A transport failure during the + slow retry takes the normal backoff and keeps `blocked_reason` until a + `hello.ok`. An explicit `connect()` while blocked retries at once. +- **Liveness watchdogs.** The handshake deadline (socket open → `hello.ok`) + is `handshake_timeout_ms`, default 15 s (was 4 s; the server's own 5 s + `1008` is transient now). A ready socket that shows no inbound frame for + `liveness_timeout_ms` is probed with a ping; no pong within 10 s trips + the watchdog. Both close as client-initiated and reconnect. The default + `liveness_timeout_ms=None` is 100 s on a socket that exposes `ping()` + (the `websockets` connection) and no watchdog otherwise; `0` disables it. +- `connect()` is a no-op while an attempt is in flight or a socket is live, + and cancels a pending backoff or blocked-retry timer to try now. +- `on_disconnect` info carries `initiator: "client" | "server"`. + +### Added + +- `RealtimeClient.state` (`RealtimeState`, exported, same names as the + TypeScript SDK: `idle | connecting | ready | backoff | blocked | stopped`), + `RealtimeClient.blocked_reason` (`" "` or `None`) and + `RealtimeClient.ack_negotiated` (this connection's `hello.ok` echoed `ack`). +- Options `handshake_timeout_ms` and `liveness_timeout_ms` (above). +- Client-raised events through `on()`: `on("state", handler)` — + `{"type": "state", "state", "previous"}` on every transition; + `on("blocked", handler)` — `{"type": "blocked", "code", "reason", "retry_in_ms"}` + once per rejection; `on("stopped", handler)` — `{"type": "stopped"}` when + `disconnect()` ran, the last event an instance raises. A server frame + typed like one of these is dropped, never dispatched. + +### Deprecated + +- `max_reconnect_attempts` (option and keyword) is ignored and warns with a + `DeprecationWarning`: the client never stops on its own. + ## [1.2.0] — 2026-11-05 **The one engine.** Everything an AgentChat integration needs to receive is diff --git a/README.md b/README.md index d70dccf..bc63794 100644 --- a/README.md +++ b/README.md @@ -427,8 +427,9 @@ realtime = RealtimeClient( client=async_client, # enables gap-fill + auto offline drain reconnect=True, # default reconnect_interval_ms=500, # initial delay - max_reconnect_interval_ms=30_000, - max_reconnect_attempts=None, # None = unlimited + max_reconnect_interval_ms=30_000, # backoff cap; widens to 5 min after 60 straight failures + handshake_timeout_ms=15_000, # socket open → hello.ok deadline + liveness_timeout_ms=None, # None = 100 s on a socket with ping(); 0 disables (see Reconnect behavior) ack_on_receipt=False, # integrations set True: ack as soon as a frame is parsed (see Delivery acks) on_sequence_gap=lambda info: print("gap", info), ) @@ -442,14 +443,21 @@ The realtime client is **async-only** because Python's WebSocket story is asynci off = realtime.on("message.new", lambda evt: ...) realtime.on_error(lambda err: ...) realtime.on_connect(lambda: ...) # fires after HELLO_ACK -realtime.on_disconnect(lambda info: ...) # {code, reason, was_clean} +realtime.on_disconnect(lambda info: ...) # {code, reason, was_clean, initiator: "client" | "server"} +realtime.on("state", lambda evt: ...) # {type, state, previous}: every state transition +realtime.on("blocked", lambda evt: ...) # {type, code, reason, retry_in_ms}: the server rejected the key +realtime.on("stopped", lambda evt: ...) # {type}: disconnect() ran off() # each on_* returns a cleanup fn await realtime.connect() await realtime.disconnect() # graceful; disposes the instance + +realtime.state # "idle" | "connecting" | "ready" | "backoff" | "blocked" | "stopped" +realtime.blocked_reason # e.g. "1008 Invalid token" while blocked, else None +realtime.ack_negotiated # True while this connection's hello.ok echoed the `ack` capability ``` -Handlers can be either sync functions or `async def` coroutines — the client awaits coroutines automatically. +Handlers can be either sync functions or `async def` coroutines — the client awaits coroutines automatically. `RealtimeState` (the `state` literal) is exported from the package root. ### Gap recovery @@ -497,11 +505,17 @@ A wake whose payload fails validation is reported through `on_error` and not dis ### Running as a guest -The SDK keeps no process-wide state: several `RealtimeClient` instances run side by side in one event loop, each with its own socket, dedup cache and timers, and `disconnect()` cancels and awaits every task that instance owned (reconnect, HELLO watchdog, stability window, gap timers, drain) and closes only its socket. +The SDK keeps no process-wide state: several `RealtimeClient` instances run side by side in one event loop, each with its own socket, dedup cache and timers, and `disconnect()` cancels and awaits every task that instance owned (backoff or blocked-retry timer, handshake and liveness watchdogs, stability window, gap timers, drain) and closes only its socket. ### Reconnect behavior -Reconnects use jittered exponential backoff and run forever by default (`max_reconnect_attempts=None`). Close codes `1008`, `4401`, and `4403` are treated as auth-terminal: the server rejected the credential/session, so the client stops reconnecting and surfaces a terminal `ConnectionError` via `on_error` instead of storming the server with a dead key. +The client has no dead state except an explicit `disconnect()`. The same rules apply to the TypeScript SDK; they came from the OpenClaw plugin's socket, which Hermes and OpenClaw now get through the SDKs. + +- **Never gives up.** Reconnects use jittered exponential backoff (±25 %) from `reconnect_interval_ms` up to `max_reconnect_interval_ms`, with no attempt cap. After 60 consecutive failures the cap widens to at least five minutes and one line is logged at `ERROR` (`agentchat.realtime`); retrying continues until the server is back. `max_reconnect_attempts` is deprecated and ignored (passing it warns). +- **Our own closes are not evidence.** A close the client initiated — handshake timeout, liveness timeout, its own stop — says nothing about the server and takes the normal backoff, whatever code travelled with it. The client closes with code `4000` and a `client ...` reason, and `on_disconnect` reports `initiator: "client"`. +- **Only a real rejection blocks.** A server close is an auth rejection only when it arrives before `hello.ok` and is `4401`, `4403`, or `1008` with the server's auth reason (`Invalid token`, case-insensitive). Any other `1008` is transient: the server also uses it for a handshake its own 5 s timer cut short (`HELLO frame required`) and for evicting a live socket after key rotation, suspension or sign-out, where the next HELLO is the evidence. +- **Blocked is not the end.** A rejection moves the client to `state == "blocked"` with `blocked_reason` set, fires `on_error` and `on("blocked")`, and retries a fresh HELLO every 10–15 minutes (jittered) until one is accepted; the next `hello.ok` clears it. The first rejection in a run logs at `ERROR`, the following ones at `DEBUG`. An explicit `connect()` while blocked (you rotated the key) retries at once. +- **Liveness.** `hello.ok` must arrive within `handshake_timeout_ms` (15 s) of the socket opening, and a ready socket must show an inbound frame within `liveness_timeout_ms` or answer a ping within 10 s of that; a tripped watchdog closes the socket as client-initiated and reconnects. `liveness_timeout_ms=None` (default) means 100 s on a socket that exposes `ping()` — the `websockets` connection does — and no watchdog on one that does not; `0` disables it. The `websockets` library's own keepalive (ping every 20 s) runs alongside. --- diff --git a/src/agentchatme/__init__.py b/src/agentchatme/__init__.py index cd152f9..c6b7b56 100644 --- a/src/agentchatme/__init__.py +++ b/src/agentchatme/__init__.py @@ -50,6 +50,7 @@ async def main(): MessageHandler, RealtimeClient, RealtimeOptions, + RealtimeState, SequenceGapHandler, SequenceGapInfo, ) @@ -130,6 +131,7 @@ async def main(): # Realtime "RealtimeClient", "RealtimeOptions", + "RealtimeState", "RecipientBackloggedError", "RequestHooks", "RequestInfo", diff --git a/src/agentchatme/_realtime.py b/src/agentchatme/_realtime.py index 119a0c2..a1c1a00 100644 --- a/src/agentchatme/_realtime.py +++ b/src/agentchatme/_realtime.py @@ -10,13 +10,35 @@ - Per-conversation seq ordering with out-of-order buffering - Gap detection with optional ``AsyncAgentChatClient``-backed recovery - ``wake`` frames (a signal with no message text: read the tray) -- Jittered exponential reconnect (halting on auth-terminal close codes), +- Jittered exponential reconnect that never gives up, a ``blocked`` state + with a slow retry after an auth rejection, handshake + liveness watchdogs, disposed flag, offline ``/sync`` drain speaking the bare-array wire The client is async-only because Python's WebSocket story is asyncio-native. Pair it with an :class:`~agentchat.AsyncAgentChatClient` for gap recovery and auto-drain on reconnect. +Connection rules (shared with the TypeScript SDK; they came from the +OpenClaw plugin's socket, which Hermes and OpenClaw now get through here): + +1. The client has no dead state except an explicit :meth:`disconnect`. + There is no reconnect cap: backoff grows exponentially with jitter, and + after ``_LONG_OUTAGE_AFTER_ATTEMPTS`` consecutive failures the cap widens + to about five minutes and one error line is logged. +2. A close the client itself initiated (handshake timeout, liveness timeout, its + own stop) is never evidence about the server: normal backoff. +3. A server close is an auth rejection only when it is ``4401``/``4403``, or + ``1008`` with the server's auth reason (``Invalid token``), during the + handshake. Any other ``1008`` (``HELLO frame required`` from the server's + own 5 s timer, an eviction after key rotation or suspension) is transient. +4. An auth rejection does not end the client: it enters ``blocked`` (exposed + as :attr:`RealtimeClient.state` with :attr:`RealtimeClient.blocked_reason`) + and retries a fresh HELLO every 10-15 minutes until a ``hello.ok`` clears it. +5. Liveness: ``hello.ok`` must arrive within ``handshake_timeout_ms`` of the + socket opening, and a ready socket must show an inbound frame or answer + a ping within ``liveness_timeout_ms``; a tripped watchdog closes the + socket as client-initiated (rule 2) and reconnects. + Guest mode: the client keeps no process-wide state. Several instances run side by side in one loop, and :meth:`RealtimeClient.disconnect` releases every task, timer and socket the instance owns, so a host process (an agent @@ -32,6 +54,7 @@ import logging import random import time +import warnings from collections import OrderedDict from collections.abc import Awaitable, Coroutine from dataclasses import dataclass, field, replace @@ -69,6 +92,29 @@ DisconnectHandler = Callable[[dict[str, Any]], Union[Awaitable[None], None]] """Handler fired on every close. Receives ``{code, reason, was_clean}``.""" +RealtimeState = Literal["idle", "connecting", "ready", "backoff", "blocked", "stopped"] +"""Where :class:`RealtimeClient` is in its connection lifecycle. Same names +as the TypeScript SDK. + +``idle`` + No socket and nothing scheduled: before the first + :meth:`RealtimeClient.connect`, or after a close with ``reconnect=False``. +``connecting`` + The transport is opening, or open with HELLO sent and ``hello.ok`` + awaited under the handshake watchdog. +``ready`` + ``hello.ok`` received; frames flow and the liveness watchdog runs. +``backoff`` + The socket closed or failed to open for a transient reason; a jittered + exponential backoff timer is pending. There is no attempt cap. +``blocked`` + The server rejected the credential during the handshake + (:attr:`RealtimeClient.blocked_reason` says how); a fresh HELLO is + retried every 10-15 minutes until one is accepted. +``stopped`` + :meth:`RealtimeClient.disconnect` ran. The only terminal state. +""" + GapReason = Literal[ "gap_filled", "gap_fill_failed", @@ -101,6 +147,22 @@ class SequenceGapInfo: # comfortably covers a reconnect/drain burst without unbounded growth. _DEFAULT_DEDUP_CACHE_SIZE = 2048 +# Handshake deadline: socket open → ``hello.ok``. The server runs its own +# 5 s HELLO timer and closes a slow handshake with 1008 'HELLO frame +# required'; that close is transient (see _is_auth_rejection_close), so there +# is no need to beat it. This watchdog covers a server that neither answers +# nor closes. A trip closes the socket as client-initiated: normal backoff. +_DEFAULT_HANDSHAKE_TIMEOUT_MS = 15_000 + +# Liveness deadline for a ready socket: this long without any inbound frame, +# the client probes with a ping; a pong within _LIVENESS_PROBE_TIMEOUT_S +# counts as liveness, anything else trips the watchdog (client-initiated +# close → normal backoff). The ``websockets`` library answers the server's +# pings and runs its own keepalive (every 20 s, see _keepalive_kwargs) +# without exposing either, so inbound frames plus an explicit probe are what +# we can count. 100 s is more than two server heartbeat cycles (45 s). +_DEFAULT_LIVENESS_TIMEOUT_MS = 100_000 + def _keepalive_kwargs(connect_fn: Any) -> dict[str, float]: """Pin the client keepalive instead of inheriting library defaults. @@ -146,13 +208,30 @@ class RealtimeOptions: reconnect: bool = True reconnect_interval_ms: int = 500 max_reconnect_interval_ms: int = 30_000 - max_reconnect_attempts: int | None = None # None → no limit + """Cap on the backoff between reconnects. After + ``_LONG_OUTAGE_AFTER_ATTEMPTS`` consecutive failures the cap widens to at + least five minutes; the client keeps retrying either way.""" + max_reconnect_attempts: int | None = None + """Deprecated and ignored: the client never stops reconnecting on its + own (only :meth:`RealtimeClient.disconnect` ends it). Passing a value + emits a :class:`DeprecationWarning`. Kept so existing calls still + construct.""" client: AsyncAgentChatClient | None = None on_sequence_gap: SequenceGapHandler | None = None auto_drain_on_connect: bool | None = None # None → True iff client set dedup_cache_size: int = _DEFAULT_DEDUP_CACHE_SIZE """Bound on the message-id dedup LRU (live + drain). ``0`` disables dedup.""" client_identity: AgentChatClientIdentity = DEFAULT_CLIENT_IDENTITY + handshake_timeout_ms: int = _DEFAULT_HANDSHAKE_TIMEOUT_MS + """Socket open → ``hello.ok`` deadline. A trip is a client-initiated + close: normal backoff, never ``blocked``.""" + liveness_timeout_ms: int | None = None + """How long a ready socket may go without an inbound frame before the + client probes it with a ping (a pong within 10 s keeps it). ``None`` + (default) means 100 s on a socket that exposes ``ping()`` — the + ``websockets`` connection does — and no watchdog on one that does not, + since frames alone cannot tell a quiet conversation from a dead peer. + ``0`` disables it. An explicit value arms it on any socket.""" ack_on_receipt: bool = False """Acknowledge each ``message.new`` the moment its frame is parsed, before any handler runs. This is the integration setting: "delivered" @@ -177,9 +256,18 @@ class RealtimeOptions: # ───────────────────────── Internal constants ───────────────────────── -# Client-side ceiling on the HELLO ack wait. Must stay below the server -# HELLO_TIMEOUT_MS (5s) so our reconnect kicks in first. -_HELLO_ACK_TIMEOUT_S = 4.0 +# How long the liveness probe waits for its pong. +_LIVENESS_PROBE_TIMEOUT_S = 10.0 + +# The code on every close this client initiates (watchdogs, a HELLO it could +# not send). Private-use range, so server telemetry can tell our closes from +# the standard codes; the reason says which watchdog. +_CLIENT_CLOSE_CODE = 4000 + +# Events the client raises itself through ``on()``. A server frame that +# happens to carry one of these types is dropped, never dispatched as if it +# were ours. +_LOCAL_EVENTS = frozenset({"state", "blocked", "stopped"}) # How long a connection must SURVIVE before we call it healthy and clear # the reconnect backoff. @@ -232,11 +320,67 @@ class RealtimeOptions: # locally (a ``hello.ok`` without ``capabilities`` means legacy server). _CLIENT_CAPABILITIES = ("ack",) -# Close codes after which reconnecting is pointless: the server rejected -# this credential/session, not this connection. 1008 is the standard -# policy-violation code used for auth rejects; 4401/4403 are the -# application-level unauthorized/forbidden codes. -_AUTH_TERMINAL_CLOSE_CODES = frozenset({1008, 4401, 4403}) +# After this many consecutive failed attempts (about half an hour at the +# default 30 s cap) the outage is long: one error line is logged for the +# operator and the backoff cap widens to _LONG_OUTAGE_MAX_INTERVAL_MS. +# Retrying continues until the server is back — a deploy, a DNS outage or a +# laptop closed overnight is not a reason to stay dead afterwards. +_LONG_OUTAGE_AFTER_ATTEMPTS = 60 +_LONG_OUTAGE_MAX_INTERVAL_MS = 5 * 60_000 + +# Backoff jitter: symmetric ±25 %, so a fleet that drops at the same instant +# re-enters the connect queue spread across ms * [0.75, 1.25]. +_BACKOFF_JITTER = 0.25 + +# After the server rejects the key, retry a fresh HELLO at this cadence +# (±_AUTH_RETRY_JITTER: 10-15 min). A rejected key is not a dead end: keys +# get rotated back, suspended accounts get reinstated, and a server-side +# lookup failure during HELLO presents as a reject too. +_AUTH_RETRY_INTERVAL_MS = 750_000 +_AUTH_RETRY_JITTER = 0.2 + +# Close codes that mean "the server rejected this session" when the SERVER +# sends them during the handshake. 4401/4403 are unambiguous. 1008 (policy +# violation) is not: the AgentChat server closes a rejected HELLO with +# 1008 'Invalid token', but also uses 1008 for a handshake that was merely +# too slow ('HELLO frame required', from its own 5 s timer) — which a slow +# key lookup or one slow network leg produces with a perfectly good key — +# and for evicting a live socket after key rotation, suspension or sign-out. +# So 1008 counts as a rejection only with the server's auth reason; any +# other 1008 takes the normal backoff. A close this client initiated itself +# never counts (see ``_close_initiated_locally``), and these codes after +# ``hello.ok`` are an eviction: the next HELLO is the evidence, so they take +# the normal reconnect path too. +_UNAMBIGUOUS_AUTH_CLOSE_CODES = frozenset({4401, 4403}) +_AMBIGUOUS_AUTH_CLOSE_CODE = 1008 +_SERVER_AUTH_CLOSE_REASONS = ("invalid token",) + + +def _is_auth_rejection_close(code: int, reason: str) -> bool: + """True iff a server-originated close with this code and reason, received + during the handshake, means the key was rejected. Pure.""" + if code in _UNAMBIGUOUS_AUTH_CLOSE_CODES: + return True + if code != _AMBIGUOUS_AUTH_CLOSE_CODE: + return False + lower = reason.lower() + return any(known in lower for known in _SERVER_AUTH_CLOSE_REASONS) + + +def _jittered_ms(ms: int, ratio: float) -> int: + """``ms`` scaled by a uniform factor in ``[1 - ratio, 1 + ratio]``.""" + factor = 1.0 - ratio + random.random() * 2.0 * ratio + return max(0, int(ms * factor)) + + +async def _default_sleep(delay_s: float) -> None: + await asyncio.sleep(delay_s) + + +def _consume_future(future: asyncio.Future[Any]) -> None: + """Retrieve a future's outcome so an unawaited failure is not logged.""" + if not future.cancelled(): + future.exception() _Routed = Literal["delivered", "duplicate", "buffered"] @@ -337,6 +481,8 @@ def __init__( auto_drain_on_connect: bool | None = None, dedup_cache_size: int | None = None, client_identity: AgentChatClientIdentity | None = None, + handshake_timeout_ms: int | None = None, + liveness_timeout_ms: int | None = None, ack_on_receipt: bool | None = None, websocket_connect: Callable[..., Awaitable[Any]] | None = None, ) -> None: @@ -365,6 +511,12 @@ def __init__( else _DEFAULT_DEDUP_CACHE_SIZE ), client_identity=client_identity or DEFAULT_CLIENT_IDENTITY, + handshake_timeout_ms=( + handshake_timeout_ms + if handshake_timeout_ms is not None + else _DEFAULT_HANDSHAKE_TIMEOUT_MS + ), + liveness_timeout_ms=liveness_timeout_ms, ack_on_receipt=bool(ack_on_receipt), ) @@ -374,32 +526,56 @@ def __init__( options = replace(options) if options.auto_drain_on_connect is None: options.auto_drain_on_connect = options.client is not None + if options.max_reconnect_attempts is not None: + warnings.warn( + "RealtimeClient ignores max_reconnect_attempts: the client never" + " stops reconnecting on its own; call disconnect() to end it.", + DeprecationWarning, + stacklevel=2, + ) self._opts = options self._websocket_connect = websocket_connect self._ws: Any = None self._recv_task: asyncio.Task[None] | None = None + # One slot for both timers: the backoff before a reconnect and the + # slow retry while blocked. Never both at once. self._reconnect_task: asyncio.Task[None] | None = None - self._hello_ack_task: asyncio.Task[None] | None = None + self._handshake_task: asyncio.Task[None] | None = None + self._liveness_task: asyncio.Task[None] | None = None self._stability_task: asyncio.Task[None] | None = None self._bg_tasks: set[asyncio.Task[Any]] = set() + self._state: RealtimeState = "idle" + self._connecting = False self._reconnect_attempts = 0 # Consecutive connections that failed to reach _STABLE_CONNECTION_S. # Drives the operator-facing instability warning only; the backoff # itself is governed by _reconnect_attempts. self._rapid_reconnects = 0 self._last_connect_monotonic: float | None = None + self._last_inbound_monotonic = 0.0 self._authenticated = False self._disposed = False self._order_states: dict[str, _OrderState] = {} # Delivery-ack mode: on only when THIS connection's hello.ok echoed # the `ack` capability. Reset on every connect/close. - self._ack_mode = False - # Set by the HELLO watchdog before it closes the socket, so the - # recv loop can tell our own 1008 self-close (transient handshake - # timeout → keep reconnecting) from a server-sent auth-terminal one. - self._watchdog_closed_socket = False + self._ack_negotiated = False + # Set by _close_socket before it closes the current socket, read and + # cleared by the recv loop's close handling. A close this client + # asked for says nothing about the server, whatever code travelled + # with it. + self._close_initiated_locally = False + # The last auth rejection, as " ", until a hello.ok + # clears it. Survives a transport failure during the slow retry. + self._blocked_reason: str | None = None + # One error line per long outage / per run of rejected keys, then + # debug. + self._long_outage_announced = False + self._blocked_announced = False + # The wait used by the reconnect and blocked-retry timers. Tests swap + # it for a controllable clock; everything else sleeps for real. + self._sleep: Callable[[float], Awaitable[None]] = _default_sleep # Message-id dedup shared by the live and drain paths. Survives # reconnects deliberately: redelivery across connections is exactly # the duplicate source it exists to absorb. @@ -436,30 +612,57 @@ async def connect(self) -> None: Returns once the HELLO has been transmitted — NOT after ``hello.ok``. Register :meth:`on_connect` to react to a fully authenticated session. Raises :class:`ConnectionError` if the - socket fails to open or the ``websockets`` package is missing. + socket fails to open or the ``websockets`` package is missing; a + failed open also schedules the normal reconnect. + + A no-op while an attempt is in flight or a socket is live. A pending + backoff or blocked-retry timer is cancelled and the attempt made now, + so a caller that has just rotated its key does not wait out the slow + retry. """ if self._disposed: raise _RealtimeConnectionError( "RealtimeClient has been disposed; create a new instance to reconnect." ) + if self._connecting or ( + self._recv_task is not None and not self._recv_task.done() + ): + _log.debug("connect() ignored: an attempt or a live socket exists") + return + pending = self._reconnect_task + if pending is not None and pending is not asyncio.current_task() and not pending.done(): + pending.cancel() + self._reconnect_task = None connect_fn = self._resolve_connect_fn() url = f"{self._opts.base_url}/v1/ws" + self._connecting = True + await self._set_state("connecting") try: - self._ws = await connect_fn(url, **_keepalive_kwargs(connect_fn)) + ws = await connect_fn(url, **_keepalive_kwargs(connect_fn)) except Exception as err: + self._connecting = False error = _RealtimeConnectionError(f"WebSocket connection failed: {err}") await self._emit_error(error) - self._schedule_reconnect() + await self._schedule_reconnect() raise error from err + finally: + self._connecting = False + + if self._disposed: + # disconnect() ran while the transport was opening. + with contextlib.suppress(Exception): + await ws.close(code=1000, reason="client stopped") + return + self._ws = ws self._authenticated = False - self._ack_mode = False - self._watchdog_closed_socket = False + self._ack_negotiated = False + self._close_initiated_locally = False try: - await self._ws.send( + await ws.send( json.dumps( { "type": "hello", @@ -479,12 +682,17 @@ async def connect(self) -> None: ) ) except Exception as err: + # A socket we cannot write HELLO to is useless; drop it as our + # own close and take the normal backoff rather than sit on it. await self._emit_error( _RealtimeConnectionError(f"HELLO send failed: {err}") ) + self._ws = None + await self._close_socket(ws, _CLIENT_CLOSE_CODE, "client HELLO send failed") + await self._schedule_reconnect() return - self._hello_ack_task = asyncio.create_task(self._hello_ack_watchdog()) + self._handshake_task = asyncio.create_task(self._handshake_watchdog(ws)) self._recv_task = asyncio.create_task(self._recv_loop()) def _resolve_connect_fn(self) -> Callable[..., Awaitable[Any]]: @@ -527,6 +735,7 @@ async def _mark_stable() -> None: return self._reconnect_attempts = 0 self._rapid_reconnects = 0 + self._long_outage_announced = False self._stability_task = None try: @@ -568,22 +777,124 @@ def _note_connection_ended(self) -> None: self._opts.max_reconnect_interval_ms / 1000.0, ) - async def _hello_ack_watchdog(self) -> None: + # ─── Watchdogs ─────────────────────────────────────────────────── + + async def _close_socket(self, ws: Any, code: int, reason: str) -> None: + """Close ``ws`` as a client-initiated close. + + The flag is set before the close so the recv loop, which may see + the close before this coroutine resumes, never reads our own code + as a verdict from the server (rule 2). + """ + self._close_initiated_locally = True + with contextlib.suppress(Exception): + await ws.close(code=code, reason=reason) + + async def _handshake_watchdog(self, ws: Any) -> None: + """Bound socket-open → ``hello.ok``. A trip is our close: backoff.""" + timeout_ms = self._opts.handshake_timeout_ms try: - await asyncio.sleep(_HELLO_ACK_TIMEOUT_S) + await asyncio.sleep(timeout_ms / 1000.0) except asyncio.CancelledError: return - if self._authenticated or self._disposed: + if self._authenticated or self._disposed or self._ws is not ws: + return + await self._emit_error( + _RealtimeConnectionError(f"handshake timeout: no hello.ok within {timeout_ms}ms") + ) + # Spawned, not awaited: the recv loop cancels this task as part of + # the close, and a cancelled close could leave the socket half shut. + self._close_initiated_locally = True + self._spawn(self._close_socket(ws, _CLIENT_CLOSE_CODE, "client handshake timeout")) + + def _liveness_timeout_ms(self, ws: Any) -> int: + """The liveness deadline for ``ws``; ``0`` means no watchdog.""" + configured = self._opts.liveness_timeout_ms + if configured is not None: + return max(0, configured) + if callable(getattr(ws, "ping", None)): + return _DEFAULT_LIVENESS_TIMEOUT_MS + # Nothing to probe with: inbound frames alone cannot separate a quiet + # conversation from a dead peer, so do not guess. + _log.debug("liveness watchdog not armed: the socket exposes no ping()") + return 0 + + def _start_liveness_watchdog(self, ws: Any) -> None: + self._cancel_liveness_watchdog() + timeout_ms = self._liveness_timeout_ms(ws) + if timeout_ms <= 0: + return + self._last_inbound_monotonic = time.monotonic() + self._liveness_task = asyncio.create_task(self._liveness_watchdog(ws, timeout_ms)) + + def _cancel_liveness_watchdog(self) -> None: + if self._liveness_task is not None: + self._liveness_task.cancel() + self._liveness_task = None + + async def _liveness_watchdog(self, ws: Any, timeout_ms: int) -> None: + """Trip when ``timeout_ms`` passes with no inbound frame and the peer + does not answer a ping either. A trip is our close: backoff.""" + timeout_s = timeout_ms / 1000.0 + try: + while not self._disposed and self._ws is ws: + remaining = timeout_s - (time.monotonic() - self._last_inbound_monotonic) + if remaining > 0: + await asyncio.sleep(remaining) + continue + if await self._probe_liveness(ws): + self._last_inbound_monotonic = time.monotonic() + continue + if self._disposed or self._ws is not ws: + return + await self._emit_error( + _RealtimeConnectionError( + f"liveness timeout: nothing received for {timeout_ms}ms" + ) + ) + self._close_initiated_locally = True + self._spawn(self._close_socket(ws, _CLIENT_CLOSE_CODE, "client liveness timeout")) + return + except asyncio.CancelledError: return - await self._emit_error(_RealtimeConnectionError("HELLO ack timeout")) - ws = self._ws - if ws is not None: - # Mark the close as self-inflicted: 1008 is also an - # auth-terminal code when the SERVER sends it, but a handshake - # timeout is transient and must keep the reconnect loop alive. - self._watchdog_closed_socket = True - with contextlib.suppress(Exception): - await ws.close(code=1008, reason="HELLO ack timeout") + + async def _probe_liveness(self, ws: Any) -> bool: + """Send a ping; ``True`` iff a pong arrives in ``_LIVENESS_PROBE_TIMEOUT_S``. + + ``websockets`` (both implementations) returns an awaitable that + resolves on the matching pong. Its future is never cancelled from + here — the library settles it on pong or on close — only left + behind with its outcome consumed. A socket without ``ping()`` (an + explicit ``liveness_timeout_ms`` armed the watchdog anyway) counts + inbound frames alone. + """ + ping = getattr(ws, "ping", None) + if not callable(ping): + return False + try: + waiter = await ping() + except Exception: + return False + if not inspect.isawaitable(waiter): + # A transport whose ping() gives no pong signal cannot be probed; + # treat the successful send as the best evidence available. + return True + future = asyncio.ensure_future(waiter) + done, _ = await asyncio.wait({future}, timeout=_LIVENESS_PROBE_TIMEOUT_S) + if future in done: + return not future.cancelled() and future.exception() is None + future.add_done_callback(_consume_future) + return False + + async def _set_state(self, state: RealtimeState) -> None: + """Record a transition and raise ``on("state")`` for it.""" + previous = self._state + if state == previous: + return + _log.debug("realtime state %s -> %s", previous, state) + self._state = state + if self._handlers.get("state"): + await self._dispatch({"type": "state", "state": state, "previous": previous}) async def _recv_loop(self) -> None: ws = self._ws @@ -591,6 +902,7 @@ async def _recv_loop(self) -> None: return try: async for raw in ws: + self._last_inbound_monotonic = time.monotonic() try: message = json.loads(raw) except (ValueError, TypeError): @@ -612,10 +924,21 @@ async def _recv_loop(self) -> None: # capabilities list is a legacy server → stay in # mark-on-send semantics and send no ack frames. caps = message.get("capabilities") - self._ack_mode = isinstance(caps, list) and "ack" in caps - if self._hello_ack_task is not None: - self._hello_ack_task.cancel() - self._hello_ack_task = None + self._ack_negotiated = isinstance(caps, list) and "ack" in caps + if self._handshake_task is not None: + self._handshake_task.cancel() + self._handshake_task = None + # A hello.ok is the proof that the key is accepted: + # whatever rejection came before is over. + if self._blocked_reason is not None: + _log.info( + "AgentChat realtime: key accepted again, blocked cleared (was %s)", + self._blocked_reason, + ) + self._blocked_reason = None + self._blocked_announced = False + await self._set_state("ready") + self._start_liveness_watchdog(ws) await self._emit_connect() if ( self._opts.auto_drain_on_connect @@ -649,6 +972,10 @@ async def _recv_loop(self) -> None: await self._dispatch_wake(message) continue + if frame_type in _LOCAL_EVENTS: + _log.debug("dropping server frame typed like a local event: %s", frame_type) + continue + await self._dispatch(message) except asyncio.CancelledError: raise @@ -666,23 +993,36 @@ async def _recv_loop(self) -> None: close_code = code_attr if isinstance(reason_attr, str): close_reason = reason_attr - - if self._hello_ack_task is not None: - self._hello_ack_task.cancel() - self._hello_ack_task = None + # Read both BEFORE anything below resets them: only a close the + # server sent, before hello.ok, can be a rejection. + ours = self._close_initiated_locally + self._close_initiated_locally = False + handshake_done = self._authenticated + + if self._handshake_task is not None: + self._handshake_task.cancel() + self._handshake_task = None + self._cancel_liveness_watchdog() # Cancel BEFORE clearing _authenticated: a connection that dies # young must not clear the backoff counter, which is the whole # point of the stability window. self._cancel_stability_timer() self._note_connection_ended() self._authenticated = False - self._ack_mode = False + self._ack_negotiated = False + if self._ws is ws: + self._ws = None await self._emit_disconnect( { "code": close_code, "reason": close_reason, "was_clean": close_code == 1000, + # Who ended the socket. "client" is this client's own + # watchdog or stop: such a close says nothing about the + # server. "server" arrived on the wire: the server, a + # proxy, or the network. + "initiator": "client" if ours else "server", } ) @@ -697,26 +1037,27 @@ async def _recv_loop(self) -> None: await self._flush_parked_frames("connection_closed") self._reset_order_states() if not self._disposed: + _log.debug( + "realtime socket closed (code=%s reason=%r ours=%s handshake_done=%s)", + close_code, + close_reason, + ours, + handshake_done, + ) if ( - close_code in _AUTH_TERMINAL_CLOSE_CODES - and not self._watchdog_closed_socket + not ours + and not handshake_done + and _is_auth_rejection_close(close_code, close_reason) ): - # The server rejected the session itself (bad/revoked - # key, forbidden account state) — retrying the same - # credential forever is a reconnect storm, not - # resilience. Surface a terminal error and stop; a - # deliberate `connect()` after fixing the credential - # still works. - await self._emit_error( - _RealtimeConnectionError( - f"WebSocket closed with auth-terminal code {close_code}" - f" ({close_reason or 'no reason given'}); reconnect" - " disabled — fix the API key or account state and" - " reconnect explicitly." - ) - ) + # The server rejected the key during the handshake. Not + # the end: keys get rotated back and accounts reinstated, + # so retry slowly rather than storm or die (rule 4). + await self._enter_blocked(close_code, close_reason) else: - self._schedule_reconnect() + # Our own close, a transport drop, a slow handshake the + # server timed out, or an eviction after hello.ok: the + # next HELLO is the evidence (rules 2 and 3). + await self._schedule_reconnect() # ─── Offline drain ─────────────────────────────────────────────── @@ -880,22 +1221,86 @@ def _drain_row_ackable(self, envelope: dict[str, Any], outcome: _Routed) -> bool # ─── Reconnect ─────────────────────────────────────────────────── - def _schedule_reconnect(self) -> None: - if self._disposed or not self._opts.reconnect: + async def _schedule_reconnect(self) -> None: + """Arm the normal backoff. No attempt cap: the client never gives up + on its own (rule 1).""" + if self._disposed: return - max_attempts = self._opts.max_reconnect_attempts - if max_attempts is not None and self._reconnect_attempts >= max_attempts: + if not self._opts.reconnect: + # The caller runs reconnects itself; a blocked verdict stays + # visible so its logic can see it. + if self._state != "blocked": + await self._set_state("idle") return if self._reconnect_task is not None and not self._reconnect_task.done(): return self._reconnect_attempts += 1 - delay_s = self._compute_reconnect_delay_ms(self._reconnect_attempts) / 1000.0 + attempt = self._reconnect_attempts + if attempt > _LONG_OUTAGE_AFTER_ATTEMPTS and not self._long_outage_announced: + # Once per outage, at error level: the operator should know the + # server has been unreachable this long. Retrying goes on. + self._long_outage_announced = True + _log.error( + "AgentChat realtime: still reconnecting after %d consecutive" + " failures; backing off up to %.0fs between attempts and not" + " giving up. The server is unreachable from here, or every" + " connection dies before it is stable.", + _LONG_OUTAGE_AFTER_ATTEMPTS, + max(self._opts.max_reconnect_interval_ms, _LONG_OUTAGE_MAX_INTERVAL_MS) / 1000.0, + ) + delay_s = self._compute_reconnect_delay_ms(attempt) / 1000.0 + _log.debug("realtime reconnect %d scheduled in %.1fs", attempt, delay_s) + await self._set_state("backoff") self._reconnect_task = asyncio.create_task(self._reconnect_after(delay_s)) + async def _enter_blocked(self, code: int, reason: str) -> None: + """The server rejected the key during the handshake. + + Records the verdict, tells the operator once (then debug), fires + ``on_error`` and ``on("blocked")``, and arms the slow retry. A + ``hello.ok`` clears it; :meth:`disconnect` ends it. + """ + detail = reason or "no reason given" + self._blocked_reason = f"{code} {reason}".strip() + retry_ms: int | None = None + if self._opts.reconnect: + retry_ms = self._compute_auth_retry_delay_ms() + log = _log.debug if self._blocked_announced else _log.error + log( + "AgentChat realtime: the server rejected this API key (close %d %s);" + " blocked, retrying a fresh HELLO %s until it is accepted again." + " Check the key and the account state; disconnect() stops the retry.", + code, + detail, + f"in ~{retry_ms // 60_000} min" if retry_ms is not None else "when connect() is called", + ) + self._blocked_announced = True + await self._set_state("blocked") + await self._emit_error( + _RealtimeConnectionError( + f"WebSocket closed with auth rejection {code} ({detail}); blocked —" + + ( + f" retrying a fresh HELLO in ~{retry_ms // 60_000} min" + if retry_ms is not None + else " reconnect is off, call connect() to retry" + ) + + " until the key is accepted again; disconnect() stops it." + ) + ) + await self._dispatch( + {"type": "blocked", "code": code, "reason": detail, "retry_in_ms": retry_ms} + ) + if self._disposed or retry_ms is None: + return + pending = self._reconnect_task + if pending is not None and pending is not asyncio.current_task() and not pending.done(): + pending.cancel() + self._reconnect_task = asyncio.create_task(self._reconnect_after(retry_ms / 1000.0)) + async def _reconnect_after(self, delay_s: float) -> None: try: - await asyncio.sleep(delay_s) + await self._sleep(delay_s) except asyncio.CancelledError: return self._reconnect_task = None @@ -904,18 +1309,23 @@ async def _reconnect_after(self, delay_s: float) -> None: try: await self.connect() except Exception as err: - await self._emit_error( - err - if isinstance(err, _RealtimeConnectionError) - else _RealtimeConnectionError(str(err)) - ) + # connect() already emitted and rescheduled; a repeat here would + # double the error for the same failure. + _log.debug("realtime reconnect attempt failed: %s", err) def _compute_reconnect_delay_ms(self, attempt: int) -> int: - exp = self._opts.reconnect_interval_ms * (2 ** min(attempt - 1, 10)) - capped = min(exp, self._opts.max_reconnect_interval_ms) - # ±25% jitter defeats thundering-herd reconnect. - jitter = 0.75 + random.random() * 0.5 - return max(0, int(capped * jitter)) + """Exponential from ``reconnect_interval_ms``, capped at + ``max_reconnect_interval_ms`` — or at least five minutes once the + outage is long — then jittered ±25 %.""" + exp = self._opts.reconnect_interval_ms * (2 ** min(attempt - 1, 20)) + cap = self._opts.max_reconnect_interval_ms + if attempt > _LONG_OUTAGE_AFTER_ATTEMPTS: + cap = max(cap, _LONG_OUTAGE_MAX_INTERVAL_MS) + return _jittered_ms(min(exp, cap), _BACKOFF_JITTER) + + def _compute_auth_retry_delay_ms(self) -> int: + """The blocked cadence: 12.5 min ±20 % → 10-15 min.""" + return _jittered_ms(_AUTH_RETRY_INTERVAL_MS, _AUTH_RETRY_JITTER) # ─── Handler registration ──────────────────────────────────────── @@ -925,6 +1335,22 @@ def on(self, event: str, handler: MessageHandler) -> Callable[[], None]: ``on("wake", handler)`` receives a validated ``wake`` frame (``payload`` parses as :class:`~agentchatme.types.WakePayload`): a content-free "check AgentChat" signal that is never acknowledged. + + Three events are raised by the client itself, in the same shape (a + server frame carrying one of these types is dropped, never dispatched): + + ``on("state", handler)`` + Every transition of :attr:`state`. Receives + ``{"type": "state", "state": RealtimeState, "previous": RealtimeState}``. + ``on("blocked", handler)`` + The server rejected the key during the handshake. Receives + ``{"type": "blocked", "code": int, "reason": str, + "retry_in_ms": int | None}``; ``retry_in_ms`` is the slow retry + now armed (``None`` with ``reconnect=False``). Fires once per + rejection, so once per 10-15 min while the key stays rejected. + ``on("stopped", handler)`` + :meth:`disconnect` ran; receives ``{"type": "stopped"}``. The + last event this instance ever raises. """ handlers = self._handlers.setdefault(event, set()) handlers.add(handler) @@ -967,27 +1393,56 @@ async def send(self, message: dict[str, Any]) -> None: def is_connected(self) -> bool: return bool(self._authenticated and self._ws is not None) + @property + def state(self) -> RealtimeState: + """Where the client is in its lifecycle; see :data:`RealtimeState`.""" + return self._state + + @property + def blocked_reason(self) -> str | None: + """The last auth rejection as ``" "`` (for example + ``"1008 Invalid token"``), or ``None``. Set when the client enters + ``blocked``; it stays set through a transport failure during the + slow retry and clears on the next ``hello.ok``.""" + return self._blocked_reason + + @property + def ack_negotiated(self) -> bool: + """True while the current connection's ``hello.ok`` echoed the + ``ack`` capability: the server keeps envelopes ``stored`` until this + client acknowledges them. False between connections and against a + legacy server (envelopes marked on push).""" + return self._ack_negotiated + async def disconnect(self) -> None: """Close the socket, disable reconnect, and release every task, - timer and handler this instance owns. + timer and handler this instance owns. The only way into ``stopped``. Buffered out-of-order envelopes are flushed to handlers first (see - :meth:`_flush_parked_frames`), then the socket closes, - then every background task (reconnect, HELLO watchdog, stability - timer, gap timers, drain) is cancelled **and awaited**, so nothing - of this instance survives in the loop. After this, :meth:`connect` - raises — create a fresh :class:`RealtimeClient` to reopen. + :meth:`_flush_parked_frames`), then the socket closes, then every + background task (backoff or blocked-retry timer, handshake and idle + watchdogs, stability timer, gap timers, drain) is cancelled **and + awaited**, so nothing of this instance survives in the loop. Fires + ``on("stopped")`` last. After this, :meth:`connect` raises — create + a fresh :class:`RealtimeClient` to reopen. """ self._disposed = True self._opts.reconnect = False + await self._set_state("stopped") current = asyncio.current_task() owned: list[asyncio.Task[Any]] = [] - for task in (self._reconnect_task, self._hello_ack_task, self._stability_task): + for task in ( + self._reconnect_task, + self._handshake_task, + self._liveness_task, + self._stability_task, + ): if task is not None: owned.append(task) self._reconnect_task = None - self._hello_ack_task = None + self._handshake_task = None + self._liveness_task = None self._stability_task = None for state in self._order_states.values(): if state.gap_task is not None: @@ -1013,7 +1468,8 @@ async def disconnect(self) -> None: await _release_tasks(owned, current) self._authenticated = False - self._ack_mode = False + self._ack_negotiated = False + await self._dispatch({"type": "stopped"}) self._handlers.clear() self._error_handlers.clear() self._connect_handlers.clear() @@ -1129,7 +1585,7 @@ async def _send_delivery_ack(self, message_id: str) -> None: drain/redelivery, where the dedup cache turns it into skip-dispatch + re-ack. Never raises into the dispatch pipeline. """ - if not self._ack_mode: + if not self._ack_negotiated: return ws = self._ws if ws is None: @@ -1598,6 +2054,7 @@ def _min_buffered_seq(state: _OrderState) -> int | None: "MessageHandler", "RealtimeClient", "RealtimeOptions", + "RealtimeState", "SequenceGapHandler", "SequenceGapInfo", ] diff --git a/tests/test_realtime.py b/tests/test_realtime.py index 5b192f9..95f835c 100644 --- a/tests/test_realtime.py +++ b/tests/test_realtime.py @@ -880,7 +880,11 @@ async def test_dedup_cache_size_is_configurable_and_bounded() -> None: @pytest.mark.asyncio @pytest.mark.parametrize("code", [1008, 4401, 4403]) -async def test_terminal_close_code_stops_reconnect(code: int) -> None: +async def test_auth_close_after_hello_ok_is_an_eviction_not_a_rejection(code: int) -> None: + """Key rotated / account suspended / signed out close a LIVE socket with + these codes. The next HELLO is the evidence, so the client takes the + normal reconnect and is not blocked (the close-code rules proper are in + test_realtime_resilience.py).""" connects: list[MockWebSocket] = [] async def fake_connect(_url: str, **_kw: Any) -> MockWebSocket: @@ -903,12 +907,13 @@ async def fake_connect(_url: str, **_kw: Any) -> MockWebSocket: ws = connects[0] await ws.push({"type": "hello.ok"}) await _settle() - # Server rejects the session. - await ws.close(code=code) + # Server evicts the authenticated session. + await ws.close(code=code, reason="Key rotated") await asyncio.sleep(0.1) - # No reconnect happened, and the terminal condition was surfaced. - assert len(connects) == 1 - assert any(f"auth-terminal code {code}" in str(e) for e in errors) + assert len(connects) >= 2 + assert rt.state in ("connecting", "backoff") + assert rt.blocked_reason is None + assert not any("auth rejection" in str(e) for e in errors) finally: await rt.disconnect() @@ -945,10 +950,9 @@ async def fake_connect(_url: str, **_kw: Any) -> MockWebSocket: async def test_hello_timeout_self_close_is_not_terminal( monkeypatch: pytest.MonkeyPatch, ) -> None: - # The watchdog closes with 1008 when the server never sends hello.ok. - # That self-close must NOT trip the auth-terminal stop — a slow - # handshake is transient, unlike a server-sent 1008. - monkeypatch.setattr("agentchatme._realtime._HELLO_ACK_TIMEOUT_S", 0.03) + # The watchdog closes the socket when the server never sends hello.ok. + # A close we initiated is never evidence about the server: normal + # backoff, never blocked. connects: list[MockWebSocket] = [] async def fake_connect(_url: str, **_kw: Any) -> MockWebSocket: @@ -961,6 +965,7 @@ async def fake_connect(_url: str, **_kw: Any) -> MockWebSocket: reconnect=True, reconnect_interval_ms=10, max_reconnect_interval_ms=20, + handshake_timeout_ms=30, websocket_connect=fake_connect, ) errors: list[BaseException] = [] @@ -968,9 +973,12 @@ async def fake_connect(_url: str, **_kw: Any) -> MockWebSocket: try: await rt.connect() await asyncio.sleep(0.15) - assert len(connects) >= 2 # kept reconnecting after the 1008 self-close - assert any("HELLO ack timeout" in str(e) for e in errors) - assert not any("auth-terminal" in str(e) for e in errors) + assert len(connects) >= 2 # kept reconnecting after our own close + assert connects[0].close_code == _realtime._CLIENT_CLOSE_CODE + assert connects[0].close_reason == "client handshake timeout" + assert any("handshake timeout" in str(e) for e in errors) + assert not any("auth rejection" in str(e) for e in errors) + assert rt.state != "blocked" finally: await rt.disconnect() diff --git a/tests/test_realtime_engine.py b/tests/test_realtime_engine.py index 5a012ce..0ec00ca 100644 --- a/tests/test_realtime_engine.py +++ b/tests/test_realtime_engine.py @@ -464,9 +464,9 @@ async def fake_connect(_url: str, **_kw: Any) -> MockWebSocket: async def test_disconnect_releases_the_hello_ack_timer_mid_handshake() -> None: baseline = other_tasks() rt, _ws = _make_client() - await rt.connect() # HELLO sent, 4 s watchdog armed, no hello.ok + await rt.connect() # HELLO sent, 15 s handshake watchdog armed, no hello.ok await _settle() - assert rt._hello_ack_task is not None and not rt._hello_ack_task.done() + assert rt._handshake_task is not None and not rt._handshake_task.done() await rt.disconnect() assert other_tasks() - baseline == set() diff --git a/tests/test_realtime_resilience.py b/tests/test_realtime_resilience.py new file mode 100644 index 0000000..7f16f19 --- /dev/null +++ b/tests/test_realtime_resilience.py @@ -0,0 +1,698 @@ +"""The connection rules of :class:`agentchatme.RealtimeClient`. + +No dead state except ``disconnect()``; our own closes are never evidence +about the server; only ``4401``/``4403`` or ``1008 'Invalid token'`` during +the handshake is a rejection; a rejection blocks and retries slowly; the +handshake and idle watchdogs close as client-initiated. The socket is the +in-memory stand-in from ``test_realtime``; the long waits go through the +client's ``_sleep`` hook so a test controls the clock instead of waiting. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +import pytest + +from agentchatme import ConnectionError as AgentChatConnectionError +from agentchatme import RealtimeClient, _realtime + +from .test_realtime import MockWebSocket, _make_client, _settle +from .test_realtime_engine import connected, message_new, other_tasks + +# ─────────────── Fakes ─────────────── + + +class PingingWebSocket(MockWebSocket): + """A socket that answers (or never answers) the client's liveness probe.""" + + def __init__(self, *, pong: bool = True) -> None: + super().__init__() + self.pong = pong + self.pings = 0 + + async def ping(self) -> asyncio.Future[float]: + self.pings += 1 + waiter: asyncio.Future[float] = asyncio.get_running_loop().create_future() + if self.pong: + waiter.set_result(0.001) + return waiter + + +class ControlledSleep: + """Stands in for the client's reconnect / blocked-retry wait. + + Records every requested delay. With ``hold=True`` each wait parks until + :meth:`release`; otherwise it yields once and returns, so a run of + failures plays out in a few loop iterations instead of hours. + """ + + def __init__(self, *, hold: bool = False) -> None: + self.delays: list[float] = [] + self.hold = hold + self._gate = asyncio.Event() + + async def __call__(self, delay_s: float) -> None: + self.delays.append(delay_s) + if self.hold: + await self._gate.wait() + self._gate.clear() + else: + await asyncio.sleep(0) + + def release(self) -> None: + self._gate.set() + + +def _client_with_sockets( + sockets: list[MockWebSocket], *, reconnect: bool = True, **opts: Any +) -> tuple[RealtimeClient, list[MockWebSocket]]: + """A client whose connects hand out ``sockets`` in order (the last one + repeats), recording each one handed out in the returned list.""" + handed: list[MockWebSocket] = [] + + async def fake_connect(_url: str, **_kw: Any) -> MockWebSocket: + sock = sockets[min(len(handed), len(sockets) - 1)] + handed.append(sock) + return sock + + rt = RealtimeClient( + api_key="sk_test", + reconnect=reconnect, + reconnect_interval_ms=10, + max_reconnect_interval_ms=20, + websocket_connect=fake_connect, + **opts, + ) + return rt, handed + + +async def _until(predicate: Any, *, timeout: float = 2.0) -> None: + async def _spin() -> None: + while not predicate(): + await asyncio.sleep(0) + + await asyncio.wait_for(_spin(), timeout=timeout) + + +# ─────────────── Close classification (pure) ─────────────── + + +@pytest.mark.parametrize( + ("code", "reason", "expected"), + [ + (4401, "", True), + (4403, "Forbidden", True), + (1008, "Invalid token", True), + (1008, "invalid token", True), + (1008, "HELLO frame required", False), + (1008, "HELLO frame required first", False), + (1008, "", False), + (1008, "Key rotated", False), + (1008, "Signed out", False), + (1001, "Heartbeat timeout", False), + (1006, "", False), + (1000, "Invalid token", False), + ], +) +def test_is_auth_rejection_close_contract(code: int, reason: str, expected: bool) -> None: + assert _realtime._is_auth_rejection_close(code, reason) is expected + + +# ─────────────── Rule 3: which server closes are rejections ─────────────── + + +@pytest.mark.asyncio +async def test_server_1008_hello_frame_required_is_transient() -> None: + """The server's own 5 s HELLO timer closes a slow handshake with 1008. + A good key produces that after one slow leg: normal backoff, then the + next hello.ok connects.""" + first, second = MockWebSocket(), MockWebSocket() + rt, handed = _client_with_sockets([first, second]) + blocked: list[dict[str, Any]] = [] + rt.on("blocked", lambda evt: blocked.append(evt)) + try: + await rt.connect() + await _settle() + await first.close(code=1008, reason="HELLO frame required") + await _until(lambda: len(handed) >= 2) + await _settle() + assert rt.state == "connecting" + assert rt.blocked_reason is None + assert blocked == [] + await second.push({"type": "hello.ok", "capabilities": ["ack"]}) + await _settle() + assert rt.state == "ready" + assert rt.is_connected + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("code", "reason"), + [(1008, "Invalid token"), (1008, "invalid token"), (4401, ""), (4403, "Forbidden")], +) +async def test_server_rejection_during_handshake_blocks(code: int, reason: str) -> None: + sock = MockWebSocket() + rt, handed = _client_with_sockets([sock]) + sleeper = ControlledSleep(hold=True) + rt._sleep = sleeper + errors: list[BaseException] = [] + blocked: list[dict[str, Any]] = [] + rt.on_error(lambda e: errors.append(e)) + rt.on("blocked", lambda evt: blocked.append(evt)) + try: + await rt.connect() + await _settle() + await sock.close(code=code, reason=reason) + await _settle() + assert rt.state == "blocked" + assert rt.blocked_reason == f"{code} {reason}".strip() + assert not rt.is_connected + assert len(handed) == 1 # no fast reconnect + assert [e["code"] for e in blocked] == [code] + assert blocked[0]["reason"] == (reason or "no reason given") + assert any("auth rejection" in str(e) for e in errors) + # The slow retry is armed, 10-15 min out. + assert len(sleeper.delays) == 1 + assert 600.0 <= sleeper.delays[0] <= 900.0 + assert 600.0 <= blocked[0]["retry_in_ms"] / 1000.0 <= 900.0 + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_blocked_retries_slowly_and_hello_ok_clears_it( + caplog: pytest.LogCaptureFixture, +) -> None: + """Rejected → blocked → (10-15 min) → rejected again → blocked, one error + line in total → (10-15 min) → hello.ok → connected, reason cleared.""" + first, second, third = MockWebSocket(), MockWebSocket(), MockWebSocket() + rt, handed = _client_with_sockets([first, second, third]) + sleeper = ControlledSleep(hold=True) + rt._sleep = sleeper + blocked: list[dict[str, Any]] = [] + rt.on("blocked", lambda evt: blocked.append(evt)) + try: + with caplog.at_level(logging.DEBUG, logger="agentchat.realtime"): + await rt.connect() + await _settle() + await first.close(code=1008, reason="Invalid token") + await _settle() + assert rt.state == "blocked" + + sleeper.release() # the first slow retry fires + await _until(lambda: len(handed) >= 2) + await _settle() + assert rt.state == "connecting" + assert rt.blocked_reason == "1008 Invalid token" # held until a hello.ok + await second.close(code=1008, reason="Invalid token") + await _settle() + assert rt.state == "blocked" + assert len(blocked) == 2 + assert len(sleeper.delays) == 2 + assert all(600.0 <= d <= 900.0 for d in sleeper.delays) + + sleeper.release() # the second slow retry fires + await _until(lambda: len(handed) >= 3) + await _settle() + await third.push({"type": "hello.ok", "capabilities": ["ack"]}) + await _settle() + assert rt.state == "ready" + assert rt.blocked_reason is None + assert rt.ack_negotiated is True + + rejected = [ + r for r in caplog.records if "rejected this API key" in r.getMessage() + ] + assert [r.levelno for r in rejected] == [logging.ERROR, logging.DEBUG] + assert any("blocked cleared" in r.getMessage() for r in caplog.records) + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_transport_failure_during_blocked_retry_takes_normal_backoff() -> None: + """A failed open during the slow retry is no verdict: fast backoff, and + the recorded rejection stays visible until a hello.ok.""" + sock = MockWebSocket() + handed = 0 + + async def flaky_connect(_url: str, **_kw: Any) -> MockWebSocket: + nonlocal handed + handed += 1 + if handed == 2: + raise OSError("network down") + return sock + + rt = RealtimeClient( + api_key="sk_test", reconnect=True, reconnect_interval_ms=10, + max_reconnect_interval_ms=20, websocket_connect=flaky_connect, + ) + sleeper = ControlledSleep(hold=True) + rt._sleep = sleeper + try: + await rt.connect() + await _settle() + await sock.close(code=4401, reason="") + await _settle() + assert rt.state == "blocked" + sleeper.release() + await _until(lambda: len(sleeper.delays) >= 2) + assert rt.state == "backoff" + assert rt.blocked_reason == "4401" + assert sleeper.delays[1] < 1.0 # the normal backoff, not the slow retry + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_connect_while_blocked_retries_at_once() -> None: + """An explicit connect() (the caller rotated its key) cancels the slow + retry and tries now.""" + first, second = MockWebSocket(), MockWebSocket() + rt, handed = _client_with_sockets([first, second]) + sleeper = ControlledSleep(hold=True) + rt._sleep = sleeper + try: + await rt.connect() + await _settle() + await first.close(code=1008, reason="Invalid token") + await _settle() + assert rt.state == "blocked" + timer = rt._reconnect_task + assert timer is not None and not timer.done() + await rt.connect() + await _settle() + assert timer.done() + assert len(handed) == 2 + assert rt.state == "connecting" + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_blocked_with_reconnect_off_exposes_the_verdict_without_a_timer() -> None: + sock = MockWebSocket() + rt, _handed = _client_with_sockets([sock], reconnect=False) + blocked: list[dict[str, Any]] = [] + rt.on("blocked", lambda evt: blocked.append(evt)) + try: + await rt.connect() + await _settle() + await sock.close(code=1008, reason="Invalid token") + await _settle() + assert rt.state == "blocked" + assert rt._reconnect_task is None + assert blocked[0]["retry_in_ms"] is None + finally: + await rt.disconnect() + + +# ─────────────── Rule 1: no cap, one error line ─────────────── + + +@pytest.mark.asyncio +async def test_sixty_one_failures_keep_reconnecting_at_the_cap_with_one_error_line( + caplog: pytest.LogCaptureFixture, +) -> None: + attempts = 0 + + async def failing_connect(_url: str, **_kw: Any) -> MockWebSocket: + nonlocal attempts + attempts += 1 + raise OSError("connection refused") + + rt = RealtimeClient( + api_key="sk_test", + reconnect=True, + reconnect_interval_ms=500, + max_reconnect_interval_ms=30_000, + websocket_connect=failing_connect, + ) + sleeper = ControlledSleep() + rt._sleep = sleeper + try: + with caplog.at_level(logging.DEBUG, logger="agentchat.realtime"): + with pytest.raises(AgentChatConnectionError): + await rt.connect() + await _until(lambda: attempts >= 75, timeout=5.0) + assert rt.state == "backoff" + assert rt._reconnect_task is not None and not rt._reconnect_task.done() + # Attempts 1..60 sit under the configured 30 s cap (+25 % jitter); + # from 61 on the cap is five minutes, never more, never a stop. + early = sleeper.delays[:60] + late = sleeper.delays[60:] + assert late, "the client must still be scheduling attempts past 60" + assert all(d <= 30.0 * 1.25 for d in early) + assert all(300.0 * 0.75 <= d <= 300.0 * 1.25 for d in late) + errors = [r for r in caplog.records if r.levelno == logging.ERROR] + assert len(errors) == 1 + assert "still reconnecting after 60" in errors[0].getMessage() + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_max_reconnect_attempts_is_deprecated_and_ignored() -> None: + attempts = 0 + + async def failing_connect(_url: str, **_kw: Any) -> MockWebSocket: + nonlocal attempts + attempts += 1 + raise OSError("refused") + + with pytest.warns(DeprecationWarning, match="max_reconnect_attempts"): + rt = RealtimeClient( + api_key="sk_test", + reconnect=True, + reconnect_interval_ms=1, + max_reconnect_interval_ms=2, + max_reconnect_attempts=1, + websocket_connect=failing_connect, + ) + rt._sleep = ControlledSleep() + try: + with pytest.raises(AgentChatConnectionError): + await rt.connect() + await _until(lambda: attempts >= 5) + assert rt.state == "backoff" + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_hello_send_failure_is_our_close_and_reconnects() -> None: + """A socket we cannot write HELLO to was a dead state before: nothing + read it, nothing reconnected.""" + + class _DeafSocket(MockWebSocket): + async def send(self, data: str) -> None: + raise OSError("broken pipe") + + deaf, good = _DeafSocket(), MockWebSocket() + rt, handed = _client_with_sockets([deaf, good]) + try: + await rt.connect() + await _until(lambda: len(handed) >= 2) + assert deaf.closed and deaf.close_code == _realtime._CLIENT_CLOSE_CODE + assert rt.state in ("backoff", "connecting") + assert rt.blocked_reason is None + finally: + await rt.disconnect() + + +# ─────────────── Rule 5: liveness ─────────────── + + +@pytest.mark.asyncio +async def test_liveness_watchdog_holds_while_pings_are_answered( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_realtime, "_LIVENESS_PROBE_TIMEOUT_S", 0.05) + sock = PingingWebSocket(pong=True) + rt, handed = _client_with_sockets([sock], liveness_timeout_ms=20) + try: + await rt.connect() + await _settle() + await sock.push({"type": "hello.ok"}) + await asyncio.sleep(0.2) + assert sock.pings >= 2 # probed repeatedly, every probe answered + assert not sock.closed + assert rt.state == "ready" + assert len(handed) == 1 + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_liveness_watchdog_trips_without_pongs_and_reconnects( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_realtime, "_LIVENESS_PROBE_TIMEOUT_S", 0.02) + silent, next_sock = PingingWebSocket(pong=False), PingingWebSocket(pong=True) + rt, handed = _client_with_sockets([silent, next_sock], liveness_timeout_ms=20) + errors: list[BaseException] = [] + closes: list[dict[str, Any]] = [] + rt.on_error(lambda e: errors.append(e)) + rt.on_disconnect(lambda info: closes.append(info)) + try: + await rt.connect() + await _settle() + await silent.push({"type": "hello.ok"}) + await _until(lambda: len(handed) >= 2) + assert silent.pings == 1 + assert silent.closed and silent.close_code == _realtime._CLIENT_CLOSE_CODE + assert silent.close_reason == "client liveness timeout" + assert any("liveness timeout" in str(e) for e in errors) + assert closes[0]["initiator"] == "client" + assert rt.state in ("backoff", "connecting") # ours: never blocked + assert rt.blocked_reason is None + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_inbound_frames_keep_the_liveness_watchdog_quiet( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_realtime, "_LIVENESS_PROBE_TIMEOUT_S", 0.02) + sock = PingingWebSocket(pong=False) + rt, _handed = _client_with_sockets([sock], liveness_timeout_ms=50) + rt.on("message.new", lambda m: None) + try: + await rt.connect() + await _settle() + await sock.push({"type": "hello.ok"}) + for seq in range(1, 11): + await asyncio.sleep(0.015) + await sock.push(message_new("c", seq)) + assert sock.pings == 0 # never silent long enough to probe + assert not sock.closed + assert rt.state == "ready" + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_liveness_watchdog_default_is_not_armed_on_a_socket_without_ping() -> None: + rt, _ws = await connected() + try: + assert rt.state == "ready" + assert rt._liveness_task is None + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_liveness_watchdog_zero_disables_it_on_a_pinging_socket() -> None: + sock = PingingWebSocket(pong=True) + rt, _handed = _client_with_sockets([sock], liveness_timeout_ms=0) + try: + await rt.connect() + await _settle() + await sock.push({"type": "hello.ok"}) + await _settle() + assert rt.state == "ready" + assert rt._liveness_task is None + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_explicit_liveness_timeout_counts_frames_alone_without_ping() -> None: + """The caller asked for it on a socket that cannot be probed: silence + for the whole window trips it.""" + quiet, next_sock = MockWebSocket(), MockWebSocket() + rt, handed = _client_with_sockets([quiet, next_sock], liveness_timeout_ms=20) + try: + await rt.connect() + await _settle() + await quiet.push({"type": "hello.ok"}) + await _until(lambda: len(handed) >= 2) + assert quiet.closed and quiet.close_reason == "client liveness timeout" + assert rt.blocked_reason is None + finally: + await rt.disconnect() + + +# ─────────────── Typed state and events ─────────────── + + +@pytest.mark.asyncio +async def test_state_walks_the_lifecycle_and_ack_negotiated_tracks_the_connection() -> None: + sock = MockWebSocket() + rt, _handed = _client_with_sockets([sock]) + assert rt.state == "idle" + assert rt.ack_negotiated is False + try: + await rt.connect() + await _settle() + assert rt.state == "connecting" + await sock.push({"type": "hello.ok", "capabilities": ["ack"]}) + await _settle() + assert rt.state == "ready" + assert rt.ack_negotiated is True + await sock.close(code=1006) + await _settle() + assert rt.ack_negotiated is False + assert rt.state in ("backoff", "connecting") + finally: + await rt.disconnect() + assert rt.state == "stopped" + + +@pytest.mark.asyncio +async def test_on_state_reports_every_transition_in_order() -> None: + sock = MockWebSocket() + rt, _handed = _client_with_sockets([sock]) + seen: list[tuple[str, str]] = [] + rt.on("state", lambda evt: seen.append((evt["previous"], evt["state"]))) + sleeper = ControlledSleep(hold=True) + rt._sleep = sleeper + await rt.connect() + await _settle() + await sock.push({"type": "hello.ok"}) + await _settle() + await sock.close(code=1006, reason="blip") + await _settle() + await rt.disconnect() + assert seen == [ + ("idle", "connecting"), + ("connecting", "ready"), + ("ready", "backoff"), + ("backoff", "stopped"), + ] + + +@pytest.mark.asyncio +async def test_on_disconnect_names_the_server_as_initiator_for_a_wire_close() -> None: + sock = MockWebSocket() + rt, _handed = _client_with_sockets([sock]) + closes: list[dict[str, Any]] = [] + rt.on_disconnect(lambda info: closes.append(info)) + try: + await rt.connect() + await _settle() + await sock.push({"type": "hello.ok"}) + await _settle() + await sock.close(code=1001, reason="Heartbeat timeout") + await _settle() + assert closes == [ + {"code": 1001, "reason": "Heartbeat timeout", "was_clean": False, "initiator": "server"} + ] + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_server_frames_typed_like_local_events_are_dropped() -> None: + sock = MockWebSocket() + rt, _handed = _client_with_sockets([sock]) + blocked: list[dict[str, Any]] = [] + stopped: list[dict[str, Any]] = [] + rt.on("blocked", lambda evt: blocked.append(evt)) + rt.on("stopped", lambda evt: stopped.append(evt)) + try: + await rt.connect() + await _settle() + await sock.push({"type": "hello.ok"}) + await _settle() + await sock.push({"type": "blocked", "code": 4401, "reason": "spoofed"}) + await sock.push({"type": "stopped"}) + await sock.push({"type": "state", "state": "stopped", "previous": "ready"}) + await _settle() + assert blocked == [] + assert stopped == [] + assert rt.state == "ready" + finally: + await rt.disconnect() + assert stopped == [{"type": "stopped"}] + + +@pytest.mark.asyncio +async def test_disconnect_from_blocked_stops_with_no_task_left() -> None: + baseline = other_tasks() + sock = MockWebSocket() + rt, handed = _client_with_sockets([sock]) + sleeper = ControlledSleep(hold=True) + rt._sleep = sleeper + stopped: list[dict[str, Any]] = [] + rt.on("stopped", lambda evt: stopped.append(evt)) + await rt.connect() + await _settle() + await sock.close(code=1008, reason="Invalid token") + await _settle() + assert rt.state == "blocked" + + await rt.disconnect() + assert rt.state == "stopped" + assert stopped == [{"type": "stopped"}] + assert other_tasks() - baseline == set() + sleeper.release() + await asyncio.sleep(0.05) + assert len(handed) == 1 + assert other_tasks() - baseline == set() + with pytest.raises(AgentChatConnectionError): + await rt.connect() + + +@pytest.mark.asyncio +async def test_disconnect_from_pending_backoff_stops_with_no_task_left() -> None: + baseline = other_tasks() + sock = MockWebSocket() + rt, handed = _client_with_sockets([sock]) + sleeper = ControlledSleep(hold=True) + rt._sleep = sleeper + stopped = 0 + + def on_stopped(_evt: dict[str, Any]) -> None: + nonlocal stopped + stopped += 1 + + rt.on("stopped", on_stopped) + await rt.connect() + await _settle() + await sock.push({"type": "hello.ok"}) + await _settle() + await sock.close(code=1006, reason="blip") + await _settle() + assert rt.state == "backoff" + + await rt.disconnect() + assert rt.state == "stopped" + assert stopped == 1 + assert other_tasks() - baseline == set() + sleeper.release() + await asyncio.sleep(0.05) + assert len(handed) == 1 + assert other_tasks() - baseline == set() + + +@pytest.mark.asyncio +async def test_disconnect_during_a_connected_session_releases_the_idle_watchdog() -> None: + baseline = other_tasks() + sock = PingingWebSocket(pong=True) + rt, _handed = _client_with_sockets([sock]) + await rt.connect() + await _settle() + await sock.push({"type": "hello.ok"}) + await _settle() + assert rt._liveness_task is not None and not rt._liveness_task.done() + await rt.disconnect() + assert other_tasks() - baseline == set() + + +@pytest.mark.asyncio +async def test_realtime_state_is_exported() -> None: + import agentchatme + + assert "RealtimeState" in agentchatme.__all__ + rt, _ws = _make_client() + try: + assert rt.state == "idle" + finally: + await rt.disconnect() From 9e9c2795218a9bba0caa16a0ea29a4d55498e8dc Mon Sep 17 00:00:00 2001 From: San Date: Wed, 4 Nov 2026 18:51:39 -0800 Subject: [PATCH 2/2] =?UTF-8?q?fix(realtime):=20review=20fixes=20=E2=80=94?= =?UTF-8?q?=20connect()=20from=20close=20handlers,=20on=5Ferror=20once=20p?= =?UTF-8?q?er=20blocked=20run,=20disconnect=20is=20ours,=20no=20timer=20af?= =?UTF-8?q?ter=20stop,=20jitter=20below=20the=20cap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - connect() is callable from on_disconnect / on_error / on("state"): the guard keys on a live socket or in-flight attempt, not on the recv task whose finally runs those handlers; with reconnect=False a caller's own reconnect from on_disconnect opens a second socket instead of parking in idle. A handler that reconnected during a close is honoured: the old close arms no backoff and no blocked retry on top of it. - on_error (and the ERROR log) fire once per blocked run; on("blocked") still fires per rejection, with the server's reason as sent ("" when none). - disconnect() closes through _close_socket (1000 'client disconnect'), so on_disconnect reports initiator "client". - _schedule_reconnect re-checks _disposed after the state event: a state handler that stops the client leaves no backoff timer behind. - Non-positive handshake_timeout_ms falls back to the default. - Backoff jitter goes up to 25 % below each step; 30 s and 5 min are true ceilings. The blocked retry keeps its symmetric 10-15 min range. - Stale "4 s" comments corrected; server heartbeat facts and the websockets keepalive interplay noted next to the liveness default. - Tests: connect-from-on_disconnect (reconnect on/off), disconnect from a state handler, 3 rejections -> 1 on_error / 3 blocked, non-positive handshake timeout, disconnect initiator, and one raising-on_error test per notice site (handshake, liveness, long outage, blocked, HELLO send). Frame-flow liveness test widened to a 200 ms window / 20 ms pushes. - README and CHANGELOG (Unreleased) updated; no version bump. --- CHANGELOG.md | 30 ++-- README.md | 13 +- src/agentchatme/_realtime.py | 110 +++++++++---- tests/test_realtime_resilience.py | 260 +++++++++++++++++++++++++++++- 4 files changed, 358 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb38a0e..2e4b6b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,17 +37,26 @@ slowly. No wire change; no version bump here. ### Changed -- **No reconnect cap.** Backoff is jittered exponential up to +- **No reconnect cap.** Backoff is exponential, each step jittered up to + 25 % below it (previously ±25 %, which let delays exceed the cap), up to `max_reconnect_interval_ms` for the first 60 consecutive failures, then capped at no less than five minutes, forever; the 61st failure logs one line at `ERROR` on `agentchat.realtime` and retrying continues. - **An auth rejection blocks instead of ending the client.** The client - enters `state == "blocked"` with `blocked_reason` set, fires `on_error` - and `on("blocked")`, and retries a fresh HELLO every 10–15 minutes - (jittered) until a `hello.ok` clears it. The first rejection in a run - logs at `ERROR`, later ones at `DEBUG`. A transport failure during the - slow retry takes the normal backoff and keeps `blocked_reason` until a - `hello.ok`. An explicit `connect()` while blocked retries at once. + enters `state == "blocked"` with `blocked_reason` set and retries a fresh + HELLO every 10–15 minutes (12.5 min ± 20 %) until a `hello.ok` clears + it. `on("blocked")` fires for every rejection; `on_error` and the `ERROR` + log line fire once per blocked run, later rejections of the same run log + at `DEBUG`. A transport failure during the slow retry takes the normal + backoff and keeps `blocked_reason` until a `hello.ok`. An explicit + `connect()` while blocked retries at once. +- `connect()` may be called from inside `on_disconnect`, `on_error` or + `on("state")` (the closed socket is already gone by then); with + `reconnect=False` that is how a caller runs its own reconnects. A + handler that reconnects or stops the client during a close is honoured: + no second socket, no timer left behind. +- A non-positive `handshake_timeout_ms` means the default (as in the + TypeScript SDK) instead of an open/close loop. - **Liveness watchdogs.** The handshake deadline (socket open → `hello.ok`) is `handshake_timeout_ms`, default 15 s (was 4 s; the server's own 5 s `1008` is transient now). A ready socket that shows no inbound frame for @@ -57,7 +66,9 @@ slowly. No wire change; no version bump here. (the `websockets` connection) and no watchdog otherwise; `0` disables it. - `connect()` is a no-op while an attempt is in flight or a socket is live, and cancels a pending backoff or blocked-retry timer to try now. -- `on_disconnect` info carries `initiator: "client" | "server"`. +- `on_disconnect` info carries `initiator: "client" | "server"`; + `disconnect()` closes with `1000 'client disconnect'` and reports + `"client"` like the watchdogs do. ### Added @@ -69,7 +80,8 @@ slowly. No wire change; no version bump here. - Client-raised events through `on()`: `on("state", handler)` — `{"type": "state", "state", "previous"}` on every transition; `on("blocked", handler)` — `{"type": "blocked", "code", "reason", "retry_in_ms"}` - once per rejection; `on("stopped", handler)` — `{"type": "stopped"}` when + once per rejection, `reason` being the server's close reason as sent + (`""` when none); `on("stopped", handler)` — `{"type": "stopped"}` when `disconnect()` ran, the last event an instance raises. A server frame typed like one of these is dropped, never dispatched. diff --git a/README.md b/README.md index bc63794..618f484 100644 --- a/README.md +++ b/README.md @@ -428,8 +428,8 @@ realtime = RealtimeClient( reconnect=True, # default reconnect_interval_ms=500, # initial delay max_reconnect_interval_ms=30_000, # backoff cap; widens to 5 min after 60 straight failures - handshake_timeout_ms=15_000, # socket open → hello.ok deadline - liveness_timeout_ms=None, # None = 100 s on a socket with ping(); 0 disables (see Reconnect behavior) + handshake_timeout_ms=15_000, # socket open → hello.ok deadline (default; non-positive = default) + # liveness_timeout_ms=..., # default: 100 000 when the socket has ping(), off otherwise; 0 disables ack_on_receipt=False, # integrations set True: ack as soon as a frame is parsed (see Delivery acks) on_sequence_gap=lambda info: print("gap", info), ) @@ -511,11 +511,12 @@ The SDK keeps no process-wide state: several `RealtimeClient` instances run side The client has no dead state except an explicit `disconnect()`. The same rules apply to the TypeScript SDK; they came from the OpenClaw plugin's socket, which Hermes and OpenClaw now get through the SDKs. -- **Never gives up.** Reconnects use jittered exponential backoff (±25 %) from `reconnect_interval_ms` up to `max_reconnect_interval_ms`, with no attempt cap. After 60 consecutive failures the cap widens to at least five minutes and one line is logged at `ERROR` (`agentchat.realtime`); retrying continues until the server is back. `max_reconnect_attempts` is deprecated and ignored (passing it warns). -- **Our own closes are not evidence.** A close the client initiated — handshake timeout, liveness timeout, its own stop — says nothing about the server and takes the normal backoff, whatever code travelled with it. The client closes with code `4000` and a `client ...` reason, and `on_disconnect` reports `initiator: "client"`. +- **Never gives up.** Reconnects use exponential backoff from `reconnect_interval_ms`, each step jittered up to 25 % *below* it, up to `max_reconnect_interval_ms` (a true ceiling), with no attempt cap. After 60 consecutive failures the ceiling widens to at least five minutes and one line is logged at `ERROR` (`agentchat.realtime`); retrying continues until the server is back. `max_reconnect_attempts` is deprecated and ignored (passing it warns). +- **Our own closes are not evidence.** A close the client initiated — handshake timeout, liveness timeout, a HELLO it could not send, `disconnect()` — says nothing about the server and takes the normal backoff, whatever code travelled with it. The watchdogs close with code `4000` and a `client ...` reason, `disconnect()` with `1000 'client disconnect'`, and `on_disconnect` reports `initiator: "client"` for all of them. - **Only a real rejection blocks.** A server close is an auth rejection only when it arrives before `hello.ok` and is `4401`, `4403`, or `1008` with the server's auth reason (`Invalid token`, case-insensitive). Any other `1008` is transient: the server also uses it for a handshake its own 5 s timer cut short (`HELLO frame required`) and for evicting a live socket after key rotation, suspension or sign-out, where the next HELLO is the evidence. -- **Blocked is not the end.** A rejection moves the client to `state == "blocked"` with `blocked_reason` set, fires `on_error` and `on("blocked")`, and retries a fresh HELLO every 10–15 minutes (jittered) until one is accepted; the next `hello.ok` clears it. The first rejection in a run logs at `ERROR`, the following ones at `DEBUG`. An explicit `connect()` while blocked (you rotated the key) retries at once. -- **Liveness.** `hello.ok` must arrive within `handshake_timeout_ms` (15 s) of the socket opening, and a ready socket must show an inbound frame within `liveness_timeout_ms` or answer a ping within 10 s of that; a tripped watchdog closes the socket as client-initiated and reconnects. `liveness_timeout_ms=None` (default) means 100 s on a socket that exposes `ping()` — the `websockets` connection does — and no watchdog on one that does not; `0` disables it. The `websockets` library's own keepalive (ping every 20 s) runs alongside. +- **Blocked is not the end.** A rejection moves the client to `state == "blocked"` with `blocked_reason` set and retries a fresh HELLO every 10–15 minutes (12.5 min ± 20 %) until one is accepted; the next `hello.ok` clears it. `on("blocked")` fires for every rejection (its `reason` is the server's close reason as sent, `""` when there was none); `on_error` and the `ERROR` log line fire once per blocked run, later rejections of the same run log at `DEBUG`. An explicit `connect()` while blocked (you rotated the key) retries at once. +- **Liveness.** `hello.ok` must arrive within `handshake_timeout_ms` (15 s; a non-positive value means the default) of the socket opening, and a ready socket must show an inbound frame within `liveness_timeout_ms` or answer a ping within 10 s of that; a tripped watchdog closes the socket as client-initiated and reconnects. `liveness_timeout_ms=None` (default) means 100 s on a socket that exposes `ping()` — the `websockets` connection does — and no watchdog on one that does not; `0` disables it. With the real `websockets` connection its own keepalive (ping every 20 s, 20 s pong deadline) trips first on a dead path and surfaces as an ordinary close (`1011`/`1006`) that takes the normal backoff; the server itself pings every 45 s with a 30 s pong deadline once `hello.ok` is sent. +- **A raising handler never parks the client.** Every `on_error`, `on_disconnect`, `on("state")` and `on("blocked")` handler is isolated (logged, never propagated), so an observer bug cannot stop the close, the backoff or the blocked retry that follows a notice. `connect()` may be called from inside `on_disconnect` / `on_error` / `on("state")` — the closed socket is already gone by then — which is how a caller with `reconnect=False` runs its own reconnects. --- diff --git a/src/agentchatme/_realtime.py b/src/agentchatme/_realtime.py index a1c1a00..48d49ea 100644 --- a/src/agentchatme/_realtime.py +++ b/src/agentchatme/_realtime.py @@ -2,7 +2,7 @@ Mirrors the TypeScript ``RealtimeClient`` behavior: -- HELLO handshake with 4 s ack timeout (authenticates over wire, never URL) +- HELLO handshake with a 15 s deadline (authenticates over wire, never URL) - Capability-negotiated delivery acks (``ack``): at-least-once delivery with per-message confirmation after handler dispatch, or on receipt with ``ack_on_receipt=True`` (delivered = this machine received it) @@ -160,7 +160,13 @@ class SequenceGapInfo: # close → normal backoff). The ``websockets`` library answers the server's # pings and runs its own keepalive (every 20 s, see _keepalive_kwargs) # without exposing either, so inbound frames plus an explicit probe are what -# we can count. 100 s is more than two server heartbeat cycles (45 s). +# we can count. 100 s is more than two server heartbeat cycles (the server +# pings every 45 s with a 30 s pong deadline, starting after hello.ok). With +# the real ``websockets`` connection its keepalive trips first on a dead +# path (20 s + 20 s) and surfaces as a close code 1011/1006 that takes the +# ordinary backoff; this watchdog is the backstop for a socket that keeps +# answering pings at the transport yet delivers nothing, and for injected +# transports without a keepalive. _DEFAULT_LIVENESS_TIMEOUT_MS = 100_000 @@ -282,9 +288,10 @@ class RealtimeOptions: # one agent, delivering zero messages, while six sibling agents on the same # host held single connections the whole time. # -# 30s is comfortably longer than any handshake-adjacent failure (HELLO ack -# is 4s, the server's own heartbeat cycle is 45s) and short enough that a -# genuinely healthy client clears its counter well before a second blip. +# 30s is comfortably longer than any handshake-adjacent failure (the +# handshake deadline is 15s, the server's own heartbeat cycle is 45s) and +# short enough that a genuinely healthy client clears its counter well +# before a second blip. _STABLE_CONNECTION_S = 30.0 # Reconnects within this window are "rapid" for the purposes of the @@ -328,8 +335,9 @@ class RealtimeOptions: _LONG_OUTAGE_AFTER_ATTEMPTS = 60 _LONG_OUTAGE_MAX_INTERVAL_MS = 5 * 60_000 -# Backoff jitter: symmetric ±25 %, so a fleet that drops at the same instant -# re-enters the connect queue spread across ms * [0.75, 1.25]. +# Backoff jitter: up to 25 % BELOW each step, so a fleet that drops at the +# same instant re-enters the connect queue spread across ms * [0.75, 1.0] +# and the caps (30 s, then 5 min) are true ceilings. _BACKOFF_JITTER = 0.25 # After the server rejects the key, retry a fresh HELLO at this cadence @@ -340,7 +348,9 @@ class RealtimeOptions: _AUTH_RETRY_JITTER = 0.2 # Close codes that mean "the server rejected this session" when the SERVER -# sends them during the handshake. 4401/4403 are unambiguous. 1008 (policy +# sends them during the handshake. 4401/4403 are unambiguous (the server +# today sends only 1008; the application codes are honoured if they ever +# appear). 1008 (policy # violation) is not: the AgentChat server closes a rejected HELLO with # 1008 'Invalid token', but also uses 1008 for a handshake that was merely # too slow ('HELLO frame required', from its own 5 s timer) — which a slow @@ -368,11 +378,19 @@ def _is_auth_rejection_close(code: int, reason: str) -> bool: def _jittered_ms(ms: int, ratio: float) -> int: - """``ms`` scaled by a uniform factor in ``[1 - ratio, 1 + ratio]``.""" + """``ms`` scaled by a uniform factor in ``[1 - ratio, 1 + ratio]``: a + range around ``ms`` (the blocked retry's 10-15 min), not a cap.""" factor = 1.0 - ratio + random.random() * 2.0 * ratio return max(0, int(ms * factor)) +def _jittered_below_ms(ms: int, ratio: float) -> int: + """``ms`` scaled by a uniform factor in ``[1 - ratio, 1]``: ``ms`` stays + a true ceiling (the reconnect backoff and its caps).""" + factor = 1.0 - random.random() * ratio + return max(0, int(ms * factor)) + + async def _default_sleep(delay_s: float) -> None: await asyncio.sleep(delay_s) @@ -526,6 +544,10 @@ def __init__( options = replace(options) if options.auto_drain_on_connect is None: options.auto_drain_on_connect = options.client is not None + if options.handshake_timeout_ms <= 0: + # A zero deadline would open and close sockets in a loop; a + # non-positive value means the default, as in the TypeScript SDK. + options.handshake_timeout_ms = _DEFAULT_HANDSHAKE_TIMEOUT_MS if options.max_reconnect_attempts is not None: warnings.warn( "RealtimeClient ignores max_reconnect_attempts: the client never" @@ -618,15 +640,24 @@ async def connect(self) -> None: A no-op while an attempt is in flight or a socket is live. A pending backoff or blocked-retry timer is cancelled and the attempt made now, so a caller that has just rotated its key does not wait out the slow - retry. + retry. Callable from inside ``on_disconnect``, ``on_error`` and + ``on("state")``: the closed socket is already gone by then, so a + caller running its own reconnects (``reconnect=False``) may reopen + from there. """ if self._disposed: raise _RealtimeConnectionError( "RealtimeClient has been disposed; create a new instance to reconnect." ) - if self._connecting or ( - self._recv_task is not None and not self._recv_task.done() - ): + recv_task = self._recv_task + recv_alive = ( + recv_task is not None + and not recv_task.done() + # The recv loop's own close handling (handlers run inside its + # ``finally``) is not a live socket. + and recv_task is not asyncio.current_task() + ) + if self._connecting or self._ws is not None or recv_alive: _log.debug("connect() ignored: an attempt or a live socket exists") return pending = self._reconnect_task @@ -1226,6 +1257,10 @@ async def _schedule_reconnect(self) -> None: on its own (rule 1).""" if self._disposed: return + if self._connecting or self._ws is not None: + # A handler (on_disconnect, on_error) already called connect() + # for this close; the new attempt owns what happens next. + return if not self._opts.reconnect: # The caller runs reconnects itself; a blocked verdict stays # visible so its logic can see it. @@ -1252,13 +1287,17 @@ async def _schedule_reconnect(self) -> None: delay_s = self._compute_reconnect_delay_ms(attempt) / 1000.0 _log.debug("realtime reconnect %d scheduled in %.1fs", attempt, delay_s) await self._set_state("backoff") + if self._disposed or self._connecting or self._ws is not None: + # A state handler stopped the client, or reconnected it itself. + return self._reconnect_task = asyncio.create_task(self._reconnect_after(delay_s)) async def _enter_blocked(self, code: int, reason: str) -> None: """The server rejected the key during the handshake. - Records the verdict, tells the operator once (then debug), fires - ``on_error`` and ``on("blocked")``, and arms the slow retry. A + Records the verdict, tells the operator and ``on_error`` once per + blocked run (later rejections of the same run log at debug), fires + ``on("blocked")`` for every rejection, and arms the slow retry. A ``hello.ok`` clears it; :meth:`disconnect` ends it. """ detail = reason or "no reason given" @@ -1266,7 +1305,9 @@ async def _enter_blocked(self, code: int, reason: str) -> None: retry_ms: int | None = None if self._opts.reconnect: retry_ms = self._compute_auth_retry_delay_ms() - log = _log.debug if self._blocked_announced else _log.error + first_of_run = not self._blocked_announced + self._blocked_announced = True + log = _log.error if first_of_run else _log.debug log( "AgentChat realtime: the server rejected this API key (close %d %s);" " blocked, retrying a fresh HELLO %s until it is accepted again." @@ -1275,23 +1316,24 @@ async def _enter_blocked(self, code: int, reason: str) -> None: detail, f"in ~{retry_ms // 60_000} min" if retry_ms is not None else "when connect() is called", ) - self._blocked_announced = True await self._set_state("blocked") - await self._emit_error( - _RealtimeConnectionError( - f"WebSocket closed with auth rejection {code} ({detail}); blocked —" - + ( - f" retrying a fresh HELLO in ~{retry_ms // 60_000} min" - if retry_ms is not None - else " reconnect is off, call connect() to retry" + if first_of_run: + await self._emit_error( + _RealtimeConnectionError( + f"WebSocket closed with auth rejection {code} ({detail}); blocked —" + + ( + f" retrying a fresh HELLO in ~{retry_ms // 60_000} min" + if retry_ms is not None + else " reconnect is off, call connect() to retry" + ) + + " until the key is accepted again; disconnect() stops it." ) - + " until the key is accepted again; disconnect() stops it." ) - ) + # The server's reason as sent (empty when it sent none). await self._dispatch( - {"type": "blocked", "code": code, "reason": detail, "retry_in_ms": retry_ms} + {"type": "blocked", "code": code, "reason": reason, "retry_in_ms": retry_ms} ) - if self._disposed or retry_ms is None: + if self._disposed or retry_ms is None or self._connecting or self._ws is not None: return pending = self._reconnect_task if pending is not None and pending is not asyncio.current_task() and not pending.done(): @@ -1316,12 +1358,12 @@ async def _reconnect_after(self, delay_s: float) -> None: def _compute_reconnect_delay_ms(self, attempt: int) -> int: """Exponential from ``reconnect_interval_ms``, capped at ``max_reconnect_interval_ms`` — or at least five minutes once the - outage is long — then jittered ±25 %.""" + outage is long — then jittered up to 25 % below, so the cap holds.""" exp = self._opts.reconnect_interval_ms * (2 ** min(attempt - 1, 20)) cap = self._opts.max_reconnect_interval_ms if attempt > _LONG_OUTAGE_AFTER_ATTEMPTS: cap = max(cap, _LONG_OUTAGE_MAX_INTERVAL_MS) - return _jittered_ms(min(exp, cap), _BACKOFF_JITTER) + return _jittered_below_ms(min(exp, cap), _BACKOFF_JITTER) def _compute_auth_retry_delay_ms(self) -> int: """The blocked cadence: 12.5 min ±20 % → 10-15 min.""" @@ -1456,8 +1498,8 @@ async def disconnect(self) -> None: ws = self._ws self._ws = None if ws is not None: - with contextlib.suppress(Exception): - await ws.close() + # Ours: on_disconnect reports initiator "client". + await self._close_socket(ws, 1000, "client disconnect") recv_task = self._recv_task self._recv_task = None @@ -1478,6 +1520,10 @@ async def disconnect(self) -> None: # ─── Dispatch ──────────────────────────────────────────────────── async def _emit_error(self, error: BaseException) -> None: + # ``_invoke`` isolates each handler: a raising ``on_error`` must never + # break the step that follows a notice (arming a watchdog close, the + # backoff, the blocked retry) — that would be the one way into a + # parked client. for h in list(self._error_handlers): await _invoke(h, error) diff --git a/tests/test_realtime_resilience.py b/tests/test_realtime_resilience.py index 7f16f19..a515eb8 100644 --- a/tests/test_realtime_resilience.py +++ b/tests/test_realtime_resilience.py @@ -174,7 +174,7 @@ async def test_server_rejection_during_handshake_blocks(code: int, reason: str) assert not rt.is_connected assert len(handed) == 1 # no fast reconnect assert [e["code"] for e in blocked] == [code] - assert blocked[0]["reason"] == (reason or "no reason given") + assert blocked[0]["reason"] == reason # as the server sent it, "" included assert any("auth rejection" in str(e) for e in errors) # The slow retry is armed, 10-15 min out. assert len(sleeper.delays) == 1 @@ -342,13 +342,14 @@ async def failing_connect(_url: str, **_kw: Any) -> MockWebSocket: await _until(lambda: attempts >= 75, timeout=5.0) assert rt.state == "backoff" assert rt._reconnect_task is not None and not rt._reconnect_task.done() - # Attempts 1..60 sit under the configured 30 s cap (+25 % jitter); - # from 61 on the cap is five minutes, never more, never a stop. + # Attempts 1..60 sit under the configured 30 s cap; from 61 on the + # cap is five minutes. Jitter only goes below a step, so both caps + # are true ceilings. Never a stop. early = sleeper.delays[:60] late = sleeper.delays[60:] assert late, "the client must still be scheduling attempts past 60" - assert all(d <= 30.0 * 1.25 for d in early) - assert all(300.0 * 0.75 <= d <= 300.0 * 1.25 for d in late) + assert all(d <= 30.0 for d in early) + assert all(300.0 * 0.75 <= d <= 300.0 for d in late) errors = [r for r in caplog.records if r.levelno == logging.ERROR] assert len(errors) == 1 assert "still reconnecting after 60" in errors[0].getMessage() @@ -461,14 +462,16 @@ async def test_inbound_frames_keep_the_liveness_watchdog_quiet( ) -> None: monkeypatch.setattr(_realtime, "_LIVENESS_PROBE_TIMEOUT_S", 0.02) sock = PingingWebSocket(pong=False) - rt, _handed = _client_with_sockets([sock], liveness_timeout_ms=50) + # A 200 ms window with a push every 20 ms: wide enough for Windows timer + # granularity (15.6 ms) to never let the window lapse between pushes. + rt, _handed = _client_with_sockets([sock], liveness_timeout_ms=200) rt.on("message.new", lambda m: None) try: await rt.connect() await _settle() await sock.push({"type": "hello.ok"}) - for seq in range(1, 11): - await asyncio.sleep(0.015) + for seq in range(1, 16): + await asyncio.sleep(0.02) await sock.push(message_new("c", seq)) assert sock.pings == 0 # never silent long enough to probe assert not sock.closed @@ -686,6 +689,247 @@ async def test_disconnect_during_a_connected_session_releases_the_idle_watchdog( assert other_tasks() - baseline == set() +@pytest.mark.asyncio +async def test_on_disconnect_names_the_client_as_initiator_for_disconnect() -> None: + sock = MockWebSocket() + rt, _handed = _client_with_sockets([sock]) + closes: list[dict[str, Any]] = [] + rt.on_disconnect(lambda info: closes.append(info)) + await rt.connect() + await _settle() + await sock.push({"type": "hello.ok"}) + await _settle() + await rt.disconnect() + assert sock.closed and sock.close_code == 1000 and sock.close_reason == "client disconnect" + assert closes == [ + {"code": 1000, "reason": "client disconnect", "was_clean": True, "initiator": "client"} + ] + + +# ─────────────── Review fixes: handlers that act during a close ─────────────── + + +@pytest.mark.asyncio +async def test_connect_from_on_disconnect_reopens_with_reconnect_off() -> None: + """A caller running its own reconnects (reconnect=False) does it from + on_disconnect; the closed socket is gone by then, so connect() opens.""" + first, second = MockWebSocket(), MockWebSocket() + rt, handed = _client_with_sockets([first, second], reconnect=False) + reopened = asyncio.Event() + + async def reopen(_info: dict[str, Any]) -> None: + await rt.connect() + reopened.set() + + rt.on_disconnect(reopen) + try: + await rt.connect() + await _settle() + await first.push({"type": "hello.ok"}) + await _settle() + await first.close(code=1006, reason="blip") + await asyncio.wait_for(reopened.wait(), timeout=2.0) + await _settle() + assert handed == [first, second] + assert rt.state == "connecting" # not parked in idle by the old close + assert rt._reconnect_task is None + await second.push({"type": "hello.ok"}) + await _settle() + assert rt.state == "ready" + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_connect_from_on_disconnect_with_reconnect_on_arms_no_second_timer() -> None: + first, second = MockWebSocket(), MockWebSocket() + rt, handed = _client_with_sockets([first, second]) + rt.on_disconnect(lambda _info: rt.connect()) + try: + await rt.connect() + await _settle() + await first.push({"type": "hello.ok"}) + await _settle() + await first.close(code=1006, reason="blip") + await asyncio.sleep(0.1) + assert handed == [first, second] + assert rt._reconnect_task is None + assert rt.state == "connecting" + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_disconnect_from_a_state_handler_on_backoff_arms_no_timer() -> None: + baseline = other_tasks() + sock = MockWebSocket() + rt, handed = _client_with_sockets([sock]) + sleeper = ControlledSleep(hold=True) + rt._sleep = sleeper + + async def stop_on_backoff(evt: dict[str, Any]) -> None: + if evt["state"] == "backoff": + await rt.disconnect() + + rt.on("state", stop_on_backoff) + await rt.connect() + await _settle() + await sock.push({"type": "hello.ok"}) + await _settle() + await sock.close(code=1006, reason="blip") + await asyncio.sleep(0.05) + assert rt.state == "stopped" + assert rt._reconnect_task is None + assert sleeper.delays == [] + assert len(handed) == 1 + assert other_tasks() - baseline == set() + + +@pytest.mark.asyncio +async def test_on_error_fires_once_per_blocked_run_while_blocked_fires_per_rejection() -> None: + socks = [MockWebSocket(), MockWebSocket(), MockWebSocket(), MockWebSocket()] + rt, handed = _client_with_sockets(socks) + sleeper = ControlledSleep(hold=True) + rt._sleep = sleeper + errors: list[BaseException] = [] + blocked: list[dict[str, Any]] = [] + rt.on_error(lambda e: errors.append(e)) + rt.on("blocked", lambda evt: blocked.append(evt)) + try: + await rt.connect() + for i in range(3): + await _until(lambda i=i: len(handed) >= i + 1) + await _settle() + await socks[i].close(code=1008, reason="Invalid token") + await _settle() + assert rt.state == "blocked" + sleeper.release() + await _until(lambda: len(handed) >= 4) + assert len(blocked) == 3 + assert len([e for e in errors if "auth rejection" in str(e)]) == 1 + # A hello.ok ends the run; the next rejection is a new run. + await socks[3].push({"type": "hello.ok"}) + await _settle() + assert rt.state == "ready" + await socks[3].close(code=1008, reason="Invalid token") # after hello.ok: eviction + await _settle() + assert rt.state == "backoff" + finally: + await rt.disconnect() + + +@pytest.mark.parametrize("value", [0, -1]) +def test_non_positive_handshake_timeout_falls_back_to_the_default(value: int) -> None: + from agentchatme import RealtimeOptions + + rt = RealtimeClient(api_key="k", handshake_timeout_ms=value) + assert rt._opts.handshake_timeout_ms == _realtime._DEFAULT_HANDSHAKE_TIMEOUT_MS + via_options = RealtimeClient(RealtimeOptions(api_key="k", handshake_timeout_ms=value)) + assert via_options._opts.handshake_timeout_ms == _realtime._DEFAULT_HANDSHAKE_TIMEOUT_MS + + +# ─────────────── Review fixes: a raising on_error never parks the client ─────────────── + + +def _raising(_err: BaseException) -> None: + raise RuntimeError("observer bug") + + +@pytest.mark.asyncio +async def test_raising_on_error_at_the_handshake_deadline_still_closes_and_reconnects() -> None: + first, second = MockWebSocket(), MockWebSocket() + rt, handed = _client_with_sockets([first, second], handshake_timeout_ms=20) + rt.on_error(_raising) + try: + await rt.connect() + await _until(lambda: len(handed) >= 2) + assert first.closed and first.close_reason == "client handshake timeout" + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_raising_on_error_at_the_liveness_deadline_still_closes_and_reconnects( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_realtime, "_LIVENESS_PROBE_TIMEOUT_S", 0.02) + silent, next_sock = PingingWebSocket(pong=False), PingingWebSocket(pong=True) + rt, handed = _client_with_sockets([silent, next_sock], liveness_timeout_ms=20) + rt.on_error(_raising) + try: + await rt.connect() + await _settle() + await silent.push({"type": "hello.ok"}) + await _until(lambda: len(handed) >= 2) + assert silent.closed and silent.close_reason == "client liveness timeout" + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_raising_on_error_through_the_long_outage_notice_keeps_reconnecting( + caplog: pytest.LogCaptureFixture, +) -> None: + attempts = 0 + + async def failing_connect(_url: str, **_kw: Any) -> MockWebSocket: + nonlocal attempts + attempts += 1 + raise OSError("refused") + + rt = RealtimeClient( + api_key="k", reconnect=True, reconnect_interval_ms=1, max_reconnect_interval_ms=2, + websocket_connect=failing_connect, + ) + rt._sleep = ControlledSleep() + rt.on_error(_raising) + try: + with caplog.at_level(logging.ERROR, logger="agentchat.realtime"): + with pytest.raises(AgentChatConnectionError): + await rt.connect() + await _until(lambda: attempts >= 65, timeout=5.0) + assert rt.state == "backoff" + assert sum("still reconnecting after 60" in r.getMessage() for r in caplog.records) == 1 + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_raising_on_error_when_entering_blocked_still_arms_the_slow_retry() -> None: + sock = MockWebSocket() + rt, _handed = _client_with_sockets([sock]) + sleeper = ControlledSleep(hold=True) + rt._sleep = sleeper + rt.on_error(_raising) + try: + await rt.connect() + await _settle() + await sock.close(code=1008, reason="Invalid token") + await _settle() + assert rt.state == "blocked" + assert rt._reconnect_task is not None and not rt._reconnect_task.done() + assert len(sleeper.delays) == 1 and 600.0 <= sleeper.delays[0] <= 900.0 + finally: + await rt.disconnect() + + +@pytest.mark.asyncio +async def test_raising_on_error_at_hello_send_failure_still_closes_and_reconnects() -> None: + class _DeafSocket(MockWebSocket): + async def send(self, data: str) -> None: + raise OSError("broken pipe") + + deaf, good = _DeafSocket(), MockWebSocket() + rt, handed = _client_with_sockets([deaf, good]) + rt.on_error(_raising) + try: + await rt.connect() + await _until(lambda: len(handed) >= 2) + assert deaf.closed and deaf.close_reason == "client HELLO send failed" + finally: + await rt.disconnect() + + @pytest.mark.asyncio async def test_realtime_state_is_exported() -> None: import agentchatme