From 7cc6b6995112589d6e972f6bca99b8bb8054c4d3 Mon Sep 17 00:00:00 2001 From: Ben Gotthold Date: Thu, 22 Oct 2026 22:23:51 +0000 Subject: [PATCH 01/26] feat(tts): Text-to-Speech HTTP client and WebSocket session Adds `client.tts` for the Text-to-Speech API, matching the Python SDK: `synthesize(request)` returns one text's complete audio with the response headers, `stream(request)` returns it as the API generates it (an async iterable over a web ReadableStream), and `session(params)` opens a `TextToSpeechSession`: `connect()`, `speak()`, `generate()` / `flush()` / `audio()`, `cancel()`, `keepAlive()`, `close()`, with `on()` listeners for the server frames. Failures throw `TextToSpeechError` with the API's envelope fields. New client option `ttsBaseUrl` (US zone by default) and `createTemporaryToken()` for browser sessions. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 4 + CLAUDE.md | 28 ++ README.md | 84 +++++ package.json | 2 +- samples/tts.ts | 51 +++ src/services/index.ts | 25 ++ src/services/tts/index.ts | 2 + src/services/tts/service.ts | 347 ++++++++++++++++++ src/services/tts/session.ts | 649 +++++++++++++++++++++++++++++++++ src/types/index.ts | 1 + src/types/services/index.ts | 2 + src/types/tts/index.ts | 255 +++++++++++++ src/utils/errors/index.ts | 2 + src/utils/errors/tts.ts | 47 +++ tests/unit/tts-session.test.ts | 467 ++++++++++++++++++++++++ tests/unit/tts.test.ts | 275 ++++++++++++++ tests/unit/utils.ts | 1 + 17 files changed, 2241 insertions(+), 1 deletion(-) create mode 100644 samples/tts.ts create mode 100644 src/services/tts/index.ts create mode 100644 src/services/tts/service.ts create mode 100644 src/services/tts/session.ts create mode 100644 src/types/tts/index.ts create mode 100644 src/utils/errors/tts.ts create mode 100644 tests/unit/tts-session.test.ts create mode 100644 tests/unit/tts.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a132284..be2f737 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [4.42.0] + +- Add Text-to-Speech: `client.tts.synthesize(request)` returns one text's complete audio (`SpeechResponse`: `audio` plus the response headers as `requestId`, `region`, `model`, `voiceId`, `encoding`, `sampleRate`, `billedCharacters`, `audioDurationMs`, ...), `client.tts.stream(request)` returns it as the API generates it (`TextToSpeechStream`, an async iterable over a web `ReadableStream`), and `client.tts.session(params)` opens a WebSocket `TextToSpeechSession` for voice agents: `connect()`, `generate()` / `flush()` / `audio()`, `speak()`, `cancel()`, `keepAlive()`, `close()`, with `on("audio" | "flushDone" | "wordBoundaries" | "cancelled" | "termination" | "error" | "close", ...)`. Requests take the API's fields (`text`, `voice_id`, `model_id`, `language`, `sample_rate`, `encoding`, `container`, `bit_rate`); failures throw `TextToSpeechError` (`status`, `errorCode`, `param`, `requestId`, `retryable`, `retryAfter`, `errors`, `closeCode`, `body`). New client option `ttsBaseUrl` (default `https://tts.us.assemblyai.com`); `client.tts.createTemporaryToken()` mints a token for browser sessions + ## [4.41.7] - Add `summarization` to the speech understanding feature requests and responses (`SummarizationRequest`, `SummarizationResponse`): `summary_type`, `headline_length`, `approximate_chapters` and `effort`. The feature was already accepted by the API but missing from the SDK types, so setting it failed to type check diff --git a/CLAUDE.md b/CLAUDE.md index d0b48de..41c02e2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,6 +52,7 @@ const client = new AssemblyAI({ - `client.llmGateway.chatCompletions(request)` — OpenAI-compatible chat completions - `client.llmGateway.listModels()` — List models available on the LLM Gateway - `client.llmGateway.understanding(request)` / `client.llmGateway.validateUnderstanding(request)` — Run or validate a Speech Understanding request +- `client.tts.synthesize({ text, voice_id? })` — one text's complete audio (`SpeechResponse.audio`, a Uint8Array, plus header fields); `client.tts.stream(request)` — async iterable of audio chunks as generated; `client.tts.session({ voiceId })` — WebSocket `TextToSpeechSession`: `connect()`, `speak(text)` (async iterable), `generate()`/`flush()`/`audio(flushId?)`, `cancel()`, `keepAlive()`, `close()`, `on("audio"|"flushDone"|"wordBoundaries"|"cancelled"|"termination"|"error"|"close", fn)`. Errors: `TextToSpeechError` (`errorCode`, `param`, `retryable`, `retryAfter`, `requestId`, `closeCode`, `body`) ## Common patterns @@ -425,6 +426,33 @@ const result = await client.llmGateway.understanding({ Failures throw `LlmGatewayError` with `.status`, `.requestId`, and `.errors` (validation detail strings, when present). +## Text-to-Speech + +One text per HTTP request, or a WebSocket session for voice agents. Request fields use the API's names (`voice_id`, `sample_rate`, `encoding`, `container`); session parameters are camelCase (`voiceId`, `sampleRate`, `wordBoundaries`, `inactivityTimeout`, `token`). Host: `ttsBaseUrl` (`https://tts.us.assemblyai.com`; EU: `https://tts.eu.assemblyai.com`). + +```typescript +const { audio } = await client.tts.synthesize({ + text: "Hello.", + voice_id: "aai-jane", + container: "wav", +}); + +const speech = await client.tts.stream({ + text: "Hello.", + encoding: "pcm_mulaw", + sample_rate: 8000, +}); +for await (const chunk of speech) send(chunk); // chunk sizes vary; re-frame to 160 B for 20 ms + +const session = client.tts.session({ voiceId: "aai-jane" }); +await session.connect(); +for await (const chunk of session.speak("Hello.")) play(chunk); +session.cancel(); // barge-in: ends the current audio() iteration at once +await session.close(); // Terminate -> Termination summary; rejects with an unobserved session error +``` + +Gotchas: a session streams raw PCM/G.711 only (no `mp3`, `container` or `bit_rate`); `generate()` takes at most 2,000 characters per frame (`speak()` splits); the API paces text at ~100 chars/s; an `audio` listener switches audio from pull to push; a session the API ended (idle after 120 s) can `connect()` again on the same object. + ## Important gotchas - **.transcribe() polls until complete** — use .submit() for fire-and-forget diff --git a/README.md b/README.md index 96259bd..6cbf626 100644 --- a/README.md +++ b/README.md @@ -824,6 +824,90 @@ Close the connection when you're finished. await transcriber.close(); ``` +## Text-To-Speech + +`client.tts` speaks text with the Text-to-Speech API. `synthesize()` returns one text's complete audio, `stream()` returns it as the API generates it, and `session()` opens a WebSocket session that speaks text as it is written, for voice agents. Options take the API's own field names: the `voice_id` (a voice of the AssemblyAI voice library such as `aai-jane`, the default, or a voice cloned in your dashboard), the `model_id` (`assemblyai-tts-1`), the `language` (`english` by default, `spanish`, `german`, `french`, `italian`, `portuguese`), the `sample_rate` (24000 by default) and the `encoding` (`pcm_s16le` by default, `pcm_mulaw`/`pcm_alaw` for telephony, `mp3` over HTTP only). Requests go to `ttsBaseUrl`, `https://tts.us.assemblyai.com` by default; pass `https://tts.eu.assemblyai.com` to keep them in the EU data zone. + +### Synthesize a WAV file + +```typescript +import { writeFileSync } from "fs"; + +const response = await client.tts.synthesize({ + text: "Hello from AssemblyAI.", + voice_id: "aai-jane", + container: "wav", +}); +writeFileSync("hello.wav", response.audio); +console.log(response.model, response.voiceId, response.audioDurationMs, "ms"); +``` + +`synthesize()` waits for the whole audio, so an API refusal is always a `TextToSpeechError` and the WAV header carries real sizes. The response headers are on the result: `requestId`, `region`, `model`, `voiceId`, `language`, `encoding`, `sampleRate`, `container`, `bitRate`, `billedCharacters` and `audioDurationMs`. + +### Stream audio as it is generated + +```typescript +const speech = await client.tts.stream({ + text: "Streaming straight to the speakers.", + voice_id: "aai-alba", +}); +console.log( + "speaking with", + speech.info.model, + "at", + speech.info.sampleRate, + "Hz", +); +for await (const chunk of speech) { + play(chunk); // raw 24 kHz 16-bit mono PCM, as a Uint8Array +} +``` + +`stream()` resolves once the API has sent the first audio. Chunk sizes vary and are not aligned to samples or frames: for 16-bit PCM carry an odd trailing byte over to the next chunk, and re-frame for a fixed-size sink (20 ms of 8 kHz mu-law is 160 bytes). The API commits to `200` with the first audio, so a failure after that cuts the body: the loop then throws a `TextToSpeechError` with `retryable: true`, and the whole request should be retried. `speech.body` is the underlying web `ReadableStream`. + +### Speak text as it is written (voice agents) + +A `TextToSpeechSession` keeps one WebSocket open and speaks text as it is written, so an LLM's reply can be voiced sentence by sentence with no per-request handshake. `speak()` sends one text and returns its audio; `generate()` / `flush()` do the same in two steps, so tokens can be sent as they arrive; `cancel()` is the barge-in. `close()` ends the session and resolves with its summary. + +```typescript +const session = client.tts.session({ voiceId: "aai-jane" }); +await session.connect(); + +for (const sentence of llmReplySentences()) { + for await (const chunk of session.speak(sentence)) play(chunk); +} + +const summary = await session.close(); +console.log(summary?.audio_duration_seconds, "s spoken"); +``` + +Streaming the text in as it is generated, with a barge-in. Audio starts before `flush()` once whole sentences have arrived, so read `session.audio()` with no id while the tokens are still being sent: + +```typescript +const session = client.tts.session({ voiceId: "aai-jane" }); +await session.connect(); + +const player = (async () => { + for await (const chunk of session.audio()) { + if (userStartedTalking()) { + session.cancel(); // ends this iteration at once; the server drops the rest + break; + } + play(chunk); + } +})(); +for await (const token of llmTokens()) session.generate(token); // joined as sent: include the spaces +session.flush(); +await player; +await session.close(); +``` + +Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order; every `flush()` and `cancel()` takes the next id, so ids skip one after a cancel. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters, so pace text to the audio. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes after `inactivityTimeout` seconds (5 to 120, default 120) without a frame; call `keepAlive()` between sparse requests. + +Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send, reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `TextToSpeechError.status` is undefined and `requestId` is the session id; `retryable: true` means a new session can succeed, and the same session object can `connect()` again. + +In a browser the API key must stay on your server: mint a temporary token there with `await client.tts.createTemporaryToken({ expires_in_seconds: 60 })` and open the session with `client.tts.session({ token })`. + ## Contributing If you want to contribute to the JavaScript SDK, follow the guidelines in [CONTRIBUTING.md](./CONTRIBUTING.md). diff --git a/package.json b/package.json index 7670770..d19a43e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "assemblyai", - "version": "4.41.7", + "version": "4.42.0", "description": "The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription.", "engines": { "node": ">=18" diff --git a/samples/tts.ts b/samples/tts.ts new file mode 100644 index 0000000..adb4e4e --- /dev/null +++ b/samples/tts.ts @@ -0,0 +1,51 @@ +/** + * Text-to-Speech: a WAV file over HTTP, then a voice-agent style session + * that speaks a reply sentence by sentence and is interrupted once. + */ +import { writeFileSync } from "fs" +import { AssemblyAI, TextToSpeechError } from "assemblyai" + +const client = new AssemblyAI({ apiKey: "YOUR_API_KEY" }) + +async function main() { + // One request, the whole audio + const response = await client.tts.synthesize({ + text: "Hello from AssemblyAI.", + voice_id: "aai-jane", + container: "wav", + }) + writeFileSync("hello.wav", response.audio) + console.log( + `hello.wav: ${response.audioDurationMs} ms, ${response.billedCharacters} characters`, + ) + + // A session: text as it is written, audio as it is generated + const session = client.tts.session({ voiceId: "aai-alba" }) + await session.connect() + const reply = [ + "Sure, I can help with that.", + "Your appointment is on Tuesday at three in the afternoon.", + "Is there anything else?", + ] + for (const [index, sentence] of reply.entries()) { + let bytes = 0 + for await (const chunk of session.speak(sentence)) { + bytes += chunk.byteLength + if (index === 1 && bytes > 48_000) { + session.cancel() // the user interrupted after a second of audio + break + } + } + console.log(`sentence ${index + 1}: ${bytes} bytes of 24 kHz PCM`) + } + const summary = await session.close() + console.log(`spoke ${summary?.audio_duration_seconds} s in total`) +} + +main().catch((error) => { + if (error instanceof TextToSpeechError) { + console.error(error.errorCode, error.param, error.message) + } else { + throw error + } +}) diff --git a/src/services/index.ts b/src/services/index.ts index f902500..002b1ee 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -2,6 +2,12 @@ import { BaseServiceParams } from ".."; import { SyncTranscriber, SyncLiveSession } from "./sync"; import { DictationTranscriber, DictationLiveSession } from "./dictation"; import { LlmGatewayService } from "./llm-gateway"; +import { + TextToSpeechService, + TextToSpeechSession, + TextToSpeechStream, +} from "./tts"; +import { TextToSpeechError } from "../utils/errors/tts"; import { SyncTranscriptError, DictationError, @@ -33,6 +39,7 @@ const defaultStreamingUrl = "https://streaming.assemblyai.com"; const defaultSyncUrl = "https://sync.assemblyai.com"; const defaultDictationUrl = "https://dictation.assemblyai.com"; const defaultLlmGatewayUrl = "https://llm-gateway.assemblyai.com"; +const defaultTtsUrl = "https://tts.us.assemblyai.com"; class AssemblyAI { /** @@ -70,6 +77,11 @@ class AssemblyAI { */ public llmGateway: LlmGatewayService; + /** + * The Text-to-Speech service: `synthesize()`, `stream()` and WebSocket `session()`. + */ + public tts: TextToSpeechService; + /** * Create a new AssemblyAI client. * @param params - The parameters for the service, including the API key and base URL, if any. @@ -115,6 +127,15 @@ class AssemblyAI { ...params, baseUrl: llmGatewayBaseUrl, }); + + let ttsBaseUrl = params.ttsBaseUrl || defaultTtsUrl; + if (ttsBaseUrl.endsWith("/")) { + ttsBaseUrl = ttsBaseUrl.slice(0, -1); + } + this.tts = new TextToSpeechService({ + ...params, + baseUrl: ttsBaseUrl, + }); } } @@ -128,6 +149,10 @@ export { DictationError, LlmGatewayService, LlmGatewayError, + TextToSpeechService, + TextToSpeechSession, + TextToSpeechStream, + TextToSpeechError, RealtimeTranscriberFactory, RealtimeTranscriber, RealtimeServiceFactory, diff --git a/src/services/tts/index.ts b/src/services/tts/index.ts new file mode 100644 index 0000000..2a01fb4 --- /dev/null +++ b/src/services/tts/index.ts @@ -0,0 +1,2 @@ +export { TextToSpeechService, TextToSpeechStream } from "./service"; +export { TextToSpeechSession } from "./session"; diff --git a/src/services/tts/service.ts b/src/services/tts/service.ts new file mode 100644 index 0000000..fc0fae7 --- /dev/null +++ b/src/services/tts/service.ts @@ -0,0 +1,347 @@ +import { + BaseServiceParams, + SpeechRequest, + SpeechRequestOptions, + SpeechResponse, + SpeechResponseInfo, + TextToSpeechSessionParams, + TextToSpeechTokenParams, +} from "../.."; +import { BaseService } from "../base"; +import { TextToSpeechError } from "../../utils/errors/tts"; +import { deadlineSignal } from "../../utils/live"; + +/** + * A signal that aborts when the external one does, or after `timeoutMs` + * unless `settled()` runs first: the timeout covers the request up to the + * headers, not a body the caller then reads at playback pace. + */ +function headersDeadline(timeoutMs: number, external?: AbortSignal) { + const controller = new AbortController(); + const timer = setTimeout( + () => controller.abort(new Error("timeout")), + timeoutMs, + ); + const forward = () => controller.abort(external?.reason); + if (external?.aborted) forward(); + external?.addEventListener("abort", forward, { once: true }); + return { + signal: controller.signal, + settled: () => { + clearTimeout(timer); + external?.removeEventListener("abort", forward); + }, + }; +} +import { TextToSpeechSession } from "./session"; + +const ENDPOINT_SPEECH = "/v1/speech"; +const DEFAULT_TOKEN_BASE_URL = "https://token.assemblyai.com"; +/** Each socket operation of a streamed request. */ +const DEFAULT_STREAM_TIMEOUT_MS = 30_000; +/** A buffered request waits for the whole audio; the API caps a request at 600 s. */ +const DEFAULT_SYNTHESIZE_TIMEOUT_MS = 610_000; +const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]); + +/** Builds the `TextToSpeechError` for a non-200 response. */ +export async function speechErrorFromResponse( + response: Response, +): Promise { + const status = response.status; + const requestId = response.headers.get("x-request-id") ?? undefined; + const retryAfterHeader = response.headers.get("retry-after"); + let retryAfter = retryAfterHeader ? Number(retryAfterHeader) : undefined; + if (retryAfter !== undefined && !Number.isFinite(retryAfter)) + retryAfter = undefined; + + let body: Record | undefined; + if (response.headers.get("content-type")?.startsWith("application/json")) { + try { + const parsed: unknown = await response.json(); + if (parsed && typeof parsed === "object") + body = parsed as Record; + } catch { + body = undefined; + } + } + if (body) { + const message = + (typeof body.message === "string" && body.message) || + (typeof body.error === "string" && body.error) || + `text-to-speech request failed with status ${status}`; + const hint = body.retry_after_seconds; + if (typeof hint === "number") retryAfter = Math.ceil(hint); + return new TextToSpeechError(message, { + status, + errorCode: typeof body.code === "string" ? body.code : undefined, + param: typeof body.param === "string" ? body.param : undefined, + requestId: + typeof body.request_id === "string" ? body.request_id : requestId, + retryable: + typeof body.retryable === "boolean" + ? body.retryable + : RETRYABLE_STATUSES.has(status), + retryAfter, + errors: Array.isArray(body.errors) + ? (body.errors as { message: string; param: string | null }[]) + : undefined, + body, + }); + } + const text = (await response.text().catch(() => "")).trim(); + return new TextToSpeechError( + `text-to-speech request failed with status ${status}${text ? `: ${text.slice(0, 200)}` : ""}`, + { + status, + requestId, + retryable: RETRYABLE_STATUSES.has(status), + retryAfter, + }, + ); +} + +function intHeader(headers: Headers, name: string): number | undefined { + const value = headers.get(name); + if (value === null || !/^\s*\d+\s*$/.test(value)) return undefined; + return Number(value); +} + +/** Reads the `X-TTS-*` headers of a `200`. */ +export function speechResponseInfo( + headers: Headers, + requestedVoiceId?: string, +): SpeechResponseInfo { + // `X-TTS-Voice-Id` is the voice; until the API sends it, derive it from + // `X-TTS-Voice` (`custom` is the request's own voice id, else a preset). + let voiceId = headers.get("x-tts-voice-id") ?? undefined; + if (voiceId === undefined) { + const voice = headers.get("x-tts-voice"); + if (voice === "custom") voiceId = requestedVoiceId; + else if (voice) voiceId = `aai-${voice}`; + } + return { + requestId: headers.get("x-request-id") ?? undefined, + region: headers.get("x-tts-region") ?? undefined, + model: headers.get("x-tts-model") ?? undefined, + voiceId, + language: headers.get("x-tts-language") ?? undefined, + encoding: headers.get("x-tts-encoding") ?? undefined, + sampleRate: intHeader(headers, "x-tts-sample-rate"), + container: headers.get("x-tts-container") ?? undefined, + bitRate: intHeader(headers, "x-tts-bit-rate"), + contentType: headers.get("content-type") ?? undefined, + voiceExpiresAt: intHeader(headers, "x-tts-voice-expires-at"), + billedCharacters: intHeader(headers, "x-tts-billed-characters"), + audioDurationMs: intHeader(headers, "x-tts-audio-duration-ms"), + }; +} + +/** + * The audio of one streamed request, read as the API generates it. + * + * Iterate it (`for await (const chunk of speech)`) or read `body`, a web + * `ReadableStream`. Chunk sizes vary and are not aligned to samples. A + * failure after the first audio cuts the body: the iteration then throws a + * `TextToSpeechError` with `retryable: true`. + */ +export class TextToSpeechStream implements AsyncIterable { + constructor( + /** What the response headers say: model, voice, encoding, sample rate, request id. */ + public readonly info: SpeechResponseInfo, + /** The audio bytes as they arrive. */ + public readonly body: ReadableStream, + ) {} + + async *[Symbol.asyncIterator](): AsyncIterator { + const reader = this.body.getReader(); + let finished = false; + try { + for (;;) { + let result: ReadableStreamReadResult; + try { + result = await reader.read(); + } catch { + finished = true; + throw new TextToSpeechError( + "The response was cut before the audio was complete. Retry the request.", + { status: 200, requestId: this.info.requestId, retryable: true }, + ); + } + if (result.done) { + finished = true; + return; + } + if (result.value.byteLength > 0) yield result.value; + } + } finally { + // A loop left early (a barge-in) must not keep downloading the rest. + if (!finished) await reader.cancel().catch(() => undefined); + reader.releaseLock(); + } + } + + /** Reads the rest of the audio and returns it. */ + async bytes(): Promise { + const chunks: Uint8Array[] = []; + for await (const chunk of this) chunks.push(chunk); + let size = 0; + for (const chunk of chunks) size += chunk.byteLength; + const out = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.byteLength; + } + return out; + } + + /** Stops reading; the connection is released. */ + async cancel(): Promise { + await this.body.cancel().catch(() => undefined); + } +} + +/** + * The Text-to-Speech API: text in, audio out. + * + * `synthesize()` returns one text's complete audio; `stream()` returns it as + * the API generates it; `session()` opens a WebSocket session that speaks + * text as it is written, for voice agents. Requests go to the client's + * `ttsBaseUrl` (`https://tts.us.assemblyai.com` by default; + * `https://tts.eu.assemblyai.com` keeps them in the EU). + * + * @example + * ```ts + * const client = new AssemblyAI({ apiKey: "..." }); + * const { audio } = await client.tts.synthesize({ text: "Hello.", voice_id: "aai-jane", container: "wav" }); + * writeFileSync("hello.wav", audio); + * ``` + */ +export class TextToSpeechService extends BaseService { + private readonly serviceParams: BaseServiceParams; + + constructor(params: BaseServiceParams) { + super(params); + this.serviceParams = params; + } + + /** + * Speaks `request.text` and returns the complete audio. + * + * Nothing is sent back until the audio is complete, so a long text takes + * several seconds and an API failure is always a `TextToSpeechError`, + * never a cut response. + * @throws TextToSpeechError on any API refusal. + */ + async synthesize( + request: SpeechRequest, + options: SpeechRequestOptions = {}, + ): Promise { + const response = await this.fetchResponse(ENDPOINT_SPEECH, { + method: "POST", + body: JSON.stringify({ ...request, stream: false }), + signal: deadlineSignal( + options.timeout ?? DEFAULT_SYNTHESIZE_TIMEOUT_MS, + options.signal, + ), + }); + if (!response.ok) throw await speechErrorFromResponse(response); + const audio = new Uint8Array(await response.arrayBuffer()); + return { ...speechResponseInfo(response.headers, request.voice_id), audio }; + } + + /** + * Speaks `request.text` and returns the audio as it is generated. + * + * Resolves once the API has sent the first audio. The API commits to `200` + * then, so a later failure cuts the body and the iteration throws. + * @throws TextToSpeechError if the API refuses the request before any audio. + */ + async stream( + request: SpeechRequest, + options: SpeechRequestOptions = {}, + ): Promise { + const deadline = headersDeadline( + options.timeout ?? DEFAULT_STREAM_TIMEOUT_MS, + options.signal, + ); + let response: Response; + try { + response = await this.fetchResponse(ENDPOINT_SPEECH, { + method: "POST", + body: JSON.stringify({ ...request, stream: true }), + signal: deadline.signal, + }); + } finally { + deadline.settled(); + } + if (!response.ok) throw await speechErrorFromResponse(response); + if (!response.body || typeof response.body.getReader !== "function") { + // A runtime without web streamed bodies: hand the whole audio over as one chunk. + const bytes = new Uint8Array(await response.arrayBuffer()); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(bytes); + controller.close(); + }, + }); + return new TextToSpeechStream( + speechResponseInfo(response.headers, request.voice_id), + body, + ); + } + return new TextToSpeechStream( + speechResponseInfo(response.headers, request.voice_id), + response.body, + ); + } + + /** + * Creates a WebSocket session; nothing connects until `connect()`. + * + * The client's API key is used when `params` has neither `token` nor + * `apiKey`; the session host follows the client's `ttsBaseUrl` unless + * `websocketBaseUrl` is given. + */ + session(params: TextToSpeechSessionParams = {}): TextToSpeechSession { + const sessionParams = { ...params }; + if (!sessionParams.token && !sessionParams.apiKey) { + sessionParams.apiKey = this.serviceParams.apiKey; + } + if (!sessionParams.websocketBaseUrl && this.serviceParams.baseUrl) { + const base = new URL(this.serviceParams.baseUrl); + base.protocol = base.protocol === "http:" ? "ws:" : "wss:"; + base.pathname = base.pathname.replace(/\/$/, "") + "/v1/ws"; + sessionParams.websocketBaseUrl = base.toString(); + } + return new TextToSpeechSession(sessionParams); + } + + /** + * Mints a temporary token for a session opened on a user's device, where + * the API key must not go. Call this server-side. + */ + async createTemporaryToken( + params: TextToSpeechTokenParams, + options: { tokenBaseUrl?: string } = {}, + ): Promise { + const search = new URLSearchParams({ product: "tts" }); + Object.entries(params).forEach(([key, value]) => { + if (value !== undefined && value !== null) + search.append(key, String(value)); + }); + const base = (options.tokenBaseUrl ?? DEFAULT_TOKEN_BASE_URL).replace( + /\/$/, + "", + ); + const response = await this.fetchResponse( + `${base}/v1/token?${search.toString()}`, + { + method: "GET", + }, + ); + if (!response.ok) throw await speechErrorFromResponse(response); + const data = (await response.json()) as { token: string }; + return data.token; + } +} diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts new file mode 100644 index 0000000..ff15d57 --- /dev/null +++ b/src/services/tts/session.ts @@ -0,0 +1,649 @@ +import { conditions } from "#conditions"; +import { + factory as polyfillWebSocketFactory, + PolyfillWebSocket, +} from "#websocket"; +import { CloseEvent, ErrorEvent, MessageEvent } from "ws"; +import { + TextToSpeechSessionEvents, + TextToSpeechSessionListeners, + TextToSpeechSessionParams, + TtsAudioEvent, + TtsBeginEvent, + TtsCancelledEvent, + TtsErrorMessage, + TtsFlushDoneEvent, + TtsTerminationEvent, + TtsWordBoundariesEvent, +} from "../.."; +import { TextToSpeechError } from "../../utils/errors/tts"; + +const DEFAULT_WEBSOCKET_BASE_URL = "wss://tts.us.assemblyai.com/v1/ws"; +const DEFAULT_CONNECT_TIMEOUT_MS = 10_000; +const DEFAULT_TERMINATE_TIMEOUT_MS = 10_000; +/** Characters one `Generate` frame may carry, unless `Begin` says otherwise. */ +const MAX_GENERATE_CHARS = 2000; +/** Audio frames queued for pulling before the oldest is dropped (~160 s). */ +const PULL_QUEUE_FRAMES = 2000; +/** Closes after which a new session can succeed. */ +const RETRYABLE_CLOSE_CODES = new Set([1006, 1011, 1012, 3005, 3008, 3009]); + +type PullItem = + | { kind: "audio"; flushId: number; bytes: Uint8Array } + | { kind: "done"; flushId: number } + | { kind: "cancelled"; cancelId: number } + | { kind: "error"; error: TextToSpeechError } + | { kind: "closed" }; + +/** A FIFO that hands items to one waiting reader; one per connection. */ +class PullQueue { + private items: PullItem[] = []; + private audioItems = 0; + private waiter?: (item: PullItem) => void; + dropped = false; + + get size() { + return this.items.length; + } + + push(item: PullItem) { + if (this.waiter) { + const resolve = this.waiter; + this.waiter = undefined; + resolve(item); + return; + } + if (item.kind === "audio") { + if (this.audioItems >= PULL_QUEUE_FRAMES) { + // Drop the oldest audio, never a marker, so `audio()` always ends. + const index = this.items.findIndex((queued) => queued.kind === "audio"); + this.items.splice(index, 1); + this.audioItems--; + if (!this.dropped) { + this.dropped = true; + console.warn( + "TextToSpeechSession: dropping audio nobody reads; iterate audio() or add an audio listener", + ); + } + } + this.audioItems++; + } + this.items.push(item); + } + + next(): Promise { + const item = this.items.shift(); + if (item) { + if (item.kind === "audio") this.audioItems--; + return Promise.resolve(item); + } + return new Promise((resolve) => { + this.waiter = resolve; + }); + } +} + +/** Pieces of at most `limit` Unicode code points, never split inside a surrogate pair. */ +function splitCodePoints(text: string, limit: number): string[] { + const points = Array.from(text); + const pieces: string[] = []; + for (let start = 0; start < points.length; start += limit) { + pieces.push(points.slice(start, start + limit).join("")); + } + return pieces; +} + +function codePointLength(text: string): number { + return Array.from(text).length; +} + +function base64ToBytes(data: string): Uint8Array { + const binary = atob(data); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + return bytes; +} + +function errorFromFrame(frame: TtsErrorMessage): TextToSpeechError { + return new TextToSpeechError(frame.error || "text-to-speech session failed", { + errorCode: frame.error_type, + param: frame.param ?? undefined, + retryable: frame.retryable, + retryAfter: + typeof frame.retry_after_seconds === "number" + ? Math.ceil(frame.retry_after_seconds) + : undefined, + closeCode: frame.error_code, + body: frame as unknown as Record, + }); +} + +function errorFromClose(code: number, reason: string): TextToSpeechError { + return new TextToSpeechError( + `Connection closed (code=${code})${reason ? `: ${reason}` : ""}`, + { + closeCode: code, + retryable: RETRYABLE_CLOSE_CODES.has(code), + body: { close_code: code, reason }, + }, + ); +} + +/** + * A Text-to-Speech WebSocket session: text in as it is written, audio out. + * + * `connect()` opens the socket and waits for `Begin`. Send text with + * `generate()` as it is produced, conclude each request with `flush()`, and + * read its audio with `audio()`; `speak()` does all three for one text. + * `cancel()` is the barge-in: it stops the request in flight and ends its + * `audio()` iteration at once. `close()` sends `Terminate` and waits for + * the session's `Termination` summary. + * + * Audio is delivered one of two ways: pulled with `speak()` / `audio()`, + * one request at a time in order, or pushed to an `audio` listener + * registered with `on()`. Listeners for the other events (`flushDone`, + * `wordBoundaries`, `cancelled`, `termination`, `error`, `close`) are + * optional. A refusal rejects `connect()`; a failure mid-session rejects the + * `audio()` iteration or the next send, and a failure nothing observed + * rejects `close()`. + * + * @example + * ```ts + * const session = client.tts.session({ voiceId: "aai-jane" }); + * await session.connect(); + * for await (const chunk of session.speak("Hello there.")) play(chunk); + * await session.close(); + * ``` + */ +export class TextToSpeechSession { + private socket?: PolyfillWebSocket; + private listeners: TextToSpeechSessionListeners = {}; + private queue = new PullQueue(); + private nextFlushId = 0; + private cancelIds: number[] = []; + private cancelledBefore = -1; + private stopped = false; + private errorRaised = false; + private reading = false; + private closeEmitted = false; + private terminationResolve?: () => void; + private abortConnect?: (error: TextToSpeechError) => void; + + /** The `Begin` frame, once connected. */ + begin?: TtsBeginEvent; + /** The `Termination` summary once the session ended cleanly. */ + termination?: TtsTerminationEvent; + /** The error that ended the session, if one did. */ + error?: TextToSpeechError; + + constructor(private readonly params: TextToSpeechSessionParams) {} + + on(event: "begin", listener: (event: TtsBeginEvent) => void): void; + on(event: "audio", listener: (event: TtsAudioEvent) => void): void; + on(event: "flushDone", listener: (event: TtsFlushDoneEvent) => void): void; + on( + event: "wordBoundaries", + listener: (event: TtsWordBoundariesEvent) => void, + ): void; + on(event: "cancelled", listener: (event: TtsCancelledEvent) => void): void; + on( + event: "termination", + listener: (event: TtsTerminationEvent) => void, + ): void; + on(event: "error", listener: (error: TextToSpeechError) => void): void; + on(event: "close", listener: (code: number, reason: string) => void): void; + /** Registers the listener of a server frame type; one listener per event. */ + on(event: TextToSpeechSessionEvents, listener: (...args: never[]) => void) { + (this.listeners as Record)[event] = listener; + } + + private connectionUrl(): URL { + const url = new URL( + this.params.websocketBaseUrl ?? DEFAULT_WEBSOCKET_BASE_URL, + ); + if (url.protocol !== "wss:" && url.protocol !== "ws:") { + throw new Error( + "The WebSocket base URL must use the wss:// or ws:// protocol", + ); + } + const p = this.params; + const set = (name: string, value: unknown) => { + if (value !== undefined && value !== null) { + url.searchParams.set(name, String(value)); + } + }; + set("voice_id", p.voiceId); + set("model_id", p.modelId); + set("language", p.language); + set("sample_rate", p.sampleRate); + set("encoding", p.encoding); + set("word_boundaries", p.wordBoundaries ?? false); + set("inactivity_timeout", p.inactivityTimeout); + set("extend_biometric_consent", p.extendBiometricConsent); + set("token", p.token); + return url; + } + + private reset() { + this.queue = new PullQueue(); + this.nextFlushId = 0; + this.cancelIds = []; + this.cancelledBefore = -1; + this.stopped = false; + this.errorRaised = false; + this.reading = false; + this.closeEmitted = false; + this.begin = undefined; + this.termination = undefined; + this.error = undefined; + } + + /** + * Opens the socket and waits for `Begin`. + * + * A session the API has ended can be connected again with the same object. + * @throws TextToSpeechError if the API refuses the session, the handshake + * fails, or `Begin` does not arrive in time. + */ + connect(): Promise { + if (this.socket && !this.stopped) { + return Promise.reject(new Error("The session is already connected")); + } + this.releaseSocket(); + this.reset(); + const url = this.connectionUrl(); + const timeoutMs = this.params.connectTimeout ?? DEFAULT_CONNECT_TIMEOUT_MS; + + return new Promise((resolve, reject) => { + let settled = false; + const timer = setTimeout(() => { + settle(() => { + this.releaseSocket(); + this.stopped = true; + reject( + new TextToSpeechError( + `The session did not begin within ${timeoutMs}ms`, + { retryable: true }, + ), + ); + }); + }, timeoutMs); + const settle = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + this.abortConnect = undefined; + fn(); + }; + this.abortConnect = (error) => settle(() => reject(error)); + + if (this.params.token) { + this.socket = polyfillWebSocketFactory(url.toString()); + } else { + if (conditions.browser) { + console.warn( + `API key authentication is not supported for the TextToSpeechSession in browser environments. Use temporary token authentication instead. +Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/compat.md#browser-compatibility.`, + ); + } + this.socket = polyfillWebSocketFactory(url.toString(), { + headers: { Authorization: this.params.apiKey }, + }); + } + const socket = this.socket; + socket.binaryType = "arraybuffer"; + socket.onopen = () => {}; + + socket.onmessage = ({ data }: MessageEvent) => { + if (typeof data !== "string" && !(data instanceof ArrayBuffer)) return; + const text = + typeof data === "string" ? data : new TextDecoder().decode(data); + let frame: Record; + try { + frame = JSON.parse(text) as Record; + } catch { + return; + } + if (frame.type === "Begin") { + this.begin = frame as unknown as TtsBeginEvent; + this.listeners.begin?.(this.begin); + settle(() => resolve(this.begin as TtsBeginEvent)); + return; + } + if (frame.type === "Error") { + const error = errorFromFrame(frame as unknown as TtsErrorMessage); + this.fail(error); + settle(() => reject(error)); + return; + } + try { + this.dispatch(frame); + } catch (cause) { + const error = new TextToSpeechError( + `Unreadable frame from the API: ${String(cause)}`, + ); + this.fail(error); + settle(() => reject(error)); + } + }; + + socket.onclose = ({ code, reason }: CloseEvent) => { + const text = typeof reason === "string" ? reason : ""; + if (!this.stopped && code !== 1000 && !this.error) { + this.error = errorFromClose(code, text); + if (this.begin) this.error.requestId = this.begin.id; + this.listeners.error?.(this.error); + } + this.end(); + this.emitClose(code, text); + settle(() => reject(this.error ?? errorFromClose(code, text))); + }; + + socket.onerror = (event: ErrorEvent) => { + if (settled) return; // after Begin, the close that follows reports it + const message = + (event.error as Error | undefined)?.message ?? event.message; + const error = new TextToSpeechError( + `WebSocket handshake failed: ${message}`, + { retryable: true }, + ); + this.error = error; + this.end(); + settle(() => reject(error)); + }; + }); + } + + private dispatch(frame: Record) { + switch (frame.type) { + case "Audio": { + const event: TtsAudioEvent = { + type: "Audio", + flush_id: frame.flush_id as number, + audio: base64ToBytes(frame.audio as string), + }; + if (this.listeners.audio) this.listeners.audio(event); + else { + this.enqueue({ + kind: "audio", + flushId: event.flush_id, + bytes: event.audio, + }); + } + return; + } + case "FlushDone": { + const event = frame as unknown as TtsFlushDoneEvent; + this.enqueue({ kind: "done", flushId: event.flush_id }); + this.listeners.flushDone?.(event); + return; + } + case "WordBoundaries": + this.listeners.wordBoundaries?.( + frame as unknown as TtsWordBoundariesEvent, + ); + return; + case "Cancelled": { + const cancelId = this.cancelIds.shift() ?? -1; + this.enqueue({ kind: "cancelled", cancelId }); + this.listeners.cancelled?.(frame as unknown as TtsCancelledEvent); + return; + } + case "Termination": + this.termination = frame as unknown as TtsTerminationEvent; + this.listeners.termination?.(this.termination); + this.end(); + return; + default: + return; // a frame type this SDK does not know + } + } + + private fail(error: TextToSpeechError) { + if (!this.error) { + if (!error.requestId && this.begin) error.requestId = this.begin.id; + this.error = error; + this.listeners.error?.(error); + } + this.enqueue({ kind: "error", error: this.error }); + this.end(); + } + + /** Queues an item for `audio()`; nothing is queued while an `audio` listener receives the frames. */ + private enqueue(item: PullItem) { + if (this.listeners.audio) return; + this.queue.push(item); + } + + private emitClose(code: number, reason: string) { + if (this.closeEmitted) return; + this.closeEmitted = true; + this.listeners.close?.(code, reason); + } + + /** Marks the connection over: waiters wake up, sends refuse. */ + private end() { + if (this.stopped) return; + this.stopped = true; + this.enqueue({ kind: "closed" }); + this.terminationResolve?.(); + this.terminationResolve = undefined; + } + + private releaseSocket() { + const socket = this.socket; + this.socket = undefined; + if (!socket) return; + socket.removeAllListeners?.(); + socket.onerror = () => {}; + socket.onclose = () => {}; + socket.onmessage = () => {}; + try { + socket.close(); + } catch { + // already closed + } + } + + private surfaced(): TextToSpeechError { + this.errorRaised = true; + return this.error as TextToSpeechError; + } + + private send(frame: Record) { + if ( + !this.socket || + this.stopped || + this.socket.readyState !== this.socket.OPEN + ) { + if (this.error) throw this.surfaced(); + throw new Error("The session is not open: call connect() first"); + } + this.socket.send(JSON.stringify(frame)); + } + + private generateLimit(): number { + const limits = this.begin?.configuration?.limits as + | Record + | undefined; + const limit = limits?.max_generate_text_length; + return typeof limit === "number" && limit >= 0 ? limit : MAX_GENERATE_CHARS; + } + + private checkPullable() { + if (this.listeners.audio) { + throw new Error( + "audio goes to the registered audio listener; pull with audio() or speak() only without one", + ); + } + if (this.reading) { + throw new Error( + "one audio() iteration at a time: requests are served in order, so read them in order", + ); + } + } + + /** + * Appends `text` to the current request. + * + * Consecutive texts are joined exactly as sent, so include the spaces. + * Audio may start before `flush()` once enough complete sentences have + * arrived; `flush()` is what concludes the request. + * @throws RangeError for a text over one frame's limit (2,000 characters); split it. + */ + generate(text: string) { + if (typeof text !== "string") throw new TypeError("text must be a string"); + const limit = this.generateLimit(); + const length = codePointLength(text); + if (limit && length > limit) { + throw new RangeError( + `text is ${length} characters, over the ${limit}-character limit of one Generate frame; split it across several generate() calls`, + ); + } + this.send({ type: "Generate", text }); + } + + /** + * Concludes the current request and returns its `flush_id`. + * + * Everything sent so far is spoken, then `FlushDone` arrives for this id. + * Flushes can be pipelined; the audio comes in order. + */ + flush(): number { + const flushId = this.nextFlushId++; + this.send({ type: "Flush" }); + return flushId; + } + + /** + * Barge-in: stops the request in flight and drops queued requests. + * + * An `audio()` iteration of a cancelled request, or of the text being + * written, ends at once; audio of it still in transit is discarded. Like + * `flush()`, a cancel takes a `flush_id`, so the next request's id skips one. + */ + cancel() { + const cancelId = this.nextFlushId++; + this.cancelIds.push(cancelId); + this.cancelledBefore = Math.max(this.cancelledBefore, cancelId); + this.send({ type: "Cancel" }); + } + + /** Resets the inactivity timer between sparse requests. */ + keepAlive() { + this.send({ type: "KeepAlive" }); + } + + /** + * Speaks one text: `generate()`, `flush()`, then its audio. + * + * Text longer than one frame is split across frames. Returns an async + * iterable of the request's audio that ends at its `FlushDone`, or early + * if the request is cancelled. + */ + speak(text: string): AsyncIterable { + if (typeof text !== "string") throw new TypeError("text must be a string"); + this.checkPullable(); + const limit = this.generateLimit() || Math.max(codePointLength(text), 1); + for (const piece of splitCodePoints(text, limit)) this.generate(piece); + return this.audio(this.flush()); + } + + /** + * Yields the audio of request `flushId` until its `FlushDone`. + * + * Without `flushId`, the request being written now, so audio can be read + * while `generate()` is still sending text. Requests are served in order, + * so read them in order, one iteration at a time. Audio is queued for + * pulling only while no `audio` listener is registered. + * @throws TextToSpeechError if the session fails before the request completes. + */ + async *audio(flushId?: number): AsyncGenerator { + this.checkPullable(); + if (flushId === undefined) flushId = this.nextFlushId; + else if (flushId > this.nextFlushId) { + throw new RangeError(`flush_id ${flushId} has not been flushed yet`); + } + const connection = this.queue; + this.reading = true; + try { + for (;;) { + if (this.queue !== connection) return; // the session reconnected + // A Cancel drops every request before it and the text being written, + // whose id it takes. + if (flushId <= this.cancelledBefore) return; + if (this.stopped && connection.size === 0) { + if (this.error) throw this.surfaced(); + return; + } + const item = await connection.next(); + switch (item.kind) { + case "audio": + if (item.flushId === flushId) yield item.bytes; + break; + case "done": + if (item.flushId === flushId) return; + break; + case "cancelled": + if (item.cancelId >= flushId) return; + break; + case "error": + throw this.surfaced(); + case "closed": + if (this.error) throw this.surfaced(); + return; + } + } + } finally { + if (this.queue === connection) this.reading = false; + } + } + + /** + * Ends the session: `Terminate`, then the `Termination` summary. + * + * Text sent and not flushed is spoken first. Resolves with the summary, + * or `undefined` if the session had already ended or the wait timed out. + * A `connect()` still pending is rejected. + * @param waitForTermination - Wait for the summary before closing the socket. + * @param terminateTimeout - Milliseconds to wait for it; `0` waits indefinitely. + * @throws TextToSpeechError the session's error, when nothing else surfaced it. + */ + async close( + waitForTermination = true, + terminateTimeout = this.params.terminateTimeout ?? + DEFAULT_TERMINATE_TIMEOUT_MS, + ): Promise { + const socket = this.socket; + this.abortConnect?.( + new TextToSpeechError("The session was closed before it began"), + ); + if (socket && !this.stopped && socket.readyState === socket.OPEN) { + const terminated = new Promise((resolve) => { + this.terminationResolve = resolve; + }); + try { + socket.send(JSON.stringify({ type: "Terminate" })); + } catch { + // the socket closed under us; the close handler ends the session + } + if (waitForTermination) { + if (terminateTimeout > 0) { + let timer: ReturnType | undefined; + const expired = new Promise((resolve) => { + timer = setTimeout(resolve, terminateTimeout); + }); + await Promise.race([terminated, expired]); + if (timer) clearTimeout(timer); + } else { + await terminated; + } + } + } + this.end(); + this.releaseSocket(); + if (socket) this.emitClose(1000, "Closed by the client"); + if (this.error && !this.errorRaised) throw this.surfaced(); + return this.termination; + } +} diff --git a/src/types/index.ts b/src/types/index.ts index fb7f7f5..de0373f 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -24,3 +24,4 @@ export type UserAgentItem = { export type UserAgent = { [component: string]: UserAgentItem | undefined | null | false; }; +export * from "./tts"; diff --git a/src/types/services/index.ts b/src/types/services/index.ts index 72d8653..3e76c69 100644 --- a/src/types/services/index.ts +++ b/src/types/services/index.ts @@ -7,6 +7,8 @@ type BaseServiceParams = { syncBaseUrl?: string; dictationBaseUrl?: string; llmGatewayBaseUrl?: string; + /** The Text-to-Speech API base URL. Defaults to `https://tts.us.assemblyai.com`; `https://tts.eu.assemblyai.com` keeps requests in the EU. */ + ttsBaseUrl?: string; /** * The AssemblyAI user agent to use for requests. * The provided components will be merged into the default AssemblyAI user agent. diff --git a/src/types/tts/index.ts b/src/types/tts/index.ts new file mode 100644 index 0000000..7b6f263 --- /dev/null +++ b/src/types/tts/index.ts @@ -0,0 +1,255 @@ +import type { TextToSpeechError } from "../../utils/errors/tts"; +import type { LiteralUnion } from "../helpers"; + +/** + * Types of the Text-to-Speech API: the HTTP route `POST /v1/speech` and the + * WebSocket session on `/v1/ws`. Request bodies and server frames keep the + * API's snake_case names; values the SDK derives (response headers, session + * parameters) are camelCase, as elsewhere in this SDK. + */ + +/** Codec of the audio the API returns. */ +export type TextToSpeechEncoding = + | "pcm_s16le" + | "pcm_mulaw" + | "pcm_alaw" + | "mp3"; + +/** Container of a PCM or G.711 response on the HTTP route. */ +export type TextToSpeechContainer = "raw" | "wav"; + +/** Sample rates the API produces, in Hz. */ +export type TextToSpeechSampleRate = + | 8000 + | 16000 + | 22050 + | 24000 + | 44100 + | 48000; + +/** + * The language, by its English name: `english` (the default), `spanish`, + * `german`, `french`, `italian` or `portuguese`. Case-insensitive. + */ +export type TextToSpeechLanguage = + | "english" + | "spanish" + | "german" + | "french" + | "italian" + | "portuguese"; + +/** Options shared by an HTTP request and a session: the voice and the audio. */ +export type TextToSpeechVoiceOptions = { + /** + * The voice: a voice of the AssemblyAI voice library by its slug, such as + * `aai-jane` (the default), `aai-alba` or `aai-michael`; a voice's ID (a + * UUID); or the slug of one of the account's own cloned voices. The + * dashboard shows the value that names each voice. + */ + voice_id?: string; + /** The model to run on, such as `assemblyai-tts-1`. Defaults to the region's default model. */ + model_id?: string; + /** Defaults to the voice's own language (English for `aai-jane`). */ + language?: LiteralUnion; + /** Output sample rate in Hz. Defaults to 24000. */ + sample_rate?: TextToSpeechSampleRate; + /** Output codec. Defaults to `pcm_s16le`. A session streams PCM or G.711 only. */ + encoding?: LiteralUnion; + /** + * `true` renews the biometric consent of the cloned voice in `voice_id`, + * which only the voice's owner may do; a cloned voice expires 60 days + * after its consent was last given. + */ + extend_biometric_consent?: boolean; +}; + +/** The body of `POST /v1/speech`. */ +export type SpeechRequest = TextToSpeechVoiceOptions & { + /** 1 to 5,000 characters. */ + text: string; + /** `raw` (the default) or `wav`, with a PCM or G.711 encoding only. */ + container?: TextToSpeechContainer; + /** MP3 bit rate in bits per second (32000 to 128000, default 64000), with `encoding: "mp3"` only. */ + bit_rate?: number; +}; + +/** Per-request options of the HTTP client. */ +export type SpeechRequestOptions = { + /** + * Milliseconds until the request is aborted. For `synthesize()` the whole + * request, 610 000 by default (the response waits for the complete audio; + * the API caps a request at 600 seconds). For `stream()` the wait for the + * headers and first audio, 30 000 by default; reading the body is not + * timed, so playback at its own pace never trips it. + */ + timeout?: number; + /** An external abort signal, combined with the timeout. */ + signal?: AbortSignal; +}; + +/** What the response headers of a `200` say about the audio. */ +export type SpeechResponseInfo = { + /** The request's id (`X-Request-Id`). Quote it in support requests. */ + requestId?: string; + /** The region that served the request, such as `us-east-1`. */ + region?: string; + /** The model that spoke, such as `assemblyai-tts-1`. */ + model?: string; + /** The voice that spoke: the request's `voice_id`, or the default voice's slug. */ + voiceId?: string; + language?: string; + /** The codec used, in canonical form (`pcm_mulaw`, never `mulaw`). */ + encoding?: string; + sampleRate?: number; + /** `raw` or `wav`; absent for MP3. */ + container?: string; + bitRate?: number; + /** The response's `Content-Type`, such as `audio/pcm;rate=24000;channels=1`. */ + contentType?: string; + /** For a cloned voice of the account: its expiry as a Unix time in seconds. */ + voiceExpiresAt?: number; + /** The characters the request bills. */ + billedCharacters?: number; + /** The audio's duration in milliseconds, on a complete response. */ + audioDurationMs?: number; +}; + +/** The result of `synthesize()`: the whole audio, with its headers. */ +export type SpeechResponse = SpeechResponseInfo & { + /** The audio in the requested encoding and container, complete. */ + audio: Uint8Array; +}; + +/** Parameters of `createTemporaryToken()`; the token opens one session from a browser. */ +export type TextToSpeechTokenParams = { + /** 1 to 600 seconds the token can wait before it opens a session. */ + expires_in_seconds: number; + /** At least 60: caps the duration of the session the token opens. */ + max_session_duration_seconds?: number; +}; + +/** Parameters of a `TextToSpeechSession`. */ +export type TextToSpeechSessionParams = { + /** See `TextToSpeechVoiceOptions.voice_id`. */ + voiceId?: string; + modelId?: string; + language?: LiteralUnion; + /** Defaults to 24000. */ + sampleRate?: TextToSpeechSampleRate; + /** `pcm_s16le` (default), `pcm_mulaw` or `pcm_alaw`. A session has no MP3 or container. */ + encoding?: LiteralUnion, string>; + /** Whether the server sends `wordBoundaries` events. Off by default here. */ + wordBoundaries?: boolean; + /** Seconds (5 to 120, default 120) after which an idle session is closed. */ + inactivityTimeout?: number; + /** `true` renews the consent of the cloned voice in `voiceId` (owner only). */ + extendBiometricConsent?: boolean; + /** A temporary token, for a session opened on a user's device. */ + token?: string; + /** The API key; the client's key is used when neither this nor `token` is given. */ + apiKey?: string; + /** Defaults to `wss://tts.us.assemblyai.com/v1/ws`; `wss://tts.eu.assemblyai.com/v1/ws` for the EU zone. */ + websocketBaseUrl?: string; + /** Milliseconds to wait for the handshake and `Begin`. Defaults to 10 000. */ + connectTimeout?: number; + /** Milliseconds to wait for `Termination` on `close()`. Defaults to 10 000. */ + terminateTimeout?: number; +}; + +/** The first frame of a session: it is open and configured. */ +export type TtsBeginEvent = { + type: "Begin"; + /** The session id. Quote it in support requests. */ + id: string; + /** Unix time in seconds at which the session's maximum duration ends. */ + expires_at: number; + /** The session as created: effective parameters, `model`, `voice_id`, `limits` and more. */ + configuration: Record; +}; + +/** A piece of a request's audio. */ +export type TtsAudioEvent = { + type: "Audio"; + /** Raw codec bytes, decoded from the frame's base64. */ + audio: Uint8Array; + /** The request the audio belongs to. */ + flush_id: number; +}; + +/** A request's audio has all been delivered. */ +export type TtsFlushDoneEvent = { + type: "FlushDone"; + flush_id: number; + /** The playback duration of the request's audio; `0` for nothing to speak. */ + audio_duration_ms: number; +}; + +export type TtsWord = { + /** One whitespace-delimited token of the request's text, verbatim. */ + word: string; + /** Milliseconds on the session's whole audio timeline; subtract `audio_start_ms` for the request's. */ + start: number; + end: number; + confidence: number; +}; + +/** Word timings of a request, best effort, after its `FlushDone`. */ +export type TtsWordBoundariesEvent = { + type: "WordBoundaries"; + flush_id: number; + /** Where the request's audio starts on the session's timeline. */ + audio_start_ms: number; + words: TtsWord[]; +}; + +/** The answer to a `Cancel`. */ +export type TtsCancelledEvent = { + type: "Cancelled"; + /** Audio streamed for the cancelled request before it stopped. */ + audio_duration_seconds: number; +}; + +/** The session's summary, after `Terminate`. */ +export type TtsTerminationEvent = { + type: "Termination"; + session_duration_seconds: number; + /** The billed characters. */ + total_input_char_length: number; + audio_duration_seconds: number; + voice_expires_at?: number | null; +}; + +/** The frame that precedes a failure close. */ +export type TtsErrorMessage = { + type: "Error"; + /** The WebSocket close code that follows. */ + error_code: number; + error: string; + /** The stable cause, such as `voice_not_found` or `rate_limited`. */ + error_type: string; + retryable: boolean; + param: string | null; + retry_after_seconds: number | null; +}; + +export type TextToSpeechSessionEvents = + | "begin" + | "audio" + | "flushDone" + | "wordBoundaries" + | "cancelled" + | "termination" + | "error" + | "close"; + +export type TextToSpeechSessionListeners = { + begin?: (event: TtsBeginEvent) => void; + audio?: (event: TtsAudioEvent) => void; + flushDone?: (event: TtsFlushDoneEvent) => void; + wordBoundaries?: (event: TtsWordBoundariesEvent) => void; + cancelled?: (event: TtsCancelledEvent) => void; + termination?: (event: TtsTerminationEvent) => void; + error?: (error: TextToSpeechError) => void; + close?: (code: number, reason: string) => void; +}; diff --git a/src/utils/errors/index.ts b/src/utils/errors/index.ts index 4a1db5a..757de3b 100644 --- a/src/utils/errors/index.ts +++ b/src/utils/errors/index.ts @@ -1,6 +1,8 @@ export { SyncTranscriptError } from "./sync"; export { DictationError } from "./dictation"; export { LlmGatewayError } from "./llm-gateway"; +export { TextToSpeechError } from "./tts"; +export type { TextToSpeechErrorFields } from "./tts"; export { RealtimeError, diff --git a/src/utils/errors/tts.ts b/src/utils/errors/tts.ts new file mode 100644 index 0000000..6ad343c --- /dev/null +++ b/src/utils/errors/tts.ts @@ -0,0 +1,47 @@ +/** The fields a Text-to-Speech failure carries besides its message. */ +export type TextToSpeechErrorFields = { + /** The HTTP status; absent for a session failure, which has no HTTP status. */ + status?: number; + /** The API's stable code: `invalid_parameter`, `voice_not_found`, `rate_limited`, ... */ + errorCode?: string; + /** The field at fault, if one. */ + param?: string; + /** The request's id, or the session's id on a session. */ + requestId?: string; + /** Whether the same request, or a new session, can succeed later. */ + retryable?: boolean; + /** Seconds to wait before retrying, when the API said. */ + retryAfter?: number; + /** For `invalid_parameter`, every failing field. */ + errors?: { message: string; param: string | null }[]; + /** The WebSocket close code that ended a session. */ + closeCode?: number; + /** The whole error envelope or frame the API sent. */ + body?: Record; +}; + +/** + * Error thrown when a Text-to-Speech request or session fails. + * + * Branch on `errorCode`, not on the message, which may be reworded. + */ +export class TextToSpeechError + extends Error + implements TextToSpeechErrorFields +{ + override name = "TextToSpeechError"; + status?: number; + errorCode?: string; + param?: string; + requestId?: string; + retryable?: boolean; + retryAfter?: number; + errors?: { message: string; param: string | null }[]; + closeCode?: number; + body?: Record; + + constructor(message: string, fields: TextToSpeechErrorFields = {}) { + super(message); + Object.assign(this, fields); + } +} diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts new file mode 100644 index 0000000..fc9dbf6 --- /dev/null +++ b/tests/unit/tts-session.test.ts @@ -0,0 +1,467 @@ +jest.mock("ws", () => require("./mocks/ws")); + +import WS from "jest-websocket-mock"; +import fetchMock from "jest-fetch-mock"; +import { AssemblyAI, TextToSpeechError, TextToSpeechSession } from "../../src"; +import { createClient } from "./utils"; + +fetchMock.enableMocks(); + +const websocketBaseUrl = "wss://localhost:1234/v1/ws"; + +const beginMessage = { + type: "Begin", + id: "sess-1", + expires_at: 1789556741, + configuration: { + voice_id: "aai-jane", + model: "assemblyai-tts-1", + limits: { max_generate_text_length: 2000 }, + }, +}; +const terminationMessage = { + type: "Termination", + session_duration_seconds: 3, + total_input_char_length: 20, + audio_duration_seconds: 1.5, +}; + +/** + * Plays the API: Generate/Flush answer with the text's bytes as one Audio + * frame and a FlushDone, Cancel with Cancelled, Terminate with Termination + * and a clean close. + */ +class FakeApi { + text = ""; + flushId = 0; + answerFlush = true; + sent: Record[] = []; + + constructor(public server: WS) { + server.on("connection", (socket) => { + this.text = ""; // a new connection is a new session: ids start at 0 + this.flushId = 0; + socket.on("message", (raw) => { + const frame = JSON.parse(String(raw)) as Record; + this.sent.push(frame); + this.handle(frame, socket); + }); + }); + } + + private handle( + frame: Record, + socket: { + send: (data: string) => void; + close: (options?: { + code: number; + reason: string; + wasClean: boolean; + }) => void; + }, + ) { + switch (frame.type) { + case "Generate": + this.text += frame.text as string; + return; + case "Flush": + if (this.answerFlush) { + socket.send( + JSON.stringify({ + type: "Audio", + audio: Buffer.from(this.text).toString("base64"), + flush_id: this.flushId, + }), + ); + socket.send( + JSON.stringify({ + type: "FlushDone", + flush_id: this.flushId, + audio_duration_ms: 80, + }), + ); + } + this.flushId += 1; + this.text = ""; + return; + case "Cancel": + this.flushId += 1; + this.text = ""; + socket.send( + JSON.stringify({ type: "Cancelled", audio_duration_seconds: 0.1 }), + ); + return; + case "Terminate": + socket.send(JSON.stringify(terminationMessage)); + socket.close({ code: 1000, reason: "Session ended", wasClean: true }); + return; + } + } + + frames(type: string) { + return this.sent.filter((f) => f.type === type); + } +} + +let server: WS; +let api: FakeApi; +let client: AssemblyAI; + +async function connect( + session: TextToSpeechSession, + begin: unknown = beginMessage, +) { + const promise = session.connect(); + await server.connected; + if (begin) server.send(JSON.stringify(begin)); + return promise; +} + +async function collect(iterable: AsyncIterable): Promise { + const chunks: Uint8Array[] = []; + for await (const chunk of iterable) chunks.push(chunk); + return Buffer.concat(chunks.map((c) => Buffer.from(c))); +} + +const tick = () => new Promise((resolve) => setTimeout(resolve, 20)); + +describe("tts session", () => { + beforeEach(() => { + server = new WS(websocketBaseUrl); + api = new FakeApi(server); + client = createClient(); + }); + afterEach(() => { + WS.clean(); + }); + + it("connects, waits for Begin, speaks and terminates", async () => { + const session = client.tts.session({ + websocketBaseUrl, + voiceId: "aai-alba", + sampleRate: 8000, + }); + const begin = await connect(session); + expect(begin.id).toBe("sess-1"); + expect(session.begin?.configuration.model).toBe("assemblyai-tts-1"); + const url = new URL( + (session as unknown as { socket: { url: string } }).socket.url, + ); + expect(url.pathname).toBe("/v1/ws"); + expect(Object.fromEntries(url.searchParams)).toEqual({ + voice_id: "aai-alba", + sample_rate: "8000", + word_boundaries: "false", + }); + + expect((await collect(session.speak("Hello there."))).toString()).toBe( + "Hello there.", + ); + expect(api.frames("Generate")).toEqual([ + { type: "Generate", text: "Hello there." }, + ]); + + const termination = await session.close(); + expect(termination?.total_input_char_length).toBe(20); + expect(session.termination).toEqual(termination); + expect(api.frames("Terminate")).toHaveLength(1); + await server.closed; + }); + + it("pipelines requests and reads them in flush id order", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + session.generate("One. "); + const first = session.flush(); + session.generate("Two."); + const second = session.flush(); + expect([first, second]).toEqual([0, 1]); + expect((await collect(session.audio(first))).toString()).toBe("One. "); + expect((await collect(session.audio(second))).toString()).toBe("Two."); + await session.close(); + }); + + it("cancel ends the current iteration at once and drops every pipelined request", async () => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const first = session.flush(); + const second = session.flush(); + const stream = session.audio(first); + server.send( + JSON.stringify({ + type: "Audio", + audio: Buffer.from("A").toString("base64"), + flush_id: 0, + }), + ); + expect( + Buffer.from((await stream.next()).value as Uint8Array).toString(), + ).toBe("A"); + session.cancel(); + expect((await stream.next()).done).toBe(true); + expect((await collect(session.audio(second))).length).toBe(0); // also dropped, no hang + + api.answerFlush = true; + expect((await collect(session.speak("Next."))).toString()).toBe("Next."); + expect(api.frames("Cancel")).toHaveLength(1); + await session.close(); + }); + + it("a cancel before a request does not end it", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + session.cancel(); + expect((await collect(session.speak("Hi."))).toString()).toBe("Hi."); + await session.close(); + }); + + it("audio() without an id follows the request being written", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const reading = collect(session.audio()); + session.generate("Early "); + session.generate("audio."); + session.flush(); + expect((await reading).toString()).toBe("Early audio."); + await session.close(); + }); + + it("speak splits long text across Generate frames", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const text = "a".repeat(2000) + "b".repeat(500); + expect((await collect(session.speak(text))).toString()).toBe(text); + expect( + api.frames("Generate").map((f) => (f.text as string).length), + ).toEqual([2000, 500]); + expect(() => session.generate("x".repeat(2001))).toThrow(RangeError); + await session.close(); + }); + + it("an Error frame before Begin rejects connect()", async () => { + const session = client.tts.session({ + websocketBaseUrl, + voiceId: "aai-nobody", + }); + const promise = session.connect(); + await server.connected; + server.send( + JSON.stringify({ + type: "Error", + error_code: 3012, + error: "The requested voice was not found.", + error_type: "voice_not_found", + retryable: false, + param: "voice_id", + retry_after_seconds: null, + }), + ); + server.close({ + code: 3012, + reason: "See Error message for details", + wasClean: true, + }); + const error = (await promise.catch((e) => e)) as TextToSpeechError; + expect(error).toBeInstanceOf(TextToSpeechError); + expect(error).toMatchObject({ + errorCode: "voice_not_found", + param: "voice_id", + retryable: false, + closeCode: 3012, + }); + }); + + it("an Error frame mid-session rejects the audio and reaches the error listener", async () => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + const errors: TextToSpeechError[] = []; + session.on("error", (e) => errors.push(e)); + await connect(session); + const flushId = session.flush(); + server.send( + JSON.stringify({ + type: "Error", + error_code: 3005, + error: "No room", + error_type: "at_capacity", + retryable: true, + param: null, + retry_after_seconds: 2, + }), + ); + const error = (await collect(session.audio(flushId)).catch( + (e) => e, + )) as TextToSpeechError; + expect(error.errorCode).toBe("at_capacity"); + expect(error.retryAfter).toBe(2); + expect(error.requestId).toBe("sess-1"); + expect(errors).toEqual([error]); + expect(() => session.generate("late")).toThrow(error); + await expect(session.close()).resolves.toBeUndefined(); // already surfaced + }); + + it("a failure nobody observed rejects close()", async () => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + session.on("audio", () => undefined); // push mode + await connect(session); + session.generate("Never read."); + session.flush(); + server.send( + JSON.stringify({ + type: "Error", + error_code: 3010, + error: "Too fast", + error_type: "input_rate_exceeded", + retryable: false, + param: null, + retry_after_seconds: null, + }), + ); + await tick(); + await expect(session.close()).rejects.toMatchObject({ + errorCode: "input_rate_exceeded", + }); + }); + + it("an audio listener receives the frames and pulling is refused", async () => { + const session = client.tts.session({ websocketBaseUrl }); + const received: Uint8Array[] = []; + const done: number[] = []; + session.on("audio", (e) => received.push(e.audio)); + session.on("flushDone", (e) => done.push(e.audio_duration_ms)); + await connect(session); + session.generate("Pushed."); + const flushId = session.flush(); + await expect(session.audio(flushId).next()).rejects.toThrow( + /audio listener/, + ); + await tick(); + expect(Buffer.from(received[0]).toString()).toBe("Pushed."); + expect(done).toEqual([80]); + await session.close(); + }); + + it("a lost connection is a retryable error", async () => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const flushId = session.flush(); + server.close({ code: 1006, reason: "", wasClean: false }); + const error = (await collect(session.audio(flushId)).catch( + (e) => e, + )) as TextToSpeechError; + expect(error.retryable).toBe(true); + expect(error.closeCode).toBe(1006); + }); + + it("a token goes in the query and the session can be reused", async () => { + const session = client.tts.session({ + websocketBaseUrl, + token: "tmp", + inactivityTimeout: 30, + }); + await connect(session); + const url = new URL( + (session as unknown as { socket: { url: string } }).socket.url, + ); + expect(url.searchParams.get("token")).toBe("tmp"); + expect(url.searchParams.get("inactivity_timeout")).toBe("30"); + expect((await collect(session.speak("One."))).toString()).toBe("One."); + await session.close(); + + // same object, new connection: `server.connected` already resolved once, + // so wait for the new socket before sending its Begin + const reconnect = session.connect(); + await new Promise((resolve) => setTimeout(resolve, 50)); + server.send(JSON.stringify(beginMessage)); + await reconnect; + expect((await collect(session.speak("Two."))).toString()).toBe("Two."); + await session.close(); + }); + + it("a send before connect says what to do", () => { + const session = client.tts.session({ websocketBaseUrl }); + expect(() => session.generate("Hi.")).toThrow(/connect\(\)/); + }); + + it("cancel ends a reader following the request being written", async () => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const reading = collect(session.audio()); // no id: the text being written + session.generate("Interrupted before the flush."); + session.cancel(); + expect((await reading).length).toBe(0); // ends at once instead of hanging + api.answerFlush = true; + expect((await collect(session.speak("Next."))).toString()).toBe("Next."); + await session.close(); + }); + + it("refuses a second concurrent reader instead of losing audio", async () => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const first = session.flush(); + const second = session.flush(); + const readerA = session.audio(first); + const pendingA = readerA.next(); + await expect(session.audio(second).next()).rejects.toThrow( + /one audio\(\) iteration at a time/, + ); + session.cancel(); + expect((await pendingA).done).toBe(true); + await session.close(); + }); + + it("speak with an audio listener throws before sending anything", async () => { + const session = client.tts.session({ websocketBaseUrl }); + session.on("audio", () => undefined); + await connect(session); + expect(() => session.speak("Hi.")).toThrow(/audio listener/); + expect(api.frames("Generate")).toHaveLength(0); + await session.close(); + }); + + it("speak splits on code points, never inside a surrogate pair", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const text = "a".repeat(1999) + "\u{1F600}" + "b"; + expect((await collect(session.speak(text))).toString()).toBe(text); + const pieces = api.frames("Generate").map((f) => f.text as string); + expect(pieces).toHaveLength(2); + expect(pieces[0].endsWith("\u{1F600}")).toBe(true); // the emoji stays whole + expect(() => session.generate("\u{1F600}".repeat(2001))).toThrow( + RangeError, + ); + expect(() => session.generate("\u{1F600}".repeat(2000))).not.toThrow(); + await session.close(); + }); + + it("close() rejects a connect() still pending instead of letting it time out", async () => { + const session = client.tts.session({ + websocketBaseUrl, + connectTimeout: 5000, + }); + const pending = session.connect(); + await server.connected; + const started = Date.now(); + const rejected = expect(pending).rejects.toMatchObject({ + message: /closed before it began/, + }); + await session.close(); + await rejected; + expect(Date.now() - started).toBeLessThan(1000); + }); + + it("the close listener fires when the client closes the session", async () => { + const session = client.tts.session({ websocketBaseUrl }); + const closes: [number, string][] = []; + session.on("close", (code, reason) => closes.push([code, reason])); + await connect(session); + await session.close(); + expect(closes).toHaveLength(1); + expect(closes[0][0]).toBe(1000); + }); +}); diff --git a/tests/unit/tts.test.ts b/tests/unit/tts.test.ts new file mode 100644 index 0000000..95fcbc5 --- /dev/null +++ b/tests/unit/tts.test.ts @@ -0,0 +1,275 @@ +import fetchMock from "jest-fetch-mock"; +import { AssemblyAI, TextToSpeechError } from "../../src"; +import { createClient, defaultApiKey, requestMatches } from "./utils"; + +fetchMock.enableMocks(); + +const audio = new Uint8Array(960).map((_, i) => i % 7); +const okHeaders = { + "content-type": "audio/pcm;rate=24000;channels=1", + "x-request-id": "req-1", + "x-tts-region": "us-east-1", + "x-tts-model": "assemblyai-tts-1", + "x-tts-voice-id": "aai-jane", + "x-tts-language": "english", + "x-tts-encoding": "pcm_s16le", + "x-tts-sample-rate": "24000", + "x-tts-container": "raw", + "x-tts-billed-characters": "12", + "x-tts-audio-duration-ms": "20", +}; + +let client: AssemblyAI; + +const mockAudio = (headers: Record = okHeaders) => + fetchMock.doMockOnceIf( + requestMatches({ url: "/v1/speech", method: "POST" }), + () => + Promise.resolve({ + status: 200, + headers, + body: Buffer.from(audio) as unknown as string, + }), + ); + +const sentBody = (call = 0) => + JSON.parse(fetchMock.mock.calls[call][1]!.body as string); + +describe("tts", () => { + beforeEach(() => { + fetchMock.resetMocks(); + fetchMock.doMock(); + client = createClient(); + }); + + it("synthesize posts a buffered request and returns the audio with its headers", async () => { + mockAudio(); + const response = await client.tts.synthesize({ text: "Hello there." }); + expect(fetchMock.mock.calls[0][0]).toBe("http://localhost:1234/v1/speech"); + const init = fetchMock.mock.calls[0][1]!; + expect((init.headers as Record).Authorization).toBe( + defaultApiKey, + ); + expect(sentBody()).toEqual({ text: "Hello there.", stream: false }); + expect(Buffer.from(response.audio)).toEqual(Buffer.from(audio)); + expect(response).toMatchObject({ + requestId: "req-1", + region: "us-east-1", + model: "assemblyai-tts-1", + voiceId: "aai-jane", + language: "english", + encoding: "pcm_s16le", + sampleRate: 24000, + container: "raw", + billedCharacters: 12, + audioDurationMs: 20, + }); + expect(response.bitRate).toBeUndefined(); + }); + + it("sends the request fields as given", async () => { + mockAudio(); + await client.tts.synthesize({ + text: "Hola.", + voice_id: "aai-juan", + model_id: "assemblyai-tts-1", + sample_rate: 8000, + encoding: "pcm_mulaw", + container: "wav", + }); + expect(sentBody()).toEqual({ + text: "Hola.", + voice_id: "aai-juan", + model_id: "assemblyai-tts-1", + sample_rate: 8000, + encoding: "pcm_mulaw", + container: "wav", + stream: false, + }); + }); + + it("derives the voice id from X-TTS-Voice until the API sends X-TTS-Voice-Id", async () => { + const legacy = { ...okHeaders }; + delete (legacy as Record)["x-tts-voice-id"]; + mockAudio({ ...legacy, "x-tts-voice": "jane" }); + mockAudio({ ...legacy, "x-tts-voice": "custom" }); + expect((await client.tts.synthesize({ text: "Hi." })).voiceId).toBe( + "aai-jane", + ); + expect( + (await client.tts.synthesize({ text: "Hi.", voice_id: "aai-isaiah-2" })) + .voiceId, + ).toBe("aai-isaiah-2"); + }); + + it("stream asks for a streamed response and yields the body", async () => { + mockAudio(); + const speech = await client.tts.stream({ text: "Hello there." }); + expect(sentBody()).toEqual({ text: "Hello there.", stream: true }); + expect(speech.info.model).toBe("assemblyai-tts-1"); + const chunks: Uint8Array[] = []; + for await (const chunk of speech) chunks.push(chunk); + expect(Buffer.concat(chunks.map((c) => Buffer.from(c)))).toEqual( + Buffer.from(audio), + ); + }); + + it("stream bytes() collects the whole audio", async () => { + mockAudio(); + const speech = await client.tts.stream({ text: "Hello there." }); + expect(Buffer.from(await speech.bytes())).toEqual(Buffer.from(audio)); + }); + + it("maps the error envelope to a TextToSpeechError", async () => { + fetchMock.doMockOnceIf( + requestMatches({ url: "/v1/speech", method: "POST" }), + () => + Promise.resolve({ + status: 422, + headers: { + "content-type": "application/json", + "x-request-id": "req-2", + }, + body: JSON.stringify({ + code: "invalid_parameter", + message: "sample_rate: not in supported set", + param: "sample_rate", + request_id: "req-2", + errors: [{ message: "not in supported set", param: "sample_rate" }], + retryable: false, + retry_after_seconds: null, + }), + }), + ); + const promise = client.tts.synthesize({ + text: "Hi.", + sample_rate: 12345 as never, + }); + await expect(promise).rejects.toBeInstanceOf(TextToSpeechError); + const error = (await promise.catch((e) => e)) as TextToSpeechError; + expect(error.message).toBe("sample_rate: not in supported set"); + expect(error).toMatchObject({ + status: 422, + errorCode: "invalid_parameter", + param: "sample_rate", + requestId: "req-2", + retryable: false, + }); + expect(error.errors).toEqual([ + { message: "not in supported set", param: "sample_rate" }, + ]); + expect(error.body?.code).toBe("invalid_parameter"); + }); + + it("reads Retry-After and rounds a fractional hint up", async () => { + fetchMock.doMockOnceIf( + requestMatches({ url: "/v1/speech", method: "POST" }), + () => + Promise.resolve({ + status: 429, + headers: { "content-type": "application/json", "retry-after": "17" }, + body: JSON.stringify({ + code: "rate_limited", + message: "slow down", + retryable: true, + retry_after_seconds: 1.2, + }), + }), + ); + const error = (await client.tts + .stream({ text: "Hi." }) + .catch((e) => e)) as TextToSpeechError; + expect(error.errorCode).toBe("rate_limited"); + expect(error.retryable).toBe(true); + expect(error.retryAfter).toBe(2); + }); + + it("reports a non-JSON answer by its status and text", async () => { + fetchMock.doMockOnceIf( + requestMatches({ url: "/v1/speech", method: "POST" }), + () => + Promise.resolve({ + status: 502, + headers: { "content-type": "text/html" }, + body: "502 Bad Gateway", + }), + ); + const error = (await client.tts + .synthesize({ text: "Hi." }) + .catch((e) => e)) as TextToSpeechError; + expect(error.status).toBe(502); + expect(error.errorCode).toBeUndefined(); + expect(error.retryable).toBe(true); + expect(error.message).toContain("Bad Gateway"); + }); + + it("uses the ttsBaseUrl of the client", async () => { + const eu = new AssemblyAI({ + apiKey: "k", + ttsBaseUrl: "https://tts.eu.assemblyai.com/", + }); + fetchMock.doMockOnceIf( + (req) => req.url === "https://tts.eu.assemblyai.com/v1/speech", + () => + Promise.resolve({ + status: 200, + headers: okHeaders, + body: Buffer.from(audio) as unknown as string, + }), + ); + const response = await eu.tts.synthesize({ text: "Hallo." }); + expect(response.audio.byteLength).toBe(audio.byteLength); + }); + + it("mints a temporary token on the token host", async () => { + fetchMock.doMockOnceIf( + (req) => + req.url === + "https://token.assemblyai.com/v1/token?product=tts&expires_in_seconds=60", + JSON.stringify({ token: "tmp-token", expires_in_seconds: 60 }), + ); + expect( + await client.tts.createTemporaryToken({ expires_in_seconds: 60 }), + ).toBe("tmp-token"); + }); + + it("leaving a stream early cancels the body", async () => { + const { TextToSpeechStream } = await import( + "../../src/services/tts/service" + ); + const cancelled = jest.fn(); + let pulls = 0; + const body = new ReadableStream({ + pull(controller) { + pulls++; + controller.enqueue(new Uint8Array([pulls])); + }, + cancel: cancelled, + }); + const speech = new TextToSpeechStream({ requestId: "r" }, body); + for await (const chunk of speech) { + if (chunk[0] === 2) break; // a barge-in after two chunks + } + expect(cancelled).toHaveBeenCalledTimes(1); + }); + + it("a stream timeout covers the headers, not the body", async () => { + jest.useFakeTimers(); + try { + fetchMock.doMockOnceIf( + requestMatches({ url: "/v1/speech", method: "POST" }), + () => + Promise.resolve({ + status: 200, + headers: okHeaders, + body: Buffer.from(audio) as unknown as string, + }), + ); + const speech = await client.tts.stream({ text: "Hi." }, { timeout: 10 }); + jest.advanceTimersByTime(50); // well past the timeout, after the headers arrived + expect(Buffer.from(await speech.bytes())).toEqual(Buffer.from(audio)); + } finally { + jest.useRealTimers(); + } + }); +}); diff --git a/tests/unit/utils.ts b/tests/unit/utils.ts index fcb5c1e..d1e4dc5 100644 --- a/tests/unit/utils.ts +++ b/tests/unit/utils.ts @@ -20,5 +20,6 @@ export const createClient = () => baseUrl: defaultBaseUrl, syncBaseUrl: defaultBaseUrl, llmGatewayBaseUrl: defaultBaseUrl, + ttsBaseUrl: defaultBaseUrl, apiKey: defaultApiKey, }); From a80abf1a9239dbe7ce77a0eec76715b898bf3eb1 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:36:49 +0000 Subject: [PATCH 02/26] fix(tts): own the stream() body reader from the headers on stream() now takes the response body's reader as soon as the headers arrive and hands out a stream that pulls from it. Node 20+ fetch cancels the unlocked body of a garbage-collected Response, so an iteration that started after an await could end at once with no audio and no error. TextToSpeechStream.cancel() goes through the running iteration's reader, so a barge-in callback that calls it while a for-await loop reads now ends the loop and releases the connection; before, cancelling the locked body rejected and the rejection was swallowed. Tests run the real streamed-body path for the first time: a mid-stream cut, the body locked right after stream() resolves, and cancel() from outside a running loop and before any read. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- src/services/tts/service.ts | 32 ++++++++-- tests/unit/tts.test.ts | 116 ++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 4 deletions(-) diff --git a/src/services/tts/service.ts b/src/services/tts/service.ts index fc0fae7..f627556 100644 --- a/src/services/tts/service.ts +++ b/src/services/tts/service.ts @@ -145,6 +145,9 @@ export function speechResponseInfo( * `TextToSpeechError` with `retryable: true`. */ export class TextToSpeechStream implements AsyncIterable { + /** The reader of the running iteration, which `cancel()` goes through. */ + private reader?: ReadableStreamDefaultReader; + constructor( /** What the response headers say: model, voice, encoding, sample rate, request id. */ public readonly info: SpeechResponseInfo, @@ -153,7 +156,7 @@ export class TextToSpeechStream implements AsyncIterable { ) {} async *[Symbol.asyncIterator](): AsyncIterator { - const reader = this.body.getReader(); + const reader = (this.reader = this.body.getReader()); let finished = false; try { for (;;) { @@ -177,6 +180,7 @@ export class TextToSpeechStream implements AsyncIterable { // A loop left early (a barge-in) must not keep downloading the rest. if (!finished) await reader.cancel().catch(() => undefined); reader.releaseLock(); + if (this.reader === reader) this.reader = undefined; } } @@ -195,9 +199,14 @@ export class TextToSpeechStream implements AsyncIterable { return out; } - /** Stops reading; the connection is released. */ + /** + * Stops reading and releases the connection. Safe to call from outside a + * running `for await` (a barge-in callback): the loop then ends after the + * chunk it holds. When you read `body` with your own reader, cancel that + * reader instead. + */ async cancel(): Promise { - await this.body.cancel().catch(() => undefined); + await (this.reader ?? this.body).cancel().catch(() => undefined); } } @@ -290,9 +299,24 @@ export class TextToSpeechService extends BaseService { body, ); } + // Lock the body now and read it through this reader. Node 20+ fetch + // cancels the unlocked body of a garbage-collected `Response`, which + // would end an iteration that starts after an `await` with no audio. + const reader = response.body.getReader(); + const body = new ReadableStream( + { + async pull(controller) { + const { done, value } = await reader.read(); + if (done) controller.close(); + else controller.enqueue(value); + }, + cancel: (reason) => reader.cancel(reason), + }, + { highWaterMark: 0 }, + ); return new TextToSpeechStream( speechResponseInfo(response.headers, request.voice_id), - response.body, + body, ); } diff --git a/tests/unit/tts.test.ts b/tests/unit/tts.test.ts index 95fcbc5..dd775ce 100644 --- a/tests/unit/tts.test.ts +++ b/tests/unit/tts.test.ts @@ -253,6 +253,122 @@ describe("tts", () => { expect(cancelled).toHaveBeenCalledTimes(1); }); + /** Answers the next fetch with a real web stream as the body. */ + const mockStreamedBody = ( + body: ReadableStream, + headers: Record = okHeaders, + ) => + fetchMock.mockImplementationOnce( + async () => + ({ + ok: true, + status: 200, + headers: new Headers(headers), + body, + }) as unknown as Response, + ); + + it("stream turns a body cut after the first audio into a retryable TextToSpeechError", async () => { + const body = new ReadableStream({ + start(c) { + c.enqueue(new Uint8Array([1, 2, 3, 4])); + }, + pull(c) { + c.error(new TypeError("terminated")); + }, + }); + fetchMock.mockImplementationOnce( + async () => + ({ + ok: true, + status: 200, + headers: new Headers({ "x-request-id": "req-cut" }), + body, + }) as unknown as Response, + ); + const speech = await client.tts.stream({ text: "Hi." }); + const chunks: Uint8Array[] = []; + const error = await (async () => { + try { + for await (const c of speech) chunks.push(c); + } catch (e) { + return e; + } + })(); + expect(chunks).toHaveLength(1); + expect(error).toBeInstanceOf(TextToSpeechError); + expect(error).toMatchObject({ + status: 200, + retryable: true, + requestId: "req-cut", + }); + }); + + it("stream locks the response body as soon as the headers arrive", async () => { + // Node 20+ fetch cancels the body of a garbage-collected Response only + // while that body is unlocked, so a locked one keeps its audio. + const source = new ReadableStream({ + start(c) { + c.enqueue(audio.subarray(0, 480)); + c.enqueue(audio.subarray(480)); + c.close(); + }, + }); + mockStreamedBody(source); + const speech = await client.tts.stream({ text: "Hello there." }); + expect(source.locked).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 5)); // the caller awaits something else first + expect(speech.info.requestId).toBe("req-1"); + expect(Buffer.from(await speech.bytes())).toEqual(Buffer.from(audio)); + }); + + it("stream cancel() from outside a running loop ends it and cancels the body", async () => { + const sourceCancelled = jest.fn(); + let sent = false; + const source = new ReadableStream( + { + pull(c) { + if (sent) return new Promise(() => undefined); // the rest of the audio never arrives + sent = true; + c.enqueue(new Uint8Array([1, 2, 3, 4])); + }, + cancel: sourceCancelled, + }, + { highWaterMark: 0 }, + ); + mockStreamedBody(source); + const speech = await client.tts.stream({ text: "Hello there." }); + const chunks: Uint8Array[] = []; + let firstChunk!: () => void; + const gotFirst = new Promise((resolve) => (firstChunk = resolve)); + const loop = (async () => { + for await (const chunk of speech) { + chunks.push(chunk); + firstChunk(); + } + })(); + await gotFirst; + await speech.cancel(); // a barge-in callback while the loop waits for audio + await loop; + expect(chunks).toHaveLength(1); + expect(sourceCancelled).toHaveBeenCalledTimes(1); + }); + + it("stream cancel() before any read cancels the body", async () => { + const sourceCancelled = jest.fn(); + const source = new ReadableStream({ + pull: () => new Promise(() => undefined), + cancel: sourceCancelled, + }); + mockStreamedBody(source); + const speech = await client.tts.stream({ text: "Hello there." }); + await speech.cancel(); + expect(sourceCancelled).toHaveBeenCalledTimes(1); + const chunks: Uint8Array[] = []; + for await (const chunk of speech) chunks.push(chunk); + expect(chunks).toHaveLength(0); + }); + it("a stream timeout covers the headers, not the body", async () => { jest.useFakeTimers(); try { From 0d3a214a20e93c07d550554c8613be92c141692e Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:57:35 +0000 Subject: [PATCH 03/26] fix(tts): keep the caller's abort on the stream() body and detach from shared signals synthesize() and stream() now share one request deadline. Its timer aborts with a DOMException named TimeoutError, so both methods time out with the same error, and stream() stops the timer once the headers arrive. The caller's signal stays attached until the request ends and is then removed, however it ends. stream() kept the caller's signal only until the headers arrived, so aborting it during playback did nothing: the body kept downloading and the loop kept yielding. Aborting it now stops the download, and the iteration throws the signal's reason (an AbortError by default) instead of the retryable cut error. synthesize() left one abort listener on the caller's signal per call. A voice agent that passes one call-level signal to every sentence's request now leaves none. The docs say that Node's built-in fetch waits at most 300 s for the headers (undici headersTimeout), so a longer synthesize() needs stream() or a dispatcher with a longer headersTimeout, and that network failures and timeouts reject with the runtime's own error. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- README.md | 4 +- src/services/tts/service.ts | 138 +++++++++++++++------ src/types/tts/index.ts | 25 +++- tests/unit/tts.test.ts | 240 +++++++++++++++++++++++++++++++++--- 4 files changed, 345 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 6cbf626..f24f9fc 100644 --- a/README.md +++ b/README.md @@ -842,7 +842,7 @@ writeFileSync("hello.wav", response.audio); console.log(response.model, response.voiceId, response.audioDurationMs, "ms"); ``` -`synthesize()` waits for the whole audio, so an API refusal is always a `TextToSpeechError` and the WAV header carries real sizes. The response headers are on the result: `requestId`, `region`, `model`, `voiceId`, `language`, `encoding`, `sampleRate`, `container`, `bitRate`, `billedCharacters` and `audioDurationMs`. +`synthesize()` waits for the whole audio, so an API refusal is always a `TextToSpeechError` and the WAV header carries real sizes. The response headers are on the result: `requestId`, `region`, `model`, `voiceId`, `language`, `encoding`, `sampleRate`, `container`, `bitRate`, `billedCharacters` and `audioDurationMs`. A network failure or a timeout rejects with the runtime's own error instead (a `TypeError`, or a `DOMException` named `TimeoutError` or `AbortError`). On Node, the built-in `fetch` gives up after 300 seconds without response headers, whatever `timeout` says; for longer requests use `stream()`, or raise the limit with `setGlobalDispatcher(new Agent({ headersTimeout: 610_000 }))` from the `undici` package. ### Stream audio as it is generated @@ -865,6 +865,8 @@ for await (const chunk of speech) { `stream()` resolves once the API has sent the first audio. Chunk sizes vary and are not aligned to samples or frames: for 16-bit PCM carry an odd trailing byte over to the next chunk, and re-frame for a fixed-size sink (20 ms of 8 kHz mu-law is 160 bytes). The API commits to `200` with the first audio, so a failure after that cuts the body: the loop then throws a `TextToSpeechError` with `retryable: true`, and the whole request should be retried. `speech.body` is the underlying web `ReadableStream`. +For a barge-in, pass `{ signal }` as the second argument and abort it: the download stops and the loop throws the signal's reason (an `AbortError`), not a `TextToSpeechError`. `speech.cancel()` stops it too, and the loop just ends. One signal can serve every request of a call. + ### Speak text as it is written (voice agents) A `TextToSpeechSession` keeps one WebSocket open and speaks text as it is written, so an LLM's reply can be voiced sentence by sentence with no per-request handshake. `speak()` sends one text and returns its audio; `generate()` / `flush()` do the same in two steps, so tokens can be sent as they arrive; `cancel()` is the barge-in. `close()` ends the session and resolves with its summary. diff --git a/src/services/tts/service.ts b/src/services/tts/service.ts index f627556..4617be7 100644 --- a/src/services/tts/service.ts +++ b/src/services/tts/service.ts @@ -9,31 +9,36 @@ import { } from "../.."; import { BaseService } from "../base"; import { TextToSpeechError } from "../../utils/errors/tts"; -import { deadlineSignal } from "../../utils/live"; +import { TextToSpeechSession } from "./session"; /** - * A signal that aborts when the external one does, or after `timeoutMs` - * unless `settled()` runs first: the timeout covers the request up to the - * headers, not a body the caller then reads at playback pace. + * The abort signal of one request. It aborts when `external` does, or after + * `timeoutMs` with a `TimeoutError` unless `headersArrived()` runs first, so + * a streamed body read at playback pace is not timed. `release()` stops the + * timer and detaches from `external`; call it once the request is over, so + * a signal shared by many requests does not collect listeners. */ -function headersDeadline(timeoutMs: number, external?: AbortSignal) { +function requestDeadline(timeoutMs: number, external?: AbortSignal) { const controller = new AbortController(); const timer = setTimeout( - () => controller.abort(new Error("timeout")), + () => + controller.abort( + new DOMException(`no audio within ${timeoutMs} ms`, "TimeoutError"), + ), timeoutMs, ); const forward = () => controller.abort(external?.reason); if (external?.aborted) forward(); - external?.addEventListener("abort", forward, { once: true }); + else external?.addEventListener("abort", forward, { once: true }); return { signal: controller.signal, - settled: () => { + headersArrived: () => clearTimeout(timer), + release: () => { clearTimeout(timer); external?.removeEventListener("abort", forward); }, }; } -import { TextToSpeechSession } from "./session"; const ENDPOINT_SPEECH = "/v1/speech"; const DEFAULT_TOKEN_BASE_URL = "https://token.assemblyai.com"; @@ -142,28 +147,39 @@ export function speechResponseInfo( * Iterate it (`for await (const chunk of speech)`) or read `body`, a web * `ReadableStream`. Chunk sizes vary and are not aligned to samples. A * failure after the first audio cuts the body: the iteration then throws a - * `TextToSpeechError` with `retryable: true`. + * `TextToSpeechError` with `retryable: true`. Aborting the request's + * `signal` stops the download, and the iteration throws the signal's reason + * (an `AbortError` unless you gave another) instead. */ export class TextToSpeechStream implements AsyncIterable { /** The reader of the running iteration, which `cancel()` goes through. */ private reader?: ReadableStreamDefaultReader; + private readonly signal?: AbortSignal; constructor( /** What the response headers say: model, voice, encoding, sample rate, request id. */ public readonly info: SpeechResponseInfo, /** The audio bytes as they arrive. */ public readonly body: ReadableStream, - ) {} + /** `signal`: the request's abort signal, whose abort ends the iteration with its reason. */ + options: { signal?: AbortSignal } = {}, + ) { + this.signal = options.signal; + } async *[Symbol.asyncIterator](): AsyncIterator { const reader = (this.reader = this.body.getReader()); + const signal = this.signal; let finished = false; try { for (;;) { + // An abort is the caller's own stop, not a cut to retry. + if (signal?.aborted) throw signal.reason; let result: ReadableStreamReadResult; try { result = await reader.read(); } catch { + if (signal?.aborted) throw signal.reason; finished = true; throw new TextToSpeechError( "The response was cut before the audio was complete. Retry the request.", @@ -240,23 +256,40 @@ export class TextToSpeechService extends BaseService { * Nothing is sent back until the audio is complete, so a long text takes * several seconds and an API failure is always a `TextToSpeechError`, * never a cut response. - * @throws TextToSpeechError on any API refusal. + * + * On Node, the built-in `fetch` waits at most 300 seconds for the response + * headers (undici's `headersTimeout`), whatever `options.timeout` says. For + * longer requests use `stream()`, or raise that limit with + * `setGlobalDispatcher(new Agent({ headersTimeout: 610_000 }))` from the + * `undici` package. + * @throws TextToSpeechError on any API refusal. A network failure or a + * timeout rejects with the runtime's own error instead: a `TypeError`, a + * `DOMException` named `TimeoutError` once `options.timeout` passes, or the + * reason of `options.signal` (an `AbortError` unless you gave another). */ async synthesize( request: SpeechRequest, options: SpeechRequestOptions = {}, ): Promise { - const response = await this.fetchResponse(ENDPOINT_SPEECH, { - method: "POST", - body: JSON.stringify({ ...request, stream: false }), - signal: deadlineSignal( - options.timeout ?? DEFAULT_SYNTHESIZE_TIMEOUT_MS, - options.signal, - ), - }); - if (!response.ok) throw await speechErrorFromResponse(response); - const audio = new Uint8Array(await response.arrayBuffer()); - return { ...speechResponseInfo(response.headers, request.voice_id), audio }; + const deadline = requestDeadline( + options.timeout ?? DEFAULT_SYNTHESIZE_TIMEOUT_MS, + options.signal, + ); + try { + const response = await this.fetchResponse(ENDPOINT_SPEECH, { + method: "POST", + body: JSON.stringify({ ...request, stream: false }), + signal: deadline.signal, + }); + if (!response.ok) throw await speechErrorFromResponse(response); + const audio = new Uint8Array(await response.arrayBuffer()); + return { + ...speechResponseInfo(response.headers, request.voice_id), + audio, + }; + } finally { + deadline.release(); + } } /** @@ -264,13 +297,17 @@ export class TextToSpeechService extends BaseService { * * Resolves once the API has sent the first audio. The API commits to `200` * then, so a later failure cuts the body and the iteration throws. + * `options.signal` keeps covering the body: aborting it stops the download. * @throws TextToSpeechError if the API refuses the request before any audio. + * A network failure or a timeout rejects with the runtime's own error + * instead: a `TypeError`, a `DOMException` named `TimeoutError` once + * `options.timeout` passes, or the reason of `options.signal`. */ async stream( request: SpeechRequest, options: SpeechRequestOptions = {}, ): Promise { - const deadline = headersDeadline( + const deadline = requestDeadline( options.timeout ?? DEFAULT_STREAM_TIMEOUT_MS, options.signal, ); @@ -281,43 +318,64 @@ export class TextToSpeechService extends BaseService { body: JSON.stringify({ ...request, stream: true }), signal: deadline.signal, }); - } finally { - deadline.settled(); + } catch (error) { + deadline.release(); + throw error; } - if (!response.ok) throw await speechErrorFromResponse(response); + deadline.headersArrived(); + if (!response.ok) { + try { + throw await speechErrorFromResponse(response); + } finally { + deadline.release(); + } + } + const info = speechResponseInfo(response.headers, request.voice_id); if (!response.body || typeof response.body.getReader !== "function") { // A runtime without web streamed bodies: hand the whole audio over as one chunk. - const bytes = new Uint8Array(await response.arrayBuffer()); + let bytes: Uint8Array; + try { + bytes = new Uint8Array(await response.arrayBuffer()); + } finally { + deadline.release(); + } const body = new ReadableStream({ start(controller) { controller.enqueue(bytes); controller.close(); }, }); - return new TextToSpeechStream( - speechResponseInfo(response.headers, request.voice_id), - body, - ); + return new TextToSpeechStream(info, body, { signal: options.signal }); } // Lock the body now and read it through this reader. Node 20+ fetch // cancels the unlocked body of a garbage-collected `Response`, which // would end an iteration that starts after an `await` with no audio. + // The request stays tied to `options.signal` until the body ends, however + // it ends, also when the caller reads `body` with a reader of its own. const reader = response.body.getReader(); const body = new ReadableStream( { async pull(controller) { - const { done, value } = await reader.read(); - if (done) controller.close(); - else controller.enqueue(value); + let result: ReadableStreamReadResult; + try { + result = await reader.read(); + } catch (error) { + deadline.release(); + throw error; + } + if (result.done) { + deadline.release(); + controller.close(); + } else controller.enqueue(result.value); + }, + cancel: (reason) => { + deadline.release(); + return reader.cancel(reason); }, - cancel: (reason) => reader.cancel(reason), }, { highWaterMark: 0 }, ); - return new TextToSpeechStream( - speechResponseInfo(response.headers, request.voice_id), - body, - ); + return new TextToSpeechStream(info, body, { signal: options.signal }); } /** diff --git a/src/types/tts/index.ts b/src/types/tts/index.ts index 7b6f263..013e296 100644 --- a/src/types/tts/index.ts +++ b/src/types/tts/index.ts @@ -77,14 +77,27 @@ export type SpeechRequest = TextToSpeechVoiceOptions & { /** Per-request options of the HTTP client. */ export type SpeechRequestOptions = { /** - * Milliseconds until the request is aborted. For `synthesize()` the whole - * request, 610 000 by default (the response waits for the complete audio; - * the API caps a request at 600 seconds). For `stream()` the wait for the - * headers and first audio, 30 000 by default; reading the body is not - * timed, so playback at its own pace never trips it. + * Milliseconds until the request is aborted with a `DOMException` named + * `TimeoutError`. For `synthesize()` the whole request, 610 000 by default + * (the response waits for the complete audio; the API caps a request at + * 600 seconds). For `stream()` the wait for the headers and first audio, + * 30 000 by default; reading the body is not timed, so playback at its own + * pace never trips it. + * + * On Node, the built-in `fetch` also gives up after 300 seconds without + * response headers (undici's `headersTimeout`), so a `synthesize()` longer + * than that fails with a `TypeError` first. For longer requests use + * `stream()`, or raise that limit with + * `setGlobalDispatcher(new Agent({ headersTimeout: 610_000 }))` from the + * `undici` package. */ timeout?: number; - /** An external abort signal, combined with the timeout. */ + /** + * An external abort signal, combined with the timeout. Aborting it rejects + * the request with the signal's reason; for `stream()` it also stops the + * download of the body, and the iteration throws that reason. One signal + * can serve many requests: each detaches from it when it ends. + */ signal?: AbortSignal; }; diff --git a/tests/unit/tts.test.ts b/tests/unit/tts.test.ts index dd775ce..017a79e 100644 --- a/tests/unit/tts.test.ts +++ b/tests/unit/tts.test.ts @@ -1,3 +1,4 @@ +import { getEventListeners } from "events"; import fetchMock from "jest-fetch-mock"; import { AssemblyAI, TextToSpeechError } from "../../src"; import { createClient, defaultApiKey, requestMatches } from "./utils"; @@ -268,6 +269,16 @@ describe("tts", () => { }) as unknown as Response, ); + /** The test audio as a web stream of two chunks. */ + const audioSource = () => + new ReadableStream({ + start(c) { + c.enqueue(audio.slice(0, 480)); + c.enqueue(audio.slice(480)); + c.close(); + }, + }); + it("stream turns a body cut after the first audio into a retryable TextToSpeechError", async () => { const body = new ReadableStream({ start(c) { @@ -369,23 +380,222 @@ describe("tts", () => { expect(chunks).toHaveLength(0); }); - it("a stream timeout covers the headers, not the body", async () => { - jest.useFakeTimers(); + it.each([ + ["a streamed body", () => mockStreamedBody(audioSource())], + ["a whole body", () => mockAudio()], + ])( + "a stream timeout covers the headers, not %s", + async (_name, mockResponse) => { + jest.useFakeTimers(); + try { + mockResponse(); + const speech = await client.tts.stream( + { text: "Hi." }, + { timeout: 10 }, + ); + jest.advanceTimersByTime(50); // well past the timeout, after the headers arrived + expect( + (fetchMock.mock.calls[0][1]!.signal as AbortSignal).aborted, + ).toBe(false); + expect(Buffer.from(await speech.bytes())).toEqual(Buffer.from(audio)); + } finally { + jest.useRealTimers(); + } + }, + ); + + /** + * Answers the next fetch the way fetch does: once the request's signal + * aborts, the body errors with the signal's reason. The body sends `total` + * one-byte chunks, then ends, or with `hang` waits forever for more. + */ + const mockAbortableBody = (total: number, { hang = false } = {}) => { + const source = { sent: 0, signal: undefined as AbortSignal | undefined }; + fetchMock.mockImplementationOnce(async (_input, init) => { + const signal = init!.signal!; + source.signal = signal; + const body = new ReadableStream( + { + start(c) { + signal.addEventListener("abort", () => c.error(signal.reason), { + once: true, + }); + }, + pull(c) { + if (source.sent === total) { + if (hang) return new Promise(() => undefined); + return c.close(); + } + source.sent++; + c.enqueue(new Uint8Array([source.sent])); + }, + }, + { highWaterMark: 0 }, + ); + return { + ok: true, + status: 200, + headers: new Headers(okHeaders), + body, + } as unknown as Response; + }); + return source; + }; + + /** Runs `loop` and returns what it threw, if anything. */ + const thrownBy = async (loop: () => Promise) => { try { - fetchMock.doMockOnceIf( - requestMatches({ url: "/v1/speech", method: "POST" }), - () => - Promise.resolve({ - status: 200, - headers: okHeaders, - body: Buffer.from(audio) as unknown as string, - }), + await loop(); + } catch (e) { + return e; + } + }; + + it("stream stops at the caller's abort after the first chunk, as an AbortError and not a cut", async () => { + const source = mockAbortableBody(50); + const controller = new AbortController(); + const speech = await client.tts.stream( + { text: "Hi." }, + { signal: controller.signal }, + ); + const chunks: Uint8Array[] = []; + const error = await thrownBy(async () => { + for await (const chunk of speech) { + chunks.push(chunk); + controller.abort(); // a barge-in + } + }); + expect(chunks).toHaveLength(1); + expect(error).not.toBeInstanceOf(TextToSpeechError); + expect((error as Error).name).toBe("AbortError"); + expect(source.signal!.aborted).toBe(true); // the download itself stopped + expect(source.sent).toBe(1); + }); + + it("stream ends a read that waits for audio at the caller's abort", async () => { + mockAbortableBody(1, { hang: true }); + const controller = new AbortController(); + const speech = await client.tts.stream( + { text: "Hi." }, + { signal: controller.signal }, + ); + const chunks: Uint8Array[] = []; + const error = await thrownBy(async () => { + for await (const chunk of speech) { + chunks.push(chunk); + setTimeout(() => controller.abort(new Error("barge-in")), 5); + } + }); + expect(chunks).toHaveLength(1); + expect(error).toEqual(new Error("barge-in")); + }); + + it("stream ends at an abort before the first read even when the audio is already here", async () => { + mockAudio(); // a body read whole, as on runtimes without streamed bodies + const controller = new AbortController(); + const speech = await client.tts.stream( + { text: "Hi." }, + { signal: controller.signal }, + ); + controller.abort(); + const chunks: Uint8Array[] = []; + const error = await thrownBy(async () => { + for await (const chunk of speech) chunks.push(chunk); + }); + expect(chunks).toHaveLength(0); + expect((error as Error).name).toBe("AbortError"); + }); + + /** A fetch that never answers and rejects with its signal's reason, as fetch does. */ + const mockNoAnswer = () => + fetchMock.mockImplementationOnce( + (_input, init) => + new Promise((_resolve, reject) => { + const signal = init!.signal!; + signal.addEventListener("abort", () => reject(signal.reason), { + once: true, + }); + }), + ); + + it("stream and synthesize time out with the same TimeoutError", async () => { + mockNoAnswer(); + mockNoAnswer(); + const errors = await Promise.all([ + client.tts.stream({ text: "Hi." }, { timeout: 20 }).catch((e) => e), + client.tts.synthesize({ text: "Hi." }, { timeout: 20 }).catch((e) => e), + ]); + for (const error of errors) { + expect(error).toBeInstanceOf(DOMException); + expect(error.name).toBe("TimeoutError"); + expect(error.message).toBe("no audio within 20 ms"); + } + }); + + it("synthesize calls sharing one signal leave no listener on it", async () => { + const controller = new AbortController(); + for (let i = 0; i < 13; i++) { + mockAudio(); + await client.tts.synthesize( + { text: "Hi." }, + { signal: controller.signal }, ); - const speech = await client.tts.stream({ text: "Hi." }, { timeout: 10 }); - jest.advanceTimersByTime(50); // well past the timeout, after the headers arrived - expect(Buffer.from(await speech.bytes())).toEqual(Buffer.from(audio)); - } finally { - jest.useRealTimers(); } + fetchMock.mockResponseOnce("busy", { status: 503 }); + await client.tts + .synthesize({ text: "Hi." }, { signal: controller.signal }) + .catch(() => undefined); + fetchMock.mockRejectOnce(new TypeError("fetch failed")); + await client.tts + .synthesize({ text: "Hi." }, { signal: controller.signal }) + .catch(() => undefined); + expect(fetchMock).toHaveBeenCalledTimes(15); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); + + it("stream calls sharing one signal leave no listener on it, however they end", async () => { + const controller = new AbortController(); + const signal = controller.signal; + const listeners = () => getEventListeners(signal, "abort").length; + + mockStreamedBody(audioSource()); // read to the end + await (await client.tts.stream({ text: "Hi." }, { signal })).bytes(); + mockStreamedBody(audioSource()); // read to the end with the caller's own reader + const own = (await client.tts.stream({ text: "Hi." }, { signal })).body; + const reader = own.getReader(); + while (!(await reader.read()).done); + mockStreamedBody(audioSource()); // left early + for await (const chunk of await client.tts.stream( + { text: "Hi." }, + { signal }, + )) { + if (chunk) break; + } + mockStreamedBody(audioSource()); // cancelled before any read + const cancelled = await client.tts.stream({ text: "Hi." }, { signal }); + expect(listeners()).toBe(1); // a body nobody has read yet still listens + await cancelled.cancel(); + mockStreamedBody( + new ReadableStream({ + start(c) { + c.enqueue(new Uint8Array([1, 2])); + }, + pull(c) { + c.error(new TypeError("terminated")); + }, + }), + ); // cut + await (await client.tts.stream({ text: "Hi." }, { signal })) + .bytes() + .catch(() => undefined); + mockAudio(); // a body read whole + await (await client.tts.stream({ text: "Hi." }, { signal })).bytes(); + fetchMock.mockResponseOnce("busy", { status: 503 }); // refused + await client.tts.stream({ text: "Hi." }, { signal }).catch(() => undefined); + fetchMock.mockRejectOnce(new TypeError("fetch failed")); // no answer + await client.tts.stream({ text: "Hi." }, { signal }).catch(() => undefined); + + expect(fetchMock).toHaveBeenCalledTimes(8); + expect(listeners()).toBe(0); }); }); From ebf05e2f6a96a3119e20db4f2781728f43063b67 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:16:46 +0000 Subject: [PATCH 04/26] fix(tts): carry each session connection's outcome and report the real close - Each connection keeps its own outcome (ended, error, close) and its `closed` item carries the error, so an audio() iteration drains its own connection and still rejects after a close listener reconnects. - fail() and the close handler end the connection and settle connect() before the error listener runs, so connect() works from on("error") and a throwing listener cannot leave a reader waiting. - connect() rejects with the session's error marked observed, so close() in a finally does not throw a handled refusal again; close() decides its outcome before the close listener runs. - The close listener gets the socket's close, or the error's close code (1000 "Session ended" after Termination) when the socket was released first; connect() reports a previous close that was never delivered. - The connect timeout ends the connection and reports its close, and audio() before the first connect() throws instead of waiting forever. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- README.md | 2 +- src/services/tts/session.ts | 181 +++++++++++++------- tests/unit/tts-session.test.ts | 300 +++++++++++++++++++++++++++++++-- 3 files changed, 410 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index f24f9fc..61d6877 100644 --- a/README.md +++ b/README.md @@ -906,7 +906,7 @@ await session.close(); Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order; every `flush()` and `cancel()` takes the next id, so ids skip one after a cancel. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters, so pace text to the audio. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes after `inactivityTimeout` seconds (5 to 120, default 120) without a frame; call `keepAlive()` between sparse requests. -Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send, reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `TextToSpeechError.status` is undefined and `requestId` is the session id; `retryable: true` means a new session can succeed, and the same session object can `connect()` again. +Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send, reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `TextToSpeechError.status` is undefined and `requestId` is the session id; `retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. In a browser the API key must stay on your server: mint a temporary token there with `await client.tts.createTemporaryToken({ expires_in_seconds: 60 })` and open the session with `client.tts.session({ token })`. diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index ff15d57..e7a2a8a 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -32,20 +32,47 @@ type PullItem = | { kind: "audio"; flushId: number; bytes: Uint8Array } | { kind: "done"; flushId: number } | { kind: "cancelled"; cancelId: number } - | { kind: "error"; error: TextToSpeechError } - | { kind: "closed" }; + | { kind: "closed"; error?: TextToSpeechError }; -/** A FIFO that hands items to one waiting reader; one per connection. */ +/** + * One connection: a FIFO that hands items to one waiting reader, and how the + * connection ended. A reader keeps the connection it started on, so a + * reconnect cannot change what it sees. + */ class PullQueue { private items: PullItem[] = []; private audioItems = 0; private waiter?: (item: PullItem) => void; dropped = false; + /** A socket was opened for this connection. */ + hadSocket = false; + /** The connection is over and its `closed` item is queued. */ + ended = false; + /** The error the connection ended with, if any. */ + error?: TextToSpeechError; + /** The connection ended with the API's `Termination`. */ + terminated = false; + /** The socket's own close event, when it arrived. */ + closedWith?: [code: number, reason: string]; + closeEmitted = false; + /** Requests up to this flush id were cancelled. */ + cancelledBefore = -1; get size() { return this.items.length; } + /** + * The socket's close code and reason; when the socket was released before + * its close event, the close the API sends after this ending. + */ + close(): [code: number, reason: string] { + if (this.closedWith) return this.closedWith; + const code = this.error?.closeCode ?? 1000; + if (code !== 1000) return [code, "See Error message for details"]; + return [code, this.terminated ? "Session ended" : "Closed by the client"]; + } + push(item: PullItem) { if (this.waiter) { const resolve = this.waiter; @@ -145,7 +172,8 @@ function errorFromClose(code: number, reason: string): TextToSpeechError { * `wordBoundaries`, `cancelled`, `termination`, `error`, `close`) are * optional. A refusal rejects `connect()`; a failure mid-session rejects the * `audio()` iteration or the next send, and a failure nothing observed - * rejects `close()`. + * rejects `close()`. The `error` and `close` listeners run once the + * connection has ended, so either can `connect()` again. * * @example * ```ts @@ -161,11 +189,8 @@ export class TextToSpeechSession { private queue = new PullQueue(); private nextFlushId = 0; private cancelIds: number[] = []; - private cancelledBefore = -1; - private stopped = false; private errorRaised = false; private reading = false; - private closeEmitted = false; private terminationResolve?: () => void; private abortConnect?: (error: TextToSpeechError) => void; @@ -178,6 +203,11 @@ export class TextToSpeechSession { constructor(private readonly params: TextToSpeechSessionParams) {} + /** The current connection is over: sends refuse and readers drain. */ + private get stopped(): boolean { + return this.queue.ended; + } + on(event: "begin", listener: (event: TtsBeginEvent) => void): void; on(event: "audio", listener: (event: TtsAudioEvent) => void): void; on(event: "flushDone", listener: (event: TtsFlushDoneEvent) => void): void; @@ -228,11 +258,8 @@ export class TextToSpeechSession { this.queue = new PullQueue(); this.nextFlushId = 0; this.cancelIds = []; - this.cancelledBefore = -1; - this.stopped = false; this.errorRaised = false; this.reading = false; - this.closeEmitted = false; this.begin = undefined; this.termination = undefined; this.error = undefined; @@ -246,11 +273,18 @@ export class TextToSpeechSession { * fails, or `Begin` does not arrive in time. */ connect(): Promise { + if (this.stopped) { + // The previous connection's close, when its socket was released before + // the close event arrived. The close listener may connect again. + this.releaseSocket(); + this.emitClose(this.queue); + } if (this.socket && !this.stopped) { return Promise.reject(new Error("The session is already connected")); } this.releaseSocket(); this.reset(); + const connection = this.queue; const url = this.connectionUrl(); const timeoutMs = this.params.connectTimeout ?? DEFAULT_CONNECT_TIMEOUT_MS; @@ -258,15 +292,15 @@ export class TextToSpeechSession { let settled = false; const timer = setTimeout(() => { settle(() => { - this.releaseSocket(); - this.stopped = true; - reject( - new TextToSpeechError( - `The session did not begin within ${timeoutMs}ms`, - { retryable: true }, - ), + this.error = new TextToSpeechError( + `The session did not begin within ${timeoutMs}ms`, + { retryable: true }, ); + this.end(); + this.releaseSocket(); + reject(this.surfaced()); }); + this.emitClose(connection); }, timeoutMs); const settle = (fn: () => void) => { if (settled) return; @@ -276,6 +310,13 @@ export class TextToSpeechSession { fn(); }; this.abortConnect = (error) => settle(() => reject(error)); + // Ends the connection and settles connect() before the error listener + // runs, so the listener sees an ended session it can connect again. + const fail = (error: TextToSpeechError) => { + const raised = this.fail(error); + settle(() => reject(this.surfaced())); + if (raised) this.listeners.error?.(raised); + }; if (this.params.token) { this.socket = polyfillWebSocketFactory(url.toString()); @@ -291,6 +332,7 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c }); } const socket = this.socket; + connection.hadSocket = true; socket.binaryType = "arraybuffer"; socket.onopen = () => {}; @@ -311,32 +353,37 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c return; } if (frame.type === "Error") { - const error = errorFromFrame(frame as unknown as TtsErrorMessage); - this.fail(error); - settle(() => reject(error)); + fail(errorFromFrame(frame as unknown as TtsErrorMessage)); return; } try { this.dispatch(frame); } catch (cause) { - const error = new TextToSpeechError( - `Unreadable frame from the API: ${String(cause)}`, + fail( + new TextToSpeechError( + `Unreadable frame from the API: ${String(cause)}`, + ), ); - this.fail(error); - settle(() => reject(error)); } }; socket.onclose = ({ code, reason }: CloseEvent) => { const text = typeof reason === "string" ? reason : ""; + connection.closedWith = [code, text]; + let raised: TextToSpeechError | undefined; if (!this.stopped && code !== 1000 && !this.error) { - this.error = errorFromClose(code, text); - if (this.begin) this.error.requestId = this.begin.id; - this.listeners.error?.(this.error); + raised = this.error = errorFromClose(code, text); + if (this.begin) raised.requestId = this.begin.id; } this.end(); - this.emitClose(code, text); - settle(() => reject(this.error ?? errorFromClose(code, text))); + settle(() => + reject(this.error ? this.surfaced() : errorFromClose(code, text)), + ); + try { + if (raised) this.listeners.error?.(raised); + } finally { + this.emitClose(connection); + } }; socket.onerror = (event: ErrorEvent) => { @@ -349,7 +396,7 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c ); this.error = error; this.end(); - settle(() => reject(error)); + settle(() => reject(this.surfaced())); }; }); } @@ -391,22 +438,24 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c } case "Termination": this.termination = frame as unknown as TtsTerminationEvent; - this.listeners.termination?.(this.termination); this.end(); + this.listeners.termination?.(this.termination); return; default: return; // a frame type this SDK does not know } } - private fail(error: TextToSpeechError) { - if (!this.error) { - if (!error.requestId && this.begin) error.requestId = this.begin.id; - this.error = error; - this.listeners.error?.(error); - } - this.enqueue({ kind: "error", error: this.error }); + /** + * Ends the connection with `error`; returns it when it is the session's + * first error, for the caller to pass to the error listener. + */ + private fail(error: TextToSpeechError): TextToSpeechError | undefined { + if (this.error) return undefined; // the connection already ended with it + if (!error.requestId && this.begin) error.requestId = this.begin.id; + this.error = error; this.end(); + return error; } /** Queues an item for `audio()`; nothing is queued while an `audio` listener receives the frames. */ @@ -415,17 +464,22 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c this.queue.push(item); } - private emitClose(code: number, reason: string) { - if (this.closeEmitted) return; - this.closeEmitted = true; + /** Reports a connection's close to the close listener, once. */ + private emitClose(connection: PullQueue) { + if (!connection.hadSocket || connection.closeEmitted) return; + connection.closeEmitted = true; + const [code, reason] = connection.close(); this.listeners.close?.(code, reason); } - /** Marks the connection over: waiters wake up, sends refuse. */ + /** Marks the connection over with the session's error: readers drain, sends refuse. */ private end() { - if (this.stopped) return; - this.stopped = true; - this.enqueue({ kind: "closed" }); + const connection = this.queue; + if (connection.ended) return; + connection.ended = true; + connection.error = this.error; + connection.terminated = this.termination !== undefined; + this.enqueue({ kind: "closed", error: this.error }); this.terminationResolve?.(); this.terminationResolve = undefined; } @@ -445,9 +499,14 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c } } + /** Marks `error` observed when it is the session's own error, and returns it. */ + private surface(error: TextToSpeechError): TextToSpeechError { + if (error === this.error) this.errorRaised = true; + return error; + } + private surfaced(): TextToSpeechError { - this.errorRaised = true; - return this.error as TextToSpeechError; + return this.surface(this.error as TextToSpeechError); } private send(frame: Record) { @@ -525,7 +584,7 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c cancel() { const cancelId = this.nextFlushId++; this.cancelIds.push(cancelId); - this.cancelledBefore = Math.max(this.cancelledBefore, cancelId); + this.queue.cancelledBefore = Math.max(this.queue.cancelledBefore, cancelId); this.send({ type: "Cancel" }); } @@ -559,21 +618,25 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c * @throws TextToSpeechError if the session fails before the request completes. */ async *audio(flushId?: number): AsyncGenerator { + if (!this.queue.hadSocket) { + throw new Error("The session is not open: call connect() first"); + } this.checkPullable(); if (flushId === undefined) flushId = this.nextFlushId; else if (flushId > this.nextFlushId) { throw new RangeError(`flush_id ${flushId} has not been flushed yet`); } + // The connection this request was sent on; a reconnect leaves it to + // drain to its own `closed` item. const connection = this.queue; this.reading = true; try { for (;;) { - if (this.queue !== connection) return; // the session reconnected // A Cancel drops every request before it and the text being written, // whose id it takes. - if (flushId <= this.cancelledBefore) return; - if (this.stopped && connection.size === 0) { - if (this.error) throw this.surfaced(); + if (flushId <= connection.cancelledBefore) return; + if (connection.ended && connection.size === 0) { + if (connection.error) throw this.surface(connection.error); return; } const item = await connection.next(); @@ -587,10 +650,8 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c case "cancelled": if (item.cancelId >= flushId) return; break; - case "error": - throw this.surfaced(); case "closed": - if (this.error) throw this.surfaced(); + if (item.error) throw this.surface(item.error); return; } } @@ -642,8 +703,12 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c } this.end(); this.releaseSocket(); - if (socket) this.emitClose(1000, "Closed by the client"); - if (this.error && !this.errorRaised) throw this.surfaced(); - return this.termination; + const termination = this.termination; + const unobserved = + this.error && !this.errorRaised ? this.surfaced() : undefined; + // Reported last: the close listener may connect again. + this.emitClose(this.queue); + if (unobserved) throw unobserved; + return termination; } } diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index fc9dbf6..23d0fe0 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -2,7 +2,12 @@ jest.mock("ws", () => require("./mocks/ws")); import WS from "jest-websocket-mock"; import fetchMock from "jest-fetch-mock"; -import { AssemblyAI, TextToSpeechError, TextToSpeechSession } from "../../src"; +import { + AssemblyAI, + TextToSpeechError, + TextToSpeechSession, + TtsBeginEvent, +} from "../../src"; import { createClient } from "./utils"; fetchMock.enableMocks(); @@ -26,6 +31,15 @@ const terminationMessage = { audio_duration_seconds: 1.5, }; +type ServerSocket = { + send: (data: string) => void; + close: (options?: { + code: number; + reason: string; + wasClean: boolean; + }) => void; +}; + /** * Plays the API: Generate/Flush answer with the text's bytes as one Audio * frame and a FlushDone, Cancel with Cancelled, Terminate with Termination @@ -35,10 +49,14 @@ class FakeApi { text = ""; flushId = 0; answerFlush = true; + closeOnTerminate = true; sent: Record[] = []; + /** The server side of each connection, in order. */ + sockets: ServerSocket[] = []; constructor(public server: WS) { server.on("connection", (socket) => { + this.sockets.push(socket); this.text = ""; // a new connection is a new session: ids start at 0 this.flushId = 0; socket.on("message", (raw) => { @@ -49,17 +67,7 @@ class FakeApi { }); } - private handle( - frame: Record, - socket: { - send: (data: string) => void; - close: (options?: { - code: number; - reason: string; - wasClean: boolean; - }) => void; - }, - ) { + private handle(frame: Record, socket: ServerSocket) { switch (frame.type) { case "Generate": this.text += frame.text as string; @@ -93,7 +101,9 @@ class FakeApi { return; case "Terminate": socket.send(JSON.stringify(terminationMessage)); - socket.close({ code: 1000, reason: "Session ended", wasClean: true }); + if (this.closeOnTerminate) { + socket.close({ code: 1000, reason: "Session ended", wasClean: true }); + } return; } } @@ -125,6 +135,37 @@ async function collect(iterable: AsyncIterable): Promise { const tick = () => new Promise((resolve) => setTimeout(resolve, 20)); +/** The server side of connection `n` (1-based), once the client opened it. */ +async function connection(n: number): Promise { + for (let waited = 0; api.sockets.length < n; waited += 5) { + if (waited > 500) throw new Error(`connection ${n} was never opened`); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + return api.sockets[n - 1]; +} + +/** What the promise settled with, or "pending" if it did not settle in time. */ +function outcome(promise: Promise, ms = 300) { + return Promise.race([ + promise.catch((error: unknown) => error), + new Promise<"pending">((resolve) => + setTimeout(() => resolve("pending"), ms), + ), + ]); +} + +function errorFrame(code: number, errorType: string, retryable: boolean) { + return JSON.stringify({ + type: "Error", + error_code: code, + error: `Failed: ${errorType}`, + error_type: errorType, + retryable, + param: null, + retry_after_seconds: null, + }); +} + describe("tts session", () => { beforeEach(() => { server = new WS(websocketBaseUrl); @@ -461,7 +502,238 @@ describe("tts session", () => { session.on("close", (code, reason) => closes.push([code, reason])); await connect(session); await session.close(); + expect(closes).toEqual([[1000, "Session ended"]]); + }); + + it.each([ + { name: "waits for audio", chunkFirst: false }, + { name: "is between chunks", chunkFirst: true }, + ])( + "a close listener that connects again does not hide the cut from a reader that $name", + async ({ chunkFirst }) => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + const closes: number[] = []; + let reconnecting: Promise | undefined; + session.on("close", (code) => { + closes.push(code); + if (session.error?.retryable) reconnecting = session.connect(); + }); + await connect(session); + const flushId = session.flush(); + const stream = session.audio(flushId); + let reading: Promise | undefined; + if (chunkFirst) { + // The reader stays suspended at this chunk through the cut, the + // reconnect and a barge-in on the new connection. + api.sockets[0].send( + JSON.stringify({ type: "Audio", audio: "QQ==", flush_id: flushId }), + ); + expect((await stream.next()).done).toBe(false); + } else { + reading = outcome(stream.next()); + await tick(); + } + api.sockets[0].close({ code: 3005, reason: "", wasClean: true }); + (await connection(2)).send(JSON.stringify(beginMessage)); + await expect(reconnecting).resolves.toMatchObject({ id: "sess-1" }); + session.cancel(); + + const error = await (reading ?? outcome(stream.next())); + expect(error).toBeInstanceOf(TextToSpeechError); + expect(error).toMatchObject({ closeCode: 3005, retryable: true }); + expect(closes).toEqual([3005]); + + api.answerFlush = true; + expect((await collect(session.speak("Again."))).toString()).toBe( + "Again.", + ); + // Reading the old connection's error does not count as observing a + // failure of the new one. + api.sockets[1].send(errorFrame(3010, "input_rate_exceeded", false)); + await expect(session.close()).rejects.toMatchObject({ + errorCode: "input_rate_exceeded", + }); + expect(closes).toEqual([3005, 3010]); + }, + ); + + it("close() rejects with an unobserved error although the close listener connects again", async () => { + const session = client.tts.session({ websocketBaseUrl }); + let reconnecting: Promise | undefined; + session.on("audio", () => undefined); // push mode: nothing reads the error + session.on("close", () => { + reconnecting ??= session.connect(); + }); + await connect(session); + api.sockets[0].send(errorFrame(3005, "at_capacity", true)); + await expect(session.close()).rejects.toMatchObject({ + errorCode: "at_capacity", + }); + + (await connection(2)).send(JSON.stringify(beginMessage)); + await expect(reconnecting).resolves.toMatchObject({ id: "sess-1" }); + await expect(session.close()).resolves.toEqual(terminationMessage); + }); + + it.each([ + { + name: "an Error frame and its close", + end: (socket: ServerSocket) => { + socket.send(errorFrame(1012, "service_restart", true)); + socket.close({ code: 1012, reason: "", wasClean: true }); + }, + }, + { + name: "a dropped connection", + end: (socket: ServerSocket) => + socket.close({ code: 1006, reason: "", wasClean: false }), + }, + ])("the error listener can connect again after $name", async ({ end }) => { + const session = client.tts.session({ websocketBaseUrl }); + const errors: TextToSpeechError[] = []; + let reconnecting: Promise | undefined; + session.on("error", (error) => { + errors.push(error); + reconnecting = session.connect(); + }); + await connect(session); + end(api.sockets[0]); + await tick(); + expect(errors).toHaveLength(1); + expect(errors[0].retryable).toBe(true); + + (await connection(2)).send(JSON.stringify(beginMessage)); + await expect(reconnecting).resolves.toMatchObject({ id: "sess-1" }); + expect((await collect(session.speak("Back."))).toString()).toBe("Back."); + await session.close(); + }); + + it("a throwing error listener does not leave a reader waiting", async () => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + session.on("error", () => { + throw new Error("listener failed"); + }); + await connect(session); + const flushId = session.flush(); + const reading = outcome(collect(session.audio(flushId))); + await tick(); + expect(() => + api.sockets[0].send(errorFrame(3005, "at_capacity", true)), + ).toThrow("listener failed"); + expect(await reading).toMatchObject({ errorCode: "at_capacity" }); + await session.close(); + }); + + it.each([ + { + name: "an Error frame", + refuse: (socket: ServerSocket) => { + socket.send(errorFrame(3012, "voice_not_found", false)); + socket.close({ code: 3012, reason: "", wasClean: true }); + }, + closeCode: 3012, + }, + { + name: "a close", + refuse: (socket: ServerSocket) => + socket.close({ code: 1008, reason: "Unauthorized", wasClean: true }), + closeCode: 1008, + }, + ])( + "close() after connect() rejected with $name does not throw it again", + async ({ refuse, closeCode }) => { + const session = client.tts.session({ websocketBaseUrl }); + const connecting = session.connect(); + refuse(await connection(1)); + let caught: unknown; + let closing: Promise | undefined; + try { + await connecting; + } catch (error) { + caught = error; + } finally { + closing = session.close(); + } + expect(caught).toMatchObject({ closeCode }); + await expect(closing).resolves.toBeUndefined(); + }, + ); + + it("close() reports the close of an Error frame whose close is still in flight", async () => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + const closes: [number, string][] = []; + session.on("close", (code, reason) => closes.push([code, reason])); + await connect(session); + const flushId = session.flush(); + api.sockets[0].send(errorFrame(3005, "at_capacity", true)); + api.sockets[0].close({ code: 3005, reason: "", wasClean: true }); + await expect(collect(session.audio(flushId))).rejects.toMatchObject({ + closeCode: 3005, + }); + await session.close(); + expect(session.error?.closeCode).toBe(3005); + expect(closes).toEqual([[3005, "See Error message for details"]]); + }); + + it("close() after Termination reports the session's end", async () => { + api.closeOnTerminate = false; // the server's close is still in flight + const session = client.tts.session({ websocketBaseUrl }); + const closes: [number, string][] = []; + session.on("close", (code, reason) => closes.push([code, reason])); + await connect(session); + await session.close(); + expect(closes).toEqual([[1000, "Session ended"]]); + }); + + it("connect() reports the previous connection's close before opening a new one", async () => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + const events: string[] = []; + session.on("close", (code) => events.push(`close ${code}`)); + session.on("begin", () => events.push("begin")); + await connect(session); + const flushId = session.flush(); + api.sockets[0].send(errorFrame(3005, "at_capacity", true)); // no close yet + await expect(collect(session.audio(flushId))).rejects.toMatchObject({ + closeCode: 3005, + }); + const reconnecting = session.connect(); + (await connection(2)).send(JSON.stringify(beginMessage)); + await reconnecting; + expect(events).toEqual(["begin", "close 3005", "begin"]); + await session.close(); + }); + + it("a connect timeout ends a reader waiting for the session", async () => { + const session = client.tts.session({ + websocketBaseUrl, + connectTimeout: 50, + }); + const closes: [number, string][] = []; + session.on("close", (code, reason) => closes.push([code, reason])); + const connecting = outcome(session.connect()); + await connection(1); // no Begin follows + const reading = outcome(collect(session.audio())); + const error = await reading; + expect(error).toBeInstanceOf(TextToSpeechError); + expect(error).toMatchObject({ + retryable: true, + message: expect.stringMatching(/did not begin within 50ms/), + }); + expect(await connecting).toBe(error); + expect(closes).toEqual([[1000, "Closed by the client"]]); + await expect(outcome(session.close())).resolves.toBeUndefined(); expect(closes).toHaveLength(1); - expect(closes[0][0]).toBe(1000); + }); + + it("audio() before connect() says what to do", async () => { + const session = client.tts.session({ websocketBaseUrl }); + const first = outcome((async () => session.audio().next())()); + expect(await first).toMatchObject({ + message: expect.stringMatching(/call connect\(\) first/), + }); }); }); From 7543d0bb258f46a1d6f8648b1994741aa820f65a Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:24:39 +0000 Subject: [PATCH 05/26] fix(tts): bind audio() to its request when called and keep its markers flowing audio() was an async generator, so it picked its request at the first next(). An iterator made before generate()/flush() and iterated later (handed to a player, or to Readable.from) bound to the next request, dropped all audio of the one just flushed and waited until the session ended. audio() is now a plain method: it takes the request being written (or the given id) and the connection when called, throws a RangeError at once for an id not flushed yet, and returns the reading generator, which still checks for an audio listener or a second reader at its first next(). Readers record the highest FlushDone they consume on the connection, so audio() refuses a request already read (or skipped by reading a later one) with a RangeError instead of waiting for the session to end. A FlushDone still queued keeps its pipelined request readable. Registering an audio listener while an audio() iteration waits no longer drops the iteration's FlushDone, Cancelled and closed markers, so it ends at its FlushDone or at close(). With a listener and no iteration running, nothing is queued. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- README.md | 2 +- src/services/tts/session.ts | 82 ++++++++++++++++------ tests/unit/tts-session.test.ts | 121 +++++++++++++++++++++++++++++++-- 3 files changed, 180 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 61d6877..8c94f7d 100644 --- a/README.md +++ b/README.md @@ -904,7 +904,7 @@ await player; await session.close(); ``` -Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order; every `flush()` and `cancel()` takes the next id, so ids skip one after a cancel. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters, so pace text to the audio. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes after `inactivityTimeout` seconds (5 to 120, default 120) without a frame; call `keepAlive()` between sparse requests. +`audio()` fixes its request when it is called, so the iterator can be made before the first `generate()` and handed to a player that starts later. Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order, once; `audio()` throws a `RangeError` for an id not flushed yet or already read. Every `flush()` and `cancel()` takes the next id, so ids skip one after a cancel. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters, so pace text to the audio. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes after `inactivityTimeout` seconds (5 to 120, default 120) without a frame; call `keepAlive()` between sparse requests. Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send, reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `TextToSpeechError.status` is undefined and `requestId` is the session id; `retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index e7a2a8a..c741037 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -57,6 +57,8 @@ class PullQueue { closeEmitted = false; /** Requests up to this flush id were cancelled. */ cancelledBefore = -1; + /** Readers consumed the `FlushDone` of the requests up to this flush id. */ + doneThrough = -1; get size() { return this.items.length; @@ -458,9 +460,15 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c return error; } - /** Queues an item for `audio()`; nothing is queued while an `audio` listener receives the frames. */ + /** + * Queues an item for `audio()`. While an `audio` listener receives the + * frames, audio is never queued, and markers are queued only for an + * `audio()` iteration already running, so it still ends. + */ private enqueue(item: PullItem) { - if (this.listeners.audio) return; + if (this.listeners.audio && (item.kind === "audio" || !this.reading)) { + return; + } this.queue.push(item); } @@ -609,27 +617,57 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c } /** - * Yields the audio of request `flushId` until its `FlushDone`. + * The audio of request `flushId`: an async iterator that ends at its + * `FlushDone`, or early if the request is cancelled. * - * Without `flushId`, the request being written now, so audio can be read - * while `generate()` is still sending text. Requests are served in order, - * so read them in order, one iteration at a time. Audio is queued for - * pulling only while no `audio` listener is registered. - * @throws TextToSpeechError if the session fails before the request completes. + * The request is fixed when `audio()` is called, not when it is first + * iterated. Without `flushId` it is the request being written now, whose + * id the next `flush()` returns, so the iterator can be made before the + * first `generate()` and read while text is still being sent. Requests + * are served in order, so read each one once, in order, one iteration at + * a time. Audio is queued for pulling only while no `audio` listener is + * registered. The iteration rejects with a `TextToSpeechError` if the + * session fails before the request completes. + * @throws Error if the session was never connected. + * @throws RangeError for a request not flushed yet, or one already read + * (its `FlushDone` was consumed by an earlier iteration). */ - async *audio(flushId?: number): AsyncGenerator { - if (!this.queue.hadSocket) { + audio(flushId?: number): AsyncGenerator { + // The connection this request is sent on. + const connection = this.queue; + if (!connection.hadSocket) { throw new Error("The session is not open: call connect() first"); } - this.checkPullable(); - if (flushId === undefined) flushId = this.nextFlushId; - else if (flushId > this.nextFlushId) { - throw new RangeError(`flush_id ${flushId} has not been flushed yet`); + const id = flushId ?? this.nextFlushId; + if (id > this.nextFlushId) { + throw new RangeError(`flush_id ${id} has not been flushed yet`); + } + this.checkUnread(connection, id); + return this.readAudio(connection, id); + } + + private checkUnread(connection: PullQueue, flushId: number) { + if (flushId <= connection.doneThrough) { + throw new RangeError( + `flush_id ${flushId} was already read: read each request once, in order`, + ); + } + } + + /** Reads request `flushId` from `connection`, until its end. */ + private async *readAudio( + connection: PullQueue, + flushId: number, + ): AsyncGenerator { + // After a reconnect the reader stays on its own connection, which drains + // to its `closed` item, and leaves the new connection's readers alone. + const current = this.queue === connection; + if (current) { + this.checkPullable(); + // Another iterator may have read past this request since audio(). + this.checkUnread(connection, flushId); + this.reading = true; } - // The connection this request was sent on; a reconnect leaves it to - // drain to its own `closed` item. - const connection = this.queue; - this.reading = true; try { for (;;) { // A Cancel drops every request before it and the text being written, @@ -645,6 +683,12 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c if (item.flushId === flushId) yield item.bytes; break; case "done": + // Counted when read, so a FlushDone still queued for a pipelined + // request leaves it readable. + connection.doneThrough = Math.max( + connection.doneThrough, + item.flushId, + ); if (item.flushId === flushId) return; break; case "cancelled": @@ -656,7 +700,7 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c } } } finally { - if (this.queue === connection) this.reading = false; + if (current && this.queue === connection) this.reading = false; } } diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index 23d0fe0..d0b1ee7 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -729,11 +729,122 @@ describe("tts session", () => { expect(closes).toHaveLength(1); }); - it("audio() before connect() says what to do", async () => { + it("audio() before connect() says what to do", () => { const session = client.tts.session({ websocketBaseUrl }); - const first = outcome((async () => session.audio().next())()); - expect(await first).toMatchObject({ - message: expect.stringMatching(/call connect\(\) first/), - }); + expect(() => session.audio()).toThrow(/call connect\(\) first/); + }); + + it("audio() takes its request when called, not at the first next()", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const stream = session.audio(); // e.g. handed to a player that starts later + session.generate("Made before the text."); + expect(session.flush()).toBe(0); + await tick(); // the request's audio and FlushDone are queued + expect(await outcome(collect(stream))).toEqual( + Buffer.from("Made before the text."), + ); + expect((await collect(session.speak("Next."))).toString()).toBe("Next."); + await session.close(); + }); + + it("audio() refuses a request not flushed yet when called", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + expect(() => session.audio(99)).toThrow(RangeError); + expect(() => session.audio(99)).toThrow(/flush_id 99 has not been flushed/); + expect(() => session.audio(1)).toThrow(RangeError); // 0 is being written + await session.close(); + }); + + it("audio() refuses a request already read instead of waiting for the session to end", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + expect((await collect(session.speak("Read me once."))).toString()).toBe( + "Read me once.", + ); + expect(() => session.audio(0)).toThrow(RangeError); + expect(() => session.audio(0)).toThrow(/flush_id 0 was already read/); + + // Reading a later request skips the FlushDone of an earlier one. + session.generate("Skipped."); + const skipped = session.flush(); + const early = session.audio(skipped); // made before the skip + session.generate("Read."); + expect((await collect(session.audio(session.flush()))).toString()).toBe( + "Read.", + ); + expect(() => session.audio(skipped)).toThrow(/already read/); + await expect(early.next()).rejects.toThrow(/already read/); + await session.close(); + }); + + it("a pipelined request whose FlushDone is queued stays readable", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + session.generate("One. "); + const first = session.flush(); + session.generate("Two."); + const second = session.flush(); + await tick(); // both FlushDones are queued, neither read + const readSecond = session.audio(second); // made before the first is read + expect((await collect(session.audio(first))).toString()).toBe("One. "); + expect((await collect(readSecond)).toString()).toBe("Two."); + await session.close(); + }); + + it("an iterator that is never iterated does not block the next reader", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + session.audio(); // dropped without a next() + expect((await collect(session.speak("Hi."))).toString()).toBe("Hi."); + await session.close(); + }); + + it("an audio listener registered while a reader waits still lets it end at FlushDone", async () => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const flushId = session.flush(); + const reading = outcome(collect(session.audio(flushId))); + await tick(); + const received: Uint8Array[] = []; + session.on("audio", (e) => received.push(e.audio)); + api.sockets[0].send( + JSON.stringify({ type: "Audio", audio: "QQ==", flush_id: flushId }), + ); + api.sockets[0].send( + JSON.stringify({ type: "FlushDone", flush_id: flushId }), + ); + expect(await reading).toEqual(Buffer.alloc(0)); // the audio went to the listener + expect(received.map((a) => Buffer.from(a).toString())).toEqual(["A"]); + await session.close(); + }); + + it("an audio listener registered while a reader waits still lets close() end it", async () => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const reading = outcome(collect(session.audio(session.flush()))); + await tick(); + session.on("audio", () => undefined); + await session.close(); + expect(await reading).toEqual(Buffer.alloc(0)); + }); + + it("an audio listener with no reader queues nothing", async () => { + const session = client.tts.session({ websocketBaseUrl }); + session.on("audio", () => undefined); + await connect(session); + for (const text of ["One.", "Two.", "Three."]) { + session.generate(text); + session.flush(); + } + session.cancel(); + await tick(); + const queue = (session as unknown as { queue: { size: number } }).queue; + expect(queue.size).toBe(0); + await session.close(); + expect(queue.size).toBe(0); }); }); From 7cbcb55cd34939b4db5a69153fb2ca78e682c1fc Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:34:17 +0000 Subject: [PATCH 06/26] fix(tts): reconnect a token session with a new token and mint tokens in the client's zone A session opened with a temporary token kept the token in its readonly params, so every reconnect resent it. The API refuses a used token where tokens are single use and an expired one always (1008 unauthorized, not retryable), so a browser session could not come back after a retryable close. connect() now takes { token }: the new token replaces the session's token for that connection and the next ones, and is sent alone, never next to the API key. createTemporaryToken() mints on the token host of the client's zone: token.eu.assemblyai.com for a tts.eu.assemblyai.com ttsBaseUrl, token.us.assemblyai.com for tts.us.assemblyai.com (the default), and token.assemblyai.com for any other host. tokenBaseUrl still wins. Docs: the token parameter, connect(), TextToSpeechTokenParams, the README reconnect sentence, the CLAUDE.md gotcha, and the browser guidance, which now opens the session with new TextToSpeechSession({ token }) (no client, no API key in the page), says to mint a token per connection and per HTTP request, and gives the EU WebSocket URL. A TextToSpeechSession bullet in docs/compat.md backs the session's browser warning link. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- CLAUDE.md | 4 +- README.md | 24 ++++++- docs/compat.md | 25 +++++++ src/services/tts/service.ts | 46 ++++++++++-- src/services/tts/session.ts | 30 ++++++-- src/types/tts/index.ts | 15 +++- tests/unit/mocks/ws.ts | 6 +- tests/unit/tts-session.test.ts | 123 +++++++++++++++++++++++++++++---- tests/unit/tts.test.ts | 53 ++++++++++++++ 9 files changed, 294 insertions(+), 32 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 41c02e2..90f30d9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,7 +52,7 @@ const client = new AssemblyAI({ - `client.llmGateway.chatCompletions(request)` — OpenAI-compatible chat completions - `client.llmGateway.listModels()` — List models available on the LLM Gateway - `client.llmGateway.understanding(request)` / `client.llmGateway.validateUnderstanding(request)` — Run or validate a Speech Understanding request -- `client.tts.synthesize({ text, voice_id? })` — one text's complete audio (`SpeechResponse.audio`, a Uint8Array, plus header fields); `client.tts.stream(request)` — async iterable of audio chunks as generated; `client.tts.session({ voiceId })` — WebSocket `TextToSpeechSession`: `connect()`, `speak(text)` (async iterable), `generate()`/`flush()`/`audio(flushId?)`, `cancel()`, `keepAlive()`, `close()`, `on("audio"|"flushDone"|"wordBoundaries"|"cancelled"|"termination"|"error"|"close", fn)`. Errors: `TextToSpeechError` (`errorCode`, `param`, `retryable`, `retryAfter`, `requestId`, `closeCode`, `body`) +- `client.tts.synthesize({ text, voice_id? })` — one text's complete audio (`SpeechResponse.audio`, a Uint8Array, plus header fields); `client.tts.stream(request)` — async iterable of audio chunks as generated; `client.tts.session({ voiceId })` — WebSocket `TextToSpeechSession`: `connect({ token? })`, `speak(text)` (async iterable), `generate()`/`flush()`/`audio(flushId?)`, `cancel()`, `keepAlive()`, `close()`, `on("audio"|"flushDone"|"wordBoundaries"|"cancelled"|"termination"|"error"|"close", fn)`. Errors: `TextToSpeechError` (`errorCode`, `param`, `retryable`, `retryAfter`, `requestId`, `closeCode`, `body`) ## Common patterns @@ -451,7 +451,7 @@ session.cancel(); // barge-in: ends the current audio() iteration at once await session.close(); // Terminate -> Termination summary; rejects with an unobserved session error ``` -Gotchas: a session streams raw PCM/G.711 only (no `mp3`, `container` or `bit_rate`); `generate()` takes at most 2,000 characters per frame (`speak()` splits); the API paces text at ~100 chars/s; an `audio` listener switches audio from pull to push; a session the API ended (idle after 120 s) can `connect()` again on the same object. +Gotchas: a session streams raw PCM/G.711 only (no `mp3`, `container` or `bit_rate`); `generate()` takes at most 2,000 characters per frame (`speak()` splits); the API paces text at ~100 chars/s; an `audio` listener switches audio from pull to push; a session the API ended (idle after 120 s) can `connect()` again on the same object, but a session opened with a temporary token needs a new token for each connection: `connect({ token: newToken })` (a used or expired token is refused with 1008 `unauthorized`). In a browser use `new TextToSpeechSession({ token })` (no client, no API key); `client.tts.createTemporaryToken()` mints on the token host of the `ttsBaseUrl` zone (`token.eu`/`token.us`), `tokenBaseUrl` overrides. ## Important gotchas diff --git a/README.md b/README.md index 8c94f7d..9c530a6 100644 --- a/README.md +++ b/README.md @@ -906,9 +906,29 @@ await session.close(); `audio()` fixes its request when it is called, so the iterator can be made before the first `generate()` and handed to a player that starts later. Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order, once; `audio()` throws a `RangeError` for an id not flushed yet or already read. Every `flush()` and `cancel()` takes the next id, so ids skip one after a cancel. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters, so pace text to the audio. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes after `inactivityTimeout` seconds (5 to 120, default 120) without a frame; call `keepAlive()` between sparse requests. -Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send, reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `TextToSpeechError.status` is undefined and `requestId` is the session id; `retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. +Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send, reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `TextToSpeechError.status` is undefined and `requestId` is the session id; `retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. That works as is with an API key; a session opened with a temporary token needs a new token for each connection, passed as `connect({ token })`, because the API may refuse a token it has seen and always refuses an expired one (close code 1008, `errorCode: "unauthorized"`). The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. -In a browser the API key must stay on your server: mint a temporary token there with `await client.tts.createTemporaryToken({ expires_in_seconds: 60 })` and open the session with `client.tts.session({ token })`. +In a browser the API key must stay on your server. Mint a temporary token there, and open the session in the page with `new TextToSpeechSession({ token })`, which needs no client and no API key: + +```typescript +// Server +const token = await client.tts.createTemporaryToken({ expires_in_seconds: 60 }); +``` + +```typescript +// Page +import { TextToSpeechSession } from "assemblyai"; + +const session = new TextToSpeechSession({ + token: await getToken(), // fetches a new token from your server + voiceId: "aai-jane", + // websocketBaseUrl: "wss://tts.eu.assemblyai.com/v1/ws", // EU data zone +}); +await session.connect(); +// After a retryable close: await session.connect({ token: await getToken() }); +``` + +Mint a new token for every connection, reconnects included, and for every HTTP request (a token works as the `apiKey` of a client made for that request), and keep `expires_in_seconds` short. `createTemporaryToken()` mints on the token host of the client's zone: `https://token.eu.assemblyai.com` when `ttsBaseUrl` is `https://tts.eu.assemblyai.com`, `https://token.us.assemblyai.com` for `https://tts.us.assemblyai.com` (the default), and `https://token.assemblyai.com` for any other host; pass `{ tokenBaseUrl }` as its second argument to choose the host. A token works in every zone. ## Contributing diff --git a/docs/compat.md b/docs/compat.md index 79a37bb..0f32ced 100644 --- a/docs/compat.md +++ b/docs/compat.md @@ -47,6 +47,31 @@ However, there are still incompatibilities between Node.js and the browser. }); ``` +- `TextToSpeechSession` doesn't support the AssemblyAI API key in the browser either. + Mint a temporary token on the server with `client.tts.createTemporaryToken`, and open the session with it in the page. + + ```js + // On the server + const token = await client.tts.createTemporaryToken({ + expires_in_seconds: 60, + }); + ``` + + ```js + // In the page: no client and no API key + import { TextToSpeechSession } from "assemblyai"; + + const session = new TextToSpeechSession({ + token: await getToken(), // getToken is a function for you to implement + voiceId: "aai-jane", + }); + await session.connect(); + ``` + + A token is for one connection: to reconnect, get a new token from your server and pass it to `session.connect({ token })`. + Mint a new token for every HTTP request too, and keep `expires_in_seconds` short. + For the EU data zone, add `websocketBaseUrl: "wss://tts.eu.assemblyai.com/v1/ws"`; a client whose `ttsBaseUrl` is `https://tts.eu.assemblyai.com` mints the token on `https://token.eu.assemblyai.com`. + - You can't pass local audio file paths to `client.files.upload`, `client.transcripts.transcribe`, and `client.transcripts.submit`. If you do, you'll get the following error: "Interacting with the file system is not supported in this environment.". If you want to transcribe audio files, you must use a public URL, a stream, or a buffer. diff --git a/src/services/tts/service.ts b/src/services/tts/service.ts index 4617be7..60761be 100644 --- a/src/services/tts/service.ts +++ b/src/services/tts/service.ts @@ -42,6 +42,27 @@ function requestDeadline(timeoutMs: number, external?: AbortSignal) { const ENDPOINT_SPEECH = "/v1/speech"; const DEFAULT_TOKEN_BASE_URL = "https://token.assemblyai.com"; +/** A zone's TTS host (`tts.eu.assemblyai.com`); its zone is the first group. */ +const ZONE_TTS_HOST = /^(?:streaming-)?tts\.(us|eu)\.assemblyai\.com$/; + +/** + * The token host of the zone `ttsBaseUrl` is in, so a token is minted in the + * zone its sessions use: `https://token.eu.assemblyai.com` for + * `https://tts.eu.assemblyai.com`, `https://token.us.assemblyai.com` for + * `https://tts.us.assemblyai.com`, and `https://token.assemblyai.com` for any + * other host. + */ +function zoneTokenBaseUrl(ttsBaseUrl?: string): string { + let host = ""; + try { + if (ttsBaseUrl) host = new URL(ttsBaseUrl).hostname; + } catch { + // not a URL, so no zone + } + const zone = ZONE_TTS_HOST.exec(host)?.[1]; + return zone ? `https://token.${zone}.assemblyai.com` : DEFAULT_TOKEN_BASE_URL; +} + /** Each socket operation of a streamed request. */ const DEFAULT_STREAM_TIMEOUT_MS = 30_000; /** A buffered request waits for the whole audio; the API caps a request at 600 s. */ @@ -383,7 +404,9 @@ export class TextToSpeechService extends BaseService { * * The client's API key is used when `params` has neither `token` nor * `apiKey`; the session host follows the client's `ttsBaseUrl` unless - * `websocketBaseUrl` is given. + * `websocketBaseUrl` is given. In a browser, open the session with + * `new TextToSpeechSession({ token })` instead, so the page needs no client + * and no API key. */ session(params: TextToSpeechSessionParams = {}): TextToSpeechSession { const sessionParams = { ...params }; @@ -400,8 +423,18 @@ export class TextToSpeechService extends BaseService { } /** - * Mints a temporary token for a session opened on a user's device, where - * the API key must not go. Call this server-side. + * Mints a temporary token for a session or an HTTP request made from a + * user's device, where the API key must not go. Call this server-side, + * once per connection (reconnects included) and per HTTP request. + * + * The token is minted on the token host of the client's zone: + * `https://token.eu.assemblyai.com` when `ttsBaseUrl` is + * `https://tts.eu.assemblyai.com`, `https://token.us.assemblyai.com` when + * it is `https://tts.us.assemblyai.com` (the default), and + * `https://token.assemblyai.com` for any other host. A token works in + * every zone. + * @param options.tokenBaseUrl - The token host to use instead. + * @throws TextToSpeechError if the token service refuses the request. */ async createTemporaryToken( params: TextToSpeechTokenParams, @@ -412,10 +445,9 @@ export class TextToSpeechService extends BaseService { if (value !== undefined && value !== null) search.append(key, String(value)); }); - const base = (options.tokenBaseUrl ?? DEFAULT_TOKEN_BASE_URL).replace( - /\/$/, - "", - ); + const base = ( + options.tokenBaseUrl ?? zoneTokenBaseUrl(this.serviceParams.baseUrl) + ).replace(/\/$/, ""); const response = await this.fetchResponse( `${base}/v1/token?${search.toString()}`, { diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index c741037..04e7dc1 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -175,7 +175,12 @@ function errorFromClose(code: number, reason: string): TextToSpeechError { * optional. A refusal rejects `connect()`; a failure mid-session rejects the * `audio()` iteration or the next send, and a failure nothing observed * rejects `close()`. The `error` and `close` listeners run once the - * connection has ended, so either can `connect()` again. + * connection has ended, so either can `connect()` again; a session opened + * with a temporary token reconnects with a new one, `connect({ token })`. + * + * In a browser, create the session with a temporary token your server + * minted, `new TextToSpeechSession({ token })`: no client and no API key in + * the page. * * @example * ```ts @@ -203,7 +208,12 @@ export class TextToSpeechSession { /** The error that ended the session, if one did. */ error?: TextToSpeechError; - constructor(private readonly params: TextToSpeechSessionParams) {} + /** The temporary token of the next connection; `connect({ token })` replaces it. */ + private token?: string; + + constructor(private readonly params: TextToSpeechSessionParams) { + this.token = params.token; + } /** The current connection is over: sends refuse and readers drain. */ private get stopped(): boolean { @@ -252,7 +262,7 @@ export class TextToSpeechSession { set("word_boundaries", p.wordBoundaries ?? false); set("inactivity_timeout", p.inactivityTimeout); set("extend_biometric_consent", p.extendBiometricConsent); - set("token", p.token); + set("token", this.token); return url; } @@ -271,10 +281,18 @@ export class TextToSpeechSession { * Opens the socket and waits for `Begin`. * * A session the API has ended can be connected again with the same object. + * A temporary token opens one connection, so a session opened with a token + * needs a new one for each connection, reconnects included: mint it on + * your server and pass it here. The API may refuse a token it has already + * seen, and always refuses an expired one, with close code 1008 + * (`errorCode: "unauthorized"`, not retryable). + * @param options.token - A new temporary token. It replaces the session's + * token for this connection and the next ones, and is sent instead of the + * API key. * @throws TextToSpeechError if the API refuses the session, the handshake * fails, or `Begin` does not arrive in time. */ - connect(): Promise { + connect(options: { token?: string } = {}): Promise { if (this.stopped) { // The previous connection's close, when its socket was released before // the close event arrived. The close listener may connect again. @@ -284,6 +302,7 @@ export class TextToSpeechSession { if (this.socket && !this.stopped) { return Promise.reject(new Error("The session is already connected")); } + if (options.token) this.token = options.token; this.releaseSocket(); this.reset(); const connection = this.queue; @@ -320,7 +339,8 @@ export class TextToSpeechSession { if (raised) this.listeners.error?.(raised); }; - if (this.params.token) { + if (this.token) { + // The token alone: an Authorization header would override it. this.socket = polyfillWebSocketFactory(url.toString()); } else { if (conditions.browser) { diff --git a/src/types/tts/index.ts b/src/types/tts/index.ts index 013e296..8f6dfc0 100644 --- a/src/types/tts/index.ts +++ b/src/types/tts/index.ts @@ -134,7 +134,12 @@ export type SpeechResponse = SpeechResponseInfo & { audio: Uint8Array; }; -/** Parameters of `createTemporaryToken()`; the token opens one session from a browser. */ +/** + * Parameters of `createTemporaryToken()`. The token is a short-lived + * credential for a session or an HTTP request made from a user's device: + * mint a new one for every connection, reconnects included, and for every + * HTTP request, and keep `expires_in_seconds` short. + */ export type TextToSpeechTokenParams = { /** 1 to 600 seconds the token can wait before it opens a session. */ expires_in_seconds: number; @@ -158,7 +163,13 @@ export type TextToSpeechSessionParams = { inactivityTimeout?: number; /** `true` renews the consent of the cloned voice in `voiceId` (owner only). */ extendBiometricConsent?: boolean; - /** A temporary token, for a session opened on a user's device. */ + /** + * A temporary token, for a session opened on a user's device. It is sent + * instead of the API key and opens one connection: to reconnect, pass a + * new token to `connect({ token })`. The API may refuse a token it has + * already seen, and always refuses an expired one, with close code 1008 + * (`unauthorized`). + */ token?: string; /** The API key; the client's key is used when neither this nor `token` is given. */ apiKey?: string; diff --git a/tests/unit/mocks/ws.ts b/tests/unit/mocks/ws.ts index d5b176d..f9b0e5a 100644 --- a/tests/unit/mocks/ws.ts +++ b/tests/unit/mocks/ws.ts @@ -1,10 +1,12 @@ import { WebSocket } from "mock-socket"; export default class MockWebSocket extends WebSocket { - // the unused `options` parameter is required for properly mocking the ws WebSocket class - // eslint-disable-next-line @typescript-eslint/no-unused-vars + /** The options the client passed to the ws constructor, such as its `headers`. */ + readonly clientOptions?: unknown; + constructor(address: string | URL, options?: unknown) { super(address); + this.clientOptions = options; } removeAllListeners() { diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index d0b1ee7..25dce11 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -8,7 +8,7 @@ import { TextToSpeechSession, TtsBeginEvent, } from "../../src"; -import { createClient } from "./utils"; +import { createClient, defaultApiKey } from "./utils"; fetchMock.enableMocks(); @@ -38,6 +38,10 @@ type ServerSocket = { reason: string; wasClean: boolean; }) => void; + /** The URL the client connected to. */ + url: string; + /** The options the client passed to the ws constructor (its headers). */ + clientOptions?: unknown; }; /** @@ -50,6 +54,9 @@ class FakeApi { flushId = 0; answerFlush = true; closeOnTerminate = true; + /** Refuse a temporary token seen before, as the API may. */ + singleUseTokens = false; + private usedTokens = new Set(); sent: Record[] = []; /** The server side of each connection, in order. */ sockets: ServerSocket[] = []; @@ -59,6 +66,25 @@ class FakeApi { this.sockets.push(socket); this.text = ""; // a new connection is a new session: ids start at 0 this.flushId = 0; + const token = new URL(socket.url).searchParams.get("token"); + if (this.singleUseTokens && token !== null) { + if (this.usedTokens.has(token)) { + socket.send( + JSON.stringify({ + type: "Error", + error_code: 1008, + error: "Unauthorized: token already used", + error_type: "unauthorized", + retryable: false, + param: null, + retry_after_seconds: null, + }), + ); + socket.close({ code: 1008, reason: "", wasClean: true }); + return; + } + this.usedTokens.add(token); + } socket.on("message", (raw) => { const frame = JSON.parse(String(raw)) as Record; this.sent.push(frame); @@ -397,29 +423,102 @@ describe("tts session", () => { expect(error.closeCode).toBe(1006); }); - it("a token goes in the query and the session can be reused", async () => { + it("a token session reconnects with a new token, sent in the query alone", async () => { + api.singleUseTokens = true; const session = client.tts.session({ websocketBaseUrl, - token: "tmp", + token: "tmp-1", inactivityTimeout: 30, }); await connect(session); - const url = new URL( - (session as unknown as { socket: { url: string } }).socket.url, - ); - expect(url.searchParams.get("token")).toBe("tmp"); + const first = await connection(1); + const url = new URL(first.url); + expect(url.searchParams.get("token")).toBe("tmp-1"); expect(url.searchParams.get("inactivity_timeout")).toBe("30"); + expect(first.clientOptions).toBeUndefined(); // no Authorization header expect((await collect(session.speak("One."))).toString()).toBe("One."); await session.close(); - // same object, new connection: `server.connected` already resolved once, - // so wait for the new socket before sending its Begin - const reconnect = session.connect(); - await new Promise((resolve) => setTimeout(resolve, 50)); - server.send(JSON.stringify(beginMessage)); + // Same object, new connection, new token. + const reconnect = session.connect({ token: "tmp-2" }); + const second = await connection(2); + expect(new URL(second.url).searchParams.get("token")).toBe("tmp-2"); + second.send(JSON.stringify(beginMessage)); await reconnect; expect((await collect(session.speak("Two."))).toString()).toBe("Two."); await session.close(); + + // The new token stays the session's token, and a used token is refused. + const reused = session.connect().catch((error: unknown) => error); + const third = await connection(3); + expect(new URL(third.url).searchParams.get("token")).toBe("tmp-2"); + expect(await reused).toMatchObject({ + closeCode: 1008, + errorCode: "unauthorized", + retryable: false, + }); + }); + + it("the error listener reconnects a token session with a token minted then", async () => { + api.singleUseTokens = true; + // As in a browser: no client and no API key. + const session = new TextToSpeechSession({ + websocketBaseUrl, + token: "tmp-1", + }); + let minted = 1; + const mintToken = async () => `tmp-${++minted}`; + let reconnecting: Promise | undefined; + session.on("error", (error) => { + if (error.retryable) { + reconnecting = mintToken().then((token) => session.connect({ token })); + } + }); + await connect(session); + api.sockets[0].send(errorFrame(1012, "service_restart", true)); + api.sockets[0].close({ code: 1012, reason: "", wasClean: true }); + + const second = await connection(2); + expect(new URL(second.url).searchParams.get("token")).toBe("tmp-2"); + second.send(JSON.stringify(beginMessage)); + await expect(reconnecting).resolves.toMatchObject({ id: "sess-1" }); + expect((await collect(session.speak("Back."))).toString()).toBe("Back."); + await session.close(); + }); + + it("connect({ token }) sends the token instead of the API key", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const first = await connection(1); + expect(new URL(first.url).searchParams.has("token")).toBe(false); + expect(first.clientOptions).toEqual({ + headers: { Authorization: defaultApiKey }, + }); + await session.close(); + + const reconnect = session.connect({ token: "tmp" }); + const second = await connection(2); + expect(new URL(second.url).searchParams.get("token")).toBe("tmp"); + expect(second.clientOptions).toBeUndefined(); + second.send(JSON.stringify(beginMessage)); + await reconnect; + await session.close(); + }); + + it("a connect({ token }) refused while connected keeps the session's token", async () => { + const session = client.tts.session({ websocketBaseUrl, token: "tmp-1" }); + await connect(session); + await expect(session.connect({ token: "tmp-2" })).rejects.toThrow( + /already connected/, + ); + await session.close(); + + const reconnect = session.connect(); + const second = await connection(2); + expect(new URL(second.url).searchParams.get("token")).toBe("tmp-1"); + second.send(JSON.stringify(beginMessage)); + await reconnect; + await session.close(); }); it("a send before connect says what to do", () => { diff --git a/tests/unit/tts.test.ts b/tests/unit/tts.test.ts index 017a79e..eef5142 100644 --- a/tests/unit/tts.test.ts +++ b/tests/unit/tts.test.ts @@ -234,6 +234,59 @@ describe("tts", () => { ).toBe("tmp-token"); }); + it.each([ + { + ttsBaseUrl: "https://tts.eu.assemblyai.com/", + tokenHost: "https://token.eu.assemblyai.com", + }, + { + ttsBaseUrl: "https://streaming-tts.eu.assemblyai.com", + tokenHost: "https://token.eu.assemblyai.com", + }, + { + ttsBaseUrl: "https://tts.us.assemblyai.com", + tokenHost: "https://token.us.assemblyai.com", + }, + { ttsBaseUrl: undefined, tokenHost: "https://token.us.assemblyai.com" }, + { + ttsBaseUrl: "https://tts.example.com", + tokenHost: "https://token.assemblyai.com", + }, + ])( + "mints the token in the zone of ttsBaseUrl $ttsBaseUrl", + async ({ ttsBaseUrl, tokenHost }) => { + const zoned = new AssemblyAI({ apiKey: "k", ttsBaseUrl }); + fetchMock.mockResponseOnce( + JSON.stringify({ token: "zone-token", expires_in_seconds: 60 }), + ); + expect( + await zoned.tts.createTemporaryToken({ expires_in_seconds: 60 }), + ).toBe("zone-token"); + expect(fetchMock.mock.calls[0][0]).toBe( + `${tokenHost}/v1/token?product=tts&expires_in_seconds=60`, + ); + }, + ); + + it("an explicit tokenBaseUrl wins over the client's zone", async () => { + const eu = new AssemblyAI({ + apiKey: "k", + ttsBaseUrl: "https://tts.eu.assemblyai.com", + }); + fetchMock.mockResponseOnce( + JSON.stringify({ token: "own-token", expires_in_seconds: 60 }), + ); + expect( + await eu.tts.createTemporaryToken( + { expires_in_seconds: 60 }, + { tokenBaseUrl: "https://token.example.com/" }, + ), + ).toBe("own-token"); + expect(fetchMock.mock.calls[0][0]).toBe( + "https://token.example.com/v1/token?product=tts&expires_in_seconds=60", + ); + }); + it("leaving a stream early cancels the body", async () => { const { TextToSpeechStream } = await import( "../../src/services/tts/service" From d961376bdfd163a92a84e5be7b75a9bc9e0562d7 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:44:57 +0000 Subject: [PATCH 07/26] fix(tts): isolate session listeners and close the socket on an unreadable frame Every listener (begin, audio, flushDone, wordBoundaries, cancelled, termination, error, close) now runs through one emit() that logs a throw with console.error and lets the session go on, as the Python session's _emit does. A bug in an app's player no longer fails the session as "Unreadable frame from the API", a throwing begin listener no longer leaves connect() waiting for its timeout, and a throwing close listener no longer rejects close(). Only parsing and decoding can make a frame unreadable: a JSON message that is not an object is ignored, an Audio frame needs base64 audio and an integer flush_id, a FlushDone an integer flush_id. An unreadable frame fails the session, releases the socket so the API stops synthesizing, then runs the error and close listeners; the error listener may connect again. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- README.md | 2 +- src/services/tts/session.ts | 180 ++++++++++++++++++++++++--------- tests/unit/tts-session.test.ts | 174 ++++++++++++++++++++++++++++++- 3 files changed, 300 insertions(+), 56 deletions(-) diff --git a/README.md b/README.md index 9c530a6..0b0d89b 100644 --- a/README.md +++ b/README.md @@ -906,7 +906,7 @@ await session.close(); `audio()` fixes its request when it is called, so the iterator can be made before the first `generate()` and handed to a player that starts later. Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order, once; `audio()` throws a `RangeError` for an id not flushed yet or already read. Every `flush()` and `cancel()` takes the next id, so ids skip one after a cancel. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters, so pace text to the audio. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes after `inactivityTimeout` seconds (5 to 120, default 120) without a frame; call `keepAlive()` between sparse requests. -Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send, reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `TextToSpeechError.status` is undefined and `requestId` is the session id; `retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. That works as is with an API key; a session opened with a temporary token needs a new token for each connection, passed as `connect({ token })`, because the API may refuse a token it has seen and always refuses an expired one (close code 1008, `errorCode: "unauthorized"`). The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. +Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A listener that throws is logged with `console.error` and the session goes on; a frame the SDK cannot read fails the session with `Unreadable frame from the API` and closes the socket. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send, reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `TextToSpeechError.status` is undefined and `requestId` is the session id; `retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. That works as is with an API key; a session opened with a temporary token needs a new token for each connection, passed as `connect({ token })`, because the API may refuse a token it has seen and always refuses an expired one (close code 1008, `errorCode: "unauthorized"`). The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. In a browser the API key must stay on your server. Mint a temporary token there, and open the session in the page with `new TextToSpeechSession({ token })`, which needs no client and no API key: diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index 04e7dc1..7bce6ec 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -126,6 +126,7 @@ function codePointLength(text: string): number { return Array.from(text).length; } +/** @throws DOMException for text that is not base64, which makes its frame unreadable. */ function base64ToBytes(data: string): Uint8Array { const binary = atob(data); const bytes = new Uint8Array(binary.length); @@ -133,6 +134,71 @@ function base64ToBytes(data: string): Uint8Array { return bytes; } +/** A server frame after `Begin`, read into the event its listener receives. */ +type SessionFrame = + | TtsAudioEvent + | TtsFlushDoneEvent + | TtsWordBoundariesEvent + | TtsCancelledEvent + | TtsTerminationEvent; + +/** + * The message, text or UTF-8 bytes, as a JSON object; `undefined` for + * anything else, which carries nothing this SDK reads. + */ +function parseFrame(data: unknown): Record | undefined { + if (typeof data !== "string" && !(data instanceof ArrayBuffer)) return; + const text = typeof data === "string" ? data : new TextDecoder().decode(data); + let frame: unknown; + try { + frame = JSON.parse(text); + } catch { + return; + } + if (typeof frame !== "object" || frame === null || Array.isArray(frame)) { + return; + } + return frame as Record; +} + +function flushIdOf(frame: Record): number { + const flushId = frame.flush_id; + if (typeof flushId !== "number" || !Number.isInteger(flushId)) { + throw new TypeError( + `${String(frame.type)} frame without an integer flush_id`, + ); + } + return flushId; +} + +/** + * Checks the fields this SDK reads and decodes the audio; `undefined` for a + * frame type this SDK does not know. + * @throws Error for a frame the SDK cannot read. + */ +function readFrame(frame: Record): SessionFrame | undefined { + switch (frame.type) { + case "Audio": + if (typeof frame.audio !== "string") { + throw new TypeError("Audio frame without base64 audio"); + } + return { + type: "Audio", + flush_id: flushIdOf(frame), + audio: base64ToBytes(frame.audio), + }; + case "FlushDone": + flushIdOf(frame); + return frame as unknown as TtsFlushDoneEvent; + case "WordBoundaries": + case "Cancelled": + case "Termination": + return frame as unknown as SessionFrame; + default: + return; + } +} + function errorFromFrame(frame: TtsErrorMessage): TextToSpeechError { return new TextToSpeechError(frame.error || "text-to-speech session failed", { errorCode: frame.error_type, @@ -172,9 +238,10 @@ function errorFromClose(code: number, reason: string): TextToSpeechError { * one request at a time in order, or pushed to an `audio` listener * registered with `on()`. Listeners for the other events (`flushDone`, * `wordBoundaries`, `cancelled`, `termination`, `error`, `close`) are - * optional. A refusal rejects `connect()`; a failure mid-session rejects the - * `audio()` iteration or the next send, and a failure nothing observed - * rejects `close()`. The `error` and `close` listeners run once the + * optional. A listener that throws is logged with `console.error` and the + * session goes on. A refusal rejects `connect()`; a failure mid-session + * rejects the `audio()` iteration or the next send, and a failure nothing + * observed rejects `close()`. The `error` and `close` listeners run once the * connection has ended, so either can `connect()` again; a session opened * with a temporary token reconnects with a new one, `connect({ token })`. * @@ -336,7 +403,21 @@ export class TextToSpeechSession { const fail = (error: TextToSpeechError) => { const raised = this.fail(error); settle(() => reject(this.surfaced())); - if (raised) this.listeners.error?.(raised); + if (raised) this.emit("error", raised); + }; + // A frame the SDK cannot read fails the session, and closing the socket + // stops the API from synthesizing for it. The socket is released before + // the error listener runs, which may connect again. + const unreadable = (cause: unknown) => { + const raised = this.fail( + new TextToSpeechError( + `Unreadable frame from the API: ${String(cause)}`, + ), + ); + this.releaseSocket(); + settle(() => reject(this.surfaced())); + if (raised) this.emit("error", raised); + this.emitClose(connection); }; if (this.token) { @@ -359,34 +440,27 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c socket.onopen = () => {}; socket.onmessage = ({ data }: MessageEvent) => { - if (typeof data !== "string" && !(data instanceof ArrayBuffer)) return; - const text = - typeof data === "string" ? data : new TextDecoder().decode(data); - let frame: Record; - try { - frame = JSON.parse(text) as Record; - } catch { - return; - } + const frame = parseFrame(data); + if (!frame) return; if (frame.type === "Begin") { - this.begin = frame as unknown as TtsBeginEvent; - this.listeners.begin?.(this.begin); - settle(() => resolve(this.begin as TtsBeginEvent)); + const begin = frame as unknown as TtsBeginEvent; + this.begin = begin; + settle(() => resolve(begin)); + this.emit("begin", begin); return; } if (frame.type === "Error") { fail(errorFromFrame(frame as unknown as TtsErrorMessage)); return; } + let event: SessionFrame | undefined; try { - this.dispatch(frame); + event = readFrame(frame); } catch (cause) { - fail( - new TextToSpeechError( - `Unreadable frame from the API: ${String(cause)}`, - ), - ); + unreadable(cause); + return; } + if (event) this.dispatch(event); }; socket.onclose = ({ code, reason }: CloseEvent) => { @@ -401,11 +475,8 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c settle(() => reject(this.error ? this.surfaced() : errorFromClose(code, text)), ); - try { - if (raised) this.listeners.error?.(raised); - } finally { - this.emitClose(connection); - } + if (raised) this.emit("error", raised); + this.emitClose(connection); }; socket.onerror = (event: ErrorEvent) => { @@ -423,15 +494,11 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c }); } - private dispatch(frame: Record) { - switch (frame.type) { - case "Audio": { - const event: TtsAudioEvent = { - type: "Audio", - flush_id: frame.flush_id as number, - audio: base64ToBytes(frame.audio as string), - }; - if (this.listeners.audio) this.listeners.audio(event); + /** Applies a frame to the session's state, then calls its listener. */ + private dispatch(event: SessionFrame) { + switch (event.type) { + case "Audio": + if (this.listeners.audio) this.emit("audio", event); else { this.enqueue({ kind: "audio", @@ -440,31 +507,44 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c }); } return; - } - case "FlushDone": { - const event = frame as unknown as TtsFlushDoneEvent; + case "FlushDone": this.enqueue({ kind: "done", flushId: event.flush_id }); - this.listeners.flushDone?.(event); + this.emit("flushDone", event); return; - } case "WordBoundaries": - this.listeners.wordBoundaries?.( - frame as unknown as TtsWordBoundariesEvent, - ); + this.emit("wordBoundaries", event); return; case "Cancelled": { const cancelId = this.cancelIds.shift() ?? -1; this.enqueue({ kind: "cancelled", cancelId }); - this.listeners.cancelled?.(frame as unknown as TtsCancelledEvent); + this.emit("cancelled", event); return; } case "Termination": - this.termination = frame as unknown as TtsTerminationEvent; + this.termination = event; this.end(); - this.listeners.termination?.(this.termination); + this.emit("termination", event); return; - default: - return; // a frame type this SDK does not know + } + } + + /** + * Calls the listener of `event`. A listener that throws is logged and the + * session goes on: an error in the app never fails the session. + */ + private emit( + event: E, + ...args: Parameters> + ) { + const listener = this.listeners[event] as + | (( + ...args: Parameters> + ) => void) + | undefined; + try { + listener?.(...args); + } catch (error) { + console.error(`TextToSpeechSession: the ${event} listener threw`, error); } } @@ -497,7 +577,7 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c if (!connection.hadSocket || connection.closeEmitted) return; connection.closeEmitted = true; const [code, reason] = connection.close(); - this.listeners.close?.(code, reason); + this.emit("close", code, reason); } /** Marks the connection over with the session's error: readers drain, sends refuse. */ diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index 25dce11..5026986 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -60,8 +60,13 @@ class FakeApi { sent: Record[] = []; /** The server side of each connection, in order. */ sockets: ServerSocket[] = []; + /** Closes the server saw, from either side. */ + closes = 0; constructor(public server: WS) { + server.on("close", () => { + this.closes += 1; + }); server.on("connection", (socket) => { this.sockets.push(socket); this.text = ""; // a new connection is a new session: ids start at 0 @@ -180,6 +185,14 @@ function outcome(promise: Promise, ms = 300) { ]); } +let consoleError: jest.SpyInstance | undefined; + +/** Records console.error calls without printing them; restored after each test. */ +function spyConsoleError(): jest.SpyInstance { + consoleError = jest.spyOn(console, "error").mockImplementation(() => {}); + return consoleError; +} + function errorFrame(code: number, errorType: string, retryable: boolean) { return JSON.stringify({ type: "Error", @@ -200,6 +213,8 @@ describe("tts session", () => { }); afterEach(() => { WS.clean(); + consoleError?.mockRestore(); + consoleError = undefined; }); it("connects, waits for Begin, speaks and terminates", async () => { @@ -708,20 +723,169 @@ describe("tts session", () => { await session.close(); }); - it("a throwing error listener does not leave a reader waiting", async () => { + it("a throwing error listener is logged and does not leave a reader waiting", async () => { + const logged = spyConsoleError(); api.answerFlush = false; const session = client.tts.session({ websocketBaseUrl }); + const thrown = new Error("listener failed"); session.on("error", () => { - throw new Error("listener failed"); + throw thrown; }); await connect(session); const flushId = session.flush(); const reading = outcome(collect(session.audio(flushId))); await tick(); - expect(() => - api.sockets[0].send(errorFrame(3005, "at_capacity", true)), - ).toThrow("listener failed"); + api.sockets[0].send(errorFrame(3005, "at_capacity", true)); expect(await reading).toMatchObject({ errorCode: "at_capacity" }); + expect(logged.mock.calls).toEqual([ + ["TextToSpeechSession: the error listener threw", thrown], + ]); + await session.close(); + }); + + it("an audio listener that throws keeps receiving frames and the session stays usable", async () => { + const logged = spyConsoleError(); + const session = client.tts.session({ websocketBaseUrl }); + const received: string[] = []; + const thrown = new Error("bug in my player"); + session.on("audio", (e) => { + received.push(Buffer.from(e.audio).toString()); + if (received.length === 1) throw thrown; + }); + const errors: TextToSpeechError[] = []; + session.on("error", (e) => errors.push(e)); + await connect(session); + for (const text of ["One.", "Two.", "Three."]) { + session.generate(text); + session.flush(); + } + await tick(); + expect(received).toEqual(["One.", "Two.", "Three."]); + expect(logged.mock.calls).toEqual([ + ["TextToSpeechSession: the audio listener threw", thrown], + ]); + expect(errors).toEqual([]); + expect(session.error).toBeUndefined(); + + session.generate("Four."); + session.flush(); + await tick(); + expect(received).toEqual(["One.", "Two.", "Three.", "Four."]); + await expect(session.close()).resolves.toMatchObject({ + type: "Termination", + }); + }); + + it("every other listener that throws is logged and the session goes on", async () => { + const logged = spyConsoleError(); + api.closeOnTerminate = false; // the close listener runs inside close() + const session = client.tts.session({ websocketBaseUrl }); + const events = [ + "begin", + "flushDone", + "wordBoundaries", + "cancelled", + "termination", + "close", + ] as const; + for (const event of events) { + session.on(event as "begin", () => { + throw new Error(`${event} listener failed`); + }); + } + await expect(connect(session)).resolves.toMatchObject({ id: "sess-1" }); + expect((await collect(session.speak("Hi."))).toString()).toBe("Hi."); + api.sockets[0].send( + JSON.stringify({ + type: "WordBoundaries", + flush_id: 0, + audio_start_ms: 0, + words: [], + }), + ); + session.cancel(); + await tick(); + await expect(session.close()).resolves.toMatchObject({ + type: "Termination", + }); + expect(session.error).toBeUndefined(); + expect(logged.mock.calls).toEqual( + events.map((event) => [ + `TextToSpeechSession: the ${event} listener threw`, + new Error(`${event} listener failed`), + ]), + ); + }); + + it.each([ + { + name: "audio that is not base64", + frame: { type: "Audio", audio: "not base64!", flush_id: 0 }, + }, + { + name: "an Audio frame without audio", + frame: { type: "Audio", flush_id: 0 }, + }, + { + name: "an Audio frame without a flush_id", + frame: { type: "Audio", audio: "QQ==" }, + }, + { + name: "a FlushDone without a flush_id", + frame: { type: "FlushDone", audio_duration_ms: 80 }, + }, + ])("$name fails the session and closes the socket", async ({ frame }) => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + const errors: TextToSpeechError[] = []; + const closes: [number, string][] = []; + session.on("error", (e) => errors.push(e)); + session.on("close", (code, reason) => closes.push([code, reason])); + await connect(session); + const flushId = session.flush(); + const reading = outcome(collect(session.audio(flushId))); + await tick(); + api.sockets[0].send(JSON.stringify(frame)); + const error = await reading; + expect(error).toBeInstanceOf(TextToSpeechError); + expect(error).toMatchObject({ + message: expect.stringMatching(/^Unreadable frame from the API: /), + requestId: "sess-1", + }); + expect(errors).toEqual([error]); + await tick(); + expect(api.closes).toBe(1); // the client closed the socket + expect(closes).toEqual([[1000, "Closed by the client"]]); + expect(() => session.generate("late")).toThrow(error as Error); + await expect(session.close()).resolves.toBeUndefined(); // already surfaced + expect(closes).toHaveLength(1); + }); + + it("the error listener reconnects after an unreadable frame", async () => { + const session = client.tts.session({ websocketBaseUrl }); + let reconnecting: Promise | undefined; + session.on("error", () => { + reconnecting = session.connect(); + }); + await connect(session); + api.sockets[0].send( + JSON.stringify({ type: "Audio", audio: "not base64!", flush_id: 0 }), + ); + (await connection(2)).send(JSON.stringify(beginMessage)); + await expect(reconnecting).resolves.toMatchObject({ id: "sess-1" }); + expect((await collect(session.speak("Back."))).toString()).toBe("Back."); + await session.close(); + }); + + it("a JSON message that is not an object is ignored", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + for (const message of ["null", "42", '"Audio"', "[]", "not json"]) { + api.sockets[0].send(message); + } + expect((await collect(session.speak("Still here."))).toString()).toBe( + "Still here.", + ); await session.close(); }); From e1ee30c83c53e65c830094f59a816e8545e72ec3 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 14:52:08 +0000 Subject: [PATCH 08/26] fix(tts): report a refused session upgrade's status and settle every connect() failure A session's handshake error now carries the HTTP status that ws reports for a refused upgrade, and is retryable only for 408, 429 and 5xx, so a 404 from a wrong websocketBaseUrl path no longer looks retryable. A browser error event with no message gets a readable one. The session constructor requires an API key or a temporary token. A socket the runtime refuses to create rejects connect() with a TextToSpeechError and clears the connect timer, and a base URL that is not ws:// or wss:// rejects connect() before anything about the previous connection changes, instead of throwing synchronously. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- README.md | 2 +- src/services/tts/service.ts | 3 +- src/services/tts/session.ts | 94 +++++++++--- src/utils/errors/tts.ts | 10 +- tests/unit/tts-session-handshake.test.ts | 57 +++++++ tests/unit/tts-session.test.ts | 185 +++++++++++++++++++++++ 6 files changed, 323 insertions(+), 28 deletions(-) create mode 100644 tests/unit/tts-session-handshake.test.ts diff --git a/README.md b/README.md index 0b0d89b..45f2b10 100644 --- a/README.md +++ b/README.md @@ -906,7 +906,7 @@ await session.close(); `audio()` fixes its request when it is called, so the iterator can be made before the first `generate()` and handed to a player that starts later. Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order, once; `audio()` throws a `RangeError` for an id not flushed yet or already read. Every `flush()` and `cancel()` takes the next id, so ids skip one after a cancel. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters, so pace text to the audio. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes after `inactivityTimeout` seconds (5 to 120, default 120) without a frame; call `keepAlive()` between sparse requests. -Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A listener that throws is logged with `console.error` and the session goes on; a frame the SDK cannot read fails the session with `Unreadable frame from the API` and closes the socket. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send, reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `TextToSpeechError.status` is undefined and `requestId` is the session id; `retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. That works as is with an API key; a session opened with a temporary token needs a new token for each connection, passed as `connect({ token })`, because the API may refuse a token it has seen and always refuses an expired one (close code 1008, `errorCode: "unauthorized"`). The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. +Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A listener that throws is logged with `console.error` and the session goes on; a frame the SDK cannot read fails the session with `Unreadable frame from the API` and closes the socket. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send, reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `requestId` is the session id, and `status` is set only when the server answered the WebSocket upgrade with an HTTP status the runtime reports (Node does, a browser does not): a 404 for a wrong `websocketBaseUrl` path is not retryable, a 503 is. `retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. That works as is with an API key; a session opened with a temporary token needs a new token for each connection, passed as `connect({ token })`, because the API may refuse a token it has seen and always refuses an expired one (close code 1008, `errorCode: "unauthorized"`). The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. In a browser the API key must stay on your server. Mint a temporary token there, and open the session in the page with `new TextToSpeechSession({ token })`, which needs no client and no API key: diff --git a/src/services/tts/service.ts b/src/services/tts/service.ts index 60761be..c13619a 100644 --- a/src/services/tts/service.ts +++ b/src/services/tts/service.ts @@ -8,7 +8,7 @@ import { TextToSpeechTokenParams, } from "../.."; import { BaseService } from "../base"; -import { TextToSpeechError } from "../../utils/errors/tts"; +import { RETRYABLE_STATUSES, TextToSpeechError } from "../../utils/errors/tts"; import { TextToSpeechSession } from "./session"; /** @@ -67,7 +67,6 @@ function zoneTokenBaseUrl(ttsBaseUrl?: string): string { const DEFAULT_STREAM_TIMEOUT_MS = 30_000; /** A buffered request waits for the whole audio; the API caps a request at 600 s. */ const DEFAULT_SYNTHESIZE_TIMEOUT_MS = 610_000; -const RETRYABLE_STATUSES = new Set([408, 429, 500, 502, 503, 504]); /** Builds the `TextToSpeechError` for a non-200 response. */ export async function speechErrorFromResponse( diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index 7bce6ec..d0b017b 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -16,7 +16,7 @@ import { TtsTerminationEvent, TtsWordBoundariesEvent, } from "../.."; -import { TextToSpeechError } from "../../utils/errors/tts"; +import { RETRYABLE_STATUSES, TextToSpeechError } from "../../utils/errors/tts"; const DEFAULT_WEBSOCKET_BASE_URL = "wss://tts.us.assemblyai.com/v1/ws"; const DEFAULT_CONNECT_TIMEOUT_MS = 10_000; @@ -27,6 +27,8 @@ const MAX_GENERATE_CHARS = 2000; const PULL_QUEUE_FRAMES = 2000; /** Closes after which a new session can succeed. */ const RETRYABLE_CLOSE_CODES = new Set([1006, 1011, 1012, 3005, 3008, 3009]); +/** How the `ws` package reports an upgrade answered with an HTTP status. */ +const UPGRADE_STATUS = /Unexpected server response: (\d{3})/; type PullItem = | { kind: "audio"; flushId: number; bytes: Uint8Array } @@ -213,6 +215,24 @@ function errorFromFrame(frame: TtsErrorMessage): TextToSpeechError { }); } +/** + * The error of a WebSocket that failed before it opened. Node's `ws` names + * the HTTP status of a refused upgrade in its message; a browser's error + * event carries no message and no status. + */ +function errorFromHandshake(event: ErrorEvent): TextToSpeechError { + const message = + (event.error as Error | undefined)?.message || + event.message || + "network error or the server refused the upgrade"; + const match = UPGRADE_STATUS.exec(message); + const status = match ? Number(match[1]) : undefined; + return new TextToSpeechError(`WebSocket handshake failed: ${message}`, { + status, + retryable: status === undefined || RETRYABLE_STATUSES.has(status), + }); +} + function errorFromClose(code: number, reason: string): TextToSpeechError { return new TextToSpeechError( `Connection closed (code=${code})${reason ? `: ${reason}` : ""}`, @@ -278,7 +298,11 @@ export class TextToSpeechSession { /** The temporary token of the next connection; `connect({ token })` replaces it. */ private token?: string; + /** @throws Error if neither `apiKey` nor `token` is given. */ constructor(private readonly params: TextToSpeechSessionParams) { + if (!params.token && !params.apiKey) { + throw new Error("API key or temporary token is required."); + } this.token = params.token; } @@ -306,7 +330,8 @@ export class TextToSpeechSession { (this.listeners as Record)[event] = listener; } - private connectionUrl(): URL { + /** @throws Error if `websocketBaseUrl` is not a ws:// or wss:// URL. */ + private baseUrl(): URL { const url = new URL( this.params.websocketBaseUrl ?? DEFAULT_WEBSOCKET_BASE_URL, ); @@ -315,6 +340,11 @@ export class TextToSpeechSession { "The WebSocket base URL must use the wss:// or ws:// protocol", ); } + return url; + } + + /** `url` with the session's parameters and token in its query. */ + private connectionUrl(url: URL): URL { const p = this.params; const set = (name: string, value: unknown) => { if (value !== undefined && value !== null) { @@ -357,9 +387,19 @@ export class TextToSpeechSession { * token for this connection and the next ones, and is sent instead of the * API key. * @throws TextToSpeechError if the API refuses the session, the handshake - * fails, or `Begin` does not arrive in time. + * fails, the socket cannot be opened, or `Begin` does not arrive in time. + * `status` is set when the API or a proxy answered the upgrade with an + * HTTP status the runtime reports. + * @throws Error, before anything else changes, if `websocketBaseUrl` is + * not a ws:// or wss:// URL. */ connect(options: { token?: string } = {}): Promise { + let base: URL; + try { + base = this.baseUrl(); + } catch (error) { + return Promise.reject(error); + } if (this.stopped) { // The previous connection's close, when its socket was released before // the close event arrived. The close listener may connect again. @@ -370,10 +410,11 @@ export class TextToSpeechSession { return Promise.reject(new Error("The session is already connected")); } if (options.token) this.token = options.token; + const token = this.token; this.releaseSocket(); this.reset(); const connection = this.queue; - const url = this.connectionUrl(); + const url = this.connectionUrl(base); const timeoutMs = this.params.connectTimeout ?? DEFAULT_CONNECT_TIMEOUT_MS; return new Promise((resolve, reject) => { @@ -420,21 +461,32 @@ export class TextToSpeechSession { this.emitClose(connection); }; - if (this.token) { - // The token alone: an Authorization header would override it. - this.socket = polyfillWebSocketFactory(url.toString()); - } else { - if (conditions.browser) { - console.warn( - `API key authentication is not supported for the TextToSpeechSession in browser environments. Use temporary token authentication instead. + if (!token && conditions.browser) { + console.warn( + `API key authentication is not supported for the TextToSpeechSession in browser environments. Use temporary token authentication instead. Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/compat.md#browser-compatibility.`, - ); - } - this.socket = polyfillWebSocketFactory(url.toString(), { - headers: { Authorization: this.params.apiKey }, - }); + ); + } + let socket: PolyfillWebSocket; + try { + socket = token + ? // The token alone: an Authorization header would override it. + polyfillWebSocketFactory(url.toString()) + : polyfillWebSocketFactory(url.toString(), { + headers: { Authorization: this.params.apiKey }, + }); + } catch (cause) { + // The runtime refused to create the socket (a browser's WebSocket + // takes no headers, for one), so there is no connection to close. + this.error = new TextToSpeechError( + `Could not open the WebSocket: ${cause instanceof Error ? cause.message : String(cause)}`, + { retryable: false }, + ); + this.end(); + settle(() => reject(this.surfaced())); + return; } - const socket = this.socket; + this.socket = socket; connection.hadSocket = true; socket.binaryType = "arraybuffer"; socket.onopen = () => {}; @@ -481,13 +533,7 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c socket.onerror = (event: ErrorEvent) => { if (settled) return; // after Begin, the close that follows reports it - const message = - (event.error as Error | undefined)?.message ?? event.message; - const error = new TextToSpeechError( - `WebSocket handshake failed: ${message}`, - { retryable: true }, - ); - this.error = error; + this.error = errorFromHandshake(event); this.end(); settle(() => reject(this.surfaced())); }; diff --git a/src/utils/errors/tts.ts b/src/utils/errors/tts.ts index 6ad343c..43fe515 100644 --- a/src/utils/errors/tts.ts +++ b/src/utils/errors/tts.ts @@ -1,6 +1,9 @@ /** The fields a Text-to-Speech failure carries besides its message. */ export type TextToSpeechErrorFields = { - /** The HTTP status; absent for a session failure, which has no HTTP status. */ + /** + * The HTTP status. On a session, only a refused WebSocket upgrade has one, + * and only where the runtime reports it (Node does, a browser does not). + */ status?: number; /** The API's stable code: `invalid_parameter`, `voice_not_found`, `rate_limited`, ... */ errorCode?: string; @@ -20,6 +23,11 @@ export type TextToSpeechErrorFields = { body?: Record; }; +/** HTTP statuses at which the same request, or a new session, can succeed later. */ +export const RETRYABLE_STATUSES: ReadonlySet = new Set([ + 408, 429, 500, 502, 503, 504, +]); + /** * Error thrown when a Text-to-Speech request or session fails. * diff --git a/tests/unit/tts-session-handshake.test.ts b/tests/unit/tts-session-handshake.test.ts new file mode 100644 index 0000000..c8ad562 --- /dev/null +++ b/tests/unit/tts-session-handshake.test.ts @@ -0,0 +1,57 @@ +import { createServer, Server } from "http"; +import { AddressInfo, Socket } from "net"; +import { TextToSpeechError, TextToSpeechSession } from "../../src"; + +/** + * The real `ws` client against a server that answers the upgrade with an + * HTTP status, so the status is read from the message `ws` actually writes. + */ +describe("tts session handshake refused with an HTTP status", () => { + let server: Server; + let sockets: Socket[]; + let status: number; + + beforeEach(async () => { + sockets = []; + server = createServer(); + server.on("upgrade", (_request, socket: Socket) => { + socket.end( + `HTTP/1.1 ${status} Refused\r\nContent-Length: 0\r\nConnection: close\r\n\r\n`, + ); + }); + server.on("connection", (socket) => sockets.push(socket)); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve), + ); + }); + + afterEach(async () => { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(resolve)); + }); + + it.each([ + [404, false], + [503, true], + ])( + "an upgrade answered %i sets the status and retryable %s", + async (code, retryable) => { + status = code; + const { port } = server.address() as AddressInfo; + const session = new TextToSpeechSession({ + websocketBaseUrl: `ws://127.0.0.1:${port}/v1/ws`, + apiKey: "apikey_123", + }); + + const error = await session.connect().catch((e: unknown) => e); + + expect(error).toBeInstanceOf(TextToSpeechError); + expect(error).toMatchObject({ + message: `WebSocket handshake failed: Unexpected server response: ${code}`, + status: code, + retryable, + }); + await expect(session.close()).resolves.toBeUndefined(); + }, + ); +}); diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index 5026986..29a59ee 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -8,6 +8,7 @@ import { TextToSpeechSession, TtsBeginEvent, } from "../../src"; +import * as websocketPolyfill from "../../src/polyfills/websocket/default"; import { createClient, defaultApiKey } from "./utils"; fetchMock.enableMocks(); @@ -1110,4 +1111,188 @@ describe("tts session", () => { await session.close(); expect(queue.size).toBe(0); }); + + describe("connect() failures before the socket opens", () => { + type FakeSocket = { + OPEN: number; + readyState: number; + binaryType: string; + onopen: (() => void) | null; + onerror: ((event: unknown) => void) | null; + onclose: ((event: unknown) => void) | null; + onmessage: ((event: unknown) => void) | null; + send: jest.Mock; + close: jest.Mock; + }; + + /** Hands the session a socket the test drives, as the runtime would. */ + function fakeSocket(): FakeSocket { + const socket: FakeSocket = { + OPEN: 1, + readyState: 0, + binaryType: "blob", + onopen: null, + onerror: null, + onclose: null, + onmessage: null, + send: jest.fn(), + close: jest.fn(), + }; + jest + .spyOn(websocketPolyfill, "factory") + .mockReturnValueOnce( + socket as unknown as ReturnType, + ); + return socket; + } + + afterEach(() => { + jest.restoreAllMocks(); + jest.useRealTimers(); + }); + + it.each([ + [404, false], + [400, false], + [401, false], + [403, false], + [408, true], + [429, true], + [502, true], + [503, true], + ])( + "an upgrade answered %i carries the status and retryable %s", + async (status, retryable) => { + // Given a session whose upgrade the server answers with an HTTP status + const socket = fakeSocket(); + const session = client.tts.session({ websocketBaseUrl }); + const connecting = session.connect(); + + // When ws reports the refused upgrade + const message = `Unexpected server response: ${status}`; + socket.onerror?.({ error: new Error(message), message }); + + // Then connect() rejects with the status, retryable only when transient + const error = await connecting.catch((e: unknown) => e); + expect(error).toBeInstanceOf(TextToSpeechError); + expect(error).toMatchObject({ + message: `WebSocket handshake failed: ${message}`, + status, + retryable, + }); + expect(session.error).toBe(error); + }, + ); + + it("a handshake error event without a message says what may have happened", async () => { + // Given a browser, whose error event has neither error nor message + const socket = fakeSocket(); + const session = client.tts.session({ websocketBaseUrl }); + const connecting = session.connect(); + + // When the socket fails before it opens + socket.onerror?.({ type: "error" }); + + // Then the error explains it and stays retryable, with no status + const error = await connecting.catch((e: unknown) => e); + expect(error).toMatchObject({ + message: + "WebSocket handshake failed: network error or the server refused the upgrade", + status: undefined, + retryable: true, + }); + }); + + it("a session needs an API key or a temporary token", () => { + expect(() => new TextToSpeechSession({ websocketBaseUrl })).toThrow( + "API key or temporary token is required.", + ); + expect( + () => + new TextToSpeechSession({ websocketBaseUrl, apiKey: "", token: "" }), + ).toThrow("API key or temporary token is required."); + expect( + () => new TextToSpeechSession({ websocketBaseUrl, token: "tmp-1" }), + ).not.toThrow(); + // The client fills in its own key. + expect(() => client.tts.session({ websocketBaseUrl })).not.toThrow(); + }); + + it("a socket the runtime refuses to create rejects connect() and leaves no timer", async () => { + // Given a runtime whose WebSocket constructor throws, as a browser's + // does for options it cannot take + jest.useFakeTimers({ doNotFake: ["nextTick", "queueMicrotask"] }); + jest.spyOn(websocketPolyfill, "factory").mockImplementationOnce(() => { + throw new SyntaxError("An invalid or illegal string was specified"); + }); + const session = client.tts.session({ + websocketBaseUrl, + connectTimeout: 3000, + }); + + // When the session connects + const error = await session.connect().catch((e: unknown) => e); + + // Then connect() rejects with a TextToSpeechError and no timer is left + expect(error).toBeInstanceOf(TextToSpeechError); + expect(error).toMatchObject({ + message: + "Could not open the WebSocket: An invalid or illegal string was specified", + retryable: false, + }); + expect(jest.getTimerCount()).toBe(0); + jest.useRealTimers(); + + // And the session sends nothing, closes quietly and can connect again + expect(() => session.keepAlive()).toThrow(error as TextToSpeechError); + await expect(session.close()).resolves.toBeUndefined(); + await connect(session); + expect((await collect(session.speak("Again."))).toString()).toBe( + "Again.", + ); + await session.close(); + }); + + it.each(["https://localhost:1234/v1/ws", "not a url"])( + "connect() with base URL %p rejects and leaves the previous connection as it was", + async (badUrl) => { + // Given a session whose last connection failed and whose close is + // still to be reported + api.answerFlush = false; + const params = { websocketBaseUrl, apiKey: defaultApiKey }; + const session = new TextToSpeechSession(params); + const events: string[] = []; + session.on("close", (code) => events.push(`close ${code}`)); + await connect(session); + const flushId = session.flush(); + api.sockets[0].send(errorFrame(3005, "at_capacity", true)); // no close yet + await expect(collect(session.audio(flushId))).rejects.toMatchObject({ + closeCode: 3005, + }); + const failed = session.error; + + // When it connects to a base URL that is not ws:// or wss:// + params.websocketBaseUrl = badUrl; + let connecting: Promise | undefined; + expect(() => { + connecting = session.connect(); + }).not.toThrow(); + + // Then connect() rejects and nothing about the last connection changed + await expect(connecting).rejects.toThrow(); + expect(session.error).toBe(failed); + expect(session.begin?.id).toBe("sess-1"); + expect(events).toEqual([]); + expect(api.sockets).toHaveLength(1); + + // And a valid URL connects again, reporting the last close first + params.websocketBaseUrl = websocketBaseUrl; + const reconnecting = session.connect(); + (await connection(2)).send(JSON.stringify(beginMessage)); + await reconnecting; + expect(events).toEqual(["close 3005"]); + await session.close(); + }, + ); + }); }); From f175b79e49928bd6d35d29f6efd2b0a1ed7e7b0b Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:06:49 +0000 Subject: [PATCH 09/26] fix(tts): share one close() per session connection and wait while the API drains close() is now shared per connection: the first call sends Terminate, and every other call on the same connection waits for that close and resolves when the connection ends. Before, each call sent its own Terminate and replaced the pending waiter, so a hang-up handler and a finally that both closed the session left the first caller waiting the whole terminateTimeout (forever with 0). Only the first call throws an unobserved session error; later calls resolve with the summary. terminateTimeout now counts milliseconds without a message from the API, not time since Terminate. The wait restarts on every frame, so a Terminate that speaks thousands of pending characters is no longer cut at 10 s while audio is still arriving. The default is now 35,000, a little over the API's own 30 s limit for a stalled drain, so the API's 3005 Error normally arrives first. When the wait does run out, the connection ends with a retryable TextToSpeechError ("No Termination within the terminate timeout; the remaining audio was dropped") on its closed item. A running audio() iteration then rejects instead of ending as if complete. close() still resolves undefined, session.error stays unset, and the close listener gets 1000 "Closed by the client". A close() that waited while a termination or error listener connected again now resolves with its own connection's summary. Before, it ended the new connection and closed its socket. The close listener and the close reason now read each connection's own Termination. Docs: close(), TextToSpeechSessionParams.terminateTimeout and the README session paragraph. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- README.md | 2 +- src/services/tts/session.ts | 125 +++++++++++++++++++++++------- src/types/tts/index.ts | 7 +- tests/unit/tts-session.test.ts | 135 ++++++++++++++++++++++++++++++++- 4 files changed, 235 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 45f2b10..cd24ab9 100644 --- a/README.md +++ b/README.md @@ -869,7 +869,7 @@ For a barge-in, pass `{ signal }` as the second argument and abort it: the downl ### Speak text as it is written (voice agents) -A `TextToSpeechSession` keeps one WebSocket open and speaks text as it is written, so an LLM's reply can be voiced sentence by sentence with no per-request handshake. `speak()` sends one text and returns its audio; `generate()` / `flush()` do the same in two steps, so tokens can be sent as they arrive; `cancel()` is the barge-in. `close()` ends the session and resolves with its summary. +A `TextToSpeechSession` keeps one WebSocket open and speaks text as it is written, so an LLM's reply can be voiced sentence by sentence with no per-request handshake. `speak()` sends one text and returns its audio; `generate()` / `flush()` do the same in two steps, so tokens can be sent as they arrive; `cancel()` is the barge-in. `close()` ends the session and resolves with its summary once the API has spoken the text still pending. It stops waiting after `terminateTimeout` milliseconds (default 35,000) without a message from the API; an `audio()` iteration still running then rejects with a retryable `TextToSpeechError`, and `close()` resolves `undefined`. Calling `close()` again, or from two places at once, waits for the same end. ```typescript const session = client.tts.session({ voiceId: "aai-jane" }); diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index d0b017b..8112de3 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -20,7 +20,12 @@ import { RETRYABLE_STATUSES, TextToSpeechError } from "../../utils/errors/tts"; const DEFAULT_WEBSOCKET_BASE_URL = "wss://tts.us.assemblyai.com/v1/ws"; const DEFAULT_CONNECT_TIMEOUT_MS = 10_000; -const DEFAULT_TERMINATE_TIMEOUT_MS = 10_000; +/** + * Milliseconds without a message from the API after which `close()` stops + * waiting: a little over the API's own 30 s drain limit, so the API's 3005 + * Error for a stalled drain normally arrives first. + */ +const DEFAULT_TERMINATE_TIMEOUT_MS = 35_000; /** Characters one `Generate` frame may carry, unless `Begin` says otherwise. */ const MAX_GENERATE_CHARS = 2000; /** Audio frames queued for pulling before the oldest is dropped (~160 s). */ @@ -52,8 +57,12 @@ class PullQueue { ended = false; /** The error the connection ended with, if any. */ error?: TextToSpeechError; - /** The connection ended with the API's `Termination`. */ - terminated = false; + /** The API's `Termination` summary, when the connection ended with it. */ + termination?: TtsTerminationEvent; + /** When the API last sent a message; `close()` starts it at `Terminate`. */ + heardAt = 0; + /** The connection's one `close()`, which every call shares. */ + closing?: Promise; /** The socket's own close event, when it arrived. */ closedWith?: [code: number, reason: string]; closeEmitted = false; @@ -74,7 +83,7 @@ class PullQueue { if (this.closedWith) return this.closedWith; const code = this.error?.closeCode ?? 1000; if (code !== 1000) return [code, "See Error message for details"]; - return [code, this.terminated ? "Session ended" : "Closed by the client"]; + return [code, this.termination ? "Session ended" : "Closed by the client"]; } push(item: PullItem) { @@ -492,6 +501,7 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c socket.onopen = () => {}; socket.onmessage = ({ data }: MessageEvent) => { + connection.heardAt = Date.now(); const frame = parseFrame(data); if (!frame) return; if (frame.type === "Begin") { @@ -626,14 +636,17 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c this.emit("close", code, reason); } - /** Marks the connection over with the session's error: readers drain, sends refuse. */ - private end() { + /** + * Marks the connection over: readers drain, sends refuse. It ends with the + * session's error, else with `cut`, the audio a `close()` stopped waiting for. + */ + private end(cut?: TextToSpeechError) { const connection = this.queue; if (connection.ended) return; connection.ended = true; - connection.error = this.error; - connection.terminated = this.termination !== undefined; - this.enqueue({ kind: "closed", error: this.error }); + connection.error = this.error ?? cut; + connection.termination = this.termination; + this.enqueue({ kind: "closed", error: connection.error }); this.terminationResolve?.(); this.terminationResolve = undefined; } @@ -853,52 +866,108 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c /** * Ends the session: `Terminate`, then the `Termination` summary. * - * Text sent and not flushed is spoken first. Resolves with the summary, - * or `undefined` if the session had already ended or the wait timed out. - * A `connect()` still pending is rejected. + * Text sent and not flushed is spoken first, and the wait lasts as long as + * the API keeps sending: it stops after `terminateTimeout` milliseconds + * without a message from the API. Then the socket is closed, the API drops + * the audio still to come, and an `audio()` iteration still running + * rejects with a retryable `TextToSpeechError`. Resolves with the summary, + * or `undefined` if the session had already ended or the wait stopped. A + * `connect()` still pending is rejected. All calls on one connection share + * one close: one `Terminate` is sent, and every call resolves when the + * connection ends; the arguments of the first call apply. * @param waitForTermination - Wait for the summary before closing the socket. - * @param terminateTimeout - Milliseconds to wait for it; `0` waits indefinitely. - * @throws TextToSpeechError the session's error, when nothing else surfaced it. + * @param terminateTimeout - Milliseconds without a message from the API + * after which the wait stops; `0` waits indefinitely. Defaults to the + * session's `terminateTimeout`, else 35 000. + * @throws TextToSpeechError the session's error, when nothing else surfaced + * it; only the first call throws it. */ - async close( + close( waitForTermination = true, terminateTimeout = this.params.terminateTimeout ?? DEFAULT_TERMINATE_TIMEOUT_MS, + ): Promise { + const connection = this.queue; + if (connection.closing) { + // The first call is the one that throws the session's error. + return connection.closing.catch(() => connection.termination); + } + connection.closing = this.closeConnection( + connection, + waitForTermination, + terminateTimeout, + ); + return connection.closing; + } + + private async closeConnection( + connection: PullQueue, + waitForTermination: boolean, + terminateTimeout: number, ): Promise { const socket = this.socket; this.abortConnect?.( new TextToSpeechError("The session was closed before it began"), ); if (socket && !this.stopped && socket.readyState === socket.OPEN) { - const terminated = new Promise((resolve) => { + const ended = new Promise((resolve) => { this.terminationResolve = resolve; }); + connection.heardAt = Date.now(); try { socket.send(JSON.stringify({ type: "Terminate" })); } catch { // the socket closed under us; the close handler ends the session } - if (waitForTermination) { - if (terminateTimeout > 0) { - let timer: ReturnType | undefined; - const expired = new Promise((resolve) => { - timer = setTimeout(resolve, terminateTimeout); - }); - await Promise.race([terminated, expired]); - if (timer) clearTimeout(timer); - } else { - await terminated; - } + if ( + waitForTermination && + !(await this.drained(connection, ended, terminateTimeout)) && + this.queue === connection + ) { + this.end( + new TextToSpeechError( + "No Termination within the terminate timeout; the remaining audio was dropped", + { retryable: true, requestId: this.begin?.id }, + ), + ); } } + if (this.queue !== connection) { + // A listener connected again while this close waited; the new + // connection is not this close's to end. + return connection.termination; + } this.end(); this.releaseSocket(); const termination = this.termination; const unobserved = this.error && !this.errorRaised ? this.surfaced() : undefined; // Reported last: the close listener may connect again. - this.emitClose(this.queue); + this.emitClose(connection); if (unobserved) throw unobserved; return termination; } + + /** + * Resolves `true` when `ended` does, or `false` once the API has sent + * nothing for `idleMs` milliseconds; `0` waits indefinitely. + */ + private drained( + connection: PullQueue, + ended: Promise, + idleMs: number, + ): Promise { + const done = ended.then(() => true); + if (!(idleMs > 0)) return done; + let timer: ReturnType | undefined; + const idle = new Promise((resolve) => { + const check = () => { + const quiet = Date.now() - connection.heardAt; + if (quiet >= idleMs) resolve(false); + else timer = setTimeout(check, idleMs - quiet); + }; + timer = setTimeout(check, idleMs); + }); + return Promise.race([done, idle]).finally(() => clearTimeout(timer)); + } } diff --git a/src/types/tts/index.ts b/src/types/tts/index.ts index 8f6dfc0..5d89591 100644 --- a/src/types/tts/index.ts +++ b/src/types/tts/index.ts @@ -177,7 +177,12 @@ export type TextToSpeechSessionParams = { websocketBaseUrl?: string; /** Milliseconds to wait for the handshake and `Begin`. Defaults to 10 000. */ connectTimeout?: number; - /** Milliseconds to wait for `Termination` on `close()`. Defaults to 10 000. */ + /** + * Milliseconds without a message from the API after which `close()` stops + * waiting for `Termination` and drops the audio still to come; `0` waits + * indefinitely. Defaults to 35 000, a little over the API's own 30 s limit + * for a stalled drain. + */ terminateTimeout?: number; }; diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index 29a59ee..b3d8536 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -54,6 +54,8 @@ class FakeApi { text = ""; flushId = 0; answerFlush = true; + /** Off, Terminate gets no answer: the test plays the drain. */ + answerTerminate = true; closeOnTerminate = true; /** Refuse a temporary token seen before, as the API may. */ singleUseTokens = false; @@ -132,6 +134,7 @@ class FakeApi { ); return; case "Terminate": + if (!this.answerTerminate) return; socket.send(JSON.stringify(terminationMessage)); if (this.closeOnTerminate) { socket.close({ code: 1000, reason: "Session ended", wasClean: true }); @@ -178,12 +181,13 @@ async function connection(n: number): Promise { /** What the promise settled with, or "pending" if it did not settle in time. */ function outcome(promise: Promise, ms = 300) { + let timer: ReturnType | undefined; return Promise.race([ promise.catch((error: unknown) => error), - new Promise<"pending">((resolve) => - setTimeout(() => resolve("pending"), ms), - ), - ]); + new Promise<"pending">((resolve) => { + timer = setTimeout(() => resolve("pending"), ms); + }), + ]).finally(() => clearTimeout(timer)); } let consoleError: jest.SpyInstance | undefined; @@ -952,6 +956,129 @@ describe("tts session", () => { expect(closes).toEqual([[1000, "Session ended"]]); }); + it("two close() calls at once send one Terminate and both resolve at Termination", async () => { + api.answerTerminate = false; + const session = client.tts.session({ websocketBaseUrl }); + const closes: [number, string][] = []; + session.on("close", (code, reason) => closes.push([code, reason])); + await connect(session); + const hangUp = session.close(); // e.g. a hang-up handler + const cleanUp = session.close(); // and a finally + await tick(); + expect(api.frames("Terminate")).toHaveLength(1); + expect(await outcome(hangUp, 20)).toBe("pending"); + + api.sockets[0].send(JSON.stringify(terminationMessage)); + api.sockets[0].close({ + code: 1000, + reason: "Session ended", + wasClean: true, + }); + const started = Date.now(); + expect(await outcome(hangUp)).toEqual(terminationMessage); + expect(await outcome(cleanUp)).toEqual(terminationMessage); + expect(Date.now() - started).toBeLessThan(250); // not the terminate timeout + await expect(session.close()).resolves.toEqual(terminationMessage); + expect(api.frames("Terminate")).toHaveLength(1); + expect(closes).toEqual([[1000, "Session ended"]]); + }); + + it("only the first of two close() calls throws an unobserved error", async () => { + api.answerTerminate = false; + const session = client.tts.session({ websocketBaseUrl }); + session.on("audio", () => undefined); // push mode: nothing reads the error + await connect(session); + const first = outcome(session.close()); + const second = outcome(session.close()); + await tick(); + api.sockets[0].send(errorFrame(3005, "at_capacity", true)); + expect(await first).toMatchObject({ errorCode: "at_capacity" }); + expect(await second).toBeUndefined(); + expect(api.frames("Terminate")).toHaveLength(1); + }); + + it("close() keeps waiting while the API sends audio past terminateTimeout", async () => { + api.answerFlush = false; + api.answerTerminate = false; + const session = client.tts.session({ + websocketBaseUrl, + terminateTimeout: 150, + }); + await connect(session); + session.generate("Pending text."); + const flushId = session.flush(); + const reading = outcome(collect(session.audio(flushId)), 2000); + const started = Date.now(); + const closing = outcome(session.close(), 2000); + await tick(); + // 450 ms of drain, a frame every 50 ms: longer than terminateTimeout, + // never that long without a frame. + for (let i = 0; i < 9; i++) { + api.sockets[0].send( + JSON.stringify({ type: "Audio", audio: "QQ==", flush_id: flushId }), + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + api.sockets[0].send( + JSON.stringify({ type: "FlushDone", flush_id: flushId }), + ); + api.sockets[0].send(JSON.stringify(terminationMessage)); + expect(await closing).toEqual(terminationMessage); + expect(Date.now() - started).toBeGreaterThanOrEqual(450); + expect(await reading).toEqual(Buffer.from("A".repeat(9))); + expect(session.error).toBeUndefined(); + }); + + it("a silent drain stops after terminateTimeout and the waiting reader rejects", async () => { + api.answerFlush = false; + api.answerTerminate = false; + const session = client.tts.session({ + websocketBaseUrl, + terminateTimeout: 100, + }); + const closes: [number, string][] = []; + session.on("close", (code, reason) => closes.push([code, reason])); + await connect(session); + session.generate("Never spoken."); + const flushId = session.flush(); + const reading = outcome(collect(session.audio(flushId)), 1000); + await tick(); + const started = Date.now(); + await expect(outcome(session.close(), 1000)).resolves.toBeUndefined(); + expect(Date.now() - started).toBeGreaterThanOrEqual(90); + const error = await reading; + expect(error).toBeInstanceOf(TextToSpeechError); + expect(error).toMatchObject({ + message: + "No Termination within the terminate timeout; the remaining audio was dropped", + retryable: true, + requestId: "sess-1", + }); + expect(session.error).toBeUndefined(); // close() reports no failure + expect(closes).toEqual([[1000, "Closed by the client"]]); + await tick(); + expect(api.closes).toBe(1); // the client closed the socket + await expect(session.close()).resolves.toBeUndefined(); + }); + + it("close() does not end a connection a listener opened while it waited", async () => { + const session = client.tts.session({ websocketBaseUrl }); + let reconnecting: Promise | undefined; + session.on("termination", () => { + reconnecting ??= session.connect(); + }); + await connect(session); + const closing = outcome(session.close()); + (await connection(2)).send(JSON.stringify(beginMessage)); + expect(await closing).toEqual(terminationMessage); + await expect(reconnecting).resolves.toMatchObject({ id: "sess-1" }); + expect((await collect(session.speak("Still open."))).toString()).toBe( + "Still open.", + ); + await expect(session.close()).resolves.toEqual(terminationMessage); + expect(api.frames("Terminate")).toHaveLength(2); + }); + it("connect() reports the previous connection's close before opening a new one", async () => { api.answerFlush = false; const session = client.tts.session({ websocketBaseUrl }); From 7c0c034ef0926d4d9e4eb2aae8afb7f6d7ecce03 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:18:27 +0000 Subject: [PATCH 10/26] fix(tts): refuse sends after close() and while the socket closes, and speak() past the text backlog - Once close() has sent Terminate, generate(), flush(), cancel() and speak() throw "The session is closing: close() was called" instead of sending frames the API never reads; keepAlive() does nothing, so a keepalive timer still running during the drain cannot throw. The flag lives on the connection, so a reconnect starts clear. - A send after Begin while the socket is closing (close frame in, close event not run yet: 1006, 1011, 1001, 1012 without an Error frame) throws a retryable TextToSpeechError "The connection is closing" with the session id, not the plain "call connect() first" Error. - flush() and cancel() send first, so a refused send changes nothing: a refused cancel no longer ends the running audio() iteration. - speak() refuses text over Begin's limits.text_max_buffer_chars (when text_throttle_enabled) with a RangeError before sending anything; the API would end the session with a non-retryable 3010. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- README.md | 6 +- src/services/tts/session.ts | 96 ++++++++++++++---- tests/unit/tts-session.test.ts | 171 +++++++++++++++++++++++++++++++++ 3 files changed, 251 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index cd24ab9..f1a8ad7 100644 --- a/README.md +++ b/README.md @@ -869,7 +869,7 @@ For a barge-in, pass `{ signal }` as the second argument and abort it: the downl ### Speak text as it is written (voice agents) -A `TextToSpeechSession` keeps one WebSocket open and speaks text as it is written, so an LLM's reply can be voiced sentence by sentence with no per-request handshake. `speak()` sends one text and returns its audio; `generate()` / `flush()` do the same in two steps, so tokens can be sent as they arrive; `cancel()` is the barge-in. `close()` ends the session and resolves with its summary once the API has spoken the text still pending. It stops waiting after `terminateTimeout` milliseconds (default 35,000) without a message from the API; an `audio()` iteration still running then rejects with a retryable `TextToSpeechError`, and `close()` resolves `undefined`. Calling `close()` again, or from two places at once, waits for the same end. +A `TextToSpeechSession` keeps one WebSocket open and speaks text as it is written, so an LLM's reply can be voiced sentence by sentence with no per-request handshake. `speak()` sends one text and returns its audio; `generate()` / `flush()` do the same in two steps, so tokens can be sent as they arrive; `cancel()` is the barge-in. `close()` ends the session and resolves with its summary once the API has spoken the text still pending. It stops waiting after `terminateTimeout` milliseconds (default 35,000) without a message from the API; an `audio()` iteration still running then rejects with a retryable `TextToSpeechError`, and `close()` resolves `undefined`. Calling `close()` again, or from two places at once, waits for the same end. Once `close()` has sent `Terminate` the API reads nothing more, so `generate()`, `flush()`, `cancel()` and `speak()` throw instead of sending text that would never be spoken, and `keepAlive()` does nothing. ```typescript const session = client.tts.session({ voiceId: "aai-jane" }); @@ -904,9 +904,9 @@ await player; await session.close(); ``` -`audio()` fixes its request when it is called, so the iterator can be made before the first `generate()` and handed to a player that starts later. Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order, once; `audio()` throws a `RangeError` for an id not flushed yet or already read. Every `flush()` and `cancel()` takes the next id, so ids skip one after a cancel. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters, so pace text to the audio. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes after `inactivityTimeout` seconds (5 to 120, default 120) without a frame; call `keepAlive()` between sparse requests. +`audio()` fixes its request when it is called, so the iterator can be made before the first `generate()` and handed to a player that starts later. Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order, once; `audio()` throws a `RangeError` for an id not flushed yet or already read. Every `flush()` and `cancel()` takes the next id, so ids skip one after a cancel. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters (`limits.text_max_buffer_chars` in `Begin`), ending the session with a non-retryable 3010 past that, so pace text to the audio. `speak()` throws a `RangeError` for a longer text before sending any of it; split it and speak each part once the previous part's audio is read. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes after `inactivityTimeout` seconds (5 to 120, default 120) without a frame; call `keepAlive()` between sparse requests. -Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A listener that throws is logged with `console.error` and the session goes on; a frame the SDK cannot read fails the session with `Unreadable frame from the API` and closes the socket. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send, reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `requestId` is the session id, and `status` is set only when the server answered the WebSocket upgrade with an HTTP status the runtime reports (Node does, a browser does not): a 404 for a wrong `websocketBaseUrl` path is not retryable, a 503 is. `retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. That works as is with an API key; a session opened with a temporary token needs a new token for each connection, passed as `connect({ token })`, because the API may refuse a token it has seen and always refuses an expired one (close code 1008, `errorCode: "unauthorized"`). The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. +Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A listener that throws is logged with `console.error` and the session goes on; a frame the SDK cannot read fails the session with `Unreadable frame from the API` and closes the socket. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send (a send in the moment between a lost connection and its close event throws a retryable `TextToSpeechError` saying the connection is closing), reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `requestId` is the session id, and `status` is set only when the server answered the WebSocket upgrade with an HTTP status the runtime reports (Node does, a browser does not): a 404 for a wrong `websocketBaseUrl` path is not retryable, a 503 is. `retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. That works as is with an API key; a session opened with a temporary token needs a new token for each connection, passed as `connect({ token })`, because the API may refuse a token it has seen and always refuses an expired one (close code 1008, `errorCode: "unauthorized"`). The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. In a browser the API key must stay on your server. Mint a temporary token there, and open the session in the page with `new TextToSpeechSession({ token })`, which needs no client and no API key: diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index 8112de3..05c9aec 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -63,6 +63,8 @@ class PullQueue { heardAt = 0; /** The connection's one `close()`, which every call shares. */ closing?: Promise; + /** `close()` sent `Terminate`; the API reads no frame after it. */ + terminating = false; /** The socket's own close event, when it arrived. */ closedWith?: [code: number, reason: string]; closeEmitted = false; @@ -676,26 +678,59 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c return this.surface(this.error as TextToSpeechError); } + /** `close()` sent `Terminate` and the connection has not ended yet. */ + private get terminating(): boolean { + return this.queue.terminating && !this.stopped; + } + + /** + * @throws Error once `close()` has sent `Terminate`, since the API reads + * no frame after it, and when the session is not connected. + * @throws TextToSpeechError the error that ended the session; or, while + * the socket is closing and its close event has not run yet, a retryable + * one: a close without an `Error` frame, such as 1006 or 1011. + */ private send(frame: Record) { - if ( - !this.socket || - this.stopped || - this.socket.readyState !== this.socket.OPEN - ) { - if (this.error) throw this.surfaced(); - throw new Error("The session is not open: call connect() first"); + const socket = this.socket; + if (this.terminating) { + throw new Error("The session is closing: close() was called"); + } + if (socket && !this.stopped && socket.readyState === socket.OPEN) { + socket.send(JSON.stringify(frame)); + return; + } + if (this.error) throw this.surfaced(); + if (socket && this.begin && !this.stopped) { + throw new TextToSpeechError("The connection is closing", { + retryable: true, + requestId: this.begin.id, + }); } - this.socket.send(JSON.stringify(frame)); + throw new Error("The session is not open: call connect() first"); } - private generateLimit(): number { - const limits = this.begin?.configuration?.limits as + private limits(): Record | undefined { + return this.begin?.configuration?.limits as | Record | undefined; - const limit = limits?.max_generate_text_length; + } + + private generateLimit(): number { + const limit = this.limits()?.max_generate_text_length; return typeof limit === "number" && limit >= 0 ? limit : MAX_GENERATE_CHARS; } + /** + * The unspoken characters the API holds before it ends the session with + * 3010, as `Begin` reports it; `0` for no limit. + */ + private backlogLimit(): number { + const limits = this.limits(); + if (limits?.text_throttle_enabled === false) return 0; + const limit = limits?.text_max_buffer_chars; + return typeof limit === "number" && limit > 0 ? limit : 0; + } + private checkPullable() { if (this.listeners.audio) { throw new Error( @@ -716,6 +751,7 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c * Audio may start before `flush()` once enough complete sentences have * arrived; `flush()` is what concludes the request. * @throws RangeError for a text over one frame's limit (2,000 characters); split it. + * @throws Error once `close()` was called: the API would not read the text. */ generate(text: string) { if (typeof text !== "string") throw new TypeError("text must be a string"); @@ -736,9 +772,8 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c * Flushes can be pipelined; the audio comes in order. */ flush(): number { - const flushId = this.nextFlushId++; this.send({ type: "Flush" }); - return flushId; + return this.nextFlushId++; } /** @@ -747,30 +782,52 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c * An `audio()` iteration of a cancelled request, or of the text being * written, ends at once; audio of it still in transit is discarded. Like * `flush()`, a cancel takes a `flush_id`, so the next request's id skips one. + * A cancel the session refuses (after `close()`, or once the connection + * is lost) changes nothing: the iterations go on to their own end. */ cancel() { + this.send({ type: "Cancel" }); const cancelId = this.nextFlushId++; this.cancelIds.push(cancelId); this.queue.cancelledBefore = Math.max(this.queue.cancelledBefore, cancelId); - this.send({ type: "Cancel" }); } - /** Resets the inactivity timer between sparse requests. */ + /** + * Resets the inactivity timer between sparse requests. Once `close()` was + * called it does nothing, so a keepalive timer still running while the + * session drains does not throw. + */ keepAlive() { + if (this.terminating) return; this.send({ type: "KeepAlive" }); } /** * Speaks one text: `generate()`, `flush()`, then its audio. * - * Text longer than one frame is split across frames. Returns an async - * iterable of the request's audio that ends at its `FlushDone`, or early - * if the request is cancelled. + * Text longer than one frame is split across frames, sent at once. The + * API holds a limited backlog of text it has not spoken yet (`Begin`'s + * `limits.text_max_buffer_chars`, 8,000 characters by default) and ends + * the session with a non-retryable 3010 when a burst passes it, so longer + * text is refused before anything is sent. The backlog counts all the + * session's unspoken text, so this check prevents a sure 3010 without + * guaranteeing success: split long text and `speak()` each part once the + * previous part's audio is read, or pace it with `generate()`. Returns an + * async iterable of the request's audio that ends at its `FlushDone`, or + * early if the request is cancelled. + * @throws RangeError for text over the backlog limit. */ speak(text: string): AsyncIterable { if (typeof text !== "string") throw new TypeError("text must be a string"); this.checkPullable(); - const limit = this.generateLimit() || Math.max(codePointLength(text), 1); + const length = codePointLength(text); + const backlog = this.backlogLimit(); + if (backlog && length > backlog) { + throw new RangeError( + `text is ${length} characters, over the ${backlog} unspoken characters the API holds before it ends the session with 3010 input_rate_exceeded; split it and speak() each part once the previous part's audio is read, or pace it with generate()`, + ); + } + const limit = this.generateLimit() || Math.max(length, 1); for (const piece of splitCodePoints(text, limit)) this.generate(piece); return this.audio(this.flush()); } @@ -914,6 +971,7 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c this.terminationResolve = resolve; }); connection.heardAt = Date.now(); + connection.terminating = true; try { socket.send(JSON.stringify({ type: "Terminate" })); } catch { diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index b3d8536..adda033 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -546,6 +546,23 @@ describe("tts session", () => { expect(() => session.generate("Hi.")).toThrow(/connect\(\)/); }); + it("a send while the socket is still opening is not reported as a lost connection", async () => { + const session = client.tts.session({ websocketBaseUrl }); + const connecting = session.connect(); + let thrown: unknown; + try { + session.generate("Hi."); + } catch (error) { + thrown = error; + } + expect(thrown).not.toBeInstanceOf(TextToSpeechError); + expect(thrown).toMatchObject({ message: /connect\(\)/ }); + await server.connected; + server.send(JSON.stringify(beginMessage)); + await connecting; + await session.close(); + }); + it("cancel ends a reader following the request being written", async () => { api.answerFlush = false; const session = client.tts.session({ websocketBaseUrl }); @@ -599,6 +616,69 @@ describe("tts session", () => { await session.close(); }); + describe("speak() and the API's text backlog", () => { + const withLimits = (limits: Record) => ({ + ...beginMessage, + configuration: { + ...beginMessage.configuration, + limits: { max_generate_text_length: 2000, ...limits }, + }, + }); + + it("refuses text over the backlog before sending anything", async () => { + // Given a session whose Begin reports an 8,000-character backlog + const session = client.tts.session({ websocketBaseUrl }); + await connect( + session, + withLimits({ + text_throttle_enabled: true, + text_max_buffer_chars: 8000, + }), + ); + + // When the app speaks 8,001 characters at once + const speaking = () => session.speak("a".repeat(8001)); + + // Then speak() throws a RangeError naming the limit, and sends nothing + expect(speaking).toThrow(RangeError); + expect(speaking).toThrow( + /^text is 8001 characters, over the 8000 unspoken characters .* 3010 input_rate_exceeded; split it and speak\(\) each part/, + ); + await tick(); + expect(api.sent).toEqual([]); + + // And text at the limit, counted in code points as the API counts it, + // is spoken: 8,000 emoji are 16,000 UTF-16 code units + const text = "\u{1F600}".repeat(8000); + expect((await collect(session.speak(text))).toString()).toBe(text); + expect(api.frames("Generate")).toHaveLength(4); + await session.close(); + }); + + it.each([ + [ + "the throttle is off", + { text_throttle_enabled: false, text_max_buffer_chars: 8000 }, + ], + ["Begin reports no backlog", {}], + [ + "the backlog is 0", + { text_throttle_enabled: true, text_max_buffer_chars: 0 }, + ], + ])("sends long text when %s", async (_case, limits) => { + // Given a session whose API sets no text backlog + const session = client.tts.session({ websocketBaseUrl }); + await connect(session, withLimits(limits)); + + // When the app speaks 8,001 characters + const text = "a".repeat(8001); + + // Then the text is sent and spoken + expect((await collect(session.speak(text))).toString()).toBe(text); + await session.close(); + }); + }); + it("close() rejects a connect() still pending instead of letting it time out", async () => { const session = client.tts.session({ websocketBaseUrl, @@ -1079,6 +1159,97 @@ describe("tts session", () => { expect(api.frames("Terminate")).toHaveLength(2); }); + it("sends after close() sent Terminate throw, and the drain still reaches its reader", async () => { + // Given a session whose close() waits for the API to speak pending text + api.answerTerminate = false; + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + session.generate("Pending text."); + const reading = outcome(collect(session.audio()), 1000); // Terminate flushes it + const closing = outcome(session.close(), 1000); + + // When the app goes on sending, as an LLM token loop still running would + const closed = "The session is closing: close() was called"; + + // Then text, flushes and cancels throw instead of being dropped unread, + // a keepalive does nothing, and the API sees nothing after Terminate + expect(() => session.generate("Too late.")).toThrow(closed); + expect(() => session.flush()).toThrow(closed); + expect(() => session.cancel()).toThrow(closed); + expect(() => session.keepAlive()).not.toThrow(); + await tick(); + expect(api.sent.map((frame) => frame.type)).toEqual([ + "Generate", + "Terminate", + ]); + + // And the refused cancel did not end the reader of the drained request + for (const audio of ["Pending ", "text."]) { + api.sockets[0].send( + JSON.stringify({ + type: "Audio", + audio: Buffer.from(audio).toString("base64"), + flush_id: 0, + }), + ); + await tick(); + } + api.sockets[0].send(JSON.stringify({ type: "FlushDone", flush_id: 0 })); + api.sockets[0].send(JSON.stringify(terminationMessage)); + expect(await closing).toEqual(terminationMessage); + expect(await reading).toEqual(Buffer.from("Pending text.")); + + // And once the session ended, a send says to connect again + expect(() => session.generate("Later.")).toThrow(/call connect\(\) first/); + }); + + it("a send while the socket closes without an Error frame throws a retryable error", async () => { + // Given a connection whose close frame (1011, no Error frame) arrived + // and whose close event has not run yet, as with ws between the two + const session = client.tts.session({ websocketBaseUrl }); + const errors: TextToSpeechError[] = []; + session.on("error", (error) => errors.push(error)); + await connect(session); + const socket = (session as unknown as { socket: { readyState: number } }) + .socket; + socket.readyState = 2; // CLOSING + + // When the app sends + const sends = [ + () => session.generate("Hello."), + () => session.flush(), + () => session.cancel(), + () => session.keepAlive(), + ]; + + // Then each send throws a retryable TextToSpeechError with the session id + for (const send of sends) { + let thrown: unknown; + try { + send(); + } catch (error) { + thrown = error; + } + expect(thrown).toBeInstanceOf(TextToSpeechError); + expect(thrown).toMatchObject({ + message: "The connection is closing", + retryable: true, + requestId: "sess-1", + }); + } + + // And the close event that follows reports the close itself + api.sockets[0].close({ + code: 1011, + reason: "keepalive ping timeout", + wasClean: false, + }); + await tick(); + expect(errors).toHaveLength(1); + expect(errors[0]).toMatchObject({ closeCode: 1011, retryable: true }); + expect(() => session.generate("Hello.")).toThrow(errors[0]); + }); + it("connect() reports the previous connection's close before opening a new one", async () => { api.answerFlush = false; const session = client.tts.session({ websocketBaseUrl }); From 85fe8aaf88158c2dd2c32feab15992a99e1533cd Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:27:22 +0000 Subject: [PATCH 11/26] fix(tts): drop audio of cancelled requests still in transit and return the cancel's id The API can send audio of a request after the client's Cancel and before its Cancelled, and the API reference says to drop it. The session passed it on: an audio listener kept playing the interrupted reply, and an audio() reader waiting for its next chunk yielded the first stale frame. - dispatch() drops Audio whose flush_id is at most the last cancel's id, before the audio listener and before the pull queue. - cancel() wakes a reader waiting for its next chunk, so it ends at once instead of at Cancelled, and a reader handed a frame just before cancel() checks again before yielding it. - cancel() returns the flush_id it takes, as flush() does. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- README.md | 2 +- src/services/tts/session.ts | 44 ++++++++--- tests/unit/tts-session.test.ts | 130 ++++++++++++++++++++++++++++++++- 3 files changed, 163 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index f1a8ad7..a7933d8 100644 --- a/README.md +++ b/README.md @@ -904,7 +904,7 @@ await player; await session.close(); ``` -`audio()` fixes its request when it is called, so the iterator can be made before the first `generate()` and handed to a player that starts later. Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order, once; `audio()` throws a `RangeError` for an id not flushed yet or already read. Every `flush()` and `cancel()` takes the next id, so ids skip one after a cancel. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters (`limits.text_max_buffer_chars` in `Begin`), ending the session with a non-retryable 3010 past that, so pace text to the audio. `speak()` throws a `RangeError` for a longer text before sending any of it; split it and speak each part once the previous part's audio is read. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes after `inactivityTimeout` seconds (5 to 120, default 120) without a frame; call `keepAlive()` between sparse requests. +`audio()` fixes its request when it is called, so the iterator can be made before the first `generate()` and handed to a player that starts later. Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order, once; `audio()` throws a `RangeError` for an id not flushed yet or already read. Every `flush()` and `cancel()` takes the next id and returns it, so ids skip one after a cancel, and the requests a cancel retires are those up to its id. Audio of a retired request that the API sent before it read the `Cancel` can still arrive; the session drops it, so neither `audio()` nor the `audio` listener keeps playing the interrupted reply. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters (`limits.text_max_buffer_chars` in `Begin`), ending the session with a non-retryable 3010 past that, so pace text to the audio. `speak()` throws a `RangeError` for a longer text before sending any of it; split it and speak each part once the previous part's audio is read. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes after `inactivityTimeout` seconds (5 to 120, default 120) without a frame; call `keepAlive()` between sparse requests. Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A listener that throws is logged with `console.error` and the session goes on; a frame the SDK cannot read fails the session with `Unreadable frame from the API` and closes the socket. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send (a send in the moment between a lost connection and its close event throws a retryable `TextToSpeechError` saying the connection is closing), reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `requestId` is the session id, and `status` is set only when the server answered the WebSocket upgrade with an HTTP status the runtime reports (Node does, a browser does not): a 404 for a wrong `websocketBaseUrl` path is not retryable, a 503 is. `retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. That works as is with an API key; a session opened with a temporary token needs a new token for each connection, passed as `connect({ token })`, because the API may refuse a token it has seen and always refuses an expired one (close code 1008, `errorCode: "unauthorized"`). The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index 05c9aec..570622e 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -39,7 +39,9 @@ type PullItem = | { kind: "audio"; flushId: number; bytes: Uint8Array } | { kind: "done"; flushId: number } | { kind: "cancelled"; cancelId: number } - | { kind: "closed"; error?: TextToSpeechError }; + | { kind: "closed"; error?: TextToSpeechError } + /** Handed only to a waiting reader, never queued: check the state again. */ + | { kind: "wake" }; /** * One connection: a FIFO that hands items to one waiting reader, and how the @@ -113,6 +115,14 @@ class PullQueue { this.items.push(item); } + /** Hands the waiting reader, if any, a `wake` item. */ + wake() { + const resolve = this.waiter; + if (!resolve) return; + this.waiter = undefined; + resolve({ kind: "wake" }); + } + next(): Promise { const item = this.items.shift(); if (item) { @@ -556,6 +566,9 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c private dispatch(event: SessionFrame) { switch (event.type) { case "Audio": + // Audio of a request a Cancel retired, which the API sent before it + // read the Cancel. + if (event.flush_id <= this.queue.cancelledBefore) return; if (this.listeners.audio) this.emit("audio", event); else { this.enqueue({ @@ -777,19 +790,27 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c } /** - * Barge-in: stops the request in flight and drops queued requests. + * Barge-in: stops the request in flight, drops queued requests and the + * text being written, and returns the `flush_id` the cancel takes. * - * An `audio()` iteration of a cancelled request, or of the text being - * written, ends at once; audio of it still in transit is discarded. Like - * `flush()`, a cancel takes a `flush_id`, so the next request's id skips one. - * A cancel the session refuses (after `close()`, or once the connection - * is lost) changes nothing: the iterations go on to their own end. + * Like `flush()`, a cancel takes the id of the text being written, so the + * cancelled requests are those with an id up to the one returned, and the + * next request's id is one more. An `audio()` iteration of a cancelled + * request ends at once, even one waiting for its next chunk. Audio of a + * cancelled request that the API sent before it read the `Cancel` can + * still arrive; neither an `audio()` iteration nor the `audio` listener + * receives it. A cancel the session refuses (after `close()`, or once the + * connection is lost) changes nothing: the iterations go on to their own end. */ - cancel() { + cancel(): number { this.send({ type: "Cancel" }); const cancelId = this.nextFlushId++; this.cancelIds.push(cancelId); this.queue.cancelledBefore = Math.max(this.queue.cancelledBefore, cancelId); + // A reader waiting for a frame reads a request this cancel retired (no + // reader's id is past the cancel's), so it ends now. + this.queue.wake(); + return cancelId; } /** @@ -896,7 +917,10 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c const item = await connection.next(); switch (item.kind) { case "audio": - if (item.flushId === flushId) yield item.bytes; + if (item.flushId !== flushId) break; + // cancel() may have run since the frame was handed over. + if (flushId <= connection.cancelledBefore) return; + yield item.bytes; break; case "done": // Counted when read, so a FlushDone still queued for a pipelined @@ -913,6 +937,8 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c case "closed": if (item.error) throw this.surface(item.error); return; + case "wake": + break; } } } finally { diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index adda033..7619fa7 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -54,6 +54,8 @@ class FakeApi { text = ""; flushId = 0; answerFlush = true; + /** Off, Cancel gets no Cancelled: the test plays the audio still in transit. */ + answerCancel = true; /** Off, Terminate gets no answer: the test plays the drain. */ answerTerminate = true; closeOnTerminate = true; @@ -129,9 +131,11 @@ class FakeApi { case "Cancel": this.flushId += 1; this.text = ""; - socket.send( - JSON.stringify({ type: "Cancelled", audio_duration_seconds: 0.1 }), - ); + if (this.answerCancel) { + socket.send( + JSON.stringify({ type: "Cancelled", audio_duration_seconds: 0.1 }), + ); + } return; case "Terminate": if (!this.answerTerminate) return; @@ -1410,6 +1414,126 @@ describe("tts session", () => { expect(queue.size).toBe(0); }); + describe("cancel() and audio still in transit", () => { + const audioFrame = (text: string, flushId: number) => + JSON.stringify({ + type: "Audio", + audio: Buffer.from(text).toString("base64"), + flush_id: flushId, + }); + const cancelledFrame = JSON.stringify({ + type: "Cancelled", + audio_duration_seconds: 0.4, + }); + const textOf = (chunk: IteratorResult) => + Buffer.from(chunk.value as Uint8Array).toString(); + + beforeEach(() => { + // The test sends the Cancelled, after the audio the API had already sent. + api.answerCancel = false; + }); + + it("cancel() returns the id it takes, and the next flush() the one after", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + expect(session.cancel()).toBe(0); + expect(session.flush()).toBe(1); + expect(session.cancel()).toBe(2); + await session.close(); + }); + + it("an audio listener gets no audio of a cancelled request still in transit", async () => { + // Given an audio listener playing the request being written + const session = client.tts.session({ websocketBaseUrl }); + const heard: string[] = []; + session.on("audio", (event) => + heard.push(`${event.flush_id}:${Buffer.from(event.audio).toString()}`), + ); + const cancelled = jest.fn(); + session.on("cancelled", cancelled); + await connect(session); + session.generate("A reply the user talks over. "); + server.send(audioFrame("first", 0)); + expect(heard).toEqual(["0:first"]); + + // When the user barges in, and audio the API sent before it read the + // Cancel arrives ahead of Cancelled + expect(session.cancel()).toBe(0); + for (let i = 0; i < 5; i++) server.send(audioFrame("stale", 0)); + server.send(cancelledFrame); + await tick(); + + // Then none of it reaches the listener + expect(cancelled).toHaveBeenCalledTimes(1); + expect(heard).toEqual(["0:first"]); + + // And the next request, which takes the id after the cancel, does + session.generate("Sure, go ahead."); + expect(session.flush()).toBe(1); + await tick(); + expect(heard).toEqual(["0:first", "1:Sure, go ahead."]); + await session.close(); + }); + + it("a reader waiting when cancel() is called ends at once and yields nothing in transit", async () => { + // Given a reader of request 0 waiting for its next frame + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + session.generate("A reply the user talks over. "); + const reader = session.audio(session.flush()); + server.send(audioFrame("first", 0)); + expect(textOf(await reader.next())).toBe("first"); + const waiting = reader.next(); + await tick(); + + // When the user barges in + expect(session.cancel()).toBe(1); + + // Then the reader ends without waiting for Cancelled + expect(await outcome(waiting, 100)).toEqual({ + value: undefined, + done: true, + }); + + // And the audio still in transit is dropped, not queued: only the + // Cancelled marker is + for (let i = 0; i < 5; i++) server.send(audioFrame("stale", 0)); + server.send(cancelledFrame); + await tick(); + const queue = (session as unknown as { queue: { size: number } }).queue; + expect(queue.size).toBe(1); + + // And the next request reads only its own audio + api.answerFlush = true; + expect((await collect(session.speak("Next."))).toString()).toBe("Next."); + await session.close(); + }); + + it("a reader yields no frame that reached it just before cancel()", async () => { + // Given a reader of the text being written, waiting for its first frame + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const reader = session.audio(); + session.generate("A reply the user talks over. "); + const waiting = reader.next(); + await tick(); + + // When a frame is handed to it and the app cancels before the reader + // runs again + server.send(audioFrame("stale", 0)); + expect(session.cancel()).toBe(0); + + // Then the reader ends instead of yielding the frame + expect(await outcome(waiting, 100)).toEqual({ + value: undefined, + done: true, + }); + server.send(cancelledFrame); + await session.close(); + }); + }); + describe("connect() failures before the socket opens", () => { type FakeSocket = { OPEN: number; From 6ae3ef7c216c1be97abfb3289789f042ee7a7e2d Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:41:33 +0000 Subject: [PATCH 12/26] fix(tts): give up a stream() body that stalls with a read timeout A streamed body that stops sending without a reset (a dropped NAT mapping, a hung proxy) was noticed only through undici's 300 s bodyTimeout on Node, and never in a browser. stream() now takes `readTimeout` (30 000 ms by default, as in the Python SDK and the public docs; 0 turns it off): each read of the body waits at most that long, then the body is cancelled, which releases the connection, and the read rejects with the retryable cut TextToSpeechError (status 200, the request id, and the wait in the message). The timer runs only while a read waits, so playback at its own pace never trips it, and it covers a caller's own reader of `body` too. The `timeout` docs now say it covers the wait for the headers and first audio. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- CLAUDE.md | 2 +- README.md | 2 +- src/services/tts/service.ts | 66 +++++++++++++-- src/types/tts/index.ts | 13 ++- tests/unit/tts.test.ts | 163 +++++++++++++++++++++++++++++++++++- 5 files changed, 233 insertions(+), 13 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 90f30d9..761b9ae 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -451,7 +451,7 @@ session.cancel(); // barge-in: ends the current audio() iteration at once await session.close(); // Terminate -> Termination summary; rejects with an unobserved session error ``` -Gotchas: a session streams raw PCM/G.711 only (no `mp3`, `container` or `bit_rate`); `generate()` takes at most 2,000 characters per frame (`speak()` splits); the API paces text at ~100 chars/s; an `audio` listener switches audio from pull to push; a session the API ended (idle after 120 s) can `connect()` again on the same object, but a session opened with a temporary token needs a new token for each connection: `connect({ token: newToken })` (a used or expired token is refused with 1008 `unauthorized`). In a browser use `new TextToSpeechSession({ token })` (no client, no API key); `client.tts.createTemporaryToken()` mints on the token host of the `ttsBaseUrl` zone (`token.eu`/`token.us`), `tokenBaseUrl` overrides. +Gotchas: `stream()` gives up a body that sends no audio for `readTimeout` ms (30,000 by default, `0` off) while a read waits, with the same retryable `TextToSpeechError` as a cut; a session streams raw PCM/G.711 only (no `mp3`, `container` or `bit_rate`); `generate()` takes at most 2,000 characters per frame (`speak()` splits); the API paces text at ~100 chars/s; an `audio` listener switches audio from pull to push; a session the API ended (idle after 120 s) can `connect()` again on the same object, but a session opened with a temporary token needs a new token for each connection: `connect({ token: newToken })` (a used or expired token is refused with 1008 `unauthorized`). In a browser use `new TextToSpeechSession({ token })` (no client, no API key); `client.tts.createTemporaryToken()` mints on the token host of the `ttsBaseUrl` zone (`token.eu`/`token.us`), `tokenBaseUrl` overrides. ## Important gotchas diff --git a/README.md b/README.md index a7933d8..f986a4d 100644 --- a/README.md +++ b/README.md @@ -863,7 +863,7 @@ for await (const chunk of speech) { } ``` -`stream()` resolves once the API has sent the first audio. Chunk sizes vary and are not aligned to samples or frames: for 16-bit PCM carry an odd trailing byte over to the next chunk, and re-frame for a fixed-size sink (20 ms of 8 kHz mu-law is 160 bytes). The API commits to `200` with the first audio, so a failure after that cuts the body: the loop then throws a `TextToSpeechError` with `retryable: true`, and the whole request should be retried. `speech.body` is the underlying web `ReadableStream`. +`stream()` resolves once the API has sent the first audio. Chunk sizes vary and are not aligned to samples or frames: for 16-bit PCM carry an odd trailing byte over to the next chunk, and re-frame for a fixed-size sink (20 ms of 8 kHz mu-law is 160 bytes). The API commits to `200` with the first audio, so a failure after that cuts the body: the loop then throws a `TextToSpeechError` with `retryable: true`, and the whole request should be retried. A body that sends no audio for 30 seconds while the loop waits for it ends the same way, and its connection is released, so a stalled connection (a dropped NAT mapping, a hung proxy) cannot hang the loop. Set the limit in milliseconds with the `readTimeout` option (`0` turns it off); only the wait counts, so playing a chunk at its own pace never trips it. `speech.body` is the underlying web `ReadableStream`, and a read of it waits under the same limit. For a barge-in, pass `{ signal }` as the second argument and abort it: the download stops and the loop throws the signal's reason (an `AbortError`), not a `TextToSpeechError`. `speech.cancel()` stops it too, and the loop just ends. One signal can serve every request of a call. diff --git a/src/services/tts/service.ts b/src/services/tts/service.ts index c13619a..8473b8a 100644 --- a/src/services/tts/service.ts +++ b/src/services/tts/service.ts @@ -40,6 +40,38 @@ function requestDeadline(timeoutMs: number, external?: AbortSignal) { }; } +/** The longest delay `setTimeout` keeps; a longer one fires at once. */ +const MAX_TIMER_MS = 2_147_483_647; + +/** + * `reader.read()`, given up after `timeoutMs` without a chunk: the reader is + * then cancelled, which releases the connection, and the read rejects with + * `stalled()`. The timer runs only while this read waits. A `timeoutMs` of + * `0` (or `Infinity`) waits as long as the body does. + */ +async function readWithin( + reader: ReadableStreamDefaultReader, + timeoutMs: number, + stalled: () => Error, +): Promise> { + if (!(timeoutMs > 0 && Number.isFinite(timeoutMs))) return reader.read(); + let timer: ReturnType | undefined; + const expired = new Promise<"expired">((resolve) => { + timer = setTimeout( + () => resolve("expired"), + Math.min(timeoutMs, MAX_TIMER_MS), + ); + }); + try { + const result = await Promise.race([reader.read(), expired]); + if (result !== "expired") return result; + } finally { + clearTimeout(timer); + } + await reader.cancel().catch(() => undefined); + throw stalled(); +} + const ENDPOINT_SPEECH = "/v1/speech"; const DEFAULT_TOKEN_BASE_URL = "https://token.assemblyai.com"; /** A zone's TTS host (`tts.eu.assemblyai.com`); its zone is the first group. */ @@ -63,8 +95,10 @@ function zoneTokenBaseUrl(ttsBaseUrl?: string): string { return zone ? `https://token.${zone}.assemblyai.com` : DEFAULT_TOKEN_BASE_URL; } -/** Each socket operation of a streamed request. */ +/** A streamed request's wait for the headers and first audio. */ const DEFAULT_STREAM_TIMEOUT_MS = 30_000; +/** How long one read of a streamed body may wait for audio. */ +const DEFAULT_READ_TIMEOUT_MS = 30_000; /** A buffered request waits for the whole audio; the API caps a request at 600 s. */ const DEFAULT_SYNTHESIZE_TIMEOUT_MS = 610_000; @@ -167,9 +201,12 @@ export function speechResponseInfo( * Iterate it (`for await (const chunk of speech)`) or read `body`, a web * `ReadableStream`. Chunk sizes vary and are not aligned to samples. A * failure after the first audio cuts the body: the iteration then throws a - * `TextToSpeechError` with `retryable: true`. Aborting the request's - * `signal` stops the download, and the iteration throws the signal's reason - * (an `AbortError` unless you gave another) instead. + * `TextToSpeechError` with `retryable: true`. So does a body that sends no + * audio for the request's `readTimeout` while a read waits for it; a read + * of `body` with your own reader then rejects with that error too. + * Aborting the request's `signal` stops the download, and the iteration + * throws the signal's reason (an `AbortError` unless you gave another) + * instead. */ export class TextToSpeechStream implements AsyncIterable { /** The reader of the running iteration, which `cancel()` goes through. */ @@ -198,9 +235,11 @@ export class TextToSpeechStream implements AsyncIterable { let result: ReadableStreamReadResult; try { result = await reader.read(); - } catch { + } catch (error) { if (signal?.aborted) throw signal.reason; finished = true; + // A stalled read rejects with its own cut error, which names the wait. + if (error instanceof TextToSpeechError) throw error; throw new TextToSpeechError( "The response was cut before the audio was complete. Retry the request.", { status: 200, requestId: this.info.requestId, retryable: true }, @@ -316,8 +355,11 @@ export class TextToSpeechService extends BaseService { * Speaks `request.text` and returns the audio as it is generated. * * Resolves once the API has sent the first audio. The API commits to `200` - * then, so a later failure cuts the body and the iteration throws. - * `options.signal` keeps covering the body: aborting it stops the download. + * then, so a later failure cuts the body and the iteration throws. So does + * a body that sends nothing for `options.readTimeout` (30 000 ms by + * default) while the iteration waits for audio; the connection is then + * released. `options.signal` keeps covering the body: aborting it stops + * the download. * @throws TextToSpeechError if the API refuses the request before any audio. * A network failure or a timeout rejects with the runtime's own error * instead: a `TypeError`, a `DOMException` named `TimeoutError` once @@ -373,12 +415,20 @@ export class TextToSpeechService extends BaseService { // The request stays tied to `options.signal` until the body ends, however // it ends, also when the caller reads `body` with a reader of its own. const reader = response.body.getReader(); + // `pull` runs only while a read of `body` waits (`highWaterMark: 0`), so + // the read timeout counts that wait and never the caller's playback. + const readTimeout = options.readTimeout ?? DEFAULT_READ_TIMEOUT_MS; + const stalled = () => + new TextToSpeechError( + `The response was cut before the audio was complete: no audio for ${readTimeout} ms. Retry the request.`, + { status: 200, requestId: info.requestId, retryable: true }, + ); const body = new ReadableStream( { async pull(controller) { let result: ReadableStreamReadResult; try { - result = await reader.read(); + result = await readWithin(reader, readTimeout, stalled); } catch (error) { deadline.release(); throw error; diff --git a/src/types/tts/index.ts b/src/types/tts/index.ts index 5d89591..979e730 100644 --- a/src/types/tts/index.ts +++ b/src/types/tts/index.ts @@ -81,8 +81,8 @@ export type SpeechRequestOptions = { * `TimeoutError`. For `synthesize()` the whole request, 610 000 by default * (the response waits for the complete audio; the API caps a request at * 600 seconds). For `stream()` the wait for the headers and first audio, - * 30 000 by default; reading the body is not timed, so playback at its own - * pace never trips it. + * 30 000 by default; each later read of the body is bounded by + * `readTimeout` instead. * * On Node, the built-in `fetch` also gives up after 300 seconds without * response headers (undici's `headersTimeout`), so a `synthesize()` longer @@ -92,6 +92,15 @@ export type SpeechRequestOptions = { * `undici` package. */ timeout?: number; + /** + * For `stream()`: milliseconds a read of the body may wait for audio, + * 30 000 by default; `0` turns it off. Only the time a read spends waiting + * counts, so playback at its own pace never trips it. A body that stalls + * without being reset (a dropped NAT mapping, a hung proxy) then ends with + * the same retryable `TextToSpeechError` as a cut, and the connection is + * released. + */ + readTimeout?: number; /** * An external abort signal, combined with the timeout. Aborting it rejects * the request with the signal's reason; for `stream()` it also stops the diff --git a/tests/unit/tts.test.ts b/tests/unit/tts.test.ts index eef5142..7c6f92a 100644 --- a/tests/unit/tts.test.ts +++ b/tests/unit/tts.test.ts @@ -457,6 +457,163 @@ describe("tts", () => { }, ); + /** + * A web stream that sends one chunk and then nothing, as a body whose + * connection stalls without a reset; `cancelled` records its cancel. + */ + const stallingSource = () => { + const cancelled = jest.fn(); + let sent = false; + const source = new ReadableStream( + { + pull(c) { + if (sent) return new Promise(() => undefined); + sent = true; + c.enqueue(new Uint8Array([1, 2, 3, 4])); + }, + cancel: cancelled, + }, + { highWaterMark: 0 }, + ); + return { source, cancelled }; + }; + + it.each([ + ["the default 30 s", {}, 30_000], + ["its readTimeout", { readTimeout: 5_000 }, 5_000], + ])( + "stream gives up a body that stalls after the first audio after %s, as a retryable cut, and releases the connection", + async (_name, options, limit) => { + jest.useFakeTimers(); + try { + const { source, cancelled } = stallingSource(); + mockStreamedBody(source); + const speech = await client.tts.stream({ text: "Hi." }, options); + const chunks: Uint8Array[] = []; + let error: unknown; + const loop = (async () => { + for await (const chunk of speech) chunks.push(chunk); + })().catch((e) => { + error = e; + }); + await jest.advanceTimersByTimeAsync(limit - 1); + expect(chunks).toHaveLength(1); + expect(error).toBeUndefined(); + expect(cancelled).not.toHaveBeenCalled(); + await jest.advanceTimersByTimeAsync(1); + await loop; + expect(error).toBeInstanceOf(TextToSpeechError); + expect(error).toMatchObject({ + status: 200, + retryable: true, + requestId: "req-1", + }); + expect((error as Error).message).toContain(`no audio for ${limit} ms`); + expect(cancelled).toHaveBeenCalledTimes(1); // the connection is released + expect(jest.getTimerCount()).toBe(0); + } finally { + jest.useRealTimers(); + } + }, + ); + + it("stream bounds a read of body with the caller's own reader too", async () => { + jest.useFakeTimers(); + try { + const { source, cancelled } = stallingSource(); + mockStreamedBody(source); + const speech = await client.tts.stream( + { text: "Hi." }, + { readTimeout: 1_000 }, + ); + const reader = speech.body.getReader(); + expect((await reader.read()).value).toEqual(new Uint8Array([1, 2, 3, 4])); + const next = reader.read().catch((e) => e); + await jest.advanceTimersByTimeAsync(1_000); + expect(await next).toMatchObject({ + name: "TextToSpeechError", + status: 200, + retryable: true, + }); + expect(cancelled).toHaveBeenCalledTimes(1); + } finally { + jest.useRealTimers(); + } + }); + + it("stream never times a caller that plays each chunk at its own pace", async () => { + jest.useFakeTimers(); + try { + let pulls = 0; + const source = new ReadableStream( + { + pull(c) { + pulls++; + c.enqueue(audio.slice((pulls - 1) * 320, pulls * 320)); + if (pulls === 3) c.close(); + }, + }, + { highWaterMark: 0 }, + ); + mockStreamedBody(source); + const speech = await client.tts.stream( + { text: "Hi." }, + { readTimeout: 1_000 }, + ); + const chunks: Uint8Array[] = []; + for await (const chunk of speech) { + chunks.push(chunk); + await jest.advanceTimersByTimeAsync(60_000); // playing the chunk takes a minute + } + expect(Buffer.concat(chunks)).toEqual(Buffer.from(audio)); + expect(jest.getTimerCount()).toBe(0); // no read timer outlives its read + } finally { + jest.useRealTimers(); + } + }); + + it.each([ + [0, 0], + [Infinity, 0], + [Number.MAX_SAFE_INTEGER, 1], // past setTimeout's range, where a timer fires at once + ])( + "stream with readTimeout %p waits for a stalled body until cancelled", + async (readTimeout, timers) => { + jest.useFakeTimers(); + try { + const { source, cancelled } = stallingSource(); + mockStreamedBody(source); + const speech = await client.tts.stream( + { text: "Hi." }, + { readTimeout }, + ); + const chunks: Uint8Array[] = []; + let settled = false; + let error: unknown; + const loop = (async () => { + for await (const chunk of speech) chunks.push(chunk); + })().then( + () => (settled = true), + (e) => { + error = e; + settled = true; + }, + ); + await jest.advanceTimersByTimeAsync(3_600_000); + expect(chunks).toHaveLength(1); + expect(settled).toBe(false); + expect(jest.getTimerCount()).toBe(timers); + expect(cancelled).not.toHaveBeenCalled(); + await speech.cancel(); + await loop; + expect(error).toBeUndefined(); + expect(cancelled).toHaveBeenCalledTimes(1); + } finally { + jest.useRealTimers(); + } + }, + ); + /** * Answers the next fetch the way fetch does: once the request's signal * aborts, the body errors with the signal's reason. The body sends `total` @@ -641,6 +798,10 @@ describe("tts", () => { await (await client.tts.stream({ text: "Hi." }, { signal })) .bytes() .catch(() => undefined); + mockStreamedBody(stallingSource().source); // stalled past its read timeout + await (await client.tts.stream({ text: "Hi." }, { signal, readTimeout: 5 })) + .bytes() + .catch(() => undefined); mockAudio(); // a body read whole await (await client.tts.stream({ text: "Hi." }, { signal })).bytes(); fetchMock.mockResponseOnce("busy", { status: 503 }); // refused @@ -648,7 +809,7 @@ describe("tts", () => { fetchMock.mockRejectOnce(new TypeError("fetch failed")); // no answer await client.tts.stream({ text: "Hi." }, { signal }).catch(() => undefined); - expect(fetchMock).toHaveBeenCalledTimes(8); + expect(fetchMock).toHaveBeenCalledTimes(9); expect(listeners()).toBe(0); }); }); From 481c82330248d5516837610b1a21320447fb1a70 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:46:37 +0000 Subject: [PATCH 13/26] fix(tts): send the SDK User-Agent on the Node session handshake The API records the handshake's User-Agent with a session's usage, but the Node handshake carried only Authorization, so Node SDK sessions could not be attributed. client.tts.session() now hands the client's User-Agent to the session (none with userAgent: false), and connect() sends it on Node on both the API key and the token path. A browser sends its own and its WebSocket takes no headers, so nothing changes there; a session created without a client sends none. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- src/services/base.ts | 3 +- src/services/tts/service.ts | 4 +- src/services/tts/session.ts | 33 +++++++++--- tests/unit/tts-session.test.ts | 92 ++++++++++++++++++++++++++++++++-- 4 files changed, 118 insertions(+), 14 deletions(-) diff --git a/src/services/base.ts b/src/services/base.ts index 312645f..897d322 100644 --- a/src/services/base.ts +++ b/src/services/base.ts @@ -7,7 +7,8 @@ import { buildUserAgent } from "../utils/userAgent"; * Base class for services that communicate with the API. */ export abstract class BaseService { - private userAgent: string | undefined; + /** The User-Agent the client sends, or `undefined` with `userAgent: false`. */ + protected readonly userAgent: string | undefined; /** * Create a new service. * @param params - The parameters to use for the service. diff --git a/src/services/tts/service.ts b/src/services/tts/service.ts index 8473b8a..144ea83 100644 --- a/src/services/tts/service.ts +++ b/src/services/tts/service.ts @@ -468,7 +468,9 @@ export class TextToSpeechService extends BaseService { base.pathname = base.pathname.replace(/\/$/, "") + "/v1/ws"; sessionParams.websocketBaseUrl = base.toString(); } - return new TextToSpeechSession(sessionParams); + return new TextToSpeechSession(sessionParams, { + userAgent: this.userAgent, + }); } /** diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index 570622e..8d8af4b 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -319,8 +319,16 @@ export class TextToSpeechSession { /** The temporary token of the next connection; `connect({ token })` replaces it. */ private token?: string; - /** @throws Error if neither `apiKey` nor `token` is given. */ - constructor(private readonly params: TextToSpeechSessionParams) { + /** + * @param params - The session's parameters. + * @param internal - Set by `client.tts.session()`: `userAgent` is the + * client's User-Agent, which the handshake sends on Node. + * @throws Error if neither `apiKey` nor `token` is given. + */ + constructor( + private readonly params: TextToSpeechSessionParams, + private readonly internal: { userAgent?: string } = {}, + ) { if (!params.token && !params.apiKey) { throw new Error("API key or temporary token is required."); } @@ -488,14 +496,23 @@ export class TextToSpeechSession { Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/compat.md#browser-compatibility.`, ); } + const headers: Record = {}; + // The token alone: an Authorization header would override it. + if (!token && this.params.apiKey) { + headers.Authorization = this.params.apiKey; + } + // The API records the handshake's User-Agent with the session's usage. + // A browser sends its own and its WebSocket takes no headers: the + // second argument there is the subprotocols. + if (this.internal.userAgent && !conditions.browser) { + headers["User-Agent"] = this.internal.userAgent; + } let socket: PolyfillWebSocket; try { - socket = token - ? // The token alone: an Authorization header would override it. - polyfillWebSocketFactory(url.toString()) - : polyfillWebSocketFactory(url.toString(), { - headers: { Authorization: this.params.apiKey }, - }); + socket = + Object.keys(headers).length > 0 + ? polyfillWebSocketFactory(url.toString(), { headers }) + : polyfillWebSocketFactory(url.toString()); } catch (cause) { // The runtime refused to create the socket (a browser's WebSocket // takes no headers, for one), so there is no connection to close. diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index 7619fa7..6342ed0 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -9,7 +9,8 @@ import { TtsBeginEvent, } from "../../src"; import * as websocketPolyfill from "../../src/polyfills/websocket/default"; -import { createClient, defaultApiKey } from "./utils"; +import { conditions } from "#conditions"; +import { createClient, defaultApiKey, defaultBaseUrl } from "./utils"; fetchMock.enableMocks(); @@ -174,6 +175,11 @@ async function collect(iterable: AsyncIterable): Promise { const tick = () => new Promise((resolve) => setTimeout(resolve, 20)); +/** The User-Agent the client builds by default, as the HTTP calls send it. */ +const sdkUserAgent = expect.stringMatching( + /AssemblyAI\/1\.0 \(sdk=JavaScript\/\S+/, +); + /** The server side of connection `n` (1-based), once the client opened it. */ async function connection(n: number): Promise { for (let waited = 0; api.sockets.length < n; waited += 5) { @@ -459,7 +465,10 @@ describe("tts session", () => { const url = new URL(first.url); expect(url.searchParams.get("token")).toBe("tmp-1"); expect(url.searchParams.get("inactivity_timeout")).toBe("30"); - expect(first.clientOptions).toBeUndefined(); // no Authorization header + // No Authorization header: it would override the token. + expect(first.clientOptions).toEqual({ + headers: { "User-Agent": sdkUserAgent }, + }); expect((await collect(session.speak("One."))).toString()).toBe("One."); await session.close(); @@ -516,19 +525,94 @@ describe("tts session", () => { const first = await connection(1); expect(new URL(first.url).searchParams.has("token")).toBe(false); expect(first.clientOptions).toEqual({ - headers: { Authorization: defaultApiKey }, + headers: { Authorization: defaultApiKey, "User-Agent": sdkUserAgent }, }); await session.close(); const reconnect = session.connect({ token: "tmp" }); const second = await connection(2); expect(new URL(second.url).searchParams.get("token")).toBe("tmp"); - expect(second.clientOptions).toBeUndefined(); + expect(second.clientOptions).toEqual({ + headers: { "User-Agent": sdkUserAgent }, + }); second.send(JSON.stringify(beginMessage)); await reconnect; await session.close(); }); + describe("the handshake's User-Agent", () => { + it("is the client's, on both the API key and the token path", async () => { + const custom = new AssemblyAI({ + apiKey: defaultApiKey, + ttsBaseUrl: defaultBaseUrl, + userAgent: { integration: { name: "Pipecat", version: "1.2.3" } }, + }); + const session = custom.tts.session({ websocketBaseUrl }); + await connect(session); + const userAgent = ( + (await connection(1)).clientOptions as { + headers: Record; + } + ).headers["User-Agent"]; + expect(userAgent).toEqual(sdkUserAgent); + expect(userAgent).toContain("integration=Pipecat/1.2.3"); + await session.close(); + + const reconnect = session.connect({ token: "tmp" }); + const second = await connection(2); + expect(second.clientOptions).toEqual({ + headers: { "User-Agent": userAgent }, + }); + second.send(JSON.stringify(beginMessage)); + await reconnect; + await session.close(); + }); + + it("is not sent with userAgent: false", async () => { + const anonymous = new AssemblyAI({ + apiKey: defaultApiKey, + ttsBaseUrl: defaultBaseUrl, + userAgent: false, + }); + const session = anonymous.tts.session({ websocketBaseUrl }); + await connect(session); + expect((await connection(1)).clientOptions).toEqual({ + headers: { Authorization: defaultApiKey }, + }); + await session.close(); + + const reconnect = session.connect({ token: "tmp" }); + const second = await connection(2); + expect(second.clientOptions).toBeUndefined(); + second.send(JSON.stringify(beginMessage)); + await reconnect; + await session.close(); + }); + + it("is not sent by a session created without a client", async () => { + const session = new TextToSpeechSession({ + websocketBaseUrl, + token: "tmp", + }); + await connect(session); + expect((await connection(1)).clientOptions).toBeUndefined(); + await session.close(); + }); + + it("is left to the browser, whose WebSocket takes no headers", async () => { + // The browser polyfill passes a second argument as the subprotocols. + conditions.browser = true; + try { + const session = client.tts.session({ websocketBaseUrl, token: "tmp" }); + await connect(session); + expect((await connection(1)).clientOptions).toBeUndefined(); + await session.close(); + } finally { + conditions.browser = false; + } + }); + }); + it("a connect({ token }) refused while connected keeps the session's token", async () => { const session = client.tts.session({ websocketBaseUrl, token: "tmp-1" }); await connect(session); From b3a316734c5439606c42dabcc76bf61abe6626a9 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:52:57 +0000 Subject: [PATCH 14/26] fix(tts): put the token service's FastAPI detail in the error message and document the consent errors createTemporaryToken() refusals come from the token host as FastAPI {"detail": ...} bodies, which speechErrorFromResponse() ignored, so a bad key read "text-to-speech request failed with status 404". A string detail is now the message, and a 422 list reads ": " joined with "; ", with param set to the first entry's field. The envelope's message, then error, still win. Document extend_biometric_consent's refusals (needs voice_id; not_owner; biometric_terms_not_accepted until the account accepts the biometric data terms) on both parameter types, list the cloned-voice and consent codes with their HTTP status and close code in the errorCode doc, and add the README sentences. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- README.md | 4 +- src/services/tts/service.ts | 41 ++++++++++++- src/types/tts/index.ts | 22 +++++-- src/utils/errors/tts.ts | 24 +++++++- tests/unit/tts.test.ts | 111 ++++++++++++++++++++++++++++++++++++ 5 files changed, 193 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f986a4d..568ca85 100644 --- a/README.md +++ b/README.md @@ -844,6 +844,8 @@ console.log(response.model, response.voiceId, response.audioDurationMs, "ms"); `synthesize()` waits for the whole audio, so an API refusal is always a `TextToSpeechError` and the WAV header carries real sizes. The response headers are on the result: `requestId`, `region`, `model`, `voiceId`, `language`, `encoding`, `sampleRate`, `container`, `bitRate`, `billedCharacters` and `audioDurationMs`. A network failure or a timeout rejects with the runtime's own error instead (a `TypeError`, or a `DOMException` named `TimeoutError` or `AbortError`). On Node, the built-in `fetch` gives up after 300 seconds without response headers, whatever `timeout` says; for longer requests use `stream()`, or raise the limit with `setGlobalDispatcher(new Agent({ headersTimeout: 610_000 }))` from the `undici` package. +Branch on a refusal's `errorCode`, not its message. Your account's own cloned voices, and `extend_biometric_consent: true` (which renews the consent of the cloned voice in `voice_id` and is refused with `not_owner` on a voice your account does not own), are refused with `biometric_terms_not_accepted` (403, not retryable; close code 3006 on a session) until the account accepts the biometric data terms on the data controls page of the AssemblyAI dashboard. + ### Stream audio as it is generated ```typescript @@ -928,7 +930,7 @@ await session.connect(); // After a retryable close: await session.connect({ token: await getToken() }); ``` -Mint a new token for every connection, reconnects included, and for every HTTP request (a token works as the `apiKey` of a client made for that request), and keep `expires_in_seconds` short. `createTemporaryToken()` mints on the token host of the client's zone: `https://token.eu.assemblyai.com` when `ttsBaseUrl` is `https://tts.eu.assemblyai.com`, `https://token.us.assemblyai.com` for `https://tts.us.assemblyai.com` (the default), and `https://token.assemblyai.com` for any other host; pass `{ tokenBaseUrl }` as its second argument to choose the host. A token works in every zone. +Mint a new token for every connection, reconnects included, and for every HTTP request (a token works as the `apiKey` of a client made for that request), and keep `expires_in_seconds` short. `createTemporaryToken()` mints on the token host of the client's zone: `https://token.eu.assemblyai.com` when `ttsBaseUrl` is `https://tts.eu.assemblyai.com`, `https://token.us.assemblyai.com` for `https://tts.us.assemblyai.com` (the default), and `https://token.assemblyai.com` for any other host; pass `{ tokenBaseUrl }` as its second argument to choose the host. A token works in every zone. A refusal throws a `TextToSpeechError` whose message gives the token service's reason, such as `Invalid API key` or `expires_in_seconds: Input should be less than or equal to 600`. ## Contributing diff --git a/src/services/tts/service.ts b/src/services/tts/service.ts index 144ea83..85dbe9f 100644 --- a/src/services/tts/service.ts +++ b/src/services/tts/service.ts @@ -102,7 +102,39 @@ const DEFAULT_READ_TIMEOUT_MS = 30_000; /** A buffered request waits for the whole audio; the API caps a request at 600 s. */ const DEFAULT_SYNTHESIZE_TIMEOUT_MS = 610_000; -/** Builds the `TextToSpeechError` for a non-200 response. */ +/** + * The reason in a FastAPI `detail`, which the token host answers refusals + * with: a string, or a list of `{ loc, msg }` for a bad query parameter. A + * list entry reads `: `, its field being the last element of + * `loc` (`["query", "expires_in_seconds"]`), and the entries are joined with + * `"; "`. `param` is the first entry's field, when its `loc` names one after + * the location (`query`, `body`, ...). + */ +function fastApiDetail(detail: unknown): { text?: string; param?: string } { + if (typeof detail === "string") return { text: detail || undefined }; + if (!Array.isArray(detail)) return {}; + let param: string | undefined; + const parts = detail.flatMap((entry: unknown, index) => { + if (!entry || typeof entry !== "object") return []; + const { loc, msg } = entry as { loc?: unknown; msg?: unknown }; + const field = + Array.isArray(loc) && loc.length > 0 + ? String(loc[loc.length - 1]) + : undefined; + if (index === 0 && Array.isArray(loc) && loc.length > 1) param = field; + const part = [field, typeof msg === "string" ? msg : undefined] + .filter(Boolean) + .join(": "); + return part ? [part] : []; + }); + return { text: parts.join("; ") || undefined, param }; +} + +/** + * Builds the `TextToSpeechError` for a non-200 response. The message is the + * body's `message`, else its `error`, else the reason in a FastAPI `detail` + * (the token host's refusals), else a generic one with the status. + */ export async function speechErrorFromResponse( response: Response, ): Promise { @@ -124,16 +156,18 @@ export async function speechErrorFromResponse( } } if (body) { + const detail = fastApiDetail(body.detail); const message = (typeof body.message === "string" && body.message) || (typeof body.error === "string" && body.error) || + detail.text || `text-to-speech request failed with status ${status}`; const hint = body.retry_after_seconds; if (typeof hint === "number") retryAfter = Math.ceil(hint); return new TextToSpeechError(message, { status, errorCode: typeof body.code === "string" ? body.code : undefined, - param: typeof body.param === "string" ? body.param : undefined, + param: typeof body.param === "string" ? body.param : detail.param, requestId: typeof body.request_id === "string" ? body.request_id : requestId, retryable: @@ -485,7 +519,8 @@ export class TextToSpeechService extends BaseService { * `https://token.assemblyai.com` for any other host. A token works in * every zone. * @param options.tokenBaseUrl - The token host to use instead. - * @throws TextToSpeechError if the token service refuses the request. + * @throws TextToSpeechError if the token service refuses the request; its + * message is the service's reason, such as `Invalid API key`. */ async createTemporaryToken( params: TextToSpeechTokenParams, diff --git a/src/types/tts/index.ts b/src/types/tts/index.ts index 979e730..1a06300 100644 --- a/src/types/tts/index.ts +++ b/src/types/tts/index.ts @@ -57,9 +57,15 @@ export type TextToSpeechVoiceOptions = { /** Output codec. Defaults to `pcm_s16le`. A session streams PCM or G.711 only. */ encoding?: LiteralUnion; /** - * `true` renews the biometric consent of the cloned voice in `voice_id`, - * which only the voice's owner may do; a cloned voice expires 60 days - * after its consent was last given. + * `true` renews the biometric consent of the cloned voice in `voice_id`; a + * cloned voice expires 60 days after its consent was last given. Needs + * `voice_id` (`422 invalid_parameter` without it). Refused with `not_owner` + * (422, `param: "extend_biometric_consent"`) on a voice the account does + * not own, and with `biometric_terms_not_accepted` (403, not retryable) + * until the account accepts the biometric data terms on the data controls + * page of the AssemblyAI dashboard. `voiceExpiresAt` on the response shows + * the new expiry after `synthesize()`; after `stream()` it is the expiry as + * the request started, because the headers leave with the first audio. */ extend_biometric_consent?: boolean; }; @@ -170,7 +176,15 @@ export type TextToSpeechSessionParams = { wordBoundaries?: boolean; /** Seconds (5 to 120, default 120) after which an idle session is closed. */ inactivityTimeout?: number; - /** `true` renews the consent of the cloned voice in `voiceId` (owner only). */ + /** + * `true` renews the biometric consent of the cloned voice in `voiceId`. + * Needs `voiceId`. `connect()` is refused with `not_owner` (close 3006, + * `param: "extend_biometric_consent"`) on a voice the account does not + * own, and with `biometric_terms_not_accepted` (close 3006, not retryable) + * until the account accepts the biometric data terms on the data controls + * page of the AssemblyAI dashboard. The `Termination` summary's + * `voice_expires_at` shows the new expiry. + */ extendBiometricConsent?: boolean; /** * A temporary token, for a session opened on a user's device. It is sent diff --git a/src/utils/errors/tts.ts b/src/utils/errors/tts.ts index 43fe515..823db36 100644 --- a/src/utils/errors/tts.ts +++ b/src/utils/errors/tts.ts @@ -5,7 +5,29 @@ export type TextToSpeechErrorFields = { * and only where the runtime reports it (Node does, a browser does not). */ status?: number; - /** The API's stable code: `invalid_parameter`, `voice_not_found`, `rate_limited`, ... */ + /** + * The API's stable code, the same on HTTP and on a session. Among them: + * - `invalid_parameter` (422; close 3006): `param` and `errors` say which field. + * - `rate_limited` (429, retryable; close 3009): wait `retryAfter` seconds. + * - `voice_not_found` (422; close 3012, `param: "voice_id"`). + * - `voice_expired` (422; close 3012, `param: "voice_id"`): the cloned + * voice's consent period ended; its owner can create it again from a new + * recording. + * - `voice_not_paired` (422; close 3014, `param: "model_id"`): the cloned + * voice is not paired with the model, or the pairing is still in progress. + * - `not_owner` (422; close 3006, `param: "extend_biometric_consent"`): + * `extend_biometric_consent` on a voice another account owns. + * - `biometric_terms_not_accepted` (403, not retryable; close 3006, + * `param` `voice_id` or `extend_biometric_consent`): the account has not + * accepted the biometric data terms that its own cloned voices and + * `extend_biometric_consent` need; accept them on the data controls page + * of the AssemblyAI dashboard. + * - `auth_unavailable` (503, retryable; close 1011): the credential, or the + * account's biometric data terms, could not be checked just now. + * + * Absent on a refusal of `createTemporaryToken()`, whose message carries + * the token service's reason. + */ errorCode?: string; /** The field at fault, if one. */ param?: string; diff --git a/tests/unit/tts.test.ts b/tests/unit/tts.test.ts index 7c6f92a..67cb1ce 100644 --- a/tests/unit/tts.test.ts +++ b/tests/unit/tts.test.ts @@ -287,6 +287,117 @@ describe("tts", () => { ); }); + it("a token refusal with a FastAPI detail string carries the reason in its message", async () => { + fetchMock.mockResponseOnce(JSON.stringify({ detail: "Invalid API key" }), { + status: 404, + headers: { "content-type": "application/json" }, + }); + const error = (await client.tts + .createTemporaryToken({ expires_in_seconds: 60 }) + .catch((e) => e)) as TextToSpeechError; + expect(error).toBeInstanceOf(TextToSpeechError); + expect(error.message).toBe("Invalid API key"); + expect(error).toMatchObject({ status: 404, retryable: false }); + expect(error.param).toBeUndefined(); + expect(error.body).toEqual({ detail: "Invalid API key" }); + }); + + it("a token refusal with a FastAPI 422 list names the field and the reason", async () => { + const body = { + detail: [ + { + type: "less_than_equal", + loc: ["query", "expires_in_seconds"], + msg: "Input should be less than or equal to 600", + input: "601", + ctx: { le: 600 }, + }, + { + type: "greater_than_equal", + loc: ["query", "max_session_duration_seconds"], + msg: "Input should be greater than or equal to 60", + input: "30", + ctx: { ge: 60 }, + }, + ], + }; + fetchMock.mockResponseOnce(JSON.stringify(body), { + status: 422, + headers: { "content-type": "application/json" }, + }); + const error = (await client.tts + .createTemporaryToken({ + expires_in_seconds: 601, + max_session_duration_seconds: 30, + }) + .catch((e) => e)) as TextToSpeechError; + expect(error).toBeInstanceOf(TextToSpeechError); + expect(error.message).toBe( + "expires_in_seconds: Input should be less than or equal to 600; " + + "max_session_duration_seconds: Input should be greater than or equal to 60", + ); + expect(error).toMatchObject({ + status: 422, + param: "expires_in_seconds", + retryable: false, + }); + expect(error.body).toEqual(body); + }); + + it.each([ + { + name: "message wins over error and detail", + body: { message: "from message", error: "from error", detail: "x" }, + message: "from message", + param: undefined, + }, + { + name: "error wins over detail", + body: { + error: "from error", + detail: [{ loc: ["query", "a"], msg: "m" }], + }, + message: "from error", + param: "a", + }, + { + name: "the envelope's param wins over the detail's", + body: { param: "voice_id", detail: [{ loc: ["query", "a"], msg: "m" }] }, + message: "a: m", + param: "voice_id", + }, + { + name: "a detail entry with only a location gives no param", + body: { detail: [{ loc: ["body"], msg: "Field required" }] }, + message: "body: Field required", + param: undefined, + }, + { + name: "an empty or unrecognized detail falls back to the status", + body: { detail: [{}, null, 3] }, + message: "text-to-speech request failed with status 400", + param: undefined, + }, + { + name: "an object detail falls back to the status", + body: { detail: { reason: "nope" } }, + message: "text-to-speech request failed with status 400", + param: undefined, + }, + ])("error message precedence: $name", async ({ body, message, param }) => { + const { speechErrorFromResponse } = await import( + "../../src/services/tts/service" + ); + const error = await speechErrorFromResponse( + new Response(JSON.stringify(body), { + status: 400, + headers: { "content-type": "application/json" }, + }), + ); + expect(error.message).toBe(message); + expect(error.param).toBe(param); + }); + it("leaving a stream early cancels the body", async () => { const { TextToSpeechStream } = await import( "../../src/services/tts/service" From b84d6c000763f8da79b034403d482e5bfe471628 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 15:57:03 +0000 Subject: [PATCH 15/26] fix(tts): report X-TTS-Unknown-Params, read the voice from X-TTS-Voice-Id only, type bit_rate's values - SpeechResponseInfo.unknownParams lists the body fields the API ignored (X-TTS-Unknown-Params), and synthesize()/stream() warn once per process when it is set, so a camelCase `sampleRate` in a request held in a variable or sent from JavaScript shows up instead of 24 kHz audio. - voiceId comes from X-TTS-Voice-Id only: the API sends it on every 200 and no longer sends X-TTS-Voice, so the fallback that guessed a slug from it is gone. - billedCharacters and voiceExpiresAt document that on a stream they are the values as the stream started (a body cut by a server failure bills 0; a consent renewal shows on synthesize()). - bit_rate is TextToSpeechMp3BitRate (32000, 48000, 64000, 96000 or 128000; at most 64000 at 8000 Hz), as the API validates it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- README.md | 4 +-- src/services/tts/service.ts | 47 +++++++++++++++++++------------- src/types/tts/index.ts | 34 +++++++++++++++++++++--- tests/unit/tts.test.ts | 53 ++++++++++++++++++++++++++++++------- 4 files changed, 104 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 568ca85..2002f2e 100644 --- a/README.md +++ b/README.md @@ -842,7 +842,7 @@ writeFileSync("hello.wav", response.audio); console.log(response.model, response.voiceId, response.audioDurationMs, "ms"); ``` -`synthesize()` waits for the whole audio, so an API refusal is always a `TextToSpeechError` and the WAV header carries real sizes. The response headers are on the result: `requestId`, `region`, `model`, `voiceId`, `language`, `encoding`, `sampleRate`, `container`, `bitRate`, `billedCharacters` and `audioDurationMs`. A network failure or a timeout rejects with the runtime's own error instead (a `TypeError`, or a `DOMException` named `TimeoutError` or `AbortError`). On Node, the built-in `fetch` gives up after 300 seconds without response headers, whatever `timeout` says; for longer requests use `stream()`, or raise the limit with `setGlobalDispatcher(new Agent({ headersTimeout: 610_000 }))` from the `undici` package. +`synthesize()` waits for the whole audio, so an API refusal is always a `TextToSpeechError` and the WAV header carries real sizes. The response headers are on the result: `requestId`, `region`, `model`, `voiceId`, `language`, `encoding`, `sampleRate`, `container`, `bitRate`, `billedCharacters`, `audioDurationMs` and `unknownParams`. The API ignores body fields it does not know, and `unknownParams` names them (the SDK also warns once), so a camelCase `sampleRate` in a request held in a variable shows up instead of silently giving 24 kHz audio. A network failure or a timeout rejects with the runtime's own error instead (a `TypeError`, or a `DOMException` named `TimeoutError` or `AbortError`). On Node, the built-in `fetch` gives up after 300 seconds without response headers, whatever `timeout` says; for longer requests use `stream()`, or raise the limit with `setGlobalDispatcher(new Agent({ headersTimeout: 610_000 }))` from the `undici` package. Branch on a refusal's `errorCode`, not its message. Your account's own cloned voices, and `extend_biometric_consent: true` (which renews the consent of the cloned voice in `voice_id` and is refused with `not_owner` on a voice your account does not own), are refused with `biometric_terms_not_accepted` (403, not retryable; close code 3006 on a session) until the account accepts the biometric data terms on the data controls page of the AssemblyAI dashboard. @@ -865,7 +865,7 @@ for await (const chunk of speech) { } ``` -`stream()` resolves once the API has sent the first audio. Chunk sizes vary and are not aligned to samples or frames: for 16-bit PCM carry an odd trailing byte over to the next chunk, and re-frame for a fixed-size sink (20 ms of 8 kHz mu-law is 160 bytes). The API commits to `200` with the first audio, so a failure after that cuts the body: the loop then throws a `TextToSpeechError` with `retryable: true`, and the whole request should be retried. A body that sends no audio for 30 seconds while the loop waits for it ends the same way, and its connection is released, so a stalled connection (a dropped NAT mapping, a hung proxy) cannot hang the loop. Set the limit in milliseconds with the `readTimeout` option (`0` turns it off); only the wait counts, so playing a chunk at its own pace never trips it. `speech.body` is the underlying web `ReadableStream`, and a read of it waits under the same limit. +`stream()` resolves once the API has sent the first audio. Chunk sizes vary and are not aligned to samples or frames: for 16-bit PCM carry an odd trailing byte over to the next chunk, and re-frame for a fixed-size sink (20 ms of 8 kHz mu-law is 160 bytes). The API commits to `200` with the first audio, so a failure after that cuts the body: the loop then throws a `TextToSpeechError` with `retryable: true`, and the whole request should be retried. The headers leave with the first audio too, so `speech.info.billedCharacters` and `speech.info.voiceExpiresAt` are the values as the stream started: a body cut by a server failure bills 0. A body that sends no audio for 30 seconds while the loop waits for it ends the same way, and its connection is released, so a stalled connection (a dropped NAT mapping, a hung proxy) cannot hang the loop. Set the limit in milliseconds with the `readTimeout` option (`0` turns it off); only the wait counts, so playing a chunk at its own pace never trips it. `speech.body` is the underlying web `ReadableStream`, and a read of it waits under the same limit. For a barge-in, pass `{ signal }` as the second argument and abort it: the download stops and the loop throws the signal's reason (an `AbortError`), not a `TextToSpeechError`. `speech.cancel()` stops it too, and the loop just ends. One signal can serve every request of a call. diff --git a/src/services/tts/service.ts b/src/services/tts/service.ts index 85dbe9f..3fe7a36 100644 --- a/src/services/tts/service.ts +++ b/src/services/tts/service.ts @@ -200,23 +200,17 @@ function intHeader(headers: Headers, name: string): number | undefined { } /** Reads the `X-TTS-*` headers of a `200`. */ -export function speechResponseInfo( - headers: Headers, - requestedVoiceId?: string, -): SpeechResponseInfo { - // `X-TTS-Voice-Id` is the voice; until the API sends it, derive it from - // `X-TTS-Voice` (`custom` is the request's own voice id, else a preset). - let voiceId = headers.get("x-tts-voice-id") ?? undefined; - if (voiceId === undefined) { - const voice = headers.get("x-tts-voice"); - if (voice === "custom") voiceId = requestedVoiceId; - else if (voice) voiceId = `aai-${voice}`; - } +export function speechResponseInfo(headers: Headers): SpeechResponseInfo { + const unknownParams = headers + .get("x-tts-unknown-params") + ?.split(",") + .map((name) => name.trim()) + .filter(Boolean); return { requestId: headers.get("x-request-id") ?? undefined, region: headers.get("x-tts-region") ?? undefined, model: headers.get("x-tts-model") ?? undefined, - voiceId, + voiceId: headers.get("x-tts-voice-id") ?? undefined, language: headers.get("x-tts-language") ?? undefined, encoding: headers.get("x-tts-encoding") ?? undefined, sampleRate: intHeader(headers, "x-tts-sample-rate"), @@ -226,9 +220,26 @@ export function speechResponseInfo( voiceExpiresAt: intHeader(headers, "x-tts-voice-expires-at"), billedCharacters: intHeader(headers, "x-tts-billed-characters"), audioDurationMs: intHeader(headers, "x-tts-audio-duration-ms"), + unknownParams: unknownParams?.length ? unknownParams : undefined, }; } +let warnedUnknownParams = false; + +/** + * Warns once per process that the API ignored body fields it does not know, + * which is how a camelCase `sampleRate` in a request held in a variable (or + * sent from plain JavaScript) shows up. The names stay on `unknownParams`. + */ +function warnUnknownParams(info: SpeechResponseInfo) { + if (warnedUnknownParams || !info.unknownParams) return; + warnedUnknownParams = true; + console.warn( + `TextToSpeechService: the API ignored unknown request fields: ${info.unknownParams.join(", ")}. ` + + "Request fields are snake_case, such as sample_rate; see unknownParams on the response.", + ); +} + /** * The audio of one streamed request, read as the API generates it. * @@ -376,10 +387,9 @@ export class TextToSpeechService extends BaseService { }); if (!response.ok) throw await speechErrorFromResponse(response); const audio = new Uint8Array(await response.arrayBuffer()); - return { - ...speechResponseInfo(response.headers, request.voice_id), - audio, - }; + const info = speechResponseInfo(response.headers); + warnUnknownParams(info); + return { ...info, audio }; } finally { deadline.release(); } @@ -426,7 +436,8 @@ export class TextToSpeechService extends BaseService { deadline.release(); } } - const info = speechResponseInfo(response.headers, request.voice_id); + const info = speechResponseInfo(response.headers); + warnUnknownParams(info); if (!response.body || typeof response.body.getReader !== "function") { // A runtime without web streamed bodies: hand the whole audio over as one chunk. let bytes: Uint8Array; diff --git a/src/types/tts/index.ts b/src/types/tts/index.ts index 1a06300..82cc3b9 100644 --- a/src/types/tts/index.ts +++ b/src/types/tts/index.ts @@ -27,6 +27,12 @@ export type TextToSpeechSampleRate = | 44100 | 48000; +/** + * MP3 bit rates the API accepts, in bits per second. At most 64000 at a + * `sample_rate` of 8000. + */ +export type TextToSpeechMp3BitRate = 32000 | 48000 | 64000 | 96000 | 128000; + /** * The language, by its English name: `english` (the default), `spanish`, * `german`, `french`, `italian` or `portuguese`. Case-insensitive. @@ -76,8 +82,12 @@ export type SpeechRequest = TextToSpeechVoiceOptions & { text: string; /** `raw` (the default) or `wav`, with a PCM or G.711 encoding only. */ container?: TextToSpeechContainer; - /** MP3 bit rate in bits per second (32000 to 128000, default 64000), with `encoding: "mp3"` only. */ - bit_rate?: number; + /** + * MP3 bit rate in bits per second: 32000, 48000, 64000 (the default), + * 96000 or 128000; at most 64000 at a `sample_rate` of 8000. With + * `encoding: "mp3"` only. + */ + bit_rate?: TextToSpeechMp3BitRate; }; /** Per-request options of the HTTP client. */ @@ -135,12 +145,28 @@ export type SpeechResponseInfo = { bitRate?: number; /** The response's `Content-Type`, such as `audio/pcm;rate=24000;channels=1`. */ contentType?: string; - /** For a cloned voice of the account: its expiry as a Unix time in seconds. */ + /** + * For a cloned voice of the account: its expiry as a Unix time in seconds, + * after the request's consent renewal on `synthesize()`. On a stream, the + * value as the stream started: the headers leave with the first audio, + * before any renewal. + */ voiceExpiresAt?: number; - /** The characters the request bills. */ + /** + * The characters the request bills. On a stream, the value as the stream + * started: the headers leave with the first audio, before the outcome is + * known, so a body cut by a server failure bills 0 instead. + */ billedCharacters?: number; /** The audio's duration in milliseconds, on a complete response. */ audioDurationMs?: number; + /** + * Body fields the API did not recognize and ignored + * (`X-TTS-Unknown-Params`), such as a camelCase `sampleRate` for + * `sample_rate`. The API lists at most 16, sorted, and shows a name it + * cannot echo as ``. Absent when every field was known. + */ + unknownParams?: string[]; }; /** The result of `synthesize()`: the whole audio, with its headers. */ diff --git a/tests/unit/tts.test.ts b/tests/unit/tts.test.ts index 67cb1ce..32dfaa2 100644 --- a/tests/unit/tts.test.ts +++ b/tests/unit/tts.test.ts @@ -89,18 +89,51 @@ describe("tts", () => { }); }); - it("derives the voice id from X-TTS-Voice until the API sends X-TTS-Voice-Id", async () => { - const legacy = { ...okHeaders }; - delete (legacy as Record)["x-tts-voice-id"]; - mockAudio({ ...legacy, "x-tts-voice": "jane" }); - mockAudio({ ...legacy, "x-tts-voice": "custom" }); - expect((await client.tts.synthesize({ text: "Hi." })).voiceId).toBe( - "aai-jane", - ); + it("takes the voice id from X-TTS-Voice-Id only", async () => { + const withoutVoiceId: Record = { ...okHeaders }; + delete withoutVoiceId["x-tts-voice-id"]; + mockAudio({ ...withoutVoiceId, "x-tts-voice": "jane" }); + mockAudio({ ...okHeaders, "x-tts-voice-id": "aai-isaiah-2" }); expect( - (await client.tts.synthesize({ text: "Hi.", voice_id: "aai-isaiah-2" })) + (await client.tts.synthesize({ text: "Hi.", voice_id: "aai-jane" })) .voiceId, - ).toBe("aai-isaiah-2"); + ).toBeUndefined(); + const speech = await client.tts.stream({ text: "Hi." }); + expect(speech.info.voiceId).toBe("aai-isaiah-2"); + await speech.cancel(); + }); + + it("reads X-TTS-Unknown-Params and warns once per process", async () => { + const warn = jest.spyOn(console, "warn").mockImplementation(() => {}); + try { + await jest.isolateModulesAsync(async () => { + // A fresh module, so the once-per-process warning has not fired yet. + const { AssemblyAI: FreshAssemblyAI } = await import("../../src"); + const fresh = new FreshAssemblyAI({ + ttsBaseUrl: "http://localhost:1234", + apiKey: defaultApiKey, + }); + mockAudio({ ...okHeaders, "x-tts-unknown-params": "sampleRate" }); + mockAudio({ ...okHeaders, "x-tts-unknown-params": "foo,,sampleRate" }); + mockAudio(okHeaders); + // A request held in a variable gets no excess-property check. + const request = { text: "Hi.", sampleRate: 8000 }; + const first = await fresh.tts.synthesize(request); + expect(first.unknownParams).toEqual(["sampleRate"]); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0][0]).toContain( + "ignored unknown request fields: sampleRate", + ); + const speech = await fresh.tts.stream(request); + expect(speech.info.unknownParams).toEqual(["foo", "sampleRate"]); + await speech.cancel(); + const clean = await fresh.tts.synthesize({ text: "Hi." }); + expect(clean.unknownParams).toBeUndefined(); + expect(warn).toHaveBeenCalledTimes(1); + }); + } finally { + warn.mockRestore(); + } }); it("stream asks for a streamed response and yields the body", async () => { From a8979380e68a7e95e6b4a6c13433bdb371218b79 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:01:05 +0000 Subject: [PATCH 16/26] fix(tts): refuse stream() before sending on a runtime without ReadableStream React Native resolves the package to dist/browser.mjs, has no streamed response body and no global ReadableStream without a polyfill, so stream() downloaded (and billed) the whole audio and then threw a ReferenceError building the one-chunk fallback body. stream() now checks for a global ReadableStream first and throws a non-retryable TextToSpeechError that points to synthesize(), without sending the request. Runtimes that have ReadableStream but no streamed body keep the whole-body fallback. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- CLAUDE.md | 2 +- README.md | 2 +- src/services/tts/service.ts | 18 +++++++++++++++++- tests/unit/tts.test.ts | 20 ++++++++++++++++++++ 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 761b9ae..ec71a0f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -451,7 +451,7 @@ session.cancel(); // barge-in: ends the current audio() iteration at once await session.close(); // Terminate -> Termination summary; rejects with an unobserved session error ``` -Gotchas: `stream()` gives up a body that sends no audio for `readTimeout` ms (30,000 by default, `0` off) while a read waits, with the same retryable `TextToSpeechError` as a cut; a session streams raw PCM/G.711 only (no `mp3`, `container` or `bit_rate`); `generate()` takes at most 2,000 characters per frame (`speak()` splits); the API paces text at ~100 chars/s; an `audio` listener switches audio from pull to push; a session the API ended (idle after 120 s) can `connect()` again on the same object, but a session opened with a temporary token needs a new token for each connection: `connect({ token: newToken })` (a used or expired token is refused with 1008 `unauthorized`). In a browser use `new TextToSpeechSession({ token })` (no client, no API key); `client.tts.createTemporaryToken()` mints on the token host of the `ttsBaseUrl` zone (`token.eu`/`token.us`), `tokenBaseUrl` overrides. +Gotchas: `stream()` gives up a body that sends no audio for `readTimeout` ms (30,000 by default, `0` off) while a read waits, with the same retryable `TextToSpeechError` as a cut; `stream()` needs a global `ReadableStream` and, on a runtime without one (React Native without a polyfill), throws a non-retryable `TextToSpeechError` before sending (use `synthesize()`); a session streams raw PCM/G.711 only (no `mp3`, `container` or `bit_rate`); `generate()` takes at most 2,000 characters per frame (`speak()` splits); the API paces text at ~100 chars/s; an `audio` listener switches audio from pull to push; a session the API ended (idle after 120 s) can `connect()` again on the same object, but a session opened with a temporary token needs a new token for each connection: `connect({ token: newToken })` (a used or expired token is refused with 1008 `unauthorized`). In a browser use `new TextToSpeechSession({ token })` (no client, no API key); `client.tts.createTemporaryToken()` mints on the token host of the `ttsBaseUrl` zone (`token.eu`/`token.us`), `tokenBaseUrl` overrides. ## Important gotchas diff --git a/README.md b/README.md index 2002f2e..0ae0495 100644 --- a/README.md +++ b/README.md @@ -865,7 +865,7 @@ for await (const chunk of speech) { } ``` -`stream()` resolves once the API has sent the first audio. Chunk sizes vary and are not aligned to samples or frames: for 16-bit PCM carry an odd trailing byte over to the next chunk, and re-frame for a fixed-size sink (20 ms of 8 kHz mu-law is 160 bytes). The API commits to `200` with the first audio, so a failure after that cuts the body: the loop then throws a `TextToSpeechError` with `retryable: true`, and the whole request should be retried. The headers leave with the first audio too, so `speech.info.billedCharacters` and `speech.info.voiceExpiresAt` are the values as the stream started: a body cut by a server failure bills 0. A body that sends no audio for 30 seconds while the loop waits for it ends the same way, and its connection is released, so a stalled connection (a dropped NAT mapping, a hung proxy) cannot hang the loop. Set the limit in milliseconds with the `readTimeout` option (`0` turns it off); only the wait counts, so playing a chunk at its own pace never trips it. `speech.body` is the underlying web `ReadableStream`, and a read of it waits under the same limit. +`stream()` resolves once the API has sent the first audio. Chunk sizes vary and are not aligned to samples or frames: for 16-bit PCM carry an odd trailing byte over to the next chunk, and re-frame for a fixed-size sink (20 ms of 8 kHz mu-law is 160 bytes). The API commits to `200` with the first audio, so a failure after that cuts the body: the loop then throws a `TextToSpeechError` with `retryable: true`, and the whole request should be retried. The headers leave with the first audio too, so `speech.info.billedCharacters` and `speech.info.voiceExpiresAt` are the values as the stream started: a body cut by a server failure bills 0. A body that sends no audio for 30 seconds while the loop waits for it ends the same way, and its connection is released, so a stalled connection (a dropped NAT mapping, a hung proxy) cannot hang the loop. Set the limit in milliseconds with the `readTimeout` option (`0` turns it off); only the wait counts, so playing a chunk at its own pace never trips it. `speech.body` is the underlying web `ReadableStream`, and a read of it waits under the same limit. So `stream()` needs a global `ReadableStream`: on a runtime without one (React Native without a polyfill) it throws a `TextToSpeechError` before sending the request; use `synthesize()` there. For a barge-in, pass `{ signal }` as the second argument and abort it: the download stops and the loop throws the signal's reason (an `AbortError`), not a `TextToSpeechError`. `speech.cancel()` stops it too, and the loop just ends. One signal can serve every request of a call. diff --git a/src/services/tts/service.ts b/src/services/tts/service.ts index 3fe7a36..9906cb0 100644 --- a/src/services/tts/service.ts +++ b/src/services/tts/service.ts @@ -404,7 +404,15 @@ export class TextToSpeechService extends BaseService { * default) while the iteration waits for audio; the connection is then * released. `options.signal` keeps covering the body: aborting it stops * the download. - * @throws TextToSpeechError if the API refuses the request before any audio. + * + * The audio comes as a web `ReadableStream`, so the runtime needs a global + * `ReadableStream` (React Native has none without a polyfill). Without + * one, `stream()` rejects before sending anything; use `synthesize()` + * there. A runtime with `ReadableStream` whose `fetch` gives no streamed + * body gets the whole audio as one chunk once it has all arrived. + * @throws TextToSpeechError if the API refuses the request before any audio, + * or, with `retryable: false` and before any request, if the runtime has + * no global `ReadableStream`. * A network failure or a timeout rejects with the runtime's own error * instead: a `TypeError`, a `DOMException` named `TimeoutError` once * `options.timeout` passes, or the reason of `options.signal`. @@ -413,6 +421,14 @@ export class TextToSpeechService extends BaseService { request: SpeechRequest, options: SpeechRequestOptions = {}, ): Promise { + // Checked before sending, so a runtime that could not hand the audio + // over is not billed for it. + if (typeof ReadableStream !== "function") { + throw new TextToSpeechError( + "stream() needs web streams (a global ReadableStream), which this runtime does not have. Use synthesize() to get the whole audio, or install a ReadableStream polyfill.", + { retryable: false }, + ); + } const deadline = requestDeadline( options.timeout ?? DEFAULT_STREAM_TIMEOUT_MS, options.signal, diff --git a/tests/unit/tts.test.ts b/tests/unit/tts.test.ts index 32dfaa2..c59172b 100644 --- a/tests/unit/tts.test.ts +++ b/tests/unit/tts.test.ts @@ -154,6 +154,26 @@ describe("tts", () => { expect(Buffer.from(await speech.bytes())).toEqual(Buffer.from(audio)); }); + it("stream refuses before sending on a runtime without a global ReadableStream", async () => { + // React Native (the react-native condition) has no ReadableStream without a polyfill. + const saved = globalThis.ReadableStream; + // @ts-expect-error -- removed to emulate such a runtime + delete globalThis.ReadableStream; + try { + mockAudio(); + const error = await client.tts + .stream({ text: "Hello there." }) + .catch((e: unknown) => e); + expect(error).toBeInstanceOf(TextToSpeechError); + expect((error as TextToSpeechError).retryable).toBe(false); + expect((error as TextToSpeechError).message).toContain("synthesize()"); + expect(fetchMock).not.toHaveBeenCalled(); + } finally { + globalThis.ReadableStream = saved; + } + expect(typeof globalThis.ReadableStream).toBe("function"); + }); + it("maps the error envelope to a TextToSpeechError", async () => { fetchMock.doMockOnceIf( requestMatches({ url: "/v1/speech", method: "POST" }), From 5e5062e43353e96c808685e5ca6914778eaa92bd Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:03:53 +0000 Subject: [PATCH 17/26] fix(tts): export TextToSpeechErrorFields from the package root The type of the public TextToSpeechError constructor's second argument was only re-exported from src/utils/errors, which the root does not export, so `import type { TextToSpeechErrorFields } from "assemblyai"` failed with TS2724 and a downstream declaration naming the constructor's parameters failed with TS2742. Re-export it next to the class and name it through the root in the #105 exports check. `pnpm generate:reference` is back to 7 warnings. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- src/services/index.ts | 2 ++ tests/unit/exports.test.ts | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/services/index.ts b/src/services/index.ts index 002b1ee..b69813b 100644 --- a/src/services/index.ts +++ b/src/services/index.ts @@ -169,3 +169,5 @@ export { rollUpTurnChannel, float32ToPcm16, }; + +export type { TextToSpeechErrorFields } from "../utils/errors/tts"; diff --git a/tests/unit/exports.test.ts b/tests/unit/exports.test.ts index be7fda4..5d713d1 100644 --- a/tests/unit/exports.test.ts +++ b/tests/unit/exports.test.ts @@ -1,8 +1,10 @@ import type { LiteralUnion, + TextToSpeechErrorFields, Transcript, TranscriptParams, } from "../../src/exports"; +import { TextToSpeechError } from "../../src/exports"; // #105: every type a public field is declared with must be reachable from the // package root. `LiteralUnion` sits behind `language_code` and @@ -20,4 +22,24 @@ describe("package root exports", () => { expect(typeof languageOf).toBe("function"); expect(typeof requested).toBe("function"); }); + + it("exposes the fields type of the TextToSpeechError constructor", () => { + // `TextToSpeechErrorFields` is the constructor's second argument and the + // class implements it, so a downstream declaration that names the + // constructor's parameters needs it from the root. + const fields: TextToSpeechErrorFields = { + status: 429, + errorCode: "rate_limited", + retryable: true, + retryAfter: 2, + }; + const args: ConstructorParameters = [ + "Too many requests", + fields, + ]; + const error = new TextToSpeechError(...args); + const carried: TextToSpeechErrorFields = error; + expect(carried.errorCode).toBe("rate_limited"); + expect(carried.retryAfter).toBe(2); + }); }); From d4c75c089199e7a79dd6d514578660751421c1a3 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:13:12 +0000 Subject: [PATCH 18/26] test(tts): cover carried-over session behaviour; the fake API's Terminate speaks trailing text FakeApi now answers Terminate the way the API does: text sent and not flushed is spoken first, as Audio and a FlushDone under the next flush id, before Termination. A wordBoundaries option sends each request's WordBoundaries frame after its FlushDone. New tests for behaviour that no test covered yet: - audio nobody reads keeps the newest 2,000 frames, warns once, and drops only audio, never a FlushDone, so every audio() iteration still ends - a client with a ttsBaseUrl path prefix opens sessions under that prefix - Begin's max_generate_text_length limits generate() and speak()'s frames - a close without an Error frame: 1011, 1012, 3005, 3008 and 3009 are retryable, 1008 and 3010 are not, before Begin and after it - keepAlive() sends KeepAlive; extendBiometricConsent reaches the query - close() speaks unflushed text and resolves after its FlushDone - the wordBoundaries listener receives each request's frame Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- tests/unit/tts-session.test.ts | 370 +++++++++++++++++++++++++++++++-- 1 file changed, 347 insertions(+), 23 deletions(-) diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index 6342ed0..1fce426 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -7,6 +7,7 @@ import { TextToSpeechError, TextToSpeechSession, TtsBeginEvent, + TtsWordBoundariesEvent, } from "../../src"; import * as websocketPolyfill from "../../src/polyfills/websocket/default"; import { conditions } from "#conditions"; @@ -48,13 +49,16 @@ type ServerSocket = { /** * Plays the API: Generate/Flush answer with the text's bytes as one Audio - * frame and a FlushDone, Cancel with Cancelled, Terminate with Termination + * frame and a FlushDone, Cancel with Cancelled, and Terminate speaks the text + * not flushed yet the same way, under the next flush id, before Termination * and a clean close. */ class FakeApi { text = ""; flushId = 0; answerFlush = true; + /** On, each spoken request's FlushDone is followed by its WordBoundaries. */ + wordBoundaries = false; /** Off, Cancel gets no Cancelled: the test plays the audio still in transit. */ answerCancel = true; /** Off, Terminate gets no answer: the test plays the drain. */ @@ -110,22 +114,7 @@ class FakeApi { this.text += frame.text as string; return; case "Flush": - if (this.answerFlush) { - socket.send( - JSON.stringify({ - type: "Audio", - audio: Buffer.from(this.text).toString("base64"), - flush_id: this.flushId, - }), - ); - socket.send( - JSON.stringify({ - type: "FlushDone", - flush_id: this.flushId, - audio_duration_ms: 80, - }), - ); - } + if (this.answerFlush) this.speak(socket); this.flushId += 1; this.text = ""; return; @@ -140,6 +129,12 @@ class FakeApi { return; case "Terminate": if (!this.answerTerminate) return; + // Text sent and not flushed is flushed: its own FlushDone, next id. + if (this.text) { + this.speak(socket); + this.flushId += 1; + this.text = ""; + } socket.send(JSON.stringify(terminationMessage)); if (this.closeOnTerminate) { socket.close({ code: 1000, reason: "Session ended", wasClean: true }); @@ -148,11 +143,50 @@ class FakeApi { } } + /** Sends the pending text as the current request's Audio and FlushDone. */ + private speak(socket: ServerSocket) { + socket.send( + JSON.stringify({ + type: "Audio", + audio: Buffer.from(this.text).toString("base64"), + flush_id: this.flushId, + }), + ); + socket.send( + JSON.stringify({ + type: "FlushDone", + flush_id: this.flushId, + audio_duration_ms: 80, + }), + ); + if (this.wordBoundaries) { + socket.send(JSON.stringify(wordBoundariesFrame(this.flushId, this.text))); + } + } + frames(type: string) { return this.sent.filter((f) => f.type === type); } } +/** The WordBoundaries frame FakeApi sends for request `flushId` of `text`. */ +function wordBoundariesFrame(flushId: number, text: string) { + return { + type: "WordBoundaries", + flush_id: flushId, + audio_start_ms: flushId * 1000, + words: text + .split(/\s+/) + .filter(Boolean) + .map((word, i) => ({ + word, + start: flushId * 1000 + i * 100, + end: flushId * 1000 + i * 100 + 80, + confidence: 0.9, + })), + }; +} + let server: WS; let api: FakeApi; let client: AssemblyAI; @@ -208,6 +242,14 @@ function spyConsoleError(): jest.SpyInstance { return consoleError; } +function audioFrame(text: string, flushId: number) { + return JSON.stringify({ + type: "Audio", + audio: Buffer.from(text).toString("base64"), + flush_id: flushId, + }); +} + function errorFrame(code: number, errorType: string, retryable: boolean) { return JSON.stringify({ type: "Error", @@ -265,6 +307,30 @@ describe("tts session", () => { await server.closed; }); + it.each([ + ["http://localhost:1234/prefix", "ws://localhost:1234/prefix/v1/ws"], + ["https://localhost:1234/prefix/", "wss://localhost:1234/prefix/v1/ws"], + ])( + "a client with ttsBaseUrl %p opens its sessions at %p", + async (ttsBaseUrl, sessionUrl) => { + // Given a client whose TTS host is under a path prefix, as behind a proxy + const prefixed = new WS(sessionUrl); + const proxied = new AssemblyAI({ apiKey: defaultApiKey, ttsBaseUrl }); + + // When it opens a session without a websocketBaseUrl + const session = proxied.tts.session(); + const connecting = outcome(session.connect()); + const socket = await outcome(prefixed.connected); + prefixed.send(JSON.stringify(beginMessage)); + + // Then the session opens at the prefixed URL + expect(await connecting).toMatchObject({ id: "sess-1" }); + const url = new URL((socket as { url: string }).url); + expect(`${url.protocol}//${url.host}${url.pathname}`).toBe(sessionUrl); + await session.close(false); + }, + ); + it("pipelines requests and reads them in flush id order", async () => { const session = client.tts.session({ websocketBaseUrl }); await connect(session); @@ -336,6 +402,45 @@ describe("tts session", () => { await session.close(); }); + it("Begin's max_generate_text_length limits generate() and the frames of speak()", async () => { + // Given a session whose Begin allows 10 characters per Generate frame + const session = client.tts.session({ websocketBaseUrl }); + await connect(session, { + ...beginMessage, + configuration: { + ...beginMessage.configuration, + limits: { max_generate_text_length: 10 }, + }, + }); + + // When the app speaks 25 characters + const text = "a".repeat(10) + "b".repeat(10) + "c".repeat(5); + + // Then speak() sends them in frames of at most 10, and they are spoken + expect((await collect(session.speak(text))).toString()).toBe(text); + expect(api.frames("Generate").map((frame) => frame.text)).toEqual([ + "a".repeat(10), + "b".repeat(10), + "c".repeat(5), + ]); + + // And generate() refuses 11 characters before sending them, and sends 10 + expect(() => session.generate("x".repeat(11))).toThrow( + new RangeError( + "text is 11 characters, over the 10-character limit of one Generate frame; split it across several generate() calls", + ), + ); + session.generate("y".repeat(10)); + await tick(); + expect(api.frames("Generate").map((frame) => frame.text)).toEqual([ + "a".repeat(10), + "b".repeat(10), + "c".repeat(5), + "y".repeat(10), + ]); + await session.close(); + }); + it("an Error frame before Begin rejects connect()", async () => { const session = client.tts.session({ websocketBaseUrl, @@ -453,6 +558,48 @@ describe("tts session", () => { expect(error.closeCode).toBe(1006); }); + it.each([ + [1011, true], + [1012, true], + [3005, true], + [3008, true], + [3009, true], + [1008, false], + [3010, false], + ])( + "a close %i without an Error frame has retryable %s, before Begin and after it", + async (code, retryable) => { + // Given a session whose first connection the API closes before Begin + const session = client.tts.session({ websocketBaseUrl }); + const errors: TextToSpeechError[] = []; + session.on("error", (error) => errors.push(error)); + const connecting = session.connect(); + (await connection(1)).close({ code, reason: "", wasClean: true }); + + // Then connect() rejects with the close code and whether to retry + const refused = await connecting.catch((error: unknown) => error); + expect(refused).toBeInstanceOf(TextToSpeechError); + expect(refused).toMatchObject({ closeCode: code, retryable }); + + // And when the API closes a second connection the same way after Begin + const reconnecting = session.connect(); + const second = await connection(2); + second.send(JSON.stringify(beginMessage)); + await reconnecting; + second.close({ code, reason: "", wasClean: true }); + await tick(); + + // Then the error listener gets the same verdict, with the session id + expect(errors).toHaveLength(2); + expect(errors[0]).toBe(refused); + expect(errors[1]).toMatchObject({ + closeCode: code, + retryable, + requestId: "sess-1", + }); + }, + ); + it("a token session reconnects with a new token, sent in the query alone", async () => { api.singleUseTokens = true; const session = client.tts.session({ @@ -540,6 +687,28 @@ describe("tts session", () => { await session.close(); }); + it("extendBiometricConsent is sent in the session URL", async () => { + const session = client.tts.session({ + websocketBaseUrl, + voiceId: "my-cloned-voice", + extendBiometricConsent: true, + }); + await connect(session); + const url = new URL((await connection(1)).url); + expect(url.searchParams.get("voice_id")).toBe("my-cloned-voice"); + expect(url.searchParams.get("extend_biometric_consent")).toBe("true"); + await session.close(); + }); + + it("keepAlive() sends a KeepAlive frame", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + session.keepAlive(); + await tick(); + expect(api.sent).toEqual([{ type: "KeepAlive" }]); + await session.close(); + }); + describe("the handshake's User-Agent", () => { it("is the client's, on both the API key and the token path", async () => { const custom = new AssemblyAI({ @@ -792,6 +961,79 @@ describe("tts session", () => { expect(closes).toEqual([[1000, "Session ended"]]); }); + it("close() speaks text not flushed yet and resolves after its FlushDone", async () => { + // Given a session with one request read and text sent after it, not flushed + const session = client.tts.session({ websocketBaseUrl }); + const events: string[] = []; + session.on("flushDone", (event) => + events.push(`flushDone ${event.flush_id}`), + ); + session.on("termination", () => events.push("termination")); + session.on("close", (code) => events.push(`close ${code}`)); + await connect(session); + expect((await collect(session.speak("One."))).toString()).toBe("One."); + session.generate("Trailing "); + session.generate("text."); + const reading = outcome(collect(session.audio())); // the request being written + + // When the app closes the session + const termination = await session.close(); + + // Then the API spoke the text as the next request before Termination, + // and close() resolved once the session ended + expect(termination).toEqual(terminationMessage); + expect(events).toEqual([ + "flushDone 0", + "flushDone 1", + "termination", + "close 1000", + ]); + expect(await reading).toEqual(Buffer.from("Trailing text.")); + expect(api.sent.map((frame) => frame.type)).toEqual([ + "Generate", + "Flush", + "Generate", + "Generate", + "Terminate", + ]); + }); + + it("the wordBoundaries listener receives each request's frame, the one close() flushes included", async () => { + // Given a session that asked for word boundaries + api.wordBoundaries = true; + const session = client.tts.session({ + websocketBaseUrl, + wordBoundaries: true, + }); + const received: TtsWordBoundariesEvent[] = []; + const events: string[] = []; + session.on("wordBoundaries", (event) => { + received.push(event); + events.push(`wordBoundaries ${event.flush_id}`); + }); + session.on("termination", () => events.push("termination")); + await connect(session); + const url = new URL((await connection(1)).url); + expect(url.searchParams.get("word_boundaries")).toBe("true"); + + // When it speaks one text, then closes with another not flushed + await collect(session.speak("Hello there.")); + session.generate("Goodbye now."); + await session.close(); + + // Then the listener receives both frames as the API sent them, before + // Termination + expect(received).toEqual([ + wordBoundariesFrame(0, "Hello there."), + wordBoundariesFrame(1, "Goodbye now."), + ]); + expect(events).toEqual([ + "wordBoundaries 0", + "wordBoundaries 1", + "termination", + ]); + }); + it.each([ { name: "waits for audio", chunkFirst: false }, { name: "is between chunks", chunkFirst: true }, @@ -1498,13 +1740,95 @@ describe("tts session", () => { expect(queue.size).toBe(0); }); + describe("audio nobody reads", () => { + const warning = + "TextToSpeechSession: dropping audio nobody reads; iterate audio() or add an audio listener"; + let warn: jest.SpyInstance; + + beforeEach(() => { + api.answerFlush = false; // the test sends each request's frames + warn = jest.spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => warn.mockRestore()); + + /** Each chunk of the iteration as text, or "pending" if it does not end. */ + function chunks(iterable: AsyncIterable) { + return outcome( + (async () => { + const texts: string[] = []; + for await (const chunk of iterable) { + texts.push(Buffer.from(chunk).toString()); + } + return texts; + })(), + ); + } + + /** Requests 0 and 1, flushed and not read yet. */ + async function twoRequests() { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + session.generate("A long answer. "); + expect(session.flush()).toBe(0); + session.generate("A short one."); + expect(session.flush()).toBe(1); + return session; + } + + function sendRequest(flushId: number, frames: number) { + for (let i = 0; i < frames; i++) { + server.send(audioFrame(`${flushId}:${i}`, flushId)); + } + server.send(JSON.stringify({ type: "FlushDone", flush_id: flushId })); + } + + it("keeps the newest 2,000 frames, warns once, and each iteration ends at its FlushDone", async () => { + // Given 2,500 frames of request 0 and 5 of request 1, each with its + // FlushDone, arriving before anything reads them + const session = await twoRequests(); + sendRequest(0, 2500); + sendRequest(1, 5); + + // When the app reads both requests afterwards + const first = await chunks(session.audio(0)); + const second = await chunks(session.audio(1)); + + // Then the oldest audio was dropped to hold 2,000 frames: request 0 + // keeps its newest 1,995 and request 1 all 5, and both iterations end + expect(first).toEqual( + Array.from({ length: 1995 }, (_, i) => `0:${i + 505}`), + ); + expect(second).toEqual(["1:0", "1:1", "1:2", "1:3", "1:4"]); + + // And one warning says why + expect(warn.mock.calls).toEqual([[warning]]); + await session.close(); + }); + + it("keeps a FlushDone at the front of a full queue, so its request's iteration still ends", async () => { + // Given request 0's 3 frames and FlushDone, then 2,001 frames of + // request 1: the FlushDone is the oldest item once request 0's audio + // was dropped + const session = await twoRequests(); + sendRequest(0, 3); + sendRequest(1, 2001); + + // When the app reads both requests afterwards + const first = await chunks(session.audio(0)); + const second = await chunks(session.audio(1)); + + // Then request 0 lost its audio and still ends, and request 1 keeps + // its newest 2,000 frames + expect(first).toEqual([]); + expect(second).toEqual( + Array.from({ length: 2000 }, (_, i) => `1:${i + 1}`), + ); + expect(warn.mock.calls).toEqual([[warning]]); + await session.close(); + }); + }); + describe("cancel() and audio still in transit", () => { - const audioFrame = (text: string, flushId: number) => - JSON.stringify({ - type: "Audio", - audio: Buffer.from(text).toString("base64"), - flush_id: flushId, - }); const cancelledFrame = JSON.stringify({ type: "Cancelled", audio_duration_seconds: 0.4, From 4f3dafa116a3cfea72f51dcf24e7ed713ae55482 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:20:10 +0000 Subject: [PATCH 19/26] docs(tts): stop sending after a barge-in, explain keepAlive(), mention text-to-speech in the package descriptions The README's token-streaming barge-in kept sending the LLM's tokens and then flush() after cancel(). The API reads frames in order, so that text became a new request: synthesized, billed and waited for by close(). The example now sets an interrupted flag before cancel(), stops the token loop (and the LLM stream) on it and flushes only when not interrupted. A unit test runs the example against the fake API and checks nothing follows the Cancel but Terminate. keepAlive() and inactivityTimeout now give the default (120 s), say the timer counts from the last frame in either direction, so a session can idle out while received audio still plays, and name the close (3006 inactivity_timeout, retryable). package.json's description and keywords, the README intro and CLAUDE.md mention text-to-speech, and the CLAUDE.md Key APIs entry is split into one bullet per API plus one for TextToSpeechError. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- CLAUDE.md | 8 +++-- README.md | 17 +++++++--- package.json | 3 +- src/services/tts/session.ts | 11 +++++-- src/types/tts/index.ts | 6 +++- tests/unit/tts-session.test.ts | 58 ++++++++++++++++++++++++++++++++++ 6 files changed, 91 insertions(+), 12 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ec71a0f..4b8c644 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ # AssemblyAI Node.js SDK -Speech-to-text and audio intelligence SDK for Node.js, Deno, and Bun. Supports pre-recorded transcription, real-time streaming, and audio analysis features. +Speech-to-text, text-to-speech and audio intelligence SDK for Node.js, Deno, and Bun. Supports pre-recorded transcription, real-time streaming, text-to-speech (HTTP requests and WebSocket sessions), and audio analysis features. ## Quick start @@ -52,7 +52,11 @@ const client = new AssemblyAI({ - `client.llmGateway.chatCompletions(request)` — OpenAI-compatible chat completions - `client.llmGateway.listModels()` — List models available on the LLM Gateway - `client.llmGateway.understanding(request)` / `client.llmGateway.validateUnderstanding(request)` — Run or validate a Speech Understanding request -- `client.tts.synthesize({ text, voice_id? })` — one text's complete audio (`SpeechResponse.audio`, a Uint8Array, plus header fields); `client.tts.stream(request)` — async iterable of audio chunks as generated; `client.tts.session({ voiceId })` — WebSocket `TextToSpeechSession`: `connect({ token? })`, `speak(text)` (async iterable), `generate()`/`flush()`/`audio(flushId?)`, `cancel()`, `keepAlive()`, `close()`, `on("audio"|"flushDone"|"wordBoundaries"|"cancelled"|"termination"|"error"|"close", fn)`. Errors: `TextToSpeechError` (`errorCode`, `param`, `retryable`, `retryAfter`, `requestId`, `closeCode`, `body`) +- `client.tts.synthesize({ text, voice_id? })` — Text-to-speech in one request: the complete audio (`SpeechResponse.audio`, a Uint8Array) plus the response header fields +- `client.tts.stream(request, { signal? })` — Text-to-speech audio as it is generated: a `TextToSpeechStream`, an async iterable of Uint8Array chunks with `info` (header fields), `body`, `bytes()` and `cancel()` +- `client.tts.session({ voiceId })` — WebSocket `TextToSpeechSession` that speaks text as it is written (voice agents): `connect({ token? })`, `speak(text)` (async iterable), `generate()`/`flush()`/`audio(flushId?)`, `cancel()`, `keepAlive()`, `close()`, `on("begin"|"audio"|"flushDone"|"wordBoundaries"|"cancelled"|"termination"|"error"|"close", fn)` +- `client.tts.createTemporaryToken({ expires_in_seconds })` — Mint a short-lived token, server-side, for a session opened in a browser with `new TextToSpeechSession({ token })`; one token per connection +- `TextToSpeechError` — What the `client.tts` methods throw on an API refusal: `errorCode`, `param`, `retryable`, `retryAfter`, `requestId`, `closeCode`, `body` ## Common patterns diff --git a/README.md b/README.md index 0ae0495..492de57 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ # AssemblyAI JavaScript SDK The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, -which supports async and streaming transcription. +which supports async and streaming transcription and text-to-speech. It is written primarily for Node.js in TypeScript with all types exported, but also [compatible with other runtimes](./docs/compat.md). ## Claude Code @@ -891,22 +891,29 @@ Streaming the text in as it is generated, with a barge-in. Audio starts before ` const session = client.tts.session({ voiceId: "aai-jane" }); await session.connect(); +let interrupted = false; const player = (async () => { for await (const chunk of session.audio()) { if (userStartedTalking()) { - session.cancel(); // ends this iteration at once; the server drops the rest + interrupted = true; + session.cancel(); // drops the text sent so far; text sent after it is a new request break; } play(chunk); } })(); -for await (const token of llmTokens()) session.generate(token); // joined as sent: include the spaces -session.flush(); +for await (const token of llmTokens()) { + if (interrupted) break; // stops the LLM stream too + session.generate(token); // joined as sent: include the spaces +} +if (!interrupted) session.flush(); await player; await session.close(); ``` -`audio()` fixes its request when it is called, so the iterator can be made before the first `generate()` and handed to a player that starts later. Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order, once; `audio()` throws a `RangeError` for an id not flushed yet or already read. Every `flush()` and `cancel()` takes the next id and returns it, so ids skip one after a cancel, and the requests a cancel retires are those up to its id. Audio of a retired request that the API sent before it read the `Cancel` can still arrive; the session drops it, so neither `audio()` nor the `audio` listener keeps playing the interrupted reply. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters (`limits.text_max_buffer_chars` in `Begin`), ending the session with a non-retryable 3010 past that, so pace text to the audio. `speak()` throws a `RangeError` for a longer text before sending any of it; split it and speak each part once the previous part's audio is read. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes after `inactivityTimeout` seconds (5 to 120, default 120) without a frame; call `keepAlive()` between sparse requests. +The API reads frames in the order sent, so a `cancel()` retires only the text sent before it. Stop sending the interrupted reply once the user barges in: text sent after the cancel is a new request, which is synthesized and billed, and which `close()` waits for. + +`audio()` fixes its request when it is called, so the iterator can be made before the first `generate()` and handed to a player that starts later. Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order, once; `audio()` throws a `RangeError` for an id not flushed yet or already read. Every `flush()` and `cancel()` takes the next id and returns it, so ids skip one after a cancel, and the requests a cancel retires are those up to its id. Audio of a retired request that the API sent before it read the `Cancel` can still arrive; the session drops it, so neither `audio()` nor the `audio` listener keeps playing the interrupted reply. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters (`limits.text_max_buffer_chars` in `Begin`), ending the session with a non-retryable 3010 past that, so pace text to the audio. `speak()` throws a `RangeError` for a longer text before sending any of it; split it and speak each part once the previous part's audio is read. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes with a retryable 3006 `inactivity_timeout` after `inactivityTimeout` seconds (5 to 120, default 120) without a frame in either direction. Audio arrives much faster than it plays, so the session can idle out while received audio is still playing: call `keepAlive()` at least once per timeout while the session waits. Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A listener that throws is logged with `console.error` and the session goes on; a frame the SDK cannot read fails the session with `Unreadable frame from the API` and closes the socket. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send (a send in the moment between a lost connection and its close event throws a retryable `TextToSpeechError` saying the connection is closing), reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `requestId` is the session id, and `status` is set only when the server answered the WebSocket upgrade with an HTTP status the runtime reports (Node does, a browser does not): a 404 for a wrong `websocketBaseUrl` path is not retryable, a 503 is. `retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. That works as is with an API key; a session opened with a temporary token needs a new token for each connection, passed as `connect({ token })`, because the API may refuse a token it has seen and always refuses an expired one (close code 1008, `errorCode: "unauthorized"`). The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. diff --git a/package.json b/package.json index d19a43e..f4507f2 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "assemblyai", "version": "4.42.0", - "description": "The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription.", + "description": "The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription and text-to-speech.", "engines": { "node": ">=18" }, @@ -106,6 +106,7 @@ "AssemblyAI", "Speech-to-text", "Transcription", + "Text-to-speech", "Audio", "LLM" ], diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index 8d8af4b..eaf7e74 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -831,9 +831,14 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c } /** - * Resets the inactivity timer between sparse requests. Once `close()` was - * called it does nothing, so a keepalive timer still running while the - * session drains does not throw. + * Restarts the API's idle timer (`inactivityTimeout`, 120 s by default). + * Send one at least once per timeout while the session waits, including + * while audio already received is still playing: audio arrives much faster + * than it plays, and the timer counts from the last frame in either + * direction. Otherwise the API closes the session with 3006 + * `inactivity_timeout` (retryable). Once `close()` was called it does + * nothing, so a keepalive timer still running while the session drains + * does not throw. */ keepAlive() { if (this.terminating) return; diff --git a/src/types/tts/index.ts b/src/types/tts/index.ts index 82cc3b9..b4c3225 100644 --- a/src/types/tts/index.ts +++ b/src/types/tts/index.ts @@ -200,7 +200,11 @@ export type TextToSpeechSessionParams = { encoding?: LiteralUnion, string>; /** Whether the server sends `wordBoundaries` events. Off by default here. */ wordBoundaries?: boolean; - /** Seconds (5 to 120, default 120) after which an idle session is closed. */ + /** + * Seconds (5 to 120, default 120) without a frame in either direction + * after which the API closes the session (3006 `inactivity_timeout`, + * retryable). See `keepAlive()`. + */ inactivityTimeout?: number; /** * `true` renews the biometric consent of the cloned voice in `voiceId`. diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index 1fce426..3295bdc 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -833,6 +833,64 @@ describe("tts session", () => { await session.close(); }); + it("the README's barge-in sends none of the interrupted reply after its Cancel", async () => { + // Given the README's token-streaming example, whose LLM is still + // writing when the user starts talking over the first audio + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const tokensAsked: string[] = []; + let llmStopped = false; + async function* llmTokens() { + try { + for (const token of ["Hello ", "there. ", "How ", "are ", "you?"]) { + tokensAsked.push(token); + yield token; + if (token === "there. ") { + server.send(audioFrame("Hello there. ", 0)); + await tick(); + } + } + } finally { + llmStopped = true; + } + } + const userStartedTalking = () => true; + const played: Uint8Array[] = []; + + // When it runs as written + let interrupted = false as boolean; + const player = (async () => { + for await (const chunk of session.audio()) { + if (userStartedTalking()) { + interrupted = true; + session.cancel(); + break; + } + played.push(chunk); + } + })(); + for await (const token of llmTokens()) { + if (interrupted) break; + session.generate(token); + } + if (!interrupted) session.flush(); + await player; + const summary = await session.close(); + + // Then the text after the Cancel is never sent, so no new request is + // synthesized and billed, and the LLM stream is stopped + expect(api.sent.map((f) => f.type)).toEqual([ + "Generate", + "Generate", + "Cancel", + "Terminate", + ]); + expect(tokensAsked).toEqual(["Hello ", "there. ", "How "]); + expect(llmStopped).toBe(true); + expect(played).toHaveLength(0); + expect(summary?.type).toBe("Termination"); + }); + it("refuses a second concurrent reader instead of losing audio", async () => { api.answerFlush = false; const session = client.tts.session({ websocketBaseUrl }); From 0d80cd76b4e7231cc6c98bd57ca1d7323941c946 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:24:21 +0000 Subject: [PATCH 20/26] refactor(tts): name the HTTP request and response types TextToSpeech* The package root also exports speech-to-text types (SpeechModel, SpeechUnderstandingRequest), so the generic Speech* names read as STT. Before the first release, rename them to match the rest of the product: SpeechRequest -> TextToSpeechRequest, SpeechRequestOptions -> TextToSpeechRequestOptions, SpeechResponse -> TextToSpeechResponse and SpeechResponseInfo -> TextToSpeechResponseInfo, plus the internal header reader. The exports test names the new types through the package root and checks synthesize() and stream() are declared with them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- CHANGELOG.md | 2 +- CLAUDE.md | 2 +- src/services/tts/service.ts | 30 ++++++++++++++++-------------- src/types/tts/index.ts | 8 ++++---- tests/unit/exports.test.ts | 33 ++++++++++++++++++++++++++++++++- 5 files changed, 54 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be2f737..8e3e5ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [4.42.0] -- Add Text-to-Speech: `client.tts.synthesize(request)` returns one text's complete audio (`SpeechResponse`: `audio` plus the response headers as `requestId`, `region`, `model`, `voiceId`, `encoding`, `sampleRate`, `billedCharacters`, `audioDurationMs`, ...), `client.tts.stream(request)` returns it as the API generates it (`TextToSpeechStream`, an async iterable over a web `ReadableStream`), and `client.tts.session(params)` opens a WebSocket `TextToSpeechSession` for voice agents: `connect()`, `generate()` / `flush()` / `audio()`, `speak()`, `cancel()`, `keepAlive()`, `close()`, with `on("audio" | "flushDone" | "wordBoundaries" | "cancelled" | "termination" | "error" | "close", ...)`. Requests take the API's fields (`text`, `voice_id`, `model_id`, `language`, `sample_rate`, `encoding`, `container`, `bit_rate`); failures throw `TextToSpeechError` (`status`, `errorCode`, `param`, `requestId`, `retryable`, `retryAfter`, `errors`, `closeCode`, `body`). New client option `ttsBaseUrl` (default `https://tts.us.assemblyai.com`); `client.tts.createTemporaryToken()` mints a token for browser sessions +- Add Text-to-Speech: `client.tts.synthesize(request)` returns one text's complete audio (`TextToSpeechResponse`: `audio` plus the response headers as `requestId`, `region`, `model`, `voiceId`, `encoding`, `sampleRate`, `billedCharacters`, `audioDurationMs`, ...), `client.tts.stream(request)` returns it as the API generates it (`TextToSpeechStream`, an async iterable over a web `ReadableStream`), and `client.tts.session(params)` opens a WebSocket `TextToSpeechSession` for voice agents: `connect()`, `generate()` / `flush()` / `audio()`, `speak()`, `cancel()`, `keepAlive()`, `close()`, with `on("audio" | "flushDone" | "wordBoundaries" | "cancelled" | "termination" | "error" | "close", ...)`. Requests take the API's fields (`text`, `voice_id`, `model_id`, `language`, `sample_rate`, `encoding`, `container`, `bit_rate`); failures throw `TextToSpeechError` (`status`, `errorCode`, `param`, `requestId`, `retryable`, `retryAfter`, `errors`, `closeCode`, `body`). New client option `ttsBaseUrl` (default `https://tts.us.assemblyai.com`); `client.tts.createTemporaryToken()` mints a token for browser sessions ## [4.41.7] diff --git a/CLAUDE.md b/CLAUDE.md index 4b8c644..f72c957 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,7 +52,7 @@ const client = new AssemblyAI({ - `client.llmGateway.chatCompletions(request)` — OpenAI-compatible chat completions - `client.llmGateway.listModels()` — List models available on the LLM Gateway - `client.llmGateway.understanding(request)` / `client.llmGateway.validateUnderstanding(request)` — Run or validate a Speech Understanding request -- `client.tts.synthesize({ text, voice_id? })` — Text-to-speech in one request: the complete audio (`SpeechResponse.audio`, a Uint8Array) plus the response header fields +- `client.tts.synthesize({ text, voice_id? })` — Text-to-speech in one request: the complete audio (`TextToSpeechResponse.audio`, a Uint8Array) plus the response header fields - `client.tts.stream(request, { signal? })` — Text-to-speech audio as it is generated: a `TextToSpeechStream`, an async iterable of Uint8Array chunks with `info` (header fields), `body`, `bytes()` and `cancel()` - `client.tts.session({ voiceId })` — WebSocket `TextToSpeechSession` that speaks text as it is written (voice agents): `connect({ token? })`, `speak(text)` (async iterable), `generate()`/`flush()`/`audio(flushId?)`, `cancel()`, `keepAlive()`, `close()`, `on("begin"|"audio"|"flushDone"|"wordBoundaries"|"cancelled"|"termination"|"error"|"close", fn)` - `client.tts.createTemporaryToken({ expires_in_seconds })` — Mint a short-lived token, server-side, for a session opened in a browser with `new TextToSpeechSession({ token })`; one token per connection diff --git a/src/services/tts/service.ts b/src/services/tts/service.ts index 9906cb0..303c1d8 100644 --- a/src/services/tts/service.ts +++ b/src/services/tts/service.ts @@ -1,9 +1,9 @@ import { BaseServiceParams, - SpeechRequest, - SpeechRequestOptions, - SpeechResponse, - SpeechResponseInfo, + TextToSpeechRequest, + TextToSpeechRequestOptions, + TextToSpeechResponse, + TextToSpeechResponseInfo, TextToSpeechSessionParams, TextToSpeechTokenParams, } from "../.."; @@ -200,7 +200,9 @@ function intHeader(headers: Headers, name: string): number | undefined { } /** Reads the `X-TTS-*` headers of a `200`. */ -export function speechResponseInfo(headers: Headers): SpeechResponseInfo { +export function textToSpeechResponseInfo( + headers: Headers, +): TextToSpeechResponseInfo { const unknownParams = headers .get("x-tts-unknown-params") ?.split(",") @@ -231,7 +233,7 @@ let warnedUnknownParams = false; * which is how a camelCase `sampleRate` in a request held in a variable (or * sent from plain JavaScript) shows up. The names stay on `unknownParams`. */ -function warnUnknownParams(info: SpeechResponseInfo) { +function warnUnknownParams(info: TextToSpeechResponseInfo) { if (warnedUnknownParams || !info.unknownParams) return; warnedUnknownParams = true; console.warn( @@ -260,7 +262,7 @@ export class TextToSpeechStream implements AsyncIterable { constructor( /** What the response headers say: model, voice, encoding, sample rate, request id. */ - public readonly info: SpeechResponseInfo, + public readonly info: TextToSpeechResponseInfo, /** The audio bytes as they arrive. */ public readonly body: ReadableStream, /** `signal`: the request's abort signal, whose abort ends the iteration with its reason. */ @@ -372,9 +374,9 @@ export class TextToSpeechService extends BaseService { * reason of `options.signal` (an `AbortError` unless you gave another). */ async synthesize( - request: SpeechRequest, - options: SpeechRequestOptions = {}, - ): Promise { + request: TextToSpeechRequest, + options: TextToSpeechRequestOptions = {}, + ): Promise { const deadline = requestDeadline( options.timeout ?? DEFAULT_SYNTHESIZE_TIMEOUT_MS, options.signal, @@ -387,7 +389,7 @@ export class TextToSpeechService extends BaseService { }); if (!response.ok) throw await speechErrorFromResponse(response); const audio = new Uint8Array(await response.arrayBuffer()); - const info = speechResponseInfo(response.headers); + const info = textToSpeechResponseInfo(response.headers); warnUnknownParams(info); return { ...info, audio }; } finally { @@ -418,8 +420,8 @@ export class TextToSpeechService extends BaseService { * `options.timeout` passes, or the reason of `options.signal`. */ async stream( - request: SpeechRequest, - options: SpeechRequestOptions = {}, + request: TextToSpeechRequest, + options: TextToSpeechRequestOptions = {}, ): Promise { // Checked before sending, so a runtime that could not hand the audio // over is not billed for it. @@ -452,7 +454,7 @@ export class TextToSpeechService extends BaseService { deadline.release(); } } - const info = speechResponseInfo(response.headers); + const info = textToSpeechResponseInfo(response.headers); warnUnknownParams(info); if (!response.body || typeof response.body.getReader !== "function") { // A runtime without web streamed bodies: hand the whole audio over as one chunk. diff --git a/src/types/tts/index.ts b/src/types/tts/index.ts index b4c3225..57cc848 100644 --- a/src/types/tts/index.ts +++ b/src/types/tts/index.ts @@ -77,7 +77,7 @@ export type TextToSpeechVoiceOptions = { }; /** The body of `POST /v1/speech`. */ -export type SpeechRequest = TextToSpeechVoiceOptions & { +export type TextToSpeechRequest = TextToSpeechVoiceOptions & { /** 1 to 5,000 characters. */ text: string; /** `raw` (the default) or `wav`, with a PCM or G.711 encoding only. */ @@ -91,7 +91,7 @@ export type SpeechRequest = TextToSpeechVoiceOptions & { }; /** Per-request options of the HTTP client. */ -export type SpeechRequestOptions = { +export type TextToSpeechRequestOptions = { /** * Milliseconds until the request is aborted with a `DOMException` named * `TimeoutError`. For `synthesize()` the whole request, 610 000 by default @@ -127,7 +127,7 @@ export type SpeechRequestOptions = { }; /** What the response headers of a `200` say about the audio. */ -export type SpeechResponseInfo = { +export type TextToSpeechResponseInfo = { /** The request's id (`X-Request-Id`). Quote it in support requests. */ requestId?: string; /** The region that served the request, such as `us-east-1`. */ @@ -170,7 +170,7 @@ export type SpeechResponseInfo = { }; /** The result of `synthesize()`: the whole audio, with its headers. */ -export type SpeechResponse = SpeechResponseInfo & { +export type TextToSpeechResponse = TextToSpeechResponseInfo & { /** The audio in the requested encoding and container, complete. */ audio: Uint8Array; }; diff --git a/tests/unit/exports.test.ts b/tests/unit/exports.test.ts index 5d713d1..973833a 100644 --- a/tests/unit/exports.test.ts +++ b/tests/unit/exports.test.ts @@ -1,10 +1,15 @@ import type { LiteralUnion, TextToSpeechErrorFields, + TextToSpeechRequest, + TextToSpeechRequestOptions, + TextToSpeechResponse, + TextToSpeechResponseInfo, + TextToSpeechStream, Transcript, TranscriptParams, } from "../../src/exports"; -import { TextToSpeechError } from "../../src/exports"; +import { AssemblyAI, TextToSpeechError } from "../../src/exports"; // #105: every type a public field is declared with must be reachable from the // package root. `LiteralUnion` sits behind `language_code` and @@ -42,4 +47,30 @@ describe("package root exports", () => { expect(carried.errorCode).toBe("rate_limited"); expect(carried.retryAfter).toBe(2); }); + + it("exposes the HTTP text-to-speech types under the product's prefix", () => { + // The root also exports speech-to-text types such as `SpeechModel` and + // `SpeechUnderstandingRequest`, so the types `synthesize()` and + // `stream()` are declared with carry the `TextToSpeech` prefix. + const client = new AssemblyAI({ apiKey: "key" }); + const synthesize: ( + request: TextToSpeechRequest, + options?: TextToSpeechRequestOptions, + ) => Promise = client.tts.synthesize.bind(client.tts); + const stream: ( + request: TextToSpeechRequest, + options?: TextToSpeechRequestOptions, + ) => Promise = client.tts.stream.bind(client.tts); + const infoOf = (speech: TextToSpeechStream): TextToSpeechResponseInfo => + speech.info; + const response: TextToSpeechResponse = { + audio: new Uint8Array(2), + requestId: "req-1", + }; + const info: TextToSpeechResponseInfo = response; + expect(typeof synthesize).toBe("function"); + expect(typeof stream).toBe("function"); + expect(typeof infoOf).toBe("function"); + expect(info.requestId).toBe("req-1"); + }); }); From 5ab14b67bd4d5518a7a395828fe630fde40a79ec Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 16:41:18 +0000 Subject: [PATCH 21/26] perf(tts): decode session audio with Buffer where it exists; README latency habits Audio frames decode with Node's or Bun's Buffer when one exists, read through globalThis so a browser bundle pulls in no polyfill. The bytes count only when they encode back to the frame's text exactly; any other text goes to the atob() loop, so the bytes and the refusals match atob() for every input and invalid base64 still fails the session as an unreadable frame. Chunks stay plain Uint8Arrays with an ArrayBuffer of their own. On Node 18 a 3.8 KB frame decodes in about 5 us instead of 240 us; on sandbox000 that cut the client CPU of a 6 s reply from 31.8 to 11.5 ms (p50, n=30 per arm) while text to first audio stayed the same (23.3 vs 23.1 ms). Node 20 and 22 were already fast. The TextDecoder for binary messages is made once, at the first one. README: connect the session before the reply and keep it, keepAlive() and replace it before expires_at, a spare browser token, Node fetch's 4 s idle keep-alive and undici's setGlobalDispatcher, and why Node needs no shared TLS context. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- README.md | 10 +++ src/services/tts/session.ts | 15 ++-- src/utils/base64.ts | 50 ++++++++++++ tests/unit/base64.test.ts | 140 +++++++++++++++++++++++++++++++++ tests/unit/tts-session.test.ts | 44 +++++++++++ 5 files changed, 250 insertions(+), 9 deletions(-) create mode 100644 src/utils/base64.ts create mode 100644 tests/unit/base64.test.ts diff --git a/README.md b/README.md index 492de57..e0873db 100644 --- a/README.md +++ b/README.md @@ -939,6 +939,16 @@ await session.connect(); Mint a new token for every connection, reconnects included, and for every HTTP request (a token works as the `apiKey` of a client made for that request), and keep `expires_in_seconds` short. `createTemporaryToken()` mints on the token host of the client's zone: `https://token.eu.assemblyai.com` when `ttsBaseUrl` is `https://tts.eu.assemblyai.com`, `https://token.us.assemblyai.com` for `https://tts.us.assemblyai.com` (the default), and `https://token.assemblyai.com` for any other host; pass `{ tokenBaseUrl }` as its second argument to choose the host. A token works in every zone. A refusal throws a `TextToSpeechError` whose message gives the token service's reason, such as `Invalid API key` or `expires_in_seconds: Input should be less than or equal to 600`. +### Keep the time to first audio low + +The wait for first audio is mostly synthesis on our servers, plus whatever your client does before the text reaches them. These habits take the client's part away from the moment a user is waiting: + +- **Connect the session before the reply is ready**, when the call starts or the page loads, and keep it for the whole conversation instead of opening one per turn. A turn on an open session skips the TCP, TLS and WebSocket handshakes and the API's checks before `Begin`: in the API's region that saved about 25 ms per turn, and about 75 ms on the first turn with a voice the API had not used recently, because the API loads the voice when the session opens. Farther away, each new session also costs three network round trips. Between turns call `keepAlive()` at least once per `inactivityTimeout`, and before `session.begin.expires_at` (at most an hour after `Begin`) open the next session in a pause and move the following turns to it. +- **In a browser, mint the next token before the page needs it**, for example when the pointer enters the text box after the idle session closed, so that opening the session does not wait for your server and the token service (about 32 ms per new session in our tests, before the round trip to your server). Mint it only when a session will have to be opened, use it for one connection, and drop it well before it expires. +- **On Node, keep HTTP connections between replies** if your voice agent calls `stream()` or `synthesize()` more than 4 seconds apart. Node's built-in `fetch` closes an idle connection after about 4 seconds when the server sends no `Keep-Alive` hint, and the API sends none, so a later reply opens a new TCP and TLS connection. Keep connections for 2 minutes with `setGlobalDispatcher(new Agent({ keepAliveTimeout: 120_000 }))` from the `undici` package, which applies to every `fetch` of the process. In the API's region the difference is too small to measure; farther away each new connection costs two network round trips. + +Node needs no shared TLS context for sessions: after its first TLS connection it keeps the parsed root certificates, so later connections set up their TLS context in about a tenth of a millisecond. + ## Contributing If you want to contribute to the JavaScript SDK, follow the guidelines in [CONTRIBUTING.md](./CONTRIBUTING.md). diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index eaf7e74..b3ca5a9 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -17,6 +17,7 @@ import { TtsWordBoundariesEvent, } from "../.."; import { RETRYABLE_STATUSES, TextToSpeechError } from "../../utils/errors/tts"; +import { base64ToBytes } from "../../utils/base64"; const DEFAULT_WEBSOCKET_BASE_URL = "wss://tts.us.assemblyai.com/v1/ws"; const DEFAULT_CONNECT_TIMEOUT_MS = 10_000; @@ -149,14 +150,6 @@ function codePointLength(text: string): number { return Array.from(text).length; } -/** @throws DOMException for text that is not base64, which makes its frame unreadable. */ -function base64ToBytes(data: string): Uint8Array { - const binary = atob(data); - const bytes = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); - return bytes; -} - /** A server frame after `Begin`, read into the event its listener receives. */ type SessionFrame = | TtsAudioEvent @@ -165,13 +158,17 @@ type SessionFrame = | TtsCancelledEvent | TtsTerminationEvent; +/** Decodes the binary messages; made at the first one. */ +let utf8: TextDecoder | undefined; + /** * The message, text or UTF-8 bytes, as a JSON object; `undefined` for * anything else, which carries nothing this SDK reads. */ function parseFrame(data: unknown): Record | undefined { if (typeof data !== "string" && !(data instanceof ArrayBuffer)) return; - const text = typeof data === "string" ? data : new TextDecoder().decode(data); + const text = + typeof data === "string" ? data : (utf8 ??= new TextDecoder()).decode(data); let frame: unknown; try { frame = JSON.parse(text); diff --git a/src/utils/base64.ts b/src/utils/base64.ts new file mode 100644 index 0000000..2de3a9b --- /dev/null +++ b/src/utils/base64.ts @@ -0,0 +1,50 @@ +/** The part of Node's and Bun's `Buffer` that decodes base64 into memory it is given. */ +type Base64Buffer = { + from(memory: ArrayBuffer): { + write(text: string, encoding: "base64"): number; + toString(encoding: "base64"): string; + }; +}; + +/** + * The bytes of a base64 text, as `atob()` reads it. + * + * Where a global `Buffer` exists (Node, Bun), its native decoder reads the + * text: on Node 18 the `atob()` loop costs about 240 µs for 3.8 KB of + * audio, the `Buffer` decode about 5 µs. The bytes have an `ArrayBuffer` of + * their own, never a slice of a shared pool. + * @throws DOMException for text that is not base64. + */ +export function base64ToBytes(text: string): Uint8Array { + return decodeWithBuffer(text) ?? decodeWithAtob(text); +} + +/** + * The bytes when `Buffer` decodes `text` the way `atob()` would; otherwise + * `undefined`, and `atob()` decodes or refuses it. `Buffer` skips what is not + * base64, reads the URL-safe alphabet and keeps only the low byte of other + * characters, so the bytes count only when they encode back to `text` + * exactly. `Buffer` is read from `globalThis` so that a browser bundler sees + * no reference to it and adds no polyfill. + */ +function decodeWithBuffer(text: string): Uint8Array | undefined { + const NodeBuffer = (globalThis as { Buffer?: Base64Buffer }).Buffer; + if (!NodeBuffer || text.length % 4 !== 0) return; + const padding = text.endsWith("==") ? 2 : text.endsWith("=") ? 1 : 0; + const bytes = new Uint8Array((text.length / 4) * 3 - padding); + try { + const view = NodeBuffer.from(bytes.buffer); + view.write(text, "base64"); + if (view.toString("base64") === text) return bytes; + } catch { + // A partial Buffer polyfill: atob() decodes instead. + } + return; +} + +function decodeWithAtob(text: string): Uint8Array { + const binary = atob(text); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); + return bytes; +} diff --git a/tests/unit/base64.test.ts b/tests/unit/base64.test.ts new file mode 100644 index 0000000..c8be4cc --- /dev/null +++ b/tests/unit/base64.test.ts @@ -0,0 +1,140 @@ +import { base64ToBytes } from "../../src/utils/base64"; + +/** What `atob()` makes of `text`: its bytes, or "refused". */ +function atobBytes(text: string): number[] | "refused" { + let binary: string; + try { + binary = atob(text); + } catch { + return "refused"; + } + return Array.from(binary, (char) => char.charCodeAt(0)); +} + +function decoded(text: string): number[] | "refused" { + try { + return Array.from(base64ToBytes(text)); + } catch { + return "refused"; + } +} + +/** Texts `Buffer` decodes differently from `atob()`, and canonical ones. */ +const edgeCases = [ + "", + "QQ==", + "QUI=", + "QUJD", + "QR==", // non-zero trailing bits: atob() ignores them + "QQ", // no padding: atob() accepts it + "QQ=", + "QQ==QQ==", + "====", + "A===", + "AB=C", + "QUJ-", + "QUJ_", + "QUJń", // Buffer reads U+0144 as "D" + "QUJÄ", + "QUJ\u{1f600}", + "QU JE", + "QUJD\nRA==", + " QUJD", + "not base64!", +]; + +/** Random texts over base64, padding, whitespace and other characters. */ +function randomTexts(count: number): string[] { + const alphabet = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + + "==== \n-_!Äń"; + let seed = 7; + const random = () => { + seed = (seed * 1103515245 + 12345) % 2 ** 31; + return seed / 2 ** 31; + }; + const texts: string[] = []; + for (let n = 0; n < count; n++) { + const length = Math.floor(random() * 13); + let text = ""; + for (let i = 0; i < length; i++) { + text += alphabet[Math.floor(random() * alphabet.length)]; + } + texts.push(text); + } + return texts; +} + +/** Bytes of every length remainder, the sizes of real audio frames included. */ +const audioSizes = [0, 1, 2, 3, 4, 5, 100, 3839, 3840, 3841, 7680, 19200]; + +function audio(size: number): Uint8Array { + const bytes = new Uint8Array(size); + for (let i = 0; i < size; i++) bytes[i] = (i * 2654435761) >>> 24; + return bytes; +} + +describe("base64ToBytes", () => { + const nodeBuffer = globalThis.Buffer; + afterEach(() => { + globalThis.Buffer = nodeBuffer; + jest.restoreAllMocks(); + }); + + describe.each([ + { runtime: "with Buffer (Node, Bun)", buffer: nodeBuffer }, + { runtime: "without Buffer (a browser)", buffer: undefined }, + ])("$runtime", ({ buffer }) => { + beforeEach(() => { + (globalThis as { Buffer?: unknown }).Buffer = buffer; + }); + + it("reads every text as atob() does, refusing what it refuses", () => { + const texts = [...edgeCases, ...randomTexts(5000)]; + const differ = texts.filter( + (text) => + JSON.stringify(decoded(text)) !== JSON.stringify(atobBytes(text)), + ); + expect(differ).toEqual([]); + }); + + it.each(audioSizes)( + "decodes %i bytes into an ArrayBuffer of their own", + (size) => { + const bytes = audio(size); + const text = nodeBuffer.from(bytes).toString("base64"); + const result = base64ToBytes(text); + expect(result.constructor).toBe(Uint8Array); // not a Buffer + expect(nodeBuffer.from(result).toString("hex")).toBe( + nodeBuffer.from(bytes).toString("hex"), + ); + expect(result.byteOffset).toBe(0); + expect(result.buffer.byteLength).toBe(size); + }, + ); + }); + + it("decodes canonical base64 with Buffer, not the atob() loop", () => { + const atob = jest.spyOn(globalThis, "atob"); + const bytes = audio(3840); + expect(base64ToBytes(nodeBuffer.from(bytes).toString("base64"))).toEqual( + bytes, + ); + expect(atob).not.toHaveBeenCalled(); + }); + + it("leaves text Buffer reads differently to atob()", () => { + const atob = jest.spyOn(globalThis, "atob"); + expect(() => base64ToBytes("QUJ-")).toThrow(); + expect(Array.from(base64ToBytes("QR=="))).toEqual([0x41]); + expect(atob).toHaveBeenCalledTimes(2); + }); + + it("falls back to atob() under a Buffer polyfill that cannot decode", () => { + (globalThis as { Buffer?: unknown }).Buffer = { + from: () => ({}), + }; + expect(Array.from(base64ToBytes("QUJD"))).toEqual([0x41, 0x42, 0x43]); + expect(() => base64ToBytes("QUJ-")).toThrow(); + }); +}); diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index 3295bdc..f158f60 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -1295,6 +1295,15 @@ describe("tts session", () => { name: "audio that is not base64", frame: { type: "Audio", audio: "not base64!", flush_id: 0 }, }, + { + // Node's Buffer decodes both without an error. + name: "audio in the URL-safe alphabet", + frame: { type: "Audio", audio: "QUJ-QUJD", flush_id: 0 }, + }, + { + name: "audio with a character past U+00FF", + frame: { type: "Audio", audio: "QUJ\u0144", flush_id: 0 }, + }, { name: "an Audio frame without audio", frame: { type: "Audio", flush_id: 0 }, @@ -1350,6 +1359,41 @@ describe("tts session", () => { await session.close(); }); + it("reads frames sent as binary messages, UTF-8 JSON", async () => { + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const flushId = session.flush(); + const reading = collect(session.audio(flushId)); + await tick(); + const binary = (frame: string) => { + const bytes = new TextEncoder().encode(frame); + return bytes.buffer.slice( + bytes.byteOffset, + bytes.byteOffset + bytes.byteLength, + ); + }; + for (const text of ["Grüße, ", "señor."]) { + api.sockets[0].send(binary(audioFrame(text, flushId)) as never); + } + api.sockets[0].send( + binary(JSON.stringify({ type: "FlushDone", flush_id: flushId })) as never, + ); + expect((await reading).toString()).toBe("Grüße, señor."); + await session.close(); + }); + + it("audio chunks own their memory", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + for await (const chunk of session.speak("Hello there.")) { + expect(chunk.constructor).toBe(Uint8Array); + expect(chunk.byteOffset).toBe(0); + expect(chunk.buffer.byteLength).toBe(chunk.byteLength); + } + await session.close(); + }); + it("a JSON message that is not an object is ignored", async () => { const session = client.tts.session({ websocketBaseUrl }); await connect(session); From 34771a361daa4b1b8b4d1a4e5acbf394c7b367d4 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:16:30 +0000 Subject: [PATCH 22/26] perf(tts): send session text while the socket opens instead of after Begin The API reads frames sent before Begin once it has sent Begin, but the session refused every send until connect() resolved, so a session opened for a reply sent its text one round trip after the API's checks instead of during them. generate(), flush(), speak(), cancel() and keepAlive() called while the socket is still opening are now held and sent, in order, the moment it opens. Until Begin reports limits, the API's defaults apply (2,000 characters per Generate, 8,000 unspoken characters for speak()). A connection that fails or is closed before it opens drops the held frames; an audio() iteration then rejects with the connection's error (or ends, after close()). Measured against sandbox000 with a per-turn connect (n=30 per arm, interleaved): with 60 ms of round trip added through a delay relay, first audio came 13.8 ms sooner at p50 [95% CI -19.2, -10.3]; in region the change is within noise on Node 18/20/22, Bun and the browser bundle. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- CHANGELOG.md | 2 +- CLAUDE.md | 2 +- README.md | 13 ++- src/services/tts/session.ts | 53 +++++++++- tests/unit/tts-session.test.ts | 177 +++++++++++++++++++++++++++++++-- 5 files changed, 233 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e3e5ad..afb1bfb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [4.42.0] -- Add Text-to-Speech: `client.tts.synthesize(request)` returns one text's complete audio (`TextToSpeechResponse`: `audio` plus the response headers as `requestId`, `region`, `model`, `voiceId`, `encoding`, `sampleRate`, `billedCharacters`, `audioDurationMs`, ...), `client.tts.stream(request)` returns it as the API generates it (`TextToSpeechStream`, an async iterable over a web `ReadableStream`), and `client.tts.session(params)` opens a WebSocket `TextToSpeechSession` for voice agents: `connect()`, `generate()` / `flush()` / `audio()`, `speak()`, `cancel()`, `keepAlive()`, `close()`, with `on("audio" | "flushDone" | "wordBoundaries" | "cancelled" | "termination" | "error" | "close", ...)`. Requests take the API's fields (`text`, `voice_id`, `model_id`, `language`, `sample_rate`, `encoding`, `container`, `bit_rate`); failures throw `TextToSpeechError` (`status`, `errorCode`, `param`, `requestId`, `retryable`, `retryAfter`, `errors`, `closeCode`, `body`). New client option `ttsBaseUrl` (default `https://tts.us.assemblyai.com`); `client.tts.createTemporaryToken()` mints a token for browser sessions +- Add Text-to-Speech: `client.tts.synthesize(request)` returns one text's complete audio (`TextToSpeechResponse`: `audio` plus the response headers as `requestId`, `region`, `model`, `voiceId`, `encoding`, `sampleRate`, `billedCharacters`, `audioDurationMs`, ...), `client.tts.stream(request)` returns it as the API generates it (`TextToSpeechStream`, an async iterable over a web `ReadableStream`), and `client.tts.session(params)` opens a WebSocket `TextToSpeechSession` for voice agents: `connect()`, `generate()` / `flush()` / `audio()`, `speak()`, `cancel()`, `keepAlive()`, `close()`, with `on("audio" | "flushDone" | "wordBoundaries" | "cancelled" | "termination" | "error" | "close", ...)`. Requests take the API's fields (`text`, `voice_id`, `model_id`, `language`, `sample_rate`, `encoding`, `container`, `bit_rate`); failures throw `TextToSpeechError` (`status`, `errorCode`, `param`, `requestId`, `retryable`, `retryAfter`, `errors`, `closeCode`, `body`). New client option `ttsBaseUrl` (default `https://tts.us.assemblyai.com`); `client.tts.createTemporaryToken()` mints a token for browser sessions. Text sent while `connect()` is pending is held and sent the moment the socket opens, so it need not wait for `Begin` ## [4.41.7] diff --git a/CLAUDE.md b/CLAUDE.md index f72c957..7084cc3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,7 +54,7 @@ const client = new AssemblyAI({ - `client.llmGateway.understanding(request)` / `client.llmGateway.validateUnderstanding(request)` — Run or validate a Speech Understanding request - `client.tts.synthesize({ text, voice_id? })` — Text-to-speech in one request: the complete audio (`TextToSpeechResponse.audio`, a Uint8Array) plus the response header fields - `client.tts.stream(request, { signal? })` — Text-to-speech audio as it is generated: a `TextToSpeechStream`, an async iterable of Uint8Array chunks with `info` (header fields), `body`, `bytes()` and `cancel()` -- `client.tts.session({ voiceId })` — WebSocket `TextToSpeechSession` that speaks text as it is written (voice agents): `connect({ token? })`, `speak(text)` (async iterable), `generate()`/`flush()`/`audio(flushId?)`, `cancel()`, `keepAlive()`, `close()`, `on("begin"|"audio"|"flushDone"|"wordBoundaries"|"cancelled"|"termination"|"error"|"close", fn)` +- `client.tts.session({ voiceId })` — WebSocket `TextToSpeechSession` that speaks text as it is written (voice agents): `connect({ token? })` (text sent before it resolves is held until the socket opens), `speak(text)` (async iterable), `generate()`/`flush()`/`audio(flushId?)`, `cancel()`, `keepAlive()`, `close()`, `on("begin"|"audio"|"flushDone"|"wordBoundaries"|"cancelled"|"termination"|"error"|"close", fn)` - `client.tts.createTemporaryToken({ expires_in_seconds })` — Mint a short-lived token, server-side, for a session opened in a browser with `new TextToSpeechSession({ token })`; one token per connection - `TextToSpeechError` — What the `client.tts` methods throw on an API refusal: `errorCode`, `param`, `retryable`, `retryAfter`, `requestId`, `closeCode`, `body` diff --git a/README.md b/README.md index e0873db..db0fc28 100644 --- a/README.md +++ b/README.md @@ -871,7 +871,7 @@ For a barge-in, pass `{ signal }` as the second argument and abort it: the downl ### Speak text as it is written (voice agents) -A `TextToSpeechSession` keeps one WebSocket open and speaks text as it is written, so an LLM's reply can be voiced sentence by sentence with no per-request handshake. `speak()` sends one text and returns its audio; `generate()` / `flush()` do the same in two steps, so tokens can be sent as they arrive; `cancel()` is the barge-in. `close()` ends the session and resolves with its summary once the API has spoken the text still pending. It stops waiting after `terminateTimeout` milliseconds (default 35,000) without a message from the API; an `audio()` iteration still running then rejects with a retryable `TextToSpeechError`, and `close()` resolves `undefined`. Calling `close()` again, or from two places at once, waits for the same end. Once `close()` has sent `Terminate` the API reads nothing more, so `generate()`, `flush()`, `cancel()` and `speak()` throw instead of sending text that would never be spoken, and `keepAlive()` does nothing. +A `TextToSpeechSession` keeps one WebSocket open and speaks text as it is written, so an LLM's reply can be voiced sentence by sentence with no per-request handshake. `speak()` sends one text and returns its audio; `generate()` / `flush()` do the same in two steps, so tokens can be sent as they arrive; `cancel()` is the barge-in. `close()` ends the session and resolves with its summary once the API has spoken the text still pending. It stops waiting after `terminateTimeout` milliseconds (default 35,000) without a message from the API; an `audio()` iteration still running then rejects with a retryable `TextToSpeechError`, and `close()` resolves `undefined`. Calling `close()` again, or from two places at once, waits for the same end. Once `close()` has sent `Terminate` the API reads nothing more, so `generate()`, `flush()`, `cancel()` and `speak()` throw instead of sending text that would never be spoken, and `keepAlive()` does nothing. Text need not wait for `connect()` to resolve: what is sent while the socket opens is held and sent the moment it opens (see [Keep the time to first audio low](#keep-the-time-to-first-audio-low)). ```typescript const session = client.tts.session({ voiceId: "aai-jane" }); @@ -944,6 +944,17 @@ Mint a new token for every connection, reconnects included, and for every HTTP r The wait for first audio is mostly synthesis on our servers, plus whatever your client does before the text reaches them. These habits take the client's part away from the moment a user is waiting: - **Connect the session before the reply is ready**, when the call starts or the page loads, and keep it for the whole conversation instead of opening one per turn. A turn on an open session skips the TCP, TLS and WebSocket handshakes and the API's checks before `Begin`: in the API's region that saved about 25 ms per turn, and about 75 ms on the first turn with a voice the API had not used recently, because the API loads the voice when the session opens. Farther away, each new session also costs three network round trips. Between turns call `keepAlive()` at least once per `inactivityTimeout`, and before `session.begin.expires_at` (at most an hour after `Begin`) open the next session in a pause and move the following turns to it. +- **When a session must be opened for a reply, send the text without waiting for `Begin`.** Text sent while `connect()` is still pending is held and sent the moment the socket opens, and the API reads it as soon as it has sent `Begin`, so the text travels while the API checks the session instead of after: + + ```typescript + const connecting = session.connect(); + const audio = session.speak(reply); // sent as soon as the socket opens + await connecting; // a refused session rejects here + for await (const chunk of audio) play(chunk); + ``` + + This saves up to one network round trip, and never more than the API's checks take. With 60 ms of round trip added between our test client and the API's region, first audio came 14 ms sooner (median of 30 sessions per arm); inside the region the difference was too small to measure. Keep and await `connect()`'s promise all the same: a refused session rejects it, and a rejection nothing handles ends a Node process. + - **In a browser, mint the next token before the page needs it**, for example when the pointer enters the text box after the idle session closed, so that opening the session does not wait for your server and the token service (about 32 ms per new session in our tests, before the round trip to your server). Mint it only when a session will have to be opened, use it for one connection, and drop it well before it expires. - **On Node, keep HTTP connections between replies** if your voice agent calls `stream()` or `synthesize()` more than 4 seconds apart. Node's built-in `fetch` closes an idle connection after about 4 seconds when the server sends no `Keep-Alive` hint, and the API sends none, so a later reply opens a new TCP and TLS connection. Keep connections for 2 minutes with `setGlobalDispatcher(new Agent({ keepAliveTimeout: 120_000 }))` from the `undici` package, which applies to every `fetch` of the process. In the API's region the difference is too small to measure; farther away each new connection costs two network round trips. diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index b3ca5a9..c8e992b 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -29,6 +29,10 @@ const DEFAULT_CONNECT_TIMEOUT_MS = 10_000; const DEFAULT_TERMINATE_TIMEOUT_MS = 35_000; /** Characters one `Generate` frame may carry, unless `Begin` says otherwise. */ const MAX_GENERATE_CHARS = 2000; +/** The unspoken characters the API holds by default, the limit before `Begin` reports one. */ +const DEFAULT_TEXT_MAX_BUFFER_CHARS = 8000; +/** `readyState` of a WebSocket still opening, the same in every runtime. */ +const CONNECTING = 0; /** Audio frames queued for pulling before the oldest is dropped (~160 s). */ const PULL_QUEUE_FRAMES = 2000; /** Closes after which a new session can succeed. */ @@ -268,6 +272,8 @@ function errorFromClose(code: number, reason: string): TextToSpeechError { * `connect()` opens the socket and waits for `Begin`. Send text with * `generate()` as it is produced, conclude each request with `flush()`, and * read its audio with `audio()`; `speak()` does all three for one text. + * Text need not wait for `connect()` to resolve: sent while the socket opens, + * it is held and sent the moment it opens. * `cancel()` is the barge-in: it stops the request in flight and ends its * `audio()` iteration at once. `close()` sends `Terminate` and waits for * the session's `Termination` summary. @@ -299,6 +305,8 @@ export class TextToSpeechSession { private socket?: PolyfillWebSocket; private listeners: TextToSpeechSessionListeners = {}; private queue = new PullQueue(); + /** Frames sent while the socket was opening, sent in order once it opens. */ + private pending: string[] = []; private nextFlushId = 0; private cancelIds: number[] = []; private errorRaised = false; @@ -391,6 +399,7 @@ export class TextToSpeechSession { private reset() { this.queue = new PullQueue(); + this.pending = []; this.nextFlushId = 0; this.cancelIds = []; this.errorRaised = false; @@ -403,6 +412,18 @@ export class TextToSpeechSession { /** * Opens the socket and waits for `Begin`. * + * Frames need not wait for `Begin`: `generate()`, `flush()`, `speak()`, + * `cancel()` and `keepAlive()` called while `connect()` is pending are + * held and sent, in order, the moment the socket opens, and the API reads + * them once it has sent `Begin`. A session opened for a reply then gets + * its first audio sooner: the text travels while the API runs its checks, + * which saves up to one network round trip. Until `Begin` arrives the + * API's default limits apply (2,000 characters per `Generate`, 8,000 + * unspoken characters for `speak()`). If the connection fails, or + * `close()` is called, before the socket opens, the held frames are + * dropped and an `audio()` iteration rejects with the connection's error + * (or ends, after `close()`). + * * A session the API has ended can be connected again with the same object. * A temporary token opens one connection, so a session opened with a token * needs a new one for each connection, reconnects included: mint it on @@ -524,7 +545,7 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c this.socket = socket; connection.hadSocket = true; socket.binaryType = "arraybuffer"; - socket.onopen = () => {}; + socket.onopen = () => this.sendPending(socket); socket.onmessage = ({ data }: MessageEvent) => { connection.heardAt = Date.now(); @@ -673,6 +694,7 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c const connection = this.queue; if (connection.ended) return; connection.ended = true; + this.pending = []; connection.error = this.error ?? cut; connection.termination = this.termination; this.enqueue({ kind: "closed", error: connection.error }); @@ -683,8 +705,10 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c private releaseSocket() { const socket = this.socket; this.socket = undefined; + this.pending = []; if (!socket) return; socket.removeAllListeners?.(); + socket.onopen = () => {}; socket.onerror = () => {}; socket.onclose = () => {}; socket.onmessage = () => {}; @@ -711,6 +735,24 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c } /** + * Sends the frames held while `socket` was opening, in order. The API reads + * them once it has sent `Begin`. + */ + private sendPending(socket: PolyfillWebSocket) { + const frames = this.pending; + this.pending = []; + if (socket !== this.socket || this.stopped) return; + try { + for (const frame of frames) socket.send(frame); + } catch { + // the socket closed under us; the close handler ends the session + } + } + + /** + * Sends `frame`, or holds it while the socket is still opening: the API + * reads frames sent before `Begin` once it has sent `Begin`, so text need + * not wait for it. * @throws Error once `close()` has sent `Terminate`, since the API reads * no frame after it, and when the session is not connected. * @throws TextToSpeechError the error that ended the session; or, while @@ -723,9 +765,14 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c throw new Error("The session is closing: close() was called"); } if (socket && !this.stopped && socket.readyState === socket.OPEN) { + if (this.pending.length > 0) this.sendPending(socket); socket.send(JSON.stringify(frame)); return; } + if (socket && !this.stopped && socket.readyState === CONNECTING) { + this.pending.push(JSON.stringify(frame)); + return; + } if (this.error) throw this.surfaced(); if (socket && this.begin && !this.stopped) { throw new TextToSpeechError("The connection is closing", { @@ -749,9 +796,11 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c /** * The unspoken characters the API holds before it ends the session with - * 3010, as `Begin` reports it; `0` for no limit. + * 3010, as `Begin` reports it (the API's default before `Begin`); `0` for + * no limit. */ private backlogLimit(): number { + if (!this.begin) return DEFAULT_TEXT_MAX_BUFFER_CHARS; const limits = this.limits(); if (limits?.text_throttle_enabled === false) return 0; const limit = limits?.text_max_buffer_chars; diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index f158f60..f3827c2 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -803,20 +803,16 @@ describe("tts session", () => { expect(() => session.generate("Hi.")).toThrow(/connect\(\)/); }); - it("a send while the socket is still opening is not reported as a lost connection", async () => { + it("a send while the socket is still opening is held, not reported as a lost connection", async () => { + api.answerFlush = false; const session = client.tts.session({ websocketBaseUrl }); const connecting = session.connect(); - let thrown: unknown; - try { - session.generate("Hi."); - } catch (error) { - thrown = error; - } - expect(thrown).not.toBeInstanceOf(TextToSpeechError); - expect(thrown).toMatchObject({ message: /connect\(\)/ }); + expect(() => session.generate("Hi.")).not.toThrow(); await server.connected; server.send(JSON.stringify(beginMessage)); await connecting; + await tick(); + expect(api.frames("Generate")).toEqual([{ type: "Generate", text: "Hi." }]); await session.close(); }); @@ -2227,4 +2223,167 @@ describe("tts session", () => { }, ); }); + + describe("text sent before Begin", () => { + type DrivenSocket = { + OPEN: number; + readyState: number; + binaryType: string; + onopen: (() => void) | null; + onerror: ((event: unknown) => void) | null; + onclose: ((event: unknown) => void) | null; + onmessage: ((event: unknown) => void) | null; + send: jest.Mock; + close: jest.Mock; + }; + + /** Hands the session a socket the test opens or fails, as the runtime would. */ + function drivenSocket(): DrivenSocket { + const socket: DrivenSocket = { + OPEN: 1, + readyState: 0, // CONNECTING + binaryType: "blob", + onopen: null, + onerror: null, + onclose: null, + onmessage: null, + send: jest.fn(), + close: jest.fn(), + }; + jest + .spyOn(websocketPolyfill, "factory") + .mockReturnValueOnce( + socket as unknown as ReturnType, + ); + return socket; + } + + const sentFrames = (socket: DrivenSocket) => + socket.send.mock.calls.map(([data]) => JSON.parse(data as string)); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + it("holds text sent while the socket opens and sends it in order, before Begin", async () => { + // Given a session whose connect() has not resolved yet + api.answerFlush = false; + const session = client.tts.session({ websocketBaseUrl }); + const connecting = session.connect(); + + // When the app sends text right away + session.generate("Hello "); + const audio = session.speak("early."); + + // Then nothing is sent before the socket opens, and everything is sent, + // in order, once it opens, without waiting for Begin + expect(api.sent).toEqual([]); + await server.connected; + await tick(); + expect(api.sent).toEqual([ + { type: "Generate", text: "Hello " }, + { type: "Generate", text: "early." }, + { type: "Flush" }, + ]); + expect(session.begin).toBeUndefined(); + + // And the audio the API sends after Begin reaches the iterator made before it + server.send(JSON.stringify(beginMessage)); + await expect(connecting).resolves.toMatchObject({ id: "sess-1" }); + server.send(audioFrame("Hello early.", 0)); + server.send( + JSON.stringify({ + type: "FlushDone", + flush_id: 0, + audio_duration_ms: 80, + }), + ); + expect((await collect(audio)).toString()).toBe("Hello early."); + await session.close(); + }); + + it("keeps the order of frames held and frames sent once the socket is open", async () => { + // Given text held while the socket opened + const socket = drivenSocket(); + const session = client.tts.session({ websocketBaseUrl }); + void session.connect().catch(() => undefined); + expect(session.speak("a".repeat(4500))).toBeDefined(); + expect(session.cancel()).toBe(1); + + // When the socket is open and the app sends before its open event ran + socket.readyState = 1; + session.keepAlive(); + + // Then the held frames go first, split as speak() splits, then the new one + expect(sentFrames(socket)).toEqual([ + { type: "Generate", text: "a".repeat(2000) }, + { type: "Generate", text: "a".repeat(2000) }, + { type: "Generate", text: "a".repeat(500) }, + { type: "Flush" }, + { type: "Cancel" }, + { type: "KeepAlive" }, + ]); + + // And the open event sends nothing twice + socket.onopen?.(); + expect(socket.send).toHaveBeenCalledTimes(6); + await session.close(false); + }); + + it("speak() before Begin refuses text over the API's default backlog", async () => { + // Given a session still connecting, so no Begin reports the backlog + const socket = drivenSocket(); + const session = client.tts.session({ websocketBaseUrl }); + void session.connect().catch(() => undefined); + + // When the app speaks more than the API's default 8,000 characters + // Then speak() throws before holding any of it + expect(() => session.speak("a".repeat(8001))).toThrow( + /^text is 8001 characters, over the 8000 unspoken characters/, + ); + socket.readyState = 1; + socket.onopen?.(); + expect(socket.send).not.toHaveBeenCalled(); + await session.close(false); + }); + + it("a connection that fails before it opens drops the held text and fails its audio", async () => { + // Given text held while the socket opens + const socket = drivenSocket(); + const session = client.tts.session({ websocketBaseUrl }); + const connecting = session.connect(); + const audio = session.speak("Never sent."); + + // When the socket fails before it opens + socket.onerror?.({ type: "error" }); + + // Then connect() and the audio reject with the same error, and nothing is sent + const error = await connecting.catch((e: unknown) => e); + expect(error).toBeInstanceOf(TextToSpeechError); + await expect(collect(audio)).rejects.toBe(error); + socket.readyState = 1; + socket.onopen?.(); + expect(socket.send).not.toHaveBeenCalled(); + }); + + it("close() while the socket opens drops the held text", async () => { + // Given text held while the socket opens + const socket = drivenSocket(); + const session = client.tts.session({ websocketBaseUrl }); + const connecting = session.connect(); + const audio = session.speak("Dropped."); + + // When the app closes the session before it began + await expect(session.close()).resolves.toBeUndefined(); + + // Then connect() rejects, the audio ends, and the held text is never sent + await expect(connecting).rejects.toThrow( + "The session was closed before it began", + ); + expect((await collect(audio)).byteLength).toBe(0); + socket.readyState = 1; + socket.onopen?.(); + expect(socket.send).not.toHaveBeenCalled(); + }); + }); }); From 75236dae3a36d7f5479d2d588a6ef9335c8724b0 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:25:26 +0000 Subject: [PATCH 23/26] perf(tts): turn Nagle's algorithm off before the session's upgrade request The ws package turns TCP_NODELAY on only once the socket is open. Under Deno's Node compatibility the upgrade request before that waits for the server's delayed ACK in most handshakes, about 40 ms. The session now passes noDelay: true to the ws socket on every runtime but the browser, whose WebSocket takes no options. Measured against sandbox000 with a per-turn connect (n=30 per arm, interleaved, in region): on Deno 2.9.7 connect to Begin went from 60.7 to 19.2 ms at p50 (-41.5 [95% CI -42.6, -40.5]) and first audio from 88.9 to 54.8 ms (-34.1 [-41.7, -16.4]); with the text sent before Begin, first audio -42.0 [-44.9, -38.7]. Node 18 and 22 and Bun (whose ws ignores the option) show no change at p50. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- src/services/tts/session.ts | 13 +++++++++++-- tests/unit/tts-session.test.ts | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index c8e992b..e41dac8 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -525,11 +525,20 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c if (this.internal.userAgent && !conditions.browser) { headers["User-Agent"] = this.internal.userAgent; } + const options: { headers?: Record; noDelay?: boolean } = + {}; + if (Object.keys(headers).length > 0) options.headers = headers; + // Nagle's algorithm off before the upgrade request is written, not only + // once the socket is open as `ws` does. Under Deno's Node compatibility + // the request otherwise waits about 40 ms in most handshakes (connect to + // Begin 60 ms instead of 19 ms in our tests). A browser's WebSocket + // takes no options. + if (!conditions.browser) options.noDelay = true; let socket: PolyfillWebSocket; try { socket = - Object.keys(headers).length > 0 - ? polyfillWebSocketFactory(url.toString(), { headers }) + Object.keys(options).length > 0 + ? polyfillWebSocketFactory(url.toString(), options) : polyfillWebSocketFactory(url.toString()); } catch (cause) { // The runtime refused to create the socket (a browser's WebSocket diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index f3827c2..1410b85 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -615,6 +615,7 @@ describe("tts session", () => { // No Authorization header: it would override the token. expect(first.clientOptions).toEqual({ headers: { "User-Agent": sdkUserAgent }, + noDelay: true, }); expect((await collect(session.speak("One."))).toString()).toBe("One."); await session.close(); @@ -673,6 +674,7 @@ describe("tts session", () => { expect(new URL(first.url).searchParams.has("token")).toBe(false); expect(first.clientOptions).toEqual({ headers: { Authorization: defaultApiKey, "User-Agent": sdkUserAgent }, + noDelay: true, }); await session.close(); @@ -681,6 +683,7 @@ describe("tts session", () => { expect(new URL(second.url).searchParams.get("token")).toBe("tmp"); expect(second.clientOptions).toEqual({ headers: { "User-Agent": sdkUserAgent }, + noDelay: true, }); second.send(JSON.stringify(beginMessage)); await reconnect; @@ -700,6 +703,29 @@ describe("tts session", () => { await session.close(); }); + it.each([ + ["an API key", { websocketBaseUrl }], + ["a temporary token, without a client", { websocketBaseUrl, token: "tmp" }], + ])( + "a session opened with %s turns Nagle's algorithm off before the upgrade request", + async (_, params) => { + // Given a runtime where the socket comes from the ws package + const session = + "token" in params + ? new TextToSpeechSession(params) + : client.tts.session(params); + + // When it connects + await connect(session); + + // Then the socket is asked for TCP_NODELAY from the start + expect((await connection(1)).clientOptions).toMatchObject({ + noDelay: true, + }); + await session.close(); + }, + ); + it("keepAlive() sends a KeepAlive frame", async () => { const session = client.tts.session({ websocketBaseUrl }); await connect(session); @@ -731,6 +757,7 @@ describe("tts session", () => { const second = await connection(2); expect(second.clientOptions).toEqual({ headers: { "User-Agent": userAgent }, + noDelay: true, }); second.send(JSON.stringify(beginMessage)); await reconnect; @@ -747,12 +774,13 @@ describe("tts session", () => { await connect(session); expect((await connection(1)).clientOptions).toEqual({ headers: { Authorization: defaultApiKey }, + noDelay: true, }); await session.close(); const reconnect = session.connect({ token: "tmp" }); const second = await connection(2); - expect(second.clientOptions).toBeUndefined(); + expect(second.clientOptions).toEqual({ noDelay: true }); second.send(JSON.stringify(beginMessage)); await reconnect; await session.close(); @@ -764,7 +792,7 @@ describe("tts session", () => { token: "tmp", }); await connect(session); - expect((await connection(1)).clientOptions).toBeUndefined(); + expect((await connection(1)).clientOptions).toEqual({ noDelay: true }); await session.close(); }); From e053c4294d3cf5ac1d22310f0abe3019c25e0393 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:25:51 +0000 Subject: [PATCH 24/26] docs(tts): measured numbers for HTTP keep-alive and token minting Replaces two estimates in "Keep the time to first audio low" with measurements against sandbox000 (n=30 per arm, interleaved): - Node fetch with a 2-minute keep-alive vs Node's default, stream() requests 6 s apart: first audio -5.1 ms p50 [95% CI -9.1, -3.7] on Node 22, -9.6 [-28.9, -5.5] on Node 20 (Node 18 -4.1, CI includes 0), and -125.1 [-129.5, -120.3] with 60 ms of round trip added. Every default request opened a new TLS connection, none with keep-alive. An idle kept connection does not hold the process open; Bun and Deno fetch kept idle connections for over a minute without any setting. - A temporary token minted on the turn added 24.4 ms [22.5, 25.5] before Begin (the README said about 32 ms). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index db0fc28..d0448a4 100644 --- a/README.md +++ b/README.md @@ -955,8 +955,8 @@ The wait for first audio is mostly synthesis on our servers, plus whatever your This saves up to one network round trip, and never more than the API's checks take. With 60 ms of round trip added between our test client and the API's region, first audio came 14 ms sooner (median of 30 sessions per arm); inside the region the difference was too small to measure. Keep and await `connect()`'s promise all the same: a refused session rejects it, and a rejection nothing handles ends a Node process. -- **In a browser, mint the next token before the page needs it**, for example when the pointer enters the text box after the idle session closed, so that opening the session does not wait for your server and the token service (about 32 ms per new session in our tests, before the round trip to your server). Mint it only when a session will have to be opened, use it for one connection, and drop it well before it expires. -- **On Node, keep HTTP connections between replies** if your voice agent calls `stream()` or `synthesize()` more than 4 seconds apart. Node's built-in `fetch` closes an idle connection after about 4 seconds when the server sends no `Keep-Alive` hint, and the API sends none, so a later reply opens a new TCP and TLS connection. Keep connections for 2 minutes with `setGlobalDispatcher(new Agent({ keepAliveTimeout: 120_000 }))` from the `undici` package, which applies to every `fetch` of the process. In the API's region the difference is too small to measure; farther away each new connection costs two network round trips. +- **In a browser, mint the next token before the page needs it**, for example when the pointer enters the text box after the idle session closed, so that opening the session does not wait for your server and the token service (about 24 ms per new session in our tests, before the round trip to your server). Mint it only when a session will have to be opened, use it for one connection, and drop it well before it expires. +- **On Node, keep HTTP connections between replies** if your voice agent calls `stream()` or `synthesize()` more than 4 seconds apart. Node's built-in `fetch` closes an idle connection after about 4 seconds when the server sends no `Keep-Alive` hint, and the API sends none, so a later reply opens a new TCP and TLS connection. Keep connections for 2 minutes with `setGlobalDispatcher(new Agent({ keepAliveTimeout: 120_000 }))` from the `undici` package, which applies to every `fetch` of the process. With requests 6 seconds apart, that brought the first audio 5 to 10 ms sooner in the API's region (median of 30 requests per arm, Node 20 and 22) and 125 ms sooner with 60 ms of round trip added, since each new connection costs two network round trips. An idle kept connection does not keep the process from exiting. Bun and Deno already keep idle connections for over a minute. Node needs no shared TLS context for sessions: after its first TLS connection it keeps the parsed root certificates, so later connections set up their TLS context in about a tenth of a millisecond. From d5d89780ee8f0ecb0423946b0e89d59880414907 Mon Sep 17 00:00:00 2001 From: Aleksandar Mitov <140423361+aleks-mitov@users.noreply.github.com> Date: Sun, 25 Oct 2026 17:48:54 +0000 Subject: [PATCH 25/26] fix(tts): release the connection of a stream() result dropped unread stream() locks the response body at the headers so that Node 20+ fetch cannot cancel it while the caller awaits something else. That lock also turned off the runtime's release of a body nobody reads: a stream dropped without being read to its end or cancelled (a reply requested ahead that a barge-in made obsolete) kept its connection, and its listener on a shared signal, until the server gave up on the download. Before the lock, Node 20 and 22 released it at the next garbage collection. The body handed out is now registered with a FinalizationRegistry: once it is garbage-collected unread, the response body is cancelled and the request's signal released. A body that ends, however it ends, is unregistered. Reading it, iterating it or holding its reader keeps it reachable, so a stream in use is never cancelled. Checked against a local server with forced GCs: a dropped stream now releases its connection on Node 18, 20 and 22, Bun, Deno and the browser bundle (before: only on Deno), and a stream iterated, or read with the caller's own reader, through repeated GCs still delivers all its audio. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01KcyY6MAEUVjqBt5pMpetvD --- README.md | 2 +- src/services/tts/service.ts | 44 +++++++++++++++++++++--- tests/unit/tts.test.ts | 67 ++++++++++++++++++++++++++++++++++++- 3 files changed, 107 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d0448a4..1da66fb 100644 --- a/README.md +++ b/README.md @@ -867,7 +867,7 @@ for await (const chunk of speech) { `stream()` resolves once the API has sent the first audio. Chunk sizes vary and are not aligned to samples or frames: for 16-bit PCM carry an odd trailing byte over to the next chunk, and re-frame for a fixed-size sink (20 ms of 8 kHz mu-law is 160 bytes). The API commits to `200` with the first audio, so a failure after that cuts the body: the loop then throws a `TextToSpeechError` with `retryable: true`, and the whole request should be retried. The headers leave with the first audio too, so `speech.info.billedCharacters` and `speech.info.voiceExpiresAt` are the values as the stream started: a body cut by a server failure bills 0. A body that sends no audio for 30 seconds while the loop waits for it ends the same way, and its connection is released, so a stalled connection (a dropped NAT mapping, a hung proxy) cannot hang the loop. Set the limit in milliseconds with the `readTimeout` option (`0` turns it off); only the wait counts, so playing a chunk at its own pace never trips it. `speech.body` is the underlying web `ReadableStream`, and a read of it waits under the same limit. So `stream()` needs a global `ReadableStream`: on a runtime without one (React Native without a polyfill) it throws a `TextToSpeechError` before sending the request; use `synthesize()` there. -For a barge-in, pass `{ signal }` as the second argument and abort it: the download stops and the loop throws the signal's reason (an `AbortError`), not a `TextToSpeechError`. `speech.cancel()` stops it too, and the loop just ends. One signal can serve every request of a call. +For a barge-in, pass `{ signal }` as the second argument and abort it: the download stops and the loop throws the signal's reason (an `AbortError`), not a `TextToSpeechError`. `speech.cancel()` stops it too, and the loop just ends. One signal can serve every request of a call. Cancel a stream you will not read, such as a reply requested ahead that a barge-in made obsolete: a stream dropped unread keeps its connection until the runtime garbage-collects it. ### Speak text as it is written (voice agents) diff --git a/src/services/tts/service.ts b/src/services/tts/service.ts index 303c1d8..4d6b829 100644 --- a/src/services/tts/service.ts +++ b/src/services/tts/service.ts @@ -43,6 +43,27 @@ function requestDeadline(timeoutMs: number, external?: AbortSignal) { /** The longest delay `setTimeout` keeps; a longer one fires at once. */ const MAX_TIMER_MS = 2_147_483_647; +/** + * Releases the response of a `stream()` result that nothing can read any + * more, when it is garbage-collected. `stream()` locks the response body at + * the headers, which turns off the runtime's own release of an unread body + * (Node 20+ `fetch` cancels it with its `Response`), so a stream dropped + * without being read or cancelled would otherwise keep its connection and + * the request's abort listener until the API gave up on the download. + */ +const unreadBodies = (() => { + // Read from `globalThis`, so the ES2015 builds need no ES2021 types. + const Registry = ( + globalThis as { + FinalizationRegistry?: new (cleanup: (release: () => void) => void) => { + register(target: object, release: () => void, token: object): void; + unregister(token: object): void; + }; + } + ).FinalizationRegistry; + return Registry ? new Registry((release) => release()) : undefined; +})(); + /** * `reader.read()`, given up after `timeoutMs` without a chunk: the reader is * then cancelled, which releases the connection, and the read rejects with @@ -405,7 +426,9 @@ export class TextToSpeechService extends BaseService { * a body that sends nothing for `options.readTimeout` (30 000 ms by * default) while the iteration waits for audio; the connection is then * released. `options.signal` keeps covering the body: aborting it stops - * the download. + * the download. A stream dropped before its end without `cancel()` keeps + * its connection until it is garbage-collected, so cancel one you will + * not read. * * The audio comes as a web `ReadableStream`, so the runtime needs a global * `ReadableStream` (React Native has none without a polyfill). Without @@ -486,6 +509,12 @@ export class TextToSpeechService extends BaseService { `The response was cut before the audio was complete: no audio for ${readTimeout} ms. Retry the request.`, { status: 200, requestId: info.requestId, retryable: true }, ); + // The body's registration with `unreadBodies`, dropped once it ends. + const registration = {}; + const ended = () => { + unreadBodies?.unregister(registration); + deadline.release(); + }; const body = new ReadableStream( { async pull(controller) { @@ -493,21 +522,28 @@ export class TextToSpeechService extends BaseService { try { result = await readWithin(reader, readTimeout, stalled); } catch (error) { - deadline.release(); + ended(); throw error; } if (result.done) { - deadline.release(); + ended(); controller.close(); } else controller.enqueue(result.value); }, cancel: (reason) => { - deadline.release(); + ended(); return reader.cancel(reason); }, }, { highWaterMark: 0 }, ); + // What runs once `body` is garbage-collected unread; it must not refer + // to `body`, or `body` would never be collected. + const releaseUnread = () => { + deadline.release(); + reader.cancel().catch(() => undefined); + }; + unreadBodies?.register(body, releaseUnread, registration); return new TextToSpeechStream(info, body, { signal: options.signal }); } diff --git a/tests/unit/tts.test.ts b/tests/unit/tts.test.ts index c59172b..d08c760 100644 --- a/tests/unit/tts.test.ts +++ b/tests/unit/tts.test.ts @@ -1,6 +1,6 @@ import { getEventListeners } from "events"; import fetchMock from "jest-fetch-mock"; -import { AssemblyAI, TextToSpeechError } from "../../src"; +import { AssemblyAI, TextToSpeechError, TextToSpeechStream } from "../../src"; import { createClient, defaultApiKey, requestMatches } from "./utils"; fetchMock.enableMocks(); @@ -927,6 +927,71 @@ describe("tts", () => { expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); }); + it("stream releases the connection and the signal of a stream dropped unread once it is garbage-collected", async () => { + // Given a stream nobody reads or cancels + const register = jest.spyOn(FinalizationRegistry.prototype, "register"); + const { source, cancelled } = stallingSource(); + mockStreamedBody(source); + const controller = new AbortController(); + let speech: TextToSpeechStream; + let calls: Parameters["register"]>[]; + try { + speech = await client.tts.stream( + { text: "Hi." }, + { signal: controller.signal }, + ); + } finally { + calls = register.mock.calls.slice(); + register.mockRestore(); + } + const registered = calls.find(([target]) => target === speech.body); + expect(registered).toBeDefined(); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(1); + + // When the runtime collects its body and runs the cleanup + const release = registered![1] as () => void; + release(); + await new Promise((resolve) => setTimeout(resolve, 0)); + + // Then the response body is cancelled, which releases the connection, + // and the shared signal keeps no listener + expect(cancelled).toHaveBeenCalledTimes(1); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); + + it.each([ + [ + "read to the end", + async (speech: TextToSpeechStream) => void (await speech.bytes()), + ], + ["cancelled", (speech: TextToSpeechStream) => speech.cancel()], + [ + "left early", + async (speech: TextToSpeechStream) => { + for await (const chunk of speech) if (chunk) break; + }, + ], + ])("a stream %s drops its garbage-collection cleanup", async (_name, end) => { + const register = jest.spyOn(FinalizationRegistry.prototype, "register"); + const unregister = jest.spyOn(FinalizationRegistry.prototype, "unregister"); + try { + mockStreamedBody(audioSource()); + const speech = await client.tts.stream({ text: "Hi." }); + const registered = register.mock.calls.find( + ([target]) => target === speech.body, + ); + expect(registered).toBeDefined(); + expect(unregister).not.toHaveBeenCalled(); + + await end(speech); + + expect(unregister).toHaveBeenCalledWith(registered![2]); + } finally { + register.mockRestore(); + unregister.mockRestore(); + } + }); + it("stream calls sharing one signal leave no listener on it, however they end", async () => { const controller = new AbortController(); const signal = controller.signal; From 2cc017e9a42a0e07aa3397e4e95376e81eac12a7 Mon Sep 17 00:00:00 2001 From: bgotthold-aai Date: Mon, 26 Oct 2026 14:37:45 +0000 Subject: [PATCH 26/26] fix(tts): say why a session refuses a send, and document the error codes A send or an audio() read on a session that close() or the API ended said "call connect() first"; it now says the session is closed and that connect() opens a new one, and audio() with no id on an ended session throws instead of yielding nothing, unless close() spoke the text being written, which stays readable. Reading the later of two speak() iterators first said the earlier request "was already read"; it now says which request an iteration read through. A connect() still pending, or the connection's own error, is reported as such. requestId on a session comes from Begin, so a refusal before Begin has none: documented, with closeCode, in the error type and the README. The README gains a table of the request options and one of the error codes with their HTTP statuses and close codes, says that stream() info never carries audioDurationMs, and states when the API speaks text before a flush. Co-Authored-By: Claude Fable 5.1 --- README.md | 50 ++++++++++++++++++++++++----- src/services/tts/session.ts | 57 ++++++++++++++++++++++++++++------ src/utils/errors/tts.ts | 7 +++-- tests/unit/tts-session.test.ts | 44 +++++++++++++++++++++++--- 4 files changed, 134 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 1da66fb..27b3db0 100644 --- a/README.md +++ b/README.md @@ -826,7 +826,23 @@ await transcriber.close(); ## Text-To-Speech -`client.tts` speaks text with the Text-to-Speech API. `synthesize()` returns one text's complete audio, `stream()` returns it as the API generates it, and `session()` opens a WebSocket session that speaks text as it is written, for voice agents. Options take the API's own field names: the `voice_id` (a voice of the AssemblyAI voice library such as `aai-jane`, the default, or a voice cloned in your dashboard), the `model_id` (`assemblyai-tts-1`), the `language` (`english` by default, `spanish`, `german`, `french`, `italian`, `portuguese`), the `sample_rate` (24000 by default) and the `encoding` (`pcm_s16le` by default, `pcm_mulaw`/`pcm_alaw` for telephony, `mp3` over HTTP only). Requests go to `ttsBaseUrl`, `https://tts.us.assemblyai.com` by default; pass `https://tts.eu.assemblyai.com` to keep them in the EU data zone. +`client.tts` speaks text with the Text-to-Speech API. `synthesize()` returns one text's complete audio, `stream()` returns it as the API generates it, and `session()` opens a WebSocket session that speaks text as it is written, for voice agents. Requests go to `ttsBaseUrl`, `https://tts.us.assemblyai.com` by default; pass `https://tts.eu.assemblyai.com` to keep them in the EU data zone. + +An HTTP request takes the API's own field names. A session takes the same options in camelCase, like the other session clients of this SDK. + +| HTTP request | Session | Values | Default | +| -------------------------- | ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | +| `text` | | 1 to 5,000 characters | | +| `voice_id` | `voiceId` | a voice of the AssemblyAI voice library by its slug, such as `aai-jane`, `aai-alba` or `aai-michael`, or a voice cloned in your dashboard | `aai-jane` | +| `model_id` | `modelId` | `assemblyai-tts-1` | the region's default model | +| `language` | `language` | `english`, `spanish`, `german`, `french`, `italian`, `portuguese` | the voice's own language | +| `sample_rate` | `sampleRate` | 8000, 16000, 22050, 24000, 44100, 48000 | 24000 | +| `encoding` | `encoding` | `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` for telephony, `mp3` over HTTP only | `pcm_s16le` | +| `container` | | `raw` or `wav`, with a PCM or G.711 encoding | `raw` | +| `bit_rate` | | MP3 bits per second: 32000, 48000, 64000, 96000, 128000 | 64000 | +| `extend_biometric_consent` | `extendBiometricConsent` | `true` renews the consent of the cloned voice in `voice_id` | `false` | +| | `wordBoundaries` | `true` sends word timings per request | `false` | +| | `inactivityTimeout` | seconds without a frame before the API closes the session, 5 to 120 | 120 | ### Synthesize a WAV file @@ -844,7 +860,23 @@ console.log(response.model, response.voiceId, response.audioDurationMs, "ms"); `synthesize()` waits for the whole audio, so an API refusal is always a `TextToSpeechError` and the WAV header carries real sizes. The response headers are on the result: `requestId`, `region`, `model`, `voiceId`, `language`, `encoding`, `sampleRate`, `container`, `bitRate`, `billedCharacters`, `audioDurationMs` and `unknownParams`. The API ignores body fields it does not know, and `unknownParams` names them (the SDK also warns once), so a camelCase `sampleRate` in a request held in a variable shows up instead of silently giving 24 kHz audio. A network failure or a timeout rejects with the runtime's own error instead (a `TypeError`, or a `DOMException` named `TimeoutError` or `AbortError`). On Node, the built-in `fetch` gives up after 300 seconds without response headers, whatever `timeout` says; for longer requests use `stream()`, or raise the limit with `setGlobalDispatcher(new Agent({ headersTimeout: 610_000 }))` from the `undici` package. -Branch on a refusal's `errorCode`, not its message. Your account's own cloned voices, and `extend_biometric_consent: true` (which renews the consent of the cloned voice in `voice_id` and is refused with `not_owner` on a voice your account does not own), are refused with `biometric_terms_not_accepted` (403, not retryable; close code 3006 on a session) until the account accepts the biometric data terms on the data controls page of the AssemblyAI dashboard. +Branch on a refusal's `errorCode`, not its message, and on `retryable` to decide whether to try again (`retryAfter` says how many seconds to wait). The same codes come back from a session, where the error's `closeCode` is the WebSocket close code that ended it. + +| `errorCode` | HTTP | Session | Meaning | +| ------------------------------ | ---- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `invalid_parameter` | 422 | 3006 | `param` and `errors` name the fields at fault | +| `rate_limited` | 429 | 3009 | retryable after `retryAfter` seconds | +| `voice_not_found` | 422 | 3012 | `param: "voice_id"` | +| `voice_expired` | 422 | 3012 | the cloned voice's consent period ended; its owner can create it again from a new recording | +| `voice_not_paired` | 422 | 3014 | the cloned voice is not paired with the model, or the pairing is still in progress | +| `not_owner` | 422 | 3006 | `extend_biometric_consent: true` on a voice another account owns | +| `biometric_terms_not_accepted` | 403 | 3006 | your own cloned voices, and `extend_biometric_consent`, need the biometric data terms accepted on the data controls page of the AssemblyAI dashboard; not retryable | +| `auth_unavailable` | 503 | 1011 | the credential could not be checked just now; retryable | +| `unauthorized` | | 1008 | a temporary token the API had already seen, or an expired one | +| `inactivity_timeout` | | 3006 | no frame in either direction for `inactivityTimeout` seconds; retryable | +| `input_rate_exceeded` | | 3010 | more unspoken text than the API holds; not retryable | + +The session rows come from the `Error` frame the API sends before it closes. A connection that closes without one, such as a network cut, gives an error with `closeCode` only, retryable for 1006, 1011, 1012, 3005, 3008 and 3009. ### Stream audio as it is generated @@ -865,13 +897,13 @@ for await (const chunk of speech) { } ``` -`stream()` resolves once the API has sent the first audio. Chunk sizes vary and are not aligned to samples or frames: for 16-bit PCM carry an odd trailing byte over to the next chunk, and re-frame for a fixed-size sink (20 ms of 8 kHz mu-law is 160 bytes). The API commits to `200` with the first audio, so a failure after that cuts the body: the loop then throws a `TextToSpeechError` with `retryable: true`, and the whole request should be retried. The headers leave with the first audio too, so `speech.info.billedCharacters` and `speech.info.voiceExpiresAt` are the values as the stream started: a body cut by a server failure bills 0. A body that sends no audio for 30 seconds while the loop waits for it ends the same way, and its connection is released, so a stalled connection (a dropped NAT mapping, a hung proxy) cannot hang the loop. Set the limit in milliseconds with the `readTimeout` option (`0` turns it off); only the wait counts, so playing a chunk at its own pace never trips it. `speech.body` is the underlying web `ReadableStream`, and a read of it waits under the same limit. So `stream()` needs a global `ReadableStream`: on a runtime without one (React Native without a polyfill) it throws a `TextToSpeechError` before sending the request; use `synthesize()` there. +`stream()` resolves once the API has sent the first audio. Chunk sizes vary and are not aligned to samples or frames: for 16-bit PCM carry an odd trailing byte over to the next chunk, and re-frame for a fixed-size sink (20 ms of 8 kHz mu-law is 160 bytes). The API commits to `200` with the first audio, so a failure after that cuts the body: the loop then throws a `TextToSpeechError` with `retryable: true`, and the whole request should be retried. The headers leave with the first audio too, so `speech.info.billedCharacters` and `speech.info.voiceExpiresAt` are the values as the stream started (a body cut by a server failure bills 0), and `speech.info.audioDurationMs` is never set, since the duration is known only once the audio is complete; measure the bytes received instead. A body that sends no audio for 30 seconds while the loop waits for it ends the same way, and its connection is released, so a stalled connection (a dropped NAT mapping, a hung proxy) cannot hang the loop. Set the limit in milliseconds with the `readTimeout` option (`0` turns it off); only the wait counts, so playing a chunk at its own pace never trips it. `speech.body` is the underlying web `ReadableStream`, and a read of it waits under the same limit. So `stream()` needs a global `ReadableStream`: on a runtime without one (React Native without a polyfill) it throws a `TextToSpeechError` before sending the request; use `synthesize()` there. For a barge-in, pass `{ signal }` as the second argument and abort it: the download stops and the loop throws the signal's reason (an `AbortError`), not a `TextToSpeechError`. `speech.cancel()` stops it too, and the loop just ends. One signal can serve every request of a call. Cancel a stream you will not read, such as a reply requested ahead that a barge-in made obsolete: a stream dropped unread keeps its connection until the runtime garbage-collects it. ### Speak text as it is written (voice agents) -A `TextToSpeechSession` keeps one WebSocket open and speaks text as it is written, so an LLM's reply can be voiced sentence by sentence with no per-request handshake. `speak()` sends one text and returns its audio; `generate()` / `flush()` do the same in two steps, so tokens can be sent as they arrive; `cancel()` is the barge-in. `close()` ends the session and resolves with its summary once the API has spoken the text still pending. It stops waiting after `terminateTimeout` milliseconds (default 35,000) without a message from the API; an `audio()` iteration still running then rejects with a retryable `TextToSpeechError`, and `close()` resolves `undefined`. Calling `close()` again, or from two places at once, waits for the same end. Once `close()` has sent `Terminate` the API reads nothing more, so `generate()`, `flush()`, `cancel()` and `speak()` throw instead of sending text that would never be spoken, and `keepAlive()` does nothing. Text need not wait for `connect()` to resolve: what is sent while the socket opens is held and sent the moment it opens (see [Keep the time to first audio low](#keep-the-time-to-first-audio-low)). +A `TextToSpeechSession` keeps one WebSocket open and speaks text as it is written, so an LLM's reply can be voiced sentence by sentence with no per-request handshake. `speak()` sends one text and returns its audio; `generate()` / `flush()` do the same in two steps, so tokens can be sent as they arrive; `cancel()` is the barge-in. `close()` ends the session and resolves with its summary once the API has spoken the text still pending. It stops waiting after `terminateTimeout` milliseconds (default 35,000) without a message from the API; an `audio()` iteration still running then rejects with a retryable `TextToSpeechError`, and `close()` resolves `undefined`. Calling `close()` again, or from two places at once, waits for the same end. Once `close()` has sent `Terminate` the API reads nothing more, so `generate()`, `flush()`, `cancel()` and `speak()` throw instead of sending text that would never be spoken, and `keepAlive()` does nothing; once the session has ended they say so, and that `connect()` opens a new one. Text need not wait for `connect()` to resolve: what is sent while the socket opens is held and sent the moment it opens (see [Keep the time to first audio low](#keep-the-time-to-first-audio-low)). ```typescript const session = client.tts.session({ voiceId: "aai-jane" }); @@ -885,7 +917,7 @@ const summary = await session.close(); console.log(summary?.audio_duration_seconds, "s spoken"); ``` -Streaming the text in as it is generated, with a barge-in. Audio starts before `flush()` once whole sentences have arrived, so read `session.audio()` with no id while the tokens are still being sent: +Streaming the text in as it is generated, with a barge-in. The API speaks a complete sentence before `flush()` once more text has arrived after it and the request is long enough, about 200 characters, or 50 for the first sentence of text sent in several `generate()` calls; a shorter request, or one whose last sentence has nothing after it yet, is spoken at `flush()`. So read `session.audio()` with no id while the tokens are still being sent: ```typescript const session = client.tts.session({ voiceId: "aai-jane" }); @@ -913,9 +945,13 @@ await session.close(); The API reads frames in the order sent, so a `cancel()` retires only the text sent before it. Stop sending the interrupted reply once the user barges in: text sent after the cancel is a new request, which is synthesized and billed, and which `close()` waits for. -`audio()` fixes its request when it is called, so the iterator can be made before the first `generate()` and handed to a player that starts later. Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order, once; `audio()` throws a `RangeError` for an id not flushed yet or already read. Every `flush()` and `cancel()` takes the next id and returns it, so ids skip one after a cancel, and the requests a cancel retires are those up to its id. Audio of a retired request that the API sent before it read the `Cancel` can still arrive; the session drops it, so neither `audio()` nor the `audio` listener keeps playing the interrupted reply. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters (`limits.text_max_buffer_chars` in `Begin`), ending the session with a non-retryable 3010 past that, so pace text to the audio. `speak()` throws a `RangeError` for a longer text before sending any of it; split it and speak each part once the previous part's audio is read. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes with a retryable 3006 `inactivity_timeout` after `inactivityTimeout` seconds (5 to 120, default 120) without a frame in either direction. Audio arrives much faster than it plays, so the session can idle out while received audio is still playing: call `keepAlive()` at least once per timeout while the session waits. +`audio()` fixes its request when it is called, so the iterator can be made before the first `generate()` and handed to a player that starts later. Requests can be pipelined: `flush()` again before reading the previous request's audio, then read each `audio(flushId)` in order, once; `audio()` throws a `RangeError` for an id not flushed yet, or one an iteration already read past, since reading a later request consumes the end of the earlier ones. Every `flush()` and `cancel()` takes the next id and returns it, so ids skip one after a cancel, and the requests a cancel retires are those up to its id. Audio of a retired request that the API sent before it read the `Cancel` can still arrive; the session drops it, so neither `audio()` nor the `audio` listener keeps playing the interrupted reply. A `Generate` frame takes at most 2,000 characters (`speak()` splits longer text; `generate()` throws); the API accepts text at about 100 characters per second after an initial 1,500 and holds at most 8,000 unspoken characters (`limits.text_max_buffer_chars` in `Begin`), ending the session with a non-retryable 3010 past that, so pace text to the audio. `speak()` throws a `RangeError` for a longer text before sending any of it; split it and speak each part once the previous part's audio is read. A session streams raw `pcm_s16le`, `pcm_mulaw` or `pcm_alaw` only, ends after an hour, and closes with a retryable 3006 `inactivity_timeout` after `inactivityTimeout` seconds (5 to 120, default 120) without a frame in either direction. Audio arrives much faster than it plays, so the session can idle out while received audio is still playing: call `keepAlive()` at least once per timeout while the session waits. + +Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A listener that throws is logged with `console.error` and the session goes on. + +A failure is a `TextToSpeechError`. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send (a send in the moment between a lost connection and its close event throws a retryable error saying the connection is closing), reaches the `error` listener, and, if nothing else observed it, rejects `close()`; a frame the SDK cannot read fails the session with `Unreadable frame from the API` and closes the socket. On a session read `errorCode` and `closeCode`, the WebSocket close code the API ended it with (see the table above). `requestId` is the session id from `Begin`, so a session refused before `Begin`, such as one with an unknown voice or a bad key, has none. `status` is set only when the server answered the WebSocket upgrade itself with an HTTP status the runtime reports (Node does, a browser does not): a 404 for a wrong `websocketBaseUrl` path is not retryable, a 503 is. -Register listeners with `session.on(event, listener)`: `begin`, `flushDone`, `wordBoundaries` (word timings per request, with `wordBoundaries: true`; `start`/`end` count from the start of the session's audio, so subtract the event's `audio_start_ms` for a time within the request), `cancelled`, `termination`, `error` and `close`. An `audio` listener receives the audio frames instead of the pull iterators, which then throw. A listener that throws is logged with `console.error` and the session goes on; a frame the SDK cannot read fails the session with `Unreadable frame from the API` and closes the socket. A refusal rejects `connect()`; a failure mid-session rejects the `audio()` iteration or the next send (a send in the moment between a lost connection and its close event throws a retryable `TextToSpeechError` saying the connection is closing), reaches the `error` listener, and, if nothing else observed it, rejects `close()`. On a session `requestId` is the session id, and `status` is set only when the server answered the WebSocket upgrade with an HTTP status the runtime reports (Node does, a browser does not): a 404 for a wrong `websocketBaseUrl` path is not retryable, a 503 is. `retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. That works as is with an API key; a session opened with a temporary token needs a new token for each connection, passed as `connect({ token })`, because the API may refuse a token it has seen and always refuses an expired one (close code 1008, `errorCode: "unauthorized"`). The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. +`retryable: true` means a new session can succeed, and the same session object can `connect()` again, also from the `error` or `close` listener, which run once the connection has ended; an `audio()` iteration of the old connection still rejects. That works as is with an API key; a session opened with a temporary token needs a new token for each connection, passed as `connect({ token })`, because the API may refuse a token it has seen and always refuses an expired one (close code 1008, `errorCode: "unauthorized"`). The `close` listener gets the socket's close code, or the error's close code (1000 without one) when the session released the socket first. In a browser the API key must stay on your server. Mint a temporary token there, and open the session in the page with `new TextToSpeechSession({ token })`, which needs no client and no API key: diff --git a/src/services/tts/session.ts b/src/services/tts/session.ts index e41dac8..670a230 100644 --- a/src/services/tts/session.ts +++ b/src/services/tts/session.ts @@ -128,6 +128,15 @@ class PullQueue { resolve({ kind: "wake" }); } + /** Whether audio or the `FlushDone` of request `flushId` is queued. */ + holds(flushId: number): boolean { + return this.items.some( + (item) => + (item.kind === "audio" || item.kind === "done") && + item.flushId === flushId, + ); + } + next(): Promise { const item = this.items.shift(); if (item) { @@ -789,7 +798,23 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c requestId: this.begin.id, }); } - throw new Error("The session is not open: call connect() first"); + throw this.notOpen(); + } + + /** The error of a send or a read on a session that is not connected. */ + private notOpen(): Error { + if (this.error) return this.surfaced(); + if (this.abortConnect) { + return new Error( + "The session is not open: connect() is still pending; await it for the outcome", + ); + } + if (this.queue.hadSocket) { + return new Error( + "The session is closed: close() ended it, or the API did; connect() again to open a new one", + ); + } + return new Error("The session is not open: call connect() first"); } private limits(): Record | undefined { @@ -833,8 +858,10 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c * Appends `text` to the current request. * * Consecutive texts are joined exactly as sent, so include the spaces. - * Audio may start before `flush()` once enough complete sentences have - * arrived; `flush()` is what concludes the request. + * The API speaks a complete sentence before `flush()` once more text has + * arrived after it and the request is long enough (about 200 characters, + * or 50 for the first sentence of text sent in several calls); a shorter + * request is spoken at `flush()`, which is what concludes the request. * @throws RangeError for a text over one frame's limit (2,000 characters); split it. * @throws Error once `close()` was called: the API would not read the text. */ @@ -942,17 +969,27 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c * a time. Audio is queued for pulling only while no `audio` listener is * registered. The iteration rejects with a `TextToSpeechError` if the * session fails before the request completes. - * @throws Error if the session was never connected. - * @throws RangeError for a request not flushed yet, or one already read - * (its `FlushDone` was consumed by an earlier iteration). + * @throws Error if the session was never connected; and, without + * `flushId`, once the session has ended with nothing queued for the text + * being written, since it can no longer be flushed on that connection. + * @throws RangeError for a request not flushed yet, or one an iteration + * already read past (its `FlushDone` was consumed by that iteration). */ audio(flushId?: number): AsyncGenerator { // The connection this request is sent on. const connection = this.queue; - if (!connection.hadSocket) { - throw new Error("The session is not open: call connect() first"); - } + if (!connection.hadSocket) throw this.notOpen(); const id = flushId ?? this.nextFlushId; + // A flushed request may still drain, and so may the text being written + // that `close()` had spoken; the text being written otherwise never will. + if ( + flushId === undefined && + connection.ended && + !connection.error && + !connection.holds(id) + ) { + throw this.notOpen(); + } if (id > this.nextFlushId) { throw new RangeError(`flush_id ${id} has not been flushed yet`); } @@ -963,7 +1000,7 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c private checkUnread(connection: PullQueue, flushId: number) { if (flushId <= connection.doneThrough) { throw new RangeError( - `flush_id ${flushId} was already read: read each request once, in order`, + `flush_id ${flushId} is no longer readable: an iteration already read through flush_id ${connection.doneThrough}, and requests are served in order, so read each one once, in order`, ); } } diff --git a/src/utils/errors/tts.ts b/src/utils/errors/tts.ts index 823db36..0b712d3 100644 --- a/src/utils/errors/tts.ts +++ b/src/utils/errors/tts.ts @@ -31,7 +31,10 @@ export type TextToSpeechErrorFields = { errorCode?: string; /** The field at fault, if one. */ param?: string; - /** The request's id, or the session's id on a session. */ + /** + * The request's id; on a session, the session's id from `Begin`, so a + * session refused before `Begin` has none: read `errorCode` and `closeCode`. + */ requestId?: string; /** Whether the same request, or a new session, can succeed later. */ retryable?: boolean; @@ -39,7 +42,7 @@ export type TextToSpeechErrorFields = { retryAfter?: number; /** For `invalid_parameter`, every failing field. */ errors?: { message: string; param: string | null }[]; - /** The WebSocket close code that ended a session. */ + /** The WebSocket close code that ended a session; see `errorCode` for the codes. */ closeCode?: number; /** The whole error envelope or frame the API sent. */ body?: Record; diff --git a/tests/unit/tts-session.test.ts b/tests/unit/tts-session.test.ts index 1410b85..92892ab 100644 --- a/tests/unit/tts-session.test.ts +++ b/tests/unit/tts-session.test.ts @@ -1655,8 +1655,12 @@ describe("tts session", () => { expect(await closing).toEqual(terminationMessage); expect(await reading).toEqual(Buffer.from("Pending text.")); - // And once the session ended, a send says to connect again - expect(() => session.generate("Later.")).toThrow(/call connect\(\) first/); + // And once the session ended, a send says so, and to connect again + expect(() => session.generate("Later.")).toThrow( + /The session is closed: close\(\) ended it.*connect\(\) again/, + ); + expect(() => session.cancel()).toThrow(/The session is closed/); + expect(() => session.audio()).toThrow(/The session is closed/); }); it("a send while the socket closes without an Error frame throws a retryable error", async () => { @@ -1782,7 +1786,9 @@ describe("tts session", () => { "Read me once.", ); expect(() => session.audio(0)).toThrow(RangeError); - expect(() => session.audio(0)).toThrow(/flush_id 0 was already read/); + expect(() => session.audio(0)).toThrow( + /flush_id 0 is no longer readable: an iteration already read through flush_id 0/, + ); // Reading a later request skips the FlushDone of an earlier one. session.generate("Skipped."); @@ -1792,8 +1798,36 @@ describe("tts session", () => { expect((await collect(session.audio(session.flush()))).toString()).toBe( "Read.", ); - expect(() => session.audio(skipped)).toThrow(/already read/); - await expect(early.next()).rejects.toThrow(/already read/); + expect(() => session.audio(skipped)).toThrow( + new RegExp(`flush_id ${skipped} is no longer readable`), + ); + await expect(early.next()).rejects.toThrow( + new RegExp(`already read through flush_id ${skipped + 1}`), + ); + await session.close(); + }); + + it("audio() after close() still reads the text close() had spoken", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + session.generate("Spoken at close."); + await session.close(); + expect((await collect(session.audio())).toString()).toBe( + "Spoken at close.", + ); + // Read once; with nothing left for the text being written, it is misuse + expect(() => session.audio()).toThrow(/The session is closed/); + }); + + it("the later of two speak() iterators read first says which request it read past", async () => { + const session = client.tts.session({ websocketBaseUrl }); + await connect(session); + const first = session.speak("First."); + const second = session.speak("Second."); + expect((await collect(second)).toString()).toBe("Second."); + await expect(collect(first)).rejects.toThrow( + /flush_id 0 is no longer readable: an iteration already read through flush_id 1/, + ); await session.close(); });