From be83643b81d905c59222b18251e4158ccd360d88 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 16 Sep 2026 05:36:22 +0000 Subject: [PATCH 1/3] Stop retrying ElevenLabs access failures and add provider safeguards --- docs/transcription-operations.md | 27 +++++++++++++++++++++++++++ src/live-voice.ts | 26 +++++++++++++++++++++++--- src/voice-provider.ts | 21 +++++++++++++++++++++ test/speaker-transcription.test.ts | 16 ++++++++++++++++ test/translation-passes.test.ts | 11 +++++++---- test/voice-provider.test.ts | 25 +++++++++++++++++++++++++ 6 files changed, 119 insertions(+), 7 deletions(-) create mode 100644 docs/transcription-operations.md create mode 100644 src/voice-provider.ts create mode 100644 test/voice-provider.test.ts diff --git a/docs/transcription-operations.md b/docs/transcription-operations.md new file mode 100644 index 0000000..ad153f4 --- /dev/null +++ b/docs/transcription-operations.md @@ -0,0 +1,27 @@ +# Transcription provider controls + +Speaker transcription uses ElevenLabs Scribe v2. Ordinary `/speech/transcribe` +uses local Whisper and does not spend ElevenLabs credits. + +ElevenLabs may report exhausted credits or permission problems with HTTP 401. +The server reads the provider error code and returns a terminal 402 (provider +billing/quota) or 424 (provider access), rather than a retryable 502. It refunds +the listener's reservation when the provider rejects a request. + +After a rejection, the LiveVoice instance pauses provider requests for five +minutes for billing/access failures, 30–300 seconds for rate limits, or ten +seconds for other provider errors. This cooldown is per server process; the +existing spending limits are persisted in PostgreSQL and shared by replicas. + +`voice_provider_failure` logs the operation, HTTP status, allowlisted provider +code, and cooldown. `voice_provider_usage` records accepted audio seconds or +characters and a hashed account/source identifier. Neither contains audio, +transcript text, API keys, or raw provider error bodies. Shared streams use a +source identifier here; listener charges remain in `translation_usage`. + +Configure daily spending ceilings with `NIXAMP_DUB_DAILY_CHARS`, +`NIXAMP_DUB_DAILY_AUDIO_SECONDS`, `NIXAMP_DUB_USER_DAILY_CHARS`, and +`NIXAMP_DUB_USER_DAILY_AUDIO_SECONDS`. Audio includes overlapping context sent +to Scribe. These are unit limits, not currency limits, and do not cover use of +the provider key outside LiveVoice (including telephone voices). + diff --git a/src/live-voice.ts b/src/live-voice.ts index 6230409..7711bd6 100644 --- a/src/live-voice.ts +++ b/src/live-voice.ts @@ -6,6 +6,7 @@ import type { VoiceProfile } from "./voice-profile.ts"; import { Guard } from "./guard.ts"; import type { TranslationMeter } from "./translation-passes.ts"; import type { Queryable } from "./follows.ts"; +import { voiceProviderFailure } from "./voice-provider.ts"; export const LIVE_VOICE_MODEL = "eleven_flash_v2_5"; export const LIVE_VOICE_RATE = 16_000; @@ -37,6 +38,18 @@ export class LiveVoice { private readonly dailyAudioSeconds: number; private readonly userDailyChars: number; private readonly userDailyAudioSeconds: number; + private providerPause: { until: number; error: SpeechError } | null = null; + + private checkProvider(): void { + if (this.providerPause && this.providerPause.until > this.now()) throw this.providerPause.error; + } + + private async providerFailed(response: Response, operation: string): Promise { + const failure = await voiceProviderFailure(response); + this.providerPause = { until: this.now() + failure.cooldownMs, error: failure.error }; + console.error(JSON.stringify({ event: "voice_provider_failure", provider: "elevenlabs", operation, status: response.status, code: failure.code, cooldownMs: failure.cooldownMs })); + throw failure.error; + } constructor(options: { apiKey?: string; fetcher?: typeof fetch; now?: () => number; charsPerMinute?: number; dailyChars?: number; dailyAudioSeconds?: number; userDailyChars?: number; userDailyAudioSeconds?: number; db?: Queryable; billing?: TranslationMeter } = {}) { this.key = options.apiKey ?? process.env["ELEVENLABS_API_KEY"] ?? ""; @@ -60,6 +73,7 @@ export class LiveVoice { async hear(bytes: Uint8Array, by: string, signal?: AbortSignal, meter = this.billing, resource = ""): Promise { if (!this.available()) throw new SpeechError("speaker voices are unavailable", 503); await meter?.require(by, resource); + this.checkProvider(); const wav = decodeWav(bytes); const seconds = wav.samples.length / wav.rate; if (wav.rate !== 16000 || wav.channels !== 1 || seconds < 0.2 || seconds > 15.1) throw new SpeechError("send up to 15 seconds of mono 16 kHz WAV", 400); @@ -77,6 +91,7 @@ export class LiveVoice { await this.reserve(`scribe:user:${by}`, billed, this.userDailyAudioSeconds, 86_400_000); await this.reserve("scribe:server", billed, this.dailyAudioSeconds, 86_400_000); signal?.throwIfAborted(); + this.checkProvider(); const form = new FormData(); // Canonical PCM prevents a crafted container from billing more audio // than the duration we validated, and removes uploaded metadata. @@ -91,8 +106,9 @@ export class LiveVoice { method: "POST", headers: { "xi-api-key": this.key }, body: form, signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000), }); - if (!answer.ok) throw new SpeechError("speaker transcription could not run; check provider quota and permissions", answer.status === 429 ? 429 : 502); + if (!answer.ok) await this.providerFailed(answer, "transcription"); accepted = true; + console.info(JSON.stringify({ event: "voice_provider_usage", provider: "elevenlabs", operation: "transcription", account: createHash("sha256").update(by).digest("hex").slice(0, 16), audioSeconds: seconds })); if (reservation) await meter!.commit(reservation); return speakerTurns(await answer.json() as ScribeResult, wav); } catch (error) { @@ -103,12 +119,13 @@ export class LiveVoice { async voices(): Promise { if (!this.available()) throw new SpeechError("translated audio needs ELEVENLABS_API_KEY on the account server", 503); + this.checkProvider(); if (!this.catalog || this.now() >= this.catalogUntil) { this.catalogUntil = this.now() + 3600_000; this.catalog = this.fetcher("https://api.elevenlabs.io/v2/voices?page_size=100&voice_type=default", { headers: { "xi-api-key": this.key }, signal: AbortSignal.timeout(8000), }).then(async response => { - if (!response.ok) throw new SpeechError("ElevenLabs could not list voices; check the server key and its voice permissions", 503); + if (!response.ok) await this.providerFailed(response, "voices"); const body = await response.json() as { voices: { voice_id: string; name: string; labels?: Record }[] }; return body.voices.filter(voice => /^[a-zA-Z0-9_-]+$/.test(voice.voice_id)).map(voice => ({ id: voice.voice_id, name: voice.name, gender: voice.labels?.["gender"] ?? "neutral", language: voice.labels?.["language"] ?? "", @@ -238,6 +255,7 @@ export class LiveVoice { let accepted = false; try { await this.charge(by, ask.channel ?? "direct", text.length); + this.checkProvider(); signal?.throwIfAborted(); reservation = await meter?.reserve(by, "voice", text.length, ask.channel); const response = await this.fetcher(`https://api.elevenlabs.io/v1/text-to-speech/${voice.id}/stream?output_format=pcm_16000`, { @@ -246,9 +264,11 @@ export class LiveVoice { body: JSON.stringify({ text, model_id: LIVE_VOICE_MODEL, language_code: ask.language }), signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000), }); - if (!response.ok || !response.body) throw new SpeechError(response.status === 429 ? "ElevenLabs audio quota is temporarily exhausted" : "ElevenLabs could not generate audio; check the server key and quota", response.status === 429 ? 429 : 502); + if (!response.ok) await this.providerFailed(response, "synthesis"); + if (!response.body) throw new SpeechError("ElevenLabs returned no audio", 502); // Once the provider accepts, aborting playback cannot refund heard audio. accepted = true; + console.info(JSON.stringify({ event: "voice_provider_usage", provider: "elevenlabs", operation: "synthesis", account: createHash("sha256").update(by).digest("hex").slice(0, 16), characters: text.length })); if (reservation) await meter!.commit(reservation); const [play, keep] = response.body.tee(); void (async () => { diff --git a/src/voice-provider.ts b/src/voice-provider.ts new file mode 100644 index 0000000..058d95e --- /dev/null +++ b/src/voice-provider.ts @@ -0,0 +1,21 @@ +import { SpeechError } from "./speech.ts"; + +/** Only fixed messages/codes leave this boundary: provider bodies can contain + * account information. Authorization and billing errors must not be retried + * as transient 502s by the live player. */ +export async function voiceProviderFailure(response: Response): Promise<{ error: SpeechError; code: string; cooldownMs: number }> { + let code = "unknown"; + try { + const body = await response.json() as { detail?: { status?: unknown } }; + const status = body?.detail?.status; + if (typeof status === "string" && ["quota_exceeded", "invalid_api_key", "missing_permissions", "subscription_required", "subscription_expired", "payment_required", "too_many_concurrent_requests", "rate_limit_exceeded"].includes(status)) code = status; + } catch { /* Non-JSON provider failures still have an HTTP status. */ } + if (code === "quota_exceeded") return { code, cooldownMs: 300_000, error: new SpeechError("ElevenLabs credits are exhausted. The site owner needs to check the provider balance.", 402) }; + if (["subscription_required", "subscription_expired", "payment_required"].includes(code) || response.status === 402) return { code, cooldownMs: 300_000, error: new SpeechError("ElevenLabs billing needs attention. The site owner needs to check the provider subscription.", 402) }; + if (["invalid_api_key", "missing_permissions"].includes(code) || [401, 403].includes(response.status)) return { code, cooldownMs: 300_000, error: new SpeechError("ElevenLabs rejected transcription or voice access. The site owner needs to check the provider key, permissions, and billing.", 424) }; + if (response.status === 429) { + const retry = Number(response.headers.get("retry-after")); + return { code, cooldownMs: Math.min(300_000, Math.max(30_000, Number.isFinite(retry) ? retry * 1000 : 0)), error: new SpeechError("ElevenLabs is rate limited. Wait before enabling translated audio again.", 429) }; + } + return { code, cooldownMs: 10_000, error: new SpeechError("ElevenLabs is temporarily unavailable. Try translated audio again shortly.", 502) }; +} diff --git a/test/speaker-transcription.test.ts b/test/speaker-transcription.test.ts index aa71c44..51a3e3d 100644 --- a/test/speaker-transcription.test.ts +++ b/test/speaker-transcription.test.ts @@ -15,6 +15,22 @@ const result = { language_code: 'spa', words: [ { type: 'word', text: 'La pelea sigue.', start: 2.5, end: 4.9, speaker_id: 'speaker_1' }, ] }; +test('a provider rejection refunds once, blocks other accounts during cooldown, then recovers', async () => { + let now = 0, calls = 0, reserved = 0, refunded = 0, committed = 0; + const billing = { require: async () => {}, reserve: async () => { reserved++; return 'reservation'; }, refund: async () => { refunded++; }, commit: async () => { committed++; } }; + const voice = new LiveVoice({ apiKey: 'key', now: () => now, billing, fetcher: (async () => { + calls++; + return calls === 1 ? Response.json({ detail: { status: 'quota_exceeded' } }, { status: 401 }) : Response.json(result); + }) as typeof fetch }); + await assert.rejects(voice.hear(audio(), 'alice'), error => error instanceof SpeechError && error.status === 402); + await assert.rejects(voice.hear(audio(), 'bob'), /credits are exhausted/); + await assert.rejects(voice.voices(), /credits are exhausted/); + assert.deepEqual([calls, reserved, refunded, committed], [1, 1, 1, 0]); + now = 300_001; + await voice.hear(audio(), 'bob'); + assert.deepEqual([calls, reserved, refunded, committed], [2, 2, 1, 1]); +}); + test('Scribe auto-detects native language, separates speakers, and receives canonical bounded WAV only', async () => { let calls = 0; const voice = new LiveVoice({ apiKey: 'key', fetcher: (async (url, init) => { diff --git a/test/translation-passes.test.ts b/test/translation-passes.test.ts index da241d0..6194f09 100644 --- a/test/translation-passes.test.ts +++ b/test/translation-passes.test.ts @@ -22,9 +22,9 @@ test("400% markup is five times the base cost; only exact, confirmed USD payment }); test("paid provider calls reserve before use, refund failures, and cached audio requires paid access", async () => { - const calls: string[] = []; let funded = true, providerOK = true; + const calls: string[] = []; let funded = true, providerOK = true, now = 0; const meter = { require: async () => { if (!funded) throw new SpeechError("Buy a pass",402); }, reserve: async (_by: string,kind:string,units:number) => { calls.push(`reserve:${kind}:${units}`); return "r"; }, commit: async () => { calls.push("commit"); }, refund: async () => { calls.push("refund"); } }; - const voice = new LiveVoice({apiKey:"test",billing:meter,fetcher:(async (url) => { + const voice = new LiveVoice({apiKey:"test",billing:meter,now:()=>now,fetcher:(async (url) => { if (String(url).includes("/voices?")) return Response.json({voices:[{voice_id:"stock",name:"Voice"}]}); calls.push("provider"); if (!providerOK) return new Response("",{status:500}); @@ -42,10 +42,13 @@ test("paid provider calls reserve before use, refund failures, and cached audio await assert.rejects(voice.grant("alice","channel"),/Buy a pass/); await assert.rejects(voice.hear(new Uint8Array(encodeWav(new Float32Array(32000).fill(.1))),"alice"),/Buy a pass/); funded=true;providerOK=false;calls.length=0; - await assert.rejects(voice.stream({...ask,text:"Unavailable"},"alice"),/could not generate/); + await assert.rejects(voice.stream({...ask,text:"Unavailable"},"alice"),/temporarily unavailable/); assert.deepEqual(calls,["reserve:voice:11","provider","refund"]); calls.length=0; - await assert.rejects(voice.hear(new Uint8Array(encodeWav(new Float32Array(32000).fill(.1))),"alice"),/could not run/); + await assert.rejects(voice.hear(new Uint8Array(encodeWav(new Float32Array(32000).fill(.1))),"alice"),/temporarily unavailable/); + assert.deepEqual(calls,[],"a provider cooldown does not reserve credit or call the provider"); + now=10_001; + await assert.rejects(voice.hear(new Uint8Array(encodeWav(new Float32Array(32000).fill(.1))),"alice"),/temporarily unavailable/); assert.deepEqual(calls,["reserve:transcription:32000","provider","refund"]); }); diff --git a/test/voice-provider.test.ts b/test/voice-provider.test.ts new file mode 100644 index 0000000..3206bbc --- /dev/null +++ b/test/voice-provider.test.ts @@ -0,0 +1,25 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { voiceProviderFailure } from "../src/voice-provider.ts"; +import { transientAudioError } from "../src/live-recovery.ts"; + +test("provider quota, billing and permissions are terminal even when ElevenLabs returns 401", async () => { + for (const [code, status] of [["quota_exceeded", 402], ["subscription_required", 402], ["invalid_api_key", 424], ["missing_permissions", 424]] as const) { + const failure = await voiceProviderFailure(Response.json({ detail: { status: code, message: "private provider account data" } }, { status: 401 })); + assert.equal(failure.error.status, status); + assert.equal(transientAudioError(failure.error), false); + assert.equal(failure.cooldownMs, 300_000); + assert.doesNotMatch(failure.error.message, /private provider/); + } +}); + +test("malformed failures are safe and rate-limit cooldowns are bounded", async () => { + const unknown = await voiceProviderFailure(new Response("private body", { status: 401 })); + assert.equal(unknown.error.status, 424); + assert.equal(unknown.code, "unknown"); + const retry = await voiceProviderFailure(new Response("busy", { status: 429, headers: { "retry-after": "9000" } })); + assert.equal(retry.cooldownMs, 300_000); + assert.equal(retry.error.status, 429); + const outage = await voiceProviderFailure(new Response("failure", { status: 500 })); + assert.equal(transientAudioError(outage.error), true); +}); From d9ec85eca1b83927f02730bb9cc65b3d18f82651 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 16 Sep 2026 05:36:50 +0000 Subject: [PATCH 2/3] Trim operations document whitespace --- docs/transcription-operations.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/transcription-operations.md b/docs/transcription-operations.md index ad153f4..e43e6ac 100644 --- a/docs/transcription-operations.md +++ b/docs/transcription-operations.md @@ -24,4 +24,3 @@ Configure daily spending ceilings with `NIXAMP_DUB_DAILY_CHARS`, `NIXAMP_DUB_USER_DAILY_AUDIO_SECONDS`. Audio includes overlapping context sent to Scribe. These are unit limits, not currency limits, and do not cover use of the provider key outside LiveVoice (including telephone voices). - From 8aa512d6d0f1c5cd659b9d864bbcfa69799c303c Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Wed, 16 Sep 2026 05:37:42 +0000 Subject: [PATCH 3/3] Preserve billing cooldown during concurrent failures and run regression tests in CI --- .github/workflows/translation.yml | 2 +- src/live-voice.ts | 4 +++- test/speaker-transcription.test.ts | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/translation.yml b/.github/workflows/translation.yml index 5ba13b8..e9557e3 100644 --- a/.github/workflows/translation.yml +++ b/.github/workflows/translation.yml @@ -36,4 +36,4 @@ jobs: bun-version: latest - run: bun install --frozen-lockfile - run: bun run typecheck - - run: bun test test/upgrade-allowance.test.ts test/translation-passes.test.ts test/shared-translation.test.ts test/live-voice.test.ts web/test/interpreter.test.ts web/test/live-voice.test.ts web/test/shared-translation.test.ts web/test/background.test.ts + - run: bun test test/upgrade-allowance.test.ts test/translation-passes.test.ts test/shared-translation.test.ts test/live-voice.test.ts test/voice-provider.test.ts test/speaker-transcription.test.ts web/test/interpreter.test.ts web/test/live-voice.test.ts web/test/shared-translation.test.ts web/test/background.test.ts diff --git a/src/live-voice.ts b/src/live-voice.ts index 7711bd6..55deb3a 100644 --- a/src/live-voice.ts +++ b/src/live-voice.ts @@ -46,7 +46,9 @@ export class LiveVoice { private async providerFailed(response: Response, operation: string): Promise { const failure = await voiceProviderFailure(response); - this.providerPause = { until: this.now() + failure.cooldownMs, error: failure.error }; + const until = this.now() + failure.cooldownMs; + // Another in-flight response must not shorten an existing billing pause. + if (!this.providerPause || this.providerPause.until < until) this.providerPause = { until, error: failure.error }; console.error(JSON.stringify({ event: "voice_provider_failure", provider: "elevenlabs", operation, status: response.status, code: failure.code, cooldownMs: failure.cooldownMs })); throw failure.error; } diff --git a/test/speaker-transcription.test.ts b/test/speaker-transcription.test.ts index 51a3e3d..b2e4982 100644 --- a/test/speaker-transcription.test.ts +++ b/test/speaker-transcription.test.ts @@ -31,6 +31,22 @@ test('a provider rejection refunds once, blocks other accounts during cooldown, assert.deepEqual([calls, reserved, refunded, committed], [2, 2, 1, 1]); }); +test('an in-flight transient failure cannot shorten a provider billing cooldown', async () => { + let now = 0; + const finish: ((response: Response) => void)[] = []; + const voice = new LiveVoice({ apiKey: 'key', now: () => now, fetcher: (() => new Promise(resolve => finish.push(resolve))) as typeof fetch }); + const first = assert.rejects(voice.hear(audio(), 'alice'), /credits are exhausted/); + const second = assert.rejects(voice.hear(audio(), 'bob'), /temporarily unavailable/); + while (finish.length < 2) await new Promise(resolve => setTimeout(resolve, 1)); + finish[0]!(Response.json({ detail: { status: 'quota_exceeded' } }, { status: 401 })); + await first; + finish[1]!(new Response('outage', { status: 500 })); + await second; + now = 11_000; + await assert.rejects(voice.hear(audio(), 'charlie'), /credits are exhausted/); + assert.equal(finish.length, 2); +}); + test('Scribe auto-detects native language, separates speakers, and receives canonical bounded WAV only', async () => { let calls = 0; const voice = new LiveVoice({ apiKey: 'key', fetcher: (async (url, init) => {