Stop retrying ElevenLabs access failures and add provider safeguards

.github/workflows/translation.yml

@@ -36,4 +36,4 @@ jobs:
           bun-version: latest
       - run: bun install --frozen-lockfile
       - run: bun run typecheck
-      - run: bun test test/upgrade-allowance.test.ts test/translation-passes.test.ts test/shared-translation.test.ts test/live-voice.test.ts web/test/interpreter.test.ts web/test/live-voice.test.ts web/test/shared-translation.test.ts web/test/background.test.ts
+      - run: bun test test/upgrade-allowance.test.ts test/translation-passes.test.ts test/shared-translation.test.ts test/live-voice.test.ts test/voice-provider.test.ts test/speaker-transcription.test.ts web/test/interpreter.test.ts web/test/live-voice.test.ts web/test/shared-translation.test.ts web/test/background.test.ts

docs/transcription-operations.md

@@ -0,0 +1,26 @@
+# Transcription provider controls
+
+Speaker transcription uses ElevenLabs Scribe v2. Ordinary `/speech/transcribe`
+uses local Whisper and does not spend ElevenLabs credits.
+
+ElevenLabs may report exhausted credits or permission problems with HTTP 401.
+The server reads the provider error code and returns a terminal 402 (provider
+billing/quota) or 424 (provider access), rather than a retryable 502. It refunds
+the listener's reservation when the provider rejects a request.
+
+After a rejection, the LiveVoice instance pauses provider requests for five
+minutes for billing/access failures, 30–300 seconds for rate limits, or ten
+seconds for other provider errors. This cooldown is per server process; the
+existing spending limits are persisted in PostgreSQL and shared by replicas.
+
+`voice_provider_failure` logs the operation, HTTP status, allowlisted provider
+code, and cooldown. `voice_provider_usage` records accepted audio seconds or
+characters and a hashed account/source identifier. Neither contains audio,
+transcript text, API keys, or raw provider error bodies. Shared streams use a
+source identifier here; listener charges remain in `translation_usage`.
+
+Configure daily spending ceilings with `NIXAMP_DUB_DAILY_CHARS`,
+`NIXAMP_DUB_DAILY_AUDIO_SECONDS`, `NIXAMP_DUB_USER_DAILY_CHARS`, and
+`NIXAMP_DUB_USER_DAILY_AUDIO_SECONDS`. Audio includes overlapping context sent
+to Scribe. These are unit limits, not currency limits, and do not cover use of
+the provider key outside LiveVoice (including telephone voices).

src/live-voice.ts

@@ -6,6 +6,7 @@ import type { VoiceProfile } from "./voice-profile.ts";
 import { Guard } from "./guard.ts";
 import type { TranslationMeter } from "./translation-passes.ts";
 import type { Queryable } from "./follows.ts";
+import { voiceProviderFailure } from "./voice-provider.ts";
 
 export const LIVE_VOICE_MODEL = "eleven_flash_v2_5";
 export const LIVE_VOICE_RATE = 16_000;
@@ -37,6 +38,20 @@ export class LiveVoice {
   private readonly dailyAudioSeconds: number;
   private readonly userDailyChars: number;
   private readonly userDailyAudioSeconds: number;
+  private providerPause: { until: number; error: SpeechError } | null = null;
+
+  private checkProvider(): void {
+    if (this.providerPause && this.providerPause.until > this.now()) throw this.providerPause.error;
+  }
+
+  private async providerFailed(response: Response, operation: string): Promise<never> {
+    const failure = await voiceProviderFailure(response);
+    const until = this.now() + failure.cooldownMs;
+    // Another in-flight response must not shorten an existing billing pause.
+    if (!this.providerPause || this.providerPause.until < until) this.providerPause = { until, error: failure.error };
+    console.error(JSON.stringify({ event: "voice_provider_failure", provider: "elevenlabs", operation, status: response.status, code: failure.code, cooldownMs: failure.cooldownMs }));
+    throw failure.error;
+  }
 
   constructor(options: { apiKey?: string; fetcher?: typeof fetch; now?: () => number; charsPerMinute?: number; dailyChars?: number; dailyAudioSeconds?: number; userDailyChars?: number; userDailyAudioSeconds?: number; db?: Queryable; billing?: TranslationMeter } = {}) {
     this.key = options.apiKey ?? process.env["ELEVENLABS_API_KEY"] ?? "";
@@ -60,6 +75,7 @@ export class LiveVoice {
   async hear(bytes: Uint8Array, by: string, signal?: AbortSignal, meter = this.billing, resource = ""): Promise<SpeakerTranscript> {
     if (!this.available()) throw new SpeechError("speaker voices are unavailable", 503);
     await meter?.require(by, resource);
+    this.checkProvider();
     const wav = decodeWav(bytes);
     const seconds = wav.samples.length / wav.rate;
     if (wav.rate !== 16000 || wav.channels !== 1 || seconds < 0.2 || seconds > 15.1) throw new SpeechError("send up to 15 seconds of mono 16 kHz WAV", 400);
@@ -77,6 +93,7 @@ export class LiveVoice {
       await this.reserve(`scribe:user:${by}`, billed, this.userDailyAudioSeconds, 86_400_000);
       await this.reserve("scribe:server", billed, this.dailyAudioSeconds, 86_400_000);
       signal?.throwIfAborted();
+      this.checkProvider();
       const form = new FormData();
       // Canonical PCM prevents a crafted container from billing more audio
       // than the duration we validated, and removes uploaded metadata.
@@ -91,8 +108,9 @@ export class LiveVoice {
         method: "POST", headers: { "xi-api-key": this.key }, body: form,
         signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000),
       });
-      if (!answer.ok) throw new SpeechError("speaker transcription could not run; check provider quota and permissions", answer.status === 429 ? 429 : 502);
+      if (!answer.ok) await this.providerFailed(answer, "transcription");
       accepted = true;
+      console.info(JSON.stringify({ event: "voice_provider_usage", provider: "elevenlabs", operation: "transcription", account: createHash("sha256").update(by).digest("hex").slice(0, 16), audioSeconds: seconds }));
       if (reservation) await meter!.commit(reservation);
       return speakerTurns(await answer.json() as ScribeResult, wav);
     } catch (error) {
@@ -103,12 +121,13 @@ export class LiveVoice {
 
   async voices(): Promise<LiveVoiceChoice[]> {
     if (!this.available()) throw new SpeechError("translated audio needs ELEVENLABS_API_KEY on the account server", 503);
+    this.checkProvider();
     if (!this.catalog || this.now() >= this.catalogUntil) {
       this.catalogUntil = this.now() + 3600_000;
       this.catalog = this.fetcher("https://api.elevenlabs.io/v2/voices?page_size=100&voice_type=default", {
         headers: { "xi-api-key": this.key }, signal: AbortSignal.timeout(8000),
       }).then(async response => {
-        if (!response.ok) throw new SpeechError("ElevenLabs could not list voices; check the server key and its voice permissions", 503);
+        if (!response.ok) await this.providerFailed(response, "voices");
         const body = await response.json() as { voices: { voice_id: string; name: string; labels?: Record<string, string> }[] };
         return body.voices.filter(voice => /^[a-zA-Z0-9_-]+$/.test(voice.voice_id)).map(voice => ({
           id: voice.voice_id, name: voice.name, gender: voice.labels?.["gender"] ?? "neutral", language: voice.labels?.["language"] ?? "",
@@ -238,6 +257,7 @@ export class LiveVoice {
     let accepted = false;
     try {
       await this.charge(by, ask.channel ?? "direct", text.length);
+      this.checkProvider();
       signal?.throwIfAborted();
       reservation = await meter?.reserve(by, "voice", text.length, ask.channel);
       const response = await this.fetcher(`https://api.elevenlabs.io/v1/text-to-speech/${voice.id}/stream?output_format=pcm_16000`, {
@@ -246,9 +266,11 @@ export class LiveVoice {
         body: JSON.stringify({ text, model_id: LIVE_VOICE_MODEL, language_code: ask.language }),
         signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(10_000)]) : AbortSignal.timeout(10_000),
       });
-      if (!response.ok || !response.body) throw new SpeechError(response.status === 429 ? "ElevenLabs audio quota is temporarily exhausted" : "ElevenLabs could not generate audio; check the server key and quota", response.status === 429 ? 429 : 502);
+      if (!response.ok) await this.providerFailed(response, "synthesis");
+      if (!response.body) throw new SpeechError("ElevenLabs returned no audio", 502);
       // Once the provider accepts, aborting playback cannot refund heard audio.
       accepted = true;
+      console.info(JSON.stringify({ event: "voice_provider_usage", provider: "elevenlabs", operation: "synthesis", account: createHash("sha256").update(by).digest("hex").slice(0, 16), characters: text.length }));
       if (reservation) await meter!.commit(reservation);
       const [play, keep] = response.body.tee();
       void (async () => {

src/voice-provider.ts

@@ -0,0 +1,21 @@
+import { SpeechError } from "./speech.ts";
+
+/** Only fixed messages/codes leave this boundary: provider bodies can contain
+ * account information. Authorization and billing errors must not be retried
+ * as transient 502s by the live player. */
+export async function voiceProviderFailure(response: Response): Promise<{ error: SpeechError; code: string; cooldownMs: number }> {
+  let code = "unknown";
+  try {
+    const body = await response.json() as { detail?: { status?: unknown } };
+    const status = body?.detail?.status;
+    if (typeof status === "string" && ["quota_exceeded", "invalid_api_key", "missing_permissions", "subscription_required", "subscription_expired", "payment_required", "too_many_concurrent_requests", "rate_limit_exceeded"].includes(status)) code = status;
+  } catch { /* Non-JSON provider failures still have an HTTP status. */ }
+  if (code === "quota_exceeded") return { code, cooldownMs: 300_000, error: new SpeechError("ElevenLabs credits are exhausted. The site owner needs to check the provider balance.", 402) };
+  if (["subscription_required", "subscription_expired", "payment_required"].includes(code) || response.status === 402) return { code, cooldownMs: 300_000, error: new SpeechError("ElevenLabs billing needs attention. The site owner needs to check the provider subscription.", 402) };
+  if (["invalid_api_key", "missing_permissions"].includes(code) || [401, 403].includes(response.status)) return { code, cooldownMs: 300_000, error: new SpeechError("ElevenLabs rejected transcription or voice access. The site owner needs to check the provider key, permissions, and billing.", 424) };
+  if (response.status === 429) {
+    const retry = Number(response.headers.get("retry-after"));
+    return { code, cooldownMs: Math.min(300_000, Math.max(30_000, Number.isFinite(retry) ? retry * 1000 : 0)), error: new SpeechError("ElevenLabs is rate limited. Wait before enabling translated audio again.", 429) };
+  }
+  return { code, cooldownMs: 10_000, error: new SpeechError("ElevenLabs is temporarily unavailable. Try translated audio again shortly.", 502) };
+}

test/speaker-transcription.test.ts

@@ -15,6 +15,38 @@ const result = { language_code: 'spa', words: [
   { type: 'word', text: 'La pelea sigue.', start: 2.5, end: 4.9, speaker_id: 'speaker_1' },
 ] };
 
+test('a provider rejection refunds once, blocks other accounts during cooldown, then recovers', async () => {
+  let now = 0, calls = 0, reserved = 0, refunded = 0, committed = 0;
+  const billing = { require: async () => {}, reserve: async () => { reserved++; return 'reservation'; }, refund: async () => { refunded++; }, commit: async () => { committed++; } };
+  const voice = new LiveVoice({ apiKey: 'key', now: () => now, billing, fetcher: (async () => {
+    calls++;
+    return calls === 1 ? Response.json({ detail: { status: 'quota_exceeded' } }, { status: 401 }) : Response.json(result);
+  }) as typeof fetch });
+  await assert.rejects(voice.hear(audio(), 'alice'), error => error instanceof SpeechError && error.status === 402);
+  await assert.rejects(voice.hear(audio(), 'bob'), /credits are exhausted/);
+  await assert.rejects(voice.voices(), /credits are exhausted/);
+  assert.deepEqual([calls, reserved, refunded, committed], [1, 1, 1, 0]);
+  now = 300_001;
+  await voice.hear(audio(), 'bob');
+  assert.deepEqual([calls, reserved, refunded, committed], [2, 2, 1, 1]);
+});
+
+test('an in-flight transient failure cannot shorten a provider billing cooldown', async () => {
+  let now = 0;
+  const finish: ((response: Response) => void)[] = [];
+  const voice = new LiveVoice({ apiKey: 'key', now: () => now, fetcher: (() => new Promise<Response>(resolve => finish.push(resolve))) as typeof fetch });
+  const first = assert.rejects(voice.hear(audio(), 'alice'), /credits are exhausted/);
+  const second = assert.rejects(voice.hear(audio(), 'bob'), /temporarily unavailable/);
+  while (finish.length < 2) await new Promise(resolve => setTimeout(resolve, 1));
+  finish[0]!(Response.json({ detail: { status: 'quota_exceeded' } }, { status: 401 }));
+  await first;
+  finish[1]!(new Response('outage', { status: 500 }));
+  await second;
+  now = 11_000;
+  await assert.rejects(voice.hear(audio(), 'charlie'), /credits are exhausted/);
+  assert.equal(finish.length, 2);
+});
+
 test('Scribe auto-detects native language, separates speakers, and receives canonical bounded WAV only', async () => {
   let calls = 0;
   const voice = new LiveVoice({ apiKey: 'key', fetcher: (async (url, init) => {

test/translation-passes.test.ts

@@ -22,9 +22,9 @@ test("400% markup is five times the base cost; only exact, confirmed USD payment
 });
 
 test("paid provider calls reserve before use, refund failures, and cached audio requires paid access", async () => {
-  const calls: string[] = []; let funded = true, providerOK = true;
+  const calls: string[] = []; let funded = true, providerOK = true, now = 0;
   const meter = { require: async () => { if (!funded) throw new SpeechError("Buy a pass",402); }, reserve: async (_by: string,kind:string,units:number) => { calls.push(`reserve:${kind}:${units}`); return "r"; }, commit: async () => { calls.push("commit"); }, refund: async () => { calls.push("refund"); } };
-  const voice = new LiveVoice({apiKey:"test",billing:meter,fetcher:(async (url) => {
+  const voice = new LiveVoice({apiKey:"test",billing:meter,now:()=>now,fetcher:(async (url) => {
     if (String(url).includes("/voices?")) return Response.json({voices:[{voice_id:"stock",name:"Voice"}]});
     calls.push("provider");
     if (!providerOK) return new Response("",{status:500});
@@ -42,10 +42,13 @@ test("paid provider calls reserve before use, refund failures, and cached audio
   await assert.rejects(voice.grant("alice","channel"),/Buy a pass/);
   await assert.rejects(voice.hear(new Uint8Array(encodeWav(new Float32Array(32000).fill(.1))),"alice"),/Buy a pass/);
   funded=true;providerOK=false;calls.length=0;
-  await assert.rejects(voice.stream({...ask,text:"Unavailable"},"alice"),/could not generate/);
+  await assert.rejects(voice.stream({...ask,text:"Unavailable"},"alice"),/temporarily unavailable/);
   assert.deepEqual(calls,["reserve:voice:11","provider","refund"]);
   calls.length=0;
-  await assert.rejects(voice.hear(new Uint8Array(encodeWav(new Float32Array(32000).fill(.1))),"alice"),/could not run/);
+  await assert.rejects(voice.hear(new Uint8Array(encodeWav(new Float32Array(32000).fill(.1))),"alice"),/temporarily unavailable/);
+  assert.deepEqual(calls,[],"a provider cooldown does not reserve credit or call the provider");
+  now=10_001;
+  await assert.rejects(voice.hear(new Uint8Array(encodeWav(new Float32Array(32000).fill(.1))),"alice"),/temporarily unavailable/);
   assert.deepEqual(calls,["reserve:transcription:32000","provider","refund"]);
 });
 

test/voice-provider.test.ts

@@ -0,0 +1,25 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { voiceProviderFailure } from "../src/voice-provider.ts";
+import { transientAudioError } from "../src/live-recovery.ts";
+
+test("provider quota, billing and permissions are terminal even when ElevenLabs returns 401", async () => {
+  for (const [code, status] of [["quota_exceeded", 402], ["subscription_required", 402], ["invalid_api_key", 424], ["missing_permissions", 424]] as const) {
+    const failure = await voiceProviderFailure(Response.json({ detail: { status: code, message: "private provider account data" } }, { status: 401 }));
+    assert.equal(failure.error.status, status);
+    assert.equal(transientAudioError(failure.error), false);
+    assert.equal(failure.cooldownMs, 300_000);
+    assert.doesNotMatch(failure.error.message, /private provider/);
+  }
+});
+
+test("malformed failures are safe and rate-limit cooldowns are bounded", async () => {
+  const unknown = await voiceProviderFailure(new Response("private body", { status: 401 }));
+  assert.equal(unknown.error.status, 424);
+  assert.equal(unknown.code, "unknown");
+  const retry = await voiceProviderFailure(new Response("busy", { status: 429, headers: { "retry-after": "9000" } }));
+  assert.equal(retry.cooldownMs, 300_000);
+  assert.equal(retry.error.status, 429);
+  const outage = await voiceProviderFailure(new Response("failure", { status: 500 }));
+  assert.equal(transientAudioError(outage.error), true);
+});