From 77026f229b0aee4cdecc56e8a4b797fc67a9d698 Mon Sep 17 00:00:00 2001 From: Lucas Proxmox Date: Wed, 7 Oct 2026 14:27:25 +0200 Subject: [PATCH 1/3] feat(tts): a fallback speech service with a short leash on the first TTS_FALLBACK_URL names a second OpenAI-compatible speech service (Kokoro) that speaks when the voice (TTS_URL, now Chatterbox Turbo on the Arc B580) fails or does not answer within TTS_PRIMARY_TIMEOUT_MS (default 10 s; the full 15 s applies without a fallback). The B580 is shared with Ollama, and Chatterbox slows ~8x while Ollama's 9B runs; without this the plugin would fall back to the Mac's local voice instead. Fallback settings are read per call. Nothing else changes when TTS_FALLBACK_URL is unset. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 5 +- packages/backend/src/ai/__tests__/tts.test.ts | 46 ++++++++++++++ packages/backend/src/ai/tts.ts | 63 ++++++++++++------- 3 files changed, 91 insertions(+), 23 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 14b83fa..8d554c8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -221,8 +221,9 @@ Session tokens and cost (migration 055, `db/tokenUsageQueries.ts`). Costs are ** The plugin's voice announcer speaks through `/api/ai/voice-audio`, so users get a natural voice without installing a model. Both voice endpoints live in `routes/voice.ts` (mounted under `/api/ai`; the Gemini guard and rate limiter are shared with `routes/ai.ts` via `routes/aiGuards.ts`). -- **Service:** [Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI) CPU image (`ghcr.io/remsky/kokoro-fastapi-cpu`, ~3.3 GB) next to the backend on the internal network only, never exposed. It speaks the OpenAI-compatible `/v1/audio/speech`; `ai/tts.ts` never throws and returns null on any failure. -- **Env:** `TTS_URL` (e.g. `http://kokoro:8880`; unset disables the feature), `TTS_VOICE` (default `am_michael`), `TTS_SPEED` (default `1.2`; the plugin sends its own, also 1.2 by default), `TTS_VOLUME` (Kokoro `volume_multiplier`, default `2`: twice the default loudness, peaks stay limited), `TTS_MODEL` (default `kokoro`). +- **Services:** the voice is **Chatterbox Turbo** (350M) on the homelab's Intel Arc B580, a small server in the deployment stack (`/opt/stacks/devscope/chatterbox`, built locally, Intel PyTorch XPU) that speaks the same OpenAI-compatible `/v1/audio/speech` and does speed with ffmpeg atempo and volume as a gain. **Kokoro** (Kokoro-FastAPI CPU image) stays as the fallback. Both run on the internal network only, never exposed. `ai/tts.ts` never throws and returns null when neither answers. +- **Env:** `TTS_URL` (the voice, e.g. `http://chatterbox:8880`; unset disables the feature), `TTS_FALLBACK_URL` (e.g. `http://kokoro:8880`; tried when the voice fails or is slow), `TTS_PRIMARY_TIMEOUT_MS` (how long the voice gets when there is a fallback, default 10000), `TTS_VOICE` (Kokoro voice, default `am_michael`; Chatterbox ignores it), `TTS_SPEED` (default `1.2`; the plugin sends its own, also 1.2 by default), `TTS_VOLUME` (default `2`: Kokoro's `volume_multiplier`, and Chatterbox's baseline gain), `TTS_MODEL` (default `kokoro`). +- **Why a fallback:** the B580 is shared with Ollama. While Ollama's 9B model runs, Chatterbox slows about 8x (a 15 s summary took about 40 s); its server answers 503 at once when busy, and the backend gives it `TTS_PRIMARY_TIMEOUT_MS`, so Kokoro speaks instead. The voice then differs for that sentence, which is the trade for not waiting. - **Pronunciation:** `services/speakable.ts` rewrites every voiced text (and every voice summary) before Kokoro reads it: word-like acronyms (JSON as jay-son, README, YAML), file extensions (`voice dot jay-son`), version numbers, units after numbers (`5s`, `150ms`, `1.2x`, `30m`), `e.g.`/`vs`, and camel/snake/PascalCase identifiers. All-caps acronyms (API, CLI, PR) and vowel-less short words (npm, jq) are left alone: Kokoro already spells them, and spacing letters out reads "A" as the article. Check a spelling against Kokoro's phonemizer: `POST http://kokoro:8880/dev/phonemize {"text", "language": "a"}`. - **Privacy:** what is voiced is either the text the backend itself wrote, or a spoken explanation Claude wrote for `/devscope:explain` (sent in pieces of at most 440 characters, the next one fetched while the current one plays). The plugin never sends `private` sessions here; they use a local voice. Nothing is stored. diff --git a/packages/backend/src/ai/__tests__/tts.test.ts b/packages/backend/src/ai/__tests__/tts.test.ts index 26488a1..4696fdb 100644 --- a/packages/backend/src/ai/__tests__/tts.test.ts +++ b/packages/backend/src/ai/__tests__/tts.test.ts @@ -55,4 +55,50 @@ describe("tts client", () => { }); expect(await tts.synthesize("x")).toBeNull(); }); + + describe("with a fallback (TTS_FALLBACK_URL)", () => { + afterEach(() => { + delete process.env.TTS_FALLBACK_URL; + delete process.env.TTS_PRIMARY_TIMEOUT_MS; + }); + + test("the voice answers: the fallback is never asked", async () => { + process.env.TTS_FALLBACK_URL = "http://fallback.test:8880"; + const f = stubFetch(async () => wav()); + expect((await tts.synthesize("x"))?.byteLength).toBe(4); + expect(f.mock.calls.map((c) => c[0])).toEqual(["http://kokoro.test:8880/v1/audio/speech"]); + }); + + test("the voice is busy or broken: the fallback speaks, with the same request", async () => { + process.env.TTS_FALLBACK_URL = "http://fallback.test:8880/"; + const f = stubFetch(async (url: string) => + url.startsWith("http://kokoro.test") ? Response.json({ error: "busy" }, { status: 503 }) : wav(), + ); + expect((await tts.synthesize("cloud needs you", "am_michael", 1.2, 2))?.byteLength).toBe(4); + expect(f.mock.calls.map((c) => c[0])).toEqual([ + "http://kokoro.test:8880/v1/audio/speech", + "http://fallback.test:8880/v1/audio/speech", + ]); + expect(f.mock.calls[1]![1].body).toBe(f.mock.calls[0]![1].body); + }); + + test("the voice gets only TTS_PRIMARY_TIMEOUT_MS before the fallback", async () => { + process.env.TTS_FALLBACK_URL = "http://fallback.test:8880"; + process.env.TTS_PRIMARY_TIMEOUT_MS = "50"; + const started = Date.now(); + stubFetch((url: string, init: RequestInit) => + url.startsWith("http://kokoro.test") + ? new Promise((_, reject) => init.signal!.addEventListener("abort", () => reject(new Error("timeout")))) + : Promise.resolve(wav()), + ); + expect((await tts.synthesize("x"))?.byteLength).toBe(4); + expect(Date.now() - started).toBeLessThan(2000); + }); + + test("both fail: null, so the plugin uses a local voice", async () => { + process.env.TTS_FALLBACK_URL = "http://fallback.test:8880"; + stubFetch(async () => new Response("nope", { status: 500 })); + expect(await tts.synthesize("x")).toBeNull(); + }); + }); }); diff --git a/packages/backend/src/ai/tts.ts b/packages/backend/src/ai/tts.ts index eb36834..9a05739 100644 --- a/packages/backend/src/ai/tts.ts +++ b/packages/backend/src/ai/tts.ts @@ -1,16 +1,26 @@ /** - * Local text-to-speech client (Kokoro-FastAPI on the homelab). + * Local text-to-speech client: the homelab's speech services. * - * Voices the plugin's "a session needs you" announcements, so users get a - * natural voice without installing a model. The service speaks the - * OpenAI-compatible `/v1/audio/speech` API. Text never leaves the box. + * Voices the plugin's announcements, reply summaries and explanations, so + * users get a natural voice without installing a model. Each service speaks + * the OpenAI-compatible `/v1/audio/speech` API. Text never leaves the box. + * + * TTS_URL is the voice (Chatterbox Turbo on the Arc B580 in production); + * TTS_FALLBACK_URL, when set, is tried when it fails or answers too slowly + * (Kokoro on the CPU). The GPU is shared with Ollama and Chatterbox slows ~8x + * while both run, so with a fallback the first service gets only + * TTS_PRIMARY_TIMEOUT_MS. Kokoro-only fields (voice name, volume_multiplier) + * are sent to both; Chatterbox's server ignores the voice name. * * Like the embeddings client, nothing here throws: it returns null on a - * missing URL, timeout, non-200 or non-audio body, and the route answers 503 - * so the plugin falls back to a local voice. + * missing URL, timeout, non-200 or non-audio body from every service, and the + * route answers 503 so the plugin falls back to a local voice. */ -const TTS_URL = process.env.TTS_URL?.replace(/\/+$/, ""); +const trimUrl = (url: string | undefined) => url?.replace(/\/+$/, "") || undefined; +const TTS_URL = trimUrl(process.env.TTS_URL); +// Read per call, so the fallback can be configured without a code path at import. +const fallbackUrl = () => trimUrl(process.env.TTS_FALLBACK_URL); if (!TTS_URL) { console.warn("[tts] TTS_URL not set — server voice disabled"); } @@ -26,12 +36,14 @@ export const TTS_DEFAULTS = { } as const; const TIMEOUT_MS = 15_000; +/** How long the voice gets before the fallback speaks instead (only when there is one). */ +const primaryTimeoutMs = () => Number(process.env.TTS_PRIMARY_TIMEOUT_MS ?? 10_000); export function isTtsAvailable(): boolean { return !!TTS_URL; } -/** WAV audio for `text`, or null when the service is unavailable. */ +/** WAV audio for `text` from the voice, else the fallback; null when neither answers. */ export async function synthesize( text: string, voice: string = TTS_DEFAULTS.voice, @@ -39,32 +51,41 @@ export async function synthesize( volume: number = TTS_DEFAULTS.volume, ): Promise { if (!TTS_URL) return null; + const body = JSON.stringify({ + model: TTS_MODEL, + input: text, + voice, + speed, + volume_multiplier: volume, + response_format: "wav", + }); + const fallback = fallbackUrl(); + const audio = await speak(TTS_URL, body, fallback ? primaryTimeoutMs() : TIMEOUT_MS); + if (audio || !fallback) return audio; + console.warn("[tts] voice unavailable, using the fallback"); + return speak(fallback, body, TIMEOUT_MS); +} + +async function speak(url: string, body: string, timeoutMs: number): Promise { try { - const res = await fetch(`${TTS_URL}/v1/audio/speech`, { + const res = await fetch(`${url}/v1/audio/speech`, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ - model: TTS_MODEL, - input: text, - voice, - speed, - volume_multiplier: volume, - response_format: "wav", - }), - signal: AbortSignal.timeout(TIMEOUT_MS), + body, + signal: AbortSignal.timeout(timeoutMs), }); if (!res.ok) { - console.warn(`[tts] HTTP ${res.status} from TTS service`); + console.warn(`[tts] HTTP ${res.status} from ${url}`); return null; } if (!res.headers.get("content-type")?.startsWith("audio/")) { - console.warn("[tts] non-audio response from TTS service"); + console.warn(`[tts] non-audio response from ${url}`); return null; } const audio = await res.arrayBuffer(); return audio.byteLength > 0 ? audio : null; } catch (err) { - console.warn("[tts] request failed:", err instanceof Error ? err.message : err); + console.warn(`[tts] request to ${url} failed:`, err instanceof Error ? err.message : err); return null; } } From 14183db4a91ef27a1b59a3ecc9c330ebd9465bfc Mon Sep 17 00:00:00 2001 From: Lucas Proxmox Date: Wed, 7 Oct 2026 14:29:56 +0200 Subject: [PATCH 2/3] feat(tts): named speech services; a request can pick its voice TTS_SERVICES=chatterbox=http://chatterbox:8880,kokoro=http://kokoro:8880 names the services in order: the first is the default, the rest are fallbacks. /api/ai/voice-audio takes an optional `model` (a service name) that is tried first, so the plugin's `/devscope:voice model kokoro` keeps Kokoro for one user while Chatterbox stays the default. Every service but the last gets TTS_PRIMARY_TIMEOUT_MS. GET /api/ai/voice-models lists them. TTS_URL/TTS_FALLBACK_URL still work as the unnamed form. Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/backend/src/ai/__tests__/tts.test.ts | 36 +++++++++ packages/backend/src/ai/tts.ts | 73 +++++++++++++------ .../src/routes/__tests__/voice.test.ts | 14 +++- packages/backend/src/routes/voice.ts | 9 ++- packages/backend/src/services/voiceSummary.ts | 2 + 5 files changed, 109 insertions(+), 25 deletions(-) diff --git a/packages/backend/src/ai/__tests__/tts.test.ts b/packages/backend/src/ai/__tests__/tts.test.ts index 4696fdb..f5ee397 100644 --- a/packages/backend/src/ai/__tests__/tts.test.ts +++ b/packages/backend/src/ai/__tests__/tts.test.ts @@ -101,4 +101,40 @@ describe("tts client", () => { expect(await tts.synthesize("x")).toBeNull(); }); }); + + describe("named services (TTS_SERVICES)", () => { + afterEach(() => { + delete process.env.TTS_SERVICES; + delete process.env.TTS_PRIMARY_TIMEOUT_MS; + }); + const urls = (f: { mock: { calls: any[][] } }) => f.mock.calls.map((c) => c[0]); + + test("parses name=url pairs in order and skips malformed ones", () => { + process.env.TTS_SERVICES = " chatterbox = http://chatterbox:8880/ ,bad entry,Kokoro=http://kokoro:8880,=http://x"; + expect(tts.ttsServices()).toEqual([ + { name: "chatterbox", url: "http://chatterbox:8880" }, + { name: "kokoro", url: "http://kokoro:8880" }, + ]); + }); + + test("the default is the first; a requested one goes first; an unknown name is ignored", async () => { + process.env.TTS_SERVICES = "chatterbox=http://c.test,kokoro=http://k.test"; + let f = stubFetch(async () => wav()); + await tts.synthesize("x"); + expect(urls(f)).toEqual(["http://c.test/v1/audio/speech"]); + f = stubFetch(async () => wav()); + await tts.synthesize("x", undefined, undefined, undefined, "kokoro"); + expect(urls(f)).toEqual(["http://k.test/v1/audio/speech"]); + f = stubFetch(async () => wav()); + await tts.synthesize("x", undefined, undefined, undefined, "nope"); + expect(urls(f)).toEqual(["http://c.test/v1/audio/speech"]); + }); + + test("a requested voice that fails falls back to the others", async () => { + process.env.TTS_SERVICES = "chatterbox=http://c.test,kokoro=http://k.test"; + const f = stubFetch(async (url: string) => (url.startsWith("http://k.test") ? new Response("x", { status: 500 }) : wav())); + expect((await tts.synthesize("x", undefined, undefined, undefined, "kokoro"))?.byteLength).toBe(4); + expect(urls(f)).toEqual(["http://k.test/v1/audio/speech", "http://c.test/v1/audio/speech"]); + }); + }); }); diff --git a/packages/backend/src/ai/tts.ts b/packages/backend/src/ai/tts.ts index 9a05739..24f94dc 100644 --- a/packages/backend/src/ai/tts.ts +++ b/packages/backend/src/ai/tts.ts @@ -5,24 +5,46 @@ * users get a natural voice without installing a model. Each service speaks * the OpenAI-compatible `/v1/audio/speech` API. Text never leaves the box. * - * TTS_URL is the voice (Chatterbox Turbo on the Arc B580 in production); - * TTS_FALLBACK_URL, when set, is tried when it fails or answers too slowly - * (Kokoro on the CPU). The GPU is shared with Ollama and Chatterbox slows ~8x - * while both run, so with a fallback the first service gets only - * TTS_PRIMARY_TIMEOUT_MS. Kokoro-only fields (voice name, volume_multiplier) - * are sent to both; Chatterbox's server ignores the voice name. + * TTS_SERVICES names the services in order, `name=url,name=url`: the first is + * the default voice (Chatterbox Turbo on the Arc B580 in production), the rest + * are fallbacks (Kokoro on the CPU). A request may name the one it wants + * (`/devscope:voice model kokoro`); it is tried first, the others after it. + * TTS_URL (+ TTS_FALLBACK_URL) is the unnamed older form. The GPU is shared + * with Ollama and Chatterbox slows ~8x while both run, so every service but + * the last gets only TTS_PRIMARY_TIMEOUT_MS. Kokoro-only fields (voice name, + * volume_multiplier) are sent to each; Chatterbox's server ignores the name. * * Like the embeddings client, nothing here throws: it returns null on a * missing URL, timeout, non-200 or non-audio body from every service, and the * route answers 503 so the plugin falls back to a local voice. */ -const trimUrl = (url: string | undefined) => url?.replace(/\/+$/, "") || undefined; -const TTS_URL = trimUrl(process.env.TTS_URL); -// Read per call, so the fallback can be configured without a code path at import. -const fallbackUrl = () => trimUrl(process.env.TTS_FALLBACK_URL); -if (!TTS_URL) { - console.warn("[tts] TTS_URL not set — server voice disabled"); +const trimUrl = (url: string | undefined) => url?.trim().replace(/\/+$/, "") || undefined; +export const SERVICE_NAME = /^[a-z0-9-]{1,32}$/; + +export type TtsService = { name: string; url: string }; + +/** The configured speech services, default first. Read per call. */ +export function ttsServices(): TtsService[] { + const list = process.env.TTS_SERVICES; + if (list) { + return list.split(",").flatMap((entry) => { + const at = entry.indexOf("="); + const name = entry.slice(0, at).trim().toLowerCase(); + const url = trimUrl(entry.slice(at + 1)); + return at > 0 && url && SERVICE_NAME.test(name) ? [{ name, url }] : []; + }); + } + const primary = trimUrl(process.env.TTS_URL); + const fallback = trimUrl(process.env.TTS_FALLBACK_URL); + return [ + ...(primary ? [{ name: "default", url: primary }] : []), + ...(primary && fallback ? [{ name: "fallback", url: fallback }] : []), + ]; +} + +if (ttsServices().length === 0) { + console.warn("[tts] neither TTS_SERVICES nor TTS_URL set — server voice disabled"); } export const TTS_MODEL = process.env.TTS_MODEL ?? "kokoro"; @@ -36,21 +58,28 @@ export const TTS_DEFAULTS = { } as const; const TIMEOUT_MS = 15_000; -/** How long the voice gets before the fallback speaks instead (only when there is one). */ +/** How long a service gets before the next one speaks instead (all but the last). */ const primaryTimeoutMs = () => Number(process.env.TTS_PRIMARY_TIMEOUT_MS ?? 10_000); export function isTtsAvailable(): boolean { - return !!TTS_URL; + return ttsServices().length > 0; } -/** WAV audio for `text` from the voice, else the fallback; null when neither answers. */ +/** + * WAV audio for `text` from the requested service (`model`, by name) or the + * default, else the next service in order; null when none answers. An unknown + * name is ignored. + */ export async function synthesize( text: string, voice: string = TTS_DEFAULTS.voice, speed: number = TTS_DEFAULTS.speed, volume: number = TTS_DEFAULTS.volume, + model?: string, ): Promise { - if (!TTS_URL) return null; + const services = ttsServices(); + const chosen = services.findIndex((s) => s.name === model); + const order = chosen > 0 ? [services[chosen]!, ...services.filter((_, i) => i !== chosen)] : services; const body = JSON.stringify({ model: TTS_MODEL, input: text, @@ -59,11 +88,13 @@ export async function synthesize( volume_multiplier: volume, response_format: "wav", }); - const fallback = fallbackUrl(); - const audio = await speak(TTS_URL, body, fallback ? primaryTimeoutMs() : TIMEOUT_MS); - if (audio || !fallback) return audio; - console.warn("[tts] voice unavailable, using the fallback"); - return speak(fallback, body, TIMEOUT_MS); + for (const [i, service] of order.entries()) { + const last = i === order.length - 1; + const audio = await speak(service.url, body, last ? TIMEOUT_MS : primaryTimeoutMs()); + if (audio) return audio; + if (!last) console.warn(`[tts] ${service.name} unavailable, trying ${order[i + 1]!.name}`); + } + return null; } async function speak(url: string, body: string, timeoutMs: number): Promise { diff --git a/packages/backend/src/routes/__tests__/voice.test.ts b/packages/backend/src/routes/__tests__/voice.test.ts index b898d9c..2086e97 100644 --- a/packages/backend/src/routes/__tests__/voice.test.ts +++ b/packages/backend/src/routes/__tests__/voice.test.ts @@ -9,6 +9,7 @@ const mockSynthesize = mock(async () => new Uint8Array([82, 73, 70, 70]).buffer mock.module("../../ai/tts", () => ({ isTtsAvailable: () => available, synthesize: mockSynthesize, + ttsServices: () => [{ name: "chatterbox", url: "http://c" }, { name: "kokoro", url: "http://k" }], TTS_DEFAULTS: { voice: "am_michael", speed: 1.2, volume: 2 }, TTS_MODEL: "kokoro", })); @@ -45,7 +46,18 @@ describe("POST /api/ai/voice-audio", () => { expect(res.status).toBe(200); expect(res.headers.get("content-type")).toBe("audio/wav"); expect((await res.arrayBuffer()).byteLength).toBe(4); - expect(mockSynthesize).toHaveBeenCalledWith("cloud needs you", "am_michael", 1.5, 2.5); + expect(mockSynthesize).toHaveBeenCalledWith("cloud needs you", "am_michael", 1.5, 2.5, undefined); + }); + + test("passes the requested voice model, and rejects a malformed name", async () => { + await post(buildApp(), { text: "hi", model: "kokoro" }); + expect((mockSynthesize.mock.calls[0] as unknown[])[4]).toBe("kokoro"); + expect((await post(buildApp(), { text: "hi", model: "../etc" })).status).toBe(400); + }); + + test("GET /voice-models lists the voices, default first", async () => { + const res = await buildApp().request("/api/ai/voice-models"); + expect(await res.json()).toEqual({ models: ["chatterbox", "kokoro"] }); }); test("503 when no TTS service is configured, without calling it", async () => { diff --git a/packages/backend/src/routes/voice.ts b/packages/backend/src/routes/voice.ts index 4962858..b82982d 100644 --- a/packages/backend/src/routes/voice.ts +++ b/packages/backend/src/routes/voice.ts @@ -2,7 +2,7 @@ import type { SQL } from "bun"; import { Hono } from "hono"; import { zValidator } from "@hono/zod-validator"; import { callGemini, DEFAULT_MODEL } from "../ai/gemini"; -import { isTtsAvailable, synthesize } from "../ai/tts"; +import { isTtsAvailable, synthesize, ttsServices } from "../ai/tts"; import { speakable } from "../services/speakable"; import { recordTokenUsage } from "../db"; import { @@ -60,12 +60,15 @@ export function voiceRoutes(sql: SQL) { if (!checkRateLimit(`tts:${rateLimitKey(c)}`)) { return c.json({ error: "Rate limit exceeded. Max 20 voice requests/minute." }, 429); } - const { text, voice, speed, volume } = c.req.valid("json"); + const { text, voice, speed, volume, model } = c.req.valid("json"); // Every voiced text, Claude's explanations included, is said the way a person would say it. - const audio = await synthesize(speakable(text), voice, speed, volume); + const audio = await synthesize(speakable(text), voice, speed, volume, model); if (!audio) return c.json({ error: "Voice unavailable" }, 503); return new Response(audio, { headers: { "content-type": "audio/wav", "cache-control": "no-store" } }); }); + // The voices this server offers, default first, for `/devscope:voice model`. + app.get("/voice-models", (c) => c.json({ models: ttsServices().map((s) => s.name) })); + return app; } diff --git a/packages/backend/src/services/voiceSummary.ts b/packages/backend/src/services/voiceSummary.ts index 7d7d7ea..a1a53a3 100644 --- a/packages/backend/src/services/voiceSummary.ts +++ b/packages/backend/src/services/voiceSummary.ts @@ -47,6 +47,8 @@ export const voiceAudioBody = z.object({ voice: z.string().regex(/^[a-z]{2}_[a-z0-9_]{2,30}$/).optional(), speed: z.number().min(0.5).max(2).optional(), volume: z.number().min(0.5).max(3).optional(), + /** Which speech service to use first, by its TTS_SERVICES name (e.g. "kokoro"). */ + model: z.string().regex(/^[a-z0-9-]{1,32}$/).optional(), }); type Trigger = VoiceSummaryInput["trigger"]; From ffa253b23c776200b86991807165376473af30f6 Mon Sep 17 00:00:00 2001 From: Lucas Proxmox Date: Wed, 7 Oct 2026 14:30:21 +0200 Subject: [PATCH 3/3] docs: TTS_SERVICES, the voice-audio model field and /voice-models Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8d554c8..d5857c8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -89,8 +89,9 @@ WebSocket message types: `event.new`, `session.update`, `developer.update`. | `/api/live/vcs` | POST | `{session_id, kind: commit\|pr, ref, repo_remote?}`: a commit or PR the session produced (`session_vcs_links`) | | `/api/live/vcs/open-prs?repo_remote=` | GET | The caller's PR links in that repo still unknown/open and not checked for an hour; the mod resolves them with `gh` | | `/api/live/vcs/status` | POST | `{ref, state: open\|merged\|closed, merged_at?, closed_at?}`: PR state for the caller's own links | +| `/api/ai/voice-models` | GET | `{models}`: the voices this server offers (`TTS_SERVICES` names), default first | | `/api/ai/voice-summary` | POST | Spoken text for the plugin's voice features (`trigger`, `project`, optional `tool`/`detail`/`last_message`): one sentence for the announcer, or with `trigger: "reply"` a summary of Claude's reply for `/devscope:voice auto`, as detailed as the optional `length` asks (`short` one sentence ≤ 20 words, `normal` 2-3 sentences ≤ 50 words, `long` 4-6 sentences ≤ 110 words; `REPLY_LENGTHS`); stateless, only token usage is recorded as `voice_summary`. The plugin never calls it for `private` sessions | -| `/api/ai/voice-audio` | POST | That text (or one piece of a `/devscope:explain` explanation) as WAV (`text`, optional Kokoro `voice`/`speed`/`volume`), voiced by the homelab TTS service; 503 when `TTS_URL` is unset or the service fails, so the plugin falls back to a local voice. Own 20/min bucket, no Gemini budget | +| `/api/ai/voice-audio` | POST | That text (or one piece of a `/devscope:voice explain` explanation) as WAV (`text`, optional `model` = a `TTS_SERVICES` name, Kokoro `voice`, `speed`, `volume`), voiced by the homelab speech services; 503 when none is configured or all fail, so the plugin falls back to a local voice. Own 20/min bucket, no Gemini budget | | `/api/sessions/usage/backfill` | POST | Exact token usage per transcript and model from `/devscope:backfill-usage` (`{items: [{transcriptId, sessionId?, byModel}]}`, max 500); only the caller's own sessions are updated, the rest are counted as `skipped` | | `/api/health` | GET | Health check + WS client count | | `/ws` | WS | Real-time event stream | @@ -222,7 +223,7 @@ Session tokens and cost (migration 055, `db/tokenUsageQueries.ts`). Costs are ** The plugin's voice announcer speaks through `/api/ai/voice-audio`, so users get a natural voice without installing a model. Both voice endpoints live in `routes/voice.ts` (mounted under `/api/ai`; the Gemini guard and rate limiter are shared with `routes/ai.ts` via `routes/aiGuards.ts`). - **Services:** the voice is **Chatterbox Turbo** (350M) on the homelab's Intel Arc B580, a small server in the deployment stack (`/opt/stacks/devscope/chatterbox`, built locally, Intel PyTorch XPU) that speaks the same OpenAI-compatible `/v1/audio/speech` and does speed with ffmpeg atempo and volume as a gain. **Kokoro** (Kokoro-FastAPI CPU image) stays as the fallback. Both run on the internal network only, never exposed. `ai/tts.ts` never throws and returns null when neither answers. -- **Env:** `TTS_URL` (the voice, e.g. `http://chatterbox:8880`; unset disables the feature), `TTS_FALLBACK_URL` (e.g. `http://kokoro:8880`; tried when the voice fails or is slow), `TTS_PRIMARY_TIMEOUT_MS` (how long the voice gets when there is a fallback, default 10000), `TTS_VOICE` (Kokoro voice, default `am_michael`; Chatterbox ignores it), `TTS_SPEED` (default `1.2`; the plugin sends its own, also 1.2 by default), `TTS_VOLUME` (default `2`: Kokoro's `volume_multiplier`, and Chatterbox's baseline gain), `TTS_MODEL` (default `kokoro`). +- **Env:** `TTS_SERVICES` (named services in order, `chatterbox=http://chatterbox:8880,kokoro=http://kokoro:8880`: the first is the default voice, the rest fallbacks; a request's `model` names the one to try first, which is how `/devscope:voice model kokoro` works), or the older unnamed `TTS_URL` + `TTS_FALLBACK_URL` (neither set disables the feature), `TTS_PRIMARY_TIMEOUT_MS` (how long every service but the last gets, default 10000), `TTS_VOICE` (Kokoro voice, default `am_michael`; Chatterbox ignores it), `TTS_SPEED` (default `1.2`; the plugin sends its own, also 1.2 by default), `TTS_VOLUME` (default `2`: Kokoro's `volume_multiplier`, and Chatterbox's baseline gain), `TTS_MODEL` (default `kokoro`). - **Why a fallback:** the B580 is shared with Ollama. While Ollama's 9B model runs, Chatterbox slows about 8x (a 15 s summary took about 40 s); its server answers 503 at once when busy, and the backend gives it `TTS_PRIMARY_TIMEOUT_MS`, so Kokoro speaks instead. The voice then differs for that sentence, which is the trade for not waiting. - **Pronunciation:** `services/speakable.ts` rewrites every voiced text (and every voice summary) before Kokoro reads it: word-like acronyms (JSON as jay-son, README, YAML), file extensions (`voice dot jay-son`), version numbers, units after numbers (`5s`, `150ms`, `1.2x`, `30m`), `e.g.`/`vs`, and camel/snake/PascalCase identifiers. All-caps acronyms (API, CLI, PR) and vowel-less short words (npm, jq) are left alone: Kokoro already spells them, and spacing letters out reads "A" as the article. Check a spelling against Kokoro's phonemizer: `POST http://kokoro:8880/dev/phonemize {"text", "language": "a"}`. - **Privacy:** what is voiced is either the text the backend itself wrote, or a spoken explanation Claude wrote for `/devscope:explain` (sent in pieces of at most 440 characters, the next one fetched while the current one plays). The plugin never sends `private` sessions here; they use a local voice. Nothing is stored.