diff --git a/CHANGELOG.md b/CHANGELOG.md index c8fb054..71cdcf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- **Voiced audio ends cleanly.** `/api/ai/voice-audio` rewrites the speech + service's WAV into a plain 44-byte-header PCM file with its real sizes + (Kokoro streams its WAV with `0xFFFFFFFF` sizes and a LIST chunk, so players + had to guess where it ends) and appends trailing silence, 400 ms by default + (`VOICE.padMs`), so outputs that close the stream early, such as Bluetooth + headphones, keep the last word. An optional `pad_ms` (0-1000) overrides it; + the plugin (0.33.0) sends `0` for pieces in the middle of long speech. +- **Summaries that run long end at a sentence.** `toSpokenText` cuts an + over-long summary at its last full sentence when that keeps at least half, + and a character cut ends at a whole word, instead of stopping mid-sentence. + - **Voice says which session it is about.** `/api/ai/voice-summary` takes an optional `session_id` and `label`. For the caller's own session it builds a spoken label, the project plus what the session works on (its title, else diff --git a/CLAUDE.md b/CLAUDE.md index 15213dc..f1362fd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -91,7 +91,7 @@ WebSocket message types: `event.new`, `session.update`, `developer.update`. | `/api/live/vcs/status` | POST | `{ref, state: open\|merged\|closed, merged_at?, closed_at?}`: PR state for the caller's own links | | `/api/ai/voice-models` | GET | `{models}`: the voices this server offers (`TTS_SERVICES` names), default first | | `/api/ai/voice-summary` | POST | Spoken text for the plugin's voice features (`trigger`, `project`, optional `tool`/`detail`/`last_message`/`session_id`/`label`), starting with the session's label (`project, topic` from the caller's own session title, else branch; a sent `label` is reused as is, so the name never changes) and returning it as `label`: one sentence for the announcer, or with `trigger: "reply"` a summary of Claude's reply for `/devscope:voice auto`, as detailed as the optional `length` asks (`short` one sentence ≤ 20 words, `normal` 2-3 sentences ≤ 50 words, `long` 4-6 sentences ≤ 110 words; `REPLY_LENGTHS`); stateless, only token usage is recorded as `voice_summary`. The plugin never calls it for `private` sessions | -| `/api/ai/voice-audio` | POST | That text (or one piece of a `/devscope:voice explain` explanation) as WAV (`text`, optional `model` = a `TTS_SERVICES` name, Kokoro `voice`, `speed`, `volume`), voiced by the homelab speech services; 503 when none is configured or all fail, so the plugin falls back to a local voice. Own 20/min bucket, no Gemini budget | +| `/api/ai/voice-audio` | POST | That text (or one piece of a `/devscope:voice explain` explanation) as WAV (`text`, optional `model` = a `TTS_SERVICES` name, Kokoro `voice`, `speed`, `volume`, `pad_ms` trailing silence, default 400), rewritten as a canonical PCM WAV with real sizes (`services/wav.ts`), voiced by the homelab speech services; 503 when none is configured or all fail, so the plugin falls back to a local voice. Own 20/min bucket, no Gemini budget | | `/api/sessions/usage/backfill` | POST | Exact token usage per transcript and model from `/devscope:backfill-usage` (`{items: [{transcriptId, sessionId?, byModel}]}`, max 500); only the caller's own sessions are updated, the rest are counted as `skipped` | | `/api/health` | GET | Health check + WS client count | | `/ws` | WS | Real-time event stream | diff --git a/packages/backend/src/routes/__tests__/voice.test.ts b/packages/backend/src/routes/__tests__/voice.test.ts index 01e3bfb..c346bf3 100644 --- a/packages/backend/src/routes/__tests__/voice.test.ts +++ b/packages/backend/src/routes/__tests__/voice.test.ts @@ -61,6 +61,27 @@ describe("POST /api/ai/voice-audio", () => { expect(mockSynthesize).toHaveBeenCalledWith("cloud needs you", "am_michael", 1.5, 2.5, undefined); }); + test("pads the end with silence; pad_ms 0 for mid-speech pieces", async () => { + const pcm = (frames: number) => { + const b = new ArrayBuffer(44 + frames * 2); + const v = new DataView(b); + [..."RIFF"].forEach((c, i) => v.setUint8(i, c.charCodeAt(0))); + v.setUint32(4, 36 + frames * 2, true); + [..."WAVEfmt "].forEach((c, i) => v.setUint8(8 + i, c.charCodeAt(0))); + v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true); + v.setUint32(24, 24000, true); v.setUint32(28, 48000, true); v.setUint16(32, 2, true); v.setUint16(34, 16, true); + [..."data"].forEach((c, i) => v.setUint8(36 + i, c.charCodeAt(0))); + v.setUint32(40, frames * 2, true); + return b; + }; + mockSynthesize.mockImplementation(async () => pcm(240)); + const padded = await (await post(buildApp(), { text: "hi" })).arrayBuffer(); + expect(padded.byteLength).toBe(44 + (240 + 9600) * 2); // default 400 ms at 24 kHz + const bare = await (await post(buildApp(), { text: "hi", pad_ms: 0 })).arrayBuffer(); + expect(bare.byteLength).toBe(44 + 240 * 2); + expect((await post(buildApp(), { text: "hi", pad_ms: 5000 })).status).toBe(400); + }); + test("passes the requested voice model, and rejects a malformed name", async () => { await post(buildApp(), { text: "hi", model: "kokoro" }); expect((mockSynthesize.mock.calls[0] as unknown[])[4]).toBe("kokoro"); diff --git a/packages/backend/src/routes/voice.ts b/packages/backend/src/routes/voice.ts index c31ee31..bdc6a7c 100644 --- a/packages/backend/src/routes/voice.ts +++ b/packages/backend/src/routes/voice.ts @@ -4,6 +4,7 @@ import { zValidator } from "@hono/zod-validator"; import { callGemini, DEFAULT_MODEL } from "../ai/gemini"; import { isTtsAvailable, synthesize, ttsServices } from "../ai/tts"; import { speakable } from "../services/speakable"; +import { finishWav } from "../services/wav"; import { recordTokenUsage } from "../db"; import { VOICE, @@ -67,11 +68,11 @@ export function voiceRoutes(sql: SQL) { if (!checkRateLimit(`tts:${rateLimitKey(c)}`)) { return c.json({ error: "Rate limit exceeded. Max 20 voice requests/minute." }, 429); } - const { text, voice, speed, volume, model } = c.req.valid("json"); + const { text, voice, speed, volume, model, pad_ms } = c.req.valid("json"); // Every voiced text, Claude's explanations included, is said the way a person would say it. const audio = await synthesize(speakable(text), voice, speed, volume, model); if (!audio) return c.json({ error: "Voice unavailable" }, 503); - return new Response(audio, { headers: { "content-type": "audio/wav", "cache-control": "no-store" } }); + return new Response(finishWav(audio, pad_ms ?? VOICE.padMs), { headers: { "content-type": "audio/wav", "cache-control": "no-store" } }); }); // The voices this server offers, default first, for `/devscope:voice model`. diff --git a/packages/backend/src/services/__tests__/voiceSummary.test.ts b/packages/backend/src/services/__tests__/voiceSummary.test.ts index adff2ef..25acb32 100644 --- a/packages/backend/src/services/__tests__/voiceSummary.test.ts +++ b/packages/backend/src/services/__tests__/voiceSummary.test.ts @@ -130,6 +130,25 @@ describe("toSpokenText", () => { expect(out.endsWith(".")).toBe(true); }); + test("a summary that runs long ends at its last full sentence, not mid-sentence", () => { + const first = Array.from({ length: 15 }, (_, i) => `a${i}`).join(" ") + "."; + const second = Array.from({ length: 30 }, (_, i) => `b${i}`).join(" ") + "."; + expect(toSpokenText(`${first} ${second}`)).toBe(first); + }); + + test("falls back to a word cut when the first sentence is too short to keep", () => { + const out = toSpokenText("Ok. " + Array.from({ length: 40 }, (_, i) => `w${i}`).join(" ") + "."); + expect(out.split(" ").length).toBe(VOICE.maxWords); + expect(out.endsWith(".")).toBe(true); + }); + + test("a character cut ends at a sentence or a whole word, never mid-word", () => { + const long = Array.from({ length: 24 }, () => "abcdefghijk").join(" "); + const out = toSpokenText(long, { maxWords: 100, maxChars: 50 }); + expect(out.endsWith("…")).toBe(true); + expect(out.slice(0, -1).split(" ").every((w) => w === "abcdefghijk")).toBe(true); + }); + test("takes the reply limits", () => { const out = toSpokenText( Array.from({ length: 80 }, (_, i) => `w${i}`).join(" "), diff --git a/packages/backend/src/services/__tests__/wav.test.ts b/packages/backend/src/services/__tests__/wav.test.ts new file mode 100644 index 0000000..f4d95f3 --- /dev/null +++ b/packages/backend/src/services/__tests__/wav.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, test } from "bun:test"; +import { finishWav } from "../wav"; + +/** A 16-bit mono WAV: `samples` frames of value 1000, optionally streamed like Kokoro's. */ +function wav(samples: number, opts: { rate?: number; streamed?: boolean; list?: boolean } = {}): ArrayBuffer { + const rate = opts.rate ?? 24000; + const list = opts.list ? 34 : 0; // "LIST" + size + 26 bytes + const out = new ArrayBuffer(44 + list + samples * 2); + const v = new DataView(out); + const put = (at: number, s: string) => [...s].forEach((c, i) => v.setUint8(at + i, c.charCodeAt(0))); + put(0, "RIFF"); + v.setUint32(4, opts.streamed ? 0xffffffff : 36 + list + samples * 2, true); + put(8, "WAVE"); + put(12, "fmt "); + v.setUint32(16, 16, true); + v.setUint16(20, 1, true); + v.setUint16(22, 1, true); + v.setUint32(24, rate, true); + v.setUint32(28, rate * 2, true); + v.setUint16(32, 2, true); + v.setUint16(34, 16, true); + let at = 36; + if (list) { + put(at, "LIST"); + v.setUint32(at + 4, 26, true); + at += 34; + } + put(at, "data"); + v.setUint32(at + 4, opts.streamed ? 0xffffffff : samples * 2, true); + for (let i = 0; i < samples; i++) v.setInt16(at + 8 + i * 2, 1000, true); + return out; +} + +const dataSize = (b: ArrayBuffer) => new DataView(b).getUint32(40, true); + +describe("finishWav", () => { + test("appends the requested silence and states the new size", () => { + const out = finishWav(wav(2400), 400); // 0.1 s of audio + 0.4 s pad at 24 kHz + expect(dataSize(out)).toBe((2400 + 9600) * 2); + expect(new DataView(out).getUint32(4, true)).toBe(36 + dataSize(out)); + const v = new DataView(out); + expect(v.getInt16(44, true)).toBe(1000); // audio kept + expect(v.getInt16(44 + 2400 * 2, true)).toBe(0); // silence after it + expect(out.byteLength).toBe(44 + dataSize(out)); + }); + + test("a streamed WAV (Kokoro: 0xFFFFFFFF sizes, LIST chunk) gets real sizes and a 44-byte header", () => { + const out = finishWav(wav(4800, { streamed: true, list: true }), 0); + expect(dataSize(out)).toBe(4800 * 2); + expect(out.byteLength).toBe(44 + 4800 * 2); + expect(new DataView(out).getInt16(44, true)).toBe(1000); + }); + + test("padding 0 keeps the audio length", () => { + expect(dataSize(finishWav(wav(100), 0))).toBe(200); + }); + + test("a body shorter than its header claims keeps whole frames only", () => { + const b = wav(10); + const truncated = b.slice(0, b.byteLength - 1); + expect(dataSize(finishWav(truncated, 0))).toBe(18); + }); + + test("anything else passes through untouched", () => { + const notWav = new Uint8Array([82, 73, 70, 70]).buffer; + expect(finishWav(notWav, 400)).toBe(notWav); + const floatWav = wav(10); + new DataView(floatWav).setUint16(20, 3, true); // IEEE float + expect(finishWav(floatWav, 400)).toBe(floatWav); + }); +}); diff --git a/packages/backend/src/services/voiceSummary.ts b/packages/backend/src/services/voiceSummary.ts index 5f2be5c..fa2354c 100644 --- a/packages/backend/src/services/voiceSummary.ts +++ b/packages/backend/src/services/voiceSummary.ts @@ -33,6 +33,8 @@ export const VOICE = { // tokens) was cut mid-word at 1024. Room for thinking where it can't be off. maxOutputTokens: 4096, temperature: 0.4, + /** Silence after the last word of a voiced text, so no output drops its tail. */ + padMs: 400, } as const; /** Longest label; the topic part is cut to whole words to fit. */ @@ -63,6 +65,11 @@ export const voiceAudioBody = z.object({ volume: z.number().min(0.5).max(3).optional(), /** Which speech service to use first, by its TTS_SERVICES name (e.g. "kokoro"). */ model: z.string().regex(/^[a-z0-9-]{1,32}$/).optional(), + /** + * Trailing silence in ms; VOICE.padMs when absent. The plugin sends 0 for a + * piece in the middle of long speech, where the next piece follows at once. + */ + pad_ms: z.number().int().min(0).max(1000).optional(), }); type Trigger = VoiceSummaryInput["trigger"]; @@ -192,8 +199,27 @@ export function toSpokenText( .trim(); if (!text) return ""; const words = text.split(" "); - const capped = words.length > maxWords ? `${words.slice(0, maxWords).join(" ")}.` : text; - return capped.length > maxChars ? `${capped.slice(0, maxChars - 1)}…` : capped; + let capped = text; + if (words.length > maxWords) { + const kept = words.slice(0, maxWords).join(" "); + capped = atSentenceEnd(kept, kept.length / 2) ?? `${kept.replace(/[,;:]$/, "")}.`; + } + if (capped.length <= maxChars) return capped; + const head = capped.slice(0, maxChars); + return atSentenceEnd(head, maxChars / 2) ?? `${head.slice(0, Math.max(head.lastIndexOf(" "), 1)).replace(/[,;:]$/, "")}…`; +} + +/** + * A cut text ends at its last full sentence when that keeps at least + * `minLength` characters, so a summary the model let run long is not spoken + * stopping mid-sentence. Null when no sentence ends late enough. + */ +function atSentenceEnd(text: string, minLength: number): string | null { + const ends = [...text.matchAll(/[.!?](?=\s|$)/g)]; + const last = ends.at(-1); + if (!last || last.index === undefined) return null; + const cut = text.slice(0, last.index + 1); + return cut.length >= minLength ? cut : null; } // --- Session labels --- diff --git a/packages/backend/src/services/wav.ts b/packages/backend/src/services/wav.ts new file mode 100644 index 0000000..334212f --- /dev/null +++ b/packages/backend/src/services/wav.ts @@ -0,0 +1,65 @@ +/** + * Makes the speech services' WAV safe to play to the very end. + * + * - Kokoro streams its WAV, so the RIFF and data sizes are the "unknown" + * placeholder 0xFFFFFFFF and a LIST chunk sits before the data. Players then + * have to guess where the audio ends; the rewritten file states it. + * - A player that exits as soon as the last sample is handed over lets some + * outputs (Bluetooth headphones in particular, ~200-300 ms behind) drop the + * tail when the stream closes. Trailing silence keeps the last word whole. + * + * Anything that is not 16-bit PCM RIFF/WAVE is returned unchanged. + */ +export function finishWav(audio: ArrayBuffer, padMs: number): ArrayBuffer { + const view = new DataView(audio); + if (audio.byteLength < 12 || tag(view, 0) !== "RIFF" || tag(view, 8) !== "WAVE") return audio; + + let fmt: { channels: number; rate: number; bits: number } | null = null; + let offset = 12; + while (offset + 8 <= audio.byteLength) { + const id = tag(view, offset); + const size = view.getUint32(offset + 4, true); + const body = offset + 8; + if (id === "fmt " && body + 16 <= audio.byteLength) { + if (view.getUint16(body, true) !== 1) return audio; // not PCM + fmt = { channels: view.getUint16(body + 2, true), rate: view.getUint32(body + 4, true), bits: view.getUint16(body + 14, true) }; + } else if (id === "data") { + if (!fmt || fmt.bits !== 16 || fmt.channels < 1 || fmt.rate < 1) return audio; + // A streamed size (0xFFFFFFFF) or a short body: the bytes present are the audio. + const blockAlign = fmt.channels * 2; + let length = Math.min(size, audio.byteLength - body); + length -= length % blockAlign; + const pad = Math.round((Math.max(0, padMs) / 1000) * fmt.rate) * blockAlign; + return pcmWav(new Uint8Array(audio, body, length), fmt.channels, fmt.rate, pad); + } + offset = body + size + (size & 1); + } + return audio; +} + +function tag(view: DataView, at: number): string { + return String.fromCharCode(view.getUint8(at), view.getUint8(at + 1), view.getUint8(at + 2), view.getUint8(at + 3)); +} + +/** A canonical 44-byte-header PCM WAV: the samples, then `padBytes` of silence. */ +function pcmWav(samples: Uint8Array, channels: number, rate: number, padBytes: number): ArrayBuffer { + const dataLength = samples.byteLength + padBytes; + const out = new ArrayBuffer(44 + dataLength); + const v = new DataView(out); + const put = (at: number, s: string) => [...s].forEach((c, i) => v.setUint8(at + i, c.charCodeAt(0))); + put(0, "RIFF"); + v.setUint32(4, 36 + dataLength, true); + put(8, "WAVE"); + put(12, "fmt "); + v.setUint32(16, 16, true); + v.setUint16(20, 1, true); + v.setUint16(22, channels, true); + v.setUint32(24, rate, true); + v.setUint32(28, rate * channels * 2, true); + v.setUint16(32, channels * 2, true); + v.setUint16(34, 16, true); + put(36, "data"); + v.setUint32(40, dataLength, true); + new Uint8Array(out, 44).set(samples); // the padding stays zero: silence + return out; +}