Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).

### Added

- **Voiced audio ends cleanly.** `/api/ai/voice-audio` rewrites the speech
service's WAV into a plain 44-byte-header PCM file with its real sizes
(Kokoro streams its WAV with `0xFFFFFFFF` sizes and a LIST chunk, so players
had to guess where it ends) and appends trailing silence, 400 ms by default
(`VOICE.padMs`), so outputs that close the stream early, such as Bluetooth
headphones, keep the last word. An optional `pad_ms` (0-1000) overrides it;
the plugin (0.33.0) sends `0` for pieces in the middle of long speech.
- **Summaries that run long end at a sentence.** `toSpokenText` cuts an
over-long summary at its last full sentence when that keeps at least half,
and a character cut ends at a whole word, instead of stopping mid-sentence.

- **Voice says which session it is about.** `/api/ai/voice-summary` takes an
optional `session_id` and `label`. For the caller's own session it builds a
spoken label, the project plus what the session works on (its title, else
Expand Down
2 changes: 1 addition & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ WebSocket message types: `event.new`, `session.update`, `developer.update`.
| `/api/live/vcs/status` | POST | `{ref, state: open\|merged\|closed, merged_at?, closed_at?}`: PR state for the caller's own links |
| `/api/ai/voice-models` | GET | `{models}`: the voices this server offers (`TTS_SERVICES` names), default first |
| `/api/ai/voice-summary` | POST | Spoken text for the plugin's voice features (`trigger`, `project`, optional `tool`/`detail`/`last_message`/`session_id`/`label`), starting with the session's label (`project, topic` from the caller's own session title, else branch; a sent `label` is reused as is, so the name never changes) and returning it as `label`: one sentence for the announcer, or with `trigger: "reply"` a summary of Claude's reply for `/devscope:voice auto`, as detailed as the optional `length` asks (`short` one sentence ≤ 20 words, `normal` 2-3 sentences ≤ 50 words, `long` 4-6 sentences ≤ 110 words; `REPLY_LENGTHS`); stateless, only token usage is recorded as `voice_summary`. The plugin never calls it for `private` sessions |
| `/api/ai/voice-audio` | POST | That text (or one piece of a `/devscope:voice explain` explanation) as WAV (`text`, optional `model` = a `TTS_SERVICES` name, Kokoro `voice`, `speed`, `volume`), voiced by the homelab speech services; 503 when none is configured or all fail, so the plugin falls back to a local voice. Own 20/min bucket, no Gemini budget |
| `/api/ai/voice-audio` | POST | That text (or one piece of a `/devscope:voice explain` explanation) as WAV (`text`, optional `model` = a `TTS_SERVICES` name, Kokoro `voice`, `speed`, `volume`, `pad_ms` trailing silence, default 400), rewritten as a canonical PCM WAV with real sizes (`services/wav.ts`), voiced by the homelab speech services; 503 when none is configured or all fail, so the plugin falls back to a local voice. Own 20/min bucket, no Gemini budget |
| `/api/sessions/usage/backfill` | POST | Exact token usage per transcript and model from `/devscope:backfill-usage` (`{items: [{transcriptId, sessionId?, byModel}]}`, max 500); only the caller's own sessions are updated, the rest are counted as `skipped` |
| `/api/health` | GET | Health check + WS client count |
| `/ws` | WS | Real-time event stream |
Expand Down
21 changes: 21 additions & 0 deletions packages/backend/src/routes/__tests__/voice.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,27 @@ describe("POST /api/ai/voice-audio", () => {
expect(mockSynthesize).toHaveBeenCalledWith("cloud needs you", "am_michael", 1.5, 2.5, undefined);
});

test("pads the end with silence; pad_ms 0 for mid-speech pieces", async () => {
const pcm = (frames: number) => {
const b = new ArrayBuffer(44 + frames * 2);
const v = new DataView(b);
[..."RIFF"].forEach((c, i) => v.setUint8(i, c.charCodeAt(0)));
v.setUint32(4, 36 + frames * 2, true);
[..."WAVEfmt "].forEach((c, i) => v.setUint8(8 + i, c.charCodeAt(0)));
v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true);
v.setUint32(24, 24000, true); v.setUint32(28, 48000, true); v.setUint16(32, 2, true); v.setUint16(34, 16, true);
[..."data"].forEach((c, i) => v.setUint8(36 + i, c.charCodeAt(0)));
v.setUint32(40, frames * 2, true);
return b;
};
mockSynthesize.mockImplementation(async () => pcm(240));
const padded = await (await post(buildApp(), { text: "hi" })).arrayBuffer();
expect(padded.byteLength).toBe(44 + (240 + 9600) * 2); // default 400 ms at 24 kHz
const bare = await (await post(buildApp(), { text: "hi", pad_ms: 0 })).arrayBuffer();
expect(bare.byteLength).toBe(44 + 240 * 2);
expect((await post(buildApp(), { text: "hi", pad_ms: 5000 })).status).toBe(400);
});

test("passes the requested voice model, and rejects a malformed name", async () => {
await post(buildApp(), { text: "hi", model: "kokoro" });
expect((mockSynthesize.mock.calls[0] as unknown[])[4]).toBe("kokoro");
Expand Down
5 changes: 3 additions & 2 deletions packages/backend/src/routes/voice.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { zValidator } from "@hono/zod-validator";
import { callGemini, DEFAULT_MODEL } from "../ai/gemini";
import { isTtsAvailable, synthesize, ttsServices } from "../ai/tts";
import { speakable } from "../services/speakable";
import { finishWav } from "../services/wav";
import { recordTokenUsage } from "../db";
import {
VOICE,
Expand Down Expand Up @@ -67,11 +68,11 @@ export function voiceRoutes(sql: SQL) {
if (!checkRateLimit(`tts:${rateLimitKey(c)}`)) {
return c.json({ error: "Rate limit exceeded. Max 20 voice requests/minute." }, 429);
}
const { text, voice, speed, volume, model } = c.req.valid("json");
const { text, voice, speed, volume, model, pad_ms } = c.req.valid("json");
// Every voiced text, Claude's explanations included, is said the way a person would say it.
const audio = await synthesize(speakable(text), voice, speed, volume, model);
if (!audio) return c.json({ error: "Voice unavailable" }, 503);
return new Response(audio, { headers: { "content-type": "audio/wav", "cache-control": "no-store" } });
return new Response(finishWav(audio, pad_ms ?? VOICE.padMs), { headers: { "content-type": "audio/wav", "cache-control": "no-store" } });
});

// The voices this server offers, default first, for `/devscope:voice model`.
Expand Down
19 changes: 19 additions & 0 deletions packages/backend/src/services/__tests__/voiceSummary.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,25 @@ describe("toSpokenText", () => {
expect(out.endsWith(".")).toBe(true);
});

test("a summary that runs long ends at its last full sentence, not mid-sentence", () => {
const first = Array.from({ length: 15 }, (_, i) => `a${i}`).join(" ") + ".";
const second = Array.from({ length: 30 }, (_, i) => `b${i}`).join(" ") + ".";
expect(toSpokenText(`${first} ${second}`)).toBe(first);
});

test("falls back to a word cut when the first sentence is too short to keep", () => {
const out = toSpokenText("Ok. " + Array.from({ length: 40 }, (_, i) => `w${i}`).join(" ") + ".");
expect(out.split(" ").length).toBe(VOICE.maxWords);
expect(out.endsWith(".")).toBe(true);
});

test("a character cut ends at a sentence or a whole word, never mid-word", () => {
const long = Array.from({ length: 24 }, () => "abcdefghijk").join(" ");
const out = toSpokenText(long, { maxWords: 100, maxChars: 50 });
expect(out.endsWith("…")).toBe(true);
expect(out.slice(0, -1).split(" ").every((w) => w === "abcdefghijk")).toBe(true);
});

test("takes the reply limits", () => {
const out = toSpokenText(
Array.from({ length: 80 }, (_, i) => `w${i}`).join(" "),
Expand Down
71 changes: 71 additions & 0 deletions packages/backend/src/services/__tests__/wav.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import { describe, expect, test } from "bun:test";
import { finishWav } from "../wav";

/** A 16-bit mono WAV: `samples` frames of value 1000, optionally streamed like Kokoro's. */
function wav(samples: number, opts: { rate?: number; streamed?: boolean; list?: boolean } = {}): ArrayBuffer {
const rate = opts.rate ?? 24000;
const list = opts.list ? 34 : 0; // "LIST" + size + 26 bytes
const out = new ArrayBuffer(44 + list + samples * 2);
const v = new DataView(out);
const put = (at: number, s: string) => [...s].forEach((c, i) => v.setUint8(at + i, c.charCodeAt(0)));
put(0, "RIFF");
v.setUint32(4, opts.streamed ? 0xffffffff : 36 + list + samples * 2, true);
put(8, "WAVE");
put(12, "fmt ");
v.setUint32(16, 16, true);
v.setUint16(20, 1, true);
v.setUint16(22, 1, true);
v.setUint32(24, rate, true);
v.setUint32(28, rate * 2, true);
v.setUint16(32, 2, true);
v.setUint16(34, 16, true);
let at = 36;
if (list) {
put(at, "LIST");
v.setUint32(at + 4, 26, true);
at += 34;
}
put(at, "data");
v.setUint32(at + 4, opts.streamed ? 0xffffffff : samples * 2, true);
for (let i = 0; i < samples; i++) v.setInt16(at + 8 + i * 2, 1000, true);
return out;
}

const dataSize = (b: ArrayBuffer) => new DataView(b).getUint32(40, true);

describe("finishWav", () => {
test("appends the requested silence and states the new size", () => {
const out = finishWav(wav(2400), 400); // 0.1 s of audio + 0.4 s pad at 24 kHz
expect(dataSize(out)).toBe((2400 + 9600) * 2);
expect(new DataView(out).getUint32(4, true)).toBe(36 + dataSize(out));
const v = new DataView(out);
expect(v.getInt16(44, true)).toBe(1000); // audio kept
expect(v.getInt16(44 + 2400 * 2, true)).toBe(0); // silence after it
expect(out.byteLength).toBe(44 + dataSize(out));
});

test("a streamed WAV (Kokoro: 0xFFFFFFFF sizes, LIST chunk) gets real sizes and a 44-byte header", () => {
const out = finishWav(wav(4800, { streamed: true, list: true }), 0);
expect(dataSize(out)).toBe(4800 * 2);
expect(out.byteLength).toBe(44 + 4800 * 2);
expect(new DataView(out).getInt16(44, true)).toBe(1000);
});

test("padding 0 keeps the audio length", () => {
expect(dataSize(finishWav(wav(100), 0))).toBe(200);
});

test("a body shorter than its header claims keeps whole frames only", () => {
const b = wav(10);
const truncated = b.slice(0, b.byteLength - 1);
expect(dataSize(finishWav(truncated, 0))).toBe(18);
});

test("anything else passes through untouched", () => {
const notWav = new Uint8Array([82, 73, 70, 70]).buffer;
expect(finishWav(notWav, 400)).toBe(notWav);
const floatWav = wav(10);
new DataView(floatWav).setUint16(20, 3, true); // IEEE float
expect(finishWav(floatWav, 400)).toBe(floatWav);
});
});
30 changes: 28 additions & 2 deletions packages/backend/src/services/voiceSummary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ export const VOICE = {
// tokens) was cut mid-word at 1024. Room for thinking where it can't be off.
maxOutputTokens: 4096,
temperature: 0.4,
/** Silence after the last word of a voiced text, so no output drops its tail. */
padMs: 400,
} as const;

/** Longest label; the topic part is cut to whole words to fit. */
Expand Down Expand Up @@ -63,6 +65,11 @@ export const voiceAudioBody = z.object({
volume: z.number().min(0.5).max(3).optional(),
/** Which speech service to use first, by its TTS_SERVICES name (e.g. "kokoro"). */
model: z.string().regex(/^[a-z0-9-]{1,32}$/).optional(),
/**
* Trailing silence in ms; VOICE.padMs when absent. The plugin sends 0 for a
* piece in the middle of long speech, where the next piece follows at once.
*/
pad_ms: z.number().int().min(0).max(1000).optional(),
});

type Trigger = VoiceSummaryInput["trigger"];
Expand Down Expand Up @@ -192,8 +199,27 @@ export function toSpokenText(
.trim();
if (!text) return "";
const words = text.split(" ");
const capped = words.length > maxWords ? `${words.slice(0, maxWords).join(" ")}.` : text;
return capped.length > maxChars ? `${capped.slice(0, maxChars - 1)}…` : capped;
let capped = text;
if (words.length > maxWords) {
const kept = words.slice(0, maxWords).join(" ");
capped = atSentenceEnd(kept, kept.length / 2) ?? `${kept.replace(/[,;:]$/, "")}.`;
}
if (capped.length <= maxChars) return capped;
const head = capped.slice(0, maxChars);
return atSentenceEnd(head, maxChars / 2) ?? `${head.slice(0, Math.max(head.lastIndexOf(" "), 1)).replace(/[,;:]$/, "")}…`;
}

/**
* A cut text ends at its last full sentence when that keeps at least
* `minLength` characters, so a summary the model let run long is not spoken
* stopping mid-sentence. Null when no sentence ends late enough.
*/
function atSentenceEnd(text: string, minLength: number): string | null {
const ends = [...text.matchAll(/[.!?](?=\s|$)/g)];
const last = ends.at(-1);
if (!last || last.index === undefined) return null;
const cut = text.slice(0, last.index + 1);
return cut.length >= minLength ? cut : null;
}

// --- Session labels ---
Expand Down
65 changes: 65 additions & 0 deletions packages/backend/src/services/wav.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/**
* Makes the speech services' WAV safe to play to the very end.
*
* - Kokoro streams its WAV, so the RIFF and data sizes are the "unknown"
* placeholder 0xFFFFFFFF and a LIST chunk sits before the data. Players then
* have to guess where the audio ends; the rewritten file states it.
* - A player that exits as soon as the last sample is handed over lets some
* outputs (Bluetooth headphones in particular, ~200-300 ms behind) drop the
* tail when the stream closes. Trailing silence keeps the last word whole.
*
* Anything that is not 16-bit PCM RIFF/WAVE is returned unchanged.
*/
export function finishWav(audio: ArrayBuffer, padMs: number): ArrayBuffer {
const view = new DataView(audio);
if (audio.byteLength < 12 || tag(view, 0) !== "RIFF" || tag(view, 8) !== "WAVE") return audio;

let fmt: { channels: number; rate: number; bits: number } | null = null;
let offset = 12;
while (offset + 8 <= audio.byteLength) {
const id = tag(view, offset);
const size = view.getUint32(offset + 4, true);
const body = offset + 8;
if (id === "fmt " && body + 16 <= audio.byteLength) {
if (view.getUint16(body, true) !== 1) return audio; // not PCM
fmt = { channels: view.getUint16(body + 2, true), rate: view.getUint32(body + 4, true), bits: view.getUint16(body + 14, true) };
} else if (id === "data") {
if (!fmt || fmt.bits !== 16 || fmt.channels < 1 || fmt.rate < 1) return audio;
// A streamed size (0xFFFFFFFF) or a short body: the bytes present are the audio.
const blockAlign = fmt.channels * 2;
let length = Math.min(size, audio.byteLength - body);
length -= length % blockAlign;
const pad = Math.round((Math.max(0, padMs) / 1000) * fmt.rate) * blockAlign;
return pcmWav(new Uint8Array(audio, body, length), fmt.channels, fmt.rate, pad);
}
offset = body + size + (size & 1);
}
return audio;
}

function tag(view: DataView, at: number): string {
return String.fromCharCode(view.getUint8(at), view.getUint8(at + 1), view.getUint8(at + 2), view.getUint8(at + 3));
}

/** A canonical 44-byte-header PCM WAV: the samples, then `padBytes` of silence. */
function pcmWav(samples: Uint8Array, channels: number, rate: number, padBytes: number): ArrayBuffer {
const dataLength = samples.byteLength + padBytes;
const out = new ArrayBuffer(44 + dataLength);
const v = new DataView(out);
const put = (at: number, s: string) => [...s].forEach((c, i) => v.setUint8(at + i, c.charCodeAt(0)));
put(0, "RIFF");
v.setUint32(4, 36 + dataLength, true);
put(8, "WAVE");
put(12, "fmt ");
v.setUint32(16, 16, true);
v.setUint16(20, 1, true);
v.setUint16(22, channels, true);
v.setUint32(24, rate, true);
v.setUint32(28, rate * channels * 2, true);
v.setUint16(32, channels * 2, true);
v.setUint16(34, 16, true);
put(36, "data");
v.setUint32(40, dataLength, true);
new Uint8Array(out, 44).set(samples); // the padding stays zero: silence
return out;
}
Loading