From 71773735a7f2a61fde3ccdfcdb5b5d33953fc837 Mon Sep 17 00:00:00 2001 From: Lucas Proxmox Date: Wed, 21 Oct 2026 12:55:36 +0200 Subject: [PATCH] feat(voice): let auto voice stay quiet on background noise Reply summaries can now come back as {skip: true} when the model judges the turn has no news worth saying: progress updates, monitors that expired or were re-armed, or a status the previous summary already gave. The decision is made in the same Gemini call (the model answers SKIP), so it adds no latency. Opt-in via may_skip, so older plugins keep a summary every time. The plugin also sends background (the turn was not started by the developer) and previous (the last summary spoken for the session). Failures are never skipped, and turns the developer started are always summarized unless they are a bare acknowledgement or repeat the previous summary. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 10 ++++ CLAUDE.md | 2 +- .../src/routes/__tests__/voice.test.ts | 13 +++++ packages/backend/src/routes/voice.ts | 5 ++ .../services/__tests__/voiceSummary.test.ts | 42 ++++++++++++++++ packages/backend/src/services/voiceSummary.ts | 49 +++++++++++++++++-- 6 files changed, 116 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71cdcf2..1d663d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- **Auto voice can stay quiet.** A `reply` request to `/api/ai/voice-summary` + with `may_skip: true` lets the model answer `SKIP` when the reply brings no + news, and the endpoint then returns `{skip: true}` (plus `label`) instead of + text. The plugin (0.36.0) also sends `background` (the turn was started by a + monitor or task notification, not the developer) and `previous` (the summary + it last spoke for the session). Background turns need real news (finished + work, a failure, a new question); progress, expired monitors and restated + state are skipped. Turns the developer started are always summarized unless + they only acknowledge or repeat, and failures are never skipped. Older + plugins send no `may_skip` and are unaffected. - **Voiced audio ends cleanly.** `/api/ai/voice-audio` rewrites the speech service's WAV into a plain 44-byte-header PCM file with its real sizes (Kokoro streams its WAV with `0xFFFFFFFF` sizes and a LIST chunk, so players diff --git a/CLAUDE.md b/CLAUDE.md index f1362fd..babca51 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -90,7 +90,7 @@ WebSocket message types: `event.new`, `session.update`, `developer.update`. | `/api/live/vcs/open-prs?repo_remote=` | GET | The caller's PR links in that repo still unknown/open and not checked for an hour; the mod resolves them with `gh` | | `/api/live/vcs/status` | POST | `{ref, state: open\|merged\|closed, merged_at?, closed_at?}`: PR state for the caller's own links | | `/api/ai/voice-models` | GET | `{models}`: the voices this server offers (`TTS_SERVICES` names), default first | -| `/api/ai/voice-summary` | POST | Spoken text for the plugin's voice features (`trigger`, `project`, optional `tool`/`detail`/`last_message`/`session_id`/`label`), starting with the session's label (`project, topic` from the caller's own session title, else branch; a sent `label` is reused as is, so the name never changes) and returning it as `label`: one sentence for the announcer, or with `trigger: "reply"` a summary of Claude's reply for `/devscope:voice auto`, as detailed as the optional `length` asks (`short` one sentence ≤ 20 words, `normal` 2-3 sentences ≤ 50 words, `long` 4-6 sentences ≤ 110 words; `REPLY_LENGTHS`); stateless, only token usage is recorded as `voice_summary`. The plugin never calls it for `private` sessions | +| `/api/ai/voice-summary` | POST | Spoken text for the plugin's voice features (`trigger`, `project`, optional `tool`/`detail`/`last_message`/`session_id`/`label`), starting with the session's label (`project, topic` from the caller's own session title, else branch; a sent `label` is reused as is, so the name never changes) and returning it as `label`: one sentence for the announcer, or with `trigger: "reply"` a summary of Claude's reply for `/devscope:voice auto`, as detailed as the optional `length` asks (`short` one sentence ≤ 20 words, `normal` 2-3 sentences ≤ 50 words, `long` 4-6 sentences ≤ 110 words; `REPLY_LENGTHS`); with `may_skip: true` (plus `background`, `previous` = the summary last spoken for the session) a `reply` the model judges not worth saying returns `{skip: true}` and the plugin stays quiet (`SKIP_RULES`); stateless, only token usage is recorded as `voice_summary`. The plugin never calls it for `private` sessions | | `/api/ai/voice-audio` | POST | That text (or one piece of a `/devscope:voice explain` explanation) as WAV (`text`, optional `model` = a `TTS_SERVICES` name, Kokoro `voice`, `speed`, `volume`, `pad_ms` trailing silence, default 400), rewritten as a canonical PCM WAV with real sizes (`services/wav.ts`), voiced by the homelab speech services; 503 when none is configured or all fail, so the plugin falls back to a local voice. Own 20/min bucket, no Gemini budget | | `/api/sessions/usage/backfill` | POST | Exact token usage per transcript and model from `/devscope:backfill-usage` (`{items: [{transcriptId, sessionId?, byModel}]}`, max 500); only the caller's own sessions are updated, the rest are counted as `skipped` | | `/api/health` | GET | Health check + WS client count | diff --git a/packages/backend/src/routes/__tests__/voice.test.ts b/packages/backend/src/routes/__tests__/voice.test.ts index c346bf3..972e554 100644 --- a/packages/backend/src/routes/__tests__/voice.test.ts +++ b/packages/backend/src/routes/__tests__/voice.test.ts @@ -191,6 +191,19 @@ describe("POST /api/ai/voice-summary (session labels)", () => { expect(prompt).toContain("Start with the project name"); }); + test("a reply the model judges not worth saying comes back as skip, with the label", async () => { + mockGemini.mockImplementationOnce(async () => ({ text: "SKIP", inputTokens: 10, outputTokens: 1 })); + const res = await summarize({ trigger: "reply", project: "api-service", session_id: "s1", last_message: "Monitor expired.", may_skip: true, background: true }); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ skip: true, label: "api-service, Rate limiter fix" }); + }); + + test("without may_skip a SKIP answer is not honoured as silence", async () => { + mockGemini.mockImplementationOnce(async () => ({ text: "SKIP", inputTokens: 10, outputTokens: 1 })); + const res = await summarize({ trigger: "reply", project: "api-service", last_message: "Monitor expired." }); + expect((await res.json()).skip).toBeUndefined(); + }); + test("a failed lookup still answers, labelled with the project", async () => { const failing = (async () => { throw new Error("db down"); }) as any; const app = new Hono(); diff --git a/packages/backend/src/routes/voice.ts b/packages/backend/src/routes/voice.ts index bdc6a7c..f435e6a 100644 --- a/packages/backend/src/routes/voice.ts +++ b/packages/backend/src/routes/voice.ts @@ -9,6 +9,7 @@ import { recordTokenUsage } from "../db"; import { VOICE, buildVoicePrompt, + isSkip, sessionLabel, sessionTopic, spokenLimits, @@ -50,6 +51,10 @@ export function voiceRoutes(sql: SQL) { recordTokenUsage(sql, "voice_summary", DEFAULT_MODEL, result.inputTokens, result.outputTokens, orgId).catch( (err) => console.error("[ai] voice usage not recorded:", err), ); + // The model judged the reply not worth saying; only plugins that can stay quiet are offered that. + if (input.trigger === "reply" && input.may_skip && isSkip(result.text)) { + return c.json(label ? { skip: true, label } : { skip: true }); + } // Cap the model's own words first; speakable() then adds words ("5s" -> // "5 seconds") that must not count against the cap and chop the summary. const capped = toSpokenText(result.text, spokenLimits(input.trigger, input.length)); diff --git a/packages/backend/src/services/__tests__/voiceSummary.test.ts b/packages/backend/src/services/__tests__/voiceSummary.test.ts index 25acb32..5616072 100644 --- a/packages/backend/src/services/__tests__/voiceSummary.test.ts +++ b/packages/backend/src/services/__tests__/voiceSummary.test.ts @@ -2,9 +2,11 @@ import { describe, expect, test } from "bun:test"; import { LABEL_MAX_CHARS, REPLY_LENGTHS, + SKIP_WORD, VOICE, branchTopic, buildVoicePrompt, + isSkip, sessionLabel, sessionTopic, spokenLimits, @@ -74,6 +76,46 @@ describe("buildVoicePrompt (reply)", () => { }); }); +describe("buildVoicePrompt (reply, may skip)", () => { + const prompt = (extra: Record) => + buildVoicePrompt(voiceSummaryBody.parse({ trigger: "reply", project: "p", last_message: "Monitor expired.", ...extra }))[0] + .parts![0].text!; + + test("older plugins are never offered silence", () => { + const text = prompt({}); + expect(text).not.toContain(SKIP_WORD); + expect(text).not.toContain("Turn started by"); + }); + + test("offers SKIP and says who started the turn", () => { + const bg = prompt({ may_skip: true, background: true }); + expect(bg).toContain(`answer with the single word ${SKIP_WORD}`); + expect(bg).toContain("need real news to be spoken"); + expect(bg).toContain("Turn started by: a background event"); + expect(prompt({ may_skip: true })).toContain("Turn started by: the developer"); + }); + + test("includes the previous summary to judge what is new", () => { + expect(prompt({ may_skip: true, previous: "The report waits for approval." })).toContain( + "Previous summary spoken for this session: The report waits for approval.", + ); + expect(prompt({ previous: "The report waits for approval." })).not.toContain("Previous summary"); + }); + + test("rejects an oversized previous summary", () => { + expect(voiceSummaryBody.safeParse({ trigger: "reply", project: "p", previous: "x".repeat(1001) }).success).toBe(false); + }); +}); + +describe("isSkip", () => { + test("the bare word, any case or punctuation", () => { + for (const raw of ["SKIP", "skip", " SKIP.\n", "**Skip**"]) expect(isSkip(raw)).toBe(true); + }); + test("a summary that mentions skipping is spoken", () => { + for (const raw of ["Skipped the flaky test.", "SKIP the migration? It asks.", ""]) expect(isSkip(raw)).toBe(false); + }); +}); + describe("reply length (verbosity)", () => { const prompt = (length?: "short" | "normal" | "long") => buildVoicePrompt({ trigger: "reply", project: "p", last_message: "Done.", ...(length ? { length } : {}) })[0].parts![0].text!; diff --git a/packages/backend/src/services/voiceSummary.ts b/packages/backend/src/services/voiceSummary.ts index fa2354c..0c4f882 100644 --- a/packages/backend/src/services/voiceSummary.ts +++ b/packages/backend/src/services/voiceSummary.ts @@ -11,7 +11,9 @@ import { clip } from "./promptRecall"; * - Reply summaries (`reply`, `/devscope:voice auto on`): what Claude just * answered, read after every finished turn, as long as the developer's * `length` (`/devscope:voice verbosity`) asks: one sentence, two or three, - * or a fuller account. + * or a fuller account. With `may_skip` the model may instead decide the + * reply is not worth saying (a background turn with routine news, or nothing + * beyond what the previous summary said) and the plugin stays quiet. * * Every text starts with the session's label, project plus what it is working * on ("api-service, rate limiter fix"), so a developer running several @@ -53,6 +55,12 @@ export const voiceSummaryBody = z.object({ session_id: z.string().trim().min(1).max(200).optional(), /** A label this server returned earlier for the session; reused as is so the session's name stays put. */ label: z.string().trim().min(1).max(LABEL_MAX_CHARS).optional(), + /** `reply` only: the plugin can stay quiet, so the model may answer SKIP. Older plugins always get a summary. */ + may_skip: z.boolean().optional(), + /** `reply` only: the turn was started by a background event (monitor, task notification), not the developer. */ + background: z.boolean().optional(), + /** `reply` only: the summary last spoken for this session, to judge what is new. */ + previous: z.string().trim().max(1000).optional(), }); export type VoiceSummaryInput = z.infer; @@ -155,20 +163,53 @@ export function buildVoicePrompt(input: VoiceSummaryInput, labelled = false): Co return [{ role: "user", parts: [{ text: `${instructions}\n\n${facts.join("\n")}` }] }]; } +/** What the model answers when a reply is not worth saying out loud. */ +export const SKIP_WORD = "SKIP"; + +/** True when the model chose silence: the bare word, give or take punctuation and case. */ +export function isSkip(raw: string): boolean { + return raw.replace(/[^a-z]/gi, "").toUpperCase() === SKIP_WORD; +} + +/** + * When to stay quiet. The developer hears every finished turn, so a voice that + * reports bookkeeping teaches them to stop listening. What counts is news: a + * waiting approval the previous summary already announced is not news again. + */ +const SKIP_RULES = + `First decide whether the reply brings news worth saying out loud. If it does not, answer with the single word ${SKIP_WORD} and nothing else. ` + + "News is: work the developer asked for or was waiting on is done, a failure, an error or an unexpected result, " + + "or a new question, decision or approval for them that the previous summary did not already mention. " + + `Answer ${SKIP_WORD} for everything else, in particular: progress updates and anything still running or still waiting; ` + + "a monitor, watch or timer that expired, fired with nothing new, or was re-armed or left alone; " + + "a reply that mostly restates the state the previous summary described, even when that state needs their approval; " + + "and bare acknowledgements. " + + `Never answer ${SKIP_WORD} when the reply reports a failure, an error or failing tests, whoever started the turn. ` + + "These rules are for turns started by a background event, which need real news to be spoken. " + + "A turn the developer started themselves is the answer they are waiting for: always summarize it, " + + `and answer ${SKIP_WORD} only when it is a bare acknowledgement or says almost exactly what the previous summary said.`; + function buildReplyPrompt(input: VoiceSummaryInput, labelled: boolean): Content[] { + const maySkip = input.may_skip === true; const facts = [ `Project: ${input.project}`, + maySkip && + `Turn started by: ${input.background ? "a background event (monitor, timer or task notification), not the developer" : "the developer"}`, + maySkip && input.previous && `Previous summary spoken for this session: ${clip(input.previous, 1000)}`, `Claude's reply: ${clip(input.last_message ?? "", 4000)}`, - ]; + ].filter(Boolean); const instructions = [ "A text-to-speech voice reads your text to a developer right after Claude Code finished answering them.", + maySkip && SKIP_RULES, REPLY_LENGTHS[input.length ?? "normal"].ask, `${opening(labelled)} At most ${REPLY_LENGTHS[input.length ?? "normal"].maxWords} words.`, SPOKEN_STYLE, DATA_NOT_INSTRUCTIONS, - "Reply with the summary only.", - ].join(" "); + maySkip ? `Reply with the summary only, or ${SKIP_WORD}.` : "Reply with the summary only.", + ] + .filter(Boolean) + .join(" "); return [{ role: "user", parts: [{ text: `${instructions}\n\n${facts.join("\n")}` }] }]; }