diff --git a/CHANGELOG.md b/CHANGELOG.md index 71cdcf2..1d663d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Added +- **Auto voice can stay quiet.** A `reply` request to `/api/ai/voice-summary` + with `may_skip: true` lets the model answer `SKIP` when the reply brings no + news, and the endpoint then returns `{skip: true}` (plus `label`) instead of + text. The plugin (0.36.0) also sends `background` (the turn was started by a + monitor or task notification, not the developer) and `previous` (the summary + it last spoke for the session). Background turns need real news (finished + work, a failure, a new question); progress, expired monitors and restated + state are skipped. Turns the developer started are always summarized unless + they only acknowledge or repeat, and failures are never skipped. Older + plugins send no `may_skip` and are unaffected. - **Voiced audio ends cleanly.** `/api/ai/voice-audio` rewrites the speech service's WAV into a plain 44-byte-header PCM file with its real sizes (Kokoro streams its WAV with `0xFFFFFFFF` sizes and a LIST chunk, so players diff --git a/CLAUDE.md b/CLAUDE.md index f1362fd..babca51 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -90,7 +90,7 @@ WebSocket message types: `event.new`, `session.update`, `developer.update`. | `/api/live/vcs/open-prs?repo_remote=` | GET | The caller's PR links in that repo still unknown/open and not checked for an hour; the mod resolves them with `gh` | | `/api/live/vcs/status` | POST | `{ref, state: open\|merged\|closed, merged_at?, closed_at?}`: PR state for the caller's own links | | `/api/ai/voice-models` | GET | `{models}`: the voices this server offers (`TTS_SERVICES` names), default first | -| `/api/ai/voice-summary` | POST | Spoken text for the plugin's voice features (`trigger`, `project`, optional `tool`/`detail`/`last_message`/`session_id`/`label`), starting with the session's label (`project, topic` from the caller's own session title, else branch; a sent `label` is reused as is, so the name never changes) and returning it as `label`: one sentence for the announcer, or with `trigger: "reply"` a summary of Claude's reply for `/devscope:voice auto`, as detailed as the optional `length` asks (`short` one sentence ≤ 20 words, `normal` 2-3 sentences ≤ 50 words, `long` 4-6 sentences ≤ 110 words; `REPLY_LENGTHS`); stateless, only token usage is recorded as `voice_summary`. The plugin never calls it for `private` sessions | +| `/api/ai/voice-summary` | POST | Spoken text for the plugin's voice features (`trigger`, `project`, optional `tool`/`detail`/`last_message`/`session_id`/`label`), starting with the session's label (`project, topic` from the caller's own session title, else branch; a sent `label` is reused as is, so the name never changes) and returning it as `label`: one sentence for the announcer, or with `trigger: "reply"` a summary of Claude's reply for `/devscope:voice auto`, as detailed as the optional `length` asks (`short` one sentence ≤ 20 words, `normal` 2-3 sentences ≤ 50 words, `long` 4-6 sentences ≤ 110 words; `REPLY_LENGTHS`); with `may_skip: true` (plus `background`, `previous` = the summary last spoken for the session) a `reply` the model judges not worth saying returns `{skip: true}` and the plugin stays quiet (`SKIP_RULES`); stateless, only token usage is recorded as `voice_summary`. The plugin never calls it for `private` sessions | | `/api/ai/voice-audio` | POST | That text (or one piece of a `/devscope:voice explain` explanation) as WAV (`text`, optional `model` = a `TTS_SERVICES` name, Kokoro `voice`, `speed`, `volume`, `pad_ms` trailing silence, default 400), rewritten as a canonical PCM WAV with real sizes (`services/wav.ts`), voiced by the homelab speech services; 503 when none is configured or all fail, so the plugin falls back to a local voice. Own 20/min bucket, no Gemini budget | | `/api/sessions/usage/backfill` | POST | Exact token usage per transcript and model from `/devscope:backfill-usage` (`{items: [{transcriptId, sessionId?, byModel}]}`, max 500); only the caller's own sessions are updated, the rest are counted as `skipped` | | `/api/health` | GET | Health check + WS client count | diff --git a/packages/backend/src/routes/__tests__/voice.test.ts b/packages/backend/src/routes/__tests__/voice.test.ts index c346bf3..972e554 100644 --- a/packages/backend/src/routes/__tests__/voice.test.ts +++ b/packages/backend/src/routes/__tests__/voice.test.ts @@ -191,6 +191,19 @@ describe("POST /api/ai/voice-summary (session labels)", () => { expect(prompt).toContain("Start with the project name"); }); + test("a reply the model judges not worth saying comes back as skip, with the label", async () => { + mockGemini.mockImplementationOnce(async () => ({ text: "SKIP", inputTokens: 10, outputTokens: 1 })); + const res = await summarize({ trigger: "reply", project: "api-service", session_id: "s1", last_message: "Monitor expired.", may_skip: true, background: true }); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ skip: true, label: "api-service, Rate limiter fix" }); + }); + + test("without may_skip a SKIP answer is not honoured as silence", async () => { + mockGemini.mockImplementationOnce(async () => ({ text: "SKIP", inputTokens: 10, outputTokens: 1 })); + const res = await summarize({ trigger: "reply", project: "api-service", last_message: "Monitor expired." }); + expect((await res.json()).skip).toBeUndefined(); + }); + test("a failed lookup still answers, labelled with the project", async () => { const failing = (async () => { throw new Error("db down"); }) as any; const app = new Hono(); diff --git a/packages/backend/src/routes/voice.ts b/packages/backend/src/routes/voice.ts index bdc6a7c..f435e6a 100644 --- a/packages/backend/src/routes/voice.ts +++ b/packages/backend/src/routes/voice.ts @@ -9,6 +9,7 @@ import { recordTokenUsage } from "../db"; import { VOICE, buildVoicePrompt, + isSkip, sessionLabel, sessionTopic, spokenLimits, @@ -50,6 +51,10 @@ export function voiceRoutes(sql: SQL) { recordTokenUsage(sql, "voice_summary", DEFAULT_MODEL, result.inputTokens, result.outputTokens, orgId).catch( (err) => console.error("[ai] voice usage not recorded:", err), ); + // The model judged the reply not worth saying; only plugins that can stay quiet are offered that. + if (input.trigger === "reply" && input.may_skip && isSkip(result.text)) { + return c.json(label ? { skip: true, label } : { skip: true }); + } // Cap the model's own words first; speakable() then adds words ("5s" -> // "5 seconds") that must not count against the cap and chop the summary. const capped = toSpokenText(result.text, spokenLimits(input.trigger, input.length)); diff --git a/packages/backend/src/services/__tests__/voiceSummary.test.ts b/packages/backend/src/services/__tests__/voiceSummary.test.ts index 25acb32..5616072 100644 --- a/packages/backend/src/services/__tests__/voiceSummary.test.ts +++ b/packages/backend/src/services/__tests__/voiceSummary.test.ts @@ -2,9 +2,11 @@ import { describe, expect, test } from "bun:test"; import { LABEL_MAX_CHARS, REPLY_LENGTHS, + SKIP_WORD, VOICE, branchTopic, buildVoicePrompt, + isSkip, sessionLabel, sessionTopic, spokenLimits, @@ -74,6 +76,46 @@ describe("buildVoicePrompt (reply)", () => { }); }); +describe("buildVoicePrompt (reply, may skip)", () => { + const prompt = (extra: Record) => + buildVoicePrompt(voiceSummaryBody.parse({ trigger: "reply", project: "p", last_message: "Monitor expired.", ...extra }))[0] + .parts![0].text!; + + test("older plugins are never offered silence", () => { + const text = prompt({}); + expect(text).not.toContain(SKIP_WORD); + expect(text).not.toContain("Turn started by"); + }); + + test("offers SKIP and says who started the turn", () => { + const bg = prompt({ may_skip: true, background: true }); + expect(bg).toContain(`answer with the single word ${SKIP_WORD}`); + expect(bg).toContain("need real news to be spoken"); + expect(bg).toContain("Turn started by: a background event"); + expect(prompt({ may_skip: true })).toContain("Turn started by: the developer"); + }); + + test("includes the previous summary to judge what is new", () => { + expect(prompt({ may_skip: true, previous: "The report waits for approval." })).toContain( + "Previous summary spoken for this session: The report waits for approval.", + ); + expect(prompt({ previous: "The report waits for approval." })).not.toContain("Previous summary"); + }); + + test("rejects an oversized previous summary", () => { + expect(voiceSummaryBody.safeParse({ trigger: "reply", project: "p", previous: "x".repeat(1001) }).success).toBe(false); + }); +}); + +describe("isSkip", () => { + test("the bare word, any case or punctuation", () => { + for (const raw of ["SKIP", "skip", " SKIP.\n", "**Skip**"]) expect(isSkip(raw)).toBe(true); + }); + test("a summary that mentions skipping is spoken", () => { + for (const raw of ["Skipped the flaky test.", "SKIP the migration? It asks.", ""]) expect(isSkip(raw)).toBe(false); + }); +}); + describe("reply length (verbosity)", () => { const prompt = (length?: "short" | "normal" | "long") => buildVoicePrompt({ trigger: "reply", project: "p", last_message: "Done.", ...(length ? { length } : {}) })[0].parts![0].text!; diff --git a/packages/backend/src/services/voiceSummary.ts b/packages/backend/src/services/voiceSummary.ts index fa2354c..0c4f882 100644 --- a/packages/backend/src/services/voiceSummary.ts +++ b/packages/backend/src/services/voiceSummary.ts @@ -11,7 +11,9 @@ import { clip } from "./promptRecall"; * - Reply summaries (`reply`, `/devscope:voice auto on`): what Claude just * answered, read after every finished turn, as long as the developer's * `length` (`/devscope:voice verbosity`) asks: one sentence, two or three, - * or a fuller account. + * or a fuller account. With `may_skip` the model may instead decide the + * reply is not worth saying (a background turn with routine news, or nothing + * beyond what the previous summary said) and the plugin stays quiet. * * Every text starts with the session's label, project plus what it is working * on ("api-service, rate limiter fix"), so a developer running several @@ -53,6 +55,12 @@ export const voiceSummaryBody = z.object({ session_id: z.string().trim().min(1).max(200).optional(), /** A label this server returned earlier for the session; reused as is so the session's name stays put. */ label: z.string().trim().min(1).max(LABEL_MAX_CHARS).optional(), + /** `reply` only: the plugin can stay quiet, so the model may answer SKIP. Older plugins always get a summary. */ + may_skip: z.boolean().optional(), + /** `reply` only: the turn was started by a background event (monitor, task notification), not the developer. */ + background: z.boolean().optional(), + /** `reply` only: the summary last spoken for this session, to judge what is new. */ + previous: z.string().trim().max(1000).optional(), }); export type VoiceSummaryInput = z.infer; @@ -155,20 +163,53 @@ export function buildVoicePrompt(input: VoiceSummaryInput, labelled = false): Co return [{ role: "user", parts: [{ text: `${instructions}\n\n${facts.join("\n")}` }] }]; } +/** What the model answers when a reply is not worth saying out loud. */ +export const SKIP_WORD = "SKIP"; + +/** True when the model chose silence: the bare word, give or take punctuation and case. */ +export function isSkip(raw: string): boolean { + return raw.replace(/[^a-z]/gi, "").toUpperCase() === SKIP_WORD; +} + +/** + * When to stay quiet. The developer hears every finished turn, so a voice that + * reports bookkeeping teaches them to stop listening. What counts is news: a + * waiting approval the previous summary already announced is not news again. + */ +const SKIP_RULES = + `First decide whether the reply brings news worth saying out loud. If it does not, answer with the single word ${SKIP_WORD} and nothing else. ` + + "News is: work the developer asked for or was waiting on is done, a failure, an error or an unexpected result, " + + "or a new question, decision or approval for them that the previous summary did not already mention. " + + `Answer ${SKIP_WORD} for everything else, in particular: progress updates and anything still running or still waiting; ` + + "a monitor, watch or timer that expired, fired with nothing new, or was re-armed or left alone; " + + "a reply that mostly restates the state the previous summary described, even when that state needs their approval; " + + "and bare acknowledgements. " + + `Never answer ${SKIP_WORD} when the reply reports a failure, an error or failing tests, whoever started the turn. ` + + "These rules are for turns started by a background event, which need real news to be spoken. " + + "A turn the developer started themselves is the answer they are waiting for: always summarize it, " + + `and answer ${SKIP_WORD} only when it is a bare acknowledgement or says almost exactly what the previous summary said.`; + function buildReplyPrompt(input: VoiceSummaryInput, labelled: boolean): Content[] { + const maySkip = input.may_skip === true; const facts = [ `Project: ${input.project}`, + maySkip && + `Turn started by: ${input.background ? "a background event (monitor, timer or task notification), not the developer" : "the developer"}`, + maySkip && input.previous && `Previous summary spoken for this session: ${clip(input.previous, 1000)}`, `Claude's reply: ${clip(input.last_message ?? "", 4000)}`, - ]; + ].filter(Boolean); const instructions = [ "A text-to-speech voice reads your text to a developer right after Claude Code finished answering them.", + maySkip && SKIP_RULES, REPLY_LENGTHS[input.length ?? "normal"].ask, `${opening(labelled)} At most ${REPLY_LENGTHS[input.length ?? "normal"].maxWords} words.`, SPOKEN_STYLE, DATA_NOT_INSTRUCTIONS, - "Reply with the summary only.", - ].join(" "); + maySkip ? `Reply with the summary only, or ${SKIP_WORD}.` : "Reply with the summary only.", + ] + .filter(Boolean) + .join(" "); return [{ role: "user", parts: [{ text: `${instructions}\n\n${facts.join("\n")}` }] }]; }