Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion packages/backend/src/ai/gemini.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ export function isAiAvailable(): boolean {
export async function callGemini(
contents: Content[],
tools?: FunctionDeclaration[],
config?: { temperature?: number; model?: string; maxOutputTokens?: number }
config?: { temperature?: number; model?: string; maxOutputTokens?: number; thinkingBudget?: number }
): Promise<GeminiResponse> {
if (!client) {
throw new Error("AI features unavailable: GEMINI_API_KEY not configured");
Expand All @@ -41,6 +41,10 @@ export async function callGemini(
temperature: config?.temperature ?? TEMPERATURE.query,
maxOutputTokens: config?.maxOutputTokens ?? 8192,
};
// Thinking tokens count against maxOutputTokens; a short text can turn it off.
if (config?.thinkingBudget !== undefined) {
generateConfig.thinkingConfig = { thinkingBudget: config.thinkingBudget };
}

if (tools && tools.length > 0) {
generateConfig.tools = [{ functionDeclarations: tools }];
Expand Down
11 changes: 9 additions & 2 deletions packages/backend/src/routes/voice.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
spokenLimits,
toSpokenText,
voiceAudioBody,
voiceThinkingBudget,
voiceSummaryBody,
} from "../services/voiceSummary";
import { checkRateLimit, rateLimitKey, requireAi } from "./aiGuards";
Expand All @@ -32,6 +33,7 @@ export function voiceRoutes(sql: SQL) {
result = await callGemini(buildVoicePrompt(input), undefined, {
temperature: VOICE.temperature,
maxOutputTokens: VOICE.maxOutputTokens,
thinkingBudget: voiceThinkingBudget(DEFAULT_MODEL),
});
} catch (err) {
console.error("[ai] voice summary failed:", err);
Expand All @@ -41,8 +43,13 @@ export function voiceRoutes(sql: SQL) {
recordTokenUsage(sql, "voice_summary", DEFAULT_MODEL, result.inputTokens, result.outputTokens, orgId).catch(
(err) => console.error("[ai] voice usage not recorded:", err),
);
// speakable first: toSpokenText strips the underscores it splits identifiers on.
const text = toSpokenText(speakable(result.text), spokenLimits(input.trigger, input.length));
// Cap the model's own words first; speakable() then adds words ("5s" ->
// "5 seconds") that must not count against the cap and chop the summary.
const capped = toSpokenText(result.text, spokenLimits(input.trigger, input.length));
const spoken = speakable(capped);
// An announcement is voiced in one request (at most maxChars * 2); a reply
// summary is voiced in pieces, so only announcements need the guard.
const text = input.trigger !== "reply" && spoken.length > VOICE.maxChars * 2 ? capped : spoken;
if (!text) return c.json({ error: "Empty summary" }, 502);
return c.json({ text });
});
Expand Down
16 changes: 16 additions & 0 deletions packages/backend/src/services/__tests__/voiceSummary.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
toSpokenText,
voiceAudioBody,
voiceSummaryBody,
voiceThinkingBudget,
} from "../voiceSummary";

describe("voiceSummaryBody", () => {
Expand Down Expand Up @@ -88,6 +89,15 @@ describe("reply length (verbosity)", () => {
});
});

describe("voiceThinkingBudget", () => {
test("off for 2.5 Flash, the model's default elsewhere", () => {
expect(voiceThinkingBudget("gemini-2.5-flash")).toBe(0);
expect(voiceThinkingBudget("gemini-2.5-flash-lite")).toBe(0);
expect(voiceThinkingBudget("gemini-2.5-pro")).toBeUndefined();
expect(voiceThinkingBudget("gemini-3-flash")).toBeUndefined();
});
});

describe("spokenLimits", () => {
test("replies may run longer than announcements but fit one audio request", () => {
expect(spokenLimits("permission")).toEqual({ maxWords: VOICE.maxWords, maxChars: VOICE.maxChars });
Expand Down Expand Up @@ -122,6 +132,12 @@ describe("toSpokenText", () => {
expect(out.split(" ").length).toBe(VOICE.replyMaxWords);
});

test("keeps identifiers' underscores for speakable, drops emphasis and an echoed label", () => {
expect(toSpokenText("Project: devscope _really_ fixed __init__ and _ds_voice_speak")).toBe(
"devscope really fixed init and ds_voice_speak",
);
});

test("returns empty for empty model output", () => {
expect(toSpokenText(" ``` ``` ")).toBe("");
});
Expand Down
25 changes: 20 additions & 5 deletions packages/backend/src/services/voiceSummary.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ export const VOICE = {
// voice-audio request (maxChars * 2). Longer ones are voiced in pieces.
replyMaxWords: 50,
replyMaxChars: 400,
// Thinking tokens count against this cap on Gemini 3; the sentence itself is ~40.
maxOutputTokens: 1024,
// Thinking tokens count against this cap, and a long reply summary (~150
// tokens) was cut mid-word at 1024. Room for thinking where it can't be off.
maxOutputTokens: 4096,
temperature: 0.4,
} as const;

Expand Down Expand Up @@ -112,7 +113,7 @@ export function buildVoicePrompt(input: VoiceSummaryInput): Content[] {
const instructions = [
"You write one sentence that a text-to-speech voice reads to a developer who is in another window.",
"It tells them which Claude Code session needs them and why, so they can decide whether to switch now.",
`Start with the project name. At most ${VOICE.maxWords} words.`,
`Start with the project name itself, as a word, never a label like "Project:". At most ${VOICE.maxWords} words.`,
SPOKEN_STYLE,
DATA_NOT_INSTRUCTIONS,
"Reply with the sentence only.",
Expand All @@ -130,7 +131,7 @@ function buildReplyPrompt(input: VoiceSummaryInput): Content[] {
const instructions = [
"A text-to-speech voice reads your text to a developer right after Claude Code finished answering them.",
REPLY_LENGTHS[input.length ?? "normal"].ask,
`Start with the project name. At most ${REPLY_LENGTHS[input.length ?? "normal"].maxWords} words.`,
`Start with the project name itself, as a word, never a label like "Project:". At most ${REPLY_LENGTHS[input.length ?? "normal"].maxWords} words.`,
SPOKEN_STYLE,
DATA_NOT_INSTRUCTIONS,
"Reply with the summary only.",
Expand All @@ -139,14 +140,28 @@ function buildReplyPrompt(input: VoiceSummaryInput): Content[] {
return [{ role: "user", parts: [{ text: `${instructions}\n\n${facts.join("\n")}` }] }];
}

/**
* Thinking off for these short texts where the model allows it (2.5 Flash
* accepts a budget of 0; other models keep their default and the larger
* output cap): it is faster, and the budget goes to the text.
*/
export function voiceThinkingBudget(model: string): number | undefined {
return /gemini-2\.5-flash/.test(model) ? 0 : undefined;
}

/** Make model output safe to hand to a TTS engine: one short line, no markup. */
export function toSpokenText(
raw: string,
{ maxWords, maxChars }: SpokenLimits = { maxWords: VOICE.maxWords, maxChars: VOICE.maxChars },
): string {
const text = raw
.replace(/```[\s\S]*?```/g, " ")
.replace(/[`*_#>"~|<>{}[\]\\]/g, "")
// An underscore inside a word is an identifier (snake_case), which
// speakable() turns into words; only emphasis underscores go.
.replace(/(?<!\w)_+|_+(?!\w)/g, "")
.replace(/[`*#>"~|<>{}[\]\\]/g, "")
// The model sometimes echoes the prompt's "Project:" label.
.replace(/^\s*project:\s*/i, "")
.replace(/\s+/g, " ")
.trim();
if (!text) return "";
Expand Down
Loading