diff --git a/benchmarks/README.md b/benchmarks/README.md index f6d57dc8..c01b27c4 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -10,6 +10,7 @@ results surface in a browser/note block. | Benchmark | Dir | Runnable? | Execution model | |-----------|-----|-----------|-----------------| | slop-code-bench | [`slopcodebench/`](slopcodebench/) | ✅ yes | **Host** (no Docker) via the fork's tmux executor | +| Open-weight TTS | [`talkito-tts/`](talkito-tts/) | ✅ yes | **Host**, one PTY; no agent, no keys; reader picks engines interactively | | SWE-bench (Lite/Verified/Full) | [`swebench/`](swebench/) | ✅ yes | Inference on host + **Docker-per-task** eval (Modal or DinD) | | SWE-bench Pro (public 731) | [`swebench-pro/`](swebench-pro/) | ✅ yes | Inference on host + **Docker-per-task** eval (`--use_local_docker` or Modal) | | Terminal-Bench 2.1 | [`terminal-bench/`](terminal-bench/) | ✅ yes | **Harbor** harness; Docker-per-task (or `--env daytona`/Modal remote) | @@ -17,6 +18,13 @@ results surface in a browser/note block. ## Two architecture families +These describe the *coding* benchmarks, where something has to run an agent under +test. The TTS benchmark belongs to neither: it measures speech engines rather than +agents, so it has no agent-under-test, needs no provider key, and its entire run is +one terminal's boot command with a browser block reading the report. It is the only +template here that skips the config wizard and the orchestrator entirely — see +[`talkito-tts/`](talkito-tts/). + **Family A — host-executable.** The agent-under-test runs directly in the Orcabot sandbox VM; no inner container. The sandbox VM *is* the isolation boundary. This is slop-code-bench (via the fork's host-tmux executor), where @@ -66,14 +74,31 @@ benchmarks/bin/seed-benchmark-templates swebench # just one, by dir name It upserts each `benchmarks/*/template.json` (delete-by-name then insert, `status='approved'`) against the desktop d1-shim (`D1_URL`, default `http://127.0.0.1:9001`). `slopcodebench/bin/scb-insert-template` is a thin -wrapper that seeds only the SlopCodeBench template. For a **remote/cloud** control -plane, export from an existing dashboard via the templates API (`POST /templates`) -instead — direct INSERT is the local/curated path. +wrapper that seeds only the SlopCodeBench template. + +For a **remote/cloud** control plane there is no code seed at all: +`templates.seedStarterTemplates` returns early unless `SURFACE_TOKEN` is set, and only +the desktop app sets it, so `starter-templates.json` never reaches production. Every +production row is a direct insert. Use the script's `--sql` mode: + +``` +AUTHOR_ID= AUTHOR_NAME=Orcabot \ + benchmarks/bin/seed-benchmark-templates --sql talkito-tts > /tmp/seed.sql +cd controlplane && npx wrangler d1 execute orcabot-db-production --remote --file /tmp/seed.sql +``` + +`AUTHOR_ID` is not optional. It defaults to `dev-desktop`, which exists on the desktop +shim but not in a cloud or wrangler-dev database, and `dashboard_templates.author_id` +is a foreign key into `users` — the insert then fails with `FOREIGN KEY constraint +failed`, quietly enough to look like success. Grep the output for `ERROR` rather than +trusting a clean exit. ## Status -All templates here are **prototypes**. slop-code-bench is validated end-to-end -(see its README). The Family-B templates are grounded in each harness's real +All templates here are **prototypes**. slop-code-bench and the open-weight TTS +benchmark are validated end-to-end (see their READMEs); the TTS one is the +cheapest to re-verify, since a piper run is about three and a half minutes and +needs no key. The Family-B templates are grounded in each harness's real commands but are **not** validated end-to-end from here, because they need a Docker-capable backend (Modal token or DinD sandbox) + provider keys that aren't available in this environment. Each dir's README states exactly what's verified diff --git a/benchmarks/talkito-tts/README.md b/benchmarks/talkito-tts/README.md new file mode 100644 index 00000000..b7f85aa4 --- /dev/null +++ b/benchmarks/talkito-tts/README.md @@ -0,0 +1,132 @@ +# talkito TTS benchmark + +Runs the open-weight TTS benchmark from +[talkito-benchmark](https://github.com/robdmac/talkito-benchmark) inside one Orcabot +terminal and shows the report it produces in a browser block beside it. This is the +runnable version of the published +[Testing open-weight Text To Speech models](https://orcabot.com/benchmarks/open-weight-tts) +table: same harness, same 84-phrase corpus, same page. + +The reader chooses which engines to measure. The terminal opens the benchmark's own +checklist — arrows move, space toggles, `n` clears, Enter runs — and everything after +that is automatic. + +## Why this one has no orchestrator + +Every other template here drives a benchmark through an agent, because those +benchmarks need an agent-under-test, provider keys and per-task containers. This one +needs none of that. The benchmark is a Python program that synthesizes a fixed corpus +and scores it, so the whole run fits in a single boot command and the only moving +parts are a terminal and a browser block. + +That also makes it the cheapest template here to run: no LLM calls, no API key, no +tokens. It costs CPU time and nothing else. + +**It ships no `setupGuide`, deliberately.** That field is not documentation — it is a +walkthrough brief for Orcabot chat, and its presence has two effects. The dashboards +page sets `orcabot_initial_prompt` to "Walk me through setting up this dashboard.", +which `ChatPanel` auto-sends on mount with the panel expanded. And the control plane +injects the guide into *every* chat message on that dashboard, under a system prompt +reading "Use your tools to do the work (… `create_terminal`/`create_browser` as +needed)". + +An earlier version carried a guide that described the dashboard, and chat did as the +wrapper told it: two terminals and two browsers, with a second benchmark writing the +same measurement files as the first. Replacing the text with "do not create anything" +only argued with the instruction wrapped around it. Removing the field removes the +auto-prompt, the walkthrough system prompt, and the reason chat was invoked at all — +so it stays minimised and silent, which is right for a dashboard with no keys, no +provider choice, no config, and a terminal that starts itself. + +## Architecture + + terminal block ──runs──> tools/benchmark.py (interactive picker) + │ │ + │ └──> tts-benchmark.html + └──serves──> python3 -m http.server 8099 <──reads──┘ + ▲ + browser block ──opens──> http://localhost:8099/ + +The terminal starts the web server *before* it starts cloning, serving a placeholder +that refreshes every ten seconds. Without that, the browser block would show a +connection error for the whole run and only become useful at the very end. When the +report is built it is copied over the placeholder and the next refresh picks it up. + +## What the sandbox forces + +All of these were found by running it, not by reading it, and two of them fail +*quietly* — the run still exits 0 and still publishes a page. + +**HTTPS clones, not submodules.** talkito's `.gitmodules` points at +`git@github.com:robdmac/...`, and the sandbox has no SSH key, so +`git clone --recurse-submodules` cannot work. The boot command clones the two public +repos over HTTPS instead and arranges them as + + /workspace/talkito/ talkito + /workspace/talkito/benchmark/ talkito-benchmark + +The harness ships inside talkito-benchmark, so `tests-repo` is not needed. +`tools/paths.py` still wants a talkito **checkout** rather than the installed package +— it looks for `talkito/tts.py` — and `benchmark/` inside `talkito/` is the layout it +resolves via `../..`. + +**`TALKITO_AUTO_APPROVE_DOWNLOADS=1`.** talkito refuses to fetch model weights when it +cannot ask a human. The picker is interactive, so it *could* ask — but the prompt would +land in the middle of a sweep the reader has walked away from, so it is answered in +advance. Without it every phrase fails with `Piper model unavailable` and the sweep +still reports `rc=0`, having published a table reading `0/84 phrases, WER 0.0%, +RTF 0.00x`. A green run measuring nothing is the worst failure available here. + +**No listening clips, and therefore no ffmpeg.** `make_samples.py` exists to embed one +playable clip per engine, and it shells out to ffmpeg for the wav-to-mp3 step. Audio +does not reach the reader through the browser block, so the clips are not synthesized +at all — which removes ffmpeg, which removes the only `apt-get` this template ran, +which removes an egress approval prompt for `deb.nodesource.com`. The sandbox image +installs Node from NodeSource at build time and leaves the apt source configured, so +*any* `apt-get update` inside the sandbox reaches for it. The table is numbers. + +**The published page's encoding is declared upstream now.** It was not: the report +carried no `` and `python3 -m http.server` sends `text/html` with no +charset either, so a browser fell back to its locale default and read valid UTF-8 as +cp1252 — `TTS Engine Benchmark — 24 configurations` arriving with garbage where the em +dash should be. `tools/page_template.html` declares it now, so the boot command simply +copies. It keeps a fallback that prepends the tag when a page lacks one, because the +failure is silent and total, and it writes to the published copy rather than the source +so the export zip stays byte-identical to what the tools wrote. + +## Choosing engines + +Everything is ticked when the picker opens, and a **full sweep is around eleven +hours** — longer on a first run, because every engine downloads its own weights. Press +`n` to clear, then space to pick a few. + +piper is the cheapest complete arm in the table: no torch at all, and scoring runs +through faster-whisper on CPU via CTranslate2, so the install is seconds rather than +gigabytes and the run is about three and a half minutes. Most other engines pull torch. + +The picker shows what each engine cost on previous runs, and `?` where it has never +been measured. + +## Status + +**Validated end to end in the Docker sandbox**, from an empty workspace: clone, +dependency install, weight download, 84-phrase sweep, Whisper scoring, page build and +serving. A piper run finishes + + piper 82/84 phrases WER 5.3% RTF 0.08x + +against 4% and 0.04x for the published Apple M2 figures — close, and real-time factor +is wall-clock so a throttled container is expected to read higher. + +The picker was driven under a real PTY to confirm it renders and takes input the way a +terminal block presents it, and the quit-without-running path was tested separately. +The report reaches the browser block as a table of numbers with its encoding declared; +nothing is playable, by design. + +## Seeding + + benchmarks/bin/seed-benchmark-templates talkito-tts + +A cloud control plane needs a direct insert instead: `seedStarterTemplates` returns +early unless `SURFACE_TOKEN` is set, and only the desktop app sets it, so the code seed +never runs in production. See [`../README.md`](../README.md). diff --git a/benchmarks/talkito-tts/template.json b/benchmarks/talkito-tts/template.json new file mode 100644 index 00000000..04ea97ea --- /dev/null +++ b/benchmarks/talkito-tts/template.json @@ -0,0 +1,40 @@ +{ + "_comment": "Importable Orcabot dashboard template. Unlike the other benchmark templates this one needs no orchestrator, no agent and no API key: the terminal's boot command runs the whole benchmark and serves its report on localhost, and the browser block is pointed at that server. Seed with bin/seed-benchmark-templates.", + "name": "TTS Benchmark Runner", + "description": "Run the open-weight TTS benchmark and read its report", + "category": "coding", + "items": [ + { + "placeholderId": "runner", + "type": "terminal", + "content": "{\"name\": \"TTS benchmark\", \"agentic\": false, \"bootCommand\": \"PORT=8099\\n# talkito refuses to fetch model weights when it cannot ask a human. The picker\\n# below is interactive, but the download prompt would land in the middle of a\\n# sweep the reader has walked away from, so it is answered in advance.\\nexport TALKITO_AUTO_APPROVE_DOWNLOADS=1\\nROOT=/workspace/talkito\\nBENCH=$ROOT/benchmark\\nSERVE=/workspace/tts-report\\necho \\\"== Orcabot: talkito TTS benchmark ==\\\"\\n\\n# Serve a placeholder first, so the browser block has something at a stable URL from\\n# the first second rather than an error until the run finishes.\\nmkdir -p \\\"$SERVE\\\"\\n[ -f \\\"$SERVE/index.html\\\" ] || printf '%s' 'TTS benchmark

Waiting for a run...

Pick engines in the terminal. This page reloads every 10 seconds and the report replaces it when the run finishes.

' > \\\"$SERVE/index.html\\\"\\ncurl -sf -o /dev/null --max-time 2 \\\"http://127.0.0.1:$PORT/\\\" || ( cd \\\"$SERVE\\\" && nohup python3 -m http.server \\\"$PORT\\\" >/workspace/.tts-serve.log 2>&1 & )\\n\\n# talkito's .gitmodules uses git@github.com: URLs and the sandbox has no SSH key,\\n# so --recurse-submodules fails; both repos are cloned over HTTPS instead.\\n# The harness ships inside talkito-benchmark, so tests-repo is not needed -- but\\n# tools/paths.py still wants a talkito CHECKOUT (it looks for talkito/tts.py, not\\n# the installed package), and benchmark/ inside talkito/ is the layout it resolves.\\n[ -d \\\"$ROOT/.git\\\" ] || git clone --depth 1 https://github.com/robdmac/talkito.git \\\"$ROOT\\\"\\n[ -d \\\"$BENCH/.git\\\" ] || git clone --depth 1 https://github.com/robdmac/talkito-benchmark.git \\\"$BENCH\\\"\\n\\n# piper needs no torch at all, and faster-whisper scores on CPU through CTranslate2,\\n# so the cheapest arm installs in seconds. Heavier engines pull their own weights on\\n# first use and the picker warns what each has cost before.\\npython3 -c 'import piper' 2>/dev/null || pip3 install --break-system-packages -q piper-tts\\npython3 -c 'import faster_whisper' 2>/dev/null || pip3 install --break-system-packages -q faster-whisper\\n\\ncd \\\"$BENCH\\\" || exec bash\\ncat <<'BANNER'\\n\\n Everything is selected by default and a full sweep is around eleven hours.\\n Press n to clear, then space to pick a few. piper alone takes about three minutes.\\n\\nBANNER\\n# No --only: this is the interactive picker, and a terminal block is a real PTY, so\\n# the arrow-key menu works exactly as it does on a laptop.\\npython3 tools/benchmark.py\\nrc=$?\\n\\n# No listening clips: make_samples.py exists to embed one per engine, but audio does\\n# not reach the reader through the browser block, and it is the only reason this\\n# needed ffmpeg -- which in turn was the only reason the sandbox ran apt-get, which\\n# reaches deb.nodesource.com through a leftover apt source and trips egress approval.\\n# Dropping playback drops all three.\\n\\nif [ -f tts-benchmark.html ]; then\\n # page_template.html declares utf-8 upstream, so this normally just copies. The\\n # fallback stays because the failure it guards is silent and total: python3 -m\\n # http.server sends \\\"text/html\\\" with no charset, and a page that also declares\\n # none is read as the browser's locale default, turning every em dash into\\n # mojibake. It writes to the published copy rather than the source, so the export\\n # zip stays byte-identical to what the tools wrote, and never doubles the tag.\\n if grep -qi ']*charset' tts-benchmark.html; then\\n cp -f tts-benchmark.html \\\"$SERVE/index.html\\\"\\n else\\n { printf '%s\\\\n' ''; cat tts-benchmark.html; } > \\\"$SERVE/index.html\\\"\\n fi\\n echo \\\"report published -> http://localhost:$PORT/\\\"\\nelse\\n echo \\\"no tts-benchmark.html was produced; measurements (if any) are under $BENCH/tools/\\\"\\nfi\\necho \\\"[tts benchmark done rc=$rc]\\\"\\nexec bash\", \"subagentIds\": [], \"skillIds\": [], \"mcpToolIds\": []}", + "position": { + "x": 0, + "y": 0 + }, + "size": { + "width": 560, + "height": 560 + } + }, + { + "placeholderId": "report", + "type": "browser", + "content": "http://localhost:8099/", + "position": { + "x": 600, + "y": 0 + }, + "size": { + "width": 900, + "height": 620 + } + } + ], + "edges": [], + "viewport": { + "x": 0, + "y": 0, + "zoom": 1 + } +} diff --git a/controlplane/src/tts/handler.test.ts b/controlplane/src/tts/handler.test.ts new file mode 100644 index 00000000..d3516525 --- /dev/null +++ b/controlplane/src/tts/handler.test.ts @@ -0,0 +1,100 @@ +// Copyright 2026 Rob Macrae. All rights reserved. +// SPDX-License-Identifier: LicenseRef-Proprietary + +// Ballot selection is weighted toward the configurations ranked least, because +// uniform sampling left the counts Poisson-spread: at 43 real ballots that was 2 +// to 9 appearances around a mean of 4.7, with six configurations stranded below +// the display cutoff while others were measured twice over. +// +// The properties worth pinning are the ones a plausible rewrite would break. +// Duplicates are the sharpest: a ballot showing the same clip twice asks the +// reader to rank it against itself. The bias is the point of the function. And +// the bias must stay a bias rather than becoming an ordering, because always +// serving the k rarest as a set would leave them only ever ranked against each +// other, and Bradley-Terry cannot place a disconnected clique on the same scale +// as the rest of the table. + +import { describe, expect, it } from 'vitest'; +import { pickLeastRanked } from './handler'; + +const counts = (o: Record) => new Map(Object.entries(o)); + +describe('pickLeastRanked', () => { + it('returns exactly k items', () => { + const pool = ['a', 'b', 'c', 'd', 'e']; + for (const k of [1, 2, 3, 4]) { + expect(pickLeastRanked(pool, k, counts({})).length).toBe(k); + } + }); + + it('never repeats a configuration within one ballot', () => { + // Sampling without replacement, not k independent draws. With a heavy skew + // an implementation that drew independently would return the rare item + // several times, and the ballot would ask for it to be ranked against itself. + const pool = ['rare', 'common1', 'common2', 'common3']; + const c = counts({ rare: 0, common1: 40, common2: 40, common3: 40 }); + for (let i = 0; i < 400; i++) { + const got = pickLeastRanked(pool, 3, c); + expect(new Set(got).size).toBe(got.length); + } + }); + + it('returns the whole pool when k covers it, without dropping any', () => { + const pool = ['a', 'b', 'c']; + const got = pickLeastRanked(pool, 3, counts({ a: 9, b: 0, c: 4 })); + expect([...got].sort()).toEqual(['a', 'b', 'c']); + expect(pickLeastRanked(pool, 9, counts({})).length).toBe(3); + }); + + it('picks the least-ranked far more often than the most-ranked', () => { + const pool = ['rare', 'mid', 'common']; + const c = counts({ rare: 0, mid: 4, common: 12 }); + const seen: Record = { rare: 0, mid: 0, common: 0 }; + for (let i = 0; i < 3000; i++) { + for (const p of pickLeastRanked(pool, 1, c)) seen[p]++; + } + expect(seen.rare).toBeGreaterThan(seen.mid); + expect(seen.mid).toBeGreaterThan(seen.common); + // Comfortably wide of the 3000/3 a uniform pick would give each. + expect(seen.rare).toBeGreaterThan(1500); + }); + + it('still reaches a well-ranked configuration sometimes', () => { + // A strict least-first ordering would return `rare` every time and starve + // the rest, which disconnects the comparison graph. The weighting has to + // leave every pairing reachable. + const pool = ['rare', 'common']; + const c = counts({ rare: 0, common: 8 }); + let common = 0; + for (let i = 0; i < 3000; i++) { + if (pickLeastRanked(pool, 1, c)[0] === 'common') common++; + } + expect(common).toBeGreaterThan(0); + }); + + it('treats an unseen configuration as the rarest', () => { + // Absent from the map means never ranked, which is a zero rather than a gap. + // Defaulting it the other way would starve exactly the new arrivals a fresh + // export adds. + const pool = ['brand-new', 'established']; + const c = counts({ established: 6 }); // brand-new absent entirely + let fresh = 0; + for (let i = 0; i < 2000; i++) { + if (pickLeastRanked(pool, 1, c)[0] === 'brand-new') fresh++; + } + expect(fresh).toBeGreaterThan(1400); + }); + + it('is near-uniform once counts are high and close together', () => { + // 1/(count+1) self-attenuates: the bias matters when 2 against 9 decides + // visibility, and should fade when everything is well measured. + const pool = ['a', 'b']; + const c = counts({ a: 60, b: 64 }); + let a = 0; + for (let i = 0; i < 4000; i++) { + if (pickLeastRanked(pool, 1, c)[0] === 'a') a++; + } + expect(a).toBeGreaterThan(1900); + expect(a).toBeLessThan(2600); + }); +}); diff --git a/controlplane/src/tts/handler.ts b/controlplane/src/tts/handler.ts index f00ed8b2..42bde6d5 100644 --- a/controlplane/src/tts/handler.ts +++ b/controlplane/src/tts/handler.ts @@ -1,10 +1,10 @@ // Copyright 2026 Rob Macrae. All rights reserved. // SPDX-License-Identifier: LicenseRef-Proprietary -// REVISION: tts-ballots-v2-full-config-set +// REVISION: tts-ballots-v3-balanced-selection import type { Env } from '../types'; -const MODULE_REVISION = 'tts-ballots-v2-full-config-set'; +const MODULE_REVISION = 'tts-ballots-v3-balanced-selection'; console.log(`[tts] REVISION: ${MODULE_REVISION} loaded at ${new Date().toISOString()}`); /** The run these ballots score. Bumped when a new TTS run replaces the samples. */ @@ -21,14 +21,16 @@ const RUN = '2026-08'; * reader can complete and the server is then obliged to reject. */ const CONFIGS = [ - 'supertonic', 'piper', 'chatterbox-turbo', 'kittentts-micro', - 'kittentts-nano-int8', 'qwen3-tts-vd', 'kittentts-mini', 'dots-tts', - 'qwen3-tts', 'kittentts-nano', 'chatterbox', 'f5-tts', 'kokoro', 'xtts', - 'orpheus-q4', 'nt-2e-q8-metal', 'nt-2e-fp32-mps', 'chatterbox-q8', - 'vibevoice', 'nt-2e-q4-metal', 'cosyvoice3-rl', 'chatterbox-q4', - 'styletts2', 'melotts', 'omnivoice', 'fastpitch', 'cosyvoice3', - 'bananamind-tts', 'tada-3b', 'zonos', 'csm', 'tada-1b', 'speecht5', - 'mms-tts', 'vibevoice-1.5b', 'bark', + 'supertonic', 'piper', 'indextts', 'kittentts-micro', 'chatterbox-turbo', + 'fish-s2-pro', 'kittentts-nano-int8', 'qwen3-tts-vd', 'kittentts-mini', + 'dots-tts', 'qwen3-tts', 'pocket-tts', 'nt-2e-q8-metal', 'kittentts-nano', + 'f5-tts', 'magpie-q6_k', 'magpie-q5_k', 'magpie-f16', 'dots-tts-mf', + 'dots-soar-acpp', 'kokoro', 'magpie-f32', 'magpie-q4_k', 'magpie-q8_0', + 'chatterbox', 'orpheus-q4', 'orpheus-q8', 'voxcpm2-tts', 'xtts', + 'nt-2e-fp32-mps', 'kani-tts', 'chatterbox-q8', 'vibevoice', + 'nt-2e-q4-metal', 'cosyvoice3-rl', 'chatterbox-q4', 'styletts2', 'melotts', + 'omnivoice', 'cosyvoice3', 'bananamind-tts', 'tada-3b', 'zonos', 'csm', + 'tada-1b', 'speecht5', 'mms-tts', 'vibevoice-1.5b', 'bark', ] as const; /** @@ -79,6 +81,57 @@ function shuffle(a: T[]): T[] { return out; } +/** How often each configuration has already been ranked, across counted ballots. + * + * Aggregated in SQL rather than by reading every ballot into the worker, because + * unlike the score computation this runs on every ballot issued, not only when + * the table is rendered. + * + * Counted ballots only, matching what the scores are built from: an issued and + * abandoned ballot has told us nothing about the clips it offered, so treating + * it as coverage would starve exactly the configurations that were offered and + * never ranked. */ +async function appearanceCounts(env: Env): Promise> { + const { results } = await env.DB.prepare( + `SELECT je.value AS config, COUNT(*) AS n + FROM tts_ballots, json_each(tts_ballots.ranking_json) je + WHERE run = ? AND status = 'counted' + GROUP BY je.value` + ).bind(RUN).all<{ config: string; n: number }>(); + return new Map((results ?? []).map((r) => [String(r.config), Number(r.n)])); +} + +/** Weighted sample without replacement, favouring the configurations ranked least. + * + * Efraimidis-Spirakis: give each item the key u^(1/w) for u uniform on (0,1) and + * take the k largest. One pass, no rejection loop, and a correct weighted sample + * without replacement - which matters because a ballot must never offer the same + * clip twice. + * + * The weight is 1/(count+1), not a strict least-ranked-first ordering, and the + * randomness is the point rather than a compromise. Always serving the k rarest + * together would make the same under-sampled configurations appear as a set every + * time, and Bradley-Terry needs the comparison graph connected: a clique that is + * only ever ranked against itself cannot be placed on the same scale as the rest, + * however many ballots it collects. Weighting pulls the counts level while every + * pairing stays reachable. + * + * 1/(count+1) also self-attenuates. Early on, when 2 against 9 is the difference + * between invisible and well-measured, it biases by more than 3x; once everything + * sits in the twenties the ratio is near 1 and selection is effectively uniform + * again, which is what you want when the deficits no longer matter. */ +export function pickLeastRanked(pool: string[], k: number, counts: Map): string[] { + if (k >= pool.length) return shuffle(pool); + return pool + .map((config) => ({ + config, + key: Math.random() ** ((counts.get(config) ?? 0) + 1), + })) + .sort((a, b) => b.key - a.key) + .slice(0, k) + .map((x) => x.config); +} + /** Coarse voter fingerprint for rate limiting and dedup. Hashed, never stored raw. */ async function voterHash(request: Request): Promise { const ip = request.headers.get('cf-connecting-ip') ?? request.headers.get('x-forwarded-for') ?? ''; @@ -123,17 +176,24 @@ export async function issueBallot(env: Env, request: Request): Promise return Response.json({ exhausted: 'clips', submitted, max: MAX_BALLOTS_PER_VOTER }); } + // Selection is biased toward whatever has been ranked least. Uniform sampling + // leaves the counts Poisson-spread - at 43 ballots that was 2 to 9 appearances + // against a mean of 4.7, with six configurations stranded under the cutoff and + // therefore invisible, while others were measured twice over. + const counts = await appearanceCounts(env); + // The anchor has to be one they have not heard either, or the attention check - // is just a memory test. + // is just a memory test. Which of the two, though, follows the same rule as + // everything else. const freshAnchors = ANCHORS.filter((a) => fresh.includes(a)); const anchor = freshAnchors.length && Math.random() < ANCHOR_RATE - ? freshAnchors[Math.floor(Math.random() * freshAnchors.length)] + ? pickLeastRanked(freshAnchors, 1, counts)[0] : null; const pool = fresh.filter((c) => c !== anchor); const want = Math.min(ITEMS_PER_BALLOT, fresh.length) - (anchor ? 1 : 0); - const picked = shuffle(pool).slice(0, want); + const picked = pickLeastRanked(pool, want, counts); const items = shuffle(anchor ? [...picked, anchor] : picked); const id = crypto.randomUUID(); diff --git a/frontend/content/benchmarks/open-weight-tts.md b/frontend/content/benchmarks/open-weight-tts.md index a95a19a1..e39ec5dd 100644 --- a/frontend/content/benchmarks/open-weight-tts.md +++ b/frontend/content/benchmarks/open-weight-tts.md @@ -1,8 +1,9 @@ --- title: Testing open-weight Text To Speech models -date: 2026-08-17 +date: 2026-08-22 description: Every open-weight TTS engine that can run on a consumer laptop, tested for WER using Whisper. author: Rob Macrae +ogImage: /benchmarks/og-open-weight-tts.png toc: false --- @@ -10,6 +11,13 @@ We tested all the open weight TTS models against the same corpus, using Whisper Every row can be played, so you can hear what a word error rate actually sounds like. +**You can run this yourself.** The harness, the corpus and the scoring are open, and +[talkito-benchmark](https://github.com/robdmac/talkito-benchmark) regenerates every number on this +page from nothing. A single engine takes minutes +(`python3 tools/benchmark.py --only piper`); the full sweep is around eleven hours, most of it +spent downloading weights on the first run. In Orcabot, the **TTS Benchmark Runner** template +does the whole thing in one terminal and shows the report in the built-in browser. + ```chart tts-vendors ``` @@ -37,13 +45,13 @@ face value. - **RTF** is compute seconds per second of audio, the conventional measure. It **flatters any engine that emits excess silence**, because padding inflates the denominator. Prefer - **x̄ synth**, which is compute per phrase: every engine speaks the same corpus, so it + **x̄ _t_**, the mean compute per phrase: every engine speaks the same corpus, so it compares directly and cannot be gamed by padding. - **WER** is the round-trip word error rate, scored by Whisper `base.en`. The stronger - `medium.en` was run too, but it did not finish for twenty-one of the thirty-six + `medium.en` was run too, but it did not finish for thirty-four of the forty-nine configurations, so `base.en` is the one recogniser that covers the whole table. Being the weaker model it hallucinates words onto trailing silence, which understates good engines - more than bad ones — so the true spread between engines is **wider** than this column + more than bad ones, so the true spread between engines is **wider** than this column shows, not narrower. - **PESQ** is a no-reference perceptual quality estimate from torchaudio's SQUIM, scored on the sample in the first column. It is a second axis word error cannot see, because word @@ -64,15 +72,27 @@ face value. **Machine.** Apple M2, macOS. One machine, one English corpus, one recogniser family. -**Nothing is filtered out.** Every configuration compared is listed, including the twelve +**Nothing is filtered out.** Every configuration compared is listed, including the seventeen that cannot keep up with their own speech, because "how far off is it" is a real question and a table that quietly omits the answer cannot be checked. The **Real time only** control -above the table hides anything above 2x for readers who only care about what can be spoken -live, and sorting by RTF or compute per phrase draws a line at the cutoff so the two groups -are visible at once. - -The line falls at 2x exactly. Nothing sits awkwardly against it: the slowest engine that -keeps up is OmniVoice at 1.51x, and the fastest that does not is Chatterbox Q4 at 2.03x. +above the table hides anything above 2x for readers who only care about what can be +spoken live, and sorting by either speed column draws a line at the cutoff so the two groups +are visible at once. The line is cut on whichever column you sorted by, because the two do +not agree: real-time factor is compute per second of *audio* and engines differ in how long +they speak, so cutting one while sorting by the other leaves the groups interleaved and the +line with nowhere honest to go. + +The line falls at 2x real time, or 7 seconds of compute a phrase. Nothing sits awkwardly +against the real-time line: 1.99x against 2.21x. The compute line is tighter — Kani at 6.86s +against Chatterbox Q8 at 7.03s — and 0.17s is well inside what three passes can resolve, so +read those two as sharing a position rather than falling either side of a line. Each number +is picked so the two measures classify the table the same way: they now agree about every +configuration but one, Magpie's full-precision build, which is quick per phrase at 3.79s and +slow per second of audio at 2.31x. It briefly sat at 2.05x, because Magpie's quantisations +landed at 2.01x and 2.02x and splitting a ladder this table reports as indistinguishable is +not a distinction the data can carry. Measuring every +autoregressive configuration over at least three passes moved them to between 1.53x and +1.81x, so the round number is honest again. **Corpus.** 84 phrases, spoken identically by every configuration, spanning core, edge and long-form categories. @@ -85,7 +105,14 @@ scored on **trimmed** audio: scoring untrimmed rewards models that pad with sile to 0.78, because silence pulls the estimate toward the ceiling and only mediocre audio has room to rise. -**Passes.** Autoregressive engines are averaged over two passes, feed-forward over one. +**Passes.** Autoregressive engines are averaged over at least three passes, feed-forward +engines over one. Sampling is what makes the difference: a deterministic forward pass gives +the same audio every run, so a second one measures nothing, while an autoregressive model +draws tokens and lands somewhere new each time. One run of those was not enough to separate +six Magpie quantisations of a single checkpoint, and it was not enough to place them against +the real-time line either; a few configurations that still moved between runs were given +four passes or five. Chatterbox Turbo is the plainest illustration: one pass put it +joint-best on word error at 4%, and three put it at 5%, behind three other engines. Differences under roughly two points are not resolvable at this sample size. ### Limitations diff --git a/frontend/package.json b/frontend/package.json index 2ebbc0b0..d179ced7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -9,6 +9,7 @@ "start": "next start", "lint": "eslint src/", "check-catalogs": "node scripts/check-catalogs.mjs", + "build-og-tts": "node scripts/build-og-tts.mjs", "test": "vitest", "test:e2e": "playwright test", "test:e2e:ui": "playwright test --ui", diff --git a/frontend/public/benchmarks/og-open-weight-tts.png b/frontend/public/benchmarks/og-open-weight-tts.png new file mode 100644 index 00000000..dfcbbf4d Binary files /dev/null and b/frontend/public/benchmarks/og-open-weight-tts.png differ diff --git a/frontend/public/benchmarks/tts/dots-soar-acpp.mp3 b/frontend/public/benchmarks/tts/dots-soar-acpp.mp3 new file mode 100644 index 00000000..30b86f39 Binary files /dev/null and b/frontend/public/benchmarks/tts/dots-soar-acpp.mp3 differ diff --git a/frontend/public/benchmarks/tts/dots-tts-mf.mp3 b/frontend/public/benchmarks/tts/dots-tts-mf.mp3 new file mode 100644 index 00000000..8f4a55b7 Binary files /dev/null and b/frontend/public/benchmarks/tts/dots-tts-mf.mp3 differ diff --git a/frontend/public/benchmarks/tts/fastpitch.mp3 b/frontend/public/benchmarks/tts/fastpitch.mp3 deleted file mode 100644 index 10d0ab4b..00000000 Binary files a/frontend/public/benchmarks/tts/fastpitch.mp3 and /dev/null differ diff --git a/frontend/public/benchmarks/tts/fish-s2-pro.mp3 b/frontend/public/benchmarks/tts/fish-s2-pro.mp3 new file mode 100644 index 00000000..f0bd3993 Binary files /dev/null and b/frontend/public/benchmarks/tts/fish-s2-pro.mp3 differ diff --git a/frontend/public/benchmarks/tts/indextts.mp3 b/frontend/public/benchmarks/tts/indextts.mp3 new file mode 100644 index 00000000..92c5c467 Binary files /dev/null and b/frontend/public/benchmarks/tts/indextts.mp3 differ diff --git a/frontend/public/benchmarks/tts/kani-tts.mp3 b/frontend/public/benchmarks/tts/kani-tts.mp3 new file mode 100644 index 00000000..5a59babc Binary files /dev/null and b/frontend/public/benchmarks/tts/kani-tts.mp3 differ diff --git a/frontend/public/benchmarks/tts/magpie-f16.mp3 b/frontend/public/benchmarks/tts/magpie-f16.mp3 new file mode 100644 index 00000000..d387d270 Binary files /dev/null and b/frontend/public/benchmarks/tts/magpie-f16.mp3 differ diff --git a/frontend/public/benchmarks/tts/magpie-f32.mp3 b/frontend/public/benchmarks/tts/magpie-f32.mp3 new file mode 100644 index 00000000..c1e7ef30 Binary files /dev/null and b/frontend/public/benchmarks/tts/magpie-f32.mp3 differ diff --git a/frontend/public/benchmarks/tts/magpie-q4_k.mp3 b/frontend/public/benchmarks/tts/magpie-q4_k.mp3 new file mode 100644 index 00000000..ddacb64b Binary files /dev/null and b/frontend/public/benchmarks/tts/magpie-q4_k.mp3 differ diff --git a/frontend/public/benchmarks/tts/magpie-q5_k.mp3 b/frontend/public/benchmarks/tts/magpie-q5_k.mp3 new file mode 100644 index 00000000..5e441b47 Binary files /dev/null and b/frontend/public/benchmarks/tts/magpie-q5_k.mp3 differ diff --git a/frontend/public/benchmarks/tts/magpie-q6_k.mp3 b/frontend/public/benchmarks/tts/magpie-q6_k.mp3 new file mode 100644 index 00000000..36ee2a7a Binary files /dev/null and b/frontend/public/benchmarks/tts/magpie-q6_k.mp3 differ diff --git a/frontend/public/benchmarks/tts/magpie-q8_0.mp3 b/frontend/public/benchmarks/tts/magpie-q8_0.mp3 new file mode 100644 index 00000000..2ad79b34 Binary files /dev/null and b/frontend/public/benchmarks/tts/magpie-q8_0.mp3 differ diff --git a/frontend/public/benchmarks/tts/orpheus-q8.mp3 b/frontend/public/benchmarks/tts/orpheus-q8.mp3 new file mode 100644 index 00000000..d5d00ed5 Binary files /dev/null and b/frontend/public/benchmarks/tts/orpheus-q8.mp3 differ diff --git a/frontend/public/benchmarks/tts/pocket-tts.mp3 b/frontend/public/benchmarks/tts/pocket-tts.mp3 new file mode 100644 index 00000000..98369e16 Binary files /dev/null and b/frontend/public/benchmarks/tts/pocket-tts.mp3 differ diff --git a/frontend/public/benchmarks/tts/voxcpm2-tts.mp3 b/frontend/public/benchmarks/tts/voxcpm2-tts.mp3 new file mode 100644 index 00000000..ed8946d6 Binary files /dev/null and b/frontend/public/benchmarks/tts/voxcpm2-tts.mp3 differ diff --git a/frontend/public/icons/tts/OpenBMB.png b/frontend/public/icons/tts/OpenBMB.png new file mode 100644 index 00000000..b7441ee3 Binary files /dev/null and b/frontend/public/icons/tts/OpenBMB.png differ diff --git a/frontend/public/icons/tts/fishaudio.png b/frontend/public/icons/tts/fishaudio.png new file mode 100644 index 00000000..5c4fa09b Binary files /dev/null and b/frontend/public/icons/tts/fishaudio.png differ diff --git a/frontend/public/icons/tts/index-tts.png b/frontend/public/icons/tts/index-tts.png new file mode 100644 index 00000000..f29957c5 Binary files /dev/null and b/frontend/public/icons/tts/index-tts.png differ diff --git a/frontend/public/icons/tts/kyutai-labs.png b/frontend/public/icons/tts/kyutai-labs.png new file mode 100644 index 00000000..e5b61561 Binary files /dev/null and b/frontend/public/icons/tts/kyutai-labs.png differ diff --git a/frontend/public/icons/tts/nineninesix-ai.png b/frontend/public/icons/tts/nineninesix-ai.png new file mode 100644 index 00000000..9ff7fed5 Binary files /dev/null and b/frontend/public/icons/tts/nineninesix-ai.png differ diff --git a/frontend/public/icons/tts/yl4579.png b/frontend/public/icons/tts/yl4579.png new file mode 100644 index 00000000..bd34086a Binary files /dev/null and b/frontend/public/icons/tts/yl4579.png differ diff --git a/frontend/scripts/build-og-tts.mjs b/frontend/scripts/build-og-tts.mjs new file mode 100644 index 00000000..f22100ac --- /dev/null +++ b/frontend/scripts/build-og-tts.mjs @@ -0,0 +1,250 @@ +// Copyright 2026 Rob Macrae. All rights reserved. +// SPDX-License-Identifier: LicenseRef-Proprietary + +// Builds the social preview card for the open-weight TTS benchmark: +// public/benchmarks/og-open-weight-tts.png, 1200x630. +// +// A script rather than a hand-made PNG because the y-axis is live. Reader +// preference comes from blind rankings that keep arriving, so the cloud moves +// and the card goes stale on its own; the two existing cards were drawn once and +// cannot be re-cut without redrawing them. +// +// The x-axis is LOGARITHMIC, which the on-page chart's is not, and that is the +// one deliberate difference between them. The page fixes x at 6.8s because that +// covers the engines that keep up with real time. Ten of the thirty rated +// engines are slower than that -- Bark, Zonos and CSM by a wide margin, up to +// 51s a phrase -- so on a 6.8s axis they are drawn off-canvas rather than +// clipped or dropped. A card is a single glance with no filter controls and no +// hover, so it has to show every point it counts: log x fits 0.12s and 51s in +// one frame, and the flatness of the cloud across two and a half decades is the +// finding. +// +// Usage: +// node scripts/build-og-tts.mjs # live scores from the API +// node scripts/build-og-tts.mjs --scores f.json # a saved /tts/scores response +// node scripts/build-og-tts.mjs --html-only # write the HTML, skip the shot + +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.join(HERE, ".."); +const RUN = path.join(ROOT, "src/data/benchmarks/open-weight-tts/2026-08.json"); +const OUT_PNG = path.join(ROOT, "public/benchmarks/og-open-weight-tts.png"); +const OUT_HTML = path.join(HERE, ".og-tts.html"); +const SCORES_URL = process.env.TTS_SCORES_URL || "https://api.orcabot.com/tts/scores"; + +const argv = process.argv.slice(2); +const arg = (name) => { + const i = argv.indexOf(name); + return i === -1 ? null : argv[i + 1]; +}; + +// Card geometry and palette, both sampled from the existing cards so a new one +// sits beside them rather than beneath them. +const CARD = { w: 1200, h: 630 }; +const BG = "#0b1a2e"; +const SURFACE = "#101e33"; +const GRID = "#1e3354"; +const AXIS = "#2a4570"; +const INK = { primary: "#e8edf5", secondary: "#c3cee0", muted: "#94a3c0" }; + +/** Fixed order, never cycled -- the same three the page uses. */ +const CLASSES = [ + { id: "det-ff", label: "Deterministic feed-forward", color: "#3987e5" }, + { id: "stoch-ff", label: "Stochastic feed-forward", color: "#5ecfb0" }, + { id: "ar-lm", label: "Autoregressive LM", color: "#d95926" }, +]; + +const run = JSON.parse(fs.readFileSync(RUN, "utf8")); +const INDEX = Object.fromEntries(run.columns.map((c, i) => [c, i])); +function col(name) { + const i = INDEX[name]; + if (i === undefined) { + throw new Error(`no "${name}" column in run ${run.run}: have ${run.columns.join(", ")}`); + } + return i; +} + +async function loadRatings() { + const file = arg("--scores"); + const body = file + ? JSON.parse(fs.readFileSync(file, "utf8")) + : await fetch(SCORES_URL).then((r) => { + if (!r.ok) throw new Error(`${SCORES_URL} -> HTTP ${r.status}`); + return r.json(); + }); + const raw = body.scores ?? {}; + const out = new Map(); + const entries = Array.isArray(raw) ? raw.map((s) => [s.config, s]) : Object.entries(raw); + for (const [config, v] of entries) { + const rating = typeof v === "number" ? v : v?.rating; + if (rating != null) out.set(config, Number(rating)); + } + return { ratings: out, minBallots: body.minBallots ?? null }; +} + +const { ratings, minBallots } = await loadRatings(); + +const points = run.rows + .map((r) => ({ + config: r.config, + name: r.display, + cls: r.cells[col("Class")].sort, + synth: Number(r.cells[col("x̄ t")].sort), + disk: Number(r.cells[col("Disk")].sort), + rating: ratings.get(r.config), + })) + .filter((p) => p.rating != null && Number.isFinite(p.synth) && p.synth > 0) + .map((p) => ({ ...p, color: CLASSES.find((c) => c.id === p.cls)?.color ?? INK.muted })); + +if (points.length < 6) { + console.error(`only ${points.length} rated engines; the page hides the chart below 6, so no card`); + process.exit(1); +} + +// Plot box inside the panel. The header (title, subtitle, legend) takes the top +// 128px and the footer the bottom 32, so the SVG has to fit in what is left -- +// getting this wrong pushes the x-axis title off the card and prints the footer +// on top of the plot. +const SVG_TOP = 128; +const SVG_H = CARD.h - SVG_TOP - 32; +const PANEL_H = SVG_H - 42; +const P = { x: 74, y: 26, w: CARD.w - 74 - 52, h: PANEL_H - 100 }; +// X_MAX sits well beyond the slowest engine (51s) so its marker has room to draw +// inside the plot rather than half over the border. +const X_MIN = 0.09, X_MAX = 80; +const lx = (v) => Math.log10(v); +const sx = (v) => P.x + ((lx(v) - lx(X_MIN)) / (lx(X_MAX) - lx(X_MIN))) * P.w; +// Ratings run 0-100 but the axis goes to 105, so the best-rated engine sits a +// little below the frame instead of against it. Without that headroom its label +// has nowhere to go but down, into the densest part of the cloud. +const Y_MAX = 105; +const sy = (v) => P.y + P.h - (v / Y_MAX) * P.h; + +const DISK = { + min: Math.min(...points.map((p) => p.disk)), + max: Math.max(...points.map((p) => p.disk)), +}; +/** Area-proportional, floored so the smallest engine is still visible. */ +const radius = (d) => 5 + 7 * Math.sqrt((d - DISK.min) / (DISK.max - DISK.min || 1)); + +const X_TICKS = [0.1, 0.3, 1, 3, 10, 30]; +const Y_TICKS = [0, 25, 50, 75, 100]; +const fmtX = (t) => (t < 1 ? `${t}s` : `${t}s`); + +// Pearson r, quoted in the subtitle. Computed rather than typed, so a re-cut +// after more ballots cannot leave a stale number sitting next to fresh points. +const xs = points.map((p) => Math.log10(p.synth)); +const ys = points.map((p) => p.rating); +const mean = (a) => a.reduce((s, v) => s + v, 0) / a.length; +const mx = mean(xs), my = mean(ys); +const cov = xs.reduce((s, x, i) => s + (x - mx) * (ys[i] - my), 0); +const sd = (a, m) => Math.sqrt(a.reduce((s, v) => s + (v - m) ** 2, 0)); +const r = cov / (sd(xs, mx) * sd(ys, my)); + +// Label only the corners of the story: best, worst, cheapest, slowest. +const byRating = [...points].sort((a, b) => b.rating - a.rating); +const labelled = new Set( + [ + byRating[0], + byRating[byRating.length - 1], + [...points].sort((a, b) => a.synth - b.synth)[0], + [...points].sort((a, b) => b.synth - a.synth)[0], + ].map((p) => p.config) +); + +const sub = points.length; +const fast = points.filter((p) => p.synth <= 1); +// "1-94" reads as a subtraction at a glance, so the range is spelled out. +const fastSpread = `${Math.round(Math.min(...fast.map((p) => p.rating)))} to ${Math.round( + Math.max(...fast.map((p) => p.rating)) +)}`; + +const svg = ` + + + ${Y_TICKS.map((t) => ``).join("")} + ${X_TICKS.map((t) => ``).join("")} + + + ${Y_TICKS.map((t) => `${t}`).join("")} + ${X_TICKS.map((t) => `${fmtX(t)}`).join("")} + Compute per phrase, log scale (seconds, Apple M2) + Reader preference + ${/* Markers first, labels after, in two passes. In one pass a marker drawn + later paints over an earlier point's label, which is how "NeuTTS-2E Q8" + rendered as "euTTS-2E Q8" with a circle sitting on the missing letters. */ ""} + ${points + .map((p) => { + const rr = radius(p.disk); + return ` + + + `; + }) + .join("")} + ${points + .filter((p) => labelled.has(p.config)) + .map((p) => { + const rr = radius(p.disk); + // Always above the marker: the y headroom guarantees room, so there is no + // ceiling case to special-case. Only the right border flips the anchor. + const right = sx(p.synth) > P.x + P.w * 0.72; + return `${p.name}`; + }) + .join("")} +`; + +const html = ` + + +

Open-weight TTS: reader preference vs compute per phrase

+

${sub} engines ranked blind. Compute barely predicts preference (r = ${r.toFixed(2)}); the sub-second engines alone span ${fastSpread}.

+
+ ${CLASSES.map((c) => `${c.label}`).join("")} +
+
${svg}
+
orcabot.com/benchmarks
+`; + +fs.writeFileSync(OUT_HTML, html); +console.log(`[og-tts] ${sub} engines, r=${r.toFixed(3)}, minBallots=${minBallots}`); +console.log(`[og-tts] widest x: ${Math.min(...points.map((p) => p.synth))}s .. ${Math.max(...points.map((p) => p.synth))}s`); + +if (argv.includes("--html-only")) { + console.log(`[og-tts] wrote ${OUT_HTML}`); + process.exit(0); +} + +const { chromium } = await import("playwright"); +const browser = await chromium.launch(); +// 1x, not retina: generateMetadata declares og:image:width=1200/height=630, and +// the two existing cards are exactly that. A 2x file would contradict the +// declaration it ships with. +const page = await browser.newPage({ + viewport: { width: CARD.w, height: CARD.h }, + deviceScaleFactor: 1, +}); +await page.goto("file://" + OUT_HTML, { waitUntil: "networkidle" }); +await page.screenshot({ path: OUT_PNG }); +await browser.close(); +fs.unlinkSync(OUT_HTML); + +const { size } = fs.statSync(OUT_PNG); +console.log(`[og-tts] wrote ${path.relative(ROOT, OUT_PNG)} (${Math.round(size / 1024)} KB)`); diff --git a/frontend/scripts/import-tts-export.mjs b/frontend/scripts/import-tts-export.mjs index 8b9f726d..33b10592 100644 --- a/frontend/scripts/import-tts-export.mjs +++ b/frontend/scripts/import-tts-export.mjs @@ -45,11 +45,48 @@ const OUT_AUDIO = "public/benchmarks/tts"; * the methodology instead. */ const DROP_COLUMNS = new Set(["Frame rate", "Passed", "Libs", "Lead-in", "WER med", "Avg audio"]); +/** Columns the page re-colours itself, keyed by the export's own header name, and + * the displayed value at or below which a cell is green. + * + * The export tones word error on the unrounded rate while the column prints whole + * percents, so its threshold lands inside a displayed band: three rows all read + * "8%" and only the first was green, because 7.70 was under the cutoff and 8.03 + * and 8.23 were over it. Two cells showing the same number in different colours + * reads as a rendering fault, not as a distinction, and no reader can recover the + * third decimal that justifies it. + * + * So the threshold is applied to what is actually on the page. Identical text is + * then guaranteed identical colour, whatever the export's own rounding does next. + * + * warn and bad are left exactly where the export put them. They mark the 24%-and- + * worse rows, which are not in question, and re-deriving them here would mean + * owning a second threshold to keep in step with the first. */ +const GREEN_AT_OR_BELOW = { "WER base": 6 }; + +/** The tone a cell should carry: the export's, unless this column is re-coloured. */ +function retone(header, cell) { + const limit = GREEN_AT_OR_BELOW[header]; + if (limit === undefined) return cell.tone; + if (cell.tone === "warn" || cell.tone === "bad") return cell.tone; + const shown = Number(String(cell.v).replace("%", "").trim()); + // A cell that is not a number at all - "·" for unmeasured - keeps whatever it + // had rather than being silently promoted to green by a NaN comparison. + if (!Number.isFinite(shown)) return cell.tone; + return shown <= limit ? "good" : ""; +} + /** Configurations the comparison does not show. The NeuTTS-2E CPU builds are * the same weights as the rows that remain, run on a slower path: they tripled * the size of the NeuTTS band while saying nothing about the model, only about * the hardware it was pointed at. Their measurements stay in the raw export. */ -const DROP_ROWS = new Set(["nt-2e-fp32-cpu", "nt-2e-q4-cpu", "nt-2e-q8-cpu"]); +const DROP_ROWS = new Set([ + "nt-2e-fp32-cpu", "nt-2e-q4-cpu", "nt-2e-q8-cpu", + // NVIDIA's own Magpie supersedes it: same vendor, an autoregressive model with a + // published quantisation ladder against a 2023 FastPitch checkpoint that only + // reached the table by speaking an AI disclosure the harness had to strip. Kept + // in the raw export, where the measurement still stands. + "fastpitch", +]); /** Shorter headers where the export's are longer than they need to be. * Only one word error rate is shown, so it does not need qualifying: base.en @@ -61,9 +98,12 @@ const RENAME_COLUMNS = { // "Peak" is the only kind of RSS reported here, so it says nothing the column // does not already imply. "Peak RSS": "RSS", - // x-bar: it is a mean, and "Avg" spent five characters saying so in a column - // whose values are four wide. - "Avg synth": "x\u0304 synth", + // x-bar t: a mean of a time. "Avg synth" spent ten characters on a column whose + // values are four wide, and "synth" named the operation rather than the + // quantity - what is being reported is seconds, and the header now says so in + // the notation the rest of the page already uses for a mean. The table renders + // the t italic, as a variable. + "Avg synth": "x\u0304 t", }; /** Where each project states its own licence: its LICENSE file, or the model @@ -116,7 +156,28 @@ const LICENCE_PROOF_URLS = { // weights carry, which is what this column states. Supertonic is the standing // reminder that the two can differ. "qwen3-tts-vd": "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign", + // Both quantizations are the same weights off the same card, so they cite it. "orpheus-q4": "https://huggingface.co/canopylabs/orpheus-3b-0.1-ft", + "orpheus-q8": "https://huggingface.co/canopylabs/orpheus-3b-0.1-ft", + "indextts": "https://huggingface.co/IndexTeam/Index-TTS", + "kani-tts": "https://huggingface.co/nineninesix/kani-tts-370m", + "fish-s2-pro": "https://huggingface.co/fishaudio/s2-pro", + "dots-tts-mf": "https://github.com/studio-dots-ai/dots.tts/blob/main/LICENSE", + "dots-soar-acpp": "https://github.com/studio-dots-ai/dots.tts/blob/main/LICENSE", + // Six quantisations of one checkpoint, so one card evidences all of them. It + // declares nvidia-open-model-license, which is what the Licence column prints. + "magpie-f32": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m", + "magpie-f16": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m", + "magpie-q8_0": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m", + "magpie-q6_k": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m", + "magpie-q5_k": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m", + "magpie-q4_k": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m", + // Kyutai splits these: the code is MIT on GitHub, the weights CC-BY-4.0 on the + // card. This column states the weights licence, so the card is the proof. + "pocket-tts": "https://huggingface.co/kyutai/pocket-tts", + // The bare openbmb/VoxCPM path 401s; the 0.5B card is the one that exists, and + // it matches the 0.5B this row measures. + "voxcpm2-tts": "https://huggingface.co/openbmb/VoxCPM-0.5B", "speecht5": "https://huggingface.co/microsoft/speecht5_tts/blob/main/README.md", "styletts2": "https://github.com/yl4579/StyleTTS2/blob/main/LICENSE", "dots-tts": "https://github.com/studio-dots-ai/dots.tts/blob/main/LICENSE", @@ -165,6 +226,20 @@ const MODEL_URLS = { // family repository the base model uses. "qwen3-tts-vd": "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign", "orpheus-q4": "https://github.com/canopyai/Orpheus-TTS", + "orpheus-q8": "https://github.com/canopyai/Orpheus-TTS", + "indextts": "https://github.com/index-tts/index-tts", + "kani-tts": "https://github.com/nineninesix-ai/kani-tts", + "fish-s2-pro": "https://huggingface.co/fishaudio/s2-pro", + "dots-tts-mf": "https://github.com/studio-dots-ai/dots.tts", + "dots-soar-acpp": "https://github.com/studio-dots-ai/dots.tts", + "magpie-f32": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m", + "magpie-f16": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m", + "magpie-q8_0": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m", + "magpie-q6_k": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m", + "magpie-q5_k": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m", + "magpie-q4_k": "https://huggingface.co/nvidia/magpie_tts_multilingual_357m", + "pocket-tts": "https://github.com/kyutai-labs/pocket-tts", + "voxcpm2-tts": "https://github.com/OpenBMB/VoxCPM", "omnivoice": "https://github.com/k2-fsa/OmniVoice", "bananamind-tts": "https://huggingface.co/Banaxi-Tech/BananaMind-TTS-V2", "dots-tts": "https://github.com/studio-dots-ai/dots.tts", @@ -228,10 +303,14 @@ const CELL_OVERRIDES = { const REWRITE = { // "stoch-ff" was the widest value in a column of six-character codes. Class: (v) => (v === "stoch-ff" ? "st-ff" : v), - // Replacement is a function, not a string: "$5m" as a string literal is a - // capture-group reference to any future group 5 in that pattern, and would - // silently start substituting instead of printing. - Licence: (v) => v.replace(/under \$5M/i, () => "<$5M"), + // Any "under $NM", not just the NeuTTS $5M this started as: kani-tts arrived + // under LFM1.0 with a $10M ceiling and would otherwise have rendered "under + // $10M" beside NeuTTS's "<$5M", two spellings of one idea in one column. + // + // Replacement is a function, not a string: "<$" followed by a digit reads as a + // capture-group reference in a string literal, so "$5M" would silently + // substitute group 5 instead of printing a dollar sign. + Licence: (v) => v.replace(/under \$(\d+)M/gi, (_, m) => `<$${m}M`), }; /** Each vendor's own capitalisation. The export uses lowercase run ids; showing @@ -267,6 +346,20 @@ const DISPLAY = { "qwen3-tts": "Qwen3-TTS", "qwen3-tts-vd": "Qwen3-TTS VoiceDesign", "orpheus-q4": "Orpheus 3B Q4", + "orpheus-q8": "Orpheus 3B Q8", + "indextts": "IndexTTS", + "kani-tts": "Kani TTS", + "fish-s2-pro": "Fish Audio S2 Pro", + "dots-tts-mf": "Dots-TTS MeanFlow", + "dots-soar-acpp": "Dots-TTS SOAR q8", + "magpie-f32": "Magpie TTS F32", + "magpie-f16": "Magpie TTS F16", + "magpie-q8_0": "Magpie TTS Q8_0", + "magpie-q6_k": "Magpie TTS Q6_K", + "magpie-q5_k": "Magpie TTS Q5_K", + "magpie-q4_k": "Magpie TTS Q4_K", + "pocket-tts": "Pocket TTS", + "voxcpm2-tts": "VoxCPM 0.5B", "speecht5": "SpeechT5", "styletts2": "StyleTTS2", "vibevoice": "VibeVoice", @@ -413,11 +506,12 @@ const rows = rawRows.map(([, rowAttrs, inner]) => { const withOverride = override ? { ...cell, v: override.v, sort: override.sort ?? override.v } : cell; + // After the override, so a corrected value is coloured on what it now says. + const toned = { ...withOverride, tone: retone(headers[i], withOverride) }; if (headers[i] === "Licence" && LICENCE_PROOF_URLS[config]) { - return { ...withOverride, href: LICENCE_PROOF_URLS[config] }; + return { ...toned, href: LICENCE_PROOF_URLS[config] }; } - if (override) return withOverride; - return cell; + return toned; }) .filter((_, i) => keep[i]); cells[0] = { ...cells[0], v: display, ...(MODEL_URLS[config] ? { href: MODEL_URLS[config] } : {}) }; @@ -435,8 +529,46 @@ const rows = rawRows.map(([, rowAttrs, inner]) => { const rtfCol = columns.indexOf("RTF"); const rtfOf = (r) => Number(r.cells[rtfCol].sort); +/** Where "keeps up with its own speech" is drawn. + * + * Back to a round 2 after a spell at 2.05. That two-hundredth shift existed + * because Magpie's quantisations landed at 2.01x and 2.02x, splitting a ladder + * the table reports as indistinguishable. Re-measuring every autoregressive row + * over at least three passes moved them to 1.53-1.81x, so nothing sits in the + * gap any more and the round number is honest again: 1.99x on one side, + * 2.21x on the other, with 0.22 between them. + * + * Worth knowing that the cutoff no longer controls whether the boundary rule is + * drawn under the default sort. RTF is compute per second of audio and the sort + * is compute per phrase, and with audio durations this varied the two orderings + * interleave at every cutoff between 1.8 and 2.5 - magpie-f32 and chatterbox + * each sit on the wrong side of their neighbours. The rule is drawn when sorting + * by RTF, where the groups are contiguous by construction, and left out + * otherwise. */ const RTF_CUTOFF = 2; +/** Where the same line falls when the table is sorted by compute per phrase. + * + * The boundary rule is cut on whatever the reader is sorting by, so it needs a + * limit in each measure. Cutting on real-time factor while sorting by per-phrase + * compute is what left the two orderings interleaved and the rule undrawn: the + * two disagree because RTF is compute per second of *audio* and engines differ + * in how long they speak. + * + * 7s classifies the table the same way 2x does but for a single row - Magpie's + * full-precision build, quick per phrase at 3.79s and slow per second at 2.31x. + * That agreement is what the number is chosen for, and it is why 7 is preferred + * to 6.5: at 6.5 Kani also disagreed, fast per second of audio at 1.50x and slow + * per phrase at 6.86s, so the two measures split over two rows instead of one. + * + * The cost is a tight boundary. 7s sits in a 0.17s gap, between Kani at 6.86s + * and Chatterbox Q8 at 7.03s, where 6.5s had 0.59s of clear air. That pair is + * closer than three passes resolve, so the line runs between two rows this + * benchmark cannot actually order - stated plainly in the methodology rather + * than presented as a clean split. 7.5 is not a way out: it pulls Q8 inside at + * 2.21x and buys a fresh disagreement for the one it removes. */ +const SYNTH_CUTOFF = 7; + const run = { benchmark: "open-weight-tts", run: runId, @@ -446,6 +578,7 @@ const run = { // Kept as data rather than hardcoded in the component: it drives the // real-time filter, and where the line sits is an editorial choice. rtfCutoff: RTF_CUTOFF, + synthCutoff: SYNTH_CUTOFF, caption: "Sorted by compute per phrase, fastest first. Every configuration measured is listed, " + "including the ones far too slow to keep up with their own speech; the filter above hides " + diff --git a/frontend/src/app/(app)/dashboards/[id]/page.tsx b/frontend/src/app/(app)/dashboards/[id]/page.tsx index 709a1cce..d22599ce 100644 --- a/frontend/src/app/(app)/dashboards/[id]/page.tsx +++ b/frontend/src/app/(app)/dashboards/[id]/page.tsx @@ -3,8 +3,8 @@ "use client"; -// REVISION: dashboard-v58-cancel-pending-edge -console.log(`[dashboard] REVISION: dashboard-v58-cancel-pending-edge loaded at ${new Date().toISOString()}`); +// REVISION: dashboard-v59-frame-on-load +console.log(`[dashboard] REVISION: dashboard-v59-frame-on-load loaded at ${new Date().toISOString()}`); import * as React from "react"; import { useParams, useRouter } from "next/navigation"; @@ -81,7 +81,7 @@ import { Canvas } from "@/components/canvas"; import { MobileFullscreenCanvas } from "@/components/mobile/MobileFullscreenCanvas"; import { FullscreenControlBar } from "@/components/mobile/FullscreenControlBar"; import { isCoarsePointerDevice } from "@/lib/openAuthPopup"; -import { findAvailableSpace } from "@/lib/canvas/placement"; +import { findAvailableSpace, frameToFit } from "@/lib/canvas/placement"; import { DesktopVersionBadge } from "@/components/DesktopVersionBadge"; import { CursorOverlay, PresenceList } from "@/components/multiplayer"; import { useAuthStore } from "@/stores/auth-store"; @@ -822,6 +822,59 @@ export default function DashboardPage() { tryApply(); }, [data, dashboardId]); + // Frame every block once, the first time a dashboard's items render. + // + // ReactFlow's own fitView is off (see fitViewEnabled below) and cannot simply be + // turned on: it fits at mount, when the items are still in flight with the query, + // so it would frame an empty canvas and never run again. Blocks the reader creates + // are placed relative to the current viewport by findAvailableSpace and are visible + // by construction, which is why nothing framed them and nobody noticed. + // + // It shows up on a dashboard whose coordinates came from somewhere else. Every + // hand-written template ships viewport {x:0,y:0,zoom:1} - a typed placeholder + // rather than a computed frame - so a block at the origin renders underneath the + // header and behind the workspace sidebar. The templates exported from a real + // dashboard carry a genuine viewport and looked fine, which is why this survived. + // + // The sidebar overlays the canvas rather than shrinking it, so ReactFlow's padding + // cannot know about it; the centre is shifted right by half its width, the same + // correction ensureVisible makes. + const didInitialFrame = React.useRef(false); + React.useEffect(() => { + if (didInitialFrame.current || !data || items.length === 0) return; + let cancelled = false; + const tryFrame = () => { + if (cancelled || didInitialFrame.current) return; + const instance = reactFlowInstanceRef.current; + const container = canvasContainerRef.current; + // Same rAF retry as the template viewport above: the instance is not ready on + // the first render after the data lands. + if (!instance || !container) { + requestAnimationFrame(tryFrame); + return; + } + const frame = frameToFit( + items, + container.clientWidth, + container.clientHeight, + getSidebarWidth(sidebarCollapsed), + ); + // null means the container has not been measured yet; try again next frame + // rather than locking in a zoom computed from nothing. + if (!frame) { + requestAnimationFrame(tryFrame); + return; + } + didInitialFrame.current = true; + instance.setCenter(frame.centerX, frame.centerY, { zoom: frame.zoom, duration: 0 }); + viewportRef.current = instance.getViewport(); + }; + tryFrame(); + return () => { + cancelled = true; + }; + }, [data, items, sidebarCollapsed]); + // Workspace sidebar session: pick the first active session from any terminal const workspaceSessionId = React.useMemo(() => { // Only browse the workspace through a LIVE session — a stopped session's sandbox diff --git a/frontend/src/app/layout.tsx b/frontend/src/app/layout.tsx index c55937b7..b5f08fc7 100644 --- a/frontend/src/app/layout.tsx +++ b/frontend/src/app/layout.tsx @@ -6,6 +6,7 @@ import { Inter, JetBrains_Mono } from "next/font/google"; import "./globals.css"; import { Providers } from "./providers"; import { SplashTransitionOverlay } from "@/components/SplashTransitionOverlay"; +import { WebAnalytics } from "@/components/WebAnalytics"; const inter = Inter({ variable: "--font-inter", @@ -102,6 +103,9 @@ export default function RootLayout({ `, }} /> + {/* Page views for readers who never sign in; the app's own analytics + only see authenticated users. No-ops without a beacon token. */} + + ); +} diff --git a/frontend/src/components/charts/TtsErrorCostChart.tsx b/frontend/src/components/charts/TtsErrorCostChart.tsx index 45041c14..4d325e10 100644 --- a/frontend/src/components/charts/TtsErrorCostChart.tsx +++ b/frontend/src/components/charts/TtsErrorCostChart.tsx @@ -7,7 +7,7 @@ // Word error against what it costs to say a phrase, for every configuration in // the table. // -// x is Avg synth (compute per phrase), NOT the conventional RTF. RTF is compute +// x is x̄ t (mean compute per phrase), NOT the conventional RTF. RTF is compute // per second of *audio*, so it flatters any engine that pads with silence. // Every engine speaks the same 84 phrases, so per-phrase compute compares // directly and cannot be gamed. @@ -76,12 +76,12 @@ const POINTS: Point[] = ( name: r.display, cls, color: CLASSES.find((c) => c.id === cls)?.color ?? INK.muted, - synth: Number(r.cells[col("x̄ synth")].sort), + synth: Number(r.cells[col("x̄ t")].sort), wer: Number(r.cells[col("WER")].sort) * 100, disk: Number(r.cells[col("Disk")].sort), rtf: Number(r.cells[col("RTF")].sort), werText: r.cells[col("WER")].v, - synthText: r.cells[col("x̄ synth")].v, + synthText: r.cells[col("x̄ t")].v, }; }); diff --git a/frontend/src/components/charts/TtsPreferenceChart.tsx b/frontend/src/components/charts/TtsPreferenceChart.tsx index 0e0b68d6..7be97091 100644 --- a/frontend/src/components/charts/TtsPreferenceChart.tsx +++ b/frontend/src/components/charts/TtsPreferenceChart.tsx @@ -3,7 +3,7 @@ "use client"; -// REVISION: tts-preference-chart-v2-human-axis +// REVISION: tts-preference-chart-v3-log-x // What compute actually buys, plotted against what it costs: reader preference // on the y-axis, compute per phrase on the x. // @@ -23,7 +23,7 @@ // measured, and asserting the finding before the data is in would be inventing // it. Sharpen the title once the shape is known. // -// x is Avg synth (compute per phrase), NOT the conventional RTF. RTF is compute +// x is x̄ t (mean compute per phrase), NOT the conventional RTF. RTF is compute // per second of *audio*, so it flatters any engine that pads with silence. // Every engine here speaks the same 84 phrases, so per-phrase compute compares // directly and cannot be gamed. @@ -40,7 +40,7 @@ import run from "@/data/benchmarks/open-weight-tts/2026-08.json"; import { useClassFilter } from "./useClassFilter"; import { useTtsScores } from "./useTtsScores"; -const MODULE_REVISION = "tts-preference-chart-v2-human-axis"; +const MODULE_REVISION = "tts-preference-chart-v3-log-x"; if (typeof window !== "undefined") { console.log(`[tts-preference-chart] REVISION: ${MODULE_REVISION} loaded at ${new Date().toISOString()}`); } @@ -85,7 +85,7 @@ const ENGINES: Engine[] = ( name: r.display, cls, color: CLASSES.find((c) => c.id === cls)?.color ?? INK.muted, - synth: Number(r.cells[col("x̄ synth")].sort), + synth: Number(r.cells[col("x̄ t")].sort), disk: Number(r.cells[col("Disk")].sort), rtf: Number(r.cells[col("RTF")].sort), }; @@ -96,16 +96,32 @@ const M = { top: 18, right: 26, bottom: 56, left: 58 }; const PW = W - M.left - M.right; const PH = H - M.top - M.bottom; -const X_MAX = 6.8; // Chatterbox Q4 at 6.39 is the slowest that survives the filter const Y_MAX = 100; // ratings are already scaled 0-100 server-side -const sx = (v: number) => M.left + (v / X_MAX) * PW; const sy = (v: number) => M.top + PH - (v / Y_MAX) * PH; +/** Compute per phrase spans two and a half decades - 0.12s to 51s - so the x-axis + * is logarithmic, the same as the word-error chart above it. A linear axis here + * was not merely cramped: it was pinned at 6.8s, which is roughly where the + * engines that keep up with real time end, and every slower engine was drawn + * outside the plot. Ten of thirty rated engines rendered off-canvas, silently, + * while the accessible label still counted them. + * + * The bound derives from the data for the same reason the sibling chart's does: + * a fixed one goes stale the moment an export moves a configuration past it, and + * the failure is a marker that simply is not there. */ +const X_MIN = 0.1; +const xScale = (xMax: number) => { + const lo = Math.log10(X_MIN); + const span = Math.log10(xMax) - lo; + return (v: number) => M.left + ((Math.log10(v) - lo) / span) * PW; +}; + const DISK = { min: Math.min(...ENGINES.map((e) => e.disk)), max: Math.max(...ENGINES.map((e) => e.disk)) }; /** Area-proportional, floored at 5px radius so the smallest is still a target. */ const radius = (disk: number) => 5 + 7 * Math.sqrt((disk - DISK.min) / (DISK.max - DISK.min)); -const X_TICKS = [0, 1, 2, 3, 4, 5, 6]; +// Decade and half-decade ticks, keeping only those the axis actually spans. +const TICK_CANDIDATES = [0.1, 0.3, 1, 3, 10, 30, 100, 300]; const Y_TICKS = [0, 25, 50, 75, 100]; interface Point extends Engine { rating: number } @@ -127,6 +143,16 @@ export function TtsPreferenceChart() { [ratings] ); + // Scaled to the engines actually plotted, so nothing can fall off the right-hand + // end. Computed before the early return below because hooks cannot be conditional. + const { sx, X_TICKS } = React.useMemo(() => { + const xMax = points.length ? Math.max(...points.map((p) => p.synth)) * 1.18 : 1; + return { + sx: xScale(xMax), + X_TICKS: TICK_CANDIDATES.filter((t) => t >= X_MIN && t <= xMax), + }; + }, [points]); + // Nothing to say yet, so say nothing. An empty axis box would read as a bug, // and a placeholder would be a promise the page cannot keep on its own. if (points.length < MIN_ENGINES_TO_PLOT) return null; @@ -205,7 +231,7 @@ export function TtsPreferenceChart() { ))} - Compute per phrase (seconds, Apple M2) + Compute per phrase, log scale (seconds, Apple M2) X_MAX * 0.75; + // Position, not value: on a log axis a fraction of the maximum sits far + // to the right of the same fraction across, so a value-based test flips + // almost nothing. + const right = sx(p.synth) > M.left + PW * 0.72; // Labels sit above the marker, except near the ceiling where that // puts them through the top border. The best-rated engine is by // definition up there, so this is the common case, not an edge one. @@ -256,7 +285,7 @@ export function TtsPreferenceChart() { position: "absolute", pointerEvents: "none", left: `${(sx(hover.synth) / W) * 100}%`, top: `${(sy(hover.rating) / H) * 100}%`, - transform: `translate(${hover.synth > X_MAX * 0.6 ? "-105%" : "12px"}, -110%)`, + transform: `translate(${sx(hover.synth) > M.left + PW * 0.62 ? "-105%" : "12px"}, -110%)`, background: "#0b1a2e", border: `1px solid ${hover.color}`, borderRadius: 6, padding: "0.45rem 0.6rem", fontSize: "0.76rem", color: INK.secondary, whiteSpace: "nowrap", boxShadow: "0 6px 18px rgba(0,0,0,0.45)", diff --git a/frontend/src/components/charts/TtsResultsTable.tsx b/frontend/src/components/charts/TtsResultsTable.tsx index 4f9bbd58..088d03a9 100644 --- a/frontend/src/components/charts/TtsResultsTable.tsx +++ b/frontend/src/components/charts/TtsResultsTable.tsx @@ -3,7 +3,7 @@ "use client"; -// REVISION: tts-results-table-v7-min-three +// REVISION: tts-results-table-v12-vendor-logos // The open-weight TTS comparison: every configuration that keeps pace with real // time, and every row playable so the reader can hear the engine that produced // the numbers. @@ -15,13 +15,15 @@ // server-side with Bradley-Terry. It is live, so it changes under the reader. import * as React from "react"; + +import { ICON_BASE, vendorForUrl } from "./ttsVendors"; import run from "@/data/benchmarks/open-weight-tts/2026-08.json"; import { TtsPreferenceDialog } from "./TtsPreferenceDialog"; import { useSamplePlayer } from "./useSamplePlayer"; import { useClassFilter } from "./useClassFilter"; import { useTtsScores } from "./useTtsScores"; -const MODULE_REVISION = "tts-results-table-v7-min-three"; +const MODULE_REVISION = "tts-results-table-v12-vendor-logos"; if (typeof window !== "undefined") { console.log(`[tts-results-table] REVISION: ${MODULE_REVISION} loaded at ${new Date().toISOString()}`); } @@ -40,12 +42,61 @@ const AUDIO_BASE = "/benchmarks/tts/"; const RTF_COL = run.columns.indexOf("RTF"); if (RTF_COL < 0) throw new Error(`[tts] no "RTF" column in run ${run.run}: have ${run.columns.join(", ")}`); const RTF_CUTOFF = run.rtfCutoff; +const SYNTH_CUTOFF = run.synthCutoff; const rtfOf = (r: Row) => Number(r.cells[RTF_COL].sort); -/** The Human column is spliced in here, right after the engine name, so the - * live number is visible without scrolling the table sideways. */ -const HUMAN_COL = 1; -const COLUMNS: string[] = [run.columns[0], "Human", ...run.columns.slice(1)]; +const RSS_COL = run.columns.indexOf("RSS"); +if (RSS_COL < 0) throw new Error(`[tts] no "RSS" column in run ${run.run}: have ${run.columns.join(", ")}`); +const rssOf = (r: Row) => Number(r.cells[RSS_COL].sort); + +/** Stops for the memory filter, in MB, with no limit as the last one. + * + * Spaced by ratio rather than evenly. Peak RSS runs 246 MB to 8.6 GB, but + * twenty of the forty-six engines fit inside a gigabyte and only three sit above + * four, so an even track would spend most of its length separating the handful + * of largest models and almost none of it where the reader is actually choosing. + * + * A slider rather than a box to type in: the question this answers is "what fits + * on the machine I have", which is a budget being felt out against the count, not + * a number known in advance. It starts at no limit so the table is unfiltered + * until someone asks for less. */ +const RSS_STEPS = [256, 384, 512, 768, 1024, 1536, 2048, 3072, 4096, 6144, 8192, Infinity]; +const fmtMB = (mb: number) => + mb === Infinity ? "any" : mb >= 1024 ? `${Math.round((mb / 1024) * 10) / 10} GB` : `${mb} MB`; + +/** The logo column's key. Empty on purpose - it is the column's visible title, + * and there is none - but it still has to be unique among COLUMNS, because the + * headers are keyed by it and every lookup in this file is COLUMNS.indexOf. */ +const LOGO_HEAD = "​"; + +/** Headers that are not plain text. The mean of a time is written the way the + * rest of the page writes it, with the variable italic. */ +const HEADER_LABEL: Record = { + "x̄ t": ( + <> + x̄ t + + ), +}; + +/** Two columns are spliced in client-side, both right after the engine name. + * + * The logo says whose model a row is without spending a column on the name - + * that information was previously only in the chip row above the table, which + * does not tell you which chip goes with which row. It carries no data, so it + * has no title and does not sort; the name is on the image for anyone who + * cannot see it. + * + * Human sits next to it so the live number is visible without scrolling the + * table sideways. */ +/** Headers that do not sort: Notes is an icon, and the logo carries no value + * to order by. Both keep a plain with no button, so there is nothing to + * tab onto that does nothing when activated. */ +const inertHeader = (i: number) => i === NOTES_COL || i === LOGO_COL; + +const LOGO_COL = 1; +const HUMAN_COL = 2; +const COLUMNS: string[] = [run.columns[0], LOGO_HEAD, "Human", ...run.columns.slice(1)]; /** Notes became an icon: as text it was the widest column in the table and the * only one that wrapped, which set the height of every row for a sentence most @@ -97,20 +148,44 @@ const LICENCE_COL = COLUMNS.indexOf("Licence"); * of a column that is otherwise four characters wide. */ const LICENCE_MAX = 92; -/** RTF index in the *rendered* columns, which is one further along than in the - * run data because Human is spliced in ahead of it. Reading the un-spliced - * index here would test the wrong column entirely. */ -const RTF_COL_RENDERED = COLUMNS.indexOf("RTF"); +/** Visually hidden but read aloud - the logo column has no visible title and a + * bare would leave its cells unlabelled in a screen reader's table mode. */ +const SR_ONLY: React.CSSProperties = { + position: "absolute", width: 1, height: 1, padding: 0, margin: -1, + overflow: "hidden", clip: "rect(0 0 0 0)", whiteSpace: "nowrap", border: 0, +}; + /** The two orderings where "can it keep up" is the axis, and so the only two * where a line drawn at the cutoff means anything. Under any other sort the * fast and slow engines interleave and the boundary is not a place. */ -const BOUNDARY_SORTS = new Set(["RTF", "x\u0304 synth"]); +/** How the real-time boundary is cut, per sort column. + * + * The line is cut on whatever the reader is sorting by, which is what makes the + * two groups contiguous: sorting by a measure and splitting on that same measure + * can only cross once. Cutting on RTF while sorting by compute per phrase was + * what left three crossings and no rule at all, because RTF is compute per + * second of *audio* and engines differ in how long they speak. + * + * The two limits classify the table identically but for one row, so the reader + * is not shown a different population depending on which header they clicked. */ +const BOUNDARY_BY_SORT: Record = { + RTF: { + col: COLUMNS.indexOf("RTF"), + limit: RTF_CUTOFF, + over: `slower than real time, over ${RTF_CUTOFF}\u00d7`, + }, + "x\u0304 t": { + col: COLUMNS.indexOf("x\u0304 t"), + limit: SYNTH_CUTOFF, + over: `over ${SYNTH_CUTOFF}s of compute a phrase`, + }, +}; /** Opening sort: compute per phrase, cheapest first. Word error was the old * default, but it saturates once speech is intelligible and two engines have no * score under the stronger transcriber, so it is a poor way to meet the table. * Cost per phrase is the axis every reader is actually shopping on. */ -const DEFAULT_SORT = { col: COLUMNS.indexOf("x̄ synth"), dir: "asc" as const }; +const DEFAULT_SORT = { col: COLUMNS.indexOf("x̄ t"), dir: "asc" as const }; /** The table's natural width, with Notes an icon, Avg audio gone and the cell * padding trimmed. Past this the columns just stretch, so the breakout stops @@ -162,6 +237,8 @@ export function TtsResultsTable() { const [sort, setSort] = React.useState<{ col: number; dir: "asc" | "desc" } | null>(DEFAULT_SORT); const [realtimeOnly, setRealtimeOnly] = React.useState(false); + const [rssStep, setRssStep] = React.useState(RSS_STEPS.length - 1); + const rssLimit = RSS_STEPS[rssStep]; // Shared with the charts: one class selection governs the whole page, so // filtering to ar-lm in the table narrows the plot below it too. const { active: classFilter, toggle: toggleClass, clear: clearClasses, shows: classShows } = useClassFilter(); @@ -212,6 +289,7 @@ export function TtsResultsTable() { const visible = ROWS.filter( (r) => (!realtimeOnly || rtfOf(r) <= RTF_CUTOFF) && + rssOf(r) <= rssLimit && classShows(classOf(r)) ); const withHuman = visible.map((r) => { @@ -223,7 +301,15 @@ export function TtsResultsTable() { // twenty either way is a precision the ballots cannot support, and a // decimal reads as one. Sorting still uses the full value. : { v: String(Math.round(rating)), sort: String(rating), tone: "", align: "" }; - return { ...r, cells: [r.cells[0], human, ...r.cells.slice(1)] }; + // The logo is derived from the row's own model link, so it cannot point + // at a different vendor than the name does. sort is the vendor name, + // which is unused today - the column does not sort - but keeps the cell + // the same shape as every other one. + const vendor = vendorForUrl(r.cells[0].href); + const logo: Cell = { + v: vendor?.icon ?? "", sort: vendor?.name ?? "", tone: "", align: "left", + }; + return { ...r, cells: [r.cells[0], logo, human, ...r.cells.slice(1)] }; }); if (!sort) return withHuman; return [...withHuman].sort((a, b) => { @@ -237,7 +323,7 @@ export function TtsResultsTable() { typeof ka === "number" && typeof kb === "number" ? ka - kb : String(ka).localeCompare(String(kb)); return sort.dir === "asc" ? cmp : -cmp; }); - }, [classShows, ratings, realtimeOnly, sort]); + }, [classShows, ratings, realtimeOnly, rssLimit, sort]); // asc -> desc -> back to the default, never to "unsorted". Unsorted rendered // the export's own row order, which is neither the default the reader arrived @@ -321,21 +407,48 @@ export function TtsResultsTable() { * not, or null where no rule belongs. Only shown when everything is listed - * with the filter on there is nothing on the far side to divide from - and * only under a sort where the two groups are actually contiguous. */ + /** The single index the rule belongs after, or null when the ordering does not + * actually separate the two groups. + * + * Contiguity used to be safe to assume from the sort column alone. Sorting by + * RTF separates them by construction, and sorting by compute per phrase did + * too, because the two agreed on order. Magpie's quantisations ended that: + * RTF is compute per second of audio, so builds that differ in how long they + * speak can order one way per phrase and the other way per second. Under the + * default sort the cutoff is now crossed three times, and a rule drawn at + * every crossing rendered the same band three times in one table, each one + * claiming everything below it was slower. + * + * So it is counted rather than assumed, which is what the comment above + * BOUNDARY_SORTS always said the rule was. Exactly one crossing means the + * groups are contiguous and the line is a place; more than one means they + * interleave and there is no line to draw. */ + const boundaryIndex = React.useMemo(() => { + const b = sort ? BOUNDARY_BY_SORT[COLUMNS[sort.col]] : undefined; + if (realtimeOnly || !b) return null; + const keepsUp = (r: (typeof rows)[number]) => Number(r.cells[b.col].sort) <= b.limit; + let only: number | null = null; + for (let i = 0; i + 1 < rows.length; i++) { + if (keepsUp(rows[i]) === keepsUp(rows[i + 1])) continue; + if (only !== null) return null; + only = i; + } + return only; + }, [realtimeOnly, sort, rows]); + const boundaryAfter = (i: number): string | null => { - if (realtimeOnly || !sort || !BOUNDARY_SORTS.has(COLUMNS[sort.col])) return null; + if (boundaryIndex === null || i !== boundaryIndex) return null; const next = rows[i + 1]; if (!next) return null; - const here = Number(rows[i].cells[RTF_COL_RENDERED].sort) <= RTF_CUTOFF; - const there = Number(next.cells[RTF_COL_RENDERED].sort) <= RTF_CUTOFF; - if (here === there) return null; + const b = sort ? BOUNDARY_BY_SORT[COLUMNS[sort.col]] : undefined; + if (!b) return null; + const here = Number(rows[i].cells[b.col].sort) <= b.limit; // Which side is which depends on the sort direction, so say it rather than // leaving the reader to infer it from a bare line. // The subject stays put and the direction flips. Naming whichever group // happens to be underneath instead means the sentence changes meaning as // well as direction, and the reader has to re-read it after every sort. - return here - ? `below: slower than real time, over ${RTF_CUTOFF}\u00d7` - : `above: slower than real time, over ${RTF_CUTOFF}\u00d7`; + return `${here ? "below" : "above"}: ${b.over}`; }; const th: React.CSSProperties = { @@ -431,15 +544,46 @@ export function TtsResultsTable() { ); })} + {/* Memory budget. The value is shown as a size rather than a step index, + and carried on aria-valuetext for the same reason: "8" tells a reader + nothing, and the count beside it only says how many survived, not what + they survived. minWidth holds the label steady so the row does not + shuffle sideways while the handle is being dragged. */} + {rows.length === ROWS.length ? `${ROWS.length} configurations` : `${rows.length} of ${ROWS.length} configurations`} - {(realtimeOnly || classFilter.size > 0) && ( + {(realtimeOnly || classFilter.size > 0 || rssLimit !== Infinity) && (