From daa74a5a1915ce5e6bd47d6ddd1056f2afb1891a Mon Sep 17 00:00:00 2001 From: ChenYingHan1121 <246545182+ChenYingHan1121@users.noreply.github.com> Date: Mon, 16 Nov 2026 02:21:05 +0800 Subject: [PATCH 1/4] Add browser audio response timing diagnostics Measure the browser-observed delay between the interviewer's speaking state and detectable remote audio activity. Keep the diagnostic opt-in and independent of avatar playback so developers can investigate response timing without changing the candidate interface. Close #712 Change-Id: Iac1dc45850391ef264036f503c263cfd97a58bcf --- .../audio-response-timing-integration.test.js | 163 ++++++ tests/browser/audio-response-timing.test.js | 549 ++++++++++++++++++ tests/browser/interview-lifecycle.test.js | 3 + tests/browser/report-recovery.test.js | 14 + web/audio-response-timing.js | 160 +++++ web/interview.js | 93 +++ 6 files changed, 982 insertions(+) create mode 100644 tests/browser/audio-response-timing-integration.test.js create mode 100644 tests/browser/audio-response-timing.test.js create mode 100644 web/audio-response-timing.js diff --git a/tests/browser/audio-response-timing-integration.test.js b/tests/browser/audio-response-timing-integration.test.js new file mode 100644 index 000000000..6541d1319 --- /dev/null +++ b/tests/browser/audio-response-timing-integration.test.js @@ -0,0 +1,163 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { runInNewContext } from "node:vm"; +import { functionBody, read } from "./source.js"; + +function fixture({ enabled = true } = {}) { + const page = read("web/interview.js"); + + const monitors = []; + let cancellations = 0; + + const context = { + audioTiming: enabled + ? { + cancelPending: () => { + cancellations++; + }, + audioActivity: () => {}, + } + : null, + + startAudioActivityMonitor(track, onActivity, options) { + const monitor = { + track, + onActivity, + options, + stopped: false, + }; + + monitors.push(monitor); + + return () => { + monitor.stopped = true; + }; + }, + + console: { + warn() {}, + }, + }; + + const code = ` + let audioTimingTrack = null; + let stopAudioTimingMonitor = null; + let audioTimingReady = false; + + ${functionBody(page, "releaseAudioTiming")} + } + + ${functionBody(page, "attachAudioTiming")} + } + + globalThis.timingApi = { + attachAudioTiming, + releaseAudioTiming, + currentTrack: () => audioTimingTrack, + ready: () => audioTimingReady, + }; + `; + + runInNewContext(code, context); + + return { + api: context.timingApi, + monitors, + cancellations: () => cancellations, + }; +} + +function remoteTrack(id) { + return { + id, + mediaStreamTrack: { + kind: "audio", + readyState: "live", + }, + }; +} + +test("replacing a remote track stops the old monitor", () => { + const f = fixture(); + + const trackA = remoteTrack("A"); + const trackB = remoteTrack("B"); + + f.api.attachAudioTiming(trackA); + + assert.equal(f.monitors.length, 1); + assert.equal(f.monitors[0].stopped, false); + + f.api.attachAudioTiming(trackB); + + assert.equal(f.monitors.length, 2); + assert.equal(f.monitors[0].stopped, true); + assert.equal(f.monitors[1].stopped, false); + + assert.equal(f.api.currentTrack(), trackB); + + f.api.releaseAudioTiming(); +}); + +test("late unsubscribe from an old track preserves the new monitor", () => { + const f = fixture(); + + const trackA = remoteTrack("A"); + const trackB = remoteTrack("B"); + + f.api.attachAudioTiming(trackA); + f.api.attachAudioTiming(trackB); + + // The old unsubscribe arrives after the replacement is active. + f.api.releaseAudioTiming(trackA); + + assert.equal(f.monitors.length, 2); + assert.equal(f.monitors[1].stopped, false); + assert.equal(f.api.currentTrack(), trackB); + + // Releasing the current track must still work. + f.api.releaseAudioTiming(trackB); + + assert.equal(f.monitors[1].stopped, true); + assert.equal(f.api.currentTrack(), null); +}); + +test("reconnect can recreate a monitor for the same remote track", () => { + const f = fixture(); + + const track = remoteTrack("A"); + + f.api.attachAudioTiming(track); + + assert.equal(f.monitors.length, 1); + + // Simulate the Reconnecting handler. + f.api.releaseAudioTiming(); + + assert.equal(f.monitors[0].stopped, true); + assert.equal(f.api.currentTrack(), null); + + // Simulate Reconnected with an existing live track. + f.api.attachAudioTiming(track); + + assert.equal(f.monitors.length, 2); + assert.equal(f.monitors[1].stopped, false); + assert.equal(f.api.currentTrack(), track); + + f.api.releaseAudioTiming(); +}); + +test("disabled audio timing creates no monitor", () => { + const f = fixture({ enabled: false }); + + const track = remoteTrack("A"); + + f.api.attachAudioTiming(track); + + assert.equal(f.monitors.length, 0); + assert.equal(f.api.currentTrack(), null); + + f.api.releaseAudioTiming(); + + assert.equal(f.monitors.length, 0); +}); diff --git a/tests/browser/audio-response-timing.test.js b/tests/browser/audio-response-timing.test.js new file mode 100644 index 000000000..e5f250d9e --- /dev/null +++ b/tests/browser/audio-response-timing.test.js @@ -0,0 +1,549 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + createAudioResponseTiming, + startAudioActivityMonitor, +} from "../../web/audio-response-timing.js"; + +function fixture() { + let time = 0; + const measurements = []; + + const timing = createAudioResponseTiming({ + now: () => time, + report: (elapsed) => measurements.push(elapsed), + }); + + return { + timing, + measurements, + advance: (ms) => { + time += ms; + }, + }; +} + +test("measures speaking-to-first-audio gap", () => { + const f = fixture(); + + f.timing.speakingChanged(true); + f.advance(180); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, [180]); +}); + +test("ignores repeated speaking updates", () => { + const f = fixture(); + + f.timing.speakingChanged(true); + f.advance(100); + + f.timing.speakingChanged(true); + f.advance(80); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, [180]); +}); + +test("reports only the first audio activity per speaking turn", () => { + const f = fixture(); + + f.timing.speakingChanged(true); + f.advance(120); + f.timing.audioActivity(); + + f.advance(50); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, [120]); +}); + +test("does not measure audio without speaking", () => { + const f = fixture(); + + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, []); +}); + +test("cancels pending measurement when speaking stops", () => { + const f = fixture(); + + f.timing.speakingChanged(true); + f.advance(100); + f.timing.speakingChanged(false); + + f.advance(50); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, []); +}); + +test("reset cancels an incomplete measurement", () => { + const f = fixture(); + + f.timing.speakingChanged(true); + f.advance(100); + f.timing.reset(); + + f.advance(50); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, []); +}); + +test("audio monitor detects a frame above threshold", () => { + const callbacks = []; + let tick; + + const analyser = { + fftSize: 1024, + getByteTimeDomainData(samples) { + samples.fill(128); + samples[0] = 192; + }, + }; + + const source = { + connect() {}, + disconnect() {}, + }; + + const context = { + state: "running", + + createMediaStreamSource() { + return source; + }, + createAnalyser() { + return analyser; + }, + close: async () => {}, + }; + + const stop = startAudioActivityMonitor( + { kind: "audio", readyState: "live" }, + () => callbacks.push("audio"), + { + createContext: () => context, + createStream: (track) => ({ getTracks: () => [track] }), + schedule: (callback) => { + tick = callback; + return 1; + }, + cancel: () => {}, + }, + ); + + tick(); + + assert.deepEqual(callbacks, ["audio"]); + + stop(); +}); +test("audio monitor ignores an ended track", () => { + const callbacks = []; + let tick; + + const track = { + kind: "audio", + readyState: "ended", + }; + + const analyser = { + fftSize: 1024, + getByteTimeDomainData(samples) { + samples.fill(128); + samples[0] = 192; + }, + }; + + const source = { + connect() {}, + disconnect() {}, + }; + + const context = { + state: "running", + createMediaStreamSource: () => source, + createAnalyser: () => analyser, + close: async () => {}, + }; + + const stop = startAudioActivityMonitor(track, () => callbacks.push("audio"), { + createContext: () => context, + createStream: () => ({}), + schedule: (callback) => { + tick = callback; + return 1; + }, + cancel: () => {}, + }); + + tick(); + + assert.deepEqual(callbacks, []); + + stop(); +}); + +test("audio monitor reports an unavailable audio context", () => { + let tick; + let unavailable = 0; + let activity = 0; + + const context = { + state: "suspended", + createMediaStreamSource: () => ({ + connect() {}, + disconnect() {}, + }), + createAnalyser: () => ({ + fftSize: 1024, + getByteTimeDomainData() { + throw new Error("Suspended context must not be sampled"); + }, + }), + close: async () => {}, + }; + + const stop = startAudioActivityMonitor( + { kind: "audio", readyState: "live" }, + () => activity++, + { + createContext: () => context, + createStream: () => ({}), + onUnavailable: () => unavailable++, + schedule: (callback) => { + tick = callback; + return 1; + }, + cancel: () => {}, + }, + ); + + tick(); + + assert.equal(unavailable, 1); + assert.equal(activity, 0); + + stop(); +}); +test("cancelPending preserves speaking state without reporting stale audio", () => { + const f = fixture(); + + f.timing.speakingChanged(true); + f.advance(100); + + f.timing.cancelPending(); + f.advance(50); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, []); + + // A repeated speaking update must not restart the cancelled measurement. + f.timing.speakingChanged(true); + f.advance(50); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, []); + + // Only a new speaking transition can start another measurement. + f.timing.speakingChanged(false); + f.timing.speakingChanged(true); + + f.advance(80); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, [80]); +}); +test("audio monitor notifies unavailability only on state changes", () => { + let tick; + let unavailable = 0; + + const context = { + state: "suspended", + createMediaStreamSource: () => ({ + connect() {}, + disconnect() {}, + }), + createAnalyser: () => ({ + fftSize: 1024, + getByteTimeDomainData(samples) { + samples.fill(128); + }, + }), + close: async () => {}, + }; + + const stop = startAudioActivityMonitor( + { kind: "audio", readyState: "live" }, + () => {}, + { + createContext: () => context, + createStream: () => ({}), + onUnavailable: () => unavailable++, + schedule: (callback) => { + tick = callback; + return 1; + }, + cancel: () => {}, + }, + ); + + tick(); + tick(); + + assert.equal(unavailable, 1); + + context.state = "running"; + tick(); + + context.state = "suspended"; + tick(); + + assert.equal(unavailable, 2); + + stop(); +}); +test("audio monitor attempts to resume a suspended context", () => { + let resumes = 0; + + const context = { + state: "suspended", + + resume: async () => { + resumes++; + }, + + createMediaStreamSource: () => ({ + connect() {}, + disconnect() {}, + }), + + createAnalyser: () => ({ + fftSize: 1024, + getByteTimeDomainData() {}, + }), + + close: async () => {}, + }; + + const stop = startAudioActivityMonitor( + { kind: "audio", readyState: "live" }, + () => {}, + { + createContext: () => context, + createStream: () => ({}), + schedule: () => 1, + cancel: () => {}, + }, + ); + + assert.equal(resumes, 1); + + stop(); +}); +test("audio monitor closes its context when setup fails", async () => { + let closes = 0; + + const context = { + state: "running", + + createMediaStreamSource() { + throw new Error("Cannot create media source"); + }, + + close: async () => { + closes++; + }, + }; + + assert.throws( + () => + startAudioActivityMonitor( + { kind: "audio", readyState: "live" }, + () => {}, + { + createContext: () => context, + createStream: () => ({}), + schedule: () => 1, + cancel: () => {}, + }, + ), + /Cannot create media source/, + ); + + // close() is asynchronous, so allow its promise to settle. + await new Promise((resolve) => setImmediate(resolve)); + + assert.equal(closes, 1); +}); +test("audio monitor resumes on a gesture and removes listeners on stop", async () => { + const gestures = new EventTarget(); + let resumes = 0; + let closes = 0; + + const context = { + state: "suspended", + + resume: async () => { + resumes++; + context.state = "running"; + }, + + createMediaStreamSource: () => ({ + connect() {}, + disconnect() {}, + }), + + createAnalyser: () => ({ + fftSize: 1024, + getByteTimeDomainData(samples) { + samples.fill(128); + }, + }), + + close: async () => { + closes++; + }, + }; + + const stop = startAudioActivityMonitor( + { kind: "audio", readyState: "live" }, + () => {}, + { + createContext: () => context, + createStream: () => ({}), + gestureTarget: gestures, + schedule: () => 1, + cancel: () => {}, + }, + ); + + // The initial resume is attempted during setup. + assert.equal(resumes, 1); + + // Simulate the browser suspending the context again. + context.state = "suspended"; + + gestures.dispatchEvent(new Event("pointerdown")); + + assert.equal(resumes, 2); + + stop(); + + // Stopped monitors must no longer react to gestures. + context.state = "suspended"; + gestures.dispatchEvent(new Event("keydown")); + + assert.equal(resumes, 2); + + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(closes, 1); +}); +test("monitor interruption does not restart the same speaking turn", () => { + const f = fixture(); + + // Jim begins speaking while the monitor is available. + f.timing.speakingChanged(true); + f.advance(100); + + // The audio monitor becomes unavailable. + f.timing.cancelPending(); + + // It later recovers, but Jim is still in the same speaking turn. + f.advance(100); + f.timing.speakingChanged(true); + + // Audio is detected after recovery. + f.advance(50); + f.timing.audioActivity(); + + // The interrupted turn must not produce a misleading measurement. + assert.deepEqual(f.measurements, []); + + // A genuine new speaking transition may be measured. + f.timing.speakingChanged(false); + f.timing.speakingChanged(true); + + f.advance(120); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, [120]); +}); + +test("hidden pages invalidate audio timing and cleanup visibility listeners", () => { + const visibilityTarget = new EventTarget(); + visibilityTarget.hidden = false; + + let tick; + let unavailable = 0; + let available = 0; + let activity = 0; + + const context = { + state: "running", + createMediaStreamSource: () => ({ + connect() {}, + disconnect() {}, + }), + createAnalyser: () => ({ + fftSize: 1024, + getByteTimeDomainData(samples) { + samples.fill(128); + samples[0] = 192; + }, + }), + close: async () => {}, + }; + + const stop = startAudioActivityMonitor( + { kind: "audio", readyState: "live" }, + () => activity++, + { + createContext: () => context, + createStream: () => ({}), + visibilityTarget, + onAvailable: () => available++, + onUnavailable: () => unavailable++, + schedule: (callback) => { + tick = callback; + return 1; + }, + cancel: () => {}, + }, + ); + + tick(); + + assert.equal(available, 1); + assert.equal(activity, 1); + + visibilityTarget.hidden = true; + visibilityTarget.dispatchEvent(new Event("visibilitychange")); + + assert.equal(unavailable, 1); + + // Even if the frame loop runs, hidden audio must not be counted. + tick(); + + assert.equal(activity, 1); + assert.equal(unavailable, 1); + + visibilityTarget.hidden = false; + visibilityTarget.dispatchEvent(new Event("visibilitychange")); + tick(); + + assert.equal(available, 2); + assert.equal(activity, 2); + + stop(); + + // Stopped monitors must not react to later visibility changes. + visibilityTarget.hidden = true; + visibilityTarget.dispatchEvent(new Event("visibilitychange")); + + assert.equal(unavailable, 1); +}); diff --git a/tests/browser/interview-lifecycle.test.js b/tests/browser/interview-lifecycle.test.js index 22ef1d8b9..2c94890c8 100644 --- a/tests/browser/interview-lifecycle.test.js +++ b/tests/browser/interview-lifecycle.test.js @@ -248,6 +248,9 @@ test("the actual connection preserves refusal reasons and gates the init deadlin // As init leaves it before connect: Practice work is guarded from page // load, and nothing in the start may arm the guard for Strict. const context = { + releaseAudioTiming: () => {}, + audioTiming: null, + protected: !strict, state: { endsAt: 0, diff --git a/tests/browser/report-recovery.test.js b/tests/browser/report-recovery.test.js index 2710785b8..b6a74ebb6 100644 --- a/tests/browser/report-recovery.test.js +++ b/tests/browser/report-recovery.test.js @@ -509,6 +509,18 @@ test("terminal room disconnect finalizes recovery before clearing the room", asy page.scope.console = { warn() {} }; const playbackRooms = []; page.scope.audioPlayback = { release: (room) => playbackRooms.push(room) }; + let audioTimingReleases = 0; + let audioTimingResets = 0; + + page.scope.releaseAudioTiming = () => { + audioTimingReleases++; + }; + + page.scope.audioTiming = { + reset: () => { + audioTimingResets++; + }, + }; page.scope.preflight = { userStream: { label: "Preflight stream" } }; page.scope.localRecording.needsMicrophone = false; const mediaCleanup = []; @@ -520,6 +532,8 @@ test("terminal room disconnect finalizes recovery before clearing the room", asy )(new Proxy(page.scope, { has: (target, key) => key in target })); disconnect(); await settle(); + assert.equal(audioTimingReleases, 1); + assert.equal(audioTimingResets, 1); assert.deepEqual(playbackRooms, [page.room]); assert.deepEqual(mediaCleanup, [ [page.scope.preflight, { preserveMicrophone: false }], diff --git a/web/audio-response-timing.js b/web/audio-response-timing.js new file mode 100644 index 000000000..228127695 --- /dev/null +++ b/web/audio-response-timing.js @@ -0,0 +1,160 @@ +import { peakLevel } from "./audio-check.js"; +/// Browser-observed gap between Jim entering "speaking" and the first +/// detectable remote audio activity. This is not end-to-end latency. +export function createAudioResponseTiming({ + now = () => performance.now(), + report = () => {}, +} = {}) { + let speaking = false; + let startedAt = null; + + function speakingChanged(nextSpeaking) { + if (nextSpeaking === speaking) return; + + speaking = nextSpeaking; + + if (speaking) { + startedAt = now(); + } else { + startedAt = null; + } + } + + function audioActivity() { + if (!speaking || startedAt === null) return; + + const elapsedMs = now() - startedAt; + startedAt = null; + + report(elapsedMs); + } + + function cancelPending() { + startedAt = null; + } + + function reset() { + speaking = false; + cancelPending(); + } + + return { + speakingChanged, + audioActivity, + cancelPending, + reset, + }; +} +/// Observe remote audio activity without routing it to another output. +/// The returned function releases the analyser and its AudioContext. +export function startAudioActivityMonitor( + track, + onActivity, + { + threshold = 0.05, + onAvailable = () => {}, + onUnavailable = () => {}, + createContext = () => new AudioContext(), + createStream = (mediaTrack) => new MediaStream([mediaTrack]), + schedule = requestAnimationFrame, + cancel = cancelAnimationFrame, + gestureTarget = globalThis.document, + visibilityTarget = globalThis.document, + } = {}, +) { + const context = createContext(); + let source; + let analyser; + + try { + if (context.state === "suspended") { + void context.resume?.().catch(() => {}); + } + + source = context.createMediaStreamSource(createStream(track)); + analyser = context.createAnalyser(); + + analyser.fftSize = 1024; + source.connect(analyser); + } catch (error) { + source?.disconnect(); + void context.close?.().catch(() => {}); + throw error; + } + + const samples = new Uint8Array(analyser.fftSize); + + let stopped = false; + let frame = null; + let unavailable = null; + + function notifyAvailability(available) { + const nextUnavailable = !available; + + if (unavailable === nextUnavailable) return; + + unavailable = nextUnavailable; + + if (available) onAvailable(); + else onUnavailable(); + } + + function visibilityChanged() { + if (stopped) return; + + if (visibilityTarget?.hidden) { + notifyAvailability(false); + } + } + + visibilityTarget?.addEventListener("visibilitychange", visibilityChanged); + + function resumeOnGesture() { + if (stopped || context.state !== "suspended") return; + void context.resume?.().catch(() => {}); + } + + gestureTarget?.addEventListener("pointerdown", resumeOnGesture); + gestureTarget?.addEventListener("keydown", resumeOnGesture); + + function tick() { + if (stopped) return; + + const available = + context.state === "running" && + track.readyState === "live" && + !visibilityTarget?.hidden; + + notifyAvailability(available); + + if (available) { + analyser.getByteTimeDomainData(samples); + + if (peakLevel(samples) > threshold) { + onActivity(); + } + } + + frame = schedule(tick); + } + frame = schedule(tick); + + return () => { + if (stopped) return; + + stopped = true; + + gestureTarget?.removeEventListener("pointerdown", resumeOnGesture); + gestureTarget?.removeEventListener("keydown", resumeOnGesture); + + visibilityTarget?.removeEventListener( + "visibilitychange", + visibilityChanged, + ); + + cancel(frame); + source.disconnect(); + + void context.close().catch(() => {}); + }; +} diff --git a/web/interview.js b/web/interview.js index 1b4a8f6f8..128feda41 100644 --- a/web/interview.js +++ b/web/interview.js @@ -222,6 +222,11 @@ import { parseCandidateCase, runBrowserTests } from "./runners.js"; import { mountBehavioralReview } from "./behavioral-review.js"; import { consumeGroundingPacket } from "./document-grounding.js"; +import { + createAudioResponseTiming, + startAudioActivityMonitor, +} from "./audio-response-timing.js"; + /// Which framework the candidate is being read against right now. The rounds /// never overlap, so this is a single value rather than a pair. Declared up /// here because `codingClosed` reads it and `init` runs before the rest of the @@ -336,6 +341,20 @@ const consentVersion = globalThis.CODETRIAL_CONSENT_VERSION || ""; const replayVersion = globalThis.CODETRIAL_REPLAY_VERSION || 1; const params = new URLSearchParams(window.location.search); +const audioTimingEnabled = params.get("audioTiming") === "1"; +const audioTiming = audioTimingEnabled + ? createAudioResponseTiming({ + report: (elapsedMs) => { + console.info( + `[audio-timing] Browser-observed speaking-to-audio gap: ${elapsedMs.toFixed(1)} ms`, + ); + }, + }) + : null; + +let audioTimingTrack = null; +let stopAudioTimingMonitor = null; +let audioTimingReady = false; const editorOptions = editorOptionsFromParams(params); const indentUnit = INDENT_UNITS[indentFromParams(params)]; const strict = params.get("selection") === "strict"; @@ -2397,6 +2416,7 @@ async function connectLiveKit(connection, preflight, presenting = false) { // on its own, so the candidate is told to wait rather than to restart. room.on(livekit.RoomEvent.Reconnecting, () => { if (state.room !== room) return; + releaseAudioTiming(); state.connected = false; if (strict) updateRunAvailability(); setBanner("connection", providerUiState("reconnecting").message); @@ -2404,6 +2424,11 @@ async function connectLiveKit(connection, preflight, presenting = false) { room.on(livekit.RoomEvent.Reconnected, () => { if (state.room !== room) return; state.connected = true; + // LiveKit may retain the same remote audio track across reconnects. + // Reattach the diagnostic even when no new TrackSubscribed event fires. + if (jimTrack?.mediaStreamTrack?.readyState === "live") { + attachAudioTiming(jimTrack); + } if (strict) updateRunAvailability(); setBanner("connection", ""); if (state.phase === "live" && state.interviewerReadiness?.pending) { @@ -2433,6 +2458,10 @@ async function connectLiveKit(connection, preflight, presenting = false) { }); room.on(livekit.RoomEvent.Disconnected, () => { audioPlayback.release(room); + if (state.room === room) { + releaseAudioTiming(); + audioTiming?.reset(); + } const startupCleanup = state.startupCleanupRoom === room; if (startupCleanup) state.startupCleanupRoom = null; if (state.room !== room) return; @@ -2680,6 +2709,8 @@ let detachSelfView = () => {}; /// left only once that notice is acknowledged or given up on. function abandonStart(preflight, { preserveMicrophone = false, flushed } = {}) { const joined = state.room; + releaseAudioTiming(); + audioTiming?.reset(); state.connected = false; state.room = null; if (joined) audioPlayback.release(joined); @@ -3004,7 +3035,56 @@ async function receiveReport(room, payload, flushed = Promise.resolve()) { reportRenderFailed(error, { offlineSummary: !renderAttempted }); } } +function releaseAudioTiming(track = null) { + if (!audioTiming) return; + + // A late unsubscribe from an old track must not remove the new monitor. + if (track && track !== audioTimingTrack) return; + + stopAudioTimingMonitor?.(); + + stopAudioTimingMonitor = null; + audioTimingTrack = null; + audioTimingReady = false; + + audioTiming.cancelPending(); +} +function attachAudioTiming(track) { + if (!audioTiming) return; + + if (audioTimingTrack === track) return; + + releaseAudioTiming(); + const mediaTrack = track.mediaStreamTrack; + if (!mediaTrack) return; + + audioTimingTrack = track; + + try { + stopAudioTimingMonitor = startAudioActivityMonitor( + mediaTrack, + () => { + if (!audioTimingReady || audioTimingTrack !== track) return; + audioTiming.audioActivity(); + }, + { + onAvailable: () => { + if (audioTimingTrack !== track) return; + audioTimingReady = true; + }, + onUnavailable: () => { + if (audioTimingTrack !== track) return; + audioTimingReady = false; + audioTiming.cancelPending(); + }, + }, + ); + } catch (error) { + releaseAudioTiming(); + console.warn("[audio-timing] Monitor unavailable", error); + } +} // One element for the whole session. LiveKit re-subscribes on reconnect, and // track.attach() with no argument mints a fresh element every time: two of // them play the same track into the shared tab, audibly doubled, while @@ -3052,10 +3132,13 @@ function playRemoteAudio(track, participant) { }); } attachAvatarAnalyser(track, participant); + attachAudioTiming(track); } function dropRemoteAudio(track) { if (track.kind !== "audio") return; + releaseAudioTiming(track); + // Cleared only if the element that went away was Jim's. Any other remote // audio track unsubscribing -- a second participant leaving -- would // otherwise drop the handle while the element is still playing, and @@ -5295,6 +5378,7 @@ function updateAgentState() { return; } if (!agent) { + audioTiming?.reset(); setAgentStateLabel("Waiting", false); // "Waiting" is honest but useless on its own: it looks identical whether // the interviewer has not arrived yet or arrived, greeted, and died. Once @@ -5375,6 +5459,15 @@ function updateAgentState() { const value = published || "listening"; state.agentSpeaking = value === "speaking"; state.agentThinking = value === "thinking"; + if (audioTiming) { + audioTiming.speakingChanged(published === "speaking"); + + // A speaking transition is measurable only if the monitor was + // already observing audio when the transition occurred. + if (!audioTimingReady || !state.connected) { + audioTiming.cancelPending(); + } + } if (state.agentSpeaking || state.agentThinking) { drawingTurn?.reset(); hideTurnRing(); From b93c715c66640915d0986e7ab615e999368f76f8 Mon Sep 17 00:00:00 2001 From: ChenYingHan1121 <246545182+ChenYingHan1121@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:33:49 +0800 Subject: [PATCH 2/4] Address audio response timing review feedback Correct timing results when remote audio is already active or unavailable, and ensure stale room events cannot release a newer monitor. Reuse the existing media meter to avoid duplicate audio analysis logic while preserving independent remote audio monitoring. Add regression tests for reconnection, cancellation, silent audio, and track lifecycle behavior. Change-Id: I20eb8e10099dc0bcb584d81431e71726a51223de --- .../audio-response-timing-integration.test.js | 381 +++++++-- tests/browser/audio-response-timing.test.js | 747 ++++++++++-------- tests/browser/mic-meter.test.js | 94 ++- web/audio-response-timing.js | 149 ++-- web/interview.js | 41 +- web/mic-meter.js | 53 +- 6 files changed, 983 insertions(+), 482 deletions(-) diff --git a/tests/browser/audio-response-timing-integration.test.js b/tests/browser/audio-response-timing-integration.test.js index 6541d1319..62592c477 100644 --- a/tests/browser/audio-response-timing-integration.test.js +++ b/tests/browser/audio-response-timing-integration.test.js @@ -1,66 +1,148 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { runInNewContext } from "node:vm"; -import { functionBody, read } from "./source.js"; +import { createAudioResponseTiming } from "../../web/audio-response-timing.js"; +import { functionBody, loadInterview, read } from "./source.js"; -function fixture({ enabled = true } = {}) { +async function fixture({ enabled = true } = {}) { const page = read("web/interview.js"); - const monitors = []; + const measurements = []; + const diagnostics = []; + const playbackReleases = []; let cancellations = 0; - + let time = 0; + const timing = createAudioResponseTiming({ + now: () => time, + report: (elapsed) => measurements.push(elapsed), + diagnose: (reason) => diagnostics.push(reason), + }); + class Room { + handlers = new Map(); + on(event, callback) { + const handlers = this.handlers.get(event) || []; + handlers.push(callback); + this.handlers.set(event, handlers); + return this; + } + emit(event, ...args) { + for (const callback of this.handlers.get(event) || []) callback(...args); + } + async connect() {} + registerTextStreamHandler() {} + } + const state = { phase: "live", connected: false, sawAgent: true }; + const agent = { + identity: "agent-test", + attributes: { "lk.agent.state": "listening" }, + }; const context = { + state, + window: { + LivekitClient: { + Room, + RoomEvent: new Proxy({}, { get: (_, event) => event }), + }, + }, + audioPlayback: { + setRoom() {}, + release: (room) => playbackReleases.push(room), + }, audioTiming: enabled ? { - cancelPending: () => { + ...timing, + cancelPending(reason) { cancellations++; + timing.cancelPending(reason); }, - audioActivity: () => {}, } : null, - startAudioActivityMonitor(track, onActivity, options) { - const monitor = { - track, - onActivity, - options, - stopped: false, - }; - + const monitor = { track, onActivity, options, stopped: false }; monitors.push(monitor); - return () => { monitor.stopped = true; }; }, - - console: { - warn() {}, - }, + strict: false, + whiteboard: false, + topics: {}, + console: { warn() {} }, + nodes: { mutedSpeechReminder: {} }, + mutedSpeech: { stop() {} }, + localRecording: { needsMicrophone: false }, + stopPreflight() {}, + dropRemoteAudio() {}, + playRemoteAudio() {}, + publishPreflightTracks: async () => {}, + paintMicrophoneState() {}, + monitorIntegrityTracks() {}, + followMicrophoneLoss() {}, + refreshAudioInputs: async () => {}, + publishIntegrityEvent: async () => {}, + startIntegrityHeartbeat() {}, + setBanner() {}, + providerUiState: () => ({ message: "connection unavailable" }), + reconnectCodingInterview: async () => {}, + exampleBoard: {}, + flushPendingPublishes() {}, + publishCode() {}, }; - const code = ` let audioTimingTrack = null; + let audioTimingRoom = null; let stopAudioTimingMonitor = null; let audioTimingReady = false; - - ${functionBody(page, "releaseAudioTiming")} - } - - ${functionBody(page, "attachAudioTiming")} - } - + let jimTrack = null; + ${["releaseAudioTiming", "attachAudioTiming", "connectLiveKit"].map((name) => functionBody(page, name) + "\n}").join("\n")} globalThis.timingApi = { - attachAudioTiming, - releaseAudioTiming, + attachAudioTiming, releaseAudioTiming, connectLiveKit, + setRemoteTrack: (track) => { jimTrack = track; attachAudioTiming(track); }, currentTrack: () => audioTimingTrack, + owner: () => audioTimingRoom, ready: () => audioTimingReady, }; `; - runInNewContext(code, context); - + const scope = { + state, + audioTiming: context.audioTiming, + get audioTimingReady() { + return context.timingApi.ready(); + }, + roomParticipants: () => [agent], + roomInterviewer: () => agent, + }; + const { updateAgentState } = loadInterview(["updateAgentState"], scope, { + stubMissing: true, + }); + context.updateAgentState = updateAgentState; + assert.equal( + await context.timingApi.connectLiveKit({}, { cameraSkipped: true }), + "joined", + ); + const room = state.room; + for (const event of ["Reconnecting", "Reconnected", "Disconnected"]) { + assert.equal( + room.handlers.get(event)?.length, + 1, + `${event} is registered by connectLiveKit`, + ); + } + agent.attributes["lk.agent.state"] = "speaking"; return { + state, + room, + agent, + timing, + diagnostics, + measurements, + playbackReleases, + advance: (ms) => { + time += ms; + }, + disconnect: () => room.emit("Disconnected"), + updateAgentState, api: context.timingApi, monitors, cancellations: () => cancellations, @@ -77,8 +159,8 @@ function remoteTrack(id) { }; } -test("replacing a remote track stops the old monitor", () => { - const f = fixture(); +test("replacing a remote track stops the old monitor", async () => { + const f = await fixture(); const trackA = remoteTrack("A"); const trackB = remoteTrack("B"); @@ -99,8 +181,8 @@ test("replacing a remote track stops the old monitor", () => { f.api.releaseAudioTiming(); }); -test("late unsubscribe from an old track preserves the new monitor", () => { - const f = fixture(); +test("late unsubscribe from an old track preserves the new monitor", async () => { + const f = await fixture(); const trackA = remoteTrack("A"); const trackB = remoteTrack("B"); @@ -122,42 +204,235 @@ test("late unsubscribe from an old track preserves the new monitor", () => { assert.equal(f.api.currentTrack(), null); }); -test("reconnect can recreate a monitor for the same remote track", () => { - const f = fixture(); +for (const readyState of ["live", "ended"]) { + test(`registered reconnect handlers respect a retained ${readyState} track`, async () => { + const f = await fixture(); + const track = remoteTrack("A"); + f.api.setRemoteTrack(track); + const old = f.monitors[0]; + old.options.onAvailable(); + f.updateAgentState(); + old.options.onQuiet(); + track.mediaStreamTrack.readyState = readyState; + const cancellations = f.cancellations(); + f.room.emit("Reconnecting"); + assert.equal(f.state.connected, false); + assert.equal(old.stopped, true); + assert.equal(f.api.ready(), false); + assert.equal(f.api.owner(), null); + assert.equal(f.cancellations(), cancellations + 1); + f.room.emit("Reconnected"); + assert.equal(f.state.connected, true); + assert.equal(f.monitors.length, readyState === "live" ? 2 : 1); + assert.equal(f.api.currentTrack(), readyState === "live" ? track : null); + assert.equal(f.api.owner(), readyState === "live" ? f.room : null); + old.options.onAvailable(); + old.options.onQuiet(); + f.advance(200); + old.onActivity(); + if (readyState === "live") { + const current = f.monitors[1]; + current.options.onAvailable(); + current.options.onQuiet(); + current.onActivity(); + assert.equal(current.stopped, false); + } + assert.deepEqual( + f.measurements, + [], + "recovery cannot finish the interrupted turn", + ); + assert.deepEqual(f.diagnostics, ["room reconnecting"]); + f.disconnect(); + assert.equal(f.api.currentTrack(), null); + assert.equal(f.api.owner(), null); + assert.equal(f.api.ready(), false); + assert.ok(f.monitors.every((monitor) => monitor.stopped)); + assert.deepEqual(f.playbackReleases, [f.room]); + }); +} + +test("disabled audio timing creates no monitor", async () => { + const f = await fixture({ enabled: false }); const track = remoteTrack("A"); f.api.attachAudioTiming(track); - assert.equal(f.monitors.length, 1); + assert.equal(f.monitors.length, 0); + assert.equal(f.api.currentTrack(), null); - // Simulate the Reconnecting handler. f.api.releaseAudioTiming(); + assert.equal(f.monitors.length, 0); +}); + +test("report disconnect releases the owning monitor after state.room is cleared", async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + f.monitors[0].options.onAvailable(); + f.timing.speakingChanged(true); + f.state.room = null; + f.disconnect(); assert.equal(f.monitors[0].stopped, true); assert.equal(f.api.currentTrack(), null); + assert.equal(f.api.ready(), false); + assert.deepEqual(f.diagnostics, ["room disconnected"]); +}); - // Simulate Reconnected with an existing live track. - f.api.attachAudioTiming(track); - - assert.equal(f.monitors.length, 2); - assert.equal(f.monitors[1].stopped, false); - assert.equal(f.api.currentTrack(), track); +for (const cleared of [false, true]) { + test(`a stale disconnect preserves the newer room's monitor (cleared=${cleared})`, async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + f.state.room = {}; + const newerTrack = remoteTrack("B"); + f.api.attachAudioTiming(newerTrack); + f.monitors[1].options.onAvailable(); + if (cleared) f.state.room = null; + f.disconnect(); + assert.equal(f.monitors[1].stopped, false); + assert.equal(f.api.currentTrack(), newerTrack); + assert.equal(f.api.ready(), true); + }); +} - f.api.releaseAudioTiming(); +test("speaking updates explain an unready monitor once and never time its recovery", async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + f.updateAgentState(); + f.updateAgentState(); + assert.deepEqual(f.diagnostics, ["monitor not ready"]); + f.monitors[0].options.onAvailable(); + f.advance(200); + f.monitors[0].onActivity(); + assert.deepEqual(f.measurements, []); }); -test("disabled audio timing creates no monitor", () => { - const f = fixture({ enabled: false }); +test("an unavailable monitor cancels a pending measurement with its reason", async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + const monitor = f.monitors[0]; + monitor.options.onAvailable(); + f.updateAgentState(); + monitor.options.onQuiet(); + const cancellations = f.cancellations(); + monitor.options.onUnavailable("audio context suspended"); + assert.equal(f.cancellations(), cancellations + 1); + monitor.options.onUnavailable("audio context suspended"); + assert.equal(f.api.ready(), false); + assert.deepEqual(f.diagnostics, ["audio context suspended"]); + monitor.options.onAvailable(); + f.advance(200); + monitor.onActivity(); + assert.deepEqual(f.measurements, []); +}); - const track = remoteTrack("A"); +test("quiet frames reach the timer through the attached monitor", async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + const monitor = f.monitors[0]; + monitor.options.onAvailable(); + f.updateAgentState(); + assert.equal(typeof monitor.options.onQuiet, "function"); + monitor.options.onQuiet(); + f.advance(80); + monitor.onActivity(); + assert.deepEqual(f.measurements, [80]); +}); - f.api.attachAudioTiming(track); +test("console diagnostics are installed only with audioTiming=1", () => { + const source = read("web/interview.js"); + const setup = source.slice( + source.indexOf("const params = new URLSearchParams"), + source.indexOf("let audioTimingTrack"), + ); + for (const search of ["", "?audioTiming=0", "?audioTiming=1"]) { + const logs = []; + let created = 0; + const context = { + URLSearchParams, + window: { location: { search } }, + console: { debug: (message) => logs.push(message) }, + createAudioResponseTiming(options) { + created++; + return createAudioResponseTiming(options); + }, + }; + runInNewContext( + setup + '; audioTiming?.speakingChanged(true, "monitor not ready");', + context, + ); + assert.equal(created, search === "?audioTiming=1" ? 1 : 0); + assert.deepEqual( + logs, + search === "?audioTiming=1" + ? ["[audio-timing] Turn not measured: monitor not ready"] + : [], + ); + } +}); - assert.equal(f.monitors.length, 0); - assert.equal(f.api.currentTrack(), null); +test("registered stale reconnect callbacks preserve the newer room and monitor", async () => { + const f = await fixture(); + f.api.setRemoteTrack(remoteTrack("A")); + const newerRoom = new f.room.constructor(); + f.state.room = newerRoom; + f.api.attachAudioTiming(remoteTrack("B")); + f.monitors[1].options.onAvailable(); + const cancellations = f.cancellations(); + f.room.emit("Reconnecting"); + f.room.emit("Reconnected"); + f.room.emit("Disconnected"); + assert.equal(f.state.room, newerRoom); + assert.equal(f.state.connected, true); + assert.equal(f.api.owner(), newerRoom); + assert.equal(f.api.ready(), true); + assert.equal(f.monitors.length, 2); + assert.equal(f.monitors[1].stopped, false); + assert.equal(f.cancellations(), cancellations); +}); +test("monitor release clears readiness and late callbacks cannot complete a pending turn", async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + const monitor = f.monitors[0]; + monitor.options.onAvailable(); + f.updateAgentState(); + monitor.options.onQuiet(); + const cancellations = f.cancellations(); f.api.releaseAudioTiming(); + assert.equal(f.cancellations(), cancellations + 1); + assert.equal(f.api.ready(), false); + assert.equal(f.api.owner(), null); + assert.equal(monitor.stopped, true); + monitor.options.onAvailable(); + monitor.options.onQuiet(); + f.advance(200); + monitor.onActivity(); + assert.equal(f.api.ready(), false); + assert.deepEqual(f.measurements, []); + assert.deepEqual(f.diagnostics, ["audio monitor released"]); +}); - assert.equal(f.monitors.length, 0); +test("callbacks from a replaced track cannot cancel or measure the current track", async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + const old = f.monitors[0]; + f.api.attachAudioTiming(remoteTrack("B")); + const current = f.monitors[1]; + current.options.onAvailable(); + f.updateAgentState(); + current.options.onQuiet(); + const cancellations = f.cancellations(); + old.options.onUnavailable("track ended"); + old.options.onAvailable(); + old.options.onQuiet(); + f.advance(80); + old.onActivity(); + assert.equal(f.cancellations(), cancellations); + assert.equal(f.api.ready(), true); + assert.equal(f.api.owner(), f.room); + assert.deepEqual(f.measurements, []); + current.onActivity(); + assert.deepEqual(f.measurements, [80]); }); diff --git a/tests/browser/audio-response-timing.test.js b/tests/browser/audio-response-timing.test.js index e5f250d9e..41b77dd50 100644 --- a/tests/browser/audio-response-timing.test.js +++ b/tests/browser/audio-response-timing.test.js @@ -1,5 +1,6 @@ import { test } from "node:test"; import assert from "node:assert/strict"; +import { startMediaMeter } from "../../web/mic-meter.js"; import { createAudioResponseTiming, startAudioActivityMonitor, @@ -8,15 +9,18 @@ import { function fixture() { let time = 0; const measurements = []; + const diagnostics = []; const timing = createAudioResponseTiming({ now: () => time, report: (elapsed) => measurements.push(elapsed), + diagnose: (reason) => diagnostics.push(reason), }); return { timing, measurements, + diagnostics, advance: (ms) => { time += ms; }, @@ -27,6 +31,7 @@ test("measures speaking-to-first-audio gap", () => { const f = fixture(); f.timing.speakingChanged(true); + f.timing.audioQuiet(); f.advance(180); f.timing.audioActivity(); @@ -37,6 +42,7 @@ test("ignores repeated speaking updates", () => { const f = fixture(); f.timing.speakingChanged(true); + f.timing.audioQuiet(); f.advance(100); f.timing.speakingChanged(true); @@ -50,6 +56,7 @@ test("reports only the first audio activity per speaking turn", () => { const f = fixture(); f.timing.speakingChanged(true); + f.timing.audioQuiet(); f.advance(120); f.timing.audioActivity(); @@ -94,140 +101,49 @@ test("reset cancels an incomplete measurement", () => { }); test("audio monitor detects a frame above threshold", () => { + const m = monitorFixture(); const callbacks = []; - let tick; - - const analyser = { - fftSize: 1024, - getByteTimeDomainData(samples) { - samples.fill(128); - samples[0] = 192; - }, - }; - - const source = { - connect() {}, - disconnect() {}, - }; - - const context = { - state: "running", - - createMediaStreamSource() { - return source; - }, - createAnalyser() { - return analyser; - }, - close: async () => {}, - }; - const stop = startAudioActivityMonitor( - { kind: "audio", readyState: "live" }, + m.track, () => callbacks.push("audio"), - { - createContext: () => context, - createStream: (track) => ({ getTracks: () => [track] }), - schedule: (callback) => { - tick = callback; - return 1; - }, - cancel: () => {}, - }, + m.options, ); - - tick(); - + assert.deepEqual( + callbacks, + [], + "remote timing waits for the first animation frame", + ); + m.tick(); assert.deepEqual(callbacks, ["audio"]); - + assert.deepEqual(m.sampleLengths, [1024]); stop(); }); test("audio monitor ignores an ended track", () => { + const m = monitorFixture({ readyState: "ended" }); const callbacks = []; - let tick; - - const track = { - kind: "audio", - readyState: "ended", - }; - - const analyser = { - fftSize: 1024, - getByteTimeDomainData(samples) { - samples.fill(128); - samples[0] = 192; - }, - }; - - const source = { - connect() {}, - disconnect() {}, - }; - - const context = { - state: "running", - createMediaStreamSource: () => source, - createAnalyser: () => analyser, - close: async () => {}, - }; - - const stop = startAudioActivityMonitor(track, () => callbacks.push("audio"), { - createContext: () => context, - createStream: () => ({}), - schedule: (callback) => { - tick = callback; - return 1; - }, - cancel: () => {}, - }); - - tick(); - + const stop = startAudioActivityMonitor( + m.track, + () => callbacks.push("audio"), + m.options, + ); + m.tick(); assert.deepEqual(callbacks, []); - + assert.deepEqual(m.sampleLengths, []); stop(); }); test("audio monitor reports an unavailable audio context", () => { - let tick; + const m = monitorFixture({ state: "suspended" }); let unavailable = 0; let activity = 0; - - const context = { - state: "suspended", - createMediaStreamSource: () => ({ - connect() {}, - disconnect() {}, - }), - createAnalyser: () => ({ - fftSize: 1024, - getByteTimeDomainData() { - throw new Error("Suspended context must not be sampled"); - }, - }), - close: async () => {}, - }; - - const stop = startAudioActivityMonitor( - { kind: "audio", readyState: "live" }, - () => activity++, - { - createContext: () => context, - createStream: () => ({}), - onUnavailable: () => unavailable++, - schedule: (callback) => { - tick = callback; - return 1; - }, - cancel: () => {}, - }, - ); - - tick(); - + const stop = startAudioActivityMonitor(m.track, () => activity++, { + ...m.options, + onUnavailable: () => unavailable++, + }); + m.tick(); assert.equal(unavailable, 1); assert.equal(activity, 0); - + assert.deepEqual(m.sampleLengths, []); stop(); }); test("cancelPending preserves speaking state without reporting stale audio", () => { @@ -251,7 +167,9 @@ test("cancelPending preserves speaking state without reporting stale audio", () // Only a new speaking transition can start another measurement. f.timing.speakingChanged(false); + f.timing.audioQuiet(); f.timing.speakingChanged(true); + f.timing.audioQuiet(); f.advance(80); f.timing.audioActivity(); @@ -259,189 +177,56 @@ test("cancelPending preserves speaking state without reporting stale audio", () assert.deepEqual(f.measurements, [80]); }); test("audio monitor notifies unavailability only on state changes", () => { - let tick; + const m = monitorFixture({ state: "suspended", peak: 0 }); let unavailable = 0; - - const context = { - state: "suspended", - createMediaStreamSource: () => ({ - connect() {}, - disconnect() {}, - }), - createAnalyser: () => ({ - fftSize: 1024, - getByteTimeDomainData(samples) { - samples.fill(128); - }, - }), - close: async () => {}, - }; - - const stop = startAudioActivityMonitor( - { kind: "audio", readyState: "live" }, - () => {}, - { - createContext: () => context, - createStream: () => ({}), - onUnavailable: () => unavailable++, - schedule: (callback) => { - tick = callback; - return 1; - }, - cancel: () => {}, - }, - ); - - tick(); - tick(); - + const stop = startAudioActivityMonitor(m.track, () => {}, { + ...m.options, + onUnavailable: () => unavailable++, + }); + m.tick(); + m.tick(); assert.equal(unavailable, 1); - - context.state = "running"; - tick(); - - context.state = "suspended"; - tick(); - + m.context.state = "running"; + m.tick(); + m.context.state = "suspended"; + m.tick(); assert.equal(unavailable, 2); - stop(); }); test("audio monitor attempts to resume a suspended context", () => { - let resumes = 0; - - const context = { - state: "suspended", - - resume: async () => { - resumes++; - }, - - createMediaStreamSource: () => ({ - connect() {}, - disconnect() {}, - }), - - createAnalyser: () => ({ - fftSize: 1024, - getByteTimeDomainData() {}, - }), - - close: async () => {}, - }; - - const stop = startAudioActivityMonitor( - { kind: "audio", readyState: "live" }, - () => {}, - { - createContext: () => context, - createStream: () => ({}), - schedule: () => 1, - cancel: () => {}, - }, - ); - - assert.equal(resumes, 1); - + const m = monitorFixture({ state: "suspended" }); + const stop = startAudioActivityMonitor(m.track, () => {}, m.options); + assert.equal(m.resumes(), 1); stop(); }); test("audio monitor closes its context when setup fails", async () => { - let closes = 0; - - const context = { - state: "running", - - createMediaStreamSource() { - throw new Error("Cannot create media source"); - }, - - close: async () => { - closes++; - }, - }; - + const m = monitorFixture({ setupError: "Cannot create media source" }); assert.throws( - () => - startAudioActivityMonitor( - { kind: "audio", readyState: "live" }, - () => {}, - { - createContext: () => context, - createStream: () => ({}), - schedule: () => 1, - cancel: () => {}, - }, - ), + () => startAudioActivityMonitor(m.track, () => {}, m.options), /Cannot create media source/, ); - - // close() is asynchronous, so allow its promise to settle. await new Promise((resolve) => setImmediate(resolve)); - - assert.equal(closes, 1); + assert.equal(m.closes(), 1); }); test("audio monitor resumes on a gesture and removes listeners on stop", async () => { - const gestures = new EventTarget(); - let resumes = 0; - let closes = 0; - - const context = { + const m = monitorFixture({ state: "suspended", - - resume: async () => { - resumes++; - context.state = "running"; - }, - - createMediaStreamSource: () => ({ - connect() {}, - disconnect() {}, - }), - - createAnalyser: () => ({ - fftSize: 1024, - getByteTimeDomainData(samples) { - samples.fill(128); - }, - }), - - close: async () => { - closes++; - }, - }; - - const stop = startAudioActivityMonitor( - { kind: "audio", readyState: "live" }, - () => {}, - { - createContext: () => context, - createStream: () => ({}), - gestureTarget: gestures, - schedule: () => 1, - cancel: () => {}, - }, - ); - - // The initial resume is attempted during setup. - assert.equal(resumes, 1); - - // Simulate the browser suspending the context again. - context.state = "suspended"; - - gestures.dispatchEvent(new Event("pointerdown")); - - assert.equal(resumes, 2); - + resumeState: "running", + peak: 0, + }); + const stop = startAudioActivityMonitor(m.track, () => {}, m.options); + assert.equal(m.resumes(), 1); + m.context.state = "suspended"; + m.gestures.dispatchEvent(new Event("pointerdown")); + assert.equal(m.resumes(), 2); stop(); - - // Stopped monitors must no longer react to gestures. - context.state = "suspended"; - gestures.dispatchEvent(new Event("keydown")); - - assert.equal(resumes, 2); - + m.context.state = "suspended"; + m.gestures.dispatchEvent(new Event("keydown")); + assert.equal(m.resumes(), 2); await new Promise((resolve) => setImmediate(resolve)); - assert.equal(closes, 1); + assert.equal(m.closes(), 1); + assert.equal(m.disconnections(), 1); + assert.equal(m.cancellations(), 1); }); test("monitor interruption does not restart the same speaking turn", () => { const f = fixture(); @@ -466,7 +251,9 @@ test("monitor interruption does not restart the same speaking turn", () => { // A genuine new speaking transition may be measured. f.timing.speakingChanged(false); + f.timing.audioQuiet(); f.timing.speakingChanged(true); + f.timing.audioQuiet(); f.advance(120); f.timing.audioActivity(); @@ -475,75 +262,367 @@ test("monitor interruption does not restart the same speaking turn", () => { }); test("hidden pages invalidate audio timing and cleanup visibility listeners", () => { - const visibilityTarget = new EventTarget(); - visibilityTarget.hidden = false; - - let tick; + const m = monitorFixture(); let unavailable = 0; let available = 0; let activity = 0; + const stop = startAudioActivityMonitor(m.track, () => activity++, { + ...m.options, + onAvailable: () => available++, + onUnavailable: () => unavailable++, + }); + m.tick(); + assert.equal(available, 1); + assert.equal(activity, 1); + m.visibility.hidden = true; + m.visibility.dispatchEvent(new Event("visibilitychange")); + assert.equal(unavailable, 1); + m.tick(); + assert.equal(activity, 1); + assert.equal(unavailable, 1); + m.visibility.hidden = false; + m.visibility.dispatchEvent(new Event("visibilitychange")); + m.tick(); + assert.equal(available, 2); + assert.equal(activity, 2); + stop(); + m.visibility.hidden = true; + m.visibility.dispatchEvent(new Event("visibilitychange")); + assert.equal(unavailable, 1); +}); +function monitorFixture({ + state = "running", + readyState = "live", + peak = 0.5, + resumeState = state, + setupError = null, +} = {}) { + let frame; + let scheduled = 0; + let closes = 0; + let resumes = 0; + let disconnections = 0; + let cancellations = 0; + const sampleLengths = []; + const streams = []; + const track = { kind: "audio", readyState, muted: false, enabled: true }; + const visibility = new EventTarget(); + visibility.hidden = false; + const gestures = new EventTarget(); + const analyser = { + fftSize: 1024, + get frequencyBinCount() { + return this.fftSize / 2; + }, + getByteTimeDomainData(samples) { + assert.equal( + context.state, + "running", + "unavailable contexts must not be sampled", + ); + sampleLengths.push(samples.length); + samples.fill(128); + samples[0] = 128 + Math.round(peak * 128); + }, + }; const context = { - state: "running", - createMediaStreamSource: () => ({ - connect() {}, - disconnect() {}, - }), - createAnalyser: () => ({ - fftSize: 1024, - getByteTimeDomainData(samples) { - samples.fill(128); - samples[0] = 192; - }, - }), - close: async () => {}, + state, + createMediaStreamSource(stream) { + streams.push(stream); + if (setupError) throw new Error(setupError); + return { + connect(target) { + assert.equal(target, analyser); + }, + disconnect() { + disconnections++; + }, + }; + }, + createAnalyser: () => analyser, + resume: async () => { + resumes++; + context.state = resumeState; + }, + close: async () => { + closes++; + }, }; - - const stop = startAudioActivityMonitor( - { kind: "audio", readyState: "live" }, - () => activity++, - { - createContext: () => context, - createStream: () => ({}), - visibilityTarget, - onAvailable: () => available++, - onUnavailable: () => unavailable++, - schedule: (callback) => { - tick = callback; - return 1; - }, - cancel: () => {}, + const options = { + createContext: () => context, + createStream: (mediaTrack) => ({ getTracks: () => [mediaTrack] }), + schedule: (callback) => { + frame = callback; + return ++scheduled; }, - ); - - tick(); + cancel: () => { + cancellations++; + }, + visibilityTarget: visibility, + gestureTarget: gestures, + }; + return { + track, + context, + visibility, + gestures, + options, + sampleLengths, + streams, + tick: () => frame(), + setPeak: (value) => { + peak = value; + }, + closes: () => closes, + resumes: () => resumes, + disconnections: () => disconnections, + cancellations: () => cancellations, + }; +} - assert.equal(available, 1); - assert.equal(activity, 1); +test("audio already active before speaking never becomes a positive gap", () => { + const f = fixture(); + f.timing.audioActivity(); + f.timing.speakingChanged(true); + f.advance(16); + f.timing.audioActivity(); + assert.deepEqual(f.measurements, []); + assert.deepEqual(f.diagnostics, ["audio already active"]); +}); - visibilityTarget.hidden = true; - visibilityTarget.dispatchEvent(new Event("visibilitychange")); +test("the first active frame cannot substitute for a post-transition quiet baseline", () => { + const f = fixture(); + f.timing.speakingChanged(true); + f.advance(16); + f.timing.audioActivity(); + assert.deepEqual(f.measurements, []); + assert.deepEqual(f.diagnostics, [ + "audio already active before a quiet frame", + ]); +}); - assert.equal(unavailable, 1); +test("only a quiet frame after speaking arms a positive measurement", () => { + const f = fixture(); + assert.equal(typeof f.timing.audioQuiet, "function"); + f.timing.audioQuiet(); + f.timing.speakingChanged(true); + f.advance(16); + f.timing.audioActivity(); + assert.deepEqual(f.measurements, []); + f.timing.audioQuiet(); + f.timing.audioActivity(); + assert.deepEqual(f.measurements, [], "a dropped turn stays dropped"); + f.timing.speakingChanged(false); + f.timing.audioQuiet(); + f.timing.speakingChanged(true); + f.advance(20); + f.timing.audioQuiet(); + f.advance(80); + f.timing.audioActivity(); + assert.deepEqual( + f.measurements, + [100], + "elapsed time still starts at the state transition", + ); +}); - // Even if the frame loop runs, hidden audio must not be counted. - tick(); +for (const [property, value, reason] of [ + ["muted", true, "track muted"], + ["enabled", false, "track disabled"], +]) { + test(`a ${property} interruption cancels the turn instead of timing recovery`, () => { + const f = fixture(); + const m = monitorFixture({ peak: 0 }); + const unavailable = []; + const stop = startAudioActivityMonitor(m.track, f.timing.audioActivity, { + ...m.options, + onQuiet: f.timing.audioQuiet, + onUnavailable: (reason) => { + unavailable.push(reason); + f.timing.cancelPending(reason); + }, + }); + m.tick(); + f.timing.speakingChanged(true); + m.tick(); + m.track[property] = value; + m.tick(); + m.tick(); + assert.deepEqual(unavailable, [reason]); + assert.deepEqual(f.diagnostics, [reason]); + m.track[property] = !value; + m.setPeak(0.5); + f.advance(200); + m.tick(); + assert.deepEqual(f.measurements, []); + m.setPeak(0); + m.tick(); + f.timing.speakingChanged(false); + f.timing.speakingChanged(true); + m.tick(); + f.advance(80); + m.setPeak(0.5); + m.tick(); + assert.deepEqual(f.measurements, [80]); + stop(); + }); +} +test("the monitor supplies quiet frames without reporting activity", () => { + const m = monitorFixture({ peak: 0 }); + let quiet = 0; + let activity = 0; + const stop = startAudioActivityMonitor(m.track, () => activity++, { + ...m.options, + onQuiet: () => quiet++, + }); + m.tick(); + m.setPeak(0.5); + m.tick(); + m.setPeak(0); + m.tick(); + assert.equal(quiet, 2); assert.equal(activity, 1); - assert.equal(unavailable, 1); + stop(); +}); + +test("skipped transitions and canceled turns explain themselves once", () => { + const f = fixture(); + f.timing.speakingChanged(true, "monitor not ready"); + f.timing.speakingChanged(true, "monitor not ready"); + f.timing.cancelPending("audio context suspended"); + assert.deepEqual(f.diagnostics, ["monitor not ready"]); + f.timing.speakingChanged(false); + f.timing.speakingChanged(true); + f.timing.cancelPending("page hidden"); + f.timing.cancelPending("page hidden"); + f.timing.audioActivity(); + assert.deepEqual(f.measurements, []); + assert.deepEqual(f.diagnostics, ["monitor not ready", "page hidden"]); +}); - visibilityTarget.hidden = false; - visibilityTarget.dispatchEvent(new Event("visibilitychange")); - tick(); +test("stopping speech and resetting explain pending measurements", () => { + const f = fixture(); + f.timing.speakingChanged(true); + f.timing.speakingChanged(false); + f.timing.speakingChanged(true); + f.timing.reset("room disconnected"); + assert.deepEqual(f.diagnostics, [ + "speaking stopped before audio", + "room disconnected", + ]); +}); - assert.equal(available, 2); - assert.equal(activity, 2); +for (const [name, peak, expected] of [ + ["silent", 0, []], + ["below threshold", 7 / 128, []], + ["at threshold", 8 / 128, []], + ["above threshold", 9 / 128, [80]], +]) { + test(`${name} frames obey the strict activity threshold`, () => { + const f = fixture(); + const m = monitorFixture({ peak: 0 }); + const stop = startAudioActivityMonitor(m.track, f.timing.audioActivity, { + ...m.options, + threshold: 8 / 128, + onQuiet: f.timing.audioQuiet, + }); + m.tick(); + f.timing.speakingChanged(true); + m.tick(); + m.setPeak(peak); + f.advance(80); + m.tick(); + m.tick(); + assert.deepEqual(f.measurements, expected); + m.setPeak(0); + m.tick(); + assert.deepEqual( + f.measurements, + expected, + "quiet frames never add a result", + ); + stop(); + }); +} - stop(); +test("microphone and remote timing meters retain separate sources and lifetimes", () => { + const f = fixture(); + const mic = monitorFixture(); + const remote = monitorFixture({ peak: 0 }); + const peaks = []; + const stopMic = startMediaMeter( + mic.options.createStream(mic.track), + (peak) => peaks.push(peak), + (error) => assert.fail(error), + () => false, + mic.options, + ); + const stopRemote = startAudioActivityMonitor( + remote.track, + f.timing.audioActivity, + { ...remote.options, onQuiet: f.timing.audioQuiet }, + ); + assert.equal(mic.streams[0].getTracks()[0], mic.track); + assert.equal(remote.streams[0].getTracks()[0], remote.track); + remote.tick(); + f.timing.speakingChanged(true); + remote.tick(); + mic.tick(); + assert.deepEqual(peaks, [0.5, 0.5]); + assert.deepEqual( + f.measurements, + [], + "microphone activity cannot satisfy remote timing", + ); + f.advance(80); + remote.setPeak(0.5); + remote.tick(); + assert.deepEqual(f.measurements, [80]); + stopRemote(); + assert.equal(remote.closes(), 1); + assert.equal(mic.closes(), 0); + mic.tick(); + assert.equal(peaks.length, 3); + stopMic(); + assert.equal(mic.closes(), 1); +}); - // Stopped monitors must not react to later visibility changes. - visibilityTarget.hidden = true; - visibilityTarget.dispatchEvent(new Event("visibilitychange")); +test("failed monitor setup leaves no gesture or visibility listeners", () => { + const m = monitorFixture({ + state: "suspended", + setupError: "Cannot create media source", + }); + let unavailable = 0; + assert.throws( + () => + startAudioActivityMonitor(m.track, () => {}, { + ...m.options, + onUnavailable: () => unavailable++, + }), + /Cannot create media source/, + ); + const resumes = m.resumes(); + m.gestures.dispatchEvent(new Event("pointerdown")); + m.gestures.dispatchEvent(new Event("keydown")); + m.visibility.hidden = true; + m.visibility.dispatchEvent(new Event("visibilitychange")); + assert.equal(m.resumes(), resumes); + assert.equal(unavailable, 0); + assert.equal(m.closes(), 1); +}); - assert.equal(unavailable, 1); +test("a synchronous resume failure releases the shared meter before throwing", () => { + const m = monitorFixture({ state: "suspended" }); + m.context.resume = () => { + throw new Error("Resume failed"); + }; + assert.throws( + () => startAudioActivityMonitor(m.track, () => {}, m.options), + /Resume failed/, + ); + assert.equal(m.closes(), 1); + assert.equal(m.disconnections(), 1); + assert.equal(m.cancellations(), 1); }); diff --git a/tests/browser/mic-meter.test.js b/tests/browser/mic-meter.test.js index d0dfc9582..4084fc14d 100644 --- a/tests/browser/mic-meter.test.js +++ b/tests/browser/mic-meter.test.js @@ -13,7 +13,8 @@ import { MIC_CONFIRM_FRAMES, preflightReadiness, } from "../../web/audio-check.js"; -import { createMicMeter } from "../../web/mic-meter.js"; +import { createMicMeter, startMediaMeter } from "../../web/mic-meter.js"; +import { stubGlobal } from "./source.js"; /// A pool with one audio track, whose readiness a test can change. function fakePool(audioState = "live") { @@ -293,3 +294,94 @@ test("a dead device of either kind reopens the request the pool makes", () => { ); } }); + +test("the real media meter samples its own stream immediately and stops on the next frame", async (t) => { + let tick; + let stopped = false; + let closes = 0; + const stream = { label: "microphone stream" }; + const streams = []; + const sampleLengths = []; + const peaks = []; + const errors = []; + const analyser = { + fftSize: 0, + get frequencyBinCount() { + return this.fftSize / 2; + }, + getByteTimeDomainData(samples) { + sampleLengths.push(samples.length); + samples.fill(128); + samples[511] = 192; + }, + }; + stubGlobal( + t, + "AudioContext", + class { + createAnalyser() { + return analyser; + } + createMediaStreamSource(input) { + streams.push(input); + return { + connect(target) { + assert.equal(target, analyser); + }, + disconnect() {}, + }; + } + async close() { + closes++; + } + }, + ); + stubGlobal(t, "requestAnimationFrame", (callback) => { + tick = callback; + return 1; + }); + stubGlobal(t, "cancelAnimationFrame", () => {}); + const stop = await startMediaMeter( + stream, + (peak) => peaks.push(peak), + (error) => errors.push(error), + () => stopped, + ); + assert.equal( + typeof stop, + "function", + "the shared loop exposes explicit cleanup", + ); + assert.deepEqual(streams, [stream]); + assert.equal(analyser.fftSize, 1024); + assert.deepEqual(sampleLengths, [512]); + assert.deepEqual(peaks, [0.5]); + tick(); + assert.deepEqual(peaks, [0.5, 0.5]); + stopped = true; + tick(); + stop(); + assert.equal(closes, 1, "cleanup is idempotent after shouldStop"); + assert.deepEqual(peaks, [0.5, 0.5]); + assert.deepEqual(errors, []); +}); + +test("the real media meter reports setup failures as strings", async (t) => { + stubGlobal( + t, + "AudioContext", + class { + constructor() { + throw new Error("No audio context"); + } + }, + ); + const errors = []; + await startMediaMeter( + {}, + () => assert.fail("no sample without a context"), + (error) => errors.push(error), + () => false, + ); + assert.deepEqual(errors, ["No audio context"]); +}); diff --git a/web/audio-response-timing.js b/web/audio-response-timing.js index 228127695..9075e57c2 100644 --- a/web/audio-response-timing.js +++ b/web/audio-response-timing.js @@ -1,27 +1,40 @@ -import { peakLevel } from "./audio-check.js"; +import { startMediaMeter } from "./mic-meter.js"; /// Browser-observed gap between Jim entering "speaking" and the first /// detectable remote audio activity. This is not end-to-end latency. export function createAudioResponseTiming({ now = () => performance.now(), report = () => {}, + diagnose = () => {}, } = {}) { let speaking = false; let startedAt = null; + let active = false; + let armed = false; - function speakingChanged(nextSpeaking) { + function speakingChanged(nextSpeaking, unavailableReason = null) { if (nextSpeaking === speaking) return; speaking = nextSpeaking; if (speaking) { - startedAt = now(); + if (unavailableReason || active) { + diagnose(unavailableReason || "audio already active"); + } else { + startedAt = now(); + armed = false; + } } else { - startedAt = null; + cancelPending("speaking stopped before audio"); } } function audioActivity() { + active = true; if (!speaking || startedAt === null) return; + if (!armed) { + cancelPending("audio already active before a quiet frame"); + return; + } const elapsedMs = now() - startedAt; startedAt = null; @@ -29,18 +42,28 @@ export function createAudioResponseTiming({ report(elapsedMs); } - function cancelPending() { + function audioQuiet() { + active = false; + // A pre-transition quiet frame cannot rule out audio arriving first. + if (startedAt !== null) armed = true; + } + + function cancelPending(reason = "measurement canceled") { + if (startedAt !== null) diagnose(reason); startedAt = null; + armed = false; } - function reset() { + function reset(reason = "timing reset") { speaking = false; - cancelPending(); + active = false; + cancelPending(reason); } return { speakingChanged, audioActivity, + audioQuiet, cancelPending, reset, }; @@ -54,6 +77,7 @@ export function startAudioActivityMonitor( threshold = 0.05, onAvailable = () => {}, onUnavailable = () => {}, + onQuiet = () => {}, createContext = () => new AudioContext(), createStream = (mediaTrack) => new MediaStream([mediaTrack]), schedule = requestAnimationFrame, @@ -62,83 +86,75 @@ export function startAudioActivityMonitor( visibilityTarget = globalThis.document, } = {}, ) { + const stream = createStream(track); const context = createContext(); - let source; - let analyser; - - try { - if (context.state === "suspended") { - void context.resume?.().catch(() => {}); - } - - source = context.createMediaStreamSource(createStream(track)); - analyser = context.createAnalyser(); - - analyser.fftSize = 1024; - source.connect(analyser); - } catch (error) { - source?.disconnect(); - void context.close?.().catch(() => {}); - throw error; - } - - const samples = new Uint8Array(analyser.fftSize); - let stopped = false; - let frame = null; - let unavailable = null; + let unavailable; - function notifyAvailability(available) { - const nextUnavailable = !available; - - if (unavailable === nextUnavailable) return; - - unavailable = nextUnavailable; - - if (available) onAvailable(); - else onUnavailable(); + function notifyAvailability(reason) { + if (unavailable === reason) return; + unavailable = reason; + if (reason === null) onAvailable(); + else onUnavailable(reason); } function visibilityChanged() { if (stopped) return; if (visibilityTarget?.hidden) { - notifyAvailability(false); + notifyAvailability("page hidden"); } } - visibilityTarget?.addEventListener("visibilitychange", visibilityChanged); - function resumeOnGesture() { if (stopped || context.state !== "suspended") return; void context.resume?.().catch(() => {}); } + const stopMeter = startMediaMeter( + stream, + (peak) => { + if (peak > threshold) onActivity(); + else onQuiet(); + }, + (message) => { + throw new Error(message); + }, + () => stopped, + { + createContext: () => context, + schedule, + cancel, + sampleSize: 1024, + deferFirstFrame: true, + shouldSample: () => { + const reason = visibilityTarget?.hidden + ? "page hidden" + : context.state !== "running" + ? `audio context ${context.state}` + : track.readyState !== "live" + ? "track ended" + : track.muted + ? "track muted" + : track.enabled === false + ? "track disabled" + : null; + + notifyAvailability(reason); + return reason === null; + }, + }, + ); + try { + resumeOnGesture(); + } catch (error) { + stopMeter(); + throw error; + } + visibilityTarget?.addEventListener("visibilitychange", visibilityChanged); gestureTarget?.addEventListener("pointerdown", resumeOnGesture); gestureTarget?.addEventListener("keydown", resumeOnGesture); - function tick() { - if (stopped) return; - - const available = - context.state === "running" && - track.readyState === "live" && - !visibilityTarget?.hidden; - - notifyAvailability(available); - - if (available) { - analyser.getByteTimeDomainData(samples); - - if (peakLevel(samples) > threshold) { - onActivity(); - } - } - - frame = schedule(tick); - } - frame = schedule(tick); - return () => { if (stopped) return; @@ -152,9 +168,6 @@ export function startAudioActivityMonitor( visibilityChanged, ); - cancel(frame); - source.disconnect(); - - void context.close().catch(() => {}); + stopMeter(); }; } diff --git a/web/interview.js b/web/interview.js index 128feda41..3def871fa 100644 --- a/web/interview.js +++ b/web/interview.js @@ -344,6 +344,8 @@ const params = new URLSearchParams(window.location.search); const audioTimingEnabled = params.get("audioTiming") === "1"; const audioTiming = audioTimingEnabled ? createAudioResponseTiming({ + diagnose: (reason) => + console.debug(`[audio-timing] Turn not measured: ${reason}`), report: (elapsedMs) => { console.info( `[audio-timing] Browser-observed speaking-to-audio gap: ${elapsedMs.toFixed(1)} ms`, @@ -353,6 +355,7 @@ const audioTiming = audioTimingEnabled : null; let audioTimingTrack = null; +let audioTimingRoom = null; let stopAudioTimingMonitor = null; let audioTimingReady = false; const editorOptions = editorOptionsFromParams(params); @@ -2416,7 +2419,7 @@ async function connectLiveKit(connection, preflight, presenting = false) { // on its own, so the candidate is told to wait rather than to restart. room.on(livekit.RoomEvent.Reconnecting, () => { if (state.room !== room) return; - releaseAudioTiming(); + releaseAudioTiming(null, "room reconnecting"); state.connected = false; if (strict) updateRunAvailability(); setBanner("connection", providerUiState("reconnecting").message); @@ -2458,8 +2461,8 @@ async function connectLiveKit(connection, preflight, presenting = false) { }); room.on(livekit.RoomEvent.Disconnected, () => { audioPlayback.release(room); - if (state.room === room) { - releaseAudioTiming(); + if (state.room === room || (!state.room && audioTimingRoom === room)) { + releaseAudioTiming(null, "room disconnected"); audioTiming?.reset(); } const startupCleanup = state.startupCleanupRoom === room; @@ -3035,7 +3038,7 @@ async function receiveReport(room, payload, flushed = Promise.resolve()) { reportRenderFailed(error, { offlineSummary: !renderAttempted }); } } -function releaseAudioTiming(track = null) { +function releaseAudioTiming(track = null, reason = "audio monitor released") { if (!audioTiming) return; // A late unsubscribe from an old track must not remove the new monitor. @@ -3045,21 +3048,23 @@ function releaseAudioTiming(track = null) { stopAudioTimingMonitor = null; audioTimingTrack = null; + audioTimingRoom = null; audioTimingReady = false; - audioTiming.cancelPending(); + audioTiming.cancelPending(reason); } function attachAudioTiming(track) { if (!audioTiming) return; if (audioTimingTrack === track) return; - releaseAudioTiming(); + releaseAudioTiming(null, "audio track replaced"); const mediaTrack = track.mediaStreamTrack; if (!mediaTrack) return; audioTimingTrack = track; + audioTimingRoom = state.room; try { stopAudioTimingMonitor = startAudioActivityMonitor( @@ -3073,10 +3078,13 @@ function attachAudioTiming(track) { if (audioTimingTrack !== track) return; audioTimingReady = true; }, - onUnavailable: () => { + onQuiet: () => { + if (audioTimingTrack === track) audioTiming.audioQuiet(); + }, + onUnavailable: (reason) => { if (audioTimingTrack !== track) return; audioTimingReady = false; - audioTiming.cancelPending(); + audioTiming.cancelPending(reason); }, }, ); @@ -5378,7 +5386,7 @@ function updateAgentState() { return; } if (!agent) { - audioTiming?.reset(); + audioTiming?.reset("interviewer unavailable"); setAgentStateLabel("Waiting", false); // "Waiting" is honest but useless on its own: it looks identical whether // the interviewer has not arrived yet or arrived, greeted, and died. Once @@ -5460,13 +5468,14 @@ function updateAgentState() { state.agentSpeaking = value === "speaking"; state.agentThinking = value === "thinking"; if (audioTiming) { - audioTiming.speakingChanged(published === "speaking"); - - // A speaking transition is measurable only if the monitor was - // already observing audio when the transition occurred. - if (!audioTimingReady || !state.connected) { - audioTiming.cancelPending(); - } + audioTiming.speakingChanged( + published === "speaking", + !state.connected + ? "room disconnected" + : !audioTimingReady + ? "monitor not ready" + : null, + ); } if (state.agentSpeaking || state.agentThinking) { drawingTurn?.reset(); diff --git a/web/mic-meter.js b/web/mic-meter.js index 136807714..1a3961cca 100644 --- a/web/mic-meter.js +++ b/web/mic-meter.js @@ -15,27 +15,60 @@ import { MIC_CONFIRM_FRAMES, peakLevel, sustainedPeak } from "./audio-check.js"; /// /// Injected rather than called directly, the way `createFaceCheck` takes its /// detector, because a test cannot build one of these and the thing worth -/// testing is not the analyser. -export async function startMediaMeter(stream, onPeak, onError, shouldStop) { +/// testing is not the analyser. The returned stop function also releases it +/// without waiting for a frame in a hidden tab. +export function startMediaMeter( + stream, + onPeak, + onError, + shouldStop, + { + createContext = () => new AudioContext(), + schedule = (callback) => requestAnimationFrame(callback), + cancel = (frame) => cancelAnimationFrame(frame), + shouldSample = () => true, + sampleSize = null, + deferFirstFrame = false, + } = {}, +) { + let context; + let source; + let frame = null; + let stopped = false; + const stop = () => { + if (stopped) return; + stopped = true; + if (frame !== null) cancel(frame); + source?.disconnect(); + void context?.close().catch(() => {}); + }; try { - const context = new AudioContext(); + context = createContext(); const analyser = context.createAnalyser(); analyser.fftSize = 1024; - context.createMediaStreamSource(stream).connect(analyser); - const samples = new Uint8Array(analyser.frequencyBinCount); + source = context.createMediaStreamSource(stream); + source.connect(analyser); + const samples = new Uint8Array(sampleSize ?? analyser.frequencyBinCount); const tick = () => { + frame = null; + if (stopped) return; if (shouldStop()) { - void context.close().catch(() => {}); + stop(); return; } - analyser.getByteTimeDomainData(samples); - onPeak(peakLevel(samples)); - requestAnimationFrame(tick); + if (shouldSample()) { + analyser.getByteTimeDomainData(samples); + onPeak(peakLevel(samples)); + } + if (!stopped) frame = schedule(tick); }; - tick(); + if (deferFirstFrame) frame = schedule(tick); + else tick(); } catch (error) { + stop(); onError(String(error?.message || error)); } + return stop; } /// One meter at a time. The pool can hand back a replacement microphone, and From 30943414e126e21602d3b2ed2c76bc39430cc18b Mon Sep 17 00:00:00 2001 From: ChenYingHan1121 <246545182+ChenYingHan1121@users.noreply.github.com> Date: Tue, 17 Nov 2026 13:39:12 +0800 Subject: [PATCH 3/4] Test audio timing error recovery Add integration tests for monitor setup failures and interviewer unavailability. Verify that both paths clear stale timing state and allow a later speaking turn to be measured. These tests cover the recovery paths identified during PR #728 review and guard against regressions in cleanup and reset logic. Change-Id: Ie603ddde2af7aa5c4650733f6df71464d0b08a8e --- .../audio-response-timing-integration.test.js | 77 ++++++++++++++++++- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/tests/browser/audio-response-timing-integration.test.js b/tests/browser/audio-response-timing-integration.test.js index 62592c477..b9fc2e5f7 100644 --- a/tests/browser/audio-response-timing-integration.test.js +++ b/tests/browser/audio-response-timing-integration.test.js @@ -4,7 +4,7 @@ import { runInNewContext } from "node:vm"; import { createAudioResponseTiming } from "../../web/audio-response-timing.js"; import { functionBody, loadInterview, read } from "./source.js"; -async function fixture({ enabled = true } = {}) { +async function fixture({ enabled = true, failFirstMonitor = false } = {}) { const page = read("web/interview.js"); const monitors = []; const measurements = []; @@ -12,6 +12,8 @@ async function fixture({ enabled = true } = {}) { const playbackReleases = []; let cancellations = 0; let time = 0; + let monitorAttempts = 0; + let interviewerPresent = true; const timing = createAudioResponseTiming({ now: () => time, report: (elapsed) => measurements.push(elapsed), @@ -58,6 +60,9 @@ async function fixture({ enabled = true } = {}) { } : null, startAudioActivityMonitor(track, onActivity, options) { + monitorAttempts++; + if (failFirstMonitor && monitorAttempts === 1) + throw new Error("monitor setup failed"); const monitor = { track, onActivity, options, stopped: false }; monitors.push(monitor); return () => { @@ -73,7 +78,9 @@ async function fixture({ enabled = true } = {}) { localRecording: { needsMicrophone: false }, stopPreflight() {}, dropRemoteAudio() {}, - playRemoteAudio() {}, + playRemoteAudio(track) { + context.timingApi.setRemoteTrack(track); + }, publishPreflightTracks: async () => {}, paintMicrophoneState() {}, monitorIntegrityTracks() {}, @@ -105,13 +112,14 @@ async function fixture({ enabled = true } = {}) { `; runInNewContext(code, context); const scope = { + Boolean, state, audioTiming: context.audioTiming, get audioTimingReady() { return context.timingApi.ready(); }, roomParticipants: () => [agent], - roomInterviewer: () => agent, + roomInterviewer: () => (interviewerPresent ? agent : null), }; const { updateAgentState } = loadInterview(["updateAgentState"], scope, { stubMissing: true, @@ -146,6 +154,10 @@ async function fixture({ enabled = true } = {}) { api: context.timingApi, monitors, cancellations: () => cancellations, + monitorAttempts: () => monitorAttempts, + setInterviewerPresent: (present) => { + interviewerPresent = present; + }, }; } @@ -436,3 +448,62 @@ test("callbacks from a replaced track cannot cancel or measure the current track current.onActivity(); assert.deepEqual(f.measurements, [80]); }); + +test("monitor setup failure clears the track and reconnect retries the diagnostic", async () => { + const f = await fixture({ failFirstMonitor: true }); + const track = remoteTrack("A"); + f.agent.attributes["lk.agent.state"] = "listening"; + f.room.emit("TrackSubscribed", track, {}, f.agent); + assert.equal(f.monitorAttempts(), 1); + assert.equal(f.monitors.length, 0); + assert.equal(f.api.currentTrack(), null); + assert.equal(f.api.owner(), null); + assert.equal(f.api.ready(), false); + + f.room.emit("Reconnected"); + assert.equal(f.monitorAttempts(), 2); + assert.equal(f.monitors.length, 1); + assert.equal(f.api.currentTrack(), track); + assert.equal(f.api.owner(), f.room); + const monitor = f.monitors[0]; + monitor.options.onAvailable(); + assert.equal(f.api.ready(), true); + f.agent.attributes["lk.agent.state"] = "speaking"; + f.room.emit("ParticipantAttributesChanged"); + monitor.options.onQuiet(); + f.advance(80); + monitor.onActivity(); + assert.deepEqual(f.measurements, [80]); + f.disconnect(); +}); + +test("an unavailable interviewer cancels a pending turn and can speak again", async () => { + const f = await fixture(); + f.agent.attributes["lk.agent.state"] = "listening"; + f.room.emit("TrackSubscribed", remoteTrack("A"), {}, f.agent); + const monitor = f.monitors[0]; + monitor.options.onAvailable(); + f.agent.attributes["lk.agent.state"] = "speaking"; + f.room.emit("ParticipantAttributesChanged"); + monitor.options.onQuiet(); + f.advance(40); + + f.setInterviewerPresent(false); + f.room.emit("ParticipantAttributesChanged"); + assert.equal(f.state.interviewerPresent, false); + assert.deepEqual(f.diagnostics, ["interviewer unavailable"]); + f.advance(80); + monitor.onActivity(); + assert.deepEqual(f.measurements, []); + + monitor.options.onQuiet(); + f.setInterviewerPresent(true); + f.room.emit("ParticipantConnected", f.agent); + assert.equal(f.state.interviewerPresent, true); + monitor.options.onQuiet(); + f.advance(60); + monitor.onActivity(); + assert.deepEqual(f.measurements, [60]); + assert.deepEqual(f.diagnostics, ["interviewer unavailable"]); + f.disconnect(); +}); From 054cb1f83a68bc0c63efa1d6a97cfdee8a6dbeac Mon Sep 17 00:00:00 2001 From: ChenYingHan1121 <246545182+ChenYingHan1121@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:22:22 +0800 Subject: [PATCH 4/4] Test interviewer disconnect recovery Exercise the registered ParticipantDisconnected callback in the interviewer-unavailable integration test, including its deferred cleanup through the queued microtask. Provide the agent and participant roster helpers required by the event-handler fixture. Keep the existing recovery assertions and avoid changes to production code. Change-Id: Ife47337809f31dd1dce206efe50713e1fd9b647c --- tests/browser/audio-response-timing-integration.test.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/browser/audio-response-timing-integration.test.js b/tests/browser/audio-response-timing-integration.test.js index b9fc2e5f7..b6aacb79c 100644 --- a/tests/browser/audio-response-timing-integration.test.js +++ b/tests/browser/audio-response-timing-integration.test.js @@ -40,6 +40,9 @@ async function fixture({ enabled = true, failFirstMonitor = false } = {}) { }; const context = { state, + isAgent: (participant) => participant === agent, + roomParticipants: () => (interviewerPresent ? [agent] : []), + queueMicrotask, window: { LivekitClient: { Room, @@ -118,7 +121,8 @@ async function fixture({ enabled = true, failFirstMonitor = false } = {}) { get audioTimingReady() { return context.timingApi.ready(); }, - roomParticipants: () => [agent], + isAgent: (participant) => participant === agent, + roomParticipants: () => (interviewerPresent ? [agent] : []), roomInterviewer: () => (interviewerPresent ? agent : null), }; const { updateAgentState } = loadInterview(["updateAgentState"], scope, { @@ -489,7 +493,8 @@ test("an unavailable interviewer cancels a pending turn and can speak again", as f.advance(40); f.setInterviewerPresent(false); - f.room.emit("ParticipantAttributesChanged"); + f.room.emit("ParticipantDisconnected", f.agent); + await Promise.resolve(); assert.equal(f.state.interviewerPresent, false); assert.deepEqual(f.diagnostics, ["interviewer unavailable"]); f.advance(80);