diff --git a/tests/browser/audio-response-timing-integration.test.js b/tests/browser/audio-response-timing-integration.test.js new file mode 100644 index 000000000..b6aacb79c --- /dev/null +++ b/tests/browser/audio-response-timing-integration.test.js @@ -0,0 +1,514 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { runInNewContext } from "node:vm"; +import { createAudioResponseTiming } from "../../web/audio-response-timing.js"; +import { functionBody, loadInterview, read } from "./source.js"; + +async function fixture({ enabled = true, failFirstMonitor = false } = {}) { + const page = read("web/interview.js"); + const monitors = []; + const measurements = []; + const diagnostics = []; + const playbackReleases = []; + let cancellations = 0; + let time = 0; + let monitorAttempts = 0; + let interviewerPresent = true; + const timing = createAudioResponseTiming({ + now: () => time, + report: (elapsed) => measurements.push(elapsed), + diagnose: (reason) => diagnostics.push(reason), + }); + class Room { + handlers = new Map(); + on(event, callback) { + const handlers = this.handlers.get(event) || []; + handlers.push(callback); + this.handlers.set(event, handlers); + return this; + } + emit(event, ...args) { + for (const callback of this.handlers.get(event) || []) callback(...args); + } + async connect() {} + registerTextStreamHandler() {} + } + const state = { phase: "live", connected: false, sawAgent: true }; + const agent = { + identity: "agent-test", + attributes: { "lk.agent.state": "listening" }, + }; + const context = { + state, + isAgent: (participant) => participant === agent, + roomParticipants: () => (interviewerPresent ? [agent] : []), + queueMicrotask, + window: { + LivekitClient: { + Room, + RoomEvent: new Proxy({}, { get: (_, event) => event }), + }, + }, + audioPlayback: { + setRoom() {}, + release: (room) => playbackReleases.push(room), + }, + audioTiming: enabled + ? { + ...timing, + cancelPending(reason) { + cancellations++; + timing.cancelPending(reason); + }, + } + : null, + startAudioActivityMonitor(track, onActivity, options) { + monitorAttempts++; + if (failFirstMonitor && monitorAttempts === 1) + throw new Error("monitor setup failed"); + const monitor = { track, onActivity, options, stopped: false }; + monitors.push(monitor); + return () => { + monitor.stopped = true; + }; + }, + strict: false, + whiteboard: false, + topics: {}, + console: { warn() {} }, + nodes: { mutedSpeechReminder: {} }, + mutedSpeech: { stop() {} }, + localRecording: { needsMicrophone: false }, + stopPreflight() {}, + dropRemoteAudio() {}, + playRemoteAudio(track) { + context.timingApi.setRemoteTrack(track); + }, + publishPreflightTracks: async () => {}, + paintMicrophoneState() {}, + monitorIntegrityTracks() {}, + followMicrophoneLoss() {}, + refreshAudioInputs: async () => {}, + publishIntegrityEvent: async () => {}, + startIntegrityHeartbeat() {}, + setBanner() {}, + providerUiState: () => ({ message: "connection unavailable" }), + reconnectCodingInterview: async () => {}, + exampleBoard: {}, + flushPendingPublishes() {}, + publishCode() {}, + }; + const code = ` + let audioTimingTrack = null; + let audioTimingRoom = null; + let stopAudioTimingMonitor = null; + let audioTimingReady = false; + let jimTrack = null; + ${["releaseAudioTiming", "attachAudioTiming", "connectLiveKit"].map((name) => functionBody(page, name) + "\n}").join("\n")} + globalThis.timingApi = { + attachAudioTiming, releaseAudioTiming, connectLiveKit, + setRemoteTrack: (track) => { jimTrack = track; attachAudioTiming(track); }, + currentTrack: () => audioTimingTrack, + owner: () => audioTimingRoom, + ready: () => audioTimingReady, + }; + `; + runInNewContext(code, context); + const scope = { + Boolean, + state, + audioTiming: context.audioTiming, + get audioTimingReady() { + return context.timingApi.ready(); + }, + isAgent: (participant) => participant === agent, + roomParticipants: () => (interviewerPresent ? [agent] : []), + roomInterviewer: () => (interviewerPresent ? agent : null), + }; + const { updateAgentState } = loadInterview(["updateAgentState"], scope, { + stubMissing: true, + }); + context.updateAgentState = updateAgentState; + assert.equal( + await context.timingApi.connectLiveKit({}, { cameraSkipped: true }), + "joined", + ); + const room = state.room; + for (const event of ["Reconnecting", "Reconnected", "Disconnected"]) { + assert.equal( + room.handlers.get(event)?.length, + 1, + `${event} is registered by connectLiveKit`, + ); + } + agent.attributes["lk.agent.state"] = "speaking"; + return { + state, + room, + agent, + timing, + diagnostics, + measurements, + playbackReleases, + advance: (ms) => { + time += ms; + }, + disconnect: () => room.emit("Disconnected"), + updateAgentState, + api: context.timingApi, + monitors, + cancellations: () => cancellations, + monitorAttempts: () => monitorAttempts, + setInterviewerPresent: (present) => { + interviewerPresent = present; + }, + }; +} + +function remoteTrack(id) { + return { + id, + mediaStreamTrack: { + kind: "audio", + readyState: "live", + }, + }; +} + +test("replacing a remote track stops the old monitor", async () => { + const f = await fixture(); + + const trackA = remoteTrack("A"); + const trackB = remoteTrack("B"); + + f.api.attachAudioTiming(trackA); + + assert.equal(f.monitors.length, 1); + assert.equal(f.monitors[0].stopped, false); + + f.api.attachAudioTiming(trackB); + + assert.equal(f.monitors.length, 2); + assert.equal(f.monitors[0].stopped, true); + assert.equal(f.monitors[1].stopped, false); + + assert.equal(f.api.currentTrack(), trackB); + + f.api.releaseAudioTiming(); +}); + +test("late unsubscribe from an old track preserves the new monitor", async () => { + const f = await fixture(); + + const trackA = remoteTrack("A"); + const trackB = remoteTrack("B"); + + f.api.attachAudioTiming(trackA); + f.api.attachAudioTiming(trackB); + + // The old unsubscribe arrives after the replacement is active. + f.api.releaseAudioTiming(trackA); + + assert.equal(f.monitors.length, 2); + assert.equal(f.monitors[1].stopped, false); + assert.equal(f.api.currentTrack(), trackB); + + // Releasing the current track must still work. + f.api.releaseAudioTiming(trackB); + + assert.equal(f.monitors[1].stopped, true); + assert.equal(f.api.currentTrack(), null); +}); + +for (const readyState of ["live", "ended"]) { + test(`registered reconnect handlers respect a retained ${readyState} track`, async () => { + const f = await fixture(); + const track = remoteTrack("A"); + f.api.setRemoteTrack(track); + const old = f.monitors[0]; + old.options.onAvailable(); + f.updateAgentState(); + old.options.onQuiet(); + track.mediaStreamTrack.readyState = readyState; + const cancellations = f.cancellations(); + f.room.emit("Reconnecting"); + assert.equal(f.state.connected, false); + assert.equal(old.stopped, true); + assert.equal(f.api.ready(), false); + assert.equal(f.api.owner(), null); + assert.equal(f.cancellations(), cancellations + 1); + f.room.emit("Reconnected"); + assert.equal(f.state.connected, true); + assert.equal(f.monitors.length, readyState === "live" ? 2 : 1); + assert.equal(f.api.currentTrack(), readyState === "live" ? track : null); + assert.equal(f.api.owner(), readyState === "live" ? f.room : null); + old.options.onAvailable(); + old.options.onQuiet(); + f.advance(200); + old.onActivity(); + if (readyState === "live") { + const current = f.monitors[1]; + current.options.onAvailable(); + current.options.onQuiet(); + current.onActivity(); + assert.equal(current.stopped, false); + } + assert.deepEqual( + f.measurements, + [], + "recovery cannot finish the interrupted turn", + ); + assert.deepEqual(f.diagnostics, ["room reconnecting"]); + f.disconnect(); + assert.equal(f.api.currentTrack(), null); + assert.equal(f.api.owner(), null); + assert.equal(f.api.ready(), false); + assert.ok(f.monitors.every((monitor) => monitor.stopped)); + assert.deepEqual(f.playbackReleases, [f.room]); + }); +} + +test("disabled audio timing creates no monitor", async () => { + const f = await fixture({ enabled: false }); + + const track = remoteTrack("A"); + + f.api.attachAudioTiming(track); + + assert.equal(f.monitors.length, 0); + assert.equal(f.api.currentTrack(), null); + + f.api.releaseAudioTiming(); + + assert.equal(f.monitors.length, 0); +}); + +test("report disconnect releases the owning monitor after state.room is cleared", async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + f.monitors[0].options.onAvailable(); + f.timing.speakingChanged(true); + f.state.room = null; + f.disconnect(); + assert.equal(f.monitors[0].stopped, true); + assert.equal(f.api.currentTrack(), null); + assert.equal(f.api.ready(), false); + assert.deepEqual(f.diagnostics, ["room disconnected"]); +}); + +for (const cleared of [false, true]) { + test(`a stale disconnect preserves the newer room's monitor (cleared=${cleared})`, async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + f.state.room = {}; + const newerTrack = remoteTrack("B"); + f.api.attachAudioTiming(newerTrack); + f.monitors[1].options.onAvailable(); + if (cleared) f.state.room = null; + f.disconnect(); + assert.equal(f.monitors[1].stopped, false); + assert.equal(f.api.currentTrack(), newerTrack); + assert.equal(f.api.ready(), true); + }); +} + +test("speaking updates explain an unready monitor once and never time its recovery", async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + f.updateAgentState(); + f.updateAgentState(); + assert.deepEqual(f.diagnostics, ["monitor not ready"]); + f.monitors[0].options.onAvailable(); + f.advance(200); + f.monitors[0].onActivity(); + assert.deepEqual(f.measurements, []); +}); + +test("an unavailable monitor cancels a pending measurement with its reason", async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + const monitor = f.monitors[0]; + monitor.options.onAvailable(); + f.updateAgentState(); + monitor.options.onQuiet(); + const cancellations = f.cancellations(); + monitor.options.onUnavailable("audio context suspended"); + assert.equal(f.cancellations(), cancellations + 1); + monitor.options.onUnavailable("audio context suspended"); + assert.equal(f.api.ready(), false); + assert.deepEqual(f.diagnostics, ["audio context suspended"]); + monitor.options.onAvailable(); + f.advance(200); + monitor.onActivity(); + assert.deepEqual(f.measurements, []); +}); + +test("quiet frames reach the timer through the attached monitor", async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + const monitor = f.monitors[0]; + monitor.options.onAvailable(); + f.updateAgentState(); + assert.equal(typeof monitor.options.onQuiet, "function"); + monitor.options.onQuiet(); + f.advance(80); + monitor.onActivity(); + assert.deepEqual(f.measurements, [80]); +}); + +test("console diagnostics are installed only with audioTiming=1", () => { + const source = read("web/interview.js"); + const setup = source.slice( + source.indexOf("const params = new URLSearchParams"), + source.indexOf("let audioTimingTrack"), + ); + for (const search of ["", "?audioTiming=0", "?audioTiming=1"]) { + const logs = []; + let created = 0; + const context = { + URLSearchParams, + window: { location: { search } }, + console: { debug: (message) => logs.push(message) }, + createAudioResponseTiming(options) { + created++; + return createAudioResponseTiming(options); + }, + }; + runInNewContext( + setup + '; audioTiming?.speakingChanged(true, "monitor not ready");', + context, + ); + assert.equal(created, search === "?audioTiming=1" ? 1 : 0); + assert.deepEqual( + logs, + search === "?audioTiming=1" + ? ["[audio-timing] Turn not measured: monitor not ready"] + : [], + ); + } +}); + +test("registered stale reconnect callbacks preserve the newer room and monitor", async () => { + const f = await fixture(); + f.api.setRemoteTrack(remoteTrack("A")); + const newerRoom = new f.room.constructor(); + f.state.room = newerRoom; + f.api.attachAudioTiming(remoteTrack("B")); + f.monitors[1].options.onAvailable(); + const cancellations = f.cancellations(); + f.room.emit("Reconnecting"); + f.room.emit("Reconnected"); + f.room.emit("Disconnected"); + assert.equal(f.state.room, newerRoom); + assert.equal(f.state.connected, true); + assert.equal(f.api.owner(), newerRoom); + assert.equal(f.api.ready(), true); + assert.equal(f.monitors.length, 2); + assert.equal(f.monitors[1].stopped, false); + assert.equal(f.cancellations(), cancellations); +}); + +test("monitor release clears readiness and late callbacks cannot complete a pending turn", async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + const monitor = f.monitors[0]; + monitor.options.onAvailable(); + f.updateAgentState(); + monitor.options.onQuiet(); + const cancellations = f.cancellations(); + f.api.releaseAudioTiming(); + assert.equal(f.cancellations(), cancellations + 1); + assert.equal(f.api.ready(), false); + assert.equal(f.api.owner(), null); + assert.equal(monitor.stopped, true); + monitor.options.onAvailable(); + monitor.options.onQuiet(); + f.advance(200); + monitor.onActivity(); + assert.equal(f.api.ready(), false); + assert.deepEqual(f.measurements, []); + assert.deepEqual(f.diagnostics, ["audio monitor released"]); +}); + +test("callbacks from a replaced track cannot cancel or measure the current track", async () => { + const f = await fixture(); + f.api.attachAudioTiming(remoteTrack("A")); + const old = f.monitors[0]; + f.api.attachAudioTiming(remoteTrack("B")); + const current = f.monitors[1]; + current.options.onAvailable(); + f.updateAgentState(); + current.options.onQuiet(); + const cancellations = f.cancellations(); + old.options.onUnavailable("track ended"); + old.options.onAvailable(); + old.options.onQuiet(); + f.advance(80); + old.onActivity(); + assert.equal(f.cancellations(), cancellations); + assert.equal(f.api.ready(), true); + assert.equal(f.api.owner(), f.room); + assert.deepEqual(f.measurements, []); + current.onActivity(); + assert.deepEqual(f.measurements, [80]); +}); + +test("monitor setup failure clears the track and reconnect retries the diagnostic", async () => { + const f = await fixture({ failFirstMonitor: true }); + const track = remoteTrack("A"); + f.agent.attributes["lk.agent.state"] = "listening"; + f.room.emit("TrackSubscribed", track, {}, f.agent); + assert.equal(f.monitorAttempts(), 1); + assert.equal(f.monitors.length, 0); + assert.equal(f.api.currentTrack(), null); + assert.equal(f.api.owner(), null); + assert.equal(f.api.ready(), false); + + f.room.emit("Reconnected"); + assert.equal(f.monitorAttempts(), 2); + assert.equal(f.monitors.length, 1); + assert.equal(f.api.currentTrack(), track); + assert.equal(f.api.owner(), f.room); + const monitor = f.monitors[0]; + monitor.options.onAvailable(); + assert.equal(f.api.ready(), true); + f.agent.attributes["lk.agent.state"] = "speaking"; + f.room.emit("ParticipantAttributesChanged"); + monitor.options.onQuiet(); + f.advance(80); + monitor.onActivity(); + assert.deepEqual(f.measurements, [80]); + f.disconnect(); +}); + +test("an unavailable interviewer cancels a pending turn and can speak again", async () => { + const f = await fixture(); + f.agent.attributes["lk.agent.state"] = "listening"; + f.room.emit("TrackSubscribed", remoteTrack("A"), {}, f.agent); + const monitor = f.monitors[0]; + monitor.options.onAvailable(); + f.agent.attributes["lk.agent.state"] = "speaking"; + f.room.emit("ParticipantAttributesChanged"); + monitor.options.onQuiet(); + f.advance(40); + + f.setInterviewerPresent(false); + f.room.emit("ParticipantDisconnected", f.agent); + await Promise.resolve(); + assert.equal(f.state.interviewerPresent, false); + assert.deepEqual(f.diagnostics, ["interviewer unavailable"]); + f.advance(80); + monitor.onActivity(); + assert.deepEqual(f.measurements, []); + + monitor.options.onQuiet(); + f.setInterviewerPresent(true); + f.room.emit("ParticipantConnected", f.agent); + assert.equal(f.state.interviewerPresent, true); + monitor.options.onQuiet(); + f.advance(60); + monitor.onActivity(); + assert.deepEqual(f.measurements, [60]); + assert.deepEqual(f.diagnostics, ["interviewer unavailable"]); + f.disconnect(); +}); diff --git a/tests/browser/audio-response-timing.test.js b/tests/browser/audio-response-timing.test.js new file mode 100644 index 000000000..41b77dd50 --- /dev/null +++ b/tests/browser/audio-response-timing.test.js @@ -0,0 +1,628 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { startMediaMeter } from "../../web/mic-meter.js"; +import { + createAudioResponseTiming, + startAudioActivityMonitor, +} from "../../web/audio-response-timing.js"; + +function fixture() { + let time = 0; + const measurements = []; + const diagnostics = []; + + const timing = createAudioResponseTiming({ + now: () => time, + report: (elapsed) => measurements.push(elapsed), + diagnose: (reason) => diagnostics.push(reason), + }); + + return { + timing, + measurements, + diagnostics, + advance: (ms) => { + time += ms; + }, + }; +} + +test("measures speaking-to-first-audio gap", () => { + const f = fixture(); + + f.timing.speakingChanged(true); + f.timing.audioQuiet(); + f.advance(180); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, [180]); +}); + +test("ignores repeated speaking updates", () => { + const f = fixture(); + + f.timing.speakingChanged(true); + f.timing.audioQuiet(); + f.advance(100); + + f.timing.speakingChanged(true); + f.advance(80); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, [180]); +}); + +test("reports only the first audio activity per speaking turn", () => { + const f = fixture(); + + f.timing.speakingChanged(true); + f.timing.audioQuiet(); + f.advance(120); + f.timing.audioActivity(); + + f.advance(50); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, [120]); +}); + +test("does not measure audio without speaking", () => { + const f = fixture(); + + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, []); +}); + +test("cancels pending measurement when speaking stops", () => { + const f = fixture(); + + f.timing.speakingChanged(true); + f.advance(100); + f.timing.speakingChanged(false); + + f.advance(50); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, []); +}); + +test("reset cancels an incomplete measurement", () => { + const f = fixture(); + + f.timing.speakingChanged(true); + f.advance(100); + f.timing.reset(); + + f.advance(50); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, []); +}); + +test("audio monitor detects a frame above threshold", () => { + const m = monitorFixture(); + const callbacks = []; + const stop = startAudioActivityMonitor( + m.track, + () => callbacks.push("audio"), + m.options, + ); + assert.deepEqual( + callbacks, + [], + "remote timing waits for the first animation frame", + ); + m.tick(); + assert.deepEqual(callbacks, ["audio"]); + assert.deepEqual(m.sampleLengths, [1024]); + stop(); +}); +test("audio monitor ignores an ended track", () => { + const m = monitorFixture({ readyState: "ended" }); + const callbacks = []; + const stop = startAudioActivityMonitor( + m.track, + () => callbacks.push("audio"), + m.options, + ); + m.tick(); + assert.deepEqual(callbacks, []); + assert.deepEqual(m.sampleLengths, []); + stop(); +}); + +test("audio monitor reports an unavailable audio context", () => { + const m = monitorFixture({ state: "suspended" }); + let unavailable = 0; + let activity = 0; + const stop = startAudioActivityMonitor(m.track, () => activity++, { + ...m.options, + onUnavailable: () => unavailable++, + }); + m.tick(); + assert.equal(unavailable, 1); + assert.equal(activity, 0); + assert.deepEqual(m.sampleLengths, []); + stop(); +}); +test("cancelPending preserves speaking state without reporting stale audio", () => { + const f = fixture(); + + f.timing.speakingChanged(true); + f.advance(100); + + f.timing.cancelPending(); + f.advance(50); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, []); + + // A repeated speaking update must not restart the cancelled measurement. + f.timing.speakingChanged(true); + f.advance(50); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, []); + + // Only a new speaking transition can start another measurement. + f.timing.speakingChanged(false); + f.timing.audioQuiet(); + f.timing.speakingChanged(true); + f.timing.audioQuiet(); + + f.advance(80); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, [80]); +}); +test("audio monitor notifies unavailability only on state changes", () => { + const m = monitorFixture({ state: "suspended", peak: 0 }); + let unavailable = 0; + const stop = startAudioActivityMonitor(m.track, () => {}, { + ...m.options, + onUnavailable: () => unavailable++, + }); + m.tick(); + m.tick(); + assert.equal(unavailable, 1); + m.context.state = "running"; + m.tick(); + m.context.state = "suspended"; + m.tick(); + assert.equal(unavailable, 2); + stop(); +}); +test("audio monitor attempts to resume a suspended context", () => { + const m = monitorFixture({ state: "suspended" }); + const stop = startAudioActivityMonitor(m.track, () => {}, m.options); + assert.equal(m.resumes(), 1); + stop(); +}); +test("audio monitor closes its context when setup fails", async () => { + const m = monitorFixture({ setupError: "Cannot create media source" }); + assert.throws( + () => startAudioActivityMonitor(m.track, () => {}, m.options), + /Cannot create media source/, + ); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(m.closes(), 1); +}); +test("audio monitor resumes on a gesture and removes listeners on stop", async () => { + const m = monitorFixture({ + state: "suspended", + resumeState: "running", + peak: 0, + }); + const stop = startAudioActivityMonitor(m.track, () => {}, m.options); + assert.equal(m.resumes(), 1); + m.context.state = "suspended"; + m.gestures.dispatchEvent(new Event("pointerdown")); + assert.equal(m.resumes(), 2); + stop(); + m.context.state = "suspended"; + m.gestures.dispatchEvent(new Event("keydown")); + assert.equal(m.resumes(), 2); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(m.closes(), 1); + assert.equal(m.disconnections(), 1); + assert.equal(m.cancellations(), 1); +}); +test("monitor interruption does not restart the same speaking turn", () => { + const f = fixture(); + + // Jim begins speaking while the monitor is available. + f.timing.speakingChanged(true); + f.advance(100); + + // The audio monitor becomes unavailable. + f.timing.cancelPending(); + + // It later recovers, but Jim is still in the same speaking turn. + f.advance(100); + f.timing.speakingChanged(true); + + // Audio is detected after recovery. + f.advance(50); + f.timing.audioActivity(); + + // The interrupted turn must not produce a misleading measurement. + assert.deepEqual(f.measurements, []); + + // A genuine new speaking transition may be measured. + f.timing.speakingChanged(false); + f.timing.audioQuiet(); + f.timing.speakingChanged(true); + f.timing.audioQuiet(); + + f.advance(120); + f.timing.audioActivity(); + + assert.deepEqual(f.measurements, [120]); +}); + +test("hidden pages invalidate audio timing and cleanup visibility listeners", () => { + const m = monitorFixture(); + let unavailable = 0; + let available = 0; + let activity = 0; + const stop = startAudioActivityMonitor(m.track, () => activity++, { + ...m.options, + onAvailable: () => available++, + onUnavailable: () => unavailable++, + }); + m.tick(); + assert.equal(available, 1); + assert.equal(activity, 1); + m.visibility.hidden = true; + m.visibility.dispatchEvent(new Event("visibilitychange")); + assert.equal(unavailable, 1); + m.tick(); + assert.equal(activity, 1); + assert.equal(unavailable, 1); + m.visibility.hidden = false; + m.visibility.dispatchEvent(new Event("visibilitychange")); + m.tick(); + assert.equal(available, 2); + assert.equal(activity, 2); + stop(); + m.visibility.hidden = true; + m.visibility.dispatchEvent(new Event("visibilitychange")); + assert.equal(unavailable, 1); +}); + +function monitorFixture({ + state = "running", + readyState = "live", + peak = 0.5, + resumeState = state, + setupError = null, +} = {}) { + let frame; + let scheduled = 0; + let closes = 0; + let resumes = 0; + let disconnections = 0; + let cancellations = 0; + const sampleLengths = []; + const streams = []; + const track = { kind: "audio", readyState, muted: false, enabled: true }; + const visibility = new EventTarget(); + visibility.hidden = false; + const gestures = new EventTarget(); + const analyser = { + fftSize: 1024, + get frequencyBinCount() { + return this.fftSize / 2; + }, + getByteTimeDomainData(samples) { + assert.equal( + context.state, + "running", + "unavailable contexts must not be sampled", + ); + sampleLengths.push(samples.length); + samples.fill(128); + samples[0] = 128 + Math.round(peak * 128); + }, + }; + const context = { + state, + createMediaStreamSource(stream) { + streams.push(stream); + if (setupError) throw new Error(setupError); + return { + connect(target) { + assert.equal(target, analyser); + }, + disconnect() { + disconnections++; + }, + }; + }, + createAnalyser: () => analyser, + resume: async () => { + resumes++; + context.state = resumeState; + }, + close: async () => { + closes++; + }, + }; + const options = { + createContext: () => context, + createStream: (mediaTrack) => ({ getTracks: () => [mediaTrack] }), + schedule: (callback) => { + frame = callback; + return ++scheduled; + }, + cancel: () => { + cancellations++; + }, + visibilityTarget: visibility, + gestureTarget: gestures, + }; + return { + track, + context, + visibility, + gestures, + options, + sampleLengths, + streams, + tick: () => frame(), + setPeak: (value) => { + peak = value; + }, + closes: () => closes, + resumes: () => resumes, + disconnections: () => disconnections, + cancellations: () => cancellations, + }; +} + +test("audio already active before speaking never becomes a positive gap", () => { + const f = fixture(); + f.timing.audioActivity(); + f.timing.speakingChanged(true); + f.advance(16); + f.timing.audioActivity(); + assert.deepEqual(f.measurements, []); + assert.deepEqual(f.diagnostics, ["audio already active"]); +}); + +test("the first active frame cannot substitute for a post-transition quiet baseline", () => { + const f = fixture(); + f.timing.speakingChanged(true); + f.advance(16); + f.timing.audioActivity(); + assert.deepEqual(f.measurements, []); + assert.deepEqual(f.diagnostics, [ + "audio already active before a quiet frame", + ]); +}); + +test("only a quiet frame after speaking arms a positive measurement", () => { + const f = fixture(); + assert.equal(typeof f.timing.audioQuiet, "function"); + f.timing.audioQuiet(); + f.timing.speakingChanged(true); + f.advance(16); + f.timing.audioActivity(); + assert.deepEqual(f.measurements, []); + f.timing.audioQuiet(); + f.timing.audioActivity(); + assert.deepEqual(f.measurements, [], "a dropped turn stays dropped"); + f.timing.speakingChanged(false); + f.timing.audioQuiet(); + f.timing.speakingChanged(true); + f.advance(20); + f.timing.audioQuiet(); + f.advance(80); + f.timing.audioActivity(); + assert.deepEqual( + f.measurements, + [100], + "elapsed time still starts at the state transition", + ); +}); + +for (const [property, value, reason] of [ + ["muted", true, "track muted"], + ["enabled", false, "track disabled"], +]) { + test(`a ${property} interruption cancels the turn instead of timing recovery`, () => { + const f = fixture(); + const m = monitorFixture({ peak: 0 }); + const unavailable = []; + const stop = startAudioActivityMonitor(m.track, f.timing.audioActivity, { + ...m.options, + onQuiet: f.timing.audioQuiet, + onUnavailable: (reason) => { + unavailable.push(reason); + f.timing.cancelPending(reason); + }, + }); + m.tick(); + f.timing.speakingChanged(true); + m.tick(); + m.track[property] = value; + m.tick(); + m.tick(); + assert.deepEqual(unavailable, [reason]); + assert.deepEqual(f.diagnostics, [reason]); + m.track[property] = !value; + m.setPeak(0.5); + f.advance(200); + m.tick(); + assert.deepEqual(f.measurements, []); + m.setPeak(0); + m.tick(); + f.timing.speakingChanged(false); + f.timing.speakingChanged(true); + m.tick(); + f.advance(80); + m.setPeak(0.5); + m.tick(); + assert.deepEqual(f.measurements, [80]); + stop(); + }); +} + +test("the monitor supplies quiet frames without reporting activity", () => { + const m = monitorFixture({ peak: 0 }); + let quiet = 0; + let activity = 0; + const stop = startAudioActivityMonitor(m.track, () => activity++, { + ...m.options, + onQuiet: () => quiet++, + }); + m.tick(); + m.setPeak(0.5); + m.tick(); + m.setPeak(0); + m.tick(); + assert.equal(quiet, 2); + assert.equal(activity, 1); + stop(); +}); + +test("skipped transitions and canceled turns explain themselves once", () => { + const f = fixture(); + f.timing.speakingChanged(true, "monitor not ready"); + f.timing.speakingChanged(true, "monitor not ready"); + f.timing.cancelPending("audio context suspended"); + assert.deepEqual(f.diagnostics, ["monitor not ready"]); + f.timing.speakingChanged(false); + f.timing.speakingChanged(true); + f.timing.cancelPending("page hidden"); + f.timing.cancelPending("page hidden"); + f.timing.audioActivity(); + assert.deepEqual(f.measurements, []); + assert.deepEqual(f.diagnostics, ["monitor not ready", "page hidden"]); +}); + +test("stopping speech and resetting explain pending measurements", () => { + const f = fixture(); + f.timing.speakingChanged(true); + f.timing.speakingChanged(false); + f.timing.speakingChanged(true); + f.timing.reset("room disconnected"); + assert.deepEqual(f.diagnostics, [ + "speaking stopped before audio", + "room disconnected", + ]); +}); + +for (const [name, peak, expected] of [ + ["silent", 0, []], + ["below threshold", 7 / 128, []], + ["at threshold", 8 / 128, []], + ["above threshold", 9 / 128, [80]], +]) { + test(`${name} frames obey the strict activity threshold`, () => { + const f = fixture(); + const m = monitorFixture({ peak: 0 }); + const stop = startAudioActivityMonitor(m.track, f.timing.audioActivity, { + ...m.options, + threshold: 8 / 128, + onQuiet: f.timing.audioQuiet, + }); + m.tick(); + f.timing.speakingChanged(true); + m.tick(); + m.setPeak(peak); + f.advance(80); + m.tick(); + m.tick(); + assert.deepEqual(f.measurements, expected); + m.setPeak(0); + m.tick(); + assert.deepEqual( + f.measurements, + expected, + "quiet frames never add a result", + ); + stop(); + }); +} + +test("microphone and remote timing meters retain separate sources and lifetimes", () => { + const f = fixture(); + const mic = monitorFixture(); + const remote = monitorFixture({ peak: 0 }); + const peaks = []; + const stopMic = startMediaMeter( + mic.options.createStream(mic.track), + (peak) => peaks.push(peak), + (error) => assert.fail(error), + () => false, + mic.options, + ); + const stopRemote = startAudioActivityMonitor( + remote.track, + f.timing.audioActivity, + { ...remote.options, onQuiet: f.timing.audioQuiet }, + ); + assert.equal(mic.streams[0].getTracks()[0], mic.track); + assert.equal(remote.streams[0].getTracks()[0], remote.track); + remote.tick(); + f.timing.speakingChanged(true); + remote.tick(); + mic.tick(); + assert.deepEqual(peaks, [0.5, 0.5]); + assert.deepEqual( + f.measurements, + [], + "microphone activity cannot satisfy remote timing", + ); + f.advance(80); + remote.setPeak(0.5); + remote.tick(); + assert.deepEqual(f.measurements, [80]); + stopRemote(); + assert.equal(remote.closes(), 1); + assert.equal(mic.closes(), 0); + mic.tick(); + assert.equal(peaks.length, 3); + stopMic(); + assert.equal(mic.closes(), 1); +}); + +test("failed monitor setup leaves no gesture or visibility listeners", () => { + const m = monitorFixture({ + state: "suspended", + setupError: "Cannot create media source", + }); + let unavailable = 0; + assert.throws( + () => + startAudioActivityMonitor(m.track, () => {}, { + ...m.options, + onUnavailable: () => unavailable++, + }), + /Cannot create media source/, + ); + const resumes = m.resumes(); + m.gestures.dispatchEvent(new Event("pointerdown")); + m.gestures.dispatchEvent(new Event("keydown")); + m.visibility.hidden = true; + m.visibility.dispatchEvent(new Event("visibilitychange")); + assert.equal(m.resumes(), resumes); + assert.equal(unavailable, 0); + assert.equal(m.closes(), 1); +}); + +test("a synchronous resume failure releases the shared meter before throwing", () => { + const m = monitorFixture({ state: "suspended" }); + m.context.resume = () => { + throw new Error("Resume failed"); + }; + assert.throws( + () => startAudioActivityMonitor(m.track, () => {}, m.options), + /Resume failed/, + ); + assert.equal(m.closes(), 1); + assert.equal(m.disconnections(), 1); + assert.equal(m.cancellations(), 1); +}); diff --git a/tests/browser/interview-lifecycle.test.js b/tests/browser/interview-lifecycle.test.js index 22ef1d8b9..2c94890c8 100644 --- a/tests/browser/interview-lifecycle.test.js +++ b/tests/browser/interview-lifecycle.test.js @@ -248,6 +248,9 @@ test("the actual connection preserves refusal reasons and gates the init deadlin // As init leaves it before connect: Practice work is guarded from page // load, and nothing in the start may arm the guard for Strict. const context = { + releaseAudioTiming: () => {}, + audioTiming: null, + protected: !strict, state: { endsAt: 0, diff --git a/tests/browser/mic-meter.test.js b/tests/browser/mic-meter.test.js index d0dfc9582..4084fc14d 100644 --- a/tests/browser/mic-meter.test.js +++ b/tests/browser/mic-meter.test.js @@ -13,7 +13,8 @@ import { MIC_CONFIRM_FRAMES, preflightReadiness, } from "../../web/audio-check.js"; -import { createMicMeter } from "../../web/mic-meter.js"; +import { createMicMeter, startMediaMeter } from "../../web/mic-meter.js"; +import { stubGlobal } from "./source.js"; /// A pool with one audio track, whose readiness a test can change. function fakePool(audioState = "live") { @@ -293,3 +294,94 @@ test("a dead device of either kind reopens the request the pool makes", () => { ); } }); + +test("the real media meter samples its own stream immediately and stops on the next frame", async (t) => { + let tick; + let stopped = false; + let closes = 0; + const stream = { label: "microphone stream" }; + const streams = []; + const sampleLengths = []; + const peaks = []; + const errors = []; + const analyser = { + fftSize: 0, + get frequencyBinCount() { + return this.fftSize / 2; + }, + getByteTimeDomainData(samples) { + sampleLengths.push(samples.length); + samples.fill(128); + samples[511] = 192; + }, + }; + stubGlobal( + t, + "AudioContext", + class { + createAnalyser() { + return analyser; + } + createMediaStreamSource(input) { + streams.push(input); + return { + connect(target) { + assert.equal(target, analyser); + }, + disconnect() {}, + }; + } + async close() { + closes++; + } + }, + ); + stubGlobal(t, "requestAnimationFrame", (callback) => { + tick = callback; + return 1; + }); + stubGlobal(t, "cancelAnimationFrame", () => {}); + const stop = await startMediaMeter( + stream, + (peak) => peaks.push(peak), + (error) => errors.push(error), + () => stopped, + ); + assert.equal( + typeof stop, + "function", + "the shared loop exposes explicit cleanup", + ); + assert.deepEqual(streams, [stream]); + assert.equal(analyser.fftSize, 1024); + assert.deepEqual(sampleLengths, [512]); + assert.deepEqual(peaks, [0.5]); + tick(); + assert.deepEqual(peaks, [0.5, 0.5]); + stopped = true; + tick(); + stop(); + assert.equal(closes, 1, "cleanup is idempotent after shouldStop"); + assert.deepEqual(peaks, [0.5, 0.5]); + assert.deepEqual(errors, []); +}); + +test("the real media meter reports setup failures as strings", async (t) => { + stubGlobal( + t, + "AudioContext", + class { + constructor() { + throw new Error("No audio context"); + } + }, + ); + const errors = []; + await startMediaMeter( + {}, + () => assert.fail("no sample without a context"), + (error) => errors.push(error), + () => false, + ); + assert.deepEqual(errors, ["No audio context"]); +}); diff --git a/tests/browser/report-recovery.test.js b/tests/browser/report-recovery.test.js index 2710785b8..b6a74ebb6 100644 --- a/tests/browser/report-recovery.test.js +++ b/tests/browser/report-recovery.test.js @@ -509,6 +509,18 @@ test("terminal room disconnect finalizes recovery before clearing the room", asy page.scope.console = { warn() {} }; const playbackRooms = []; page.scope.audioPlayback = { release: (room) => playbackRooms.push(room) }; + let audioTimingReleases = 0; + let audioTimingResets = 0; + + page.scope.releaseAudioTiming = () => { + audioTimingReleases++; + }; + + page.scope.audioTiming = { + reset: () => { + audioTimingResets++; + }, + }; page.scope.preflight = { userStream: { label: "Preflight stream" } }; page.scope.localRecording.needsMicrophone = false; const mediaCleanup = []; @@ -520,6 +532,8 @@ test("terminal room disconnect finalizes recovery before clearing the room", asy )(new Proxy(page.scope, { has: (target, key) => key in target })); disconnect(); await settle(); + assert.equal(audioTimingReleases, 1); + assert.equal(audioTimingResets, 1); assert.deepEqual(playbackRooms, [page.room]); assert.deepEqual(mediaCleanup, [ [page.scope.preflight, { preserveMicrophone: false }], diff --git a/web/audio-response-timing.js b/web/audio-response-timing.js new file mode 100644 index 000000000..9075e57c2 --- /dev/null +++ b/web/audio-response-timing.js @@ -0,0 +1,173 @@ +import { startMediaMeter } from "./mic-meter.js"; +/// Browser-observed gap between Jim entering "speaking" and the first +/// detectable remote audio activity. This is not end-to-end latency. +export function createAudioResponseTiming({ + now = () => performance.now(), + report = () => {}, + diagnose = () => {}, +} = {}) { + let speaking = false; + let startedAt = null; + let active = false; + let armed = false; + + function speakingChanged(nextSpeaking, unavailableReason = null) { + if (nextSpeaking === speaking) return; + + speaking = nextSpeaking; + + if (speaking) { + if (unavailableReason || active) { + diagnose(unavailableReason || "audio already active"); + } else { + startedAt = now(); + armed = false; + } + } else { + cancelPending("speaking stopped before audio"); + } + } + + function audioActivity() { + active = true; + if (!speaking || startedAt === null) return; + if (!armed) { + cancelPending("audio already active before a quiet frame"); + return; + } + + const elapsedMs = now() - startedAt; + startedAt = null; + + report(elapsedMs); + } + + function audioQuiet() { + active = false; + // A pre-transition quiet frame cannot rule out audio arriving first. + if (startedAt !== null) armed = true; + } + + function cancelPending(reason = "measurement canceled") { + if (startedAt !== null) diagnose(reason); + startedAt = null; + armed = false; + } + + function reset(reason = "timing reset") { + speaking = false; + active = false; + cancelPending(reason); + } + + return { + speakingChanged, + audioActivity, + audioQuiet, + cancelPending, + reset, + }; +} +/// Observe remote audio activity without routing it to another output. +/// The returned function releases the analyser and its AudioContext. +export function startAudioActivityMonitor( + track, + onActivity, + { + threshold = 0.05, + onAvailable = () => {}, + onUnavailable = () => {}, + onQuiet = () => {}, + createContext = () => new AudioContext(), + createStream = (mediaTrack) => new MediaStream([mediaTrack]), + schedule = requestAnimationFrame, + cancel = cancelAnimationFrame, + gestureTarget = globalThis.document, + visibilityTarget = globalThis.document, + } = {}, +) { + const stream = createStream(track); + const context = createContext(); + let stopped = false; + let unavailable; + + function notifyAvailability(reason) { + if (unavailable === reason) return; + unavailable = reason; + if (reason === null) onAvailable(); + else onUnavailable(reason); + } + + function visibilityChanged() { + if (stopped) return; + + if (visibilityTarget?.hidden) { + notifyAvailability("page hidden"); + } + } + + function resumeOnGesture() { + if (stopped || context.state !== "suspended") return; + void context.resume?.().catch(() => {}); + } + + const stopMeter = startMediaMeter( + stream, + (peak) => { + if (peak > threshold) onActivity(); + else onQuiet(); + }, + (message) => { + throw new Error(message); + }, + () => stopped, + { + createContext: () => context, + schedule, + cancel, + sampleSize: 1024, + deferFirstFrame: true, + shouldSample: () => { + const reason = visibilityTarget?.hidden + ? "page hidden" + : context.state !== "running" + ? `audio context ${context.state}` + : track.readyState !== "live" + ? "track ended" + : track.muted + ? "track muted" + : track.enabled === false + ? "track disabled" + : null; + + notifyAvailability(reason); + return reason === null; + }, + }, + ); + try { + resumeOnGesture(); + } catch (error) { + stopMeter(); + throw error; + } + visibilityTarget?.addEventListener("visibilitychange", visibilityChanged); + gestureTarget?.addEventListener("pointerdown", resumeOnGesture); + gestureTarget?.addEventListener("keydown", resumeOnGesture); + + return () => { + if (stopped) return; + + stopped = true; + + gestureTarget?.removeEventListener("pointerdown", resumeOnGesture); + gestureTarget?.removeEventListener("keydown", resumeOnGesture); + + visibilityTarget?.removeEventListener( + "visibilitychange", + visibilityChanged, + ); + + stopMeter(); + }; +} diff --git a/web/interview.js b/web/interview.js index 1b4a8f6f8..3def871fa 100644 --- a/web/interview.js +++ b/web/interview.js @@ -222,6 +222,11 @@ import { parseCandidateCase, runBrowserTests } from "./runners.js"; import { mountBehavioralReview } from "./behavioral-review.js"; import { consumeGroundingPacket } from "./document-grounding.js"; +import { + createAudioResponseTiming, + startAudioActivityMonitor, +} from "./audio-response-timing.js"; + /// Which framework the candidate is being read against right now. The rounds /// never overlap, so this is a single value rather than a pair. Declared up /// here because `codingClosed` reads it and `init` runs before the rest of the @@ -336,6 +341,23 @@ const consentVersion = globalThis.CODETRIAL_CONSENT_VERSION || ""; const replayVersion = globalThis.CODETRIAL_REPLAY_VERSION || 1; const params = new URLSearchParams(window.location.search); +const audioTimingEnabled = params.get("audioTiming") === "1"; +const audioTiming = audioTimingEnabled + ? createAudioResponseTiming({ + diagnose: (reason) => + console.debug(`[audio-timing] Turn not measured: ${reason}`), + report: (elapsedMs) => { + console.info( + `[audio-timing] Browser-observed speaking-to-audio gap: ${elapsedMs.toFixed(1)} ms`, + ); + }, + }) + : null; + +let audioTimingTrack = null; +let audioTimingRoom = null; +let stopAudioTimingMonitor = null; +let audioTimingReady = false; const editorOptions = editorOptionsFromParams(params); const indentUnit = INDENT_UNITS[indentFromParams(params)]; const strict = params.get("selection") === "strict"; @@ -2397,6 +2419,7 @@ async function connectLiveKit(connection, preflight, presenting = false) { // on its own, so the candidate is told to wait rather than to restart. room.on(livekit.RoomEvent.Reconnecting, () => { if (state.room !== room) return; + releaseAudioTiming(null, "room reconnecting"); state.connected = false; if (strict) updateRunAvailability(); setBanner("connection", providerUiState("reconnecting").message); @@ -2404,6 +2427,11 @@ async function connectLiveKit(connection, preflight, presenting = false) { room.on(livekit.RoomEvent.Reconnected, () => { if (state.room !== room) return; state.connected = true; + // LiveKit may retain the same remote audio track across reconnects. + // Reattach the diagnostic even when no new TrackSubscribed event fires. + if (jimTrack?.mediaStreamTrack?.readyState === "live") { + attachAudioTiming(jimTrack); + } if (strict) updateRunAvailability(); setBanner("connection", ""); if (state.phase === "live" && state.interviewerReadiness?.pending) { @@ -2433,6 +2461,10 @@ async function connectLiveKit(connection, preflight, presenting = false) { }); room.on(livekit.RoomEvent.Disconnected, () => { audioPlayback.release(room); + if (state.room === room || (!state.room && audioTimingRoom === room)) { + releaseAudioTiming(null, "room disconnected"); + audioTiming?.reset(); + } const startupCleanup = state.startupCleanupRoom === room; if (startupCleanup) state.startupCleanupRoom = null; if (state.room !== room) return; @@ -2680,6 +2712,8 @@ let detachSelfView = () => {}; /// left only once that notice is acknowledged or given up on. function abandonStart(preflight, { preserveMicrophone = false, flushed } = {}) { const joined = state.room; + releaseAudioTiming(); + audioTiming?.reset(); state.connected = false; state.room = null; if (joined) audioPlayback.release(joined); @@ -3004,7 +3038,61 @@ async function receiveReport(room, payload, flushed = Promise.resolve()) { reportRenderFailed(error, { offlineSummary: !renderAttempted }); } } +function releaseAudioTiming(track = null, reason = "audio monitor released") { + if (!audioTiming) return; + + // A late unsubscribe from an old track must not remove the new monitor. + if (track && track !== audioTimingTrack) return; + + stopAudioTimingMonitor?.(); + + stopAudioTimingMonitor = null; + audioTimingTrack = null; + audioTimingRoom = null; + audioTimingReady = false; + + audioTiming.cancelPending(reason); +} +function attachAudioTiming(track) { + if (!audioTiming) return; + + if (audioTimingTrack === track) return; + + releaseAudioTiming(null, "audio track replaced"); + + const mediaTrack = track.mediaStreamTrack; + if (!mediaTrack) return; + audioTimingTrack = track; + audioTimingRoom = state.room; + + try { + stopAudioTimingMonitor = startAudioActivityMonitor( + mediaTrack, + () => { + if (!audioTimingReady || audioTimingTrack !== track) return; + audioTiming.audioActivity(); + }, + { + onAvailable: () => { + if (audioTimingTrack !== track) return; + audioTimingReady = true; + }, + onQuiet: () => { + if (audioTimingTrack === track) audioTiming.audioQuiet(); + }, + onUnavailable: (reason) => { + if (audioTimingTrack !== track) return; + audioTimingReady = false; + audioTiming.cancelPending(reason); + }, + }, + ); + } catch (error) { + releaseAudioTiming(); + console.warn("[audio-timing] Monitor unavailable", error); + } +} // One element for the whole session. LiveKit re-subscribes on reconnect, and // track.attach() with no argument mints a fresh element every time: two of // them play the same track into the shared tab, audibly doubled, while @@ -3052,10 +3140,13 @@ function playRemoteAudio(track, participant) { }); } attachAvatarAnalyser(track, participant); + attachAudioTiming(track); } function dropRemoteAudio(track) { if (track.kind !== "audio") return; + releaseAudioTiming(track); + // Cleared only if the element that went away was Jim's. Any other remote // audio track unsubscribing -- a second participant leaving -- would // otherwise drop the handle while the element is still playing, and @@ -5295,6 +5386,7 @@ function updateAgentState() { return; } if (!agent) { + audioTiming?.reset("interviewer unavailable"); setAgentStateLabel("Waiting", false); // "Waiting" is honest but useless on its own: it looks identical whether // the interviewer has not arrived yet or arrived, greeted, and died. Once @@ -5375,6 +5467,16 @@ function updateAgentState() { const value = published || "listening"; state.agentSpeaking = value === "speaking"; state.agentThinking = value === "thinking"; + if (audioTiming) { + audioTiming.speakingChanged( + published === "speaking", + !state.connected + ? "room disconnected" + : !audioTimingReady + ? "monitor not ready" + : null, + ); + } if (state.agentSpeaking || state.agentThinking) { drawingTurn?.reset(); hideTurnRing(); diff --git a/web/mic-meter.js b/web/mic-meter.js index 136807714..1a3961cca 100644 --- a/web/mic-meter.js +++ b/web/mic-meter.js @@ -15,27 +15,60 @@ import { MIC_CONFIRM_FRAMES, peakLevel, sustainedPeak } from "./audio-check.js"; /// /// Injected rather than called directly, the way `createFaceCheck` takes its /// detector, because a test cannot build one of these and the thing worth -/// testing is not the analyser. -export async function startMediaMeter(stream, onPeak, onError, shouldStop) { +/// testing is not the analyser. The returned stop function also releases it +/// without waiting for a frame in a hidden tab. +export function startMediaMeter( + stream, + onPeak, + onError, + shouldStop, + { + createContext = () => new AudioContext(), + schedule = (callback) => requestAnimationFrame(callback), + cancel = (frame) => cancelAnimationFrame(frame), + shouldSample = () => true, + sampleSize = null, + deferFirstFrame = false, + } = {}, +) { + let context; + let source; + let frame = null; + let stopped = false; + const stop = () => { + if (stopped) return; + stopped = true; + if (frame !== null) cancel(frame); + source?.disconnect(); + void context?.close().catch(() => {}); + }; try { - const context = new AudioContext(); + context = createContext(); const analyser = context.createAnalyser(); analyser.fftSize = 1024; - context.createMediaStreamSource(stream).connect(analyser); - const samples = new Uint8Array(analyser.frequencyBinCount); + source = context.createMediaStreamSource(stream); + source.connect(analyser); + const samples = new Uint8Array(sampleSize ?? analyser.frequencyBinCount); const tick = () => { + frame = null; + if (stopped) return; if (shouldStop()) { - void context.close().catch(() => {}); + stop(); return; } - analyser.getByteTimeDomainData(samples); - onPeak(peakLevel(samples)); - requestAnimationFrame(tick); + if (shouldSample()) { + analyser.getByteTimeDomainData(samples); + onPeak(peakLevel(samples)); + } + if (!stopped) frame = schedule(tick); }; - tick(); + if (deferFirstFrame) frame = schedule(tick); + else tick(); } catch (error) { + stop(); onError(String(error?.message || error)); } + return stop; } /// One meter at a time. The pool can hand back a replacement microphone, and