From 32c6c6ebbf467ec0f9e19ca4b014cec79ef22caa Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:14:43 -0500 Subject: [PATCH] web_audio: convert each write's PCM16 to float in one JS call, not sample by sample in Python --- lib/audiodev/README.md | 6 ++ lib/audiodev/web_audio.py | 113 ++++++++++++++++++++--- tests/test_web_audio.py | 189 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 295 insertions(+), 13 deletions(-) create mode 100644 tests/test_web_audio.py diff --git a/lib/audiodev/README.md b/lib/audiodev/README.md index 6770cf3e..3e361abc 100644 --- a/lib/audiodev/README.md +++ b/lib/audiodev/README.md @@ -610,6 +610,11 @@ PyScript backend. `WebPCMOutput` converts each write into an `AudioBuffer` and schedules `AudioBufferSourceNode`s back to back against `currentTime`; `WebPCMInput` captures through `getUserMedia` and a `ScriptProcessorNode`. +Each write is converted in one JS call: the PCM goes across as a typed array +and JS decodes, scales and de-interleaves it straight into the `AudioBuffer`. +If the page's FFI can't hand JS a typed array, the module converts in Python +instead, with the same result. + Browser autoplay policy is the thing to know here. A fresh `AudioContext` starts `suspended`, and while suspended `currentTime` never advances — a naive wait for playout would spin forever. The module resumes best-effort, arms a one-shot @@ -661,6 +666,7 @@ the import is a no-op. [`tests/test_audiodev.py`](../../tests/test_audiodev.py) (base contracts, and `AudioOut` against a fake transport + a fake audiosample), [`tests/test_sdl2_audio.py`](../../tests/test_sdl2_audio.py), [`tests/test_pygame_audio.py`](../../tests/test_pygame_audio.py), [`tests/test_win_audio.py`](../../tests/test_win_audio.py), +[`tests/test_web_audio.py`](../../tests/test_web_audio.py) (the PCM16 conversion, its JS run under node when installed), [`tests/test_emulated_audio.py`](../../tests/test_emulated_audio.py) and [`tests/test_auto.py`](../../tests/test_auto.py) cover these modules against SDL's `dummy` driver, and [`tests/test_audiodev_latency.py`](../../tests/test_audiodev_latency.py) covers the profile vocabulary on its own. [`tests/test_audio_playback_golden.py`](../../tests/test_audio_playback_golden.py) runs a diff --git a/lib/audiodev/web_audio.py b/lib/audiodev/web_audio.py index 04d0a0c4..de61ec00 100644 --- a/lib/audiodev/web_audio.py +++ b/lib/audiodev/web_audio.py @@ -8,8 +8,14 @@ except ImportError: asyncio = None +import sys import time +try: + from array import array +except ImportError: # pragma: no cover + array = None + from audiodev import AudioFormat, PCMInput, PCMOutput, check_latency from audiodev.bytequeue import ByteQueue @@ -22,6 +28,19 @@ navigator = None create_proxy = None +try: + from js import Function as _JSFunction +except ImportError: # pragma: no cover - non-browser import for tooling + _JSFunction = None + +try: + from pyodide.ffi import to_js as _to_js +except ImportError: + try: + from pyscript.ffi import to_js as _to_js + except ImportError: # pragma: no cover - non-browser import for tooling + _to_js = None + def _sleep_ms(milliseconds): if hasattr(time, "sleep_ms"): @@ -43,16 +62,88 @@ def _require_browser(): def _pcm16_to_f32(buf, channels): - """Convert little-endian signed 16-bit interleaved PCM to float samples.""" + """Convert little-endian signed 16-bit interleaved PCM to float samples. + + Returns one flat interleaved list, each sample divided by 32768. This is + the reference the browser path in :func:`_fill_audio_buffer` must match. + """ + return [sample / 32768.0 for sample in _pcm16_samples(buf)] + + +def _pcm16_samples(buf): + """Signed 16-bit samples of *buf* in one bulk decode (no per-sample slicing).""" view = memoryview(buf) - samples = len(view) // 2 - floats = [0.0] * samples - for i in range(samples): - sample = int.from_bytes(view[i * 2 : i * 2 + 2], "little") + count = len(view) // 2 + raw = bytes(view[: count * 2]) + if array is not None: + samples = array("h", raw) + if _BIG_ENDIAN: + samples.byteswap() + return samples + samples = [] # pragma: no cover - no array module + for i in range(count): + sample = int.from_bytes(raw[i * 2 : i * 2 + 2], "little") if sample >= 0x8000: sample -= 0x10000 - floats[i] = sample / 32768.0 - return floats + samples.append(sample) + return samples + + +_BIG_ENDIAN = getattr(sys, "byteorder", "little") == "big" + +# Fills an AudioBuffer from little-endian PCM16 bytes in one call: decode, +# divide by 32768 and de-interleave all happen in JS. A DataView reads at any +# byte offset, so the bytes need not be 2-aligned. +_FILL_JS = """ +const dv = new DataView(u8.buffer, u8.byteOffset, u8.byteLength); +const frames = Math.floor(u8.byteLength / (2 * channels)); +for (let c = 0; c < channels; c++) { + const f = new Float32Array(frames); + for (let i = 0, o = c * 2; i < frames; i++, o += 2 * channels) { + f[i] = dv.getInt16(o, true) / 32768; + } + audioBuffer.copyToChannel(f, c); +} +return frames; +""" + +_fill_js = None +_fill_js_failed = False + + +def _fill_js_function(): + global _fill_js, _fill_js_failed + if _fill_js is None and not _fill_js_failed: + try: + _fill_js = _JSFunction.new("audioBuffer", "u8", "channels", _FILL_JS) + except Exception: + _fill_js_failed = True + return _fill_js + + +def _fill_audio_buffer(audio_buffer, buf, channels): + """Write interleaved PCM16 *buf* into *audio_buffer*, one channel each. + + In the browser this is one JS call over the whole buffer. Where that is + not possible (no ``to_js``, or an FFI that cannot hand JS a typed array) + it falls back to converting in Python, with the same result. + """ + global _fill_js_failed + view = memoryview(buf) + if _JSFunction is not None and _to_js is not None and not _fill_js_failed: + fill = _fill_js_function() + if fill is not None: + try: + fill(audio_buffer, _to_js(view), channels) + return + except Exception: + _fill_js_failed = True + floats = _pcm16_to_f32(view, channels) + frames = len(floats) // channels + for channel in range(channels): + channel_data = audio_buffer.getChannelData(channel) + for i in range(frames): + channel_data[i] = floats[i * channels + channel] def _f32_to_pcm16(channel_data): @@ -197,8 +288,7 @@ def _open(self): self._next_time = float(self._ctx.currentTime) def _write(self, buf): - floats = _pcm16_to_f32(buf, self.format.channels) - frames = len(floats) // self.format.channels + frames = len(buf) // self.format.frame_size if frames <= 0: return 0 audio_buffer = self._ctx.createBuffer( @@ -206,10 +296,7 @@ def _write(self, buf): frames, self.format.rate, ) - for channel in range(self.format.channels): - channel_data = audio_buffer.getChannelData(channel) - for i in range(frames): - channel_data[i] = floats[i * self.format.channels + channel] + _fill_audio_buffer(audio_buffer, buf, self.format.channels) source = self._ctx.createBufferSource() source.buffer = audio_buffer source.connect(self._ctx.destination) diff --git a/tests/test_web_audio.py b/tests/test_web_audio.py new file mode 100644 index 00000000..2a81bb31 --- /dev/null +++ b/tests/test_web_audio.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: 2026 Brad Barnett +# +# SPDX-License-Identifier: MIT +"""PCM16-to-float conversion in ``audiodev.web_audio``. + +The conversion runs as one JS call in the browser and in Python elsewhere. +Both must give exactly what the original per-sample loop gave: each sample +divided by 32768, de-interleaved one channel per ``AudioBuffer`` channel. +""" + +import json +import random +import shutil +import struct +import subprocess +import unittest +from unittest import mock + +import _env # noqa: F401 + +from audiodev import AudioFormat, web_audio + + +def _reference_f32(buf): + """The per-sample conversion web_audio shipped before it was vectorised.""" + view = memoryview(buf) + samples = len(view) // 2 + floats = [0.0] * samples + for i in range(samples): + sample = int.from_bytes(view[i * 2 : i * 2 + 2], "little") + if sample >= 0x8000: + sample -= 0x10000 + floats[i] = sample / 32768.0 + return floats + + +def _reference_channels(buf, channels): + floats = _reference_f32(buf) + frames = len(floats) // channels + return [[floats[i * channels + c] for i in range(frames)] for c in range(channels)] + + +def _f32(values): + """Round doubles to float32, as storing into a Float32Array does.""" + fmt = f"<{len(values)}f" + return list(struct.unpack(fmt, struct.pack(fmt, *values))) + + +def _pcm(frames, channels, seed): + rng = random.Random(seed) + edge = [-32768, -32767, -1, 0, 1, 32766, 32767] + values = edge + [rng.randint(-32768, 32767) for _ in range(frames * channels - len(edge))] + return struct.pack(f"<{len(values)}h", *values) + + +class FakeAudioBuffer: + def __init__(self, channels, frames): + self.data = [[0.0] * frames for _ in range(channels)] + + def getChannelData(self, channel): + return self.data[channel] + + +class FakeContext: + currentTime = 0.0 + destination = object() + + def __init__(self): + self.buffers = [] + + def createBuffer(self, channels, frames, rate): + buf = FakeAudioBuffer(channels, frames) + self.buffers.append((channels, frames, rate, buf)) + return buf + + def createBufferSource(self): + return mock.Mock() + + +class TestPythonConversion(unittest.TestCase): + def test_flat_floats_match_reference(self): + for channels in (1, 2): + pcm = _pcm(300, channels, seed=channels) + self.assertEqual(_reference_f32(pcm), web_audio._pcm16_to_f32(pcm, channels)) + + def test_odd_trailing_byte_ignored(self): + pcm = _pcm(10, 1, seed=3) + b"\x7f" + self.assertEqual(_reference_f32(pcm), web_audio._pcm16_to_f32(pcm, 1)) + + def test_fallback_fill_matches_reference(self): + for channels in (1, 2): + pcm = _pcm(257, channels, seed=10 + channels) + target = FakeAudioBuffer(channels, 257) + with mock.patch.object(web_audio, "_JSFunction", None): + web_audio._fill_audio_buffer(target, memoryview(pcm), channels) + self.assertEqual(_reference_channels(pcm, channels), target.data) + + def test_write_builds_one_buffer_per_write(self): + for channels in (1, 2): + fmt = AudioFormat(48000, channels, 16) + out = web_audio.WebPCMOutput(fmt) + out._ctx = FakeContext() + pcm = _pcm(128, channels, seed=20 + channels) + with mock.patch.object(web_audio, "_JSFunction", None): + self.assertEqual(len(pcm), out._write(memoryview(pcm))) + ((got_channels, frames, rate, target),) = out._ctx.buffers + self.assertEqual((channels, 128, 48000), (got_channels, frames, rate)) + self.assertEqual(_reference_channels(pcm, channels), target.data) + + +class TestBrowserPath(unittest.TestCase): + def setUp(self): + web_audio._fill_js = None + web_audio._fill_js_failed = False + self.addCleanup(setattr, web_audio, "_fill_js", None) + self.addCleanup(setattr, web_audio, "_fill_js_failed", False) + + def test_one_js_call_per_buffer(self): + calls = [] + fake_function = mock.Mock() + fake_function.new.return_value = lambda *args: calls.append(args) + pcm = _pcm(256, 2, seed=5) + target = FakeAudioBuffer(2, 256) + with mock.patch.object(web_audio, "_JSFunction", fake_function), mock.patch.object( + web_audio, "_to_js", lambda view: ("u8", bytes(view)) + ): + web_audio._fill_audio_buffer(target, memoryview(pcm), 2) + web_audio._fill_audio_buffer(target, memoryview(pcm), 2) + self.assertEqual(1, fake_function.new.call_count) + self.assertEqual([(target, ("u8", pcm), 2)] * 2, calls) + + def test_js_failure_falls_back_once_and_stays_off(self): + fake_function = mock.Mock() + + def broken(*_args): + raise TypeError("FFI cannot pass a typed array") + + fake_function.new.return_value = broken + pcm = _pcm(64, 1, seed=6) + with mock.patch.object(web_audio, "_JSFunction", fake_function), mock.patch.object( + web_audio, "_to_js", lambda view: view + ): + for _ in range(2): + target = FakeAudioBuffer(1, 64) + web_audio._fill_audio_buffer(target, memoryview(pcm), 1) + self.assertEqual(_reference_channels(pcm, 1), target.data) + self.assertTrue(web_audio._fill_js_failed) + + @unittest.skipUnless(shutil.which("node"), "node not installed") + def test_js_source_matches_reference(self): + """Run the exact JS body under node, at an odd byte offset too.""" + script = """ +const input = JSON.parse(require("fs").readFileSync(0, "utf8")); +const fill = new Function("audioBuffer", "u8", "channels", input.src); +const out = []; +for (const c of input.cases) { + const raw = Buffer.from(c.hex, "hex"); + const backing = new Uint8Array(raw.length + 1); + backing.set(raw, 1); + const u8 = backing.subarray(1); + const chans = []; + const ab = { copyToChannel(f, ch) { chans[ch] = Array.from(f); } }; + const frames = fill(ab, u8, c.channels); + out.push({ frames, chans }); +} +process.stdout.write(JSON.stringify(out)); +""" + cases = [(1, _pcm(300, 1, seed=7)), (2, _pcm(300, 2, seed=8))] + payload = { + "src": web_audio._FILL_JS, + "cases": [{"channels": ch, "hex": pcm.hex()} for ch, pcm in cases], + } + result = subprocess.run( + ["node", "-e", script], + input=json.dumps(payload), + capture_output=True, + text=True, + timeout=60, + check=True, + ) + got = json.loads(result.stdout) + for (channels, pcm), case in zip(cases, got): + self.assertEqual(300, case["frames"]) + expected = [_f32(chan) for chan in _reference_channels(pcm, channels)] + self.assertEqual(expected, case["chans"]) + + +if __name__ == "__main__": + unittest.main()