From a45faf98febc93101c15f383801078fe4b654d69 Mon Sep 17 00:00:00 2001 From: Tanishq Date: Mon, 2 Nov 2026 00:05:43 +0530 Subject: [PATCH 1/3] fix(onnx): avoid UnboundLocalError in Fsmn_vad_online for chunks without feature frames Fsmn_vad_online.__call__ fetched vad_scorer from param_dict only inside "if feats.size != 0:", but always stored it back afterwards. When WavFrontendOnline returns no frames for a chunk, the call raised UnboundLocalError. This happens on the first call when the first chunk is shorter than 720 samples (45 ms at 16 kHz, so 10, 20 or 30 ms streaming frames fail at once), and later in the stream when a chunk does not complete a new 25 ms fbank window. Fetch the scorer before the check, as the frontend already is. The frontend keeps the samples of such chunks in its caches, so the segments match those of the demo's 1600-sample chunks. --- .../workflows/test-funasr-onnx-package.yml | 5 + .../python/onnxruntime/funasr_onnx/vad_bin.py | 2 +- ...est_funasr_onnx_vad_online_short_chunks.py | 95 +++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 tests/test_funasr_onnx_vad_online_short_chunks.py diff --git a/.github/workflows/test-funasr-onnx-package.yml b/.github/workflows/test-funasr-onnx-package.yml index 75d735f0b7..e2418bb07d 100644 --- a/.github/workflows/test-funasr-onnx-package.yml +++ b/.github/workflows/test-funasr-onnx-package.yml @@ -11,6 +11,7 @@ on: - "tests/test_funasr_onnx_installed.py" - "tests/test_funasr_onnx_paraformer_online_overlap.py" - "tests/test_funasr_onnx_paraformer_online_empty_final.py" + - "tests/test_funasr_onnx_vad_online_short_chunks.py" - "tests/test_rich_transcription_postprocess.py" - "tests/test_rich_transcription_event_segments.py" push: @@ -25,6 +26,7 @@ on: - "tests/test_funasr_onnx_installed.py" - "tests/test_funasr_onnx_paraformer_online_overlap.py" - "tests/test_funasr_onnx_paraformer_online_empty_final.py" + - "tests/test_funasr_onnx_vad_online_short_chunks.py" - "tests/test_rich_transcription_postprocess.py" - "tests/test_rich_transcription_event_segments.py" workflow_dispatch: @@ -105,6 +107,9 @@ jobs: - name: Verify empty split-final Paraformer result run: python tests/test_funasr_onnx_paraformer_online_empty_final.py + - name: Verify streaming VAD short chunks + run: python tests/test_funasr_onnx_vad_online_short_chunks.py + - name: Verify source distribution error paths run: | python -m pip install --force-reinstall --no-deps dist/funasr-onnx/*.tar.gz diff --git a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py index 1c98b7f92b..6bfa7ea922 100644 --- a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py +++ b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py @@ -289,11 +289,11 @@ def __call__(self, audio_in: np.ndarray, **kwargs) -> List: is_final = param_dict.get("is_final", False) frontend: WavFrontendOnline = param_dict.get("frontend", WavFrontendOnline(cmvn_file=self.cmvn_file, **self.config["frontend_conf"])) feats, feats_len = self.extract_feat(frontend=frontend, waveforms=waveforms, is_final=is_final) + vad_scorer = param_dict.get("vad_scorer", E2EVadModel(self.config["model_conf"])) segments = [] if feats.size != 0: in_cache = param_dict.get("in_cache", list()) in_cache = self.prepare_cache(in_cache) - vad_scorer = param_dict.get("vad_scorer", E2EVadModel(self.config["model_conf"])) try: inputs = [feats] inputs.extend(in_cache) diff --git a/tests/test_funasr_onnx_vad_online_short_chunks.py b/tests/test_funasr_onnx_vad_online_short_chunks.py new file mode 100644 index 0000000000..5602e1fe33 --- /dev/null +++ b/tests/test_funasr_onnx_vad_online_short_chunks.py @@ -0,0 +1,95 @@ +"""Run against an installed funasr-onnx package.""" + +import importlib.util +import itertools +import unittest + +import numpy as np + +if importlib.util.find_spec("funasr_onnx") is None: + raise unittest.SkipTest("Install the standalone funasr-onnx distribution first.") + +from funasr_onnx.vad_bin import Fsmn_vad_online + +# frontend_conf of funasr/models/fsmn_vad_streaming/template.yaml +FRONTEND_CONF = { + "fs": 16000, + "window": "hamming", + "n_mels": 80, + "frame_length": 25, + "frame_shift": 10, + "dither": 0.0, + "lfr_m": 5, + "lfr_n": 1, +} + +# Segments (ms) of make_audio() streamed in 1600-sample chunks, the step used by the demo. +EXPECTED = [[[[220, -1]]], [[[-1, 1660]]], [[[2220, -1]]], [[[-1, 3360]]]] + + +def fake_infer(inputs): + # Score loud frames as speech instead of running the ONNX session. + sil = np.where(inputs[0][0].mean(-1) > 5.0, 0.05, 0.95) + scores = np.stack((sil, 1.0 - sil), axis=-1)[None].astype(np.float32) + return scores, inputs[1:] + + +def make_model(): + # Keep __call__, WavFrontendOnline and E2EVadModel real; no model files. + model = Fsmn_vad_online.__new__(Fsmn_vad_online) + model.cmvn_file = None + # E2EVadModel's defaults equal the template's model_conf. + model.config = {"frontend_conf": FRONTEND_CONF, "model_conf": {}} + model.max_end_sil = 800 + model.encoder_conf = {"fsmn_layers": 1, "proj_dim": 2, "lorder": 2} + model.infer = fake_infer + return model + + +def make_audio(): + # 4 s at 16 kHz: noise at 0.5-1.5 s and 2.5-3.2 s, silence elsewhere. + rng = np.random.default_rng(0) + audio = np.zeros(64000, dtype=np.float32) + for start, end in ((8000, 24000), (40000, 51200)): + audio[start:end] = rng.uniform(-0.3, 0.3, end - start) + return audio + + +def stream(chunk_sizes): + # Feed the audio in chunks, cycling through chunk_sizes; the last chunk is final. + model = make_model() + audio = make_audio() + param_dict = {"in_cache": []} + results = [] + offset = 0 + for size in itertools.cycle(chunk_sizes): + chunk = audio[offset : offset + size] + offset += size + param_dict["is_final"] = offset >= len(audio) + segments = model(audio_in=chunk, param_dict=param_dict) + if segments: + results.append(segments) + if param_dict["is_final"]: + return results + + +class FsmnVadOnlineShortChunkTest(unittest.TestCase): + def test_demo_chunk_size(self): + self.assertEqual(stream([1600]), EXPECTED) + + def test_chunks_shorter_than_first_lfr_frame(self): + # The first LFR frame needs 720 samples (45 ms), so these first calls yield no features. + for size in (160, 320, 480): + with self.subTest(size=size): + self.assertEqual(stream([size]), EXPECTED) + + def test_packets_that_complete_no_fbank_frame(self): + # The first 1600-sample chunk leaves 320 samples for the next 400-sample fbank window, + # so a 40-sample packet, or the second of two 80-sample packets, completes no frame. + for chunk_sizes in ([1600, 40], [1600, 80, 80]): + with self.subTest(chunk_sizes=chunk_sizes): + self.assertEqual(stream(chunk_sizes), EXPECTED) + + +if __name__ == "__main__": + unittest.main() From f903065516742f5274daa6fe6ab9a0d5c6a7e07c Mon Sep 17 00:00:00 2001 From: Tanishq Date: Mon, 2 Nov 2026 04:37:58 +0530 Subject: [PATCH 2/3] fix(onnx): return [] and reset state at the end of short VAD streams A final Fsmn_vad_online call on a stream shorter than 720 samples (45 ms, the first LFR frame) crashed or left state in param_dict: - Under 400 samples there is no fbank frame, and the final flush called np.stack on the empty LFR splice cache (ValueError). - With 400-719 samples, a final call that completed a new fbank frame returned before cache_reset(), so the next stream on the same param_dict started with the old samples (segments 30 ms late after a 480-sample stream). - A final call that completed no new fbank frame flushed the one or two cached frames with less than one 25 ms window of waveform, so E2EVadModel.GetFrameState had no decibel value (IndexError) and the scorer was never reset. WavFrontendOnline.extract_fbank now skips the flush when no fbank frame is cached and resets its caches on every final call. GetFrameState scores a frame without a decibel value as silence, as FsmnVADStreaming.GetFrameState does. These streams now return [] and leave the frontend and scorer reset; streams of 720 samples or more behave as before. --- .../onnxruntime/funasr_onnx/utils/e2e_vad.py | 4 +++ .../onnxruntime/funasr_onnx/utils/frontend.py | 6 ++-- ...est_funasr_onnx_vad_online_short_chunks.py | 36 +++++++++++++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/runtime/python/onnxruntime/funasr_onnx/utils/e2e_vad.py b/runtime/python/onnxruntime/funasr_onnx/utils/e2e_vad.py index 41e357ee78..a991cb3baa 100644 --- a/runtime/python/onnxruntime/funasr_onnx/utils/e2e_vad.py +++ b/runtime/python/onnxruntime/funasr_onnx/utils/e2e_vad.py @@ -452,6 +452,10 @@ def LatencyFrmNumAtStartPoint(self) -> int: def GetFrameState(self, t: int) -> FrameState: frame_state = FrameState.kFrameStateInvalid + # A final call can flush frames that lack a full 25 ms window (streams under 45 ms) and + # so a decibel value; score them as silence, as FsmnVADStreaming.GetFrameState does. + if t >= len(self.decibel): + return FrameState.kFrameStateSil cur_decibel = self.decibel[t] cur_snr = cur_decibel - self.noise_average_decibel # for each frame, calc log posterior probability of each state diff --git a/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py b/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py index 54f9deb71b..f150ccbf8a 100644 --- a/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py +++ b/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py @@ -361,9 +361,11 @@ def extract_fbank( self.lfr_splice_cache[i] = np.concatenate( (self.lfr_splice_cache[i], feats[i]), axis=0 ) - return np.empty(0, dtype=np.float32), feats_lengths + # No LFR frame yet; a final call still resets the caches below. + feats = np.empty(0, dtype=np.float32) else: - if is_final: + # A stream without a full 25 ms window has no fbank frame to flush. + if is_final and self.lfr_splice_cache: self.waveforms = ( waveforms if self.reserve_waveforms is None else self.reserve_waveforms ) diff --git a/tests/test_funasr_onnx_vad_online_short_chunks.py b/tests/test_funasr_onnx_vad_online_short_chunks.py index 5602e1fe33..99b9131c21 100644 --- a/tests/test_funasr_onnx_vad_online_short_chunks.py +++ b/tests/test_funasr_onnx_vad_online_short_chunks.py @@ -55,11 +55,12 @@ def make_audio(): return audio -def stream(chunk_sizes): +def stream(chunk_sizes, param_dict=None): # Feed the audio in chunks, cycling through chunk_sizes; the last chunk is final. model = make_model() audio = make_audio() - param_dict = {"in_cache": []} + if param_dict is None: + param_dict = {"in_cache": []} results = [] offset = 0 for size in itertools.cycle(chunk_sizes): @@ -73,6 +74,16 @@ def stream(chunk_sizes): return results +def short_stream(sizes, param_dict): + # Send silent chunks of these sizes as a whole stream; the last call is final. + model = make_model() + results = [] + for i, size in enumerate(sizes): + param_dict["is_final"] = i == len(sizes) - 1 + results.append(model(audio_in=np.zeros(size, dtype=np.float32), param_dict=param_dict)) + return results + + class FsmnVadOnlineShortChunkTest(unittest.TestCase): def test_demo_chunk_size(self): self.assertEqual(stream([1600]), EXPECTED) @@ -90,6 +101,27 @@ def test_packets_that_complete_no_fbank_frame(self): with self.subTest(chunk_sizes=chunk_sizes): self.assertEqual(stream(chunk_sizes), EXPECTED) + def test_short_first_and_final_call(self): + # Under 400 samples (25 ms) the stream has no fbank frame to flush. + for size in (0, 160, 320, 399): + with self.subTest(size=size): + self.assertEqual(short_stream([size], {"in_cache": []}), [[]]) + + def test_empty_final_call_after_partial_chunk(self): + # 400-719 samples give one or two fbank frames but no LFR frame. The empty final call + # flushes them with less than one 25 ms window of waveform, so they have no decibel value. + for size in (400, 480, 560, 719): + with self.subTest(size=size): + self.assertEqual(short_stream([size, 0], {"in_cache": []}), [[], []]) + + def test_param_dict_reuse_after_short_stream(self): + # The final call of a short stream resets the frontend and scorer for the next stream. + for sizes in ([320], [480, 0], [480]): + with self.subTest(sizes=sizes): + param_dict = {"in_cache": []} + self.assertEqual(short_stream(sizes, param_dict), [[]] * len(sizes)) + self.assertEqual(stream([1600], param_dict), EXPECTED) + if __name__ == "__main__": unittest.main() From 2700fa5aba4ea19d80b88c9393928bedf13b53ff Mon Sep 17 00:00:00 2001 From: Tanishq Date: Mon, 2 Nov 2026 06:47:01 +0530 Subject: [PATCH 3/3] fix(onnx): reset the FSMN cache at the end of a VAD stream A final call now clears param_dict["in_cache"], as init_cache() in FsmnVADStreaming and Reset() in the C++ FsmnVadOnline do, so a param_dict reused after a stream starts like a fresh one. A 400-719 sample stream whose final call completes no new feature frame used to leave the cache from its one model call behind. The test stub now changes the cache it returns, and the reuse test checks that the cache is empty after the short stream. --- runtime/python/onnxruntime/funasr_onnx/vad_bin.py | 3 +++ tests/test_funasr_onnx_vad_online_short_chunks.py | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py index 6bfa7ea922..fc132d17cf 100644 --- a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py +++ b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py @@ -309,6 +309,9 @@ def __call__(self, audio_in: np.ndarray, **kwargs) -> List: logging.warning("input wav is silence or noise") segments = [] param_dict.update({"frontend": frontend, "vad_scorer": vad_scorer}) + if is_final: + # Start the next stream with a fresh FSMN cache, as init_cache() and Reset() do. + param_dict["in_cache"] = [] return segments def load_data(self, wav_content: Union[str, np.ndarray, List[str]], fs: int = None) -> List: diff --git a/tests/test_funasr_onnx_vad_online_short_chunks.py b/tests/test_funasr_onnx_vad_online_short_chunks.py index 99b9131c21..012e34b398 100644 --- a/tests/test_funasr_onnx_vad_online_short_chunks.py +++ b/tests/test_funasr_onnx_vad_online_short_chunks.py @@ -31,7 +31,8 @@ def fake_infer(inputs): # Score loud frames as speech instead of running the ONNX session. sil = np.where(inputs[0][0].mean(-1) > 5.0, 0.05, 0.95) scores = np.stack((sil, 1.0 - sil), axis=-1)[None].astype(np.float32) - return scores, inputs[1:] + # Change the cache so a cache carried into the next stream shows up. + return scores, [cache + 1.0 for cache in inputs[1:]] def make_model(): @@ -115,11 +116,12 @@ def test_empty_final_call_after_partial_chunk(self): self.assertEqual(short_stream([size, 0], {"in_cache": []}), [[], []]) def test_param_dict_reuse_after_short_stream(self): - # The final call of a short stream resets the frontend and scorer for the next stream. + # The final call of a short stream resets the frontend, scorer and cache for the next stream. for sizes in ([320], [480, 0], [480]): with self.subTest(sizes=sizes): param_dict = {"in_cache": []} self.assertEqual(short_stream(sizes, param_dict), [[]] * len(sizes)) + self.assertEqual(param_dict["in_cache"], []) self.assertEqual(stream([1600], param_dict), EXPECTED)