From 7d8855a931f5b2307072dd09c754fe4483880d33 Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Fri, 6 Nov 2026 12:33:20 +0800 Subject: [PATCH] fix(frontend): initialize consumed frames without LFR Handle default online frontend features without an unbound cursor and finalize the last nonempty offline VAD chunk. Signed-off-by: LauraGPT --- .github/workflows/test-kws-output.yml | 5 + funasr/frontends/wav_frontend.py | 1 + funasr/models/fsmn_vad_streaming/model.py | 4 +- tests/test_wav_frontend_no_lfr.py | 133 ++++++++++++++++++++++ 4 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 tests/test_wav_frontend_no_lfr.py diff --git a/.github/workflows/test-kws-output.yml b/.github/workflows/test-kws-output.yml index 55b2f4cb03..754e333e0c 100644 --- a/.github/workflows/test-kws-output.yml +++ b/.github/workflows/test-kws-output.yml @@ -8,12 +8,14 @@ on: - "funasr/models/sanm_kws/**" - "funasr/models/sanm_kws_streaming/**" - "funasr/frontends/wav_frontend.py" + - "funasr/models/fsmn_vad_streaming/model.py" - "funasr/auto/auto_model.py" - "funasr/utils/datadir_writer.py" - "funasr/utils/kws_utils.py" - "tests/test_kws_optional_output.py" - "tests/test_kws_streaming_continuity.py" - "tests/test_kws_utils.py" + - "tests/test_wav_frontend_no_lfr.py" - "tests/test_fsmn_vad_streaming_buffers.py" - "tests/test_dynamic_streaming_vad.py" - "tests/test_fsmn_vad_dynamic_silence.py" @@ -26,12 +28,14 @@ on: - "funasr/models/sanm_kws/**" - "funasr/models/sanm_kws_streaming/**" - "funasr/frontends/wav_frontend.py" + - "funasr/models/fsmn_vad_streaming/model.py" - "funasr/auto/auto_model.py" - "funasr/utils/datadir_writer.py" - "funasr/utils/kws_utils.py" - "tests/test_kws_optional_output.py" - "tests/test_kws_streaming_continuity.py" - "tests/test_kws_utils.py" + - "tests/test_wav_frontend_no_lfr.py" - "tests/test_fsmn_vad_streaming_buffers.py" - "tests/test_dynamic_streaming_vad.py" - "tests/test_fsmn_vad_dynamic_silence.py" @@ -63,6 +67,7 @@ jobs: tests/test_kws_optional_output.py \ tests/test_kws_streaming_continuity.py \ tests/test_kws_utils.py \ + tests/test_wav_frontend_no_lfr.py \ tests/test_fsmn_vad_streaming_buffers.py \ tests/test_dynamic_streaming_vad.py \ tests/test_fsmn_vad_dynamic_silence.py diff --git a/funasr/frontends/wav_frontend.py b/funasr/frontends/wav_frontend.py index 593cc14462..cb671e6df6 100644 --- a/funasr/frontends/wav_frontend.py +++ b/funasr/frontends/wav_frontend.py @@ -488,6 +488,7 @@ def forward_lfr_cmvn( lfr_splice_frame_idxs = [] for i in range(batch_size): mat = input[i, : input_lengths[i], :] + lfr_splice_frame_idx = mat.size(0) if self.lfr_m != 1 or self.lfr_n != 1: # update self.lfr_splice_cache in self.apply_lfr # mat, self.lfr_splice_cache[i], lfr_splice_frame_idx = self.apply_lfr(mat, self.lfr_m, self.lfr_n, self.lfr_splice_cache[i], diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.py index 0b01c83846..b59862e6b7 100644 --- a/funasr/models/fsmn_vad_streaming/model.py +++ b/funasr/models/fsmn_vad_streaming/model.py @@ -1005,7 +1005,9 @@ def inference( audio_sample = torch.cat((cache["prev_samples"], audio_sample_list[0])) - n = int(len(audio_sample) // chunk_stride_samples + int(_is_final)) + n = len(audio_sample) // chunk_stride_samples + if _is_final and len(audio_sample) % chunk_stride_samples: + n += 1 m = int(len(audio_sample) % chunk_stride_samples * (1 - int(_is_final))) segments = [] # Keep explicit fixed-threshold requests from being overwritten by the dynamic schedule. diff --git a/tests/test_wav_frontend_no_lfr.py b/tests/test_wav_frontend_no_lfr.py new file mode 100644 index 0000000000..a9af796626 --- /dev/null +++ b/tests/test_wav_frontend_no_lfr.py @@ -0,0 +1,133 @@ +import pytest +import torch + +from funasr.frontends.wav_frontend import WavFrontendOnline +from funasr.models.fsmn_vad_streaming import model as vad_model +from funasr.utils import fbank as kaldi + + +def make_frontend(tmp_path, cmvn): + cmvn_file = None + if cmvn: + cmvn_file = tmp_path / "no_lfr.mvn" + cmvn_file.write_text( + " 80 80\n 1 [ " + + " ".join(["0.25"] * 80) + + " ]\n 80 80\n 1 [ " + + " ".join(["2"] * 80) + + " ]\n" + ) + return WavFrontendOnline(cmvn_file=cmvn_file, dither=0.0) + + +@pytest.mark.parametrize("cmvn", (False, True)) +@pytest.mark.parametrize("is_final", (False, True)) +@pytest.mark.parametrize("lengths", ((5, 3, 0), (0, 0))) +def test_no_lfr_reports_valid_consumed_frames(tmp_path, cmvn, is_final, lengths): + frontend = make_frontend(tmp_path, cmvn) + inputs = torch.arange(len(lengths) * 7 * 80, dtype=torch.float32).reshape( + len(lengths), 7, 80 + ) + original = inputs.clone() + cache = {} + + feats, actual_lengths, consumed = frontend.forward_lfr_cmvn( + inputs, torch.tensor(lengths), is_final=is_final, cache=cache + ) + + assert actual_lengths.tolist() == list(lengths) + assert consumed.tolist() == list(lengths) + assert feats.shape == (len(lengths), max(lengths), 80) + assert feats.dtype == torch.float32 + assert cache == {} + for index, length in enumerate(lengths): + expected = original[index, :length] + if cmvn: + expected = (expected + 0.25) * 2 + torch.testing.assert_close(feats[index, :length], expected, rtol=0, atol=0) + assert torch.count_nonzero(feats[index, length:]) == 0 + + +@pytest.mark.parametrize("cmvn", (False, True)) +@pytest.mark.parametrize("empty_final", (False, True)) +@pytest.mark.parametrize("chunks", ((1600,), (400,) * 4, (160,) * 10, (80, 320, 40, 1160))) +def test_no_lfr_stream_matches_full_filterbank(tmp_path, cmvn, empty_final, chunks): + frontend = make_frontend(tmp_path, cmvn) + audio = 0.2 * torch.sin(torch.arange(1600, dtype=torch.float32) * (2 * torch.pi * 440 / 16000)) + expected = kaldi.fbank( + audio.unsqueeze(0) * (1 << 15), num_mel_bins=80, frame_length=25, frame_shift=10, + sample_frequency=16000, window_type="hamming", dither=0.0, energy_floor=0.0, + ) + if cmvn: + expected = (expected + 0.25) * 2 + cache, outputs, offset, emitted = {}, [], 0, 0 + calls = chunks + ((0,) if empty_final else ()) + for index, size in enumerate(calls): + chunk = audio[offset : offset + size] + offset += size + feats, lengths = frontend( + chunk.unsqueeze(0), torch.tensor([size]), cache=cache, + is_final=index == len(calls) - 1, return_waveform=True, + ) + count = int(lengths[0]) if lengths.numel() else 0 + if count: + outputs.append(feats[0, :count]) + start = emitted * 160 + stop = start + (count - 1) * 160 + 400 + torch.testing.assert_close(cache["aligned_waveforms"][0], audio[start:stop], rtol=0, atol=0) + emitted += count + else: + assert cache["aligned_waveforms"].numel() == 0 + assert cache["reserve_waveforms"].numel() == 0 + assert emitted == expected.shape[0] == 8 + torch.testing.assert_close(torch.cat(outputs), expected, rtol=1e-5, atol=1e-5) + + +@pytest.mark.parametrize("size", (0, 160, 399)) +@pytest.mark.parametrize("empty_final", (False, True)) +def test_no_lfr_short_stream_has_no_frames(tmp_path, size, empty_final): + frontend = make_frontend(tmp_path, False) + cache = {} + for index, length in enumerate((size, 0) if empty_final else (size,)): + feats, lengths = frontend( + torch.zeros(1, length), torch.tensor([length]), cache=cache, + is_final=not empty_final or index == 1, return_waveform=True, + ) + assert feats.numel() == 0 + assert not lengths.numel() or lengths.sum() == 0 + assert cache["aligned_waveforms"].numel() == 0 + + +@pytest.mark.parametrize("lfr_m", (1, 5)) +@pytest.mark.parametrize("offset_ms", (0, 1000)) +@pytest.mark.parametrize("chunk_ms", (20, 100, 5000)) +def test_offline_vad_finalizes_last_nonempty_chunk(lfr_m, offset_ms, chunk_ms): + class ScriptedEncoder(torch.nn.Module): + def __init__(self): + super().__init__() + self.frames = 0 + + def forward(self, feats, cache=None): + time_ms = (torch.arange(feats.shape[1]) + self.frames) * 10 + speech = ((time_ms >= 500 + offset_ms) & (time_ms < 1500 + offset_ms)) | ( + (time_ms >= 2500 + offset_ms) & (time_ms < 3200 + offset_ms) + ) + self.frames += feats.shape[1] + probability = torch.where(speech, 0.999, 0.001) + return torch.stack((1 - probability, probability), dim=-1).unsqueeze(0) + + model = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming) + torch.nn.Module.__init__(model) + model.encoder = ScriptedEncoder() + model.vad_opts = vad_model.VADXOptions() + audio = 0.2 * torch.sin( + torch.arange((4000 + offset_ms) * 16, dtype=torch.float32) + * (2 * torch.pi * 440 / 16000) + ) + frontend = WavFrontendOnline(lfr_m=lfr_m, lfr_n=1, dither=0.0) + result, _ = model.inference( + [audio], key=["fixture"], cache={}, frontend=frontend, device="cpu", + chunk_size=chunk_ms, is_streaming_input=False, is_final=True, dynamic_silence=True, + ) + assert model.encoder.frames == (4000 + offset_ms) // 10 - 2 + assert result == [{"key": "fixture", "value": [[240 + offset_ms, 3980 + offset_ms]]}]