fix(frontend): handle default no-LFR online features

.github/workflows/test-kws-output.yml

@@ -8,12 +8,14 @@ on:
       - "funasr/models/sanm_kws/**"
       - "funasr/models/sanm_kws_streaming/**"
       - "funasr/frontends/wav_frontend.py"
+      - "funasr/models/fsmn_vad_streaming/model.py"
       - "funasr/auto/auto_model.py"
       - "funasr/utils/datadir_writer.py"
       - "funasr/utils/kws_utils.py"
       - "tests/test_kws_optional_output.py"
       - "tests/test_kws_streaming_continuity.py"
       - "tests/test_kws_utils.py"
+      - "tests/test_wav_frontend_no_lfr.py"
       - "tests/test_fsmn_vad_streaming_buffers.py"
       - "tests/test_dynamic_streaming_vad.py"
       - "tests/test_fsmn_vad_dynamic_silence.py"
@@ -26,12 +28,14 @@ on:
       - "funasr/models/sanm_kws/**"
       - "funasr/models/sanm_kws_streaming/**"
       - "funasr/frontends/wav_frontend.py"
+      - "funasr/models/fsmn_vad_streaming/model.py"
       - "funasr/auto/auto_model.py"
       - "funasr/utils/datadir_writer.py"
       - "funasr/utils/kws_utils.py"
       - "tests/test_kws_optional_output.py"
       - "tests/test_kws_streaming_continuity.py"
       - "tests/test_kws_utils.py"
+      - "tests/test_wav_frontend_no_lfr.py"
       - "tests/test_fsmn_vad_streaming_buffers.py"
       - "tests/test_dynamic_streaming_vad.py"
       - "tests/test_fsmn_vad_dynamic_silence.py"
@@ -63,6 +67,7 @@ jobs:
             tests/test_kws_optional_output.py \
             tests/test_kws_streaming_continuity.py \
             tests/test_kws_utils.py \
+            tests/test_wav_frontend_no_lfr.py \
             tests/test_fsmn_vad_streaming_buffers.py \
             tests/test_dynamic_streaming_vad.py \
             tests/test_fsmn_vad_dynamic_silence.py

funasr/frontends/wav_frontend.py

@@ -488,6 +488,7 @@ def forward_lfr_cmvn(
         lfr_splice_frame_idxs = []
         for i in range(batch_size):
             mat = input[i, : input_lengths[i], :]
+            lfr_splice_frame_idx = mat.size(0)
             if self.lfr_m != 1 or self.lfr_n != 1:
                 # update self.lfr_splice_cache in self.apply_lfr
                 # mat, self.lfr_splice_cache[i], lfr_splice_frame_idx = self.apply_lfr(mat, self.lfr_m, self.lfr_n, self.lfr_splice_cache[i],

funasr/models/fsmn_vad_streaming/model.py

@@ -1005,7 +1005,9 @@ def inference(
 
         audio_sample = torch.cat((cache["prev_samples"], audio_sample_list[0]))
 
-        n = int(len(audio_sample) // chunk_stride_samples + int(_is_final))
+        n = len(audio_sample) // chunk_stride_samples
+        if _is_final and len(audio_sample) % chunk_stride_samples:
+            n += 1
         m = int(len(audio_sample) % chunk_stride_samples * (1 - int(_is_final)))
         segments = []
         # Keep explicit fixed-threshold requests from being overwritten by the dynamic schedule.

tests/test_wav_frontend_no_lfr.py

@@ -0,0 +1,133 @@
+import pytest
+import torch
+
+from funasr.frontends.wav_frontend import WavFrontendOnline
+from funasr.models.fsmn_vad_streaming import model as vad_model
+from funasr.utils import fbank as kaldi
+
+
+def make_frontend(tmp_path, cmvn):
+    cmvn_file = None
+    if cmvn:
+        cmvn_file = tmp_path / "no_lfr.mvn"
+        cmvn_file.write_text(
+            "<AddShift> 80 80\n<LearnRateCoef> 1 [ "
+            + " ".join(["0.25"] * 80)
+            + " ]\n<Rescale> 80 80\n<LearnRateCoef> 1 [ "
+            + " ".join(["2"] * 80)
+            + " ]\n"
+        )
+    return WavFrontendOnline(cmvn_file=cmvn_file, dither=0.0)
+
+
+@pytest.mark.parametrize("cmvn", (False, True))
+@pytest.mark.parametrize("is_final", (False, True))
+@pytest.mark.parametrize("lengths", ((5, 3, 0), (0, 0)))
+def test_no_lfr_reports_valid_consumed_frames(tmp_path, cmvn, is_final, lengths):
+    frontend = make_frontend(tmp_path, cmvn)
+    inputs = torch.arange(len(lengths) * 7 * 80, dtype=torch.float32).reshape(
+        len(lengths), 7, 80
+    )
+    original = inputs.clone()
+    cache = {}
+
+    feats, actual_lengths, consumed = frontend.forward_lfr_cmvn(
+        inputs, torch.tensor(lengths), is_final=is_final, cache=cache
+    )
+
+    assert actual_lengths.tolist() == list(lengths)
+    assert consumed.tolist() == list(lengths)
+    assert feats.shape == (len(lengths), max(lengths), 80)
+    assert feats.dtype == torch.float32
+    assert cache == {}
+    for index, length in enumerate(lengths):
+        expected = original[index, :length]
+        if cmvn:
+            expected = (expected + 0.25) * 2
+        torch.testing.assert_close(feats[index, :length], expected, rtol=0, atol=0)
+        assert torch.count_nonzero(feats[index, length:]) == 0
+
+
+@pytest.mark.parametrize("cmvn", (False, True))
+@pytest.mark.parametrize("empty_final", (False, True))
+@pytest.mark.parametrize("chunks", ((1600,), (400,) * 4, (160,) * 10, (80, 320, 40, 1160)))
+def test_no_lfr_stream_matches_full_filterbank(tmp_path, cmvn, empty_final, chunks):
+    frontend = make_frontend(tmp_path, cmvn)
+    audio = 0.2 * torch.sin(torch.arange(1600, dtype=torch.float32) * (2 * torch.pi * 440 / 16000))
+    expected = kaldi.fbank(
+        audio.unsqueeze(0) * (1 << 15), num_mel_bins=80, frame_length=25, frame_shift=10,
+        sample_frequency=16000, window_type="hamming", dither=0.0, energy_floor=0.0,
+    )
+    if cmvn:
+        expected = (expected + 0.25) * 2
+    cache, outputs, offset, emitted = {}, [], 0, 0
+    calls = chunks + ((0,) if empty_final else ())
+    for index, size in enumerate(calls):
+        chunk = audio[offset : offset + size]
+        offset += size
+        feats, lengths = frontend(
+            chunk.unsqueeze(0), torch.tensor([size]), cache=cache,
+            is_final=index == len(calls) - 1, return_waveform=True,
+        )
+        count = int(lengths[0]) if lengths.numel() else 0
+        if count:
+            outputs.append(feats[0, :count])
+            start = emitted * 160
+            stop = start + (count - 1) * 160 + 400
+            torch.testing.assert_close(cache["aligned_waveforms"][0], audio[start:stop], rtol=0, atol=0)
+            emitted += count
+        else:
+            assert cache["aligned_waveforms"].numel() == 0
+        assert cache["reserve_waveforms"].numel() == 0
+    assert emitted == expected.shape[0] == 8
+    torch.testing.assert_close(torch.cat(outputs), expected, rtol=1e-5, atol=1e-5)
+
+
+@pytest.mark.parametrize("size", (0, 160, 399))
+@pytest.mark.parametrize("empty_final", (False, True))
+def test_no_lfr_short_stream_has_no_frames(tmp_path, size, empty_final):
+    frontend = make_frontend(tmp_path, False)
+    cache = {}
+    for index, length in enumerate((size, 0) if empty_final else (size,)):
+        feats, lengths = frontend(
+            torch.zeros(1, length), torch.tensor([length]), cache=cache,
+            is_final=not empty_final or index == 1, return_waveform=True,
+        )
+        assert feats.numel() == 0
+        assert not lengths.numel() or lengths.sum() == 0
+        assert cache["aligned_waveforms"].numel() == 0
+
+
+@pytest.mark.parametrize("lfr_m", (1, 5))
+@pytest.mark.parametrize("offset_ms", (0, 1000))
+@pytest.mark.parametrize("chunk_ms", (20, 100, 5000))
+def test_offline_vad_finalizes_last_nonempty_chunk(lfr_m, offset_ms, chunk_ms):
+    class ScriptedEncoder(torch.nn.Module):
+        def __init__(self):
+            super().__init__()
+            self.frames = 0
+
+        def forward(self, feats, cache=None):
+            time_ms = (torch.arange(feats.shape[1]) + self.frames) * 10
+            speech = ((time_ms >= 500 + offset_ms) & (time_ms < 1500 + offset_ms)) | (
+                (time_ms >= 2500 + offset_ms) & (time_ms < 3200 + offset_ms)
+            )
+            self.frames += feats.shape[1]
+            probability = torch.where(speech, 0.999, 0.001)
+            return torch.stack((1 - probability, probability), dim=-1).unsqueeze(0)
+
+    model = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming)
+    torch.nn.Module.__init__(model)
+    model.encoder = ScriptedEncoder()
+    model.vad_opts = vad_model.VADXOptions()
+    audio = 0.2 * torch.sin(
+        torch.arange((4000 + offset_ms) * 16, dtype=torch.float32)
+        * (2 * torch.pi * 440 / 16000)
+    )
+    frontend = WavFrontendOnline(lfr_m=lfr_m, lfr_n=1, dither=0.0)
+    result, _ = model.inference(
+        [audio], key=["fixture"], cache={}, frontend=frontend, device="cpu",
+        chunk_size=chunk_ms, is_streaming_input=False, is_final=True, dynamic_silence=True,
+    )
+    assert model.encoder.frames == (4000 + offset_ms) // 10 - 2
+    assert result == [{"key": "fixture", "value": [[240 + offset_ms, 3980 + offset_ms]]}]