From 290c1335e506b0f4554398b0717d29d820816cdb Mon Sep 17 00:00:00 2001 From: BillVan1945 <307193874+BillVan1945@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:52:18 +0900 Subject: [PATCH 1/3] fix(normalize): correct UtteranceMVN variance normalization --- funasr/models/normalize/utterance_mvn.py | 3 +- tests/test_utterance_mvn.py | 268 +++++++++++++++++++++++ 2 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 tests/test_utterance_mvn.py diff --git a/funasr/models/normalize/utterance_mvn.py b/funasr/models/normalize/utterance_mvn.py index 9128c6d044..8c8353e698 100644 --- a/funasr/models/normalize/utterance_mvn.py +++ b/funasr/models/normalize/utterance_mvn.py @@ -81,9 +81,10 @@ def utterance_mvn( x -= mean if norm_vars: + x = x.masked_fill(make_pad_mask(ilens, x, 1), 0.0) var = x.pow(2).sum(dim=1, keepdim=True) / ilens_ std = torch.clamp(var.sqrt(), min=eps) - x = x / std.sqrt() + x = x / std return x, ilens else: if norm_vars: diff --git a/tests/test_utterance_mvn.py b/tests/test_utterance_mvn.py new file mode 100644 index 0000000000..f74201cd03 --- /dev/null +++ b/tests/test_utterance_mvn.py @@ -0,0 +1,268 @@ +"""Run with: python -m pytest tests/test_utterance_mvn.py -q (CPU and CUDA).""" + +from pathlib import Path + +import pytest +import soundfile as sf +import torch +from omegaconf import OmegaConf + +from funasr import AutoModel +from funasr.frontends.wav_frontend import WavFrontend +from funasr.models.normalize.utterance_mvn import UtteranceMVN, utterance_mvn +from funasr.models.transformer.model import Transformer +from funasr.tokenizer.char_tokenizer import CharTokenizer +from funasr.utils.misc import prepare_model_dir + + +@pytest.fixture(params=["cpu", "cuda"]) +def device(request): + if request.param == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA is unavailable") + return request.param + + +def features(device, dtype=torch.float32): + generator = torch.Generator().manual_seed(17) + x = torch.randn(2, 17, 3, generator=generator, dtype=dtype) * 2 + 5 + return x.to(device), torch.tensor([11, 17], device=device) + + +@pytest.mark.parametrize("dtype", [torch.float32, torch.float64, torch.float16, torch.bfloat16]) +@pytest.mark.parametrize( + "norm_means,norm_vars", [(True, True), (True, False), (False, True), (False, False)] +) +def test_formula_and_switches(device, dtype, norm_means, norm_vars): + x, lengths = features(device, dtype) + actual, actual_lengths = utterance_mvn(x.clone(), lengths, norm_means, norm_vars) + assert actual.dtype == dtype + assert actual.device == x.device + assert actual_lengths is lengths + tolerance = { + torch.float64: 1e-12, + torch.float32: 2e-6, + torch.float16: 0.02, + torch.bfloat16: 0.15, + }[dtype] + for i, length in enumerate(lengths.tolist()): + # Independent oracle: unpadded per-utterance torch.std, evaluated in double. + valid = x[i, :length].double() + mean = valid.mean(0) + expected = valid - mean if norm_means else valid + if norm_vars: + expected = expected / valid.std(0, unbiased=False).clamp_min(1e-20) + torch.testing.assert_close( + actual[i, :length].double(), expected, rtol=tolerance, atol=tolerance + ) + if norm_means and norm_vars: + torch.testing.assert_close( + actual[i, :length].double().var(0, unbiased=False), + torch.ones(3, device=device, dtype=torch.float64), + rtol=tolerance, + atol=tolerance, + ) + if norm_means and not norm_vars: + # Preserve historical mean-only padding. + torch.testing.assert_close( + actual[i, length:].double(), + -mean.expand(17 - length, -1), + rtol=tolerance, + atol=tolerance, + ) + else: + assert torch.count_nonzero(actual[i, length:]) == 0 + + +def test_batch_length_and_order_invariance(device): + x, lengths = features(device) + layer = UtteranceMVN(norm_vars=True) + batch, _ = layer(x.clone(), lengths) + single, inferred_lengths = layer(x[:1, :11].clone()) + reverse, _ = layer(x.flip(0), lengths.flip(0)) + torch.testing.assert_close(single[0], batch[0, :11], rtol=1e-6, atol=1e-6) + torch.testing.assert_close(reverse.flip(0), batch, rtol=0, atol=0) + assert inferred_lengths.tolist() == [11] + + +@pytest.mark.parametrize( + "norm_means,norm_vars,expected", + [ + (True, False, [-2.0, 2.0, -3.0, -3.0]), + (False, True, [0.5, 2.5, 0.0, 0.0]), + (False, False, [1.0, 5.0, 0.0, 0.0]), + ], +) +def test_other_branches_keep_existing_values(device, norm_means, norm_vars, expected): + x = torch.tensor([[[1.0], [5.0], [0.0], [0.0]]], device=device) + actual, _ = utterance_mvn(x, torch.tensor([2], device=device), norm_means, norm_vars) + torch.testing.assert_close(actual.flatten(), x.new_tensor(expected), rtol=0, atol=0) + + +@pytest.mark.parametrize("padding", [0, 2]) +def test_mean_variance_values(device, padding): + x = torch.tensor([1.0, 5.0] + [0.0] * padding, device=device).view(1, -1, 1) + actual, _ = UtteranceMVN(norm_vars=True)(x, torch.tensor([2], device=device)) + expected = x.new_tensor([-1.0, 1.0] + [0.0] * padding) + torch.testing.assert_close(actual.flatten(), expected, rtol=0, atol=0) + + +@pytest.mark.parametrize("norm_means", [True, False]) +def test_eps_clamps_standard_deviation(device, norm_means): + x = torch.tensor([[[1.0], [3.0]]], device=device, dtype=torch.float64) + actual, _ = utterance_mvn(x.clone(), norm_means=norm_means, norm_vars=True, eps=4.0) + expected = (x - 2.0 if norm_means else x) / 4.0 + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +def test_corrected_gradients(device): + x, lengths = features(device, torch.float64) + x.requires_grad_() + reference = x.detach().clone().requires_grad_() + actual, _ = UtteranceMVN(norm_vars=True)(x, lengths) + weights = torch.linspace(0.1, 1.3, x.numel(), device=device).reshape_as(x) + (actual * weights).sum().backward() + loss = 0 + for i, length in enumerate(lengths.tolist()): + valid = reference[i, :length] + normalized = (valid - valid.mean(0)) / valid.std(0, unbiased=False) + loss = loss + (normalized * weights[i, :length]).sum() + loss.backward() + assert torch.isfinite(x.grad).all() + assert torch.count_nonzero(x.grad[0, :11]) > 0 + assert torch.count_nonzero(x.grad[0, 11:]) == 0 + torch.testing.assert_close(x.grad, reference.grad, rtol=1e-9, atol=1e-9) + + +def model_config(normalize_conf): + return dict( + model="Transformer", + input_size=80, + vocab_size=9, + encoder="TransformerEncoder", + encoder_conf=dict( + output_size=32, + attention_heads=2, + linear_units=64, + num_blocks=1, + input_layer="linear", + dropout_rate=0.0, + positional_dropout_rate=0.0, + attention_dropout_rate=0.0, + ), + decoder="TransformerDecoder", + decoder_conf=dict( + attention_heads=2, + linear_units=64, + num_blocks=1, + dropout_rate=0.0, + positional_dropout_rate=0.0, + self_attention_dropout_rate=0.0, + src_attention_dropout_rate=0.0, + ), + ctc_weight=0.5, + normalize="UtteranceMVN", + normalize_conf=normalize_conf, + frontend="WavFrontend", + frontend_conf=dict(fs=16000, dither=0.0), + tokenizer="CharTokenizer", + tokenizer_conf=dict( + token_list=["", "", "", "你", "好", "语", "音", "识", ""] + ), + ) + + +def test_saved_config_and_checkpoint_round_trip(tmp_path): + normalize_conf = dict(norm_means=True, norm_vars=True) + config = model_config(normalize_conf) + original = Transformer(**config).eval() + # Use the training config writer and the local AutoModel config/weight loader. + prepare_model_dir(output_dir=str(tmp_path), **config) + torch.save({"state_dict": original.state_dict()}, tmp_path / "model.pt") + saved = OmegaConf.load(tmp_path / "config.yaml") + assert dict(saved.normalize_conf) == normalize_conf + restored = AutoModel( + model=str(tmp_path), device="cpu", ncpu=1, disable_update=True, ignore_init_mismatch=False + ).model + assert restored.normalize.state_dict() == {} + for key, value in original.state_dict().items(): + torch.testing.assert_close(restored.state_dict()[key], value, rtol=0, atol=0) + x = torch.randn(2, 17, 80) + lengths = torch.tensor([11, 17]) + with torch.no_grad(): + expected, _ = original.encode(x.clone(), lengths) + actual, _ = restored.encode(x.clone(), lengths) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize("norm_vars", [False, True]) +def test_real_audio_transformer_inference(device, norm_vars): + """Real loader/frontend/encoder/CTC; small random model, no downloads or training.""" + audio, fs = sf.read( + Path(__file__).resolve().parents[1] / "runtime/funasr_api/asr_example.wav", dtype="float32" + ) + short = audio[: 2 * fs].copy() + config = model_config(dict(norm_means=True, norm_vars=norm_vars)) + with torch.random.fork_rng(devices=[]): + torch.manual_seed(2026) + model = Transformer(**config).to(device).eval() + frontend = WavFrontend(**config["frontend_conf"]) + tokenizer = CharTokenizer(**config["tokenizer_conf"]) + captured = {} + + def frontend_hook(module, args, output): + captured["frontend_device"] = output[0].device.type + + def raw_hook(module, args): + captured["raw"] = args[0].detach().clone() + + def norm_hook(module, args, output): + captured["normalized"] = output[0].detach().clone() + captured["lengths"] = output[1].detach().clone() + + def encoder_hook(module, args, output): + captured["encoded"] = output[0].detach().clone() + + handles = [ + frontend.register_forward_hook(frontend_hook), + model.normalize.register_forward_pre_hook(raw_hook), + model.normalize.register_forward_hook(norm_hook), + model.encoder.register_forward_hook(encoder_hook), + ] + outputs = [] + try: + for inputs in [[short], [short, audio], [audio, short]]: + with torch.no_grad(): + results, _ = model.inference( + inputs, + batch_size=2, + frontend=frontend, + tokenizer=tokenizer, + device=device, + fs=fs, + ) + assert len(results) == len(inputs) + assert all("text" in result for result in results) + outputs.append(dict(captured)) + finally: + for handle in handles: + handle.remove() + single, batch, reverse = outputs + n = int(single["lengths"][0]) + assert batch["lengths"].tolist() == [198, 553] + assert next(model.parameters()).device.type == device + assert all(output["frontend_device"] == "cpu" for output in outputs) + for name in ["raw", "normalized", "encoded"]: + assert batch[name].device.type == device + torch.testing.assert_close(single[name][0, :n], batch[name][0, :n], rtol=0, atol=1e-5) + torch.testing.assert_close(batch[name][0, :n], reverse[name][1, :n], rtol=0, atol=1e-5) + if norm_vars: + valid = single["raw"][0, :n].double() + expected = (valid - valid.mean(0)) / valid.std(0, unbiased=False) + torch.testing.assert_close(batch["normalized"][0, :n].double(), expected, rtol=0, atol=2e-6) + torch.testing.assert_close( + batch["normalized"][0, :n].var(0, unbiased=False), + torch.ones(80, device=device), + rtol=0, + atol=1e-6, + ) + assert torch.count_nonzero(batch["normalized"][0, n:]) == 0 From 1ad7430b925ef3bdc3fcc7cf93e317fb488c088f Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Sun, 8 Nov 2026 00:44:59 +0800 Subject: [PATCH 2/3] test(normalize): enforce precision and hosted regression coverage Preserve the original PR #3853 commit. Synchronize retained main content to the exact tested integration tree without rewriting history. Use IEEE FP32 only inside audio integration tests, restore backend settings, and keep all assertions and tolerances unchanged. Add the regression file to the existing NumPy matrix. Signed-off-by: LauraGPT --- .../workflows/test-funasr-onnx-package.yml | 9 +- .github/workflows/test-kws-output.yml | 5 + .../workflows/test-numpy-compatibility.yml | 10 ++ README.md | 4 +- README_ja.md | 4 +- README_ko.md | 4 +- README_zh.md | 4 +- docs/python_api.md | 2 +- docs/python_api_zh.md | 2 +- docs/repository_roles.md | 2 +- docs/repository_roles_zh.md | 2 +- funasr/auto/auto_model.py | 6 +- funasr/frontends/wav_frontend.py | 1 + funasr/models/fsmn_vad_streaming/model.py | 44 +++-- funasr/tokenizer/sentencepiece_tokenizer.py | 69 ++++---- funasr/version.txt | 2 +- runtime/python/libtorch/README.md | 15 +- .../libtorch/funasr_torch/utils/frontend.py | 14 +- runtime/python/libtorch/setup.py | 2 +- runtime/python/onnxruntime/README.md | 15 +- .../onnxruntime/funasr_onnx/punc_bin.py | 5 + .../onnxruntime/funasr_onnx/utils/frontend.py | 12 +- .../python/onnxruntime/funasr_onnx/vad_bin.py | 11 +- runtime/python/onnxruntime/setup.py | 2 +- tests/test_fsmn_vad_dynamic_silence.py | 76 +++++++- tests/test_funasr_onnx_installed.py | 2 +- ...est_funasr_onnx_punctuation_empty_input.py | 116 +++++++++++++ tests/test_funasr_onnx_release.py | 16 +- tests/test_funasr_onnx_vad_multiple_inputs.py | 79 ++++++++- tests/test_funasr_torch_package.py | 4 +- tests/test_merge_vad_length.py | 81 +++++++++ tests/test_release_version_contract.py | 10 +- tests/test_runtime_cmvn.py | 105 ++++++++++++ tests/test_sentencepiece_tokenizer.py | 162 ++++++++++++++++++ tests/test_utterance_mvn.py | 16 +- tests/test_wav_frontend_no_lfr.py | 133 ++++++++++++++ 36 files changed, 939 insertions(+), 107 deletions(-) create mode 100644 tests/test_funasr_onnx_punctuation_empty_input.py create mode 100644 tests/test_merge_vad_length.py create mode 100644 tests/test_runtime_cmvn.py create mode 100644 tests/test_sentencepiece_tokenizer.py create mode 100644 tests/test_wav_frontend_no_lfr.py diff --git a/.github/workflows/test-funasr-onnx-package.yml b/.github/workflows/test-funasr-onnx-package.yml index 1a0e1438fd..f94b86e098 100644 --- a/.github/workflows/test-funasr-onnx-package.yml +++ b/.github/workflows/test-funasr-onnx-package.yml @@ -13,6 +13,7 @@ on: - "tests/test_funasr_onnx_paraformer_online_empty_final.py" - "tests/test_funasr_onnx_vad_online_short_chunks.py" - "tests/test_funasr_onnx_vad_multiple_inputs.py" + - "tests/test_funasr_onnx_punctuation_empty_input.py" - "tests/test_rich_transcription_postprocess.py" - "tests/test_rich_transcription_event_segments.py" push: @@ -29,6 +30,7 @@ on: - "tests/test_funasr_onnx_paraformer_online_empty_final.py" - "tests/test_funasr_onnx_vad_online_short_chunks.py" - "tests/test_funasr_onnx_vad_multiple_inputs.py" + - "tests/test_funasr_onnx_punctuation_empty_input.py" - "tests/test_rich_transcription_postprocess.py" - "tests/test_rich_transcription_event_segments.py" workflow_dispatch: @@ -84,7 +86,7 @@ jobs: import importlib.util from importlib.metadata import requires, version - assert version("funasr-onnx") == "0.4.7" + assert version("funasr-onnx") == "0.4.9" assert importlib.util.find_spec("torch") is None requirements = requires("funasr-onnx") or [] @@ -115,6 +117,9 @@ jobs: - name: Verify offline VAD results per input run: python tests/test_funasr_onnx_vad_multiple_inputs.py + - name: Verify empty punctuation streaming cache + run: python tests/test_funasr_onnx_punctuation_empty_input.py + - name: Verify source distribution error paths run: | python -m pip install --force-reinstall --no-deps dist/funasr-onnx/*.tar.gz @@ -125,7 +130,7 @@ jobs: if: matrix.python-version == '3.12' uses: actions/upload-artifact@v7 with: - name: funasr-onnx-0.4.7 + name: funasr-onnx-0.4.9 path: dist/funasr-onnx/* if-no-files-found: error diff --git a/.github/workflows/test-kws-output.yml b/.github/workflows/test-kws-output.yml index 55b2f4cb03..754e333e0c 100644 --- a/.github/workflows/test-kws-output.yml +++ b/.github/workflows/test-kws-output.yml @@ -8,12 +8,14 @@ on: - "funasr/models/sanm_kws/**" - "funasr/models/sanm_kws_streaming/**" - "funasr/frontends/wav_frontend.py" + - "funasr/models/fsmn_vad_streaming/model.py" - "funasr/auto/auto_model.py" - "funasr/utils/datadir_writer.py" - "funasr/utils/kws_utils.py" - "tests/test_kws_optional_output.py" - "tests/test_kws_streaming_continuity.py" - "tests/test_kws_utils.py" + - "tests/test_wav_frontend_no_lfr.py" - "tests/test_fsmn_vad_streaming_buffers.py" - "tests/test_dynamic_streaming_vad.py" - "tests/test_fsmn_vad_dynamic_silence.py" @@ -26,12 +28,14 @@ on: - "funasr/models/sanm_kws/**" - "funasr/models/sanm_kws_streaming/**" - "funasr/frontends/wav_frontend.py" + - "funasr/models/fsmn_vad_streaming/model.py" - "funasr/auto/auto_model.py" - "funasr/utils/datadir_writer.py" - "funasr/utils/kws_utils.py" - "tests/test_kws_optional_output.py" - "tests/test_kws_streaming_continuity.py" - "tests/test_kws_utils.py" + - "tests/test_wav_frontend_no_lfr.py" - "tests/test_fsmn_vad_streaming_buffers.py" - "tests/test_dynamic_streaming_vad.py" - "tests/test_fsmn_vad_dynamic_silence.py" @@ -63,6 +67,7 @@ jobs: tests/test_kws_optional_output.py \ tests/test_kws_streaming_continuity.py \ tests/test_kws_utils.py \ + tests/test_wav_frontend_no_lfr.py \ tests/test_fsmn_vad_streaming_buffers.py \ tests/test_dynamic_streaming_vad.py \ tests/test_fsmn_vad_dynamic_silence.py diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml index 288f6926c8..a6f2b474c6 100644 --- a/.github/workflows/test-numpy-compatibility.yml +++ b/.github/workflows/test-numpy-compatibility.yml @@ -8,6 +8,8 @@ on: - "funasr/cli.py" - "tests/test_cli.py" - "funasr/auto/auto_model.py" + - "funasr/tokenizer/sentencepiece_tokenizer.py" + - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" - "funasr/bin/realtime_ws.py" @@ -26,6 +28,8 @@ on: - "tests/test_pcm_input_format.py" - "tests/test_load_audio_bytes.py" - "funasr/frontends/**" + - "funasr/models/normalize/utterance_mvn.py" + - "tests/test_utterance_mvn.py" - "funasr/models/emotion2vec/**" - "funasr/models/campplus/**" - "funasr/models/sense_voice/model.py" @@ -45,6 +49,8 @@ on: - "funasr/cli.py" - "tests/test_cli.py" - "funasr/auto/auto_model.py" + - "funasr/tokenizer/sentencepiece_tokenizer.py" + - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" - "funasr/bin/realtime_ws.py" @@ -63,6 +69,8 @@ on: - "tests/test_pcm_input_format.py" - "tests/test_load_audio_bytes.py" - "funasr/frontends/**" + - "funasr/models/normalize/utterance_mvn.py" + - "tests/test_utterance_mvn.py" - "funasr/models/emotion2vec/**" - "funasr/models/campplus/**" - "funasr/models/sense_voice/model.py" @@ -119,12 +127,14 @@ jobs: run: | python -m pytest -q \ tests/test_numpy_compatibility.py \ + tests/test_utterance_mvn.py \ tests/test_numpy_cluster_compatibility.py \ tests/test_cluster_backend.py \ tests/test_speaker_optional_dependency.py \ tests/test_eend_feature_compatibility.py \ tests/test_pypi_metadata.py \ tests/test_sensevoice_word_timestamps.py \ + tests/test_sentencepiece_tokenizer.py \ tests/test_pcm_input_format.py \ tests/test_realtime_ws_service.py \ tests/test_realtime_ws_hf_overrides.py \ diff --git a/README.md b/README.md index 277870f0a3..f36b7a0863 100644 --- a/README.md +++ b/README.md @@ -364,7 +364,9 @@ recordings with the same evaluation scope. ## What's new - **MOSS-Transcribe-Diarize** brings long-form ASR, timestamps, and anonymous speaker labels to FunASR services, Docker, Kubernetes, vLLM/SGLang workflows, and FunClip. [Deploy MOSS ->](./docs/moss_transcribe_diarize.md) -- **FunASR 1.4.21** preserves the streaming punctuation cache on empty or whitespace chunks, returning an empty result without touching cached words or the embedding path; nonempty inference is unchanged. [Fix #3824](https://github.com/modelscope/FunASR/pull/3824). Install with `python -m pip install -U "funasr==1.4.21"`. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) +- **FunASR 1.4.23** fixes the online frontend cursor for `lfr_m=lfr_n=1` and redundant empty final offline VAD chunks. With a supplied `token_list`, SentencePiece `ids2tokens` returns model pieces; use `tokens2text` for text. Without `token_list`, legacy text decoding is unchanged. [Fix #3849](https://github.com/modelscope/FunASR/pull/3849), [fix #3783](https://github.com/modelscope/FunASR/pull/3783). Install with `python -m pip install -U "funasr==1.4.23"`. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23) + **FunASR 1.4.22** honors per-call `merge_length_s` and evaluates dynamic VAD silence thresholds per frame using the current segment duration, preserving the fixed-threshold opt-out. [Fix #3841](https://github.com/modelscope/FunASR/pull/3841), [fix #3833](https://github.com/modelscope/FunASR/pull/3833). [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22) + **FunASR 1.4.21** preserves the streaming punctuation cache on empty or whitespace chunks; nonempty inference is unchanged. [Fix #3824](https://github.com/modelscope/FunASR/pull/3824). [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) **FunASR 1.4.20** preserves the original optional speaker dependency failure in a chained `ImportError` when `spk_model` is requested, while ASR-only construction remains available. [Fix #3819](https://github.com/modelscope/FunASR/pull/3819). [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.20) **FunASR 1.4.19** handles empty-token Paraformer batches, enforces `max_speakers` for automatic clustering, and adds validated JSON-object `--hf_overrides` support to realtime WebSocket serving. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.19) Previous release **FunASR 1.4.18** fixes cross-script hotword misreplacement, filtered-token timestamp alignment, and Chinese VAD sentence/speaker drift. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.18) diff --git a/README_ja.md b/README_ja.md index 13450d8641..25ee7e109f 100644 --- a/README_ja.md +++ b/README_ja.md @@ -112,7 +112,9 @@ CER/WER をそろえて比較してください。オフラインのスループ ## 最新情報 - **MOSS-Transcribe-Diarize** を FunASR service、Docker、Kubernetes、vLLM/SGLang workflow、FunClip に統合し、長時間 ASR、timestamp、匿名 speaker label を一度に処理できます。[MOSS をデプロイ ->](./docs/moss_transcribe_diarize.md) -- **FunASR 1.4.21** はストリーミング句読点処理の空または空白入力で cache を保持し、キャッシュ済み単語や embedding 経路に触れず空結果を返します。非空推論は変更されません。[修正 #3824](https://github.com/modelscope/FunASR/pull/3824)。`python -m pip install -U "funasr==1.4.21"` でインストールできます。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) +- **FunASR 1.4.23** は `lfr_m=lfr_n=1` のオンライン特徴抽出のフレーム位置と、オフライン VAD の余分な空の最終チャンクを修正します。`token_list` 指定時の SentencePiece `ids2tokens` はモデルのピースを返し、テキストへの変換には `tokens2text` を使います。未指定時の従来のテキスト復号は維持します。[修正 #3849](https://github.com/modelscope/FunASR/pull/3849)、[修正 #3783](https://github.com/modelscope/FunASR/pull/3783)。`python -m pip install -U "funasr==1.4.23"` でインストールできます。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23) + **FunASR 1.4.22** は呼び出しごとの `merge_length_s` を反映し、現在の発話区間の長さに基づいて動的 VAD 無音しきい値をフレーム単位で更新します。固定しきい値モードも維持します。[修正 #3841](https://github.com/modelscope/FunASR/pull/3841)、[修正 #3833](https://github.com/modelscope/FunASR/pull/3833)。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22) + **FunASR 1.4.21** はストリーミング句読点処理の空入力で cache を保持します。非空推論は変更されません。[修正 #3824](https://github.com/modelscope/FunASR/pull/3824)。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) **FunASR 1.4.20** は `spk_model` 指定時に任意の話者依存モジュールを読み込めない場合、元の例外を連鎖した `ImportError` で報告します。ASR のみの構築は引き続き利用できます。[修正 #3819](https://github.com/modelscope/FunASR/pull/3819)。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.20) **FunASR 1.4.19** は Paraformer の空 token batch を処理し、自動クラスタリングの `max_speakers` 上限を適用します。リアルタイム WebSocket サービスには JSON object を検証する `--hf_overrides` を追加しました。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.19) 前のリリース **FunASR 1.4.18** は中国語とラテン文字が混在する hotword の誤置換、特殊 token 除去後の timestamp 対応、中国語 VAD 分割での文と話者のずれを修正しました。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.18) diff --git a/README_ko.md b/README_ko.md index a4a8ece06a..259840723f 100644 --- a/README_ko.md +++ b/README_ko.md @@ -111,7 +111,9 @@ checkpoint/revision, 오디오 집합, 하드웨어, 배치, 워밍업, 측정 ## 최신 소식 - **MOSS-Transcribe-Diarize**를 FunASR service, Docker, Kubernetes, vLLM/SGLang workflow, FunClip에 통합해 긴 오디오 ASR, timestamp, 익명 speaker label을 한 번에 처리합니다. [MOSS 배포 ->](./docs/moss_transcribe_diarize.md) -- **FunASR 1.4.21**은 스트리밍 문장부호 처리의 빈 입력이나 공백 입력에서 cache를 유지하고, 캐시된 단어나 embedding 경로를 건드리지 않은 채 빈 결과를 반환합니다. 비어 있지 않은 추론은 변경되지 않습니다. [수정 #3824](https://github.com/modelscope/FunASR/pull/3824). `python -m pip install -U "funasr==1.4.21"`로 설치할 수 있습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) +- **FunASR 1.4.23**은 `lfr_m=lfr_n=1` 온라인 프런트엔드의 프레임 위치와 오프라인 VAD의 불필요한 빈 마지막 청크를 수정합니다. `token_list`를 지정하면 SentencePiece `ids2tokens`는 모델 토큰을 반환하며, 텍스트 변환에는 `tokens2text`를 사용합니다. 미지정 시 기존 텍스트 디코딩은 유지됩니다. [수정 #3849](https://github.com/modelscope/FunASR/pull/3849), [수정 #3783](https://github.com/modelscope/FunASR/pull/3783). `python -m pip install -U "funasr==1.4.23"`로 설치할 수 있습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23) + **FunASR 1.4.22**는 호출별 `merge_length_s`를 반영하고 현재 발화 구간 길이에 따라 동적 VAD 무음 임계값을 프레임마다 갱신합니다. 고정 임계값 모드도 유지합니다. [수정 #3841](https://github.com/modelscope/FunASR/pull/3841), [수정 #3833](https://github.com/modelscope/FunASR/pull/3833). [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22) + **FunASR 1.4.21**은 스트리밍 문장부호 처리의 빈 입력에서 cache를 유지합니다. 비어 있지 않은 추론은 변경되지 않습니다. [수정 #3824](https://github.com/modelscope/FunASR/pull/3824). [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) **FunASR 1.4.20**은 `spk_model` 사용 시 선택적 화자 의존성 가져오기가 실패하면 원래 예외를 연결한 `ImportError`를 제공합니다. ASR 전용 구성은 계속 사용할 수 있습니다. [수정 #3819](https://github.com/modelscope/FunASR/pull/3819). [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.20) **FunASR 1.4.19**는 Paraformer의 빈 token batch를 처리하고 자동 클러스터링에 `max_speakers` 상한을 적용합니다. 실시간 WebSocket 서비스에 JSON object를 검증하는 `--hf_overrides`도 추가했습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.19) 이전 릴리스 **FunASR 1.4.18**은 중국어와 라틴 문자가 섞인 hotword의 잘못된 치환, 특수 token 제거 후 timestamp 정렬, 중국어 VAD 분할의 문장 및 화자 불일치를 수정했습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.18) diff --git a/README_zh.md b/README_zh.md index eccb4f7144..1c3199b476 100644 --- a/README_zh.md +++ b/README_zh.md @@ -157,7 +157,9 @@ checkpoint/revision、音频集、硬件、批量大小、预热、计时范围 ## 最新动态 - **MOSS-Transcribe-Diarize** 已接入 FunASR 服务、Docker、Kubernetes、vLLM/SGLang 工作流和 FunClip，一次完成长音频转写、时间戳与匿名说话人标注。[部署 MOSS ->](./docs/moss_transcribe_diarize_zh.md) -- **FunASR 1.4.21** 在流式标点收到空白或空输入时保留 cache，并在不访问缓存词或 embedding 路径的情况下返回空结果；非空推理保持不变。[修复 #3824](https://github.com/modelscope/FunASR/pull/3824)。安装：`python -m pip install -U "funasr==1.4.21"`。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) +- **FunASR 1.4.23** 修复 `lfr_m=lfr_n=1` 在线前端的帧游标和离线 VAD 多余的空尾块。提供 `token_list` 时，SentencePiece 的 `ids2tokens` 返回模型词元，请用 `tokens2text` 转为文本；未提供 `token_list` 时保持旧有文本解码行为。[修复 #3849](https://github.com/modelscope/FunASR/pull/3849)、[修复 #3783](https://github.com/modelscope/FunASR/pull/3783)。安装：`python -m pip install -U "funasr==1.4.23"`。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23) + **FunASR 1.4.22** 支持逐调用覆盖 `merge_length_s`，并根据当前语音段时长逐帧更新动态 VAD 静音阈值，同时保留固定阈值模式。[修复 #3841](https://github.com/modelscope/FunASR/pull/3841)、[修复 #3833](https://github.com/modelscope/FunASR/pull/3833)。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22) + **FunASR 1.4.21** 在流式标点收到空白或空输入时保留 cache；非空推理保持不变。[修复 #3824](https://github.com/modelscope/FunASR/pull/3824)。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) **FunASR 1.4.20** 在指定 `spk_model` 而可选说话人依赖导入失败时，通过带原始异常链的 `ImportError` 提供诊断，同时保留仅 ASR 的构造路径。[修复 #3819](https://github.com/modelscope/FunASR/pull/3819)。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.20) **FunASR 1.4.19** 修复 Paraformer 空 token 批次处理，使自动聚类遵守 `max_speakers` 上限，并为实时 WebSocket 服务新增带 JSON 对象校验的 `--hf_overrides` 参数。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.19) 上一版本 **FunASR 1.4.18** 修复跨中英文热词误替换、过滤特殊符号后的时间戳对齐，以及中文 VAD 分段的句子与说话人错位。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.18) diff --git a/docs/python_api.md b/docs/python_api.md index 160a662735..f95a5ba270 100644 --- a/docs/python_api.md +++ b/docs/python_api.md @@ -42,7 +42,7 @@ The wrapper accepts `**kwargs`, not a universally validated parameter schema. A | `batch_size_s` | `300` | VAD-path batching budget in seconds, based on longest padded segment times segment count, not a hard file-duration limit. CPU VAD processing uses single-segment batches. | | `batch_size_threshold_s` | `60` | VAD batching threshold for segment duration, not a VAD splitting setting. | | `merge_vad` | `False` | Enables merging detected VAD regions for this call. | -| `merge_length_s` | `15` | VAD merge target read before call overrides are merged into ASR config. Set this at construction in this checkout. | +| `merge_length_s` | `15` | Target length in seconds for merged VAD regions when `merge_vad=True`. A per-call value overrides the construction value. | | `cache` | No wrapper-managed session | Pass a caller-owned dictionary on every streaming call. It is not a hub/model download cache. | | `sentence_timestamp` | `False` | Requests sentence records in the VAD pipeline; depends on available timestamps and punctuation, with limited VAD fallback. | | `return_spk_res` | `True` | Includes clustered speaker results when VAD and a speaker model are configured. `False` does not avoid construction or segment embedding computation. | diff --git a/docs/python_api_zh.md b/docs/python_api_zh.md index 42d9a4a139..1f70e6d900 100644 --- a/docs/python_api_zh.md +++ b/docs/python_api_zh.md @@ -42,7 +42,7 @@ | `batch_size_s` | `300` | VAD 路径的秒级批处理预算，按最长补齐片段长度乘以片段数量估算，不是文件时长硬上限。CPU 的 VAD 路径按单片段处理。 | | `batch_size_threshold_s` | `60` | VAD 片段参与组批的时长阈值，不是 VAD 切分参数。 | | `merge_vad` | `False` | 为本次调用启用 VAD 区域合并。 | -| `merge_length_s` | `15` | VAD 合并目标，在本次参数合并进 ASR 配置之前读取。当前代码版本应在构建时设置它。 | +| `merge_length_s` | `15` | `merge_vad=True` 时合并后 VAD 片段的目标长度，单位为秒。本次调用传入的值会覆盖构建时设置的值。 | | `cache` | 包装层不维护会话 | 每次流式调用都传入调用方持有的字典。它不是模型下载缓存。 | | `sentence_timestamp` | `False` | 请求 VAD 流程输出句子记录，依赖实际时间戳与标点，部分情形可回退到 VAD 区域。 | | `return_spk_res` | `True` | 配置 VAD 与说话人模型时输出聚类结果。设为 `False` 并不能省去模型构建或片段说话人特征计算。 | diff --git a/docs/repository_roles.md b/docs/repository_roles.md index 911a2b6555..96aa91ed42 100644 --- a/docs/repository_roles.md +++ b/docs/repository_roles.md @@ -5,7 +5,7 @@ This document explains the responsibility boundaries, user entry points, and iss > **Directional roadmap, not a release promise.** > This document records shipped capabilities and active work, but does not commit to > future version numbers or dates. The current Python release is -> [`funasr==1.4.21`](https://github.com/modelscope/FunASR/releases/tag/v1.4.21). +> [`funasr==1.4.23`](https://github.com/modelscope/FunASR/releases/tag/v1.4.23). > Any future breaking release still requires a maintainer-approved milestone and > migration plan. diff --git a/docs/repository_roles_zh.md b/docs/repository_roles_zh.md index 6a847f1706..c0214965e1 100644 --- a/docs/repository_roles_zh.md +++ b/docs/repository_roles_zh.md @@ -4,7 +4,7 @@ > **方向性路线图，不是版本承诺。** > 本文档记录已交付能力与正在推进的工作，但不承诺未来版本号或日期。当前 Python -> 版本是 [`funasr==1.4.21`](https://github.com/modelscope/FunASR/releases/tag/v1.4.21)。 +> 版本是 [`funasr==1.4.23`](https://github.com/modelscope/FunASR/releases/tag/v1.4.23)。 > 任何未来的 breaking release 仍需 maintainer 确认 milestone 与迁移方案。 --- diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py index 8ec9fcbce0..b8320dfa7c 100644 --- a/funasr/auto/auto_model.py +++ b/funasr/auto/auto_model.py @@ -1043,10 +1043,10 @@ def inference_with_vad(self, input, input_len=None, **cfg): # FIX(gcf): concat the vad clips for sense vocie model for better aed if cfg.get("merge_vad", False): + # Call options are merged into kwargs only after this point. + merge_length_s = cfg.get("merge_length_s", kwargs.get("merge_length_s", 15)) for i in range(len(res)): - res[i]["value"] = merge_vad( - res[i]["value"], kwargs.get("merge_length_s", 15) * 1000 - ) + res[i]["value"] = merge_vad(res[i]["value"], merge_length_s * 1000) # step.2 compute asr model model = self.model diff --git a/funasr/frontends/wav_frontend.py b/funasr/frontends/wav_frontend.py index 593cc14462..cb671e6df6 100644 --- a/funasr/frontends/wav_frontend.py +++ b/funasr/frontends/wav_frontend.py @@ -488,6 +488,7 @@ def forward_lfr_cmvn( lfr_splice_frame_idxs = [] for i in range(batch_size): mat = input[i, : input_lengths[i], :] + lfr_splice_frame_idx = mat.size(0) if self.lfr_m != 1 or self.lfr_n != 1: # update self.lfr_splice_cache in self.apply_lfr # mat, self.lfr_splice_cache[i], lfr_splice_frame_idx = self.apply_lfr(mat, self.lfr_m, self.lfr_n, self.lfr_splice_cache[i], diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.py index d8537ed20f..b59862e6b7 100644 --- a/funasr/models/fsmn_vad_streaming/model.py +++ b/funasr/models/fsmn_vad_streaming/model.py @@ -353,6 +353,7 @@ def __init__( self.output_data_buf_offset = 0 self.frame_probs = [] self.max_end_sil_frame_cnt_thresh = max_end_sil_frame_cnt_thresh + self.silence_schedule = None self.speech_noise_thres = speech_noise_thres self.scores = None self.max_time_out = False @@ -1004,7 +1005,9 @@ def inference( audio_sample = torch.cat((cache["prev_samples"], audio_sample_list[0])) - n = int(len(audio_sample) // chunk_stride_samples + int(_is_final)) + n = len(audio_sample) // chunk_stride_samples + if _is_final and len(audio_sample) % chunk_stride_samples: + n += 1 m = int(len(audio_sample) % chunk_stride_samples * (1 - int(_is_final))) segments = [] # Keep explicit fixed-threshold requests from being overwritten by the dynamic schedule. @@ -1012,9 +1015,11 @@ def inference( "dynamic_silence", kwargs.get("max_end_silence_time") is None ) silence_schedule = kwargs.get("silence_schedule", DEFAULT_SILENCE_SCHEDULE) - speech_to_sil_ms = self.vad_opts.speech_to_sil_time_thres - accumulated_ms = cache.get("_dynamic_accumulated_ms", 0) - in_speech = cache.get("_dynamic_in_speech", False) + # The schedule is looked up per frame in DetectOneFrame from the current segment's duration. + if "stats" in cache: + cache["stats"].silence_schedule = silence_schedule if dynamic_silence else None + if dynamic_silence: + cache["stats"].speech_noise_thres = 0.5 frontend_inference_kwargs = {} if getattr(frontend, "supports_aligned_waveforms", False): frontend_inference_kwargs["return_waveform"] = True @@ -1023,21 +1028,6 @@ def inference( kwargs["is_final"] = _is_final and i == n - 1 audio_sample_i = audio_sample[i * chunk_stride_samples : (i + 1) * chunk_stride_samples] - # Apply dynamic silence threshold (only accumulate while in speech) - if dynamic_silence and "stats" in cache: - vad_state = cache["stats"].vad_state_machine - # VadStateMachine: 2=kVadInStateStartPointNotDetected, 3=kVadInStateInSpeechSegment - if vad_state.value == 2 or in_speech: # kVadInStateInSpeechSegment - accumulated_ms += chunk_size - in_speech = True - for limit_ms, silence_ms in silence_schedule: - if accumulated_ms <= limit_ms: - cache["stats"].max_end_sil_frame_cnt_thresh = max(silence_ms - speech_to_sil_ms, 0) - cache["stats"].speech_noise_thres = 0.5 - break - cache["_dynamic_accumulated_ms"] = accumulated_ms - cache["_dynamic_in_speech"] = in_speech - # extract fbank feats speech, speech_lengths = extract_fbank( [audio_sample_i], @@ -1074,11 +1064,6 @@ def inference( segments_i = self.forward(**batch) if len(segments_i) > 0: segments.extend(*segments_i) - if dynamic_silence: - accumulated_ms = 0 - in_speech = False - cache["_dynamic_accumulated_ms"] = 0 - cache["_dynamic_in_speech"] = False cache["prev_samples"] = audio_sample[-m:] if m > 0 else torch.empty(0) if _is_final: @@ -1264,6 +1249,17 @@ def DetectOneFrame( cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache), cache=cache ) elif cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: + silence_schedule = getattr(cache["stats"], "silence_schedule", None) + if silence_schedule is not None: + segment_ms = ( + cur_frm_idx - cache["stats"].confirmed_start_frame + 1 + ) * frm_shift_in_ms + for limit_ms, silence_ms in silence_schedule: + if segment_ms <= limit_ms: + cache["stats"].max_end_sil_frame_cnt_thresh = max( + silence_ms - self.vad_opts.speech_to_sil_time_thres, 0 + ) + break if ( cache["stats"].continous_silence_frame_count * frm_shift_in_ms >= cache["stats"].max_end_sil_frame_cnt_thresh diff --git a/funasr/tokenizer/sentencepiece_tokenizer.py b/funasr/tokenizer/sentencepiece_tokenizer.py index acba22c2a4..1c76c8984a 100644 --- a/funasr/tokenizer/sentencepiece_tokenizer.py +++ b/funasr/tokenizer/sentencepiece_tokenizer.py @@ -13,11 +13,11 @@ class SentencepiecesTokenizer(BaseTokenizer): def __init__(self, bpemodel: Union[Path, str], **kwargs): """Initialize SentencepiecesTokenizer. - - Args: - bpemodel: TODO. - **kwargs: Additional keyword arguments. - """ + + Args: + bpemodel: TODO. + **kwargs: Additional keyword arguments. + """ super().__init__(**kwargs) self.bpemodel = str(bpemodel) # NOTE(kamo): @@ -41,39 +41,39 @@ def _build_sentence_piece_processor(self): def text2tokens(self, line: str) -> List[str]: """Text2tokens. - - Args: - line: TODO. - """ + + Args: + line: TODO. + """ self._build_sentence_piece_processor() return self.sp.EncodeAsPieces(line) def tokens2text(self, tokens: Iterable[str]) -> str: """Tokens2text. - - Args: - tokens: TODO. - """ + + Args: + tokens: TODO. + """ self._build_sentence_piece_processor() return self.sp.DecodePieces(list(tokens)) def encode(self, line: str, **kwargs) -> List[int]: """Encode. - - Args: - line: TODO. - **kwargs: Additional keyword arguments. - """ + + Args: + line: TODO. + **kwargs: Additional keyword arguments. + """ self._build_sentence_piece_processor() return self.sp.EncodeAsIds(line) def decode(self, line: List[int], **kwargs): """Decode. - - Args: - line: TODO. - **kwargs: Additional keyword arguments. - """ + + Args: + line: TODO. + **kwargs: Additional keyword arguments. + """ self._build_sentence_piece_processor() return self.sp.DecodeIds(line) @@ -81,20 +81,17 @@ def get_vocab_size(self): """Get vocab size.""" return self.sp.GetPieceSize() - def ids2tokens(self, *args, **kwargs): - """Ids2tokens. - - Args: - *args: Variable positional arguments. - **kwargs: Additional keyword arguments. - """ - return self.decode(*args, **kwargs) + def ids2tokens(self, line, **kwargs): + """Return model-vocabulary pieces, or legacy decoded text without a token list.""" + if getattr(self, "token_list", None) is not None: + return super().ids2tokens(line) + return self.decode(line, **kwargs) def tokens2ids(self, *args, **kwargs): """Tokens2ids. - - Args: - *args: Variable positional arguments. - **kwargs: Additional keyword arguments. - """ + + Args: + *args: Variable positional arguments. + **kwargs: Additional keyword arguments. + """ return self.encode(*args, **kwargs) diff --git a/funasr/version.txt b/funasr/version.txt index 8cc8fe024b..580ef4b6c1 100644 --- a/funasr/version.txt +++ b/funasr/version.txt @@ -1 +1 @@ -1.4.21 +1.4.23 diff --git a/runtime/python/libtorch/README.md b/runtime/python/libtorch/README.md index 9136bd74f1..00808b6ba1 100644 --- a/runtime/python/libtorch/README.md +++ b/runtime/python/libtorch/README.md @@ -21,12 +21,23 @@ python -m funasr.export.export_model --model-name damo/speech_paraformer-large_a ## Install the `funasr_torch` +`funasr_torch` is a separate distribution from `funasr`; upgrading the root +package alone does not update these TorchScript wrappers. Version 0.1.5 +includes the Contextual/Seaco Paraformer hotword batching and empty +hotword-file handling fix (#3838). This is a runtime wrapper fix, not a +model-weight update or an ASR accuracy claim. + +Version 0.1.6 makes the standalone CMVN reader accept blank lines and +raise a file-specific error for missing or truncated statistics (#3842). +Valid float64 statistics and shipped feature output are unchanged. This is +a CPU parser/package fix, not a model-weight or ASR accuracy update. + install from pip ```shell -pip install -U "funasr_torch>=0.1.4" +pip install -U "funasr_torch>=0.1.6" # For the users in China, you could install with the command: -# pip install -U funasr_torch -i https://mirror.sjtu.edu.cn/pypi/web/simple +# pip install -U "funasr_torch>=0.1.6" -i https://mirror.sjtu.edu.cn/pypi/web/simple ``` or install from source code diff --git a/runtime/python/libtorch/funasr_torch/utils/frontend.py b/runtime/python/libtorch/funasr_torch/utils/frontend.py index 743ee5e5f2..2772ad95d6 100644 --- a/runtime/python/libtorch/funasr_torch/utils/frontend.py +++ b/runtime/python/libtorch/funasr_torch/utils/frontend.py @@ -125,25 +125,29 @@ def load_cmvn( self, ) -> np.ndarray: with open(self.cmvn_file, "r", encoding="utf-8") as f: - lines = f.readlines() + lines = [line for line in f.readlines() if line.split()] means_list = [] vars_list = [] for i in range(len(lines)): line_item = lines[i].split() if line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": + line_item = lines[i + 1].split() if i + 1 < len(lines) else [] + if line_item and line_item[0] == "": add_shift_line = line_item[3 : (len(line_item) - 1)] means_list = list(add_shift_line) continue elif line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": + line_item = lines[i + 1].split() if i + 1 < len(lines) else [] + if line_item and line_item[0] == "": rescale_line = line_item[3 : (len(line_item) - 1)] vars_list = list(rescale_line) continue + if not means_list or not vars_list: + raise ValueError( + f"No / statistics found in cmvn file: {self.cmvn_file}" + ) means = np.array(means_list).astype(np.float64) vars = np.array(vars_list).astype(np.float64) cmvn = np.array([means, vars]) diff --git a/runtime/python/libtorch/setup.py b/runtime/python/libtorch/setup.py index 3ba4610e47..be4b7e3c59 100644 --- a/runtime/python/libtorch/setup.py +++ b/runtime/python/libtorch/setup.py @@ -15,7 +15,7 @@ def get_readme(): setuptools.setup( name="funasr_torch", - version="0.1.4", + version="0.1.6", platforms="Any", url="https://github.com/modelscope/FunASR.git", author="Speech Lab of DAMO Academy, Alibaba Group", diff --git a/runtime/python/onnxruntime/README.md b/runtime/python/onnxruntime/README.md index 0b19d0cc5f..8794ac4c74 100644 --- a/runtime/python/onnxruntime/README.md +++ b/runtime/python/onnxruntime/README.md @@ -7,6 +7,17 @@ does not update the installed ONNX wrappers. Check their versions separately with `python -m pip show funasr-onnx funasr`; use the source installation below when a required wrapper fix is newer than the published PyPI package. +Version 0.4.9 keeps one result per input when offline VAD receives empty or sub-frame +audio, without invoking ONNX Runtime for zero-frame features (#3846). It also returns +an empty punctuation chunk while it preserves the streaming cache and avoids +tokenization and inference (#3847). These are CPU runtime-wrapper fixes, not a +model-weight or ASR accuracy update. + +Version 0.4.8 makes the standalone CMVN reader accept blank lines and +raise a file-specific error for missing or truncated statistics (#3842). +Valid float64 statistics and shipped feature output are unchanged. This is +a CPU parser/package fix, not a model-weight or ASR accuracy update. + Version 0.4.7 includes the Contextual/Seaco Paraformer hotword batching fix (#3838), including empty hotword-file handling. This is a runtime wrapper fix, not a model-weight update or an ASR accuracy claim. @@ -28,9 +39,9 @@ and postprocessing fixes, not a model-weight update or an ASR accuracy claim. install from pip ```shell -pip install -U "funasr-onnx>=0.4.7" +pip install -U "funasr-onnx>=0.4.9" # For the users in China, you could install with the command: -# pip install -U "funasr-onnx>=0.4.7" -i https://mirror.sjtu.edu.cn/pypi/web/simple +# pip install -U "funasr-onnx>=0.4.9" -i https://mirror.sjtu.edu.cn/pypi/web/simple # If you want to export .onnx file, you should install modelscope and funasr pip install -U modelscope funasr # For the users in China, you could install with the command: diff --git a/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/runtime/python/onnxruntime/funasr_onnx/punc_bin.py index 7fa62d44d0..bfcb6f80b6 100644 --- a/runtime/python/onnxruntime/funasr_onnx/punc_bin.py +++ b/runtime/python/onnxruntime/funasr_onnx/punc_bin.py @@ -99,6 +99,8 @@ def __call__(self, text: Union[list, str], split_size=20): split_text = self.code_mix_split_words_jieba(text) else: split_text = code_mix_split_words(text) + if not split_text: + return "", [] split_text_id = self.converter.tokens2ids(split_text) mini_sentences = split_to_mini_sentence(split_text, split_size) mini_sentences_id = split_to_mini_sentence(split_text_id, split_size) @@ -196,6 +198,9 @@ def __call__(self, text: str, param_dict: map, split_size=20): cache_key = "cache" assert cache_key in param_dict cache = param_dict[cache_key] + # An empty chunk adds no text and must not reprocess pending words. + if not text.strip(): + return "", [], cache if cache is not None else [] if cache is not None and len(cache) > 0: precache = "".join(cache) else: diff --git a/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py b/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py index f150ccbf8a..4218718b6c 100644 --- a/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py +++ b/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py @@ -142,24 +142,26 @@ def load_cmvn(cmvn_file: Union[str, Path]) -> np.ndarray: raise FileNotFoundError("cmvn file not exits") with open(cmvn_file, "r", encoding="utf-8") as f: - lines = f.readlines() + lines = [line for line in f.readlines() if line.split()] means_list = [] vars_list = [] for i in range(len(lines)): line_item = lines[i].split() if line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": + line_item = lines[i + 1].split() if i + 1 < len(lines) else [] + if line_item and line_item[0] == "": add_shift_line = line_item[3 : (len(line_item) - 1)] means_list = list(add_shift_line) continue elif line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": + line_item = lines[i + 1].split() if i + 1 < len(lines) else [] + if line_item and line_item[0] == "": rescale_line = line_item[3 : (len(line_item) - 1)] vars_list = list(rescale_line) continue + if not means_list or not vars_list: + raise ValueError(f"No / statistics found in cmvn file: {cmvn_file}") means = np.array(means_list).astype(np.float64) vars = np.array(vars_list).astype(np.float64) cmvn = np.array([means, vars]) diff --git a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py index cb9ba3d0d2..5d9d8fd4e2 100644 --- a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py +++ b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py @@ -103,6 +103,8 @@ def __call__(self, audio_in: Union[str, np.ndarray, List[str]], **kwargs) -> Lis end_idx = beg_idx + 1 waveform = waveform_list[beg_idx:end_idx] feats, feats_len = self.extract_feat(waveform) + if feats_len[0] == 0: + continue waveform = np.array(waveform) param_dict = kwargs.get("param_dict", dict()) in_cache = param_dict.get("in_cache", list()) @@ -183,8 +185,13 @@ def load_wav(path: str) -> np.ndarray: def extract_feat(self, waveform_list: List[np.ndarray]) -> Tuple[np.ndarray, np.ndarray]: feats, feats_len = [], [] for waveform in waveform_list: - speech, _ = self.frontend.fbank(waveform) - feat, feat_len = self.frontend.lfr_cmvn(speech) + speech, speech_len = self.frontend.fbank(waveform) + if speech_len == 0: + # LFR needs a first frame; keep the empty input at its original position. + feat = np.empty((0, speech.shape[1] * self.frontend.lfr_m), dtype=speech.dtype) + feat_len = 0 + else: + feat, feat_len = self.frontend.lfr_cmvn(speech) feats.append(feat) feats_len.append(feat_len) diff --git a/runtime/python/onnxruntime/setup.py b/runtime/python/onnxruntime/setup.py index b61aa96c63..b5236ffc16 100644 --- a/runtime/python/onnxruntime/setup.py +++ b/runtime/python/onnxruntime/setup.py @@ -13,7 +13,7 @@ def get_readme(): MODULE_NAME = "funasr_onnx" -VERSION_NUM = "0.4.7" +VERSION_NUM = "0.4.9" setuptools.setup( name=MODULE_NAME, diff --git a/tests/test_fsmn_vad_dynamic_silence.py b/tests/test_fsmn_vad_dynamic_silence.py index f396614f98..d8f39c345a 100644 --- a/tests/test_fsmn_vad_dynamic_silence.py +++ b/tests/test_fsmn_vad_dynamic_silence.py @@ -55,13 +55,83 @@ def test_explicit_max_end_silence_time_keeps_fixed_threshold_by_default(self): cache = self._run_inference(max_end_silence_time=300) self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 200) - self.assertNotIn("_dynamic_accumulated_ms", cache) + self.assertIsNone(cache["stats"].silence_schedule) + self.assertEqual(cache["stats"].speech_noise_thres, 0.6) def test_explicit_dynamic_silence_still_enables_schedule(self): cache = self._run_inference(max_end_silence_time=300, dynamic_silence=True) - self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 1900) - self.assertEqual(cache["_dynamic_accumulated_ms"], 1000) + self.assertEqual(cache["stats"].silence_schedule, vad_model.DEFAULT_SILENCE_SCHEDULE) + self.assertEqual(cache["stats"].speech_noise_thres, 0.5) + # The threshold is no longer advanced per chunk; DetectOneFrame applies the schedule. + self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 200) + self.assertNotIn("_dynamic_accumulated_ms", cache) + + +class TestFsmnVadDynamicSilencePerFrame(unittest.TestCase): + """Drive DetectOneFrame with scripted window-detector transitions.""" + + def _end_frames(self, speech_ms, silence_ms, schedule, start_frame=0, fixed_thresh_ms=650): + vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming) + torch.nn.Module.__init__(vad) + vad.vad_opts = vad_model.VADXOptions() + ends = [] + + def on_voice_end(end_frame, fake_result, is_last_frame, cache=None): + ends.append(end_frame) + + def reset_detection(cache=None): + cache["stats"].confirmed_start_frame = -1 + cache["stats"].vad_state_machine = vad_model.VadStateMachine.kVadInStateStartPointNotDetected + + vad.OnVoiceEnd = on_voice_end + vad.PopDataToOutputBuf = lambda *args, **kwargs: None + vad.PopDataBufTillFrame = lambda *args, **kwargs: None + vad.LatencyFrmNumAtStartPoint = lambda cache=None: 0 + vad.ResetDetection = reset_detection + + stats = vad_model.Stats( + sil_pdf_ids=[0], max_end_sil_frame_cnt_thresh=fixed_thresh_ms, speech_noise_thres=0.5 + ) + stats.silence_schedule = schedule + speech_frames, silence_frames = speech_ms // 10, silence_ms // 10 + changes = ( + [vad_model.AudioChangeState.kChangeStateSil2Speech] + + [vad_model.AudioChangeState.kChangeStateSpeech2Speech] * (speech_frames - 1) + + [vad_model.AudioChangeState.kChangeStateSpeech2Sil] + + [vad_model.AudioChangeState.kChangeStateSil2Sil] * silence_frames + ) + script = iter(changes) + cache = { + "stats": stats, + "windows_detector": SimpleNamespace(DetectOneFrame=lambda *a, **k: next(script)), + } + for i in range(len(changes)): + vad.DetectOneFrame(vad_model.FrameState.kFrameStateSpeech, start_frame + i, False, cache=cache) + return [e - start_frame for e in ends] + + def test_schedule_follows_current_segment_duration(self): + schedule = vad_model.DEFAULT_SILENCE_SCHEDULE + # Thresholds below are the schedule values minus speech_to_sil_time_thres (150 ms). + # 5 s into a segment: 2000 ms -> 1850 ms, so 900 ms of silence does not end it. + self.assertEqual(self._end_frames(5000, 900, schedule), []) + # 15 s into a segment: 1000 ms -> 850 ms, so 900 ms ends it and 700 ms does not. + self.assertEqual(len(self._end_frames(15000, 900, schedule)), 1) + self.assertEqual(self._end_frames(15000, 700, schedule), []) + # 55 s into a segment: 200 ms -> 50 ms, so 300 ms ends it. + self.assertEqual(len(self._end_frames(55000, 300, schedule)), 1) + + def test_schedule_does_not_depend_on_absolute_position(self): + schedule = vad_model.DEFAULT_SILENCE_SCHEDULE + for speech_ms, silence_ms in [(5000, 900), (15000, 900), (55000, 300)]: + at_zero = self._end_frames(speech_ms, silence_ms, schedule) + shifted = self._end_frames(speech_ms, silence_ms, schedule, start_frame=6000) + self.assertEqual(at_zero, shifted) + + def test_without_schedule_threshold_stays_fixed(self): + # 650 ms fixed threshold: 700 ms of silence ends a 5 s segment, 600 ms does not. + self.assertEqual(len(self._end_frames(5000, 700, None)), 1) + self.assertEqual(self._end_frames(5000, 600, None), []) if __name__ == "__main__": diff --git a/tests/test_funasr_onnx_installed.py b/tests/test_funasr_onnx_installed.py index babcf53a59..4b3c254ee9 100644 --- a/tests/test_funasr_onnx_installed.py +++ b/tests/test_funasr_onnx_installed.py @@ -38,7 +38,7 @@ class InstalledOnnxPackageTest(unittest.TestCase): def test_installed_distribution_version_and_origin(self): distribution = importlib.metadata.distribution('funasr-onnx') - self.assertEqual(distribution.version, '0.4.7') + self.assertEqual(distribution.version, '0.4.9') self.assertEqual( Path(funasr_onnx.__file__).resolve().parent, Path(distribution.locate_file('funasr_onnx')).resolve(), diff --git a/tests/test_funasr_onnx_punctuation_empty_input.py b/tests/test_funasr_onnx_punctuation_empty_input.py new file mode 100644 index 0000000000..901a228bc3 --- /dev/null +++ b/tests/test_funasr_onnx_punctuation_empty_input.py @@ -0,0 +1,116 @@ +"""Run with PYTHONPATH=runtime/python/onnxruntime python -m unittest discover +-s tests -p test_funasr_onnx_punctuation_empty_input.py (standalone ONNX dependencies). +""" + +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock + +import numpy as np + +from funasr_onnx.punc_bin import CT_Transformer, CT_Transformer_VadRealtime +from funasr_onnx.utils.utils import TokenIDConverter, code_mix_split_words_jieba + + +EMPTY_TEXTS = ("", " ", "\t\n", " \t\r\n\u3000") +PUNC_LIST = ["", "_", "，", "。", "？", "、"] + + +def make_model(model_class, segmenter=None): + # Exercise real tokenization, chunking and cache handling without model downloads. + model = model_class.__new__(model_class) + model.seg_jieba = segmenter is not None + model.code_mix_split_words_jieba = segmenter + model.converter = TokenIDConverter(["", "你", "好", "世", "界", "hello", "world"]) + model.punc_list = PUNC_LIST + model.period = PUNC_LIST.index("。") + + def infer(tokens, lengths, *masks): + logits = np.zeros((*tokens.shape, len(PUNC_LIST)), dtype=np.float32) + logits[:, :, PUNC_LIST.index("_")] = 1 + return [logits] + + model.infer = Mock(side_effect=infer) + return model + + +class OnnxPunctuationEmptyInputTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + with tempfile.TemporaryDirectory() as directory: + dictionary = Path(directory) / "jieba_usr_dict" + dictionary.write_text("你好 100\n世界 100\n", encoding="utf-8") + cls.segmenter = staticmethod(code_mix_split_words_jieba(str(dictionary))) + + def test_offline_empty_skips_inference(self): + for segmenter in (None, self.segmenter): + model = make_model(CT_Transformer, segmenter) + for text in EMPTY_TEXTS: + with self.subTest(jieba=segmenter is not None, text=text): + self.assertEqual(model(text), ("", [])) + model.infer.assert_not_called() + + def test_offline_nonempty_around_empty(self): + for segmenter in (None, self.segmenter): + for text, expected in (("你好", "你好。"), ("hello world", "hello world。")): + with self.subTest(jieba=segmenter is not None, text=text): + model = make_model(CT_Transformer, segmenter) + before = model(text) + self.assertEqual(before[0], expected) + self.assertEqual(before[1][-1], model.period) + for empty in EMPTY_TEXTS: + self.assertEqual(model(empty), ("", [])) + self.assertEqual(model(text), before) + self.assertEqual(model.infer.call_count, 2) + + def test_streaming_empty_preserves_cache(self): + for initial_cache in (None, [], ["你", "好"], ["hello", " world"]): + with self.subTest(cache=initial_cache): + model = make_model(CT_Transformer_VadRealtime) + params = {"cache": initial_cache} + expected = None if initial_cache is None else initial_cache.copy() + for text in EMPTY_TEXTS * 2: + result = model(text, params) + self.assertEqual(result, ("", [], expected or [])) + self.assertIs(params["cache"], initial_cache) + self.assertEqual(params["cache"], expected) + if initial_cache is not None: + self.assertIs(result[2], initial_cache) + model.infer.assert_not_called() + + def test_streaming_continuity_across_empty_chunks(self): + for chunks in (("你好", "世界"), ("hello", "world"), ("你好 hello", "world 世界")): + for split_size in (2, 20): + with self.subTest(chunks=chunks, split_size=split_size): + model = make_model(CT_Transformer_VadRealtime) + reference = make_model(CT_Transformer_VadRealtime) + params, reference_params = {"cache": []}, {"cache": []} + for chunk in chunks: + for empty in EMPTY_TEXTS: + cache = params["cache"] + calls = model.infer.call_count + self.assertEqual(model(empty, params), ("", [], cache)) + self.assertIs(params["cache"], cache) + self.assertEqual(model.infer.call_count, calls) + self.assertEqual( + model(chunk, params, split_size=split_size), + reference(chunk, reference_params, split_size=split_size), + ) + self.assertEqual(params, reference_params) + self.assertEqual(model.infer.call_count, reference.infer.call_count) + for actual, expected in zip( + model.infer.call_args_list, reference.infer.call_args_list + ): + for actual_arg, expected_arg in zip(actual.args, expected.args): + np.testing.assert_array_equal(actual_arg, expected_arg) + + def test_streaming_requires_cache_key_even_for_empty_text(self): + model = make_model(CT_Transformer_VadRealtime) + with self.assertRaises(AssertionError): + model("", {}) + model.infer.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_funasr_onnx_release.py b/tests/test_funasr_onnx_release.py index 34a9fdea42..160c24aa4c 100644 --- a/tests/test_funasr_onnx_release.py +++ b/tests/test_funasr_onnx_release.py @@ -7,7 +7,7 @@ ROOT = Path(__file__).resolve().parents[1] PACKAGE_ROOT = ROOT / "runtime" / "python" / "onnxruntime" SETUP_PATH = PACKAGE_ROOT / "setup.py" -EXPECTED_VERSION = "0.4.7" +EXPECTED_VERSION = "0.4.9" REQUIREMENT_NAME_PATTERN = re.compile(r"^\s*([A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?)") @@ -44,7 +44,7 @@ def normalized_requirement_name(requirement): class FunASROnnxReleaseContractTest(unittest.TestCase): - def test_release_version_is_0_4_7(self): + def test_release_version_is_0_4_9(self): self.assertEqual(assigned_literal(read_setup_tree(), "VERSION_NUM"), EXPECTED_VERSION) def test_standalone_license_preserves_repository_notice(self): @@ -58,6 +58,12 @@ def test_workflow_checks_and_artifact_use_release_version(self): self.assertIn(f'assert version("funasr-onnx") == "{EXPECTED_VERSION}"', workflow) self.assertIn(f"name: funasr-onnx-{EXPECTED_VERSION}", workflow) + def test_workflow_collects_empty_punctuation_regression(self): + workflow = (ROOT / ".github/workflows/test-funasr-onnx-package.yml").read_text() + test_path = "tests/test_funasr_onnx_punctuation_empty_input.py" + self.assertEqual(workflow.count(test_path), 3) + self.assertIn("python " + test_path, workflow) + def test_readme_names_the_standalone_fix_release(self): readme = (PACKAGE_ROOT / "README.md").read_text() self.assertIn(f'pip install -U "funasr-onnx>={EXPECTED_VERSION}"', readme) @@ -69,6 +75,12 @@ def test_readme_names_the_standalone_fix_release(self): self.assertIn("#3831", readme) self.assertIn("#3838", readme) self.assertIn("hotword", readme) + self.assertIn("#3842", readme) + self.assertIn("CMVN", readme) + self.assertIn("#3846", readme) + self.assertIn("#3847", readme) + self.assertIn("empty or sub-frame", readme) + self.assertIn("preserves the streaming cache", readme) def test_runtime_dependencies_keep_onnx_install_torch_free(self): requirements = setup_keyword_literal(read_setup_tree(), "install_requires") diff --git a/tests/test_funasr_onnx_vad_multiple_inputs.py b/tests/test_funasr_onnx_vad_multiple_inputs.py index fd24781147..a5cbaff48f 100644 --- a/tests/test_funasr_onnx_vad_multiple_inputs.py +++ b/tests/test_funasr_onnx_vad_multiple_inputs.py @@ -4,6 +4,7 @@ import os import tempfile import unittest +from unittest.mock import Mock import numpy as np @@ -44,10 +45,10 @@ def fake_infer(inputs): return scores, inputs[1:] -def make_model(batch_size=1): +def make_model(batch_size=1, **frontend_conf): # Keep __call__, WavFrontend and E2EVadModel real; no model files. model = Fsmn_vad.__new__(Fsmn_vad) - model.frontend = WavFrontend(cmvn_file=None, **FRONTEND_CONF) + model.frontend = WavFrontend(cmvn_file=None, **{**FRONTEND_CONF, **frontend_conf}) model.batch_size = batch_size # E2EVadModel's defaults equal the template's model_conf. model.vad_scorer_config = {} @@ -99,6 +100,80 @@ def test_batch_size_above_one(self): result = model([self.wav[name] for name in names]) self.assertEqual(result, [SEGMENTS[name] for name in names]) + def test_short_inputs_and_frame_boundary(self): + for fs, frame_length, lfr_m in ( + (16000, 25, 5), + (8000, 25, 5), + (16000, 20, 5), + (16000, 25, 1), + ): + window = int(fs * frame_length / 1000) + for batch_size in (1, 2, 3): + model = make_model(batch_size, fs=fs, frame_length=frame_length, lfr_m=lfr_m) + model.infer = Mock(side_effect=fake_infer) + for samples in (0, 1, window // 2, window - 1, window, window + 1, 2 * window): + audio = np.zeros(samples, dtype=np.float32) + path = os.path.join(os.path.dirname(self.wav["a"]), "short.wav") + soundfile.write(path, audio, fs) + for audio_in in (audio, path): + with self.subTest( + fs=fs, + frame_length=frame_length, + lfr_m=lfr_m, + batch_size=batch_size, + samples=samples, + input_type=type(audio_in).__name__, + ): + model.infer.reset_mock() + self.assertEqual(model(audio_in), [[]]) + self.assertEqual(model.infer.call_count, int(samples >= window)) + + def test_short_files_keep_their_input_positions(self): + for samples in (0, 160, 320, 399): + path = os.path.join(os.path.dirname(self.wav["a"]), "short.wav") + soundfile.write(path, np.zeros(samples, dtype=np.float32), 16000) + paths = {**self.wav, "short": path} + expected = {**SEGMENTS, "short": []} + for batch_size in (1, 2, 3): + model = make_model(batch_size) + model.infer = Mock(side_effect=fake_infer) + for names in ( + ["short", "a"], + ["a", "short"], + ["a", "short", "b"], + ["short", "short"], + ["short", "b", "short"], + ): + with self.subTest(samples=samples, batch_size=batch_size, names=names): + model.infer.reset_mock() + result = model([paths[name] for name in names]) + self.assertEqual(result, [expected[name] for name in names]) + self.assertEqual( + model.infer.call_count, sum(name != "short" for name in names) + ) + self.assertEqual(len({id(segments) for segments in result}), len(result)) + + def test_extract_feat_with_empty_frames(self): + for lfr_m, lfr_n in ((1, 1), (5, 1), (3, 2)): + model = make_model(lfr_m=lfr_m, lfr_n=lfr_n) + empty = np.zeros(0, dtype=np.float32) + short = np.zeros(399, dtype=np.float32) + normal = np.zeros(1600, dtype=np.float32) + expected, expected_len = model.extract_feat([normal]) + for waveforms, lengths in ( + ([empty], [0]), + ([short], [0]), + ([empty, short, normal], [0, 0, expected_len[0]]), + ): + with self.subTest(lfr_m=lfr_m, lfr_n=lfr_n, lengths=lengths): + feats, feats_len = model.extract_feat(waveforms) + np.testing.assert_array_equal(feats_len, lengths) + self.assertEqual(feats.shape, (len(waveforms), max(lengths), 80 * lfr_m)) + self.assertEqual(feats.dtype, np.float32) + if len(waveforms) > 1: + np.testing.assert_array_equal(feats[-1], expected[0]) + self.assertFalse(feats[:-1].any()) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_funasr_torch_package.py b/tests/test_funasr_torch_package.py index ce39c86137..04cdcf01f5 100644 --- a/tests/test_funasr_torch_package.py +++ b/tests/test_funasr_torch_package.py @@ -119,12 +119,12 @@ def test_torch_package_does_not_require_unrelated_runtime_engines(distributions) } assert not names & {"torch-quant", "onnxruntime", "onnxruntime-gpu"} -def test_distribution_version_excludes_empty_published_release(distributions): +def test_distribution_version_includes_cmvn_parser_fix(distributions): from packaging.version import Version wheel, sdist = distributions version = metadata(wheel)["Version"] - assert Version(version) >= Version("0.1.4") + assert Version(version) >= Version("0.1.6") with tarfile.open(sdist, "r:gz") as archive: member = next(item for item in archive.getmembers() if item.name.count("/") == 1 and item.name.endswith("/PKG-INFO")) diff --git a/tests/test_merge_vad_length.py b/tests/test_merge_vad_length.py new file mode 100644 index 0000000000..f14fc952dc --- /dev/null +++ b/tests/test_merge_vad_length.py @@ -0,0 +1,81 @@ +"""Regression tests for the merge_length_s option of AutoModel.generate().""" + +import unittest +from unittest.mock import MagicMock, patch + +import numpy as np + + +VAD_SEGMENTS = [[0, 4000], [5000, 9000], [10000, 14000]] + + +class TestMergeVadLength(unittest.TestCase): + def _make_auto_model(self, **init_kwargs): + from funasr.auto.auto_model import AutoModel + + am = AutoModel.__new__(AutoModel) + am.model = MagicMock() + am.vad_model = MagicMock() + am.punc_model = None + am.punc_kwargs = {} + am.spk_model = None + am.spk_kwargs = {} + am.cb_model = None + am.spk_mode = None + am.vad_kwargs = {} + am.kwargs = { + "batch_size_s": 300, + "batch_size_threshold_s": 60, + "device": "cpu", + "disable_pbar": True, + "frontend": MagicMock(fs=16000), + "fs": 16000, + **init_kwargs, + } + am._reset_runtime_configs = MagicMock() + + def mock_inference(data, input_len=None, model=None, kwargs=None, **cfg): + if model is am.vad_model: + return [{"key": "utt", "value": [list(s) for s in VAD_SEGMENTS]}] + return [{"text": "x"} for _ in data] + + am.inference = MagicMock(side_effect=mock_inference) + return am + + @patch("funasr.auto.auto_model.slice_padding_audio_samples") + @patch("funasr.auto.auto_model.load_audio_text_image_video") + @patch("funasr.auto.auto_model.prepare_data_iterator") + def _asr_segments(self, am, mock_prep, mock_load, mock_slice, **cfg): + mock_prep.return_value = (["utt"], [np.zeros(16000 * 14, dtype=np.float32)]) + mock_load.return_value = np.zeros(16000 * 14, dtype=np.float32) + mock_slice.side_effect = lambda speech, speech_lengths, segments: ( + [np.zeros(16000, dtype=np.float32) for _ in segments], + [16000 for _ in segments], + ) + + am.inference_with_vad("dummy_input", merge_vad=True, **cfg) + + segments = [] + for call in mock_slice.call_args_list: + segments.extend(seg for seg, _ in call.args[2]) + return sorted(segments) + + def test_default_merge_length_is_15_seconds(self): + am = self._make_auto_model() + self.assertEqual(self._asr_segments(am), [[0, 14000]]) + + def test_generate_merge_length_s_is_applied(self): + am = self._make_auto_model() + self.assertEqual(self._asr_segments(am, merge_length_s=10), [[0, 9000], [9000, 14000]]) + + def test_construction_merge_length_s_is_applied(self): + am = self._make_auto_model(merge_length_s=10) + self.assertEqual(self._asr_segments(am), [[0, 9000], [9000, 14000]]) + + def test_generate_merge_length_s_overrides_construction_value(self): + am = self._make_auto_model(merge_length_s=10) + self.assertEqual(self._asr_segments(am, merge_length_s=15), [[0, 14000]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_version_contract.py b/tests/test_release_version_contract.py index 57623d6903..621d3ed838 100644 --- a/tests/test_release_version_contract.py +++ b/tests/test_release_version_contract.py @@ -4,16 +4,18 @@ ROOT = Path(__file__).resolve().parents[1] -RELEASE_VERSION = "1.4.21" +RELEASE_VERSION = "1.4.23" README_FILES = ("README.md", "README_zh.md", "README_ja.md", "README_ko.md") REPOSITORY_ROLE_FILES = ("docs/repository_roles.md", "docs/repository_roles_zh.md") @pytest.mark.parametrize("name", README_FILES) -def test_release_summary_preserves_streaming_punctuation_cache(name): +def test_release_summary_covers_no_lfr_and_tokenizer_vocabulary(name): summaries = [line for line in (ROOT / name).read_text(encoding="utf-8").splitlines() if f"**FunASR {RELEASE_VERSION}**" in line] assert len(summaries) == 1 - assert "cache" in summaries[0].lower() - assert "https://github.com/modelscope/FunASR/pull/3824" in summaries[0] + assert "lfr_m=lfr_n=1" in summaries[0] + assert "token_list" in summaries[0] + assert "https://github.com/modelscope/FunASR/pull/3849" in summaries[0] + assert "https://github.com/modelscope/FunASR/pull/3783" in summaries[0] diff --git a/tests/test_runtime_cmvn.py b/tests/test_runtime_cmvn.py new file mode 100644 index 0000000000..97889e2fc7 --- /dev/null +++ b/tests/test_runtime_cmvn.py @@ -0,0 +1,105 @@ +"""Exercise the standalone runtime frontends without model weights or engines.""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +CMVN = " 2 2\n 1 [ 1 -2 ]\n 2 2\n 1 [ 0.5 2 ]\n" + + +@pytest.fixture(params=["onnxruntime/funasr_onnx", "libtorch/funasr_torch"]) +def frontend_class(request): + # Loading the frontend directly keeps these tests independent of the + # optional inference engines and exercises each package's real constructor. + path = ROOT / "runtime/python" / request.param / "utils/frontend.py" + spec = importlib.util.spec_from_file_location("runtime_cmvn_frontend", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.WavFrontend + + +def test_valid_cmvn_preserves_statistics_and_normalized_features(frontend_class, tmp_path): + path = tmp_path / "valid.mvn" + path.write_text(CMVN, encoding="utf-8") + frontend = frontend_class(cmvn_file=str(path)) + + assert frontend.cmvn.dtype == np.float64 + np.testing.assert_array_equal(frontend.cmvn, [[1, -2], [0.5, 2]]) + features = np.array([[0, 2], [-2, 4]], dtype=np.float32) + np.testing.assert_array_equal(frontend.apply_cmvn(features), [[0.5, 0], [-0.5, 4]]) + + +@pytest.mark.parametrize("placement", ["leading", "trailing", "between_sections", "after_tags"]) +def test_cmvn_accepts_blank_lines(frontend_class, tmp_path, placement): + blank = " \t\n\n" + if placement == "leading": + content = blank + CMVN + elif placement == "trailing": + content = CMVN + blank + elif placement == "between_sections": + content = CMVN.replace("", blank + "") + else: + content = CMVN.replace("", blank + "") + path = tmp_path / "blank lines.mvn" + path.write_text(content, encoding="utf-8") + + frontend = frontend_class(cmvn_file=str(path)) + + np.testing.assert_array_equal(frontend.cmvn, [[1, -2], [0.5, 2]]) + + +@pytest.mark.parametrize( + "content", + [ + "", + " \t\n", + "\n\n", + " 2 2\n", + " 2 2\n", + " 2 2\n 1 [ 1 -2 ]\n", + " 2 2\n 1 [ 0.5 2 ]\n", + CMVN.replace("\n", " "), + ], + ids=[ + "empty", + "blank", + "no_stats", + "truncated_shift", + "truncated_scale", + "shift_only", + "scale_only", + "compact", + ], +) +def test_incomplete_cmvn_fails_during_construction(frontend_class, tmp_path, content): + path = tmp_path / "incomplete.mvn" + path.write_text(content, encoding="utf-8") + + with pytest.raises(ValueError) as error: + frontend_class(cmvn_file=str(path)) + + assert str(path) in str(error.value) + + +def test_shipped_cmvn_is_unchanged_by_blank_lines(frontend_class, tmp_path): + shipped = ROOT / "runtime/triton_gpu/model_repo_sense_voice_small/feature_extractor/am.mvn" + original = frontend_class(cmvn_file=str(shipped), lfr_m=7, lfr_n=6, dither=0.0) + path = tmp_path / "shipped-with-blanks.mvn" + path.write_text("\n" + "\n \t\n".join(shipped.read_text().splitlines()) + "\n\n") + edited = frontend_class(cmvn_file=str(path), lfr_m=7, lfr_n=6, dither=0.0) + + assert original.cmvn.shape == (2, 560) + assert np.isfinite(original.cmvn).all() + np.testing.assert_array_equal(edited.cmvn, original.cmvn) + # Real filterbank + LFR + CMVN on a deterministic waveform, not a model stub. + waveform = np.sin(2 * np.pi * 440 * np.arange(1600) / 16000).astype(np.float32) * 0.1 + original_features, _ = original.fbank(waveform) + edited_features, _ = edited.fbank(waveform) + original_features, original_length = original.lfr_cmvn(original_features) + edited_features, edited_length = edited.lfr_cmvn(edited_features) + assert edited_length == original_length + np.testing.assert_array_equal(edited_features, original_features) diff --git a/tests/test_sentencepiece_tokenizer.py b/tests/test_sentencepiece_tokenizer.py new file mode 100644 index 0000000000..f3a4349eac --- /dev/null +++ b/tests/test_sentencepiece_tokenizer.py @@ -0,0 +1,162 @@ +"""Model vocabulary IDs must reach callers as pieces, not BPE-decoded text.""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest +import sentencepiece as spm +import torch + +from funasr.models.paraformer.model import Paraformer +from funasr.tokenizer.build_tokenizer import build_tokenizer +from funasr.tokenizer.sentencepiece_tokenizer import SentencepiecesTokenizer + + +@pytest.fixture(scope="module") +def bpe_model(tmp_path_factory): + prefix = tmp_path_factory.mktemp("sentencepiece") / "tiny" + spm.SentencePieceTrainer.train( + sentence_iterator=iter( + ["hello world", "hello there", "he tried to think how it could be", "你好 world"] * 20 + ), + model_prefix=str(prefix), + model_type="bpe", + vocab_size=48, + user_defined_symbols=["<|en|>"], + shuffle_input_sentence=False, + num_threads=1, + minloglevel=2, + ) + return str(prefix.with_suffix(".model")) + + +def vocabulary(bpe_model, reordered=True): + sp = spm.SentencePieceProcessor(model_file=bpe_model) + pieces = [sp.id_to_piece(i) for i in range(sp.get_piece_size())] + return list(reversed(pieces)) if reordered else pieces + + +@pytest.mark.parametrize("storage", ["list", "tuple", "json", "txt"]) +def test_reordered_vocabulary_decodes_expected_text(bpe_model, tmp_path, storage): + tokens = vocabulary(bpe_model) + token_list = tokens + if storage == "tuple": + token_list = tuple(tokens) + elif storage in ("json", "txt"): + path = tmp_path / ("tokens." + storage) + content = json.dumps(tokens) if storage == "json" else "\n".join(tokens) + "\n" + path.write_text(content, encoding="utf-8") + token_list = str(path) + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=token_list) + pieces = tokenizer.text2tokens("hello world") + model_ids = [tokens.index(piece) for piece in pieces] + + # Establish that the two vocabularies really assign different meanings to IDs. + assert tokenizer.decode(model_ids) != "hello world" + result = tokenizer.ids2tokens(model_ids) + + assert result == pieces + assert tokenizer.tokens2text(result) == "hello world" + + +@pytest.mark.parametrize("text", ["hello world", "你好 world", "<|en|> hello world", ""]) +def test_matching_vocabulary_preserves_text_and_piece_boundaries(bpe_model, text): + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=vocabulary(bpe_model, False)) + ids = tokenizer.encode(text) + + pieces = tokenizer.ids2tokens(ids) + + assert pieces == tokenizer.text2tokens(text) + assert tokenizer.tokens2text(pieces) == tokenizer.decode(ids) == text + + +@pytest.mark.parametrize("container", [list, tuple, iter, np.asarray]) +def test_model_ids_accept_base_tokenizer_containers(bpe_model, container): + tokens = vocabulary(bpe_model) + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=tokens) + ids = [tokens.index(piece) for piece in tokenizer.text2tokens("hello world")] + + assert tokenizer.tokens2text(tokenizer.ids2tokens(container(ids))) == "hello world" + + +def test_special_tokens_and_model_only_ids_are_preserved(bpe_model): + tokens = vocabulary(bpe_model) + [""] + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=tokens) + pieces = ["", "", "<|en|>", "▁", "▁", "", ""] + + assert tokenizer.ids2tokens([tokens.index(piece) for piece in pieces]) == pieces + assert tokenizer.ids2tokens([]) == [] + + +def test_model_ids_require_one_dimensional_numpy_array(bpe_model): + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=vocabulary(bpe_model)) + + with pytest.raises(ValueError, match="Must be 1 dim ndarray"): + tokenizer.ids2tokens(np.array([[1, 2]])) + + +@pytest.mark.parametrize("construction", ["omitted", "none", "build_tokenizer"]) +def test_without_vocabulary_preserves_legacy_text_decode(bpe_model, construction): + if construction == "build_tokenizer": + tokenizer = build_tokenizer("bpe", bpemodel=bpe_model) + elif construction == "none": + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=None) + else: + tokenizer = SentencepiecesTokenizer(bpe_model) + ids = tokenizer.encode("hello world") + + assert tokenizer.ids2tokens(ids) == "hello world" + assert tokenizer.ids2tokens(line=ids) == "hello world" + assert tokenizer.ids2tokens([ids, ids]) == ["hello world", "hello world"] + assert tokenizer.ids2tokens([]) == "" + + +def test_encode_decode_and_legacy_keyword_remain_sentencepiece_operations(bpe_model): + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=vocabulary(bpe_model)) + ids = tokenizer.encode("hello world") + model_ids = [tokenizer.token_list.index(p) for p in tokenizer.text2tokens("hello world")] + + assert tokenizer.tokens2ids("hello world") == ids + assert tokenizer.decode(ids) == "hello world" + assert tokenizer.tokens2text(tokenizer.ids2tokens(line=model_ids, unused=True)) == "hello world" + + +def test_paraformer_inference_uses_model_pieces_and_filters_control_ids(bpe_model, tmp_path): + tokens = ["", "", ""] + [ + p for p in vocabulary(bpe_model) if p not in ("", "") + ] + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=tokens) + pieces = tokenizer.text2tokens("hello world") + ids = [0, 1] + [tokens.index(p) for p in pieces] + [2, 0] + scores = torch.full((1, len(ids), len(tokens)), -10.0) + scores[0, torch.arange(len(ids)), ids] = 10.0 + lengths = torch.tensor([len(ids)]) + speech = torch.zeros(1, len(ids), 2) + # Only the acoustic stages are controlled; the actual inference method, + # tokenizer, special-ID filtering, text conversion and writer all run. + state = SimpleNamespace( + beam_search=None, + sos=1, + eos=2, + blank_id=0, + encode=lambda *args: (speech, lengths), + calc_predictor=lambda *args: (speech, lengths.float(), None, None), + cal_decoder_with_predictor=lambda *args: (scores, lengths), + ) + + results, _ = Paraformer.inference( + state, + speech, + data_lengths=lengths, + key=["sample"], + tokenizer=tokenizer, + data_type="fbank", + device="cpu", + output_dir=str(tmp_path), + ) + + assert results == [{"key": "sample", "text": "hello world"}] + assert (tmp_path / "1best_recog" / "token").read_text().strip() == ( + "sample " + " ".join(pieces) + ) diff --git a/tests/test_utterance_mvn.py b/tests/test_utterance_mvn.py index f74201cd03..851ee38868 100644 --- a/tests/test_utterance_mvn.py +++ b/tests/test_utterance_mvn.py @@ -194,8 +194,22 @@ def test_saved_config_and_checkpoint_round_trip(tmp_path): torch.testing.assert_close(actual, expected, rtol=0, atol=0) +@pytest.fixture +def ieee_fp32_matmul(): + # Batch shapes can select different TF32 kernels; test the FP32 contract. + matmul = torch.backends.cuda.matmul.allow_tf32 + cudnn = torch.backends.cudnn.allow_tf32 + try: + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + yield + finally: + torch.backends.cuda.matmul.allow_tf32 = matmul + torch.backends.cudnn.allow_tf32 = cudnn + + @pytest.mark.parametrize("norm_vars", [False, True]) -def test_real_audio_transformer_inference(device, norm_vars): +def test_real_audio_transformer_inference(device, norm_vars, ieee_fp32_matmul): """Real loader/frontend/encoder/CTC; small random model, no downloads or training.""" audio, fs = sf.read( Path(__file__).resolve().parents[1] / "runtime/funasr_api/asr_example.wav", dtype="float32" diff --git a/tests/test_wav_frontend_no_lfr.py b/tests/test_wav_frontend_no_lfr.py new file mode 100644 index 0000000000..a9af796626 --- /dev/null +++ b/tests/test_wav_frontend_no_lfr.py @@ -0,0 +1,133 @@ +import pytest +import torch + +from funasr.frontends.wav_frontend import WavFrontendOnline +from funasr.models.fsmn_vad_streaming import model as vad_model +from funasr.utils import fbank as kaldi + + +def make_frontend(tmp_path, cmvn): + cmvn_file = None + if cmvn: + cmvn_file = tmp_path / "no_lfr.mvn" + cmvn_file.write_text( + " 80 80\n 1 [ " + + " ".join(["0.25"] * 80) + + " ]\n 80 80\n 1 [ " + + " ".join(["2"] * 80) + + " ]\n" + ) + return WavFrontendOnline(cmvn_file=cmvn_file, dither=0.0) + + +@pytest.mark.parametrize("cmvn", (False, True)) +@pytest.mark.parametrize("is_final", (False, True)) +@pytest.mark.parametrize("lengths", ((5, 3, 0), (0, 0))) +def test_no_lfr_reports_valid_consumed_frames(tmp_path, cmvn, is_final, lengths): + frontend = make_frontend(tmp_path, cmvn) + inputs = torch.arange(len(lengths) * 7 * 80, dtype=torch.float32).reshape( + len(lengths), 7, 80 + ) + original = inputs.clone() + cache = {} + + feats, actual_lengths, consumed = frontend.forward_lfr_cmvn( + inputs, torch.tensor(lengths), is_final=is_final, cache=cache + ) + + assert actual_lengths.tolist() == list(lengths) + assert consumed.tolist() == list(lengths) + assert feats.shape == (len(lengths), max(lengths), 80) + assert feats.dtype == torch.float32 + assert cache == {} + for index, length in enumerate(lengths): + expected = original[index, :length] + if cmvn: + expected = (expected + 0.25) * 2 + torch.testing.assert_close(feats[index, :length], expected, rtol=0, atol=0) + assert torch.count_nonzero(feats[index, length:]) == 0 + + +@pytest.mark.parametrize("cmvn", (False, True)) +@pytest.mark.parametrize("empty_final", (False, True)) +@pytest.mark.parametrize("chunks", ((1600,), (400,) * 4, (160,) * 10, (80, 320, 40, 1160))) +def test_no_lfr_stream_matches_full_filterbank(tmp_path, cmvn, empty_final, chunks): + frontend = make_frontend(tmp_path, cmvn) + audio = 0.2 * torch.sin(torch.arange(1600, dtype=torch.float32) * (2 * torch.pi * 440 / 16000)) + expected = kaldi.fbank( + audio.unsqueeze(0) * (1 << 15), num_mel_bins=80, frame_length=25, frame_shift=10, + sample_frequency=16000, window_type="hamming", dither=0.0, energy_floor=0.0, + ) + if cmvn: + expected = (expected + 0.25) * 2 + cache, outputs, offset, emitted = {}, [], 0, 0 + calls = chunks + ((0,) if empty_final else ()) + for index, size in enumerate(calls): + chunk = audio[offset : offset + size] + offset += size + feats, lengths = frontend( + chunk.unsqueeze(0), torch.tensor([size]), cache=cache, + is_final=index == len(calls) - 1, return_waveform=True, + ) + count = int(lengths[0]) if lengths.numel() else 0 + if count: + outputs.append(feats[0, :count]) + start = emitted * 160 + stop = start + (count - 1) * 160 + 400 + torch.testing.assert_close(cache["aligned_waveforms"][0], audio[start:stop], rtol=0, atol=0) + emitted += count + else: + assert cache["aligned_waveforms"].numel() == 0 + assert cache["reserve_waveforms"].numel() == 0 + assert emitted == expected.shape[0] == 8 + torch.testing.assert_close(torch.cat(outputs), expected, rtol=1e-5, atol=1e-5) + + +@pytest.mark.parametrize("size", (0, 160, 399)) +@pytest.mark.parametrize("empty_final", (False, True)) +def test_no_lfr_short_stream_has_no_frames(tmp_path, size, empty_final): + frontend = make_frontend(tmp_path, False) + cache = {} + for index, length in enumerate((size, 0) if empty_final else (size,)): + feats, lengths = frontend( + torch.zeros(1, length), torch.tensor([length]), cache=cache, + is_final=not empty_final or index == 1, return_waveform=True, + ) + assert feats.numel() == 0 + assert not lengths.numel() or lengths.sum() == 0 + assert cache["aligned_waveforms"].numel() == 0 + + +@pytest.mark.parametrize("lfr_m", (1, 5)) +@pytest.mark.parametrize("offset_ms", (0, 1000)) +@pytest.mark.parametrize("chunk_ms", (20, 100, 5000)) +def test_offline_vad_finalizes_last_nonempty_chunk(lfr_m, offset_ms, chunk_ms): + class ScriptedEncoder(torch.nn.Module): + def __init__(self): + super().__init__() + self.frames = 0 + + def forward(self, feats, cache=None): + time_ms = (torch.arange(feats.shape[1]) + self.frames) * 10 + speech = ((time_ms >= 500 + offset_ms) & (time_ms < 1500 + offset_ms)) | ( + (time_ms >= 2500 + offset_ms) & (time_ms < 3200 + offset_ms) + ) + self.frames += feats.shape[1] + probability = torch.where(speech, 0.999, 0.001) + return torch.stack((1 - probability, probability), dim=-1).unsqueeze(0) + + model = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming) + torch.nn.Module.__init__(model) + model.encoder = ScriptedEncoder() + model.vad_opts = vad_model.VADXOptions() + audio = 0.2 * torch.sin( + torch.arange((4000 + offset_ms) * 16, dtype=torch.float32) + * (2 * torch.pi * 440 / 16000) + ) + frontend = WavFrontendOnline(lfr_m=lfr_m, lfr_n=1, dither=0.0) + result, _ = model.inference( + [audio], key=["fixture"], cache={}, frontend=frontend, device="cpu", + chunk_size=chunk_ms, is_streaming_input=False, is_final=True, dynamic_silence=True, + ) + assert model.encoder.frames == (4000 + offset_ms) // 10 - 2 + assert result == [{"key": "fixture", "value": [[240 + offset_ms, 3980 + offset_ms]]}] From a3050a3ea7e5a5516bd3cb36998ad32e42b47902 Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Sun, 8 Nov 2026 01:02:26 +0800 Subject: [PATCH 3/3] test: retain tokenizer regressions alongside normalization fixes Resolve the confirmed companion/main conflict by retaining the accepted tokenizer changes and both CI selections. All 58 combined CPU/CUDA regressions passed on the exact combined source. No history rewrite or protection bypass. Signed-off-by: LauraGPT --- .../workflows/test-numpy-compatibility.yml | 7 +++ funasr/tokenizer/char_tokenizer.py | 3 ++ funasr/tokenizer/phoneme_tokenizer.py | 3 ++ ...t_char_tokenizer_non_linguistic_symbols.py | 44 +++++++++++++++++++ 4 files changed, 57 insertions(+) create mode 100644 tests/test_char_tokenizer_non_linguistic_symbols.py diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml index a6f2b474c6..3705536689 100644 --- a/.github/workflows/test-numpy-compatibility.yml +++ b/.github/workflows/test-numpy-compatibility.yml @@ -9,6 +9,9 @@ on: - "tests/test_cli.py" - "funasr/auto/auto_model.py" - "funasr/tokenizer/sentencepiece_tokenizer.py" + - "funasr/tokenizer/char_tokenizer.py" + - "funasr/tokenizer/phoneme_tokenizer.py" + - "tests/test_char_tokenizer_non_linguistic_symbols.py" - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" @@ -50,6 +53,9 @@ on: - "tests/test_cli.py" - "funasr/auto/auto_model.py" - "funasr/tokenizer/sentencepiece_tokenizer.py" + - "funasr/tokenizer/char_tokenizer.py" + - "funasr/tokenizer/phoneme_tokenizer.py" + - "tests/test_char_tokenizer_non_linguistic_symbols.py" - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" @@ -135,6 +141,7 @@ jobs: tests/test_pypi_metadata.py \ tests/test_sensevoice_word_timestamps.py \ tests/test_sentencepiece_tokenizer.py \ + tests/test_char_tokenizer_non_linguistic_symbols.py \ tests/test_pcm_input_format.py \ tests/test_realtime_ws_service.py \ tests/test_realtime_ws_hf_overrides.py \ diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py index d25c98926f..69e8cd3a4d 100644 --- a/funasr/tokenizer/char_tokenizer.py +++ b/funasr/tokenizer/char_tokenizer.py @@ -44,6 +44,9 @@ def __init__( self.non_linguistic_symbols = set() else: self.non_linguistic_symbols = set(non_linguistic_symbols) + # An empty symbol (e.g. a blank line in the file) matches every position + # and would make text2tokens loop forever. + self.non_linguistic_symbols.discard("") self.remove_non_linguistic_symbols = remove_non_linguistic_symbols self.split_with_space = split_with_space self.seg_dict = None diff --git a/funasr/tokenizer/phoneme_tokenizer.py b/funasr/tokenizer/phoneme_tokenizer.py index ab479242a6..0005483e37 100644 --- a/funasr/tokenizer/phoneme_tokenizer.py +++ b/funasr/tokenizer/phoneme_tokenizer.py @@ -599,6 +599,9 @@ def __init__( self.non_linguistic_symbols = set() else: self.non_linguistic_symbols = set(non_linguistic_symbols) + # An empty symbol (e.g. a blank line in the file) matches every position + # and would make text2tokens loop forever. + self.non_linguistic_symbols.discard("") self.remove_non_linguistic_symbols = remove_non_linguistic_symbols def __repr__(self): diff --git a/tests/test_char_tokenizer_non_linguistic_symbols.py b/tests/test_char_tokenizer_non_linguistic_symbols.py new file mode 100644 index 0000000000..7644ba1eae --- /dev/null +++ b/tests/test_char_tokenizer_non_linguistic_symbols.py @@ -0,0 +1,44 @@ +"""Tokenizers must ignore empty non-linguistic symbols instead of looping forever.""" + +import os +import tempfile +import unittest + +from funasr.tokenizer.char_tokenizer import CharTokenizer +from funasr.tokenizer.phoneme_tokenizer import PhonemeTokenizer + + +class CharTokenizerNonLinguisticSymbolsTest(unittest.TestCase): + def test_blank_line_in_symbol_file_is_ignored(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, "nlsyms.txt") + with open(path, "w", encoding="utf-8") as f: + f.write("\n\n") + for remove in (False, True): + with self.subTest(remove_non_linguistic_symbols=remove): + tokenizer = CharTokenizer( + non_linguistic_symbols=path, remove_non_linguistic_symbols=remove + ) + self.assertEqual(tokenizer.non_linguistic_symbols, {""}) + expected = ["你", "好"] if remove else ["你", "", "好"] + self.assertEqual(tokenizer.text2tokens("你好"), expected) + + def test_empty_symbol_in_iterable_is_ignored(self): + tokenizer = CharTokenizer(non_linguistic_symbols=["", ""]) + self.assertEqual(tokenizer.non_linguistic_symbols, {""}) + self.assertEqual(tokenizer.text2tokens("a b"), ["a", "", "b"]) + + +class PhonemeTokenizerNonLinguisticSymbolsTest(unittest.TestCase): + def test_empty_symbol_is_ignored(self): + tokenizer = PhonemeTokenizer( + g2p_type=None, + non_linguistic_symbols=["", ""], + remove_non_linguistic_symbols=True, + ) + self.assertEqual(tokenizer.non_linguistic_symbols, {""}) + self.assertEqual(tokenizer.text2tokens("ab"), ["ab"]) + + +if __name__ == "__main__": + unittest.main()