diff --git a/.github/workflows/test-cif-predictor.yml b/.github/workflows/test-cif-predictor.yml new file mode 100644 index 0000000000..72bc9a9ac7 --- /dev/null +++ b/.github/workflows/test-cif-predictor.yml @@ -0,0 +1,58 @@ +name: Validate CIF predictor boundaries + +on: + pull_request: + paths: + - "funasr/models/paraformer/cif_predictor.py" + - "tests/test_cif_mixed_zero_fires.py" + - ".github/workflows/test-cif-predictor.yml" + push: + branches: [main] + paths: + - "funasr/models/paraformer/cif_predictor.py" + - "tests/test_cif_mixed_zero_fires.py" + - ".github/workflows/test-cif-predictor.yml" + +permissions: + contents: read + +jobs: + cif-boundaries: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + OMP_NUM_THREADS: "1" + MKL_NUM_THREADS: "1" + HF_HUB_OFFLINE: "1" + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install CPU and ONNX test dependencies + run: | + python -m pip install torch==2.10.0 torchaudio==2.10.0 --index-url https://download.pytorch.org/whl/cpu + python -m pip install -e . pytest numpy==1.26.4 onnx==1.17.0 onnxruntime==1.23.2 --report install-report.json + - name: Record environment and require ONNX dependencies + run: | + git rev-parse HEAD | tee source-commit.txt + python -c "import onnx, onnxruntime; assert 'CPUExecutionProvider' in onnxruntime.get_available_providers()" + python -m pip freeze | tee resolved-dependencies.txt + python -m pip check | tee pip-check.txt + - name: Validate mixed zero-fire batches, gradients and ONNX export + run: | + python -m pytest -q tests/test_cif_mixed_zero_fires.py --junitxml=cif-results.xml + python -c "import xml.etree.ElementTree as E; r=E.parse('cif-results.xml').getroot(); assert len(list(r.iter('testcase')))==6; assert not any(list(r.iter(k)) for k in ['failure','error','skipped'])" + - uses: actions/upload-artifact@v4 + if: always() + with: + name: cif-boundary-evidence + path: | + resolved-dependencies.txt + pip-check.txt + cif-results.xml + source-commit.txt + install-report.json diff --git a/.github/workflows/test-funasr-onnx-package.yml b/.github/workflows/test-funasr-onnx-package.yml index 1a0e1438fd..f94b86e098 100644 --- a/.github/workflows/test-funasr-onnx-package.yml +++ b/.github/workflows/test-funasr-onnx-package.yml @@ -13,6 +13,7 @@ on: - "tests/test_funasr_onnx_paraformer_online_empty_final.py" - "tests/test_funasr_onnx_vad_online_short_chunks.py" - "tests/test_funasr_onnx_vad_multiple_inputs.py" + - "tests/test_funasr_onnx_punctuation_empty_input.py" - "tests/test_rich_transcription_postprocess.py" - "tests/test_rich_transcription_event_segments.py" push: @@ -29,6 +30,7 @@ on: - "tests/test_funasr_onnx_paraformer_online_empty_final.py" - "tests/test_funasr_onnx_vad_online_short_chunks.py" - "tests/test_funasr_onnx_vad_multiple_inputs.py" + - "tests/test_funasr_onnx_punctuation_empty_input.py" - "tests/test_rich_transcription_postprocess.py" - "tests/test_rich_transcription_event_segments.py" workflow_dispatch: @@ -84,7 +86,7 @@ jobs: import importlib.util from importlib.metadata import requires, version - assert version("funasr-onnx") == "0.4.7" + assert version("funasr-onnx") == "0.4.9" assert importlib.util.find_spec("torch") is None requirements = requires("funasr-onnx") or [] @@ -115,6 +117,9 @@ jobs: - name: Verify offline VAD results per input run: python tests/test_funasr_onnx_vad_multiple_inputs.py + - name: Verify empty punctuation streaming cache + run: python tests/test_funasr_onnx_punctuation_empty_input.py + - name: Verify source distribution error paths run: | python -m pip install --force-reinstall --no-deps dist/funasr-onnx/*.tar.gz @@ -125,7 +130,7 @@ jobs: if: matrix.python-version == '3.12' uses: actions/upload-artifact@v7 with: - name: funasr-onnx-0.4.7 + name: funasr-onnx-0.4.9 path: dist/funasr-onnx/* if-no-files-found: error diff --git a/.github/workflows/test-kws-output.yml b/.github/workflows/test-kws-output.yml index 55b2f4cb03..754e333e0c 100644 --- a/.github/workflows/test-kws-output.yml +++ b/.github/workflows/test-kws-output.yml @@ -8,12 +8,14 @@ on: - "funasr/models/sanm_kws/**" - "funasr/models/sanm_kws_streaming/**" - "funasr/frontends/wav_frontend.py" + - "funasr/models/fsmn_vad_streaming/model.py" - "funasr/auto/auto_model.py" - "funasr/utils/datadir_writer.py" - "funasr/utils/kws_utils.py" - "tests/test_kws_optional_output.py" - "tests/test_kws_streaming_continuity.py" - "tests/test_kws_utils.py" + - "tests/test_wav_frontend_no_lfr.py" - "tests/test_fsmn_vad_streaming_buffers.py" - "tests/test_dynamic_streaming_vad.py" - "tests/test_fsmn_vad_dynamic_silence.py" @@ -26,12 +28,14 @@ on: - "funasr/models/sanm_kws/**" - "funasr/models/sanm_kws_streaming/**" - "funasr/frontends/wav_frontend.py" + - "funasr/models/fsmn_vad_streaming/model.py" - "funasr/auto/auto_model.py" - "funasr/utils/datadir_writer.py" - "funasr/utils/kws_utils.py" - "tests/test_kws_optional_output.py" - "tests/test_kws_streaming_continuity.py" - "tests/test_kws_utils.py" + - "tests/test_wav_frontend_no_lfr.py" - "tests/test_fsmn_vad_streaming_buffers.py" - "tests/test_dynamic_streaming_vad.py" - "tests/test_fsmn_vad_dynamic_silence.py" @@ -63,6 +67,7 @@ jobs: tests/test_kws_optional_output.py \ tests/test_kws_streaming_continuity.py \ tests/test_kws_utils.py \ + tests/test_wav_frontend_no_lfr.py \ tests/test_fsmn_vad_streaming_buffers.py \ tests/test_dynamic_streaming_vad.py \ tests/test_fsmn_vad_dynamic_silence.py diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml index 288f6926c8..962f78ef3a 100644 --- a/.github/workflows/test-numpy-compatibility.yml +++ b/.github/workflows/test-numpy-compatibility.yml @@ -8,6 +8,11 @@ on: - "funasr/cli.py" - "tests/test_cli.py" - "funasr/auto/auto_model.py" + - "funasr/tokenizer/sentencepiece_tokenizer.py" + - "funasr/tokenizer/char_tokenizer.py" + - "funasr/tokenizer/phoneme_tokenizer.py" + - "tests/test_char_tokenizer_non_linguistic_symbols.py" + - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" - "funasr/bin/realtime_ws.py" @@ -45,6 +50,11 @@ on: - "funasr/cli.py" - "tests/test_cli.py" - "funasr/auto/auto_model.py" + - "funasr/tokenizer/sentencepiece_tokenizer.py" + - "funasr/tokenizer/char_tokenizer.py" + - "funasr/tokenizer/phoneme_tokenizer.py" + - "tests/test_char_tokenizer_non_linguistic_symbols.py" + - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" - "funasr/bin/realtime_ws.py" @@ -125,6 +135,8 @@ jobs: tests/test_eend_feature_compatibility.py \ tests/test_pypi_metadata.py \ tests/test_sensevoice_word_timestamps.py \ + tests/test_sentencepiece_tokenizer.py \ + tests/test_char_tokenizer_non_linguistic_symbols.py \ tests/test_pcm_input_format.py \ tests/test_realtime_ws_service.py \ tests/test_realtime_ws_hf_overrides.py \ diff --git a/README.md b/README.md index 277870f0a3..f36b7a0863 100644 --- a/README.md +++ b/README.md @@ -364,7 +364,9 @@ recordings with the same evaluation scope. ## What's new - **MOSS-Transcribe-Diarize** brings long-form ASR, timestamps, and anonymous speaker labels to FunASR services, Docker, Kubernetes, vLLM/SGLang workflows, and FunClip. [Deploy MOSS ->](./docs/moss_transcribe_diarize.md) -- **FunASR 1.4.21** preserves the streaming punctuation cache on empty or whitespace chunks, returning an empty result without touching cached words or the embedding path; nonempty inference is unchanged. [Fix #3824](https://github.com/modelscope/FunASR/pull/3824). Install with `python -m pip install -U "funasr==1.4.21"`. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) +- **FunASR 1.4.23** fixes the online frontend cursor for `lfr_m=lfr_n=1` and redundant empty final offline VAD chunks. With a supplied `token_list`, SentencePiece `ids2tokens` returns model pieces; use `tokens2text` for text. Without `token_list`, legacy text decoding is unchanged. [Fix #3849](https://github.com/modelscope/FunASR/pull/3849), [fix #3783](https://github.com/modelscope/FunASR/pull/3783). Install with `python -m pip install -U "funasr==1.4.23"`. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23) + **FunASR 1.4.22** honors per-call `merge_length_s` and evaluates dynamic VAD silence thresholds per frame using the current segment duration, preserving the fixed-threshold opt-out. [Fix #3841](https://github.com/modelscope/FunASR/pull/3841), [fix #3833](https://github.com/modelscope/FunASR/pull/3833). [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22) + **FunASR 1.4.21** preserves the streaming punctuation cache on empty or whitespace chunks; nonempty inference is unchanged. [Fix #3824](https://github.com/modelscope/FunASR/pull/3824). [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) **FunASR 1.4.20** preserves the original optional speaker dependency failure in a chained `ImportError` when `spk_model` is requested, while ASR-only construction remains available. [Fix #3819](https://github.com/modelscope/FunASR/pull/3819). [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.20) **FunASR 1.4.19** handles empty-token Paraformer batches, enforces `max_speakers` for automatic clustering, and adds validated JSON-object `--hf_overrides` support to realtime WebSocket serving. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.19) Previous release **FunASR 1.4.18** fixes cross-script hotword misreplacement, filtered-token timestamp alignment, and Chinese VAD sentence/speaker drift. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.18) diff --git a/README_ja.md b/README_ja.md index 13450d8641..25ee7e109f 100644 --- a/README_ja.md +++ b/README_ja.md @@ -112,7 +112,9 @@ CER/WER をそろえて比較してください。オフラインのスループ ## 最新情報 - **MOSS-Transcribe-Diarize** を FunASR service、Docker、Kubernetes、vLLM/SGLang workflow、FunClip に統合し、長時間 ASR、timestamp、匿名 speaker label を一度に処理できます。[MOSS をデプロイ ->](./docs/moss_transcribe_diarize.md) -- **FunASR 1.4.21** はストリーミング句読点処理の空または空白入力で cache を保持し、キャッシュ済み単語や embedding 経路に触れず空結果を返します。非空推論は変更されません。[修正 #3824](https://github.com/modelscope/FunASR/pull/3824)。`python -m pip install -U "funasr==1.4.21"` でインストールできます。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) +- **FunASR 1.4.23** は `lfr_m=lfr_n=1` のオンライン特徴抽出のフレーム位置と、オフライン VAD の余分な空の最終チャンクを修正します。`token_list` 指定時の SentencePiece `ids2tokens` はモデルのピースを返し、テキストへの変換には `tokens2text` を使います。未指定時の従来のテキスト復号は維持します。[修正 #3849](https://github.com/modelscope/FunASR/pull/3849)、[修正 #3783](https://github.com/modelscope/FunASR/pull/3783)。`python -m pip install -U "funasr==1.4.23"` でインストールできます。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23) + **FunASR 1.4.22** は呼び出しごとの `merge_length_s` を反映し、現在の発話区間の長さに基づいて動的 VAD 無音しきい値をフレーム単位で更新します。固定しきい値モードも維持します。[修正 #3841](https://github.com/modelscope/FunASR/pull/3841)、[修正 #3833](https://github.com/modelscope/FunASR/pull/3833)。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22) + **FunASR 1.4.21** はストリーミング句読点処理の空入力で cache を保持します。非空推論は変更されません。[修正 #3824](https://github.com/modelscope/FunASR/pull/3824)。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) **FunASR 1.4.20** は `spk_model` 指定時に任意の話者依存モジュールを読み込めない場合、元の例外を連鎖した `ImportError` で報告します。ASR のみの構築は引き続き利用できます。[修正 #3819](https://github.com/modelscope/FunASR/pull/3819)。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.20) **FunASR 1.4.19** は Paraformer の空 token batch を処理し、自動クラスタリングの `max_speakers` 上限を適用します。リアルタイム WebSocket サービスには JSON object を検証する `--hf_overrides` を追加しました。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.19) 前のリリース **FunASR 1.4.18** は中国語とラテン文字が混在する hotword の誤置換、特殊 token 除去後の timestamp 対応、中国語 VAD 分割での文と話者のずれを修正しました。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.18) diff --git a/README_ko.md b/README_ko.md index a4a8ece06a..259840723f 100644 --- a/README_ko.md +++ b/README_ko.md @@ -111,7 +111,9 @@ checkpoint/revision, 오디오 집합, 하드웨어, 배치, 워밍업, 측정 ## 최신 소식 - **MOSS-Transcribe-Diarize**를 FunASR service, Docker, Kubernetes, vLLM/SGLang workflow, FunClip에 통합해 긴 오디오 ASR, timestamp, 익명 speaker label을 한 번에 처리합니다. [MOSS 배포 ->](./docs/moss_transcribe_diarize.md) -- **FunASR 1.4.21**은 스트리밍 문장부호 처리의 빈 입력이나 공백 입력에서 cache를 유지하고, 캐시된 단어나 embedding 경로를 건드리지 않은 채 빈 결과를 반환합니다. 비어 있지 않은 추론은 변경되지 않습니다. [수정 #3824](https://github.com/modelscope/FunASR/pull/3824). `python -m pip install -U "funasr==1.4.21"`로 설치할 수 있습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) +- **FunASR 1.4.23**은 `lfr_m=lfr_n=1` 온라인 프런트엔드의 프레임 위치와 오프라인 VAD의 불필요한 빈 마지막 청크를 수정합니다. `token_list`를 지정하면 SentencePiece `ids2tokens`는 모델 토큰을 반환하며, 텍스트 변환에는 `tokens2text`를 사용합니다. 미지정 시 기존 텍스트 디코딩은 유지됩니다. [수정 #3849](https://github.com/modelscope/FunASR/pull/3849), [수정 #3783](https://github.com/modelscope/FunASR/pull/3783). `python -m pip install -U "funasr==1.4.23"`로 설치할 수 있습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23) + **FunASR 1.4.22**는 호출별 `merge_length_s`를 반영하고 현재 발화 구간 길이에 따라 동적 VAD 무음 임계값을 프레임마다 갱신합니다. 고정 임계값 모드도 유지합니다. [수정 #3841](https://github.com/modelscope/FunASR/pull/3841), [수정 #3833](https://github.com/modelscope/FunASR/pull/3833). [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22) + **FunASR 1.4.21**은 스트리밍 문장부호 처리의 빈 입력에서 cache를 유지합니다. 비어 있지 않은 추론은 변경되지 않습니다. [수정 #3824](https://github.com/modelscope/FunASR/pull/3824). [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) **FunASR 1.4.20**은 `spk_model` 사용 시 선택적 화자 의존성 가져오기가 실패하면 원래 예외를 연결한 `ImportError`를 제공합니다. ASR 전용 구성은 계속 사용할 수 있습니다. [수정 #3819](https://github.com/modelscope/FunASR/pull/3819). [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.20) **FunASR 1.4.19**는 Paraformer의 빈 token batch를 처리하고 자동 클러스터링에 `max_speakers` 상한을 적용합니다. 실시간 WebSocket 서비스에 JSON object를 검증하는 `--hf_overrides`도 추가했습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.19) 이전 릴리스 **FunASR 1.4.18**은 중국어와 라틴 문자가 섞인 hotword의 잘못된 치환, 특수 token 제거 후 timestamp 정렬, 중국어 VAD 분할의 문장 및 화자 불일치를 수정했습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.18) diff --git a/README_zh.md b/README_zh.md index eccb4f7144..1c3199b476 100644 --- a/README_zh.md +++ b/README_zh.md @@ -157,7 +157,9 @@ checkpoint/revision、音频集、硬件、批量大小、预热、计时范围 ## 最新动态 - **MOSS-Transcribe-Diarize** 已接入 FunASR 服务、Docker、Kubernetes、vLLM/SGLang 工作流和 FunClip，一次完成长音频转写、时间戳与匿名说话人标注。[部署 MOSS ->](./docs/moss_transcribe_diarize_zh.md) -- **FunASR 1.4.21** 在流式标点收到空白或空输入时保留 cache，并在不访问缓存词或 embedding 路径的情况下返回空结果；非空推理保持不变。[修复 #3824](https://github.com/modelscope/FunASR/pull/3824)。安装：`python -m pip install -U "funasr==1.4.21"`。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) +- **FunASR 1.4.23** 修复 `lfr_m=lfr_n=1` 在线前端的帧游标和离线 VAD 多余的空尾块。提供 `token_list` 时，SentencePiece 的 `ids2tokens` 返回模型词元，请用 `tokens2text` 转为文本；未提供 `token_list` 时保持旧有文本解码行为。[修复 #3849](https://github.com/modelscope/FunASR/pull/3849)、[修复 #3783](https://github.com/modelscope/FunASR/pull/3783)。安装：`python -m pip install -U "funasr==1.4.23"`。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23) + **FunASR 1.4.22** 支持逐调用覆盖 `merge_length_s`，并根据当前语音段时长逐帧更新动态 VAD 静音阈值，同时保留固定阈值模式。[修复 #3841](https://github.com/modelscope/FunASR/pull/3841)、[修复 #3833](https://github.com/modelscope/FunASR/pull/3833)。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22) + **FunASR 1.4.21** 在流式标点收到空白或空输入时保留 cache；非空推理保持不变。[修复 #3824](https://github.com/modelscope/FunASR/pull/3824)。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21) **FunASR 1.4.20** 在指定 `spk_model` 而可选说话人依赖导入失败时，通过带原始异常链的 `ImportError` 提供诊断，同时保留仅 ASR 的构造路径。[修复 #3819](https://github.com/modelscope/FunASR/pull/3819)。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.20) **FunASR 1.4.19** 修复 Paraformer 空 token 批次处理，使自动聚类遵守 `max_speakers` 上限，并为实时 WebSocket 服务新增带 JSON 对象校验的 `--hf_overrides` 参数。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.19) 上一版本 **FunASR 1.4.18** 修复跨中英文热词误替换、过滤特殊符号后的时间戳对齐，以及中文 VAD 分段的句子与说话人错位。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.18) diff --git a/docs/python_api.md b/docs/python_api.md index 160a662735..f95a5ba270 100644 --- a/docs/python_api.md +++ b/docs/python_api.md @@ -42,7 +42,7 @@ The wrapper accepts `**kwargs`, not a universally validated parameter schema. A | `batch_size_s` | `300` | VAD-path batching budget in seconds, based on longest padded segment times segment count, not a hard file-duration limit. CPU VAD processing uses single-segment batches. | | `batch_size_threshold_s` | `60` | VAD batching threshold for segment duration, not a VAD splitting setting. | | `merge_vad` | `False` | Enables merging detected VAD regions for this call. | -| `merge_length_s` | `15` | VAD merge target read before call overrides are merged into ASR config. Set this at construction in this checkout. | +| `merge_length_s` | `15` | Target length in seconds for merged VAD regions when `merge_vad=True`. A per-call value overrides the construction value. | | `cache` | No wrapper-managed session | Pass a caller-owned dictionary on every streaming call. It is not a hub/model download cache. | | `sentence_timestamp` | `False` | Requests sentence records in the VAD pipeline; depends on available timestamps and punctuation, with limited VAD fallback. | | `return_spk_res` | `True` | Includes clustered speaker results when VAD and a speaker model are configured. `False` does not avoid construction or segment embedding computation. | diff --git a/docs/python_api_zh.md b/docs/python_api_zh.md index 42d9a4a139..1f70e6d900 100644 --- a/docs/python_api_zh.md +++ b/docs/python_api_zh.md @@ -42,7 +42,7 @@ | `batch_size_s` | `300` | VAD 路径的秒级批处理预算，按最长补齐片段长度乘以片段数量估算，不是文件时长硬上限。CPU 的 VAD 路径按单片段处理。 | | `batch_size_threshold_s` | `60` | VAD 片段参与组批的时长阈值，不是 VAD 切分参数。 | | `merge_vad` | `False` | 为本次调用启用 VAD 区域合并。 | -| `merge_length_s` | `15` | VAD 合并目标，在本次参数合并进 ASR 配置之前读取。当前代码版本应在构建时设置它。 | +| `merge_length_s` | `15` | `merge_vad=True` 时合并后 VAD 片段的目标长度，单位为秒。本次调用传入的值会覆盖构建时设置的值。 | | `cache` | 包装层不维护会话 | 每次流式调用都传入调用方持有的字典。它不是模型下载缓存。 | | `sentence_timestamp` | `False` | 请求 VAD 流程输出句子记录，依赖实际时间戳与标点，部分情形可回退到 VAD 区域。 | | `return_spk_res` | `True` | 配置 VAD 与说话人模型时输出聚类结果。设为 `False` 并不能省去模型构建或片段说话人特征计算。 | diff --git a/docs/repository_roles.md b/docs/repository_roles.md index 911a2b6555..96aa91ed42 100644 --- a/docs/repository_roles.md +++ b/docs/repository_roles.md @@ -5,7 +5,7 @@ This document explains the responsibility boundaries, user entry points, and iss > **Directional roadmap, not a release promise.** > This document records shipped capabilities and active work, but does not commit to > future version numbers or dates. The current Python release is -> [`funasr==1.4.21`](https://github.com/modelscope/FunASR/releases/tag/v1.4.21). +> [`funasr==1.4.23`](https://github.com/modelscope/FunASR/releases/tag/v1.4.23). > Any future breaking release still requires a maintainer-approved milestone and > migration plan. diff --git a/docs/repository_roles_zh.md b/docs/repository_roles_zh.md index 6a847f1706..c0214965e1 100644 --- a/docs/repository_roles_zh.md +++ b/docs/repository_roles_zh.md @@ -4,7 +4,7 @@ > **方向性路线图，不是版本承诺。** > 本文档记录已交付能力与正在推进的工作，但不承诺未来版本号或日期。当前 Python -> 版本是 [`funasr==1.4.21`](https://github.com/modelscope/FunASR/releases/tag/v1.4.21)。 +> 版本是 [`funasr==1.4.23`](https://github.com/modelscope/FunASR/releases/tag/v1.4.23)。 > 任何未来的 breaking release 仍需 maintainer 确认 milestone 与迁移方案。 --- diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py index 8ec9fcbce0..b8320dfa7c 100644 --- a/funasr/auto/auto_model.py +++ b/funasr/auto/auto_model.py @@ -1043,10 +1043,10 @@ def inference_with_vad(self, input, input_len=None, **cfg): # FIX(gcf): concat the vad clips for sense vocie model for better aed if cfg.get("merge_vad", False): + # Call options are merged into kwargs only after this point. + merge_length_s = cfg.get("merge_length_s", kwargs.get("merge_length_s", 15)) for i in range(len(res)): - res[i]["value"] = merge_vad( - res[i]["value"], kwargs.get("merge_length_s", 15) * 1000 - ) + res[i]["value"] = merge_vad(res[i]["value"], merge_length_s * 1000) # step.2 compute asr model model = self.model diff --git a/funasr/frontends/wav_frontend.py b/funasr/frontends/wav_frontend.py index 593cc14462..cb671e6df6 100644 --- a/funasr/frontends/wav_frontend.py +++ b/funasr/frontends/wav_frontend.py @@ -488,6 +488,7 @@ def forward_lfr_cmvn( lfr_splice_frame_idxs = [] for i in range(batch_size): mat = input[i, : input_lengths[i], :] + lfr_splice_frame_idx = mat.size(0) if self.lfr_m != 1 or self.lfr_n != 1: # update self.lfr_splice_cache in self.apply_lfr # mat, self.lfr_splice_cache[i], lfr_splice_frame_idx = self.apply_lfr(mat, self.lfr_m, self.lfr_n, self.lfr_splice_cache[i], diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.py index d8537ed20f..b59862e6b7 100644 --- a/funasr/models/fsmn_vad_streaming/model.py +++ b/funasr/models/fsmn_vad_streaming/model.py @@ -353,6 +353,7 @@ def __init__( self.output_data_buf_offset = 0 self.frame_probs = [] self.max_end_sil_frame_cnt_thresh = max_end_sil_frame_cnt_thresh + self.silence_schedule = None self.speech_noise_thres = speech_noise_thres self.scores = None self.max_time_out = False @@ -1004,7 +1005,9 @@ def inference( audio_sample = torch.cat((cache["prev_samples"], audio_sample_list[0])) - n = int(len(audio_sample) // chunk_stride_samples + int(_is_final)) + n = len(audio_sample) // chunk_stride_samples + if _is_final and len(audio_sample) % chunk_stride_samples: + n += 1 m = int(len(audio_sample) % chunk_stride_samples * (1 - int(_is_final))) segments = [] # Keep explicit fixed-threshold requests from being overwritten by the dynamic schedule. @@ -1012,9 +1015,11 @@ def inference( "dynamic_silence", kwargs.get("max_end_silence_time") is None ) silence_schedule = kwargs.get("silence_schedule", DEFAULT_SILENCE_SCHEDULE) - speech_to_sil_ms = self.vad_opts.speech_to_sil_time_thres - accumulated_ms = cache.get("_dynamic_accumulated_ms", 0) - in_speech = cache.get("_dynamic_in_speech", False) + # The schedule is looked up per frame in DetectOneFrame from the current segment's duration. + if "stats" in cache: + cache["stats"].silence_schedule = silence_schedule if dynamic_silence else None + if dynamic_silence: + cache["stats"].speech_noise_thres = 0.5 frontend_inference_kwargs = {} if getattr(frontend, "supports_aligned_waveforms", False): frontend_inference_kwargs["return_waveform"] = True @@ -1023,21 +1028,6 @@ def inference( kwargs["is_final"] = _is_final and i == n - 1 audio_sample_i = audio_sample[i * chunk_stride_samples : (i + 1) * chunk_stride_samples] - # Apply dynamic silence threshold (only accumulate while in speech) - if dynamic_silence and "stats" in cache: - vad_state = cache["stats"].vad_state_machine - # VadStateMachine: 2=kVadInStateStartPointNotDetected, 3=kVadInStateInSpeechSegment - if vad_state.value == 2 or in_speech: # kVadInStateInSpeechSegment - accumulated_ms += chunk_size - in_speech = True - for limit_ms, silence_ms in silence_schedule: - if accumulated_ms <= limit_ms: - cache["stats"].max_end_sil_frame_cnt_thresh = max(silence_ms - speech_to_sil_ms, 0) - cache["stats"].speech_noise_thres = 0.5 - break - cache["_dynamic_accumulated_ms"] = accumulated_ms - cache["_dynamic_in_speech"] = in_speech - # extract fbank feats speech, speech_lengths = extract_fbank( [audio_sample_i], @@ -1074,11 +1064,6 @@ def inference( segments_i = self.forward(**batch) if len(segments_i) > 0: segments.extend(*segments_i) - if dynamic_silence: - accumulated_ms = 0 - in_speech = False - cache["_dynamic_accumulated_ms"] = 0 - cache["_dynamic_in_speech"] = False cache["prev_samples"] = audio_sample[-m:] if m > 0 else torch.empty(0) if _is_final: @@ -1264,6 +1249,17 @@ def DetectOneFrame( cur_frm_idx - self.LatencyFrmNumAtStartPoint(cache=cache), cache=cache ) elif cache["stats"].vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: + silence_schedule = getattr(cache["stats"], "silence_schedule", None) + if silence_schedule is not None: + segment_ms = ( + cur_frm_idx - cache["stats"].confirmed_start_frame + 1 + ) * frm_shift_in_ms + for limit_ms, silence_ms in silence_schedule: + if segment_ms <= limit_ms: + cache["stats"].max_end_sil_frame_cnt_thresh = max( + silence_ms - self.vad_opts.speech_to_sil_time_thres, 0 + ) + break if ( cache["stats"].continous_silence_frame_count * frm_shift_in_ms >= cache["stats"].max_end_sil_frame_cnt_thresh diff --git a/funasr/models/paraformer/cif_predictor.py b/funasr/models/paraformer/cif_predictor.py index 927ae21950..70e6e2c8d9 100644 --- a/funasr/models/paraformer/cif_predictor.py +++ b/funasr/models/paraformer/cif_predictor.py @@ -655,6 +655,8 @@ def cif_v1_export(hidden, alphas, threshold: float): batch_idxs = torch.cumsum(batch_len, dim=0) shift_batch_idxs = torch.roll(batch_idxs, 1, dims=0) shift_batch_idxs[0] = 0 + # Empty samples have no frame to reset and may point past the last frame. + shift_batch_idxs = shift_batch_idxs[batch_len > 0] shift_frames[shift_batch_idxs] = 0 remains = fires - torch.floor(fires) @@ -884,6 +886,8 @@ def cif_v1(hidden, alphas, threshold): batch_idxs = torch.cumsum(batch_len, dim=0) shift_batch_idxs = torch.roll(batch_idxs, 1, dims=0) shift_batch_idxs[0] = 0 + # Empty samples have no frame to reset and may point past the last frame. + shift_batch_idxs = shift_batch_idxs[batch_len > 0] shift_frames[shift_batch_idxs] = 0 remains = fires - torch.floor(fires) diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py index d25c98926f..69e8cd3a4d 100644 --- a/funasr/tokenizer/char_tokenizer.py +++ b/funasr/tokenizer/char_tokenizer.py @@ -44,6 +44,9 @@ def __init__( self.non_linguistic_symbols = set() else: self.non_linguistic_symbols = set(non_linguistic_symbols) + # An empty symbol (e.g. a blank line in the file) matches every position + # and would make text2tokens loop forever. + self.non_linguistic_symbols.discard("") self.remove_non_linguistic_symbols = remove_non_linguistic_symbols self.split_with_space = split_with_space self.seg_dict = None diff --git a/funasr/tokenizer/phoneme_tokenizer.py b/funasr/tokenizer/phoneme_tokenizer.py index ab479242a6..0005483e37 100644 --- a/funasr/tokenizer/phoneme_tokenizer.py +++ b/funasr/tokenizer/phoneme_tokenizer.py @@ -599,6 +599,9 @@ def __init__( self.non_linguistic_symbols = set() else: self.non_linguistic_symbols = set(non_linguistic_symbols) + # An empty symbol (e.g. a blank line in the file) matches every position + # and would make text2tokens loop forever. + self.non_linguistic_symbols.discard("") self.remove_non_linguistic_symbols = remove_non_linguistic_symbols def __repr__(self): diff --git a/funasr/tokenizer/sentencepiece_tokenizer.py b/funasr/tokenizer/sentencepiece_tokenizer.py index acba22c2a4..1c76c8984a 100644 --- a/funasr/tokenizer/sentencepiece_tokenizer.py +++ b/funasr/tokenizer/sentencepiece_tokenizer.py @@ -13,11 +13,11 @@ class SentencepiecesTokenizer(BaseTokenizer): def __init__(self, bpemodel: Union[Path, str], **kwargs): """Initialize SentencepiecesTokenizer. - - Args: - bpemodel: TODO. - **kwargs: Additional keyword arguments. - """ + + Args: + bpemodel: TODO. + **kwargs: Additional keyword arguments. + """ super().__init__(**kwargs) self.bpemodel = str(bpemodel) # NOTE(kamo): @@ -41,39 +41,39 @@ def _build_sentence_piece_processor(self): def text2tokens(self, line: str) -> List[str]: """Text2tokens. - - Args: - line: TODO. - """ + + Args: + line: TODO. + """ self._build_sentence_piece_processor() return self.sp.EncodeAsPieces(line) def tokens2text(self, tokens: Iterable[str]) -> str: """Tokens2text. - - Args: - tokens: TODO. - """ + + Args: + tokens: TODO. + """ self._build_sentence_piece_processor() return self.sp.DecodePieces(list(tokens)) def encode(self, line: str, **kwargs) -> List[int]: """Encode. - - Args: - line: TODO. - **kwargs: Additional keyword arguments. - """ + + Args: + line: TODO. + **kwargs: Additional keyword arguments. + """ self._build_sentence_piece_processor() return self.sp.EncodeAsIds(line) def decode(self, line: List[int], **kwargs): """Decode. - - Args: - line: TODO. - **kwargs: Additional keyword arguments. - """ + + Args: + line: TODO. + **kwargs: Additional keyword arguments. + """ self._build_sentence_piece_processor() return self.sp.DecodeIds(line) @@ -81,20 +81,17 @@ def get_vocab_size(self): """Get vocab size.""" return self.sp.GetPieceSize() - def ids2tokens(self, *args, **kwargs): - """Ids2tokens. - - Args: - *args: Variable positional arguments. - **kwargs: Additional keyword arguments. - """ - return self.decode(*args, **kwargs) + def ids2tokens(self, line, **kwargs): + """Return model-vocabulary pieces, or legacy decoded text without a token list.""" + if getattr(self, "token_list", None) is not None: + return super().ids2tokens(line) + return self.decode(line, **kwargs) def tokens2ids(self, *args, **kwargs): """Tokens2ids. - - Args: - *args: Variable positional arguments. - **kwargs: Additional keyword arguments. - """ + + Args: + *args: Variable positional arguments. + **kwargs: Additional keyword arguments. + """ return self.encode(*args, **kwargs) diff --git a/funasr/version.txt b/funasr/version.txt index 8cc8fe024b..580ef4b6c1 100644 --- a/funasr/version.txt +++ b/funasr/version.txt @@ -1 +1 @@ -1.4.21 +1.4.23 diff --git a/runtime/python/libtorch/README.md b/runtime/python/libtorch/README.md index 9136bd74f1..00808b6ba1 100644 --- a/runtime/python/libtorch/README.md +++ b/runtime/python/libtorch/README.md @@ -21,12 +21,23 @@ python -m funasr.export.export_model --model-name damo/speech_paraformer-large_a ## Install the `funasr_torch` +`funasr_torch` is a separate distribution from `funasr`; upgrading the root +package alone does not update these TorchScript wrappers. Version 0.1.5 +includes the Contextual/Seaco Paraformer hotword batching and empty +hotword-file handling fix (#3838). This is a runtime wrapper fix, not a +model-weight update or an ASR accuracy claim. + +Version 0.1.6 makes the standalone CMVN reader accept blank lines and +raise a file-specific error for missing or truncated statistics (#3842). +Valid float64 statistics and shipped feature output are unchanged. This is +a CPU parser/package fix, not a model-weight or ASR accuracy update. + install from pip ```shell -pip install -U "funasr_torch>=0.1.4" +pip install -U "funasr_torch>=0.1.6" # For the users in China, you could install with the command: -# pip install -U funasr_torch -i https://mirror.sjtu.edu.cn/pypi/web/simple +# pip install -U "funasr_torch>=0.1.6" -i https://mirror.sjtu.edu.cn/pypi/web/simple ``` or install from source code diff --git a/runtime/python/libtorch/funasr_torch/utils/frontend.py b/runtime/python/libtorch/funasr_torch/utils/frontend.py index 743ee5e5f2..2772ad95d6 100644 --- a/runtime/python/libtorch/funasr_torch/utils/frontend.py +++ b/runtime/python/libtorch/funasr_torch/utils/frontend.py @@ -125,25 +125,29 @@ def load_cmvn( self, ) -> np.ndarray: with open(self.cmvn_file, "r", encoding="utf-8") as f: - lines = f.readlines() + lines = [line for line in f.readlines() if line.split()] means_list = [] vars_list = [] for i in range(len(lines)): line_item = lines[i].split() if line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": + line_item = lines[i + 1].split() if i + 1 < len(lines) else [] + if line_item and line_item[0] == "": add_shift_line = line_item[3 : (len(line_item) - 1)] means_list = list(add_shift_line) continue elif line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": + line_item = lines[i + 1].split() if i + 1 < len(lines) else [] + if line_item and line_item[0] == "": rescale_line = line_item[3 : (len(line_item) - 1)] vars_list = list(rescale_line) continue + if not means_list or not vars_list: + raise ValueError( + f"No / statistics found in cmvn file: {self.cmvn_file}" + ) means = np.array(means_list).astype(np.float64) vars = np.array(vars_list).astype(np.float64) cmvn = np.array([means, vars]) diff --git a/runtime/python/libtorch/setup.py b/runtime/python/libtorch/setup.py index 3ba4610e47..be4b7e3c59 100644 --- a/runtime/python/libtorch/setup.py +++ b/runtime/python/libtorch/setup.py @@ -15,7 +15,7 @@ def get_readme(): setuptools.setup( name="funasr_torch", - version="0.1.4", + version="0.1.6", platforms="Any", url="https://github.com/modelscope/FunASR.git", author="Speech Lab of DAMO Academy, Alibaba Group", diff --git a/runtime/python/onnxruntime/README.md b/runtime/python/onnxruntime/README.md index 0b19d0cc5f..8794ac4c74 100644 --- a/runtime/python/onnxruntime/README.md +++ b/runtime/python/onnxruntime/README.md @@ -7,6 +7,17 @@ does not update the installed ONNX wrappers. Check their versions separately with `python -m pip show funasr-onnx funasr`; use the source installation below when a required wrapper fix is newer than the published PyPI package. +Version 0.4.9 keeps one result per input when offline VAD receives empty or sub-frame +audio, without invoking ONNX Runtime for zero-frame features (#3846). It also returns +an empty punctuation chunk while it preserves the streaming cache and avoids +tokenization and inference (#3847). These are CPU runtime-wrapper fixes, not a +model-weight or ASR accuracy update. + +Version 0.4.8 makes the standalone CMVN reader accept blank lines and +raise a file-specific error for missing or truncated statistics (#3842). +Valid float64 statistics and shipped feature output are unchanged. This is +a CPU parser/package fix, not a model-weight or ASR accuracy update. + Version 0.4.7 includes the Contextual/Seaco Paraformer hotword batching fix (#3838), including empty hotword-file handling. This is a runtime wrapper fix, not a model-weight update or an ASR accuracy claim. @@ -28,9 +39,9 @@ and postprocessing fixes, not a model-weight update or an ASR accuracy claim. install from pip ```shell -pip install -U "funasr-onnx>=0.4.7" +pip install -U "funasr-onnx>=0.4.9" # For the users in China, you could install with the command: -# pip install -U "funasr-onnx>=0.4.7" -i https://mirror.sjtu.edu.cn/pypi/web/simple +# pip install -U "funasr-onnx>=0.4.9" -i https://mirror.sjtu.edu.cn/pypi/web/simple # If you want to export .onnx file, you should install modelscope and funasr pip install -U modelscope funasr # For the users in China, you could install with the command: diff --git a/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/runtime/python/onnxruntime/funasr_onnx/punc_bin.py index 7fa62d44d0..bfcb6f80b6 100644 --- a/runtime/python/onnxruntime/funasr_onnx/punc_bin.py +++ b/runtime/python/onnxruntime/funasr_onnx/punc_bin.py @@ -99,6 +99,8 @@ def __call__(self, text: Union[list, str], split_size=20): split_text = self.code_mix_split_words_jieba(text) else: split_text = code_mix_split_words(text) + if not split_text: + return "", [] split_text_id = self.converter.tokens2ids(split_text) mini_sentences = split_to_mini_sentence(split_text, split_size) mini_sentences_id = split_to_mini_sentence(split_text_id, split_size) @@ -196,6 +198,9 @@ def __call__(self, text: str, param_dict: map, split_size=20): cache_key = "cache" assert cache_key in param_dict cache = param_dict[cache_key] + # An empty chunk adds no text and must not reprocess pending words. + if not text.strip(): + return "", [], cache if cache is not None else [] if cache is not None and len(cache) > 0: precache = "".join(cache) else: diff --git a/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py b/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py index f150ccbf8a..4218718b6c 100644 --- a/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py +++ b/runtime/python/onnxruntime/funasr_onnx/utils/frontend.py @@ -142,24 +142,26 @@ def load_cmvn(cmvn_file: Union[str, Path]) -> np.ndarray: raise FileNotFoundError("cmvn file not exits") with open(cmvn_file, "r", encoding="utf-8") as f: - lines = f.readlines() + lines = [line for line in f.readlines() if line.split()] means_list = [] vars_list = [] for i in range(len(lines)): line_item = lines[i].split() if line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": + line_item = lines[i + 1].split() if i + 1 < len(lines) else [] + if line_item and line_item[0] == "": add_shift_line = line_item[3 : (len(line_item) - 1)] means_list = list(add_shift_line) continue elif line_item[0] == "": - line_item = lines[i + 1].split() - if line_item[0] == "": + line_item = lines[i + 1].split() if i + 1 < len(lines) else [] + if line_item and line_item[0] == "": rescale_line = line_item[3 : (len(line_item) - 1)] vars_list = list(rescale_line) continue + if not means_list or not vars_list: + raise ValueError(f"No / statistics found in cmvn file: {cmvn_file}") means = np.array(means_list).astype(np.float64) vars = np.array(vars_list).astype(np.float64) cmvn = np.array([means, vars]) diff --git a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py index cb9ba3d0d2..5d9d8fd4e2 100644 --- a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py +++ b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py @@ -103,6 +103,8 @@ def __call__(self, audio_in: Union[str, np.ndarray, List[str]], **kwargs) -> Lis end_idx = beg_idx + 1 waveform = waveform_list[beg_idx:end_idx] feats, feats_len = self.extract_feat(waveform) + if feats_len[0] == 0: + continue waveform = np.array(waveform) param_dict = kwargs.get("param_dict", dict()) in_cache = param_dict.get("in_cache", list()) @@ -183,8 +185,13 @@ def load_wav(path: str) -> np.ndarray: def extract_feat(self, waveform_list: List[np.ndarray]) -> Tuple[np.ndarray, np.ndarray]: feats, feats_len = [], [] for waveform in waveform_list: - speech, _ = self.frontend.fbank(waveform) - feat, feat_len = self.frontend.lfr_cmvn(speech) + speech, speech_len = self.frontend.fbank(waveform) + if speech_len == 0: + # LFR needs a first frame; keep the empty input at its original position. + feat = np.empty((0, speech.shape[1] * self.frontend.lfr_m), dtype=speech.dtype) + feat_len = 0 + else: + feat, feat_len = self.frontend.lfr_cmvn(speech) feats.append(feat) feats_len.append(feat_len) diff --git a/runtime/python/onnxruntime/setup.py b/runtime/python/onnxruntime/setup.py index b61aa96c63..b5236ffc16 100644 --- a/runtime/python/onnxruntime/setup.py +++ b/runtime/python/onnxruntime/setup.py @@ -13,7 +13,7 @@ def get_readme(): MODULE_NAME = "funasr_onnx" -VERSION_NUM = "0.4.7" +VERSION_NUM = "0.4.9" setuptools.setup( name=MODULE_NAME, diff --git a/tests/test_char_tokenizer_non_linguistic_symbols.py b/tests/test_char_tokenizer_non_linguistic_symbols.py new file mode 100644 index 0000000000..7644ba1eae --- /dev/null +++ b/tests/test_char_tokenizer_non_linguistic_symbols.py @@ -0,0 +1,44 @@ +"""Tokenizers must ignore empty non-linguistic symbols instead of looping forever.""" + +import os +import tempfile +import unittest + +from funasr.tokenizer.char_tokenizer import CharTokenizer +from funasr.tokenizer.phoneme_tokenizer import PhonemeTokenizer + + +class CharTokenizerNonLinguisticSymbolsTest(unittest.TestCase): + def test_blank_line_in_symbol_file_is_ignored(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, "nlsyms.txt") + with open(path, "w", encoding="utf-8") as f: + f.write("\n\n") + for remove in (False, True): + with self.subTest(remove_non_linguistic_symbols=remove): + tokenizer = CharTokenizer( + non_linguistic_symbols=path, remove_non_linguistic_symbols=remove + ) + self.assertEqual(tokenizer.non_linguistic_symbols, {""}) + expected = ["你", "好"] if remove else ["你", "", "好"] + self.assertEqual(tokenizer.text2tokens("你好"), expected) + + def test_empty_symbol_in_iterable_is_ignored(self): + tokenizer = CharTokenizer(non_linguistic_symbols=["", ""]) + self.assertEqual(tokenizer.non_linguistic_symbols, {""}) + self.assertEqual(tokenizer.text2tokens("a b"), ["a", "", "b"]) + + +class PhonemeTokenizerNonLinguisticSymbolsTest(unittest.TestCase): + def test_empty_symbol_is_ignored(self): + tokenizer = PhonemeTokenizer( + g2p_type=None, + non_linguistic_symbols=["", ""], + remove_non_linguistic_symbols=True, + ) + self.assertEqual(tokenizer.non_linguistic_symbols, {""}) + self.assertEqual(tokenizer.text2tokens("ab"), ["ab"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cif_mixed_zero_fires.py b/tests/test_cif_mixed_zero_fires.py new file mode 100644 index 0000000000..6d012d5056 --- /dev/null +++ b/tests/test_cif_mixed_zero_fires.py @@ -0,0 +1,204 @@ +"""CPU CIF regressions using synthetic tensors; no model weights are needed.""" + +import importlib.util +import itertools +from pathlib import Path +import tempfile +import unittest + +import numpy as np +import torch + +from funasr.models.paraformer.cif_predictor import ( + CifPredictorV2, + CifPredictorV2Export, + cif_v1, + cif_v1_export, +) + +FUNCTIONS = (cif_v1, cif_v1_export) +# All nonzero totals stay away from integer and rounding boundaries. +# -1 supplies exact zeros; 0 supplies positive mass below the firing threshold. +ALPHAS = { + -1: [0.0] * 5, + 0: [0.11, 0.13, 0.09, 0.08, 0.12], + 1: [0.23, 0.34, 0.12, 0.33, 0.18], + 2: [0.62, 0.52, 0.43, 0.38, 0.35], + 3: [0.71, 0.67, 0.69, 0.61, 0.52], +} +PATTERNS = ( + (1, 0), + (0, 1), + (1, 0, 1), + (1, 0, 0), + (0, 0, 1), + (0, 1, 0), + (0, 0), + (-1, -1), + (1, 1), + (2, 0, 3, 0), + (0, 2, 0, 0, 3, 0, 0), + (1,), + (0,), + (3, 2, 1), +) + + +def _inputs(pattern): + alphas = torch.tensor([ALPHAS[n] for n in pattern], dtype=torch.float32) + hidden = torch.arange(alphas.numel() * 3, dtype=torch.float32) + hidden = hidden.reshape(len(pattern), 5, 3) / 17 - 2 + return hidden, alphas + + +def _sequential_frames(hidden, alphas): + """Integrate one sample at a time, splitting each firing frame at mass 1.""" + samples = [] + for sample, weights in zip(hidden, alphas): + mass = weights.new_zeros(()) + frame = torch.zeros_like(sample[0]) + emitted = [] + for value, weight in zip(sample, weights): + if (mass + weight).item() >= 1.0: + needed = 1.0 - mass + emitted.append(frame + needed * value) + mass = weight - needed + frame = mass * value + else: + mass = mass + weight + frame = frame + weight * value + samples.append(torch.stack(emitted) if emitted else sample[:0]) + return samples + + +def _expected(hidden, alphas, function): + samples = _sequential_frames(hidden, alphas) + # Preserve the existing round (eager) versus floor (export) padding contract. + lengths = alphas.sum(-1) + width = int((lengths.round() if function is cif_v1 else lengths.floor()).max()) + padded = [torch.nn.functional.pad(sample, (0, 0, 0, width - len(sample))) for sample in samples] + return torch.stack(padded), [len(sample) for sample in samples] + + +class _ExportCif(torch.nn.Module): + def forward(self, hidden, alphas): + return cif_v1_export(hidden, alphas, 1.0) + + +class TestCifMixedZeroFires(unittest.TestCase): + def test_minimal_trailing_empty(self): + hidden = torch.tensor([[[1.0, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]) + alphas = torch.tensor([[0.6, 0.6], [0.1, 0.1]]) + for function in FUNCTIONS: + with self.subTest(function=function.__name__): + frames, fires = function(hidden, alphas, 1.0) + torch.testing.assert_close(frames, torch.tensor([[[1.8, 2.8]], [[0.0, 0.0]]])) + self.assertEqual((fires >= 1).sum(1).tolist(), [1, 0]) + + def test_predictor_callers(self): + model = CifPredictorV2(idim=2, l_order=0, r_order=0, tail_threshold=0.0).eval() + hidden = torch.arange(12, dtype=torch.float32).reshape(3, 2, 2) + mask = torch.tensor([[[1.0, 1.0]], [[0.0, 0.0]], [[0.0, 0.0]]]) + with torch.no_grad(): + # Real predictor layers produce alpha=0.6 before the padding mask. + model.cif_output.weight.zero_() + model.cif_output.bias.fill_(torch.logit(torch.tensor(0.6))) + for predictor in (model, CifPredictorV2Export(model).eval()): + with self.subTest(predictor=type(predictor).__name__): + frames, _, _, fires = predictor(hidden, mask=mask) + expected = torch.tensor([[[0.8, 1.8]], [[0.0, 0.0]], [[0.0, 0.0]]]) + torch.testing.assert_close(frames, expected) + self.assertEqual((fires >= 1).sum(1).tolist(), [1, 0, 0]) + + def test_frames_counts_and_padding(self): + for pattern, function in itertools.product(PATTERNS, FUNCTIONS): + with self.subTest(pattern=pattern, function=function.__name__): + hidden, alphas = _inputs(pattern) + expected, counts = _expected(hidden, alphas, function) + self.assertEqual(counts, [max(n, 0) for n in pattern]) + frames, fires = function(hidden, alphas, 1.0) + torch.testing.assert_close(frames, expected) + self.assertEqual((fires >= 1).sum(1).tolist(), counts) + self.assertEqual(frames.dtype, hidden.dtype) + self.assertEqual(frames.device, hidden.device) + + def test_batch_permutations(self): + hidden, alphas = _inputs((0, 2, 0, 3)) + for function in FUNCTIONS: + expected, _ = _expected(hidden, alphas, function) + _, expected_fires = function(hidden, alphas, 1.0) + for permutation in itertools.permutations(range(4)): + with self.subTest(function=function.__name__, permutation=permutation): + order = list(permutation) + frames, fires = function(hidden[order], alphas[order], 1.0) + torch.testing.assert_close(frames, expected[order]) + torch.testing.assert_close(fires, expected_fires[order]) + + def test_gradients_away_from_boundaries(self): + for pattern, function in itertools.product(((2, 0, 3, 0), (3, 2, 1)), FUNCTIONS): + with self.subTest(pattern=pattern, function=function.__name__): + hidden, alphas = _inputs(pattern) + hidden.requires_grad_() + alphas.requires_grad_() + expected, _ = _expected(hidden, alphas, function) + frames, _ = function(hidden, alphas, 1.0) + weights = torch.linspace(-0.7, 1.3, frames.numel()).reshape_as(frames) + gradients = torch.autograd.grad((frames * weights).sum(), (hidden, alphas)) + reference = torch.autograd.grad((expected * weights).sum(), (hidden, alphas)) + for actual, wanted in zip(gradients, reference): + self.assertTrue(torch.isfinite(actual).all()) + torch.testing.assert_close(actual, wanted, atol=2e-5, rtol=2e-5) + + @unittest.skipUnless( + importlib.util.find_spec("onnx") and importlib.util.find_spec("onnxruntime"), + "ONNX validation requires onnx and onnxruntime", + ) + def test_onnx_dynamic_batches(self): + import onnx + import onnxruntime as ort + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "cif.onnx" + # Keep the scripted function's data-dependent empty-batch branch. + torch.onnx.export( + _ExportCif(), + _inputs((2, 1)), + str(path), + opset_version=14, + dynamo=False, + input_names=["hidden", "alphas"], + output_names=["frames", "fires"], + dynamic_axes={ + "hidden": {0: "batch", 1: "time"}, + "alphas": {0: "batch", 1: "time"}, + "frames": {0: "batch", 1: "tokens"}, + "fires": {0: "batch", 1: "time"}, + }, + ) + onnx.checker.check_model(str(path)) + options = ort.SessionOptions() + options.intra_op_num_threads = 1 + options.inter_op_num_threads = 1 + session = ort.InferenceSession( + str(path), sess_options=options, providers=["CPUExecutionProvider"] + ) + for pattern in PATTERNS: + with self.subTest(pattern=pattern): + hidden, alphas = _inputs(pattern) + # Also exercise a time dimension different from the export example. + hidden = torch.nn.functional.pad(hidden, (0, 0, 0, 2)) + alphas = torch.nn.functional.pad(alphas, (0, 2)) + expected, counts = _expected(hidden, alphas, cif_v1_export) + frames, fires = session.run( + None, {"hidden": hidden.numpy(), "alphas": alphas.numpy()} + ) + self.assertEqual(frames.shape, tuple(expected.shape)) + self.assertEqual(fires.shape, tuple(alphas.shape)) + np.testing.assert_allclose(frames, expected.numpy(), atol=2e-5, rtol=2e-5) + self.assertEqual((fires >= 1).sum(1).tolist(), counts) + _, torch_fires = cif_v1_export(hidden, alphas, 1.0) + np.testing.assert_allclose(fires, torch_fires.numpy(), atol=2e-6, rtol=2e-6) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fsmn_vad_dynamic_silence.py b/tests/test_fsmn_vad_dynamic_silence.py index f396614f98..d8f39c345a 100644 --- a/tests/test_fsmn_vad_dynamic_silence.py +++ b/tests/test_fsmn_vad_dynamic_silence.py @@ -55,13 +55,83 @@ def test_explicit_max_end_silence_time_keeps_fixed_threshold_by_default(self): cache = self._run_inference(max_end_silence_time=300) self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 200) - self.assertNotIn("_dynamic_accumulated_ms", cache) + self.assertIsNone(cache["stats"].silence_schedule) + self.assertEqual(cache["stats"].speech_noise_thres, 0.6) def test_explicit_dynamic_silence_still_enables_schedule(self): cache = self._run_inference(max_end_silence_time=300, dynamic_silence=True) - self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 1900) - self.assertEqual(cache["_dynamic_accumulated_ms"], 1000) + self.assertEqual(cache["stats"].silence_schedule, vad_model.DEFAULT_SILENCE_SCHEDULE) + self.assertEqual(cache["stats"].speech_noise_thres, 0.5) + # The threshold is no longer advanced per chunk; DetectOneFrame applies the schedule. + self.assertEqual(cache["stats"].max_end_sil_frame_cnt_thresh, 200) + self.assertNotIn("_dynamic_accumulated_ms", cache) + + +class TestFsmnVadDynamicSilencePerFrame(unittest.TestCase): + """Drive DetectOneFrame with scripted window-detector transitions.""" + + def _end_frames(self, speech_ms, silence_ms, schedule, start_frame=0, fixed_thresh_ms=650): + vad = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming) + torch.nn.Module.__init__(vad) + vad.vad_opts = vad_model.VADXOptions() + ends = [] + + def on_voice_end(end_frame, fake_result, is_last_frame, cache=None): + ends.append(end_frame) + + def reset_detection(cache=None): + cache["stats"].confirmed_start_frame = -1 + cache["stats"].vad_state_machine = vad_model.VadStateMachine.kVadInStateStartPointNotDetected + + vad.OnVoiceEnd = on_voice_end + vad.PopDataToOutputBuf = lambda *args, **kwargs: None + vad.PopDataBufTillFrame = lambda *args, **kwargs: None + vad.LatencyFrmNumAtStartPoint = lambda cache=None: 0 + vad.ResetDetection = reset_detection + + stats = vad_model.Stats( + sil_pdf_ids=[0], max_end_sil_frame_cnt_thresh=fixed_thresh_ms, speech_noise_thres=0.5 + ) + stats.silence_schedule = schedule + speech_frames, silence_frames = speech_ms // 10, silence_ms // 10 + changes = ( + [vad_model.AudioChangeState.kChangeStateSil2Speech] + + [vad_model.AudioChangeState.kChangeStateSpeech2Speech] * (speech_frames - 1) + + [vad_model.AudioChangeState.kChangeStateSpeech2Sil] + + [vad_model.AudioChangeState.kChangeStateSil2Sil] * silence_frames + ) + script = iter(changes) + cache = { + "stats": stats, + "windows_detector": SimpleNamespace(DetectOneFrame=lambda *a, **k: next(script)), + } + for i in range(len(changes)): + vad.DetectOneFrame(vad_model.FrameState.kFrameStateSpeech, start_frame + i, False, cache=cache) + return [e - start_frame for e in ends] + + def test_schedule_follows_current_segment_duration(self): + schedule = vad_model.DEFAULT_SILENCE_SCHEDULE + # Thresholds below are the schedule values minus speech_to_sil_time_thres (150 ms). + # 5 s into a segment: 2000 ms -> 1850 ms, so 900 ms of silence does not end it. + self.assertEqual(self._end_frames(5000, 900, schedule), []) + # 15 s into a segment: 1000 ms -> 850 ms, so 900 ms ends it and 700 ms does not. + self.assertEqual(len(self._end_frames(15000, 900, schedule)), 1) + self.assertEqual(self._end_frames(15000, 700, schedule), []) + # 55 s into a segment: 200 ms -> 50 ms, so 300 ms ends it. + self.assertEqual(len(self._end_frames(55000, 300, schedule)), 1) + + def test_schedule_does_not_depend_on_absolute_position(self): + schedule = vad_model.DEFAULT_SILENCE_SCHEDULE + for speech_ms, silence_ms in [(5000, 900), (15000, 900), (55000, 300)]: + at_zero = self._end_frames(speech_ms, silence_ms, schedule) + shifted = self._end_frames(speech_ms, silence_ms, schedule, start_frame=6000) + self.assertEqual(at_zero, shifted) + + def test_without_schedule_threshold_stays_fixed(self): + # 650 ms fixed threshold: 700 ms of silence ends a 5 s segment, 600 ms does not. + self.assertEqual(len(self._end_frames(5000, 700, None)), 1) + self.assertEqual(self._end_frames(5000, 600, None), []) if __name__ == "__main__": diff --git a/tests/test_funasr_onnx_installed.py b/tests/test_funasr_onnx_installed.py index babcf53a59..4b3c254ee9 100644 --- a/tests/test_funasr_onnx_installed.py +++ b/tests/test_funasr_onnx_installed.py @@ -38,7 +38,7 @@ class InstalledOnnxPackageTest(unittest.TestCase): def test_installed_distribution_version_and_origin(self): distribution = importlib.metadata.distribution('funasr-onnx') - self.assertEqual(distribution.version, '0.4.7') + self.assertEqual(distribution.version, '0.4.9') self.assertEqual( Path(funasr_onnx.__file__).resolve().parent, Path(distribution.locate_file('funasr_onnx')).resolve(), diff --git a/tests/test_funasr_onnx_punctuation_empty_input.py b/tests/test_funasr_onnx_punctuation_empty_input.py new file mode 100644 index 0000000000..901a228bc3 --- /dev/null +++ b/tests/test_funasr_onnx_punctuation_empty_input.py @@ -0,0 +1,116 @@ +"""Run with PYTHONPATH=runtime/python/onnxruntime python -m unittest discover +-s tests -p test_funasr_onnx_punctuation_empty_input.py (standalone ONNX dependencies). +""" + +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock + +import numpy as np + +from funasr_onnx.punc_bin import CT_Transformer, CT_Transformer_VadRealtime +from funasr_onnx.utils.utils import TokenIDConverter, code_mix_split_words_jieba + + +EMPTY_TEXTS = ("", " ", "\t\n", " \t\r\n\u3000") +PUNC_LIST = ["", "_", "，", "。", "？", "、"] + + +def make_model(model_class, segmenter=None): + # Exercise real tokenization, chunking and cache handling without model downloads. + model = model_class.__new__(model_class) + model.seg_jieba = segmenter is not None + model.code_mix_split_words_jieba = segmenter + model.converter = TokenIDConverter(["", "你", "好", "世", "界", "hello", "world"]) + model.punc_list = PUNC_LIST + model.period = PUNC_LIST.index("。") + + def infer(tokens, lengths, *masks): + logits = np.zeros((*tokens.shape, len(PUNC_LIST)), dtype=np.float32) + logits[:, :, PUNC_LIST.index("_")] = 1 + return [logits] + + model.infer = Mock(side_effect=infer) + return model + + +class OnnxPunctuationEmptyInputTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + with tempfile.TemporaryDirectory() as directory: + dictionary = Path(directory) / "jieba_usr_dict" + dictionary.write_text("你好 100\n世界 100\n", encoding="utf-8") + cls.segmenter = staticmethod(code_mix_split_words_jieba(str(dictionary))) + + def test_offline_empty_skips_inference(self): + for segmenter in (None, self.segmenter): + model = make_model(CT_Transformer, segmenter) + for text in EMPTY_TEXTS: + with self.subTest(jieba=segmenter is not None, text=text): + self.assertEqual(model(text), ("", [])) + model.infer.assert_not_called() + + def test_offline_nonempty_around_empty(self): + for segmenter in (None, self.segmenter): + for text, expected in (("你好", "你好。"), ("hello world", "hello world。")): + with self.subTest(jieba=segmenter is not None, text=text): + model = make_model(CT_Transformer, segmenter) + before = model(text) + self.assertEqual(before[0], expected) + self.assertEqual(before[1][-1], model.period) + for empty in EMPTY_TEXTS: + self.assertEqual(model(empty), ("", [])) + self.assertEqual(model(text), before) + self.assertEqual(model.infer.call_count, 2) + + def test_streaming_empty_preserves_cache(self): + for initial_cache in (None, [], ["你", "好"], ["hello", " world"]): + with self.subTest(cache=initial_cache): + model = make_model(CT_Transformer_VadRealtime) + params = {"cache": initial_cache} + expected = None if initial_cache is None else initial_cache.copy() + for text in EMPTY_TEXTS * 2: + result = model(text, params) + self.assertEqual(result, ("", [], expected or [])) + self.assertIs(params["cache"], initial_cache) + self.assertEqual(params["cache"], expected) + if initial_cache is not None: + self.assertIs(result[2], initial_cache) + model.infer.assert_not_called() + + def test_streaming_continuity_across_empty_chunks(self): + for chunks in (("你好", "世界"), ("hello", "world"), ("你好 hello", "world 世界")): + for split_size in (2, 20): + with self.subTest(chunks=chunks, split_size=split_size): + model = make_model(CT_Transformer_VadRealtime) + reference = make_model(CT_Transformer_VadRealtime) + params, reference_params = {"cache": []}, {"cache": []} + for chunk in chunks: + for empty in EMPTY_TEXTS: + cache = params["cache"] + calls = model.infer.call_count + self.assertEqual(model(empty, params), ("", [], cache)) + self.assertIs(params["cache"], cache) + self.assertEqual(model.infer.call_count, calls) + self.assertEqual( + model(chunk, params, split_size=split_size), + reference(chunk, reference_params, split_size=split_size), + ) + self.assertEqual(params, reference_params) + self.assertEqual(model.infer.call_count, reference.infer.call_count) + for actual, expected in zip( + model.infer.call_args_list, reference.infer.call_args_list + ): + for actual_arg, expected_arg in zip(actual.args, expected.args): + np.testing.assert_array_equal(actual_arg, expected_arg) + + def test_streaming_requires_cache_key_even_for_empty_text(self): + model = make_model(CT_Transformer_VadRealtime) + with self.assertRaises(AssertionError): + model("", {}) + model.infer.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_funasr_onnx_release.py b/tests/test_funasr_onnx_release.py index 34a9fdea42..160c24aa4c 100644 --- a/tests/test_funasr_onnx_release.py +++ b/tests/test_funasr_onnx_release.py @@ -7,7 +7,7 @@ ROOT = Path(__file__).resolve().parents[1] PACKAGE_ROOT = ROOT / "runtime" / "python" / "onnxruntime" SETUP_PATH = PACKAGE_ROOT / "setup.py" -EXPECTED_VERSION = "0.4.7" +EXPECTED_VERSION = "0.4.9" REQUIREMENT_NAME_PATTERN = re.compile(r"^\s*([A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?)") @@ -44,7 +44,7 @@ def normalized_requirement_name(requirement): class FunASROnnxReleaseContractTest(unittest.TestCase): - def test_release_version_is_0_4_7(self): + def test_release_version_is_0_4_9(self): self.assertEqual(assigned_literal(read_setup_tree(), "VERSION_NUM"), EXPECTED_VERSION) def test_standalone_license_preserves_repository_notice(self): @@ -58,6 +58,12 @@ def test_workflow_checks_and_artifact_use_release_version(self): self.assertIn(f'assert version("funasr-onnx") == "{EXPECTED_VERSION}"', workflow) self.assertIn(f"name: funasr-onnx-{EXPECTED_VERSION}", workflow) + def test_workflow_collects_empty_punctuation_regression(self): + workflow = (ROOT / ".github/workflows/test-funasr-onnx-package.yml").read_text() + test_path = "tests/test_funasr_onnx_punctuation_empty_input.py" + self.assertEqual(workflow.count(test_path), 3) + self.assertIn("python " + test_path, workflow) + def test_readme_names_the_standalone_fix_release(self): readme = (PACKAGE_ROOT / "README.md").read_text() self.assertIn(f'pip install -U "funasr-onnx>={EXPECTED_VERSION}"', readme) @@ -69,6 +75,12 @@ def test_readme_names_the_standalone_fix_release(self): self.assertIn("#3831", readme) self.assertIn("#3838", readme) self.assertIn("hotword", readme) + self.assertIn("#3842", readme) + self.assertIn("CMVN", readme) + self.assertIn("#3846", readme) + self.assertIn("#3847", readme) + self.assertIn("empty or sub-frame", readme) + self.assertIn("preserves the streaming cache", readme) def test_runtime_dependencies_keep_onnx_install_torch_free(self): requirements = setup_keyword_literal(read_setup_tree(), "install_requires") diff --git a/tests/test_funasr_onnx_vad_multiple_inputs.py b/tests/test_funasr_onnx_vad_multiple_inputs.py index fd24781147..a5cbaff48f 100644 --- a/tests/test_funasr_onnx_vad_multiple_inputs.py +++ b/tests/test_funasr_onnx_vad_multiple_inputs.py @@ -4,6 +4,7 @@ import os import tempfile import unittest +from unittest.mock import Mock import numpy as np @@ -44,10 +45,10 @@ def fake_infer(inputs): return scores, inputs[1:] -def make_model(batch_size=1): +def make_model(batch_size=1, **frontend_conf): # Keep __call__, WavFrontend and E2EVadModel real; no model files. model = Fsmn_vad.__new__(Fsmn_vad) - model.frontend = WavFrontend(cmvn_file=None, **FRONTEND_CONF) + model.frontend = WavFrontend(cmvn_file=None, **{**FRONTEND_CONF, **frontend_conf}) model.batch_size = batch_size # E2EVadModel's defaults equal the template's model_conf. model.vad_scorer_config = {} @@ -99,6 +100,80 @@ def test_batch_size_above_one(self): result = model([self.wav[name] for name in names]) self.assertEqual(result, [SEGMENTS[name] for name in names]) + def test_short_inputs_and_frame_boundary(self): + for fs, frame_length, lfr_m in ( + (16000, 25, 5), + (8000, 25, 5), + (16000, 20, 5), + (16000, 25, 1), + ): + window = int(fs * frame_length / 1000) + for batch_size in (1, 2, 3): + model = make_model(batch_size, fs=fs, frame_length=frame_length, lfr_m=lfr_m) + model.infer = Mock(side_effect=fake_infer) + for samples in (0, 1, window // 2, window - 1, window, window + 1, 2 * window): + audio = np.zeros(samples, dtype=np.float32) + path = os.path.join(os.path.dirname(self.wav["a"]), "short.wav") + soundfile.write(path, audio, fs) + for audio_in in (audio, path): + with self.subTest( + fs=fs, + frame_length=frame_length, + lfr_m=lfr_m, + batch_size=batch_size, + samples=samples, + input_type=type(audio_in).__name__, + ): + model.infer.reset_mock() + self.assertEqual(model(audio_in), [[]]) + self.assertEqual(model.infer.call_count, int(samples >= window)) + + def test_short_files_keep_their_input_positions(self): + for samples in (0, 160, 320, 399): + path = os.path.join(os.path.dirname(self.wav["a"]), "short.wav") + soundfile.write(path, np.zeros(samples, dtype=np.float32), 16000) + paths = {**self.wav, "short": path} + expected = {**SEGMENTS, "short": []} + for batch_size in (1, 2, 3): + model = make_model(batch_size) + model.infer = Mock(side_effect=fake_infer) + for names in ( + ["short", "a"], + ["a", "short"], + ["a", "short", "b"], + ["short", "short"], + ["short", "b", "short"], + ): + with self.subTest(samples=samples, batch_size=batch_size, names=names): + model.infer.reset_mock() + result = model([paths[name] for name in names]) + self.assertEqual(result, [expected[name] for name in names]) + self.assertEqual( + model.infer.call_count, sum(name != "short" for name in names) + ) + self.assertEqual(len({id(segments) for segments in result}), len(result)) + + def test_extract_feat_with_empty_frames(self): + for lfr_m, lfr_n in ((1, 1), (5, 1), (3, 2)): + model = make_model(lfr_m=lfr_m, lfr_n=lfr_n) + empty = np.zeros(0, dtype=np.float32) + short = np.zeros(399, dtype=np.float32) + normal = np.zeros(1600, dtype=np.float32) + expected, expected_len = model.extract_feat([normal]) + for waveforms, lengths in ( + ([empty], [0]), + ([short], [0]), + ([empty, short, normal], [0, 0, expected_len[0]]), + ): + with self.subTest(lfr_m=lfr_m, lfr_n=lfr_n, lengths=lengths): + feats, feats_len = model.extract_feat(waveforms) + np.testing.assert_array_equal(feats_len, lengths) + self.assertEqual(feats.shape, (len(waveforms), max(lengths), 80 * lfr_m)) + self.assertEqual(feats.dtype, np.float32) + if len(waveforms) > 1: + np.testing.assert_array_equal(feats[-1], expected[0]) + self.assertFalse(feats[:-1].any()) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_funasr_torch_package.py b/tests/test_funasr_torch_package.py index ce39c86137..04cdcf01f5 100644 --- a/tests/test_funasr_torch_package.py +++ b/tests/test_funasr_torch_package.py @@ -119,12 +119,12 @@ def test_torch_package_does_not_require_unrelated_runtime_engines(distributions) } assert not names & {"torch-quant", "onnxruntime", "onnxruntime-gpu"} -def test_distribution_version_excludes_empty_published_release(distributions): +def test_distribution_version_includes_cmvn_parser_fix(distributions): from packaging.version import Version wheel, sdist = distributions version = metadata(wheel)["Version"] - assert Version(version) >= Version("0.1.4") + assert Version(version) >= Version("0.1.6") with tarfile.open(sdist, "r:gz") as archive: member = next(item for item in archive.getmembers() if item.name.count("/") == 1 and item.name.endswith("/PKG-INFO")) diff --git a/tests/test_merge_vad_length.py b/tests/test_merge_vad_length.py new file mode 100644 index 0000000000..f14fc952dc --- /dev/null +++ b/tests/test_merge_vad_length.py @@ -0,0 +1,81 @@ +"""Regression tests for the merge_length_s option of AutoModel.generate().""" + +import unittest +from unittest.mock import MagicMock, patch + +import numpy as np + + +VAD_SEGMENTS = [[0, 4000], [5000, 9000], [10000, 14000]] + + +class TestMergeVadLength(unittest.TestCase): + def _make_auto_model(self, **init_kwargs): + from funasr.auto.auto_model import AutoModel + + am = AutoModel.__new__(AutoModel) + am.model = MagicMock() + am.vad_model = MagicMock() + am.punc_model = None + am.punc_kwargs = {} + am.spk_model = None + am.spk_kwargs = {} + am.cb_model = None + am.spk_mode = None + am.vad_kwargs = {} + am.kwargs = { + "batch_size_s": 300, + "batch_size_threshold_s": 60, + "device": "cpu", + "disable_pbar": True, + "frontend": MagicMock(fs=16000), + "fs": 16000, + **init_kwargs, + } + am._reset_runtime_configs = MagicMock() + + def mock_inference(data, input_len=None, model=None, kwargs=None, **cfg): + if model is am.vad_model: + return [{"key": "utt", "value": [list(s) for s in VAD_SEGMENTS]}] + return [{"text": "x"} for _ in data] + + am.inference = MagicMock(side_effect=mock_inference) + return am + + @patch("funasr.auto.auto_model.slice_padding_audio_samples") + @patch("funasr.auto.auto_model.load_audio_text_image_video") + @patch("funasr.auto.auto_model.prepare_data_iterator") + def _asr_segments(self, am, mock_prep, mock_load, mock_slice, **cfg): + mock_prep.return_value = (["utt"], [np.zeros(16000 * 14, dtype=np.float32)]) + mock_load.return_value = np.zeros(16000 * 14, dtype=np.float32) + mock_slice.side_effect = lambda speech, speech_lengths, segments: ( + [np.zeros(16000, dtype=np.float32) for _ in segments], + [16000 for _ in segments], + ) + + am.inference_with_vad("dummy_input", merge_vad=True, **cfg) + + segments = [] + for call in mock_slice.call_args_list: + segments.extend(seg for seg, _ in call.args[2]) + return sorted(segments) + + def test_default_merge_length_is_15_seconds(self): + am = self._make_auto_model() + self.assertEqual(self._asr_segments(am), [[0, 14000]]) + + def test_generate_merge_length_s_is_applied(self): + am = self._make_auto_model() + self.assertEqual(self._asr_segments(am, merge_length_s=10), [[0, 9000], [9000, 14000]]) + + def test_construction_merge_length_s_is_applied(self): + am = self._make_auto_model(merge_length_s=10) + self.assertEqual(self._asr_segments(am), [[0, 9000], [9000, 14000]]) + + def test_generate_merge_length_s_overrides_construction_value(self): + am = self._make_auto_model(merge_length_s=10) + self.assertEqual(self._asr_segments(am, merge_length_s=15), [[0, 14000]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_version_contract.py b/tests/test_release_version_contract.py index 57623d6903..621d3ed838 100644 --- a/tests/test_release_version_contract.py +++ b/tests/test_release_version_contract.py @@ -4,16 +4,18 @@ ROOT = Path(__file__).resolve().parents[1] -RELEASE_VERSION = "1.4.21" +RELEASE_VERSION = "1.4.23" README_FILES = ("README.md", "README_zh.md", "README_ja.md", "README_ko.md") REPOSITORY_ROLE_FILES = ("docs/repository_roles.md", "docs/repository_roles_zh.md") @pytest.mark.parametrize("name", README_FILES) -def test_release_summary_preserves_streaming_punctuation_cache(name): +def test_release_summary_covers_no_lfr_and_tokenizer_vocabulary(name): summaries = [line for line in (ROOT / name).read_text(encoding="utf-8").splitlines() if f"**FunASR {RELEASE_VERSION}**" in line] assert len(summaries) == 1 - assert "cache" in summaries[0].lower() - assert "https://github.com/modelscope/FunASR/pull/3824" in summaries[0] + assert "lfr_m=lfr_n=1" in summaries[0] + assert "token_list" in summaries[0] + assert "https://github.com/modelscope/FunASR/pull/3849" in summaries[0] + assert "https://github.com/modelscope/FunASR/pull/3783" in summaries[0] diff --git a/tests/test_runtime_cmvn.py b/tests/test_runtime_cmvn.py new file mode 100644 index 0000000000..97889e2fc7 --- /dev/null +++ b/tests/test_runtime_cmvn.py @@ -0,0 +1,105 @@ +"""Exercise the standalone runtime frontends without model weights or engines.""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +CMVN = " 2 2\n 1 [ 1 -2 ]\n 2 2\n 1 [ 0.5 2 ]\n" + + +@pytest.fixture(params=["onnxruntime/funasr_onnx", "libtorch/funasr_torch"]) +def frontend_class(request): + # Loading the frontend directly keeps these tests independent of the + # optional inference engines and exercises each package's real constructor. + path = ROOT / "runtime/python" / request.param / "utils/frontend.py" + spec = importlib.util.spec_from_file_location("runtime_cmvn_frontend", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.WavFrontend + + +def test_valid_cmvn_preserves_statistics_and_normalized_features(frontend_class, tmp_path): + path = tmp_path / "valid.mvn" + path.write_text(CMVN, encoding="utf-8") + frontend = frontend_class(cmvn_file=str(path)) + + assert frontend.cmvn.dtype == np.float64 + np.testing.assert_array_equal(frontend.cmvn, [[1, -2], [0.5, 2]]) + features = np.array([[0, 2], [-2, 4]], dtype=np.float32) + np.testing.assert_array_equal(frontend.apply_cmvn(features), [[0.5, 0], [-0.5, 4]]) + + +@pytest.mark.parametrize("placement", ["leading", "trailing", "between_sections", "after_tags"]) +def test_cmvn_accepts_blank_lines(frontend_class, tmp_path, placement): + blank = " \t\n\n" + if placement == "leading": + content = blank + CMVN + elif placement == "trailing": + content = CMVN + blank + elif placement == "between_sections": + content = CMVN.replace("", blank + "") + else: + content = CMVN.replace("", blank + "") + path = tmp_path / "blank lines.mvn" + path.write_text(content, encoding="utf-8") + + frontend = frontend_class(cmvn_file=str(path)) + + np.testing.assert_array_equal(frontend.cmvn, [[1, -2], [0.5, 2]]) + + +@pytest.mark.parametrize( + "content", + [ + "", + " \t\n", + "\n\n", + " 2 2\n", + " 2 2\n", + " 2 2\n 1 [ 1 -2 ]\n", + " 2 2\n 1 [ 0.5 2 ]\n", + CMVN.replace("\n", " "), + ], + ids=[ + "empty", + "blank", + "no_stats", + "truncated_shift", + "truncated_scale", + "shift_only", + "scale_only", + "compact", + ], +) +def test_incomplete_cmvn_fails_during_construction(frontend_class, tmp_path, content): + path = tmp_path / "incomplete.mvn" + path.write_text(content, encoding="utf-8") + + with pytest.raises(ValueError) as error: + frontend_class(cmvn_file=str(path)) + + assert str(path) in str(error.value) + + +def test_shipped_cmvn_is_unchanged_by_blank_lines(frontend_class, tmp_path): + shipped = ROOT / "runtime/triton_gpu/model_repo_sense_voice_small/feature_extractor/am.mvn" + original = frontend_class(cmvn_file=str(shipped), lfr_m=7, lfr_n=6, dither=0.0) + path = tmp_path / "shipped-with-blanks.mvn" + path.write_text("\n" + "\n \t\n".join(shipped.read_text().splitlines()) + "\n\n") + edited = frontend_class(cmvn_file=str(path), lfr_m=7, lfr_n=6, dither=0.0) + + assert original.cmvn.shape == (2, 560) + assert np.isfinite(original.cmvn).all() + np.testing.assert_array_equal(edited.cmvn, original.cmvn) + # Real filterbank + LFR + CMVN on a deterministic waveform, not a model stub. + waveform = np.sin(2 * np.pi * 440 * np.arange(1600) / 16000).astype(np.float32) * 0.1 + original_features, _ = original.fbank(waveform) + edited_features, _ = edited.fbank(waveform) + original_features, original_length = original.lfr_cmvn(original_features) + edited_features, edited_length = edited.lfr_cmvn(edited_features) + assert edited_length == original_length + np.testing.assert_array_equal(edited_features, original_features) diff --git a/tests/test_sentencepiece_tokenizer.py b/tests/test_sentencepiece_tokenizer.py new file mode 100644 index 0000000000..f3a4349eac --- /dev/null +++ b/tests/test_sentencepiece_tokenizer.py @@ -0,0 +1,162 @@ +"""Model vocabulary IDs must reach callers as pieces, not BPE-decoded text.""" + +import json +from types import SimpleNamespace + +import numpy as np +import pytest +import sentencepiece as spm +import torch + +from funasr.models.paraformer.model import Paraformer +from funasr.tokenizer.build_tokenizer import build_tokenizer +from funasr.tokenizer.sentencepiece_tokenizer import SentencepiecesTokenizer + + +@pytest.fixture(scope="module") +def bpe_model(tmp_path_factory): + prefix = tmp_path_factory.mktemp("sentencepiece") / "tiny" + spm.SentencePieceTrainer.train( + sentence_iterator=iter( + ["hello world", "hello there", "he tried to think how it could be", "你好 world"] * 20 + ), + model_prefix=str(prefix), + model_type="bpe", + vocab_size=48, + user_defined_symbols=["<|en|>"], + shuffle_input_sentence=False, + num_threads=1, + minloglevel=2, + ) + return str(prefix.with_suffix(".model")) + + +def vocabulary(bpe_model, reordered=True): + sp = spm.SentencePieceProcessor(model_file=bpe_model) + pieces = [sp.id_to_piece(i) for i in range(sp.get_piece_size())] + return list(reversed(pieces)) if reordered else pieces + + +@pytest.mark.parametrize("storage", ["list", "tuple", "json", "txt"]) +def test_reordered_vocabulary_decodes_expected_text(bpe_model, tmp_path, storage): + tokens = vocabulary(bpe_model) + token_list = tokens + if storage == "tuple": + token_list = tuple(tokens) + elif storage in ("json", "txt"): + path = tmp_path / ("tokens." + storage) + content = json.dumps(tokens) if storage == "json" else "\n".join(tokens) + "\n" + path.write_text(content, encoding="utf-8") + token_list = str(path) + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=token_list) + pieces = tokenizer.text2tokens("hello world") + model_ids = [tokens.index(piece) for piece in pieces] + + # Establish that the two vocabularies really assign different meanings to IDs. + assert tokenizer.decode(model_ids) != "hello world" + result = tokenizer.ids2tokens(model_ids) + + assert result == pieces + assert tokenizer.tokens2text(result) == "hello world" + + +@pytest.mark.parametrize("text", ["hello world", "你好 world", "<|en|> hello world", ""]) +def test_matching_vocabulary_preserves_text_and_piece_boundaries(bpe_model, text): + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=vocabulary(bpe_model, False)) + ids = tokenizer.encode(text) + + pieces = tokenizer.ids2tokens(ids) + + assert pieces == tokenizer.text2tokens(text) + assert tokenizer.tokens2text(pieces) == tokenizer.decode(ids) == text + + +@pytest.mark.parametrize("container", [list, tuple, iter, np.asarray]) +def test_model_ids_accept_base_tokenizer_containers(bpe_model, container): + tokens = vocabulary(bpe_model) + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=tokens) + ids = [tokens.index(piece) for piece in tokenizer.text2tokens("hello world")] + + assert tokenizer.tokens2text(tokenizer.ids2tokens(container(ids))) == "hello world" + + +def test_special_tokens_and_model_only_ids_are_preserved(bpe_model): + tokens = vocabulary(bpe_model) + [""] + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=tokens) + pieces = ["", "", "<|en|>", "▁", "▁", "", ""] + + assert tokenizer.ids2tokens([tokens.index(piece) for piece in pieces]) == pieces + assert tokenizer.ids2tokens([]) == [] + + +def test_model_ids_require_one_dimensional_numpy_array(bpe_model): + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=vocabulary(bpe_model)) + + with pytest.raises(ValueError, match="Must be 1 dim ndarray"): + tokenizer.ids2tokens(np.array([[1, 2]])) + + +@pytest.mark.parametrize("construction", ["omitted", "none", "build_tokenizer"]) +def test_without_vocabulary_preserves_legacy_text_decode(bpe_model, construction): + if construction == "build_tokenizer": + tokenizer = build_tokenizer("bpe", bpemodel=bpe_model) + elif construction == "none": + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=None) + else: + tokenizer = SentencepiecesTokenizer(bpe_model) + ids = tokenizer.encode("hello world") + + assert tokenizer.ids2tokens(ids) == "hello world" + assert tokenizer.ids2tokens(line=ids) == "hello world" + assert tokenizer.ids2tokens([ids, ids]) == ["hello world", "hello world"] + assert tokenizer.ids2tokens([]) == "" + + +def test_encode_decode_and_legacy_keyword_remain_sentencepiece_operations(bpe_model): + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=vocabulary(bpe_model)) + ids = tokenizer.encode("hello world") + model_ids = [tokenizer.token_list.index(p) for p in tokenizer.text2tokens("hello world")] + + assert tokenizer.tokens2ids("hello world") == ids + assert tokenizer.decode(ids) == "hello world" + assert tokenizer.tokens2text(tokenizer.ids2tokens(line=model_ids, unused=True)) == "hello world" + + +def test_paraformer_inference_uses_model_pieces_and_filters_control_ids(bpe_model, tmp_path): + tokens = ["", "", ""] + [ + p for p in vocabulary(bpe_model) if p not in ("", "") + ] + tokenizer = SentencepiecesTokenizer(bpe_model, token_list=tokens) + pieces = tokenizer.text2tokens("hello world") + ids = [0, 1] + [tokens.index(p) for p in pieces] + [2, 0] + scores = torch.full((1, len(ids), len(tokens)), -10.0) + scores[0, torch.arange(len(ids)), ids] = 10.0 + lengths = torch.tensor([len(ids)]) + speech = torch.zeros(1, len(ids), 2) + # Only the acoustic stages are controlled; the actual inference method, + # tokenizer, special-ID filtering, text conversion and writer all run. + state = SimpleNamespace( + beam_search=None, + sos=1, + eos=2, + blank_id=0, + encode=lambda *args: (speech, lengths), + calc_predictor=lambda *args: (speech, lengths.float(), None, None), + cal_decoder_with_predictor=lambda *args: (scores, lengths), + ) + + results, _ = Paraformer.inference( + state, + speech, + data_lengths=lengths, + key=["sample"], + tokenizer=tokenizer, + data_type="fbank", + device="cpu", + output_dir=str(tmp_path), + ) + + assert results == [{"key": "sample", "text": "hello world"}] + assert (tmp_path / "1best_recog" / "token").read_text().strip() == ( + "sample " + " ".join(pieces) + ) diff --git a/tests/test_wav_frontend_no_lfr.py b/tests/test_wav_frontend_no_lfr.py new file mode 100644 index 0000000000..a9af796626 --- /dev/null +++ b/tests/test_wav_frontend_no_lfr.py @@ -0,0 +1,133 @@ +import pytest +import torch + +from funasr.frontends.wav_frontend import WavFrontendOnline +from funasr.models.fsmn_vad_streaming import model as vad_model +from funasr.utils import fbank as kaldi + + +def make_frontend(tmp_path, cmvn): + cmvn_file = None + if cmvn: + cmvn_file = tmp_path / "no_lfr.mvn" + cmvn_file.write_text( + " 80 80\n 1 [ " + + " ".join(["0.25"] * 80) + + " ]\n 80 80\n 1 [ " + + " ".join(["2"] * 80) + + " ]\n" + ) + return WavFrontendOnline(cmvn_file=cmvn_file, dither=0.0) + + +@pytest.mark.parametrize("cmvn", (False, True)) +@pytest.mark.parametrize("is_final", (False, True)) +@pytest.mark.parametrize("lengths", ((5, 3, 0), (0, 0))) +def test_no_lfr_reports_valid_consumed_frames(tmp_path, cmvn, is_final, lengths): + frontend = make_frontend(tmp_path, cmvn) + inputs = torch.arange(len(lengths) * 7 * 80, dtype=torch.float32).reshape( + len(lengths), 7, 80 + ) + original = inputs.clone() + cache = {} + + feats, actual_lengths, consumed = frontend.forward_lfr_cmvn( + inputs, torch.tensor(lengths), is_final=is_final, cache=cache + ) + + assert actual_lengths.tolist() == list(lengths) + assert consumed.tolist() == list(lengths) + assert feats.shape == (len(lengths), max(lengths), 80) + assert feats.dtype == torch.float32 + assert cache == {} + for index, length in enumerate(lengths): + expected = original[index, :length] + if cmvn: + expected = (expected + 0.25) * 2 + torch.testing.assert_close(feats[index, :length], expected, rtol=0, atol=0) + assert torch.count_nonzero(feats[index, length:]) == 0 + + +@pytest.mark.parametrize("cmvn", (False, True)) +@pytest.mark.parametrize("empty_final", (False, True)) +@pytest.mark.parametrize("chunks", ((1600,), (400,) * 4, (160,) * 10, (80, 320, 40, 1160))) +def test_no_lfr_stream_matches_full_filterbank(tmp_path, cmvn, empty_final, chunks): + frontend = make_frontend(tmp_path, cmvn) + audio = 0.2 * torch.sin(torch.arange(1600, dtype=torch.float32) * (2 * torch.pi * 440 / 16000)) + expected = kaldi.fbank( + audio.unsqueeze(0) * (1 << 15), num_mel_bins=80, frame_length=25, frame_shift=10, + sample_frequency=16000, window_type="hamming", dither=0.0, energy_floor=0.0, + ) + if cmvn: + expected = (expected + 0.25) * 2 + cache, outputs, offset, emitted = {}, [], 0, 0 + calls = chunks + ((0,) if empty_final else ()) + for index, size in enumerate(calls): + chunk = audio[offset : offset + size] + offset += size + feats, lengths = frontend( + chunk.unsqueeze(0), torch.tensor([size]), cache=cache, + is_final=index == len(calls) - 1, return_waveform=True, + ) + count = int(lengths[0]) if lengths.numel() else 0 + if count: + outputs.append(feats[0, :count]) + start = emitted * 160 + stop = start + (count - 1) * 160 + 400 + torch.testing.assert_close(cache["aligned_waveforms"][0], audio[start:stop], rtol=0, atol=0) + emitted += count + else: + assert cache["aligned_waveforms"].numel() == 0 + assert cache["reserve_waveforms"].numel() == 0 + assert emitted == expected.shape[0] == 8 + torch.testing.assert_close(torch.cat(outputs), expected, rtol=1e-5, atol=1e-5) + + +@pytest.mark.parametrize("size", (0, 160, 399)) +@pytest.mark.parametrize("empty_final", (False, True)) +def test_no_lfr_short_stream_has_no_frames(tmp_path, size, empty_final): + frontend = make_frontend(tmp_path, False) + cache = {} + for index, length in enumerate((size, 0) if empty_final else (size,)): + feats, lengths = frontend( + torch.zeros(1, length), torch.tensor([length]), cache=cache, + is_final=not empty_final or index == 1, return_waveform=True, + ) + assert feats.numel() == 0 + assert not lengths.numel() or lengths.sum() == 0 + assert cache["aligned_waveforms"].numel() == 0 + + +@pytest.mark.parametrize("lfr_m", (1, 5)) +@pytest.mark.parametrize("offset_ms", (0, 1000)) +@pytest.mark.parametrize("chunk_ms", (20, 100, 5000)) +def test_offline_vad_finalizes_last_nonempty_chunk(lfr_m, offset_ms, chunk_ms): + class ScriptedEncoder(torch.nn.Module): + def __init__(self): + super().__init__() + self.frames = 0 + + def forward(self, feats, cache=None): + time_ms = (torch.arange(feats.shape[1]) + self.frames) * 10 + speech = ((time_ms >= 500 + offset_ms) & (time_ms < 1500 + offset_ms)) | ( + (time_ms >= 2500 + offset_ms) & (time_ms < 3200 + offset_ms) + ) + self.frames += feats.shape[1] + probability = torch.where(speech, 0.999, 0.001) + return torch.stack((1 - probability, probability), dim=-1).unsqueeze(0) + + model = vad_model.FsmnVADStreaming.__new__(vad_model.FsmnVADStreaming) + torch.nn.Module.__init__(model) + model.encoder = ScriptedEncoder() + model.vad_opts = vad_model.VADXOptions() + audio = 0.2 * torch.sin( + torch.arange((4000 + offset_ms) * 16, dtype=torch.float32) + * (2 * torch.pi * 440 / 16000) + ) + frontend = WavFrontendOnline(lfr_m=lfr_m, lfr_n=1, dither=0.0) + result, _ = model.inference( + [audio], key=["fixture"], cache={}, frontend=frontend, device="cpu", + chunk_size=chunk_ms, is_streaming_input=False, is_final=True, dynamic_silence=True, + ) + assert model.encoder.frames == (4000 + offset_ms) // 10 - 2 + assert result == [{"key": "fixture", "value": [[240 + offset_ms, 3980 + offset_ms]]}]