fix(normalize): correct variance with reproducible CPU/CUDA regressions

.github/workflows/test-funasr-onnx-package.yml

@@ -13,6 +13,7 @@ on:
       - "tests/test_funasr_onnx_paraformer_online_empty_final.py"
       - "tests/test_funasr_onnx_vad_online_short_chunks.py"
       - "tests/test_funasr_onnx_vad_multiple_inputs.py"
+      - "tests/test_funasr_onnx_punctuation_empty_input.py"
       - "tests/test_rich_transcription_postprocess.py"
       - "tests/test_rich_transcription_event_segments.py"
   push:
@@ -29,6 +30,7 @@ on:
       - "tests/test_funasr_onnx_paraformer_online_empty_final.py"
       - "tests/test_funasr_onnx_vad_online_short_chunks.py"
       - "tests/test_funasr_onnx_vad_multiple_inputs.py"
+      - "tests/test_funasr_onnx_punctuation_empty_input.py"
       - "tests/test_rich_transcription_postprocess.py"
       - "tests/test_rich_transcription_event_segments.py"
   workflow_dispatch:
@@ -84,7 +86,7 @@ jobs:
           import importlib.util
           from importlib.metadata import requires, version
 
-          assert version("funasr-onnx") == "0.4.7"
+          assert version("funasr-onnx") == "0.4.9"
           assert importlib.util.find_spec("torch") is None
 
           requirements = requires("funasr-onnx") or []
@@ -115,6 +117,9 @@ jobs:
       - name: Verify offline VAD results per input
         run: python tests/test_funasr_onnx_vad_multiple_inputs.py
 
+      - name: Verify empty punctuation streaming cache
+        run: python tests/test_funasr_onnx_punctuation_empty_input.py
+
       - name: Verify source distribution error paths
         run: |
           python -m pip install --force-reinstall --no-deps dist/funasr-onnx/*.tar.gz
@@ -125,7 +130,7 @@ jobs:
         if: matrix.python-version == '3.12'
         uses: actions/upload-artifact@v7
         with:
-          name: funasr-onnx-0.4.7
+          name: funasr-onnx-0.4.9
           path: dist/funasr-onnx/*
           if-no-files-found: error
 

.github/workflows/test-kws-output.yml

@@ -8,12 +8,14 @@ on:
       - "funasr/models/sanm_kws/**"
       - "funasr/models/sanm_kws_streaming/**"
       - "funasr/frontends/wav_frontend.py"
+      - "funasr/models/fsmn_vad_streaming/model.py"
       - "funasr/auto/auto_model.py"
       - "funasr/utils/datadir_writer.py"
       - "funasr/utils/kws_utils.py"
       - "tests/test_kws_optional_output.py"
       - "tests/test_kws_streaming_continuity.py"
       - "tests/test_kws_utils.py"
+      - "tests/test_wav_frontend_no_lfr.py"
       - "tests/test_fsmn_vad_streaming_buffers.py"
       - "tests/test_dynamic_streaming_vad.py"
       - "tests/test_fsmn_vad_dynamic_silence.py"
@@ -26,12 +28,14 @@ on:
       - "funasr/models/sanm_kws/**"
       - "funasr/models/sanm_kws_streaming/**"
       - "funasr/frontends/wav_frontend.py"
+      - "funasr/models/fsmn_vad_streaming/model.py"
       - "funasr/auto/auto_model.py"
       - "funasr/utils/datadir_writer.py"
       - "funasr/utils/kws_utils.py"
       - "tests/test_kws_optional_output.py"
       - "tests/test_kws_streaming_continuity.py"
       - "tests/test_kws_utils.py"
+      - "tests/test_wav_frontend_no_lfr.py"
       - "tests/test_fsmn_vad_streaming_buffers.py"
       - "tests/test_dynamic_streaming_vad.py"
       - "tests/test_fsmn_vad_dynamic_silence.py"
@@ -63,6 +67,7 @@ jobs:
             tests/test_kws_optional_output.py \
             tests/test_kws_streaming_continuity.py \
             tests/test_kws_utils.py \
+            tests/test_wav_frontend_no_lfr.py \
             tests/test_fsmn_vad_streaming_buffers.py \
             tests/test_dynamic_streaming_vad.py \
             tests/test_fsmn_vad_dynamic_silence.py

.github/workflows/test-numpy-compatibility.yml

@@ -8,6 +8,11 @@ on:
       - "funasr/cli.py"
       - "tests/test_cli.py"
       - "funasr/auto/auto_model.py"
+      - "funasr/tokenizer/sentencepiece_tokenizer.py"
+      - "funasr/tokenizer/char_tokenizer.py"
+      - "funasr/tokenizer/phoneme_tokenizer.py"
+      - "tests/test_char_tokenizer_non_linguistic_symbols.py"
+      - "tests/test_sentencepiece_tokenizer.py"
       - "tests/test_speaker_optional_dependency.py"
       - "funasr/utils/load_utils.py"
       - "funasr/bin/realtime_ws.py"
@@ -26,6 +31,8 @@ on:
       - "tests/test_pcm_input_format.py"
       - "tests/test_load_audio_bytes.py"
       - "funasr/frontends/**"
+      - "funasr/models/normalize/utterance_mvn.py"
+      - "tests/test_utterance_mvn.py"
       - "funasr/models/emotion2vec/**"
       - "funasr/models/campplus/**"
       - "funasr/models/sense_voice/model.py"
@@ -45,6 +52,11 @@ on:
       - "funasr/cli.py"
       - "tests/test_cli.py"
       - "funasr/auto/auto_model.py"
+      - "funasr/tokenizer/sentencepiece_tokenizer.py"
+      - "funasr/tokenizer/char_tokenizer.py"
+      - "funasr/tokenizer/phoneme_tokenizer.py"
+      - "tests/test_char_tokenizer_non_linguistic_symbols.py"
+      - "tests/test_sentencepiece_tokenizer.py"
       - "tests/test_speaker_optional_dependency.py"
       - "funasr/utils/load_utils.py"
       - "funasr/bin/realtime_ws.py"
@@ -63,6 +75,8 @@ on:
       - "tests/test_pcm_input_format.py"
       - "tests/test_load_audio_bytes.py"
       - "funasr/frontends/**"
+      - "funasr/models/normalize/utterance_mvn.py"
+      - "tests/test_utterance_mvn.py"
       - "funasr/models/emotion2vec/**"
       - "funasr/models/campplus/**"
       - "funasr/models/sense_voice/model.py"
@@ -119,12 +133,15 @@ jobs:
         run: |
           python -m pytest -q \
             tests/test_numpy_compatibility.py \
+            tests/test_utterance_mvn.py \
             tests/test_numpy_cluster_compatibility.py \
             tests/test_cluster_backend.py \
             tests/test_speaker_optional_dependency.py \
             tests/test_eend_feature_compatibility.py \
             tests/test_pypi_metadata.py \
             tests/test_sensevoice_word_timestamps.py \
+            tests/test_sentencepiece_tokenizer.py \
+            tests/test_char_tokenizer_non_linguistic_symbols.py \
             tests/test_pcm_input_format.py \
             tests/test_realtime_ws_service.py \
             tests/test_realtime_ws_hf_overrides.py \

README.md

@@ -364,7 +364,9 @@ recordings with the same evaluation scope.
 ## What's new
 
 - **MOSS-Transcribe-Diarize** brings long-form ASR, timestamps, and anonymous speaker labels to FunASR services, Docker, Kubernetes, vLLM/SGLang workflows, and FunClip. [Deploy MOSS ->](./docs/moss_transcribe_diarize.md)
-- **FunASR 1.4.21** preserves the streaming punctuation cache on empty or whitespace chunks, returning an empty result without touching cached words or the embedding path; nonempty inference is unchanged. [Fix #3824](https://github.com/modelscope/FunASR/pull/3824). Install with `python -m pip install -U "funasr==1.4.21"`. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21)
+- **FunASR 1.4.23** fixes the online frontend cursor for `lfr_m=lfr_n=1` and redundant empty final offline VAD chunks. With a supplied `token_list`, SentencePiece `ids2tokens` returns model pieces; use `tokens2text` for text. Without `token_list`, legacy text decoding is unchanged. [Fix #3849](https://github.com/modelscope/FunASR/pull/3849), [fix #3783](https://github.com/modelscope/FunASR/pull/3783). Install with `python -m pip install -U "funasr==1.4.23"`. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23)
+  **FunASR 1.4.22** honors per-call `merge_length_s` and evaluates dynamic VAD silence thresholds per frame using the current segment duration, preserving the fixed-threshold opt-out. [Fix #3841](https://github.com/modelscope/FunASR/pull/3841), [fix #3833](https://github.com/modelscope/FunASR/pull/3833). [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22)
+  **FunASR 1.4.21** preserves the streaming punctuation cache on empty or whitespace chunks; nonempty inference is unchanged. [Fix #3824](https://github.com/modelscope/FunASR/pull/3824). [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21)
   **FunASR 1.4.20** preserves the original optional speaker dependency failure in a chained `ImportError` when `spk_model` is requested, while ASR-only construction remains available. [Fix #3819](https://github.com/modelscope/FunASR/pull/3819). [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.20)
   **FunASR 1.4.19** handles empty-token Paraformer batches, enforces `max_speakers` for automatic clustering, and adds validated JSON-object `--hf_overrides` support to realtime WebSocket serving. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.19)
   Previous release **FunASR 1.4.18** fixes cross-script hotword misreplacement, filtered-token timestamp alignment, and Chinese VAD sentence/speaker drift. [Release and verification scope ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.18)

README_ja.md

@@ -112,7 +112,9 @@ CER/WER をそろえて比較してください。オフラインのスループ
 ## 最新情報
 
 - **MOSS-Transcribe-Diarize** を FunASR service、Docker、Kubernetes、vLLM/SGLang workflow、FunClip に統合し、長時間 ASR、timestamp、匿名 speaker label を一度に処理できます。[MOSS をデプロイ ->](./docs/moss_transcribe_diarize.md)
-- **FunASR 1.4.21** はストリーミング句読点処理の空または空白入力で cache を保持し、キャッシュ済み単語や embedding 経路に触れず空結果を返します。非空推論は変更されません。[修正 #3824](https://github.com/modelscope/FunASR/pull/3824)。`python -m pip install -U "funasr==1.4.21"` でインストールできます。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21)
+- **FunASR 1.4.23** は `lfr_m=lfr_n=1` のオンライン特徴抽出のフレーム位置と、オフライン VAD の余分な空の最終チャンクを修正します。`token_list` 指定時の SentencePiece `ids2tokens` はモデルのピースを返し、テキストへの変換には `tokens2text` を使います。未指定時の従来のテキスト復号は維持します。[修正 #3849](https://github.com/modelscope/FunASR/pull/3849)、[修正 #3783](https://github.com/modelscope/FunASR/pull/3783)。`python -m pip install -U "funasr==1.4.23"` でインストールできます。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23)
+  **FunASR 1.4.22** は呼び出しごとの `merge_length_s` を反映し、現在の発話区間の長さに基づいて動的 VAD 無音しきい値をフレーム単位で更新します。固定しきい値モードも維持します。[修正 #3841](https://github.com/modelscope/FunASR/pull/3841)、[修正 #3833](https://github.com/modelscope/FunASR/pull/3833)。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22)
+  **FunASR 1.4.21** はストリーミング句読点処理の空入力で cache を保持します。非空推論は変更されません。[修正 #3824](https://github.com/modelscope/FunASR/pull/3824)。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21)
   **FunASR 1.4.20** は `spk_model` 指定時に任意の話者依存モジュールを読み込めない場合、元の例外を連鎖した `ImportError` で報告します。ASR のみの構築は引き続き利用できます。[修正 #3819](https://github.com/modelscope/FunASR/pull/3819)。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.20)
   **FunASR 1.4.19** は Paraformer の空 token batch を処理し、自動クラスタリングの `max_speakers` 上限を適用します。リアルタイム WebSocket サービスには JSON object を検証する `--hf_overrides` を追加しました。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.19)
   前のリリース **FunASR 1.4.18** は中国語とラテン文字が混在する hotword の誤置換、特殊 token 除去後の timestamp 対応、中国語 VAD 分割での文と話者のずれを修正しました。[リリースと検証範囲 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.18)

README_ko.md

@@ -111,7 +111,9 @@ checkpoint/revision, 오디오 집합, 하드웨어, 배치, 워밍업, 측정 
 ## 최신 소식
 
 - **MOSS-Transcribe-Diarize**를 FunASR service, Docker, Kubernetes, vLLM/SGLang workflow, FunClip에 통합해 긴 오디오 ASR, timestamp, 익명 speaker label을 한 번에 처리합니다. [MOSS 배포 ->](./docs/moss_transcribe_diarize.md)
-- **FunASR 1.4.21**은 스트리밍 문장부호 처리의 빈 입력이나 공백 입력에서 cache를 유지하고, 캐시된 단어나 embedding 경로를 건드리지 않은 채 빈 결과를 반환합니다. 비어 있지 않은 추론은 변경되지 않습니다. [수정 #3824](https://github.com/modelscope/FunASR/pull/3824). `python -m pip install -U "funasr==1.4.21"`로 설치할 수 있습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21)
+- **FunASR 1.4.23**은 `lfr_m=lfr_n=1` 온라인 프런트엔드의 프레임 위치와 오프라인 VAD의 불필요한 빈 마지막 청크를 수정합니다. `token_list`를 지정하면 SentencePiece `ids2tokens`는 모델 토큰을 반환하며, 텍스트 변환에는 `tokens2text`를 사용합니다. 미지정 시 기존 텍스트 디코딩은 유지됩니다. [수정 #3849](https://github.com/modelscope/FunASR/pull/3849), [수정 #3783](https://github.com/modelscope/FunASR/pull/3783). `python -m pip install -U "funasr==1.4.23"`로 설치할 수 있습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23)
+  **FunASR 1.4.22**는 호출별 `merge_length_s`를 반영하고 현재 발화 구간 길이에 따라 동적 VAD 무음 임계값을 프레임마다 갱신합니다. 고정 임계값 모드도 유지합니다. [수정 #3841](https://github.com/modelscope/FunASR/pull/3841), [수정 #3833](https://github.com/modelscope/FunASR/pull/3833). [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22)
+  **FunASR 1.4.21**은 스트리밍 문장부호 처리의 빈 입력에서 cache를 유지합니다. 비어 있지 않은 추론은 변경되지 않습니다. [수정 #3824](https://github.com/modelscope/FunASR/pull/3824). [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21)
   **FunASR 1.4.20**은 `spk_model` 사용 시 선택적 화자 의존성 가져오기가 실패하면 원래 예외를 연결한 `ImportError`를 제공합니다. ASR 전용 구성은 계속 사용할 수 있습니다. [수정 #3819](https://github.com/modelscope/FunASR/pull/3819). [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.20)
   **FunASR 1.4.19**는 Paraformer의 빈 token batch를 처리하고 자동 클러스터링에 `max_speakers` 상한을 적용합니다. 실시간 WebSocket 서비스에 JSON object를 검증하는 `--hf_overrides`도 추가했습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.19)
   이전 릴리스 **FunASR 1.4.18**은 중국어와 라틴 문자가 섞인 hotword의 잘못된 치환, 특수 token 제거 후 timestamp 정렬, 중국어 VAD 분할의 문장 및 화자 불일치를 수정했습니다. [릴리스 및 검증 범위 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.18)

README_zh.md

@@ -157,7 +157,9 @@ checkpoint/revision、音频集、硬件、批量大小、预热、计时范围
 ## 最新动态
 
 - **MOSS-Transcribe-Diarize** 已接入 FunASR 服务、Docker、Kubernetes、vLLM/SGLang 工作流和 FunClip，一次完成长音频转写、时间戳与匿名说话人标注。[部署 MOSS ->](./docs/moss_transcribe_diarize_zh.md)
-- **FunASR 1.4.21** 在流式标点收到空白或空输入时保留 cache，并在不访问缓存词或 embedding 路径的情况下返回空结果；非空推理保持不变。[修复 #3824](https://github.com/modelscope/FunASR/pull/3824)。安装：`python -m pip install -U "funasr==1.4.21"`。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21)
+- **FunASR 1.4.23** 修复 `lfr_m=lfr_n=1` 在线前端的帧游标和离线 VAD 多余的空尾块。提供 `token_list` 时，SentencePiece 的 `ids2tokens` 返回模型词元，请用 `tokens2text` 转为文本；未提供 `token_list` 时保持旧有文本解码行为。[修复 #3849](https://github.com/modelscope/FunASR/pull/3849)、[修复 #3783](https://github.com/modelscope/FunASR/pull/3783)。安装：`python -m pip install -U "funasr==1.4.23"`。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.23)
+  **FunASR 1.4.22** 支持逐调用覆盖 `merge_length_s`，并根据当前语音段时长逐帧更新动态 VAD 静音阈值，同时保留固定阈值模式。[修复 #3841](https://github.com/modelscope/FunASR/pull/3841)、[修复 #3833](https://github.com/modelscope/FunASR/pull/3833)。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.22)
+  **FunASR 1.4.21** 在流式标点收到空白或空输入时保留 cache；非空推理保持不变。[修复 #3824](https://github.com/modelscope/FunASR/pull/3824)。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.21)
   **FunASR 1.4.20** 在指定 `spk_model` 而可选说话人依赖导入失败时，通过带原始异常链的 `ImportError` 提供诊断，同时保留仅 ASR 的构造路径。[修复 #3819](https://github.com/modelscope/FunASR/pull/3819)。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.20)
   **FunASR 1.4.19** 修复 Paraformer 空 token 批次处理，使自动聚类遵守 `max_speakers` 上限，并为实时 WebSocket 服务新增带 JSON 对象校验的 `--hf_overrides` 参数。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.19)
   上一版本 **FunASR 1.4.18** 修复跨中英文热词误替换、过滤特殊符号后的时间戳对齐，以及中文 VAD 分段的句子与说话人错位。[发布说明与验证范围 ->](https://github.com/modelscope/FunASR/releases/tag/v1.4.18)

docs/python_api.md

@@ -42,7 +42,7 @@ The wrapper accepts `**kwargs`, not a universally validated parameter schema. A
 | `batch_size_s` | `300` | VAD-path batching budget in seconds, based on longest padded segment times segment count, not a hard file-duration limit. CPU VAD processing uses single-segment batches. |
 | `batch_size_threshold_s` | `60` | VAD batching threshold for segment duration, not a VAD splitting setting. |
 | `merge_vad` | `False` | Enables merging detected VAD regions for this call. |
-| `merge_length_s` | `15` | VAD merge target read before call overrides are merged into ASR config. Set this at construction in this checkout. |
+| `merge_length_s` | `15` | Target length in seconds for merged VAD regions when `merge_vad=True`. A per-call value overrides the construction value. |
 | `cache` | No wrapper-managed session | Pass a caller-owned dictionary on every streaming call. It is not a hub/model download cache. |
 | `sentence_timestamp` | `False` | Requests sentence records in the VAD pipeline; depends on available timestamps and punctuation, with limited VAD fallback. |
 | `return_spk_res` | `True` | Includes clustered speaker results when VAD and a speaker model are configured. `False` does not avoid construction or segment embedding computation. |