From a540211c4293b2371bf860d5f7129cdf7c5e4284 Mon Sep 17 00:00:00 2001 From: Mohammad Hijjawi Date: Thu, 5 Nov 2026 22:32:37 +0000 Subject: [PATCH] fix(auto): keep a result for inputs whose VAD speech decodes to no text inference_with_vad skipped an input when every VAD segment decoded to empty text, while the no-speech and result-count branches append a keyed empty result. generate() could then return an empty list, so the CLI and the fallback server path failed on result[0] with IndexError, and a multi-input batch lost that input's row. Append the same keyed empty result on this path and update the Python API docs, which described the skip. --- docs/python_api.md | 2 +- docs/python_api_zh.md | 2 +- funasr/auto/auto_model.py | 2 ++ tests/test_punc_model_none.py | 50 +++++++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/docs/python_api.md b/docs/python_api.md index f95a5ba27..e340724e7 100644 --- a/docs/python_api.md +++ b/docs/python_api.md @@ -161,7 +161,7 @@ For `sentence_timestamp=True` without speakers, the current wrapper can return V ### Result Fields -`generate()` returns a Python `list` of model-result dictionaries, not an OpenAI response object. Do not assume every model produces every field, or index `results[0]` without checking: some VAD empty-text paths can skip a result. No-speech VAD results commonly contain `{"key": ..., "text": "", "timestamp": []}`. +`generate()` returns a Python `list` of model-result dictionaries, not an OpenAI response object. Do not assume every model produces every field. With a VAD model, each input keeps one result: when no speech is detected, or the detected speech decodes to no text, the result is `{"key": ..., "text": "", "timestamp": []}`. | Field | Meaning | |---|---| diff --git a/docs/python_api_zh.md b/docs/python_api_zh.md index 1f70e6d90..261a75a13 100644 --- a/docs/python_api_zh.md +++ b/docs/python_api_zh.md @@ -160,7 +160,7 @@ for result in model.generate(input=str(paths["audio"]), return_spk_res=True): ### 返回字段 -`generate()` 返回 Python `list`，其中是模型结果字典，而不是 OpenAI 响应对象。不要假设所有模型都有全部字段，也不要未经检查直接使用 `results[0]`：VAD 路径中的部分空文本分支会跳过一条结果。没有检测到语音时，VAD 流程通常返回 `{"key": ..., "text": "", "timestamp": []}`。 +`generate()` 返回 Python `list`，其中是模型结果字典，而不是 OpenAI 响应对象。不要假设所有模型都有全部字段。使用 VAD 模型时，每个输入都保留一条结果：没有检测到语音，或检测到的语音解码后没有文本时，结果为 `{"key": ..., "text": "", "timestamp": []}`。 | 字段 | 含义 | |---|---| diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py index b8320dfa7..c0b6a20e4 100644 --- a/funasr/auto/auto_model.py +++ b/funasr/auto/auto_model.py @@ -1212,6 +1212,8 @@ def inference_with_vad(self, input, input_len=None, **cfg): ] if not len(result["text"].strip()): + results_ret_list.append({"key": key, "text": "", "timestamp": []}) + logging.info("decoding, utt: {}, empty text".format(key)) continue return_raw_text = kwargs.get("return_raw_text", False) aligned_words = result.get("words") diff --git a/tests/test_punc_model_none.py b/tests/test_punc_model_none.py index 42dfe34e6..eba2d99b5 100644 --- a/tests/test_punc_model_none.py +++ b/tests/test_punc_model_none.py @@ -620,6 +620,56 @@ def mock_inference(data, *args, **kwargs): ) self.assertEqual(results[0]["sentence_info"][-1]["timestamp"][-1], [1100, 1600]) + @patch("funasr.auto.auto_model.slice_padding_audio_samples") + @patch("funasr.auto.auto_model.load_audio_text_image_video") + @patch("funasr.auto.auto_model.prepare_data_iterator") + def test_empty_asr_text_keeps_a_keyed_result(self, mock_prep, mock_load, mock_slice): + """VAD speech that decodes to no text must not remove the input from the results.""" + for segments, asr_results in ( + ([[0, 1000]], [[{"text": ""}]]), + ([[0, 1000], [2000, 3500]], [[{"text": ""}], [{"text": ""}]]), + ): + with self.subTest(segments=len(segments)): + am = self._make_auto_model(punc_model=MagicMock()) + results_seq = [[{"key": "noise", "value": segments}], *asr_results] + am.inference = MagicMock(side_effect=lambda *args, **kwargs: results_seq.pop(0)) + mock_prep.return_value = (["noise"], [np.zeros(16000, dtype=np.float32)]) + mock_load.return_value = np.zeros(16000, dtype=np.float32) + mock_slice.return_value = ([np.zeros(16000, dtype=np.float32)], [16000]) + + results = am.inference_with_vad("dummy_input", sentence_timestamp=True) + + self.assertEqual(results, [{"key": "noise", "text": "", "timestamp": []}]) + + @patch("funasr.auto.auto_model.slice_padding_audio_samples") + @patch("funasr.auto.auto_model.load_audio_text_image_video") + @patch("funasr.auto.auto_model.prepare_data_iterator") + def test_empty_asr_text_keeps_input_order(self, mock_prep, mock_load, mock_slice): + """Results stay aligned with a multi-input batch when one input has no text.""" + am = self._make_auto_model() + results_seq = [ + [ + {"key": "first", "value": [[0, 1000]]}, + {"key": "noise", "value": [[0, 1000]]}, + {"key": "last", "value": [[0, 1000]]}, + ], + [{"text": "hello", "timestamp": [[0, 500]]}], + [{"text": ""}], + [{"text": "world", "timestamp": [[0, 500]]}], + ] + am.inference = MagicMock(side_effect=lambda *args, **kwargs: results_seq.pop(0)) + mock_prep.return_value = ( + ["first", "noise", "last"], + [np.zeros(16000, dtype=np.float32)] * 3, + ) + mock_load.return_value = np.zeros(16000, dtype=np.float32) + mock_slice.return_value = ([np.zeros(16000, dtype=np.float32)], [16000]) + + results = am.inference_with_vad("dummy_input") + + self.assertEqual([r["key"] for r in results], ["first", "noise", "last"]) + self.assertEqual([r["text"] for r in results], ["hello", "", "world"]) + @patch("funasr.auto.auto_model.slice_padding_audio_samples") @patch("funasr.auto.auto_model.load_audio_text_image_video") @patch("funasr.auto.auto_model.prepare_data_iterator")