diff --git a/funasr/utils/timestamp_tools.py b/funasr/utils/timestamp_tools.py index b97e405f3..8009ec757 100644 --- a/funasr/utils/timestamp_tools.py +++ b/funasr/utils/timestamp_tools.py @@ -189,6 +189,11 @@ def timestamp_sentence( punc_id = int(punc_id) if punc_id is not None else 1 sentence_end = timestamp[1] if timestamp is not None else sentence_end + # Latin tokens are inserted with a leading space. Drop one leading + # space, as timestamp_sentence_en does, so a sentence that starts + # with an English word is not stored as " iphone...". + if sentence_text.startswith(" "): + sentence_text = sentence_text[1:] if punc_id > 1: sentence_text += punc_list[punc_id - 2] if sentence_text_seg.endswith(" "): diff --git a/tests/test_timestamp_tools.py b/tests/test_timestamp_tools.py index a9dd41d18..763c14e19 100644 --- a/tests/test_timestamp_tools.py +++ b/tests/test_timestamp_tools.py @@ -57,6 +57,31 @@ def test_timestamp_sentence_en_missing_trailing_timestamp_matches_zh_fallback(): assert en_result[1]["start"] == zh_result[1]["start"] +def test_timestamp_sentence_does_not_prefix_latin_text_with_a_space(): + mixed = timestamp_sentence( + punc_id_list=[1, 2, 1, 1, 3], + timestamp_postprocessed=[[0, 100], [100, 200], [200, 300], [300, 400], [400, 500]], + text_postprocessed="我 用 iphone pro max", + return_raw_text=True, + ) + assert [sentence["text"] for sentence in mixed] == ["我用,", "iphone pro max。"] + assert [sentence["raw_text"] for sentence in mixed] == ["我 用", "iphone pro max"] + + english = timestamp_sentence( + [1, 3, 1], + [[0, 100], [100, 200], [200, 300]], + "hello world again", + ) + assert [sentence["text"] for sentence in english] == ["hello world。", "again"] + + chinese = timestamp_sentence( + [1, 3], + [[0, 100], [100, 200]], + "你好 世界", + ) + assert chinese[0]["text"] == "你好世界。" + + def test_timestamp_sentence_raw_text_keeps_token_separators(): result = timestamp_sentence( punc_id_list=[1, 2, 1, 1, 3],