From 5156f09ee35c150fa0fa5f1ac08062d5addd533a Mon Sep 17 00:00:00 2001 From: Mohammad Hijjawi Date: Sat, 7 Nov 2026 10:40:10 +0000 Subject: [PATCH 1/2] fix(tokenizer): ignore empty non-linguistic symbols to avoid a hang CharTokenizer and PhonemeTokenizer load non_linguistic_symbols with line.rstrip(), so a blank line in the symbol file (or an "" entry in a list) adds the empty string to the set. line.startswith("") is true at every position without consuming any input, so text2tokens never terminates (and keeps appending "" tokens when symbols are kept). Drop the empty string after the symbols are loaded and add a regression test covering a file with a blank line and an iterable with "". --- funasr/tokenizer/char_tokenizer.py | 3 ++ funasr/tokenizer/phoneme_tokenizer.py | 3 ++ ...t_char_tokenizer_non_linguistic_symbols.py | 44 +++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 tests/test_char_tokenizer_non_linguistic_symbols.py diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py index d25c98926f..69e8cd3a4d 100644 --- a/funasr/tokenizer/char_tokenizer.py +++ b/funasr/tokenizer/char_tokenizer.py @@ -44,6 +44,9 @@ def __init__( self.non_linguistic_symbols = set() else: self.non_linguistic_symbols = set(non_linguistic_symbols) + # An empty symbol (e.g. a blank line in the file) matches every position + # and would make text2tokens loop forever. + self.non_linguistic_symbols.discard("") self.remove_non_linguistic_symbols = remove_non_linguistic_symbols self.split_with_space = split_with_space self.seg_dict = None diff --git a/funasr/tokenizer/phoneme_tokenizer.py b/funasr/tokenizer/phoneme_tokenizer.py index ab479242a6..0005483e37 100644 --- a/funasr/tokenizer/phoneme_tokenizer.py +++ b/funasr/tokenizer/phoneme_tokenizer.py @@ -599,6 +599,9 @@ def __init__( self.non_linguistic_symbols = set() else: self.non_linguistic_symbols = set(non_linguistic_symbols) + # An empty symbol (e.g. a blank line in the file) matches every position + # and would make text2tokens loop forever. + self.non_linguistic_symbols.discard("") self.remove_non_linguistic_symbols = remove_non_linguistic_symbols def __repr__(self): diff --git a/tests/test_char_tokenizer_non_linguistic_symbols.py b/tests/test_char_tokenizer_non_linguistic_symbols.py new file mode 100644 index 0000000000..7644ba1eae --- /dev/null +++ b/tests/test_char_tokenizer_non_linguistic_symbols.py @@ -0,0 +1,44 @@ +"""Tokenizers must ignore empty non-linguistic symbols instead of looping forever.""" + +import os +import tempfile +import unittest + +from funasr.tokenizer.char_tokenizer import CharTokenizer +from funasr.tokenizer.phoneme_tokenizer import PhonemeTokenizer + + +class CharTokenizerNonLinguisticSymbolsTest(unittest.TestCase): + def test_blank_line_in_symbol_file_is_ignored(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, "nlsyms.txt") + with open(path, "w", encoding="utf-8") as f: + f.write("\n\n") + for remove in (False, True): + with self.subTest(remove_non_linguistic_symbols=remove): + tokenizer = CharTokenizer( + non_linguistic_symbols=path, remove_non_linguistic_symbols=remove + ) + self.assertEqual(tokenizer.non_linguistic_symbols, {""}) + expected = ["你", "好"] if remove else ["你", "", "好"] + self.assertEqual(tokenizer.text2tokens("你好"), expected) + + def test_empty_symbol_in_iterable_is_ignored(self): + tokenizer = CharTokenizer(non_linguistic_symbols=["", ""]) + self.assertEqual(tokenizer.non_linguistic_symbols, {""}) + self.assertEqual(tokenizer.text2tokens("a b"), ["a", "", "b"]) + + +class PhonemeTokenizerNonLinguisticSymbolsTest(unittest.TestCase): + def test_empty_symbol_is_ignored(self): + tokenizer = PhonemeTokenizer( + g2p_type=None, + non_linguistic_symbols=["", ""], + remove_non_linguistic_symbols=True, + ) + self.assertEqual(tokenizer.non_linguistic_symbols, {""}) + self.assertEqual(tokenizer.text2tokens("ab"), ["ab"]) + + +if __name__ == "__main__": + unittest.main() From a6516ae40d53a12933955dbd3ec807be76b6df9f Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Sun, 8 Nov 2026 00:36:37 +0800 Subject: [PATCH 2/2] test(tokenizer): run empty-symbol regressions in NumPy CI Preserve the original PR #3852 commit and add only missing trigger paths and regression selection. Signed-off-by: LauraGPT --- .github/workflows/test-numpy-compatibility.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml index bb5ba166ea..962f78ef3a 100644 --- a/.github/workflows/test-numpy-compatibility.yml +++ b/.github/workflows/test-numpy-compatibility.yml @@ -9,6 +9,9 @@ on: - "tests/test_cli.py" - "funasr/auto/auto_model.py" - "funasr/tokenizer/sentencepiece_tokenizer.py" + - "funasr/tokenizer/char_tokenizer.py" + - "funasr/tokenizer/phoneme_tokenizer.py" + - "tests/test_char_tokenizer_non_linguistic_symbols.py" - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" @@ -48,6 +51,9 @@ on: - "tests/test_cli.py" - "funasr/auto/auto_model.py" - "funasr/tokenizer/sentencepiece_tokenizer.py" + - "funasr/tokenizer/char_tokenizer.py" + - "funasr/tokenizer/phoneme_tokenizer.py" + - "tests/test_char_tokenizer_non_linguistic_symbols.py" - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" @@ -130,6 +136,7 @@ jobs: tests/test_pypi_metadata.py \ tests/test_sensevoice_word_timestamps.py \ tests/test_sentencepiece_tokenizer.py \ + tests/test_char_tokenizer_non_linguistic_symbols.py \ tests/test_pcm_input_format.py \ tests/test_realtime_ws_service.py \ tests/test_realtime_ws_hf_overrides.py \