fix(tokenizer): prevent empty-symbol hangs with hosted regression coverage

.github/workflows/test-numpy-compatibility.yml

@@ -9,6 +9,9 @@ on:
       - "tests/test_cli.py"
       - "funasr/auto/auto_model.py"
       - "funasr/tokenizer/sentencepiece_tokenizer.py"
+      - "funasr/tokenizer/char_tokenizer.py"
+      - "funasr/tokenizer/phoneme_tokenizer.py"
+      - "tests/test_char_tokenizer_non_linguistic_symbols.py"
       - "tests/test_sentencepiece_tokenizer.py"
       - "tests/test_speaker_optional_dependency.py"
       - "funasr/utils/load_utils.py"
@@ -48,6 +51,9 @@ on:
       - "tests/test_cli.py"
       - "funasr/auto/auto_model.py"
       - "funasr/tokenizer/sentencepiece_tokenizer.py"
+      - "funasr/tokenizer/char_tokenizer.py"
+      - "funasr/tokenizer/phoneme_tokenizer.py"
+      - "tests/test_char_tokenizer_non_linguistic_symbols.py"
       - "tests/test_sentencepiece_tokenizer.py"
       - "tests/test_speaker_optional_dependency.py"
       - "funasr/utils/load_utils.py"
@@ -130,6 +136,7 @@ jobs:
             tests/test_pypi_metadata.py \
             tests/test_sensevoice_word_timestamps.py \
             tests/test_sentencepiece_tokenizer.py \
+            tests/test_char_tokenizer_non_linguistic_symbols.py \
             tests/test_pcm_input_format.py \
             tests/test_realtime_ws_service.py \
             tests/test_realtime_ws_hf_overrides.py \

funasr/tokenizer/char_tokenizer.py

@@ -44,6 +44,9 @@ def __init__(
                 self.non_linguistic_symbols = set()
         else:
             self.non_linguistic_symbols = set(non_linguistic_symbols)
+        # An empty symbol (e.g. a blank line in the file) matches every position
+        # and would make text2tokens loop forever.
+        self.non_linguistic_symbols.discard("")
         self.remove_non_linguistic_symbols = remove_non_linguistic_symbols
         self.split_with_space = split_with_space
         self.seg_dict = None

funasr/tokenizer/phoneme_tokenizer.py

@@ -599,6 +599,9 @@ def __init__(
                 self.non_linguistic_symbols = set()
         else:
             self.non_linguistic_symbols = set(non_linguistic_symbols)
+        # An empty symbol (e.g. a blank line in the file) matches every position
+        # and would make text2tokens loop forever.
+        self.non_linguistic_symbols.discard("")
         self.remove_non_linguistic_symbols = remove_non_linguistic_symbols
 
     def __repr__(self):

tests/test_char_tokenizer_non_linguistic_symbols.py

@@ -0,0 +1,44 @@
+"""Tokenizers must ignore empty non-linguistic symbols instead of looping forever."""
+
+import os
+import tempfile
+import unittest
+
+from funasr.tokenizer.char_tokenizer import CharTokenizer
+from funasr.tokenizer.phoneme_tokenizer import PhonemeTokenizer
+
+
+class CharTokenizerNonLinguisticSymbolsTest(unittest.TestCase):
+    def test_blank_line_in_symbol_file_is_ignored(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            path = os.path.join(tmpdir, "nlsyms.txt")
+            with open(path, "w", encoding="utf-8") as f:
+                f.write("<noise>\n\n")
+            for remove in (False, True):
+                with self.subTest(remove_non_linguistic_symbols=remove):
+                    tokenizer = CharTokenizer(
+                        non_linguistic_symbols=path, remove_non_linguistic_symbols=remove
+                    )
+                    self.assertEqual(tokenizer.non_linguistic_symbols, {"<noise>"})
+                    expected = ["你", "好"] if remove else ["你", "<noise>", "好"]
+                    self.assertEqual(tokenizer.text2tokens("你<noise>好"), expected)
+
+    def test_empty_symbol_in_iterable_is_ignored(self):
+        tokenizer = CharTokenizer(non_linguistic_symbols=["", "<noise>"])
+        self.assertEqual(tokenizer.non_linguistic_symbols, {"<noise>"})
+        self.assertEqual(tokenizer.text2tokens("a <noise>b"), ["a", "<noise>", "b"])
+
+
+class PhonemeTokenizerNonLinguisticSymbolsTest(unittest.TestCase):
+    def test_empty_symbol_is_ignored(self):
+        tokenizer = PhonemeTokenizer(
+            g2p_type=None,
+            non_linguistic_symbols=["", "<noise>"],
+            remove_non_linguistic_symbols=True,
+        )
+        self.assertEqual(tokenizer.non_linguistic_symbols, {"<noise>"})
+        self.assertEqual(tokenizer.text2tokens("a<noise>b"), ["ab"])
+
+
+if __name__ == "__main__":
+    unittest.main()