diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml index bb5ba166e..962f78ef3 100644 --- a/.github/workflows/test-numpy-compatibility.yml +++ b/.github/workflows/test-numpy-compatibility.yml @@ -9,6 +9,9 @@ on: - "tests/test_cli.py" - "funasr/auto/auto_model.py" - "funasr/tokenizer/sentencepiece_tokenizer.py" + - "funasr/tokenizer/char_tokenizer.py" + - "funasr/tokenizer/phoneme_tokenizer.py" + - "tests/test_char_tokenizer_non_linguistic_symbols.py" - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" @@ -48,6 +51,9 @@ on: - "tests/test_cli.py" - "funasr/auto/auto_model.py" - "funasr/tokenizer/sentencepiece_tokenizer.py" + - "funasr/tokenizer/char_tokenizer.py" + - "funasr/tokenizer/phoneme_tokenizer.py" + - "tests/test_char_tokenizer_non_linguistic_symbols.py" - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" @@ -130,6 +136,7 @@ jobs: tests/test_pypi_metadata.py \ tests/test_sensevoice_word_timestamps.py \ tests/test_sentencepiece_tokenizer.py \ + tests/test_char_tokenizer_non_linguistic_symbols.py \ tests/test_pcm_input_format.py \ tests/test_realtime_ws_service.py \ tests/test_realtime_ws_hf_overrides.py \ diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py index d25c98926..69e8cd3a4 100644 --- a/funasr/tokenizer/char_tokenizer.py +++ b/funasr/tokenizer/char_tokenizer.py @@ -44,6 +44,9 @@ def __init__( self.non_linguistic_symbols = set() else: self.non_linguistic_symbols = set(non_linguistic_symbols) + # An empty symbol (e.g. a blank line in the file) matches every position + # and would make text2tokens loop forever. + self.non_linguistic_symbols.discard("") self.remove_non_linguistic_symbols = remove_non_linguistic_symbols self.split_with_space = split_with_space self.seg_dict = None diff --git a/funasr/tokenizer/phoneme_tokenizer.py b/funasr/tokenizer/phoneme_tokenizer.py index ab479242a..0005483e3 100644 --- a/funasr/tokenizer/phoneme_tokenizer.py +++ b/funasr/tokenizer/phoneme_tokenizer.py @@ -599,6 +599,9 @@ def __init__( self.non_linguistic_symbols = set() else: self.non_linguistic_symbols = set(non_linguistic_symbols) + # An empty symbol (e.g. a blank line in the file) matches every position + # and would make text2tokens loop forever. + self.non_linguistic_symbols.discard("") self.remove_non_linguistic_symbols = remove_non_linguistic_symbols def __repr__(self): diff --git a/tests/test_char_tokenizer_non_linguistic_symbols.py b/tests/test_char_tokenizer_non_linguistic_symbols.py new file mode 100644 index 000000000..7644ba1ea --- /dev/null +++ b/tests/test_char_tokenizer_non_linguistic_symbols.py @@ -0,0 +1,44 @@ +"""Tokenizers must ignore empty non-linguistic symbols instead of looping forever.""" + +import os +import tempfile +import unittest + +from funasr.tokenizer.char_tokenizer import CharTokenizer +from funasr.tokenizer.phoneme_tokenizer import PhonemeTokenizer + + +class CharTokenizerNonLinguisticSymbolsTest(unittest.TestCase): + def test_blank_line_in_symbol_file_is_ignored(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, "nlsyms.txt") + with open(path, "w", encoding="utf-8") as f: + f.write("\n\n") + for remove in (False, True): + with self.subTest(remove_non_linguistic_symbols=remove): + tokenizer = CharTokenizer( + non_linguistic_symbols=path, remove_non_linguistic_symbols=remove + ) + self.assertEqual(tokenizer.non_linguistic_symbols, {""}) + expected = ["你", "好"] if remove else ["你", "", "好"] + self.assertEqual(tokenizer.text2tokens("你好"), expected) + + def test_empty_symbol_in_iterable_is_ignored(self): + tokenizer = CharTokenizer(non_linguistic_symbols=["", ""]) + self.assertEqual(tokenizer.non_linguistic_symbols, {""}) + self.assertEqual(tokenizer.text2tokens("a b"), ["a", "", "b"]) + + +class PhonemeTokenizerNonLinguisticSymbolsTest(unittest.TestCase): + def test_empty_symbol_is_ignored(self): + tokenizer = PhonemeTokenizer( + g2p_type=None, + non_linguistic_symbols=["", ""], + remove_non_linguistic_symbols=True, + ) + self.assertEqual(tokenizer.non_linguistic_symbols, {""}) + self.assertEqual(tokenizer.text2tokens("ab"), ["ab"]) + + +if __name__ == "__main__": + unittest.main()