diff --git a/src/license_expression/_pyahocorasick.py b/src/license_expression/_pyahocorasick.py index 2a1f5bb..2ad4a2c 100644 --- a/src/license_expression/_pyahocorasick.py +++ b/src/license_expression/_pyahocorasick.py @@ -492,6 +492,7 @@ def filter_overlapping(tokens): else: logger_debug(" del curr_tok smaller overlap:", curr_tok) del tokens[i] + i -= 1 break j += 1 i += 1 diff --git a/tests/test__pyahocorasick.py b/tests/test__pyahocorasick.py index 2cc4213..00559ab 100644 --- a/tests/test__pyahocorasick.py +++ b/tests/test__pyahocorasick.py @@ -17,6 +17,7 @@ from license_expression._pyahocorasick import Trie from license_expression._pyahocorasick import Token +from license_expression._pyahocorasick import filter_overlapping class TestTrie(unittest.TestCase): @@ -315,3 +316,13 @@ def test_iter_simple(self): ] assert expected == result + + +def test_filter_overlapping_rechecks_replacement_token(): + tokens = [Token(0, 1), Token(1, 4), Token(4, 8), Token(10, 11)] + assert filter_overlapping(tokens) == [Token(4, 8), Token(10, 11)] + + +def test_filter_overlapping_removes_contained_token_after_replacement(): + tokens = [Token(0, 1), Token(1, 5), Token(2, 3), Token(8, 9)] + assert filter_overlapping(tokens) == [Token(1, 5), Token(8, 9)]