From eb4aad1c3ff7eeb35492141e311ee25a83781614 Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Mon, 20 Jul 2026 04:53:27 +0000 Subject: [PATCH] Fix quadratic Doxygen lookahead on single-line declarations Limit trailing-comment scans to the immediate declaration boundary instead of repeatedly rebuilding the remaining token buffer. Reorder enum initializer parsing so trailing comments are collected after the value is consumed, preserving Doxygen behavior while restoring linear parsing for variables, fields, comma-separated declarators, and enumerators. --- cxxheaderparser/lexer.py | 33 +++-- cxxheaderparser/parser.py | 22 ++- tests/test_doxygen.py | 235 ++++++++++++++++++++++++++++++ tests/test_doxygen_performance.py | 53 +++++++ 4 files changed, 323 insertions(+), 20 deletions(-) create mode 100644 tests/test_doxygen_performance.py diff --git a/cxxheaderparser/lexer.py b/cxxheaderparser/lexer.py index 748a186..5818a09 100644 --- a/cxxheaderparser/lexer.py +++ b/cxxheaderparser/lexer.py @@ -832,26 +832,35 @@ def get_doxygen_after(self) -> typing.Optional[str]: if not tokbuf: return None - # retrieve comments after non-discard elements + # retrieve comments after the current declaration boundary. Only scan + # the immediate delimiter/comment region; callers must consume any + # initializer/expression before asking for trailing comments. comments: typing.List[LexToken] = [] - new_tokbuf = typing.Deque[LexToken]() + restore = typing.Deque[LexToken]() + seen_boundary = False - # This is different: we only extract tokens here while tokbuf: tok = tokbuf.popleft() - if tok.type == "NEWLINE": + tok_type = tok.type + if tok_type == "NEWLINE": break - elif tok.type == "WHITESPACE": - new_tokbuf.append(tok) - elif tok.type in ("COMMENT_SINGLELINE", "COMMENT_MULTILINE"): + elif tok_type == "WHITESPACE": + restore.append(tok) + continue + elif tok_type in ("COMMENT_SINGLELINE", "COMMENT_MULTILINE"): comments.append(tok) + continue + elif comments or seen_boundary: + restore.append(tok) + break + + restore.append(tok) + if tok_type in (",", ";", "}"): + seen_boundary = True else: - new_tokbuf.append(tok) - if comments: - break + break - new_tokbuf.extend(tokbuf) - self.tokbuf = new_tokbuf + tokbuf.extendleft(reversed(restore)) if comments: return self._extract_comments(comments) diff --git a/cxxheaderparser/parser.py b/cxxheaderparser/parser.py index c5f0fbe..46efef5 100644 --- a/cxxheaderparser/parser.py +++ b/cxxheaderparser/parser.py @@ -1324,21 +1324,27 @@ def _parse_enumerator_list(self) -> typing.List[Enumerator]: if name_tok.value == "}": break - if doxygen is None: - doxygen = self.lex.get_doxygen_after() - name = name_tok.value value = None attributes = [] - tok = self._next_token_must_be("}", ",", "=", "DBL_LBRACKET") - if tok.type == "DBL_LBRACKET": + if doxygen is None: + doxygen = self.lex.get_doxygen_after() + + tok = self.lex.token_if("DBL_LBRACKET") + if tok: attributes = self._consume_attribute_specifier_seq(tok, record=False) - tok = self._next_token_must_be("}", ",", "=") - if tok.type == "=": + if doxygen is None: + doxygen = self.lex.get_doxygen_after() + + if self.lex.token_if("="): value = self._create_value(self._consume_value_until([], ",", "}")) - tok = self._next_token_must_be("}", ",") + + if doxygen is None: + doxygen = self.lex.get_doxygen_after() + + tok = self._next_token_must_be("}", ",") values.append(Enumerator(name, value, doxygen, attributes)) diff --git a/tests/test_doxygen.py b/tests/test_doxygen.py index a6aa64b..a13e963 100644 --- a/tests/test_doxygen.py +++ b/tests/test_doxygen.py @@ -491,3 +491,238 @@ def test_doxygen_using_alias() -> None: ] ) ) + + +def test_doxygen_oneline_variables() -> None: + content = """ + int v0; /** v0 */ int v1; /** v1 */ + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="v0")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + doxygen="/** v0 */", + ), + Variable( + name=PQName(segments=[NameSpecifier(name="v1")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + doxygen="/** v1 */", + ), + ] + ) + ) + + +def test_doxygen_oneline_class_fields() -> None: + content = """ + class C { int f0; /** f0 */ int f1; /** f1 */ }; + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + classes=[ + ClassScope( + class_decl=ClassDecl( + typename=PQName( + segments=[NameSpecifier(name="C")], classkey="class" + ) + ), + fields=[ + Field( + access="private", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="f0", + doxygen="/** f0 */", + ), + Field( + access="private", + type=Type( + typename=PQName( + segments=[FundamentalSpecifier(name="int")] + ) + ), + name="f1", + doxygen="/** f1 */", + ), + ], + ) + ] + ) + ) + + +def test_doxygen_oneline_comma_decls() -> None: + content = """ + int c0, /** c0 */ c1; /** c1 */ + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + variables=[ + Variable( + name=PQName(segments=[NameSpecifier(name="c0")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + doxygen="/** c0 */", + ), + Variable( + name=PQName(segments=[NameSpecifier(name="c1")]), + type=Type( + typename=PQName(segments=[FundamentalSpecifier(name="int")]) + ), + doxygen="/** c1 */", + ), + ] + ) + ) + + +def test_doxygen_oneline_enum_values() -> None: + content = """ + enum E { E0, /** E0 */ E1 /** E1 */ }; + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + values=[ + Enumerator(name="E0", doxygen="/** E0 */"), + Enumerator(name="E1", doxygen="/** E1 */"), + ], + ) + ] + ) + ) + + +def test_doxygen_enum_value_with_template_comma() -> None: + content = """ + enum E { E0 = Foo::value, /** E0 */ E1 }; + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + values=[ + Enumerator( + name="E0", + value=Value( + tokens=[ + Token(value="Foo"), + Token(value="<"), + Token(value="int"), + Token(value=","), + Token(value="int"), + Token(value=">"), + Token(value="::"), + Token(value="value"), + ] + ), + doxygen="/** E0 */", + ), + Enumerator(name="E1"), + ], + ) + ] + ) + ) + + +def test_doxygen_enum_value_with_less_than_expression() -> None: + content = """ + enum E { E0 = (1 < 2), /** E0 */ E1 }; + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + values=[ + Enumerator( + name="E0", + value=Value( + tokens=[ + Token(value="("), + Token(value="1"), + Token(value="<"), + Token(value="2"), + Token(value=")"), + ] + ), + doxygen="/** E0 */", + ), + Enumerator(name="E1"), + ], + ) + ] + ) + ) + + +def test_doxygen_enum_value_with_greater_than_expression() -> None: + content = """ + enum E { E0 = 2 > 1, /** E0 */ E1 }; + """ + + data = parse_string(content, cleandoc=True) + + assert data == ParsedData( + namespace=NamespaceScope( + enums=[ + EnumDecl( + typename=PQName( + segments=[NameSpecifier(name="E")], classkey="enum" + ), + values=[ + Enumerator( + name="E0", + value=Value( + tokens=[ + Token(value="2"), + Token(value=">"), + Token(value="1"), + ] + ), + doxygen="/** E0 */", + ), + Enumerator(name="E1"), + ], + ) + ] + ) + ) diff --git a/tests/test_doxygen_performance.py b/tests/test_doxygen_performance.py new file mode 100644 index 0000000..ec422ac --- /dev/null +++ b/tests/test_doxygen_performance.py @@ -0,0 +1,53 @@ +import collections + +import pytest + +import cxxheaderparser.lexer as lexer +from cxxheaderparser.simple import parse_string + + +class CountingDeque(collections.deque): + popleft_count = 0 + restore_count = 0 + + def __class_getitem__(cls, item): + return cls + + def popleft(self): + type(self).popleft_count += 1 + return super().popleft() + + def extend(self, values): + values = list(values) + type(self).restore_count += len(values) + return super().extend(values) + + def extendleft(self, values): + values = list(values) + type(self).restore_count += len(values) + return super().extendleft(values) + + @classmethod + def reset_counts(cls) -> None: + cls.popleft_count = 0 + cls.restore_count = 0 + + +def _count_token_buffer_touches( + declaration_count: int, monkeypatch: pytest.MonkeyPatch +) -> int: + monkeypatch.setattr(lexer.typing, "Deque", CountingDeque) + CountingDeque.reset_counts() + + parse_string("int x;" * declaration_count) + + return CountingDeque.popleft_count + CountingDeque.restore_count + + +def test_doxygen_after_lookahead_is_not_quadratic( + monkeypatch: pytest.MonkeyPatch, +) -> None: + small = _count_token_buffer_touches(120, monkeypatch) + large = _count_token_buffer_touches(240, monkeypatch) + + assert large < small * 3