Fix quadratic-time parsing of consecutive reference definitions

docs/changelog.md

@@ -16,6 +16,8 @@ See the [Contributing Guide](contributing.md) for details.
 
 ### Fixed
 
+* Fix quadratic-time parsing of many consecutive reference definitions, and of
+  many consecutive footnote definitions in the `footnotes` extension.
 * Support automatic links with URL schemes other than HTTP and FTP, such as
   `<ssh://example.org>` (#1244).
 * Fix quadratic-time id generation in the `toc` extension when many headers

markdown/blockprocessors.py

@@ -588,16 +588,25 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool:
         block = blocks.pop(0)
         m = self.RE.search(block)
         if m:
-            id = m.group(1).strip().lower()
-            link = m.group(2).lstrip('<').rstrip('>')
-            title = m.group(5) or m.group(6)
-            self.parser.md.references[id] = (link, title)
-            if block[m.end():].strip():
+            start = end = m.start()
+            # Record the definitions on the following lines in this pass rather
+            # than one per pass through the block processors. A definition after
+            # the first which spans lines is left for the next pass, as another
+            # processor may claim part of it.
+            for m in self.RE.finditer(block, start):
+                if m.start() > start and (block[end:m.start()].strip('\n') or '\n' in m.group()):
+                    break
+                id = m.group(1).strip().lower()
+                link = m.group(2).lstrip('<').rstrip('>')
+                title = m.group(5) or m.group(6)
+                self.parser.md.references[id] = (link, title)
+                end = m.end()
+            if block[end:].strip():
                 # Add any content after match back to blocks as separate block
-                blocks.insert(0, block[m.end():].lstrip('\n'))
-            if block[:m.start()].strip():
+                blocks.insert(0, block[end:].lstrip('\n'))
+            if block[:start].strip():
                 # Add any content before match back to blocks as separate block
-                blocks.insert(0, block[:m.start()].rstrip('\n'))
+                blocks.insert(0, block[:start].rstrip('\n'))
             return True
         # No match. Restore block.
         blocks.insert(0, block)

markdown/extensions/footnotes.py

@@ -261,6 +261,16 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool:
 
         m = self.RE.search(block)
         if m:
+            preceding = block[:m.start()]
+            # Record the definitions which are directly followed by another
+            # definition in this pass rather than one per pass through the block
+            # processors.
+            for m2 in self.RE.finditer(block, m.end()):
+                if block[m.end():m2.start()].strip('\n'):
+                    break
+                self.footnotes.setFootnote(m.group(1), m.group(2).rstrip())
+                m = m2
+
             id = m.group(1)
             fn_blocks = [m.group(2)]
 
@@ -284,9 +294,9 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool:
             footnote = "\n\n".join(fn_blocks)
             self.footnotes.setFootnote(id, footnote.rstrip())
 
-            if block[:m.start()].strip():
+            if preceding.strip():
                 # Add any content before match back to blocks as separate block
-                blocks.insert(0, block[:m.start()].rstrip('\n'))
+                blocks.insert(0, preceding.rstrip('\n'))
             return True
         # No match. Restore block.
         blocks.insert(0, block)

tests/test_syntax/blocks/test_reference_links.py

@@ -60,3 +60,154 @@ def test_malformed_reference_with_long_run_of_spaces(self):
         self.assertMarkdownRenders(
             text, f'<p>{text}</p>', expected_attrs={'references': {}}
         )
+
+    def test_reference_definitions_before_setext_underline(self):
+        """
+        A setext underline after consecutive reference definitions is a paragraph,
+        and does not turn the last definition into a header.
+        """
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\n[b]: http://example.com/b\n===',
+            '<p>===</p>',
+            expected_attrs={
+                'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)}
+            }
+        )
+
+    def test_reference_definitions_before_split_definition(self):
+        """
+        A definition split across lines after other definitions is left for the
+        next pass, so a horizontal rule on its second line is not taken as the link.
+        """
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\n[b]:\n***',
+            '<p>[b]:</p>\n<hr />',
+            expected_attrs={'references': {'a': ('http://example.com/a', None)}}
+        )
+
+    def test_reference_definitions_before_definition_split_across_lines(self):
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\n[b]:\nhttp://example.com/b',
+            '',
+            expected_attrs={
+                'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)}
+            }
+        )
+
+    def test_reference_definitions_title_on_own_line(self):
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\n[b]: http://example.com/b\n"Title"',
+            '',
+            expected_attrs={
+                'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', 'Title')}
+            }
+        )
+
+    def test_reference_definitions_before_paragraph(self):
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\n[b]: http://example.com/b\nSome text',
+            '<p>Some text</p>',
+            expected_attrs={
+                'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)}
+            }
+        )
+
+    def test_paragraph_before_reference_definitions(self):
+        self.assertMarkdownRenders(
+            'Some text\n[a]: http://example.com/a\n[b]: http://example.com/b',
+            '<p>Some text</p>',
+            expected_attrs={
+                'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)}
+            }
+        )
+
+    def test_reference_definitions_separated_by_paragraph(self):
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\nSome text\n[b]: http://example.com/b',
+            '<p>Some text</p>',
+            expected_attrs={
+                'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)}
+            }
+        )
+
+    def test_indented_reference_definitions(self):
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\n   [b]: http://example.com/b',
+            '',
+            expected_attrs={
+                'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)}
+            }
+        )
+
+    def test_indented_reference_definitions_short_tab_length(self):
+        """
+        A definition indented by `tab_length` spaces directly after another
+        definition is a definition, not an indented code block.
+        """
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\n  [b]: http://example.com/b',
+            '',
+            expected_attrs={
+                'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)}
+            },
+            tab_length=2
+        )
+
+    def test_duplicate_reference_definitions(self):
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\n[a]: http://example.com/b',
+            '',
+            expected_attrs={'references': {'a': ('http://example.com/b', None)}}
+        )
+
+    def test_reference_definitions_before_hash_header(self):
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\n[b]: http://example.com/b\n# Header',
+            '<h1>Header</h1>',
+            expected_attrs={
+                'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)}
+            }
+        )
+
+    def test_reference_definitions_before_hr(self):
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\n[b]: http://example.com/b\n***',
+            '<hr />',
+            expected_attrs={
+                'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)}
+            }
+        )
+
+    def test_reference_definitions_before_blockquote(self):
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\n[b]: http://example.com/b\n> quote',
+            '<blockquote>\n<p>quote</p>\n</blockquote>',
+            expected_attrs={
+                'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)}
+            }
+        )
+
+    def test_reference_definitions_before_code_block(self):
+        self.assertMarkdownRenders(
+            '[a]: http://example.com/a\n[b]: http://example.com/b\n    code',
+            '<pre><code>code\n</code></pre>',
+            expected_attrs={
+                'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)}
+            }
+        )
+
+    def test_many_reference_definitions(self):
+        """
+        Consecutive reference definitions should be consumed together rather
+        than returning the remainder of the block to be parsed once per
+        definition.
+        """
+        count = 10000
+        text = ''.join(f'[id{i}]: http://example.com/{i}\n' for i in range(count))
+        self.assertMarkdownRenders(
+            text,
+            '',
+            expected_attrs={
+                'references': {f'id{i}': (f'http://example.com/{i}', None) for i in range(count)}
+            }
+        )

tests/test_syntax/extensions/test_footnotes.py

@@ -265,6 +265,157 @@ def test_footnote_multple_complex_no_blank_line_between(self):
             '</div>'
         )
 
+    def test_consecutive_footnotes_last_multi_line(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                """
+                [^1]: Footnote 1
+                [^2]: Footnote 2
+                    line 2
+                """
+            ),
+            '<div class="footnote">\n'
+            '<hr />\n'
+            '<ol>\n'
+            '<li id="fn:1">\n'
+            '<p>Footnote 1&#160;<a class="footnote-backref" href="#fnref:1"'
+            ' title="Jump back to footnote 1 in the text">&#8617;</a></p>\n'
+            '</li>\n'
+            '<li id="fn:2">\n'
+            '<p>Footnote 2\nline 2&#160;<a class="footnote-backref" href="#fnref:2"'
+            ' title="Jump back to footnote 2 in the text">&#8617;</a></p>\n'
+            '</li>\n'
+            '</ol>\n'
+            '</div>'
+        )
+
+    def test_consecutive_footnotes_last_multi_line_lazy_indent(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                """
+                [^1]: Footnote 1
+                [^2]: Footnote 2
+                line 2
+                """
+            ),
+            '<div class="footnote">\n'
+            '<hr />\n'
+            '<ol>\n'
+            '<li id="fn:1">\n'
+            '<p>Footnote 1&#160;<a class="footnote-backref" href="#fnref:1"'
+            ' title="Jump back to footnote 1 in the text">&#8617;</a></p>\n'
+            '</li>\n'
+            '<li id="fn:2">\n'
+            '<p>Footnote 2\nline 2&#160;<a class="footnote-backref" href="#fnref:2"'
+            ' title="Jump back to footnote 2 in the text">&#8617;</a></p>\n'
+            '</li>\n'
+            '</ol>\n'
+            '</div>'
+        )
+
+    def test_consecutive_footnotes_last_multi_paragraph(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                """
+                [^1]: Footnote 1
+                [^2]: Footnote 2
+
+                    paragraph 2
+                """
+            ),
+            '<div class="footnote">\n'
+            '<hr />\n'
+            '<ol>\n'
+            '<li id="fn:1">\n'
+            '<p>Footnote 1&#160;<a class="footnote-backref" href="#fnref:1"'
+            ' title="Jump back to footnote 1 in the text">&#8617;</a></p>\n'
+            '</li>\n'
+            '<li id="fn:2">\n'
+            '<p>Footnote 2</p>\n'
+            '<p>paragraph 2&#160;<a class="footnote-backref" href="#fnref:2"'
+            ' title="Jump back to footnote 2 in the text">&#8617;</a></p>\n'
+            '</li>\n'
+            '</ol>\n'
+            '</div>'
+        )
+
+    def test_paragraph_before_consecutive_footnotes(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                """
+                paragraph[^1]
+                [^1]: Footnote 1
+                [^2]: Footnote 2
+                """
+            ),
+            '<p>paragraph<sup id="fnref:1"><a class="footnote-ref" href="#fn:1">1</a></sup></p>\n'
+            '<div class="footnote">\n'
+            '<hr />\n'
+            '<ol>\n'
+            '<li id="fn:1">\n'
+            '<p>Footnote 1&#160;<a class="footnote-backref" href="#fnref:1"'
+            ' title="Jump back to footnote 1 in the text">&#8617;</a></p>\n'
+            '</li>\n'
+            '<li id="fn:2">\n'
+            '<p>Footnote 2&#160;<a class="footnote-backref" href="#fnref:2"'
+            ' title="Jump back to footnote 2 in the text">&#8617;</a></p>\n'
+            '</li>\n'
+            '</ol>\n'
+            '</div>'
+        )
+
+    def test_consecutive_footnotes_before_setext_underline(self):
+        """
+        A setext underline after consecutive footnotes is a lazily indented line
+        of the last footnote, and does not turn that definition into a header.
+        """
+        self.assertMarkdownRenders(
+            self.dedent(
+                """
+                [^1]: Footnote 1
+                [^2]: Footnote 2
+                ===
+                """
+            ),
+            '<div class="footnote">\n'
+            '<hr />\n'
+            '<ol>\n'
+            '<li id="fn:1">\n'
+            '<p>Footnote 1&#160;<a class="footnote-backref" href="#fnref:1"'
+            ' title="Jump back to footnote 1 in the text">&#8617;</a></p>\n'
+            '</li>\n'
+            '<li id="fn:2">\n'
+            '<h1>Footnote 2</h1>\n'
+            '<p><a class="footnote-backref" href="#fnref:2"'
+            ' title="Jump back to footnote 2 in the text">&#8617;</a></p>\n'
+            '</li>\n'
+            '</ol>\n'
+            '</div>'
+        )
+
+    def test_many_footnote_definitions(self):
+        """
+        Consecutive footnote definitions should be consumed together rather
+        than returning the remainder of the block to be parsed once per
+        definition.
+        """
+        count = 5000
+        self.assertMarkdownRenders(
+            ''.join(f'[^{i}]: Footnote {i}\n' for i in range(count)),
+            '<div class="footnote">\n'
+            '<hr />\n'
+            '<ol>\n'
+            + ''.join(
+                f'<li id="fn:{i}">\n'
+                f'<p>Footnote {i}&#160;<a class="footnote-backref" href="#fnref:{i}"'
+                f' title="Jump back to footnote {i + 1} in the text">&#8617;</a></p>\n'
+                '</li>\n'
+                for i in range(count)
+            )
+            + '</ol>\n'
+            '</div>'
+        )
+
     def test_backlink_text(self):
         """Test back-link configuration."""