From e610432b331d5e114654dbe72f367a47b5fe2ed8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Gonz=C3=A1lez?= Date: Mon, 10 Mar 2025 11:47:41 +0100 Subject: [PATCH 1/4] add link to auto_mode in sphinx-autodoc2 fixes #1031 --- docs/syntax/code_and_apis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/syntax/code_and_apis.md b/docs/syntax/code_and_apis.md index 771b843d..aeb3ee21 100644 --- a/docs/syntax/code_and_apis.md +++ b/docs/syntax/code_and_apis.md @@ -170,7 +170,7 @@ Sphinx and MyST provide means to analyse source code and automatically generate As opposed to `sphinx.ext.autodoc`, `sphinx-autodoc2` performs static (rather than dynamic) analysis of the source code, integrates full package documenting, and also allows for docstrings to be written in both RestructureText and MyST. -The `auto_mode` will automatically generate the full API documentation, as shown . +The [sphinx-autodoc2's `auto_mode` config variable](https://sphinx-autodoc2.readthedocs.io/en/latest/config.html#confval-autodoc2_packages-auto_mode) (which per default is `True`) will automatically generate the full API documentation, as shown . Alternatively, the `autodoc2-object` directive can be used to generate documentation for a single object. To embed in a MyST document the MyST `render_plugin` should be specified, for example: From 4f9abe985d984319e7b7f83058efe364e56dad4b Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Sun, 12 Jul 2026 14:22:57 +0000 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20build-aborting=20cras?= =?UTF-8?q?hes=20on=20hostile=20input,=20and=20docutils=200.23=20compat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects where malformed or unlucky input aborted the whole build instead of degrading gracefully: - Deeply nested HTML: rendering the parsed HTML AST recurses once per nesting level, and the resulting RecursionError escaped the html_to_nodes try/except (which only covered tokenization), killing the build. Now caught, warning as myst.html_parse and falling back to the raw HTML. - Hostile YAML front matter: only ParserError/ScannerError were caught, so a ConstructorError (e.g. `title: !Ref X`), ComposerError, or the bare ValueError PyYAML leaks for out-of-range timestamps (`2021-99-99`) crashed the parse. Both read_topmatter and render_front_matter now catch yaml.YAMLError + ValueError and emit the intended myst.topmatter warning. - A footnote whose label matches any heading or target name was silently dropped: the duplicate-definition check tested against document.nameids, which holds every target name, not just footnote labels. It now checks against the names of previously rendered footnotes only. Also docutils 0.23 support (the only failure was test-side, docutils now attaches info-level system messages to the doctree when report_level allows): - normalize them out of the tree in test_link_resolution (they are still asserted via the captured report stream) - extend the pin to docutils<0.24 and add 0.23 to the myst-docutils CI test matrix --- .github/workflows/tests.yml | 2 +- myst_parser/config/main.py | 4 +- myst_parser/mdit_to_docutils/base.py | 14 +++- myst_parser/mdit_to_docutils/html_to_nodes.py | 18 +++- pyproject.toml | 2 +- tests/test_docutils.py | 84 +++++++++++++++++++ .../test_renderers/test_fixtures_docutils.py | 7 ++ 7 files changed, 125 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b4c2b3dd..d8ea2697 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -74,7 +74,7 @@ jobs: strategy: fail-fast: false matrix: - docutils-version: ["0.20", "0.21", "0.22"] + docutils-version: ["0.20", "0.21", "0.22", "0.23"] steps: - name: Checkout source diff --git a/myst_parser/config/main.py b/myst_parser/config/main.py index 2a380c49..59f80ee1 100644 --- a/myst_parser/config/main.py +++ b/myst_parser/config/main.py @@ -609,7 +609,9 @@ def read_topmatter(text: str | Iterator[str]) -> dict[str, Any] | None: top_matter.append(line.rstrip() + "\n") try: metadata = yaml.safe_load("".join(top_matter)) - except (yaml.parser.ParserError, yaml.scanner.ScannerError) as err: + # ValueError: PyYAML lets it escape for some invalid scalars, + # e.g. out-of-range timestamps like `2021-99-99` + except (yaml.YAMLError, ValueError) as err: raise TopmatterReadError("Malformed YAML") from err if not isinstance(metadata, dict): raise TopmatterReadError(f"YAML is not a dict: {type(metadata)}") diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index 8ecfb70a..ecee2d61 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -1262,7 +1262,9 @@ def render_front_matter(self, token: SyntaxTreeNode) -> None: if isinstance(token.content, str): try: data = yaml.safe_load(token.content) - except (yaml.parser.ParserError, yaml.scanner.ScannerError): + # ValueError: PyYAML lets it escape for some invalid scalars, + # e.g. out-of-range timestamps like `2021-99-99` + except (yaml.YAMLError, ValueError): self.create_warning( "Malformed YAML", MystWarnings.MD_TOPMATTER, @@ -1505,7 +1507,15 @@ def render_footnote_reference(self, token: SyntaxTreeNode) -> None: """Despite the name, this is actually a footnote definition, e.g. `[^a]: ...`""" target = token.meta["label"] - if target in self.document.nameids: + # check against existing footnote labels only, not `document.nameids`, + # which holds every target name (headings, `(name)=` targets, ...) and + # so would wrongly drop a footnote that merely shares a heading's name + existing_labels = { + name + for footnote in self.document.autofootnotes + self.document.footnotes + for name in footnote["names"] + } + if target in existing_labels: # note we chose to directly omit these footnotes in the parser, # rather than let docutils/sphinx handle them, since otherwise you end up with a confusing warning: # WARNING: Duplicate explicit target name: "x". [docutils] diff --git a/myst_parser/mdit_to_docutils/html_to_nodes.py b/myst_parser/mdit_to_docutils/html_to_nodes.py index 713cc040..46a22365 100644 --- a/myst_parser/mdit_to_docutils/html_to_nodes.py +++ b/myst_parser/mdit_to_docutils/html_to_nodes.py @@ -7,7 +7,7 @@ from docutils import nodes -from myst_parser.parsers.parse_html import Data, tokenize_html +from myst_parser.parsers.parse_html import Data, Element, tokenize_html from myst_parser.warnings_ import MystWarnings if TYPE_CHECKING: @@ -81,6 +81,22 @@ def html_to_nodes( ): return default_html(text, renderer.document["source"], line_number) + try: + return _render_nodes(root, line_number, renderer) + except RecursionError: + msg_node = renderer.create_warning( + "HTML is too deeply nested to process", + MystWarnings.HTML_PARSE, + line=line_number, + ) + return ([msg_node] if msg_node else []) + default_html( + text, renderer.document["source"], line_number + ) + + +def _render_nodes( + root: Element, line_number: int, renderer: DocutilsRenderer +) -> list[nodes.Element]: nodes_list = [] for child in root: if child.name == "img": diff --git a/pyproject.toml b/pyproject.toml index 3ff3bd2f..8c0d5d79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ keywords = [ ] requires-python = ">=3.11" dependencies = [ - "docutils>=0.20,<0.23", + "docutils>=0.20,<0.24", "jinja2", # required for substitutions, but let sphinx choose version "markdown-it-py~=4.2", "mdit-py-plugins~=0.6,>=0.6.1", diff --git a/tests/test_docutils.py b/tests/test_docutils.py index bed66709..ebe14bf3 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -1,6 +1,7 @@ import contextlib import importlib.util import io +import sys from dataclasses import dataclass, field, fields from textwrap import dedent from typing import Literal @@ -210,6 +211,89 @@ def test_linkify_no_warning_when_available(): assert "[myst.linkify]" not in stream.getvalue() +def test_html_deep_nesting_warns(): + """Deeply nested HTML degrades to raw output with a warning. + + Regression: rendering the HTML AST recurses once per nesting level, + and the resulting ``RecursionError`` escaped and aborted the build. + """ + depth = sys.getrecursionlimit() + # tags on separate lines, to stay under docutils' line-length-limit + source = ( + '

Title

\n' + + "
\n" * depth + + "content\n" + + "
\n" * depth + + "
\n" + ) + stream = io.StringIO() + doctree = publish_doctree( + source=source, + parser=Parser(), + settings_overrides={ + "myst_enable_extensions": ["html_admonition"], + "warning_stream": stream, + }, + ) + assert "HTML is too deeply nested" in stream.getvalue() + assert "[myst.html]" in stream.getvalue() + # the original text is preserved as a raw HTML node + assert list(doctree.findall(nodes.raw)) + + +@pytest.mark.parametrize( + "yaml_line", + [ + "title: !UnknownTag value", # yaml.constructor.ConstructorError + "date: 2021-99-99", # bare ValueError from an out-of-range timestamp + ], +) +def test_topmatter_hostile_yaml_warns(yaml_line): + """Front matter YAML errors beyond parser/scanner ones warn, not crash. + + Regression: only ``ParserError``/``ScannerError`` were caught, so e.g. a + ``ConstructorError`` from an unknown tag aborted the whole build. + """ + stream = io.StringIO() + doctree = publish_doctree( + source=f"---\n{yaml_line}\n---\n\ncontent\n", + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + assert "[myst.topmatter]" in stream.getvalue() + assert "content" in doctree.pformat() + + +def test_footnote_label_matching_heading_name(): + """A footnote label sharing a heading's name is not a duplicate. + + Regression: the duplicate check used ``document.nameids``, which holds + every target name, so the footnote definition was silently dropped. + """ + stream = io.StringIO() + doctree = publish_doctree( + source="# Note\n\n[^note]\n\n[^note]: the definition\n", + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + footnotes = list(doctree.findall(nodes.footnote)) + assert footnotes, "expected the footnote definition to be kept" + assert "the definition" in footnotes[0].astext() + assert "Duplicate footnote" not in stream.getvalue() + + +def test_footnote_duplicate_definition_warns(): + """A genuinely duplicated footnote definition still warns and is dropped.""" + stream = io.StringIO() + doctree = publish_doctree( + source="[^a]\n\n[^a]: first\n\n[^a]: second\n", + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + assert "Duplicate footnote definition" in stream.getvalue() + assert len(list(doctree.findall(nodes.footnote))) == 1 + + def test_definition_list_orphan_definition(): """A definition with no preceding term errors, but keeps its content. diff --git a/tests/test_renderers/test_fixtures_docutils.py b/tests/test_renderers/test_fixtures_docutils.py index 4988f078..344b34c4 100644 --- a/tests/test_renderers/test_fixtures_docutils.py +++ b/tests/test_renderers/test_fixtures_docutils.py @@ -12,6 +12,7 @@ import pytest from docutils import __version_info__ as docutils_version +from docutils import nodes from docutils.core import Publisher, publish_doctree from pytest_param_files import ParamTestData @@ -64,6 +65,12 @@ def test_link_resolution(file_params: ParamTestData, normalize_doctree_xml): parser=Parser(), settings_overrides=settings, ) + # docutils >=0.23 also inserts info-level (severity 1) system messages + # into the doctree; they are still written to the report stream, + # which is what the fixtures capture, so drop them from the tree + for msg in list(doctree.findall(nodes.system_message)): + if msg["level"] < 2: + msg.parent.remove(msg) outcome = normalize_doctree_xml(doctree.pformat()) if report_stream.getvalue().strip(): outcome += "\n\n" + report_stream.getvalue().strip() From b40d9bdab43092d54f3aead03910460542398987 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Sun, 12 Jul 2026 16:17:44 +0000 Subject: [PATCH 3/4] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20harden=20the=20cr?= =?UTF-8?q?ash=20fixes=20following=20review?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The footnote duplicate check now gates on `document.nametypes`: a label colliding with any *explicit* target name (another footnote, a `(name)=` target, a directive `:name:`, including ones created via eval-rst) keeps the previous drop-with-warning behaviour — letting it through would strip the name from both nodes and break previously working references. Only collisions with *implicit* names (headings — the actual bug) keep the footnote. This is also O(1) per definition, where the set rebuild it replaces was O(n^2) per document (+9.7s at 10k footnotes). - The YAML catches move to a shared `YAML_LOAD_ERRORS` tuple that also covers `RecursionError`: PyYAML's composer recurses per nesting level, so deeply nested front matter (e.g. 500 nested flow sequences) still aborted the build. Also applied to the directive-options YAML path (only reachable with `validate_options=False`, e.g. myst-nb). - Front matter fields are measured before being JSON-serialized for display: YAML anchors/aliases can produce structures whose expanded size is exponential in the source size ("billion laughs"), which previously hung the build in `json.dumps`; self-referential structures raised `ValueError`. Both now warn and skip the field. - Regression tests for each, plus docstring/comment accuracy fixes. --- docs/syntax/code_and_apis.md | 2 +- myst_parser/config/main.py | 6 +- myst_parser/mdit_to_docutils/base.py | 80 +++++++++++++++---- myst_parser/mdit_to_docutils/html_to_nodes.py | 12 ++- myst_parser/parsers/directives.py | 9 ++- myst_parser/warnings_.py | 2 +- tests/test_docutils.py | 67 ++++++++++++++++ .../test_renderers/test_fixtures_docutils.py | 7 +- 8 files changed, 157 insertions(+), 28 deletions(-) diff --git a/docs/syntax/code_and_apis.md b/docs/syntax/code_and_apis.md index aeb3ee21..f24ccef2 100644 --- a/docs/syntax/code_and_apis.md +++ b/docs/syntax/code_and_apis.md @@ -170,7 +170,7 @@ Sphinx and MyST provide means to analyse source code and automatically generate As opposed to `sphinx.ext.autodoc`, `sphinx-autodoc2` performs static (rather than dynamic) analysis of the source code, integrates full package documenting, and also allows for docstrings to be written in both RestructureText and MyST. -The [sphinx-autodoc2's `auto_mode` config variable](https://sphinx-autodoc2.readthedocs.io/en/latest/config.html#confval-autodoc2_packages-auto_mode) (which per default is `True`) will automatically generate the full API documentation, as shown . +[sphinx-autodoc2's `auto_mode` config option](https://sphinx-autodoc2.readthedocs.io/en/latest/config.html#confval-autodoc2_packages-auto_mode) (`True` by default) will automatically generate the full API documentation, as shown . Alternatively, the `autodoc2-object` directive can be used to generate documentation for a single object. To embed in a MyST document the MyST `render_plugin` should be specified, for example: diff --git a/myst_parser/config/main.py b/myst_parser/config/main.py index 59f80ee1..9bf637dc 100644 --- a/myst_parser/config/main.py +++ b/myst_parser/config/main.py @@ -593,6 +593,8 @@ def read_topmatter(text: str | Iterator[str]) -> dict[str, Any] | None: """ import yaml + from myst_parser.parsers.directives import YAML_LOAD_ERRORS + if isinstance(text, str): if not text.startswith("---"): # skip creating the line list in memory return None @@ -609,9 +611,7 @@ def read_topmatter(text: str | Iterator[str]) -> dict[str, Any] | None: top_matter.append(line.rstrip() + "\n") try: metadata = yaml.safe_load("".join(top_matter)) - # ValueError: PyYAML lets it escape for some invalid scalars, - # e.g. out-of-range timestamps like `2021-99-99` - except (yaml.YAMLError, ValueError) as err: + except YAML_LOAD_ERRORS as err: raise TopmatterReadError("Malformed YAML") from err if not isinstance(metadata, dict): raise TopmatterReadError(f"YAML is not a dict: {type(metadata)}") diff --git a/myst_parser/mdit_to_docutils/base.py b/myst_parser/mdit_to_docutils/base.py index ecee2d61..fe292635 100644 --- a/myst_parser/mdit_to_docutils/base.py +++ b/myst_parser/mdit_to_docutils/base.py @@ -46,7 +46,11 @@ MockState, MockStateMachine, ) -from myst_parser.parsers.directives import MarkupError, parse_directive_text +from myst_parser.parsers.directives import ( + YAML_LOAD_ERRORS, + MarkupError, + parse_directive_text, +) from myst_parser.warnings_ import MystWarnings, create_warning from .html_to_nodes import html_to_nodes @@ -1262,9 +1266,7 @@ def render_front_matter(self, token: SyntaxTreeNode) -> None: if isinstance(token.content, str): try: data = yaml.safe_load(token.content) - # ValueError: PyYAML lets it escape for some invalid scalars, - # e.g. out-of-range timestamps like `2021-99-99` - except (yaml.YAMLError, ValueError): + except YAML_LOAD_ERRORS: self.create_warning( "Malformed YAML", MystWarnings.MD_TOPMATTER, @@ -1345,7 +1347,26 @@ def dict_to_fm_field_list( for key, value in data.items(): if not isinstance(value, str | int | float | date | datetime): - value = json.dumps(value) + # YAML anchors/aliases can produce structures whose expanded + # size is exponential in the source size (a "billion laughs" + # bomb), or even self-referential; measure before serializing + if _expanded_length(value) > _FM_FIELD_MAX_LENGTH: + self.create_warning( + f"Front matter field {key!r} is too large to render", + MystWarnings.MD_TOPMATTER, + line=line, + ) + continue + try: + value = json.dumps(value) + except (ValueError, RecursionError): + # e.g. a self-referential structure via a YAML alias + self.create_warning( + f"Front matter field {key!r} could not be serialized", + MystWarnings.MD_TOPMATTER, + line=line, + ) + continue value = str(value) body = nodes.paragraph() body.source, body.line = self.document["source"], line @@ -1507,18 +1528,16 @@ def render_footnote_reference(self, token: SyntaxTreeNode) -> None: """Despite the name, this is actually a footnote definition, e.g. `[^a]: ...`""" target = token.meta["label"] - # check against existing footnote labels only, not `document.nameids`, - # which holds every target name (headings, `(name)=` targets, ...) and - # so would wrongly drop a footnote that merely shares a heading's name - existing_labels = { - name - for footnote in self.document.autofootnotes + self.document.footnotes - for name in footnote["names"] - } - if target in existing_labels: - # note we chose to directly omit these footnotes in the parser, - # rather than let docutils/sphinx handle them, since otherwise you end up with a confusing warning: - # WARNING: Duplicate explicit target name: "x". [docutils] + # A label sharing a name with an existing *explicit* target (another + # footnote, a `(name)=` target, a directive `:name:`, ...) is omitted + # here, rather than left to docutils/sphinx: registering it would + # emit a confusing warning (`Duplicate explicit target name`), + # strip the name from both nodes, and so break previously working + # references to the colliding target. + # A collision with an *implicit* target name (e.g. a section heading) + # keeps the footnote: explicit-over-implicit is standard docutils + # behaviour, reported at INFO level only. + if self.document.nametypes.get(target): # we use [ref.footnote] as the type/subtype, rather than a myst specific warning, # to make it more aligned with sphinx warnings for unreferenced footnotes self.create_warning( @@ -1955,6 +1974,33 @@ def render_substitution(self, token: SyntaxTreeNode, inline: bool) -> None: self.document.sub_references.difference_update(references) +_FM_FIELD_MAX_LENGTH = 100_000 +"""Maximum number of items a front matter field may expand to when rendered.""" + + +def _expanded_length(value: Any, _memo: dict[int, int] | None = None) -> int: + """Return the total number of items ``value`` expands to when serialized. + + Sub-structures are memoized by identity, so structures with shared + references (from YAML anchors/aliases), whose expanded size can be + exponential in the source size, are measured in linear time, + and self-referential structures terminate. + """ + if isinstance(value, dict): + children: Iterable[Any] = value.values() + elif isinstance(value, list | tuple): + children = value + else: + return 1 + if _memo is None: + _memo = {} + key = id(value) + if key not in _memo: + _memo[key] = 1 # occupied marker, in case value contains itself + _memo[key] = 1 + sum(_expanded_length(child, _memo) for child in children) + return _memo[key] + + def html_meta_to_nodes( data: dict[str, Any], document: nodes.document, line: int, reporter: Reporter ) -> list[nodes.meta | nodes.system_message]: diff --git a/myst_parser/mdit_to_docutils/html_to_nodes.py b/myst_parser/mdit_to_docutils/html_to_nodes.py index 46a22365..d8d07b09 100644 --- a/myst_parser/mdit_to_docutils/html_to_nodes.py +++ b/myst_parser/mdit_to_docutils/html_to_nodes.py @@ -82,7 +82,7 @@ def html_to_nodes( return default_html(text, renderer.document["source"], line_number) try: - return _render_nodes(root, line_number, renderer) + return _html_ast_to_nodes(root, line_number, renderer) except RecursionError: msg_node = renderer.create_warning( "HTML is too deeply nested to process", @@ -94,9 +94,17 @@ def html_to_nodes( ) -def _render_nodes( +def _html_ast_to_nodes( root: Element, line_number: int, renderer: DocutilsRenderer ) -> list[nodes.Element]: + """Convert the parsed HTML AST to docutils nodes, + by running the equivalent ``image``/``admonition`` directives. + + Recursion depth scales with the HTML nesting depth + (via ``Element.render``/``Element.strip`` and the nested parse of + directive content, which can re-enter ``html_to_nodes``), + so callers must guard against ``RecursionError``. + """ nodes_list = [] for child in root: if child.name == "img": diff --git a/myst_parser/parsers/directives.py b/myst_parser/parsers/directives.py index 4f4f7ae3..1908d139 100644 --- a/myst_parser/parsers/directives.py +++ b/myst_parser/parsers/directives.py @@ -52,6 +52,13 @@ from .options import TokenizeError, options_to_items +YAML_LOAD_ERRORS = (yaml.YAMLError, ValueError, RecursionError) +"""Errors that ``yaml.safe_load`` can raise on invalid input: +``ValueError`` escapes PyYAML for some invalid scalars +(e.g. out-of-range timestamps like ``2021-99-99``), +and ``RecursionError`` for deeply nested data. +""" + @dataclass class ParseWarnings: @@ -207,7 +214,7 @@ def _parse_directive_options( yaml_errors: list[ParseWarnings] = [] try: yaml_options = yaml.safe_load(options_block or "") or {} - except (yaml.parser.ParserError, yaml.scanner.ScannerError): + except YAML_LOAD_ERRORS: yaml_options = {} yaml_errors.append( ParseWarnings( diff --git a/myst_parser/warnings_.py b/myst_parser/warnings_.py index 54b65991..99b9072f 100644 --- a/myst_parser/warnings_.py +++ b/myst_parser/warnings_.py @@ -61,7 +61,7 @@ class MystWarnings(Enum): STRIKETHROUGH = "strikethrough" """Strikethrough warning, since only implemented in HTML.""" HTML_PARSE = "html" - """HTML could not be parsed.""" + """Issue parsing or converting HTML.""" INVALID_ATTRIBUTE = "attribute" """Invalid attribute value.""" SUBSTITUTION = "substitution" diff --git a/tests/test_docutils.py b/tests/test_docutils.py index ebe14bf3..5bdb6389 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -217,6 +217,8 @@ def test_html_deep_nesting_warns(): Regression: rendering the HTML AST recurses once per nesting level, and the resulting ``RecursionError`` escaped and aborted the build. """ + # rendering uses >=1 stack frame per nesting level, so this depth + # always overflows, whatever the currently available stack depth = sys.getrecursionlimit() # tags on separate lines, to stay under docutils' line-length-limit source = ( @@ -282,6 +284,71 @@ def test_footnote_label_matching_heading_name(): assert "Duplicate footnote" not in stream.getvalue() +def test_footnote_label_matching_explicit_target(): + """A footnote label sharing an *explicit* target's name is dropped. + + Registering it would strip the name from both nodes and so break + previously working references to the target, with confusing docutils + warnings; the pre-existing drop-with-warning behaviour is kept. + """ + stream = io.StringIO() + doctree = publish_doctree( + source="(note)=\n\ntext\n\n[^note]: the definition\n", + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + assert "Duplicate footnote definition" in stream.getvalue() + assert "Duplicate explicit target name" not in stream.getvalue() + assert not list(doctree.findall(nodes.footnote)) + # the original target keeps its name, so references to it still resolve + assert "note" in doctree.nameids + assert doctree.nameids["note"] + + +def test_topmatter_deeply_nested_yaml_warns(): + """Deeply nested front matter YAML warns instead of crashing. + + Regression: PyYAML's composer recurses per nesting level, and the + resulting ``RecursionError`` is neither a ``YAMLError`` nor a + ``ValueError``, so it escaped and aborted the build. + """ + # brackets on separate lines, to stay under docutils' line-length-limit; + # composing uses >=1 stack frame per nesting level, so this depth + # always overflows, whatever the currently available stack + depth = sys.getrecursionlimit() + source = "---\nkey:\n" + "[\n" * depth + "]\n" * depth + "---\n\ncontent\n" + stream = io.StringIO() + doctree = publish_doctree( + source=source, + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + assert "[myst.topmatter]" in stream.getvalue() + assert "content" in doctree.pformat() + + +def test_topmatter_alias_expansion_bomb_warns(): + """A YAML alias-expansion ("billion laughs") bomb warns, not hangs. + + ``yaml.safe_load`` keeps aliased structures shared, so loading is cheap, + but serializing the field for display would expand ``9**20`` items. + """ + lines = ["a0: &a0 [x, x, x, x, x, x, x, x, x]"] + for i in range(1, 20): + refs = ", ".join([f"*a{i - 1}"] * 9) + lines.append(f"a{i}: &a{i} [{refs}]") + source = "---\n" + "\n".join(lines) + "\n---\n\ncontent\n" + stream = io.StringIO() + doctree = publish_doctree( + source=source, + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + assert "too large to render" in stream.getvalue() + assert "[myst.topmatter]" in stream.getvalue() + assert "content" in doctree.pformat() + + def test_footnote_duplicate_definition_warns(): """A genuinely duplicated footnote definition still warns and is dropped.""" stream = io.StringIO() diff --git a/tests/test_renderers/test_fixtures_docutils.py b/tests/test_renderers/test_fixtures_docutils.py index 344b34c4..4e52bb6b 100644 --- a/tests/test_renderers/test_fixtures_docutils.py +++ b/tests/test_renderers/test_fixtures_docutils.py @@ -68,9 +68,10 @@ def test_link_resolution(file_params: ParamTestData, normalize_doctree_xml): # docutils >=0.23 also inserts info-level (severity 1) system messages # into the doctree; they are still written to the report stream, # which is what the fixtures capture, so drop them from the tree - for msg in list(doctree.findall(nodes.system_message)): - if msg["level"] < 2: - msg.parent.remove(msg) + if docutils_version >= (0, 23): + for msg in list(doctree.findall(nodes.system_message)): + if msg["level"] < 2: + msg.parent.remove(msg) outcome = normalize_doctree_xml(doctree.pformat()) if report_stream.getvalue().strip(): outcome += "\n\n" + report_stream.getvalue().strip() From 9748f6c80591ff9d18e736022d2cf098ce5978f3 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Sun, 12 Jul 2026 16:30:57 +0000 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=A7=AA=20Make=20the=20deeply-nested-Y?= =?UTF-8?q?AML=20regression=20test=20load-bearing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nesting must be a single-line flow sequence: spread over multiple lines it fails in PyYAML's (iterative) scanner — already caught before the fix — and never reaches the recursive composer. The recursion limit is pinned so the depth both overflows it and stays under docutils' line-length-limit on any runner. --- tests/test_docutils.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/test_docutils.py b/tests/test_docutils.py index 5bdb6389..8a716d51 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -311,18 +311,25 @@ def test_topmatter_deeply_nested_yaml_warns(): Regression: PyYAML's composer recurses per nesting level, and the resulting ``RecursionError`` is neither a ``YAMLError`` nor a ``ValueError``, so it escaped and aborted the build. + + The nesting must be a single-line flow sequence: spread over lines it + fails in the (iterative) scanner instead, never reaching the composer. """ - # brackets on separate lines, to stay under docutils' line-length-limit; - # composing uses >=1 stack frame per nesting level, so this depth - # always overflows, whatever the currently available stack - depth = sys.getrecursionlimit() - source = "---\nkey:\n" + "[\n" * depth + "]\n" * depth + "---\n\ncontent\n" + # pin the limit so the depth both overflows it and, as a single line, + # stays under docutils' line-length-limit (10000), whatever the runner + depth = 1000 + limit = sys.getrecursionlimit() + source = "---\nkey: " + "[" * depth + "]" * depth + "\n---\n\ncontent\n" stream = io.StringIO() - doctree = publish_doctree( - source=source, - parser=Parser(), - settings_overrides={"warning_stream": stream}, - ) + try: + sys.setrecursionlimit(depth) + doctree = publish_doctree( + source=source, + parser=Parser(), + settings_overrides={"warning_stream": stream}, + ) + finally: + sys.setrecursionlimit(limit) assert "[myst.topmatter]" in stream.getvalue() assert "content" in doctree.pformat()