diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2bebf04d..824737cc 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -96,7 +96,7 @@ Ready to contribute? Here's how to set up `json2xml` for local development. # Or individually: $ ruff check json2xml tests - $ mypy json2xml tests + $ uvx ty check json2xml tests $ pytest tests/ 6. Commit your changes and push your branch to GitHub:: diff --git a/Makefile b/Makefile index ad9f7510..c4d78472 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ endef export PRINT_HELP_PYSCRIPT BROWSER := python -c "$$BROWSER_PYSCRIPT" +UV_RUN := uv run --locked --extra dev help: @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) @@ -51,19 +52,19 @@ clean-test: ## remove test and coverage artifacts rm -fr coverage/ lint: ## check style with ruff - ruff check json2xml tests + $(UV_RUN) ruff check json2xml tests lint-fix: ## automatically fix ruff issues - ruff check --fix json2xml tests + $(UV_RUN) ruff check --fix json2xml tests typecheck: ## check types with ty - uvx ty check json2xml tests + $(UV_RUN) --with ty ty check json2xml tests -test: ## run tests quickly with the default Python - pytest --cov=json2xml --cov-report=xml:coverage/reports/coverage.xml --cov-report=term --cov-fail-under=100 -xvs tests +test: ## run tests with the locked development environment + $(UV_RUN) pytest --cov=json2xml --cov-report=xml:coverage/reports/coverage.xml --cov-report=term --cov-fail-under=100 -xvs tests test-simple: ## run tests without coverage - pytest -vv tests + $(UV_RUN) pytest -vv tests test-rust: ## run Rust tests cd rust && cargo test diff --git a/README.rst b/README.rst index f21730a3..4d254ae1 100644 --- a/README.rst +++ b/README.rst @@ -258,7 +258,11 @@ boolean ``True`` or choose a smaller limit: Custom Wrappers and Indentation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -By default, a wrapper `all` and pretty `True` is set. However, you can easily change this in your code like this: +By default, a wrapper `all` and compact output (``pretty=False``) are set. Pretty printing can be enabled explicitly: + +Conversions also default to a nesting limit of 100, an item limit of 100,000, +and a 10 MiB XML output limit. Pass ``max_depth``, ``max_items``, or +``max_output_bytes`` to choose smaller budgets for untrusted workloads. .. code-block:: python @@ -425,7 +429,7 @@ Using Make (recommended): make test # Run tests with coverage make lint # Run linting with ruff - make typecheck # Run type checking with mypy + make typecheck # Run type checking with ty make check-all # Run all checks (lint, typecheck, test) Using the development script: @@ -443,7 +447,7 @@ Using tools directly: pytest --cov=json2xml --cov-report=term -xvs tests -n auto ruff check json2xml tests - mypy json2xml tests + uvx ty check json2xml tests **Rust Extension Development** @@ -528,7 +532,7 @@ The ``json2xml-py`` command-line tool provides an easy way to convert JSON to XM Conversion Options: -w, --wrapper string Wrapper element name (default "all") -r, --root Include root element (default true) - -p, --pretty Pretty print output (default true) + -p, --pretty Pretty print output (default false) -t, --type Include type attributes (default true) -i, --item-wrap Wrap list items in elements (default true) -x, --xpath Use XPath 3.1 json-to-xml format diff --git a/docs/usage.rst b/docs/usage.rst index 023c2f0c..2679f099 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -100,16 +100,22 @@ The ``Json2xml`` class accepts the following parameters: * ``data`` - The JSON data (dict or list) to convert * ``wrapper`` (default: ``"all"``) - Custom root element name * ``root`` (default: ``True``) - Whether to include the XML declaration and root element -* ``pretty`` (default: ``True``) - Whether to pretty-print the XML output +* ``pretty`` (default: ``False``) - Whether to pretty-print the XML output * ``attr_type`` (default: ``True``) - Whether to include type attributes on elements * ``item_wrap`` (default: ``True``) - Whether to wrap list items in ```` tags * ``xpath_format`` (default: ``False``) - Whether to use XPath 3.1 compliant output format +* ``max_depth`` (default: ``100``) - Maximum JSON container nesting depth +* ``max_items`` (default: ``100000``) - Maximum number of JSON values and containers +* ``max_output_bytes`` (default: ``10485760``) - Maximum UTF-8 XML output size + +All three conversion limits must be positive integers. They apply to both compact and +pretty output; callers may choose smaller limits for untrusted workloads. Custom Wrappers and Indentation ------------------------------- -By default, a wrapper ``all`` and ``pretty=True`` is set. You can customize these: +By default, a wrapper ``all`` and compact output (``pretty=False``) are set. Pretty printing can be enabled explicitly: .. code-block:: python diff --git a/json2xml/cli.py b/json2xml/cli.py index 258acfbd..97ba4517 100644 --- a/json2xml/cli.py +++ b/json2xml/cli.py @@ -8,7 +8,7 @@ Flags: -w, --wrapper string Wrapper element name (default "all") -r, --root Include root element (default true) - -p, --pretty Pretty print output (default true) + -p, --pretty Pretty print output (default false) -t, --type Include type attributes (default true) -i, --item-wrap Wrap list items in elements (default true) -x, --xpath Use XPath 3.1 json-to-xml format @@ -295,8 +295,8 @@ def create_parser() -> argparse.ArgumentParser: "--pretty", dest="pretty", action="store_true", - default=True, - help="Pretty print output (default: true)", + default=False, + help="Pretty print output (default: false)", ) conv_group.add_argument( "--no-pretty", diff --git a/json2xml/json2xml.py b/json2xml/json2xml.py index 44741ca6..585b36f2 100644 --- a/json2xml/json2xml.py +++ b/json2xml/json2xml.py @@ -1,11 +1,143 @@ +from collections.abc import Mapping, Sequence from typing import Any -__lazy_modules__ = ["defusedxml.minidom", "pyexpat"] - from . import dicttoxml_fast as dicttoxml from .types import JSONValue from .utils import InvalidDataError +DEFAULT_MAX_DEPTH = 100 +DEFAULT_MAX_ITEMS = 100_000 +DEFAULT_MAX_OUTPUT_BYTES = 10 * 1024 * 1024 + + +def _positive_limit(name: str, value: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{name} must be a positive integer") + return value + + +def _validate_conversion_budget( + data: JSONValue, max_depth: int, max_items: int, max_output_bytes: int +) -> None: + """Reject inputs whose structure or conservative encoded size exceeds a limit.""" + stack: list[tuple[Any, int]] = [(data, 0)] + items = 0 + estimated_bytes = 128 + while stack: + value, depth = stack.pop() + items += 1 + if items > max_items: + raise InvalidDataError("JSON item limit exceeded") + if depth > max_depth: + raise InvalidDataError("JSON nesting depth limit exceeded") + if isinstance(value, Mapping): + estimated_bytes += 256 * len(value) + for key, child in value.items(): + estimated_bytes += 6 * len(str(key).encode("utf-8")) + stack.append((child, depth + 1)) + elif isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + estimated_bytes += 128 * len(value) + stack.extend((child, depth + 1) for child in value) + else: + estimated_bytes += 6 * len(str(value).encode("utf-8")) + 128 + if estimated_bytes > max_output_bytes: + raise InvalidDataError("XML output size limit exceeded") + + +def _pretty_xml(xml_data: bytes, max_output_bytes: int) -> str: + """Indent generated XML without constructing or reparsing a DOM.""" + text = xml_data.decode("utf-8") + if " position: + tokens.append(text[position:opening]) + if text.startswith("", opening) + closing = terminator + 3 if terminator >= 0 else -1 + elif text.startswith("", opening) + closing = terminator + 3 if terminator >= 0 else -1 + else: + quote: str | None = None + closing = opening + 1 + terminated = False + while closing < len(text): + char = text[closing] + if char in {'"', "'"}: + quote = None if quote == char else char if quote is None else quote + elif char == ">" and quote is None: + closing += 1 + terminated = True + break + closing += 1 + if not terminated: + closing = -1 + if closing < 0 or closing > len(text): + raise InvalidDataError("Malformed XML generated") + tokens.append(text[opening:closing]) + position = closing + + lines: list[str] = [] + depth = 0 + output_bytes = 0 + has_inline_content = False + open_elements: list[str] = [] + for token in tokens: + if not token.startswith("<"): + if token.strip(): + if not open_elements: + raise InvalidDataError("Malformed XML generated") + lines[-1] += token + output_bytes += len(token.encode("utf-8")) + has_inline_content = True + continue + if token.startswith("") or markup or token.startswith("") + if not element_name or token.startswith(" max_output_bytes: + raise InvalidDataError("XML output size limit exceeded") + if open_elements or depth != 0: + raise InvalidDataError("Malformed XML generated") + return "\n".join(lines) + "\n" + # @lat: [[architecture#Core pipeline]] class Json2xml: @@ -15,24 +147,30 @@ class Json2xml: are serialized. :param wrapper: The root element name used when ``root`` is enabled. :param root: Include the XML declaration and root element. - :param pretty: Reparse and indent the serialized XML, returning text instead of bytes. + :param pretty: Indent serialized XML without a DOM, returning text instead of bytes. :param attr_type: Add each value's JSON type as an XML attribute. :param item_wrap: Wrap list members in ```` elements. :param xpath_format: Emit the W3C XPath 3.1 JSON-to-XML representation. :param cdata: Wrap string values in CDATA sections. :param list_headers: Repeat the parent element for nested dictionary items in lists. + :param max_depth: Maximum JSON container nesting depth. + :param max_items: Maximum total number of JSON values and containers. + :param max_output_bytes: Maximum compact or pretty UTF-8 XML size. """ def __init__( self, data: JSONValue = None, wrapper: str = "all", root: bool = True, - pretty: bool = True, + pretty: bool = False, attr_type: bool = True, item_wrap: bool = True, xpath_format: bool = False, cdata: bool = False, list_headers: bool = False, + max_depth: int = DEFAULT_MAX_DEPTH, + max_items: int = DEFAULT_MAX_ITEMS, + max_output_bytes: int = DEFAULT_MAX_OUTPUT_BYTES, ): self.data = data self.pretty = pretty @@ -43,6 +181,9 @@ def __init__( self.xpath_format = xpath_format self.cdata = cdata self.list_headers = list_headers + self.max_depth = _positive_limit("max_depth", max_depth) + self.max_items = _positive_limit("max_items", max_items) + self.max_output_bytes = _positive_limit("max_output_bytes", max_output_bytes) # @lat: [[behavior#Conversion output]] # @lat: [[behavior#Invalid XML payloads]] @@ -51,10 +192,13 @@ def to_xml(self) -> bytes | str | None: :return: Pretty-printed XML text when ``pretty`` is enabled, UTF-8 encoded XML bytes otherwise, or ``None`` when the configured data is ``None``. - :raises InvalidDataError: If serialization rejects the data or pretty-print parsing finds - malformed XML. + :raises InvalidDataError: If a conversion limit is exceeded or serialization/formatting + rejects the data. """ if self.data is not None: + _validate_conversion_budget( + self.data, self.max_depth, self.max_items, self.max_output_bytes + ) try: xml_data = dicttoxml.dicttoxml( self.data, @@ -68,16 +212,9 @@ def to_xml(self) -> bytes | str | None: ) except ValueError as error: raise InvalidDataError from error + if len(xml_data) > self.max_output_bytes: + raise InvalidDataError("XML output size limit exceeded") if self.pretty: - # Keep parser imports off the compact-output path, which returns serializer bytes directly. - from pyexpat import ExpatError - - from defusedxml.minidom import parseString - - try: - result = parseString(xml_data).toprettyxml(encoding="UTF-8").decode() - except ExpatError: - raise InvalidDataError - return result + return _pretty_xml(xml_data, self.max_output_bytes) return xml_data return None diff --git a/lat.md/architecture.md b/lat.md/architecture.md index 1cc3e98b..e68f6a19 100644 --- a/lat.md/architecture.md +++ b/lat.md/architecture.md @@ -6,7 +6,7 @@ This file documents the main execution paths that turn JSON input into XML outpu The standard pipeline reads JSON into Python objects, passes that data through [[json2xml/json2xml.py#Json2xml]], and delegates serialization through the fast backend selector in [[json2xml/dicttoxml_fast.py#dicttoxml]]. -Library callers usually construct [[json2xml/json2xml.py#Json2xml]] with decoded JSON data. CLI callers reach the same conversion path through [[json2xml/cli.py#read_input]], which resolves the input source before creating the converter. Pretty output is produced by reparsing the generated XML so callers get indented text when requested. +Library callers usually construct [[json2xml/json2xml.py#Json2xml]] with decoded JSON data. CLI callers reach the same bounded conversion path through [[json2xml/cli.py#read_input]]. Pretty output is indented lexically without constructing a DOM. ## Conversion engine @@ -38,6 +38,12 @@ The Cargo feature layout separates normal Rust/PyO3 tests from extension-module Release and CI workflows install the pinned Rust toolchain before building wheels or running Rust checks, so hosted runners do not silently use an older default compiler. The macOS release build also provisions Python 3.10 explicitly so maturin emits wheels for the oldest supported interpreter even when runner images omit it. +## Development checks + +Make-based lint, type-check, and Python test targets run through uv's locked development environment so results do not depend on globally installed tools or optional extensions. + +The shared `UV_RUN` command installs the `dev` extra from `uv.lock`. The type-check target overlays `ty`, while test targets use the same isolated dependency set and leave Rust extension integration to its dedicated workflow. + ## Release packaging Package releases keep the Python wrapper and Rust accelerator requirements aligned so optional fast installs receive compatible wheels. diff --git a/lat.md/behavior.md b/lat.md/behavior.md index a1251bc8..058ca2e3 100644 --- a/lat.md/behavior.md +++ b/lat.md/behavior.md @@ -24,9 +24,9 @@ README and docs examples use `pretty=False` for scan-friendly output and avoid h ## Conversion output -Default output includes an XML declaration, wraps content in `all`, pretty prints the document, and annotates elements with their source type unless callers disable those features. +Default output includes an XML declaration, wraps content in `all`, stays compact, and annotates elements with their source type unless callers change those features. -[[json2xml/json2xml.py#Json2xml#to_xml]] calls [[json2xml/dicttoxml_fast.py#dicttoxml]] with the configured wrapper, root, `attr_type`, `item_wrap`, `cdata`, and `list_headers` options. It treats only `None` as absent input, so falsy JSON values still serialize. When `item_wrap=False`, list values repeat the parent tag instead of creating `` children. Pretty output is Unicode text; `pretty=False` returns the serializer's UTF-8 bytes directly. +[[json2xml/json2xml.py#Json2xml#to_xml]] calls [[json2xml/dicttoxml_fast.py#dicttoxml]] with the configured wrapper, root, `attr_type`, `item_wrap`, `cdata`, and `list_headers` options. It treats only `None` as absent input, so falsy JSON values still serialize. Compact output is the safe default and returns the serializer's UTF-8 bytes directly; explicit pretty output is Unicode text. When `item_wrap=False`, list values repeat the parent tag instead of creating `` children. The fast backend selector falls back to the pure Python serializer for root scalar payloads so values like `0`, `false`, and `""` keep the historical `` element inside the configured root wrapper. @@ -40,9 +40,9 @@ When `xpath_format=True`, [[json2xml/dicttoxml.py#dicttoxml]] delegates payload ## Invalid XML payloads -Pretty printing acts as a validation step, because the formatter reparses the generated XML before returning it. +Opt-in pretty printing indents trusted serializer output without constructing a second XML DOM. -[[json2xml/json2xml.py#Json2xml#to_xml]] imports `defusedxml.minidom.parseString` only for pretty output, then reparses before `toprettyxml`. If the generated bytes are not well-formed XML, the converter raises `InvalidDataError` instead of returning broken pretty output. +[[json2xml/json2xml.py#Json2xml#to_xml]] rejects excessive depth, item counts, conservative output estimates, and exact encoded output sizes. Its lexical formatter rejects malformed markup, DTDs, and entities while enforcing the pretty-output byte limit. ## XML output safety diff --git a/lat.md/tests.md b/lat.md/tests.md index cb8929da..8fac266d 100644 --- a/lat.md/tests.md +++ b/lat.md/tests.md @@ -178,6 +178,24 @@ The public `Json2xml` wrapper should delegate through the fast backend selector The public wrapper should return Unicode text for pretty output and UTF-8 bytes for compact output so callers can rely on the documented `to_xml()` type contract. +### Compact output is the safe default + +Default library conversion should return serializer bytes without building a second DOM copy, while pretty printing remains available through an explicit opt-in. + +### Pretty printing rejects unsafe XML constructs + +Opt-in pretty printing should reject an exponential entity-expansion payload before indentation and expose the rejection as the converter's public invalid-data error. + +### Conversion resource limits + +Conversion should reject excessive nesting, item counts, and conservative output estimates before serialization, then enforce the exact byte limit on compact and pretty results. + +Limit validation rejects booleans, non-integers, and non-positive values. Tests cover preflight estimates, exact backend bytes, and indentation added only by pretty output. + +### Pretty printing avoids DOM reparsing + +Pretty output should use bounded lexical indentation over trusted serializer output instead of constructing a second DOM, rejecting unterminated, mismatched, or unclosed markup. + ### Special keys force Python fallback Special dictionary keys such as `@attrs` and `@val` should bypass the Rust callable so the Python serializer can preserve legacy attribute semantics. diff --git a/tests/test_cli.py b/tests/test_cli.py index 7796d518..9cbdcafc 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -421,6 +421,7 @@ def test_create_parser(self) -> None: parser = create_parser() assert parser is not None assert parser.prog == "json2xml-py" + assert parser.parse_args(["-s", "{}"]).pretty is False def test_create_parser_parses_all_args(self) -> None: """Test parser handles all argument combinations.""" diff --git a/tests/test_json2xml.py b/tests/test_json2xml.py index b5ae318c..563d3f89 100644 --- a/tests/test_json2xml.py +++ b/tests/test_json2xml.py @@ -3,13 +3,14 @@ """Tests for `json2xml` package.""" from pyexpat import ExpatError -from typing import Any +from typing import Any, TypedDict from unittest.mock import Mock import pytest import xmltodict from json2xml import json2xml +from json2xml.json2xml import _positive_limit, _pretty_xml from json2xml.utils import ( InvalidDataError, JSONReadError, @@ -19,6 +20,14 @@ ) +class _ConversionLimits(TypedDict, total=False): + """Keyword limits accepted by ``Json2xml`` resource-bound tests.""" + + max_depth: int + max_items: int + max_output_bytes: int + + class TestJson2xml: """Tests for `json2xml` package.""" @@ -74,6 +83,20 @@ def test_json_to_xml_conversion(self) -> None: dict_from_xml = xmltodict.parse(xmldata) assert isinstance(dict_from_xml["all"], dict) + # @lat: [[tests#Conversion behavior#Compact output is the safe default]] + def test_json_to_xml_defaults_to_compact_output( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Default conversion avoids reparsing attacker-controlled output into a DOM.""" + parse_string = Mock(side_effect=AssertionError("pretty parser should not run")) + monkeypatch.setattr("defusedxml.minidom.parseString", parse_string) + + xmldata = json2xml.Json2xml({"name": "Ada"}).to_xml() + + assert isinstance(xmldata, bytes) + assert b'Ada' in xmldata + parse_string.assert_not_called() + def test_json_to_xml_empty_data_conversion(self) -> None: data = None xmldata = json2xml.Json2xml(data).to_xml() @@ -209,15 +232,161 @@ def test_bad_data(self) -> None: json2xml.Json2xml({"bad": decoded}).to_xml() assert pytest_wrapped_e.type == InvalidDataError - def test_pretty_print_parser_errors_are_wrapped( + def test_pretty_print_rejects_malformed_generated_xml( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The streaming formatter rejects unterminated generated markup.""" + monkeypatch.setattr( + "json2xml.json2xml.dicttoxml.dicttoxml", + Mock(return_value=b"", + b"", + b"", + b"" + ) + + result = _pretty_xml(xml, 1_000) + + assert '' in result + assert " " in result + assert "" in result + assert " " in result + + def test_pretty_formatter_rejects_trailing_text_and_unknown_declarations( + self + ) -> None: + """Text outside the root and unsupported declarations cannot be formatted as XML.""" + with pytest.raises(InvalidDataError, match="Malformed XML generated"): + _pretty_xml(b"trailing", 1_000) + with pytest.raises(InvalidDataError, match="Malformed XML generated"): + _pretty_xml(b"", 1_000) + with pytest.raises(InvalidDataError, match="Malformed XML generated"): + _pretty_xml(b"", 1_000) + + @pytest.mark.parametrize( + "unsafe_xml", [b"", b""] + ) + def test_pretty_formatter_rejects_unsafe_declarations( + self, unsafe_xml: bytes + ) -> None: + """DTD and entity declarations are rejected case-insensitively.""" + with pytest.raises(InvalidDataError, match="Unsafe XML declaration rejected"): + _pretty_xml(unsafe_xml.lower(), 1_000) + + # @lat: [[tests#Conversion behavior#Pretty printing avoids DOM reparsing]] + def test_pretty_print_does_not_reparse_a_dom( self, monkeypatch: pytest.MonkeyPatch ) -> None: - """Parser failures preserve the public InvalidDataError contract.""" - parse_string = Mock(side_effect=ExpatError("malformed XML")) + """Pretty output uses bounded lexical indentation without an XML DOM parser.""" + parse_string = Mock(side_effect=AssertionError("DOM parser must not run")) monkeypatch.setattr("defusedxml.minidom.parseString", parse_string) + result = json2xml.Json2xml({"name": "Ada"}, pretty=True).to_xml() + + assert isinstance(result, str) + assert "\n None: + """The bounded formatter rejects exponential entities before expansion.""" + entity_declarations = [''] + for level in range(1, 10): + references = f"&lol{level - 1};" * 10 + entity_declarations.append(f'') + malicious_xml = ( + '' + f'' + '&lol9;' + ).encode() + monkeypatch.setattr( + "json2xml.json2xml.dicttoxml.dicttoxml", + Mock(return_value=malicious_xml), + ) + with pytest.raises(InvalidDataError): - json2xml.Json2xml({"valid": "data"}).to_xml() + json2xml.Json2xml({"valid": "data"}, pretty=True).to_xml() def test_read_boolean_data_from_json(self) -> None: """Test correct return for boolean types."""