From d237e16bd655ef109e3c25ee189bb4f8963fe759 Mon Sep 17 00:00:00 2001 From: Nash Aschenbath Date: Sun, 8 Nov 2026 07:03:25 +0800 Subject: [PATCH] Add a multiline array option to dump and dumps --- CHANGELOG.md | 4 + docs/quickstart.rst | 26 ++++++ tests/test_api_multiline.py | 176 ++++++++++++++++++++++++++++++++++++ tomlkit/api.py | 26 ++++-- tomlkit/container.py | 66 +++++++++----- tomlkit/items.py | 60 ++++++++++-- 6 files changed, 323 insertions(+), 35 deletions(-) create mode 100644 tests/test_api_multiline.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 47bf49b0..c91021f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Add `multiline=True` to `dumps()` and `dump()` to render arrays one item per line without changing the input document. + ### Changed - Speed up repeated array appends by keeping existing indexes instead of rebuilding them. diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 84ef7782..18e66390 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -146,3 +146,29 @@ It can be created with the following code:: >>> database["enabled"] = True >>> doc["database"] = database + + +Multiline arrays +~~~~~~~~~~~~~~~~ + +Pass ``multiline=True`` to ``dumps()`` or ``dump()`` to put each array item on +its own line. This also applies to arrays nested in tables, inline tables, +arrays of tables, and other arrays:: + + >>> data = {"ports": [8001, 8002]} + >>> print(dumps(data, multiline=True), end="") + ports = [ + 8001, + 8002, + ] + >>> print(dumps(data), end="") + ports = [8001, 8002] + +The option uses the same formatting as ``Array.multiline(True)`` without +changing the input document or its arrays. Empty arrays remain ``[]``. The +default ``multiline=False`` preserves existing formatting, including arrays +already marked multiline. The option is a boolean; it does not select arrays +by item count or line width. + +Custom ``as_string()`` methods keep their existing no-argument interface and +control their own output. The option is not forwarded to those methods. diff --git a/tests/test_api_multiline.py b/tests/test_api_multiline.py new file mode 100644 index 00000000..cb115e66 --- /dev/null +++ b/tests/test_api_multiline.py @@ -0,0 +1,176 @@ +import io + +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier +from typing import Any + +import pytest + +from tomlkit import array +from tomlkit import dump +from tomlkit import dumps +from tomlkit import item +from tomlkit import parse +from tomlkit.items import Array +from tomlkit.items import Trivia +from tomlkit.toml_document import TOMLDocument + + +def test_dumps_multiline_mapping_and_dump_stream() -> None: + data = {"z": (1, 2), "a": []} + expected = "a = []\nz = [\n 1,\n 2,\n]\n" + assert dumps(data, True, multiline=True) == expected + stream = io.StringIO() + dump(data, stream, sort_keys=True, multiline=True) + assert stream.getvalue() == expected + assert data == {"z": (1, 2), "a": []} + + +@pytest.mark.parametrize( + "source, expected", + [ + ("a = [1, 2]\n", "a = [\n 1,\n 2,\n]\n"), + ("[t]\na = [1, 2]\n", "[t]\na = [\n 1,\n 2,\n]\n"), + ("a.b = [1, 2]\n", "a.b = [\n 1,\n 2,\n]\n"), + ( + "[[t]]\na = [1, 2]\n[[t]]\na = [3]\n", + "[[t]]\na = [\n 1,\n 2,\n]\n[[t]]\na = [\n 3,\n]\n", + ), + ("a = {b = [1, 2]}\n", "a = {b = [\n 1,\n 2,\n]}\n"), + ("a = {b.c = [1, 2]}\n", "a = {b.c = [\n 1,\n 2,\n]}\n"), + ( + "a = [[1, 2], [3]]\n", + "a = [\n [\n 1,\n 2,\n],\n [\n 3,\n],\n]\n", + ), + ( + "a = [{b = [1, 2]}]\n", + "a = [\n {b = [\n 1,\n 2,\n]},\n]\n", + ), + ( + "[a.b]\nx = [1, 2]\n[z]\nx = [3]\n[a.c]\nx = [4]\n", + "[a.b]\nx = [\n 1,\n 2,\n]\n[z]\nx = [\n 3,\n]\n[a.c]\nx = [\n 4,\n]\n", + ), + ], +) +def test_dumps_multiline_preserves_document(source: str, expected: str) -> None: + doc = parse(source) + assert dumps(doc, multiline=True) == expected + assert parse(expected) == doc + assert dumps(doc) == source + assert dumps(doc, multiline=False) == source + + +def test_dumps_multiline_preserves_comments_and_manual_formatting() -> None: + source = "a = [\n # before\n 1, # one\n # between\n 2 # two\n] # array\n" + doc = parse(source) + original_array = doc["a"] + result = dumps(doc, multiline=True) + assert parse(result) == doc + for comment in ["# before", "# one", "# between", "# two", "# array"]: + assert result.count(comment) == 1 + assert doc["a"] is original_array + assert original_array._multiline is False + assert dumps(doc) == source + + explicit = array("[1, 2]").multiline(True) + assert dumps({"a": explicit}, multiline=False) == "a = [\n 1,\n 2,\n]\n" + assert explicit._multiline is True + + +@pytest.mark.parametrize("multiline", [False, True]) +def test_dumps_multiline_keeps_custom_renderer_signature(multiline: bool) -> None: + class CustomArray(Array): + def as_string(self) -> str: + return "[42]" + + class CustomDocument(TOMLDocument): + def as_string(self) -> str: + return "custom = [42]\n" + + custom = CustomArray([], Trivia()) + assert dumps({"a": custom}, multiline=multiline) == "a = [42]\n" + assert dumps(CustomDocument(), multiline=multiline) == "custom = [42]\n" + + class Wrapper: + def as_string(self) -> str: + return "wrapped = [42]\n" + + assert dumps(Wrapper(), multiline=multiline) == "wrapped = [42]\n" # type: ignore[arg-type] + + +def test_nested_dumps_has_its_own_formatting_option() -> None: + inner_results = [] + + class ReentrantArray(Array): + def as_string(self) -> str: + inner_results.append(dumps({"inner": [7, 8]})) + return "[42]" + + result = dumps( + {"custom": ReentrantArray([], Trivia()), "ordinary": [1, 2]}, multiline=True + ) + assert inner_results == ["inner = [7, 8]\n"] + assert result == "custom = [42]\nordinary = [\n 1,\n 2,\n]\n" + + +def test_failed_multiline_dump_does_not_change_the_input() -> None: + class FailingArray(Array): + def as_string(self) -> str: + raise RuntimeError("render failed") + + shared = array("[1, 2]") + with pytest.raises(RuntimeError, match="render failed"): + dumps({"a": shared, "b": FailingArray([], Trivia())}, multiline=True) + assert shared._multiline is False + assert shared.as_string() == "[1, 2]" + assert dumps({"a": shared}) == "a = [1, 2]\n" + + +def test_concurrent_dumps_can_format_the_same_array_differently() -> None: + barrier = Barrier(2) + + class SynchronizedArray(Array): + def as_string(self) -> str: + barrier.wait(timeout=10) + return "[]" + + shared = array("[1, 2]") + data = {"sync": SynchronizedArray([], Trivia()), "a": shared} + with ThreadPoolExecutor(max_workers=2) as executor: + normal = executor.submit(dumps, data) + multiline = executor.submit(dumps, data, multiline=True) + assert normal.result() == "sync = []\na = [1, 2]\n" + assert multiline.result() == "sync = []\na = [\n 1,\n 2,\n]\n" + assert shared.as_string() == "[1, 2]" + + +def test_multiline_renders_table_and_inline_table_roots() -> None: + table = item({"a": [1, 2]}) + inline = parse("a = {b = [1, 2]}\n")["a"] + assert dumps(table, multiline=True) == "a = [\n 1,\n 2,\n]\n" + assert dumps(inline, multiline=True) == "{b = [\n 1,\n 2,\n]}" + assert dumps(table) == "a = [1, 2]\n" + assert dumps(inline) == "{b = [1, 2]}" + + +def test_multiline_retains_custom_encoder_arrays() -> None: + from tomlkit import register_encoder + from tomlkit import unregister_encoder + from tomlkit.exceptions import ConvertError + + class Value: + pass + + shared = array("[1, 2]") + + def encoder(value: Any) -> Array: + if isinstance(value, Value): + return shared + raise ConvertError + + register_encoder(encoder) + try: + assert dumps({"a": Value()}, multiline=True) == "a = [\n 1,\n 2,\n]\n" + assert shared.as_string() == "[1, 2]" + finally: + unregister_encoder(encoder) diff --git a/tomlkit/api.py b/tomlkit/api.py index d17324e7..333b4bb4 100644 --- a/tomlkit/api.py +++ b/tomlkit/api.py @@ -35,6 +35,7 @@ from tomlkit.items import Time from tomlkit.items import Trivia from tomlkit.items import Whitespace +from tomlkit.items import _render_item from tomlkit.items import item as item from tomlkit.parser import Parser from tomlkit.toml_document import TOMLDocument as TOMLDocument @@ -55,18 +56,24 @@ def loads(string: str | bytes) -> TOMLDocument: return parse(string) -def dumps(data: Mapping[str, Any], sort_keys: bool = False) -> str: +def dumps( + data: Mapping[str, Any], sort_keys: bool = False, *, multiline: bool = False +) -> str: """ Dumps a TOMLDocument into a string. + + :param multiline: render arrays one item per line without modifying the input. + False preserves the existing array formatting. Empty arrays stay empty. + Custom as_string() implementations retain control of their formatting. """ if isinstance(data, (Table, InlineTable, Container)): if not sort_keys: - return data.as_string() + return _render_item(data, multiline) - return item(data, _sort_keys=True).as_string() + return _render_item(item(data, _sort_keys=True), multiline) if isinstance(data, Mapping): - return item(dict(data), _sort_keys=sort_keys).as_string() + return _render_item(item(dict(data), _sort_keys=sort_keys), multiline) try: # mapping-like wrappers (e.g. dotty_dict's Dotty) delegate @@ -86,19 +93,26 @@ def load(fp: IO[str] | IO[bytes]) -> TOMLDocument: return parse(fp.read()) -def dump(data: Mapping[str, Any], fp: IO[str], *, sort_keys: bool = False) -> None: +def dump( + data: Mapping[str, Any], + fp: IO[str], + *, + sort_keys: bool = False, + multiline: bool = False, +) -> None: """ Dump a TOMLDocument into a writable file stream. :param data: a dict-like object to dump :param sort_keys: if true, sort the keys in alphabetic order + :param multiline: render arrays one item per line, as in dumps() :Example: >>> with open("output.toml", "w") as fp: ... tomlkit.dump(data, fp) """ - fp.write(dumps(data, sort_keys=sort_keys)) + fp.write(dumps(data, sort_keys=sort_keys, multiline=multiline)) def parse(string: str | bytes) -> TOMLDocument: diff --git a/tomlkit/container.py b/tomlkit/container.py index 718431a6..bcd32866 100644 --- a/tomlkit/container.py +++ b/tomlkit/container.py @@ -29,6 +29,7 @@ from tomlkit.items import Table from tomlkit.items import Trivia from tomlkit.items import Whitespace +from tomlkit.items import _render_item from tomlkit.items import item as _item @@ -661,6 +662,9 @@ def last_item(self) -> Item | None: def as_string(self) -> str: """Render as TOML string.""" + return self._as_string(multiline=False) + + def _as_string(self, multiline: bool) -> str: s = "" for path, aot in _header_less_aots(self._body): s += _render_empty_aot(path, aot) @@ -675,7 +679,7 @@ def as_string(self) -> str: and "\n" not in v.trivia.indent ): s += "\n" - s += self._render_table(k, v) + s += self._render_table(k, v, multiline=multiline) elif isinstance(v, AoT): if ( s.strip(" ") @@ -683,15 +687,22 @@ def as_string(self) -> str: and "\n" not in v.trivia.indent ): s += "\n" - s += self._render_aot(k, v) + s += self._render_aot(k, v, multiline=multiline) else: - s += self._render_simple_item(k, v) + s += self._render_simple_item(k, v, multiline=multiline) else: - s += self._render_simple_item(k, v) + s += self._render_simple_item(k, v, multiline=multiline) return s - def _render_table(self, key: Key, table: Table, prefix: str | None = None) -> str: + def _render_table( + self, + key: Key, + table: Table, + prefix: str | None = None, + *, + multiline: bool = False, + ) -> str: cur = "" if table.display_name is not None: @@ -743,11 +754,13 @@ def _render_table(self, key: Key, table: Table, prefix: str | None = None) -> st if v.is_super_table(): if k.is_dotted() and not key.is_dotted(): # Dotted key inside table - cur += self._render_table(k, v) + cur += self._render_table(k, v, multiline=multiline) else: - cur += self._render_table(k, v, prefix=_key) + cur += self._render_table( + k, v, prefix=_key, multiline=multiline + ) else: - cur += self._render_table(k, v, prefix=_key) + cur += self._render_table(k, v, prefix=_key, multiline=multiline) elif isinstance(v, AoT): if ( cur.strip(" ") @@ -756,15 +769,17 @@ def _render_table(self, key: Key, table: Table, prefix: str | None = None) -> st ): cur += "\n" assert k is not None - cur += self._render_aot(k, v, prefix=_key) + cur += self._render_aot(k, v, prefix=_key, multiline=multiline) else: cur += self._render_simple_item( - k, v, prefix=_key if key.is_dotted() else None + k, v, prefix=_key if key.is_dotted() else None, multiline=multiline ) return cur - def _render_aot(self, key: Key, aot: AoT, prefix: str | None = None) -> str: + def _render_aot( + self, key: Key, aot: AoT, prefix: str | None = None, *, multiline: bool = False + ) -> str: _key = key.as_string() if prefix is not None: _key = prefix + "." + _key @@ -772,11 +787,13 @@ def _render_aot(self, key: Key, aot: AoT, prefix: str | None = None) -> str: cur = "" _key = decode(_key) for table in aot.body: - cur += self._render_aot_table(table, prefix=_key) + cur += self._render_aot_table(table, prefix=_key, multiline=multiline) return cur - def _render_aot_table(self, table: Table, prefix: str | None = None) -> str: + def _render_aot_table( + self, table: Table, prefix: str | None = None, *, multiline: bool = False + ) -> str: cur = "" _key = prefix or "" open_, close = "[[", "]]" @@ -801,24 +818,31 @@ def _render_aot_table(self, table: Table, prefix: str | None = None) -> str: if v.is_super_table(): if k.is_dotted(): # Dotted key inside table - cur += self._render_table(k, v) + cur += self._render_table(k, v, multiline=multiline) else: - cur += self._render_table(k, v, prefix=_key) + cur += self._render_table( + k, v, prefix=_key, multiline=multiline + ) else: - cur += self._render_table(k, v, prefix=_key) + cur += self._render_table(k, v, prefix=_key, multiline=multiline) elif isinstance(v, AoT): assert k is not None - cur += self._render_aot(k, v, prefix=_key) + cur += self._render_aot(k, v, prefix=_key, multiline=multiline) else: - cur += self._render_simple_item(k, v) + cur += self._render_simple_item(k, v, multiline=multiline) return cur def _render_simple_item( - self, key: Key | None, item: Item, prefix: str | None = None + self, + key: Key | None, + item: Item, + prefix: str | None = None, + *, + multiline: bool = False, ) -> str: if key is None: - return item.as_string() + return _render_item(item, multiline=True) if multiline else item.as_string() _key = key.as_string() if prefix is not None: @@ -828,7 +852,7 @@ def _render_simple_item( f"{item.trivia.indent}" f"{decode(_key)}" f"{key.sep}" - f"{decode(item.as_string())}" + f"{decode(_render_item(item, multiline=True) if multiline else item.as_string())}" f"{item.trivia.comment_ws}" f"{decode(item.trivia.comment)}" f"{item.trivia.trail}" diff --git a/tomlkit/items.py b/tomlkit/items.py index 51eba39e..2327d809 100644 --- a/tomlkit/items.py +++ b/tomlkit/items.py @@ -1560,9 +1560,16 @@ def multiline(self, multiline: bool) -> Array: return self def as_string(self) -> str: - if not self._multiline or not self._value: + return self._as_string(multiline=False) + + def _as_string(self, multiline: bool) -> str: + if not (self._multiline or multiline) or not self._value: rendered = list(self._iter_items()) - inner = "".join(v.as_string() for v in rendered) + inner = ( + "".join(_render_item(v, multiline=True) for v in rendered) + if multiline + else "".join(v.as_string() for v in rendered) + ) if rendered and isinstance(rendered[-1], Comment): # A trailing comment (e.g. from add_line(comment=...)) carries # no newline, so keep the closing bracket off the comment line @@ -1574,7 +1581,11 @@ def as_string(self) -> str: s += "".join( self.trivia.indent + " " * 4 - + v.value.as_string() + + ( + _render_item(v.value, multiline=True) + if multiline + else v.value.as_string() + ) + ("," if not isinstance(v.value, Null) else "") + (v.comment.as_string() if v.comment is not None else "") + "\n" @@ -2164,6 +2175,9 @@ def _validate_child(self, _item: Item) -> None: raise ValueError("Inline tables cannot contain a table") def as_string(self) -> str: + return self._as_string(multiline=False) + + def _as_string(self, multiline: bool) -> str: buf = "{" emitted_key = False needs_separator = False @@ -2244,12 +2258,12 @@ def as_string(self) -> str: # A table materialized from a dotted key renders one # `prefix.child = value` pair per child, so that keys added # after parsing stay attached to the dotted prefix. - rendered = ", ".join(self._render_dotted(k, v)) + rendered = ", ".join(self._render_dotted(k, v, multiline=multiline)) else: rendered = ( f"{k.as_string() + ('.' if k.is_dotted() else '')}" f"{k.sep}" - f"{v.as_string()}" + f"{_render_item(v, multiline=True) if multiline else v.as_string()}" ) buf += f"{v.trivia.indent}{rendered}" last_value_end = len(buf) @@ -2272,7 +2286,9 @@ def as_string(self) -> str: return buf - def _render_dotted(self, key: Key, table: Table) -> list[str]: + def _render_dotted( + self, key: Key, table: Table, *, multiline: bool = False + ) -> list[str]: """Render a table materialized from a dotted key as a list of ``prefix.child = value`` strings, recursing into nested dotted children.""" @@ -2282,11 +2298,15 @@ def _render_dotted(self, key: Key, table: Table) -> list[str]: if k is None: continue if isinstance(v, Table): - parts.extend(f"{prefix}{sub}" for sub in self._render_dotted(k, v)) + parts.extend( + f"{prefix}{sub}" + for sub in self._render_dotted(k, v, multiline=multiline) + ) else: trail = v.trivia.trail.replace("\n", "") parts.append( - f"{prefix}{v.trivia.indent}{k.as_string()}{k.sep}{v.as_string()}" + f"{prefix}{v.trivia.indent}{k.as_string()}{k.sep}" + f"{_render_item(v, multiline=True) if multiline else v.as_string()}" f"{v.trivia.comment_ws}{v.trivia.comment}{trail}" ) if not parts: @@ -2524,3 +2544,27 @@ def as_string(self) -> str: def _getstate(self, protocol: int = 3) -> tuple[()]: return () + + +def _render_item(value: Item | container.Container, multiline: bool) -> str: + """Render built-in items with dump options, preserving custom renderers.""" + renderer = value.as_string + if multiline: + method = getattr(renderer, "__func__", None) + if isinstance(value, (Array, InlineTable)): + if method in (Array.as_string, InlineTable.as_string): + return value._as_string(multiline=True) + elif isinstance(value, Table): + if method is Table.as_string: + return _render_item(value.value, multiline=True) + elif isinstance(value, AoT): + if method is AoT.as_string: + return "".join( + _render_item(table, multiline=True) for table in value.body + ) + else: + from tomlkit.container import Container + + if isinstance(value, Container) and method is Container.as_string: + return value._as_string(multiline=True) + return renderer()