From cfb988c08f679ade66d218b3ecb6e554195d04de Mon Sep 17 00:00:00 2001 From: L4XB Date: Wed, 16 Sep 2026 14:39:56 +0200 Subject: [PATCH] fix(csv): read the separator the file was written with A .csv file is not always comma separated. Excel writes the list separator of the machine's locale, which is a semicolon across most of Europe, and a tab-separated export is routinely saved as .csv. Parsing either one with a comma does not fail: it returns one column holding the whole row, separators included, so every column boundary in the file is lost. Detect the separator among comma, semicolon and tab by taking the one that gives the same column count on every row of a sample, keeping the comma when none of them does, so an ambiguous or ragged file is parsed exactly as before. A leading `sep=` line, which Excel writes and honours, sets the separator directly and is no longer emitted as a table row. --- .../markitdown/converters/_csv_converter.py | 62 ++++++++++++- .../markitdown/tests/test_csv_delimiter.py | 88 +++++++++++++++++++ 2 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 packages/markitdown/tests/test_csv_delimiter.py diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py index 331b0712a3..a957803b3f 100644 --- a/packages/markitdown/src/markitdown/converters/_csv_converter.py +++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py @@ -19,6 +19,58 @@ _PIPE_ESCAPE_RE = re.compile(r"(? int: + """Columns per row under `delimiter`, or 0 when the rows disagree. + + A separator the file was not written with either does not occur at all (one + column) or occurs by accident, and then the rows do not line up. Requiring + the same count on every row is what keeps a comma inside a sentence from + being read as a separator. + """ + count = 0 + reader = csv.reader(io.StringIO(content, newline=""), delimiter=delimiter) + for index, row in enumerate(reader): + if index >= _DETECTION_SAMPLE_ROWS: + break + if not row: # a blank line says nothing about the separator + continue + if count and len(row) != count: + return 0 + count = len(row) + return count if count > 1 else 0 + + +def _detect_delimiter(content: str) -> str: + """The separator the file was written with, defaulting to a comma. + + Parsing a semicolon-separated export with a comma does not fail -- it yields + one column holding the whole row, separators and all. + """ + sample = content[:_DETECTION_SAMPLE_CHARS] + best_delimiter, best_columns = ",", 0 + for delimiter in _CANDIDATE_DELIMITERS: + columns = _consistent_column_count(sample, delimiter) + if columns > best_columns: + best_delimiter, best_columns = delimiter, columns + return best_delimiter + + def _escape_table_cell(value: str) -> str: r"""Escape a CSV value so it is safe inside a Markdown table cell. @@ -99,8 +151,16 @@ def convert( # it does not end up inside the first header cell. content = content.lstrip("\ufeff") + # A "sep=" line declares the separator and is not part of the table. + directive = _SEP_DIRECTIVE_RE.match(content) + if directive: + delimiter = directive.group(1) + content = content[directive.end() :] + else: + delimiter = _detect_delimiter(content) + # Parse CSV content - reader = csv.reader(io.StringIO(content, newline="")) + reader = csv.reader(io.StringIO(content, newline=""), delimiter=delimiter) rows = list(reader) _trim_outer_blank_rows(rows) diff --git a/packages/markitdown/tests/test_csv_delimiter.py b/packages/markitdown/tests/test_csv_delimiter.py new file mode 100644 index 0000000000..342ed55cbb --- /dev/null +++ b/packages/markitdown/tests/test_csv_delimiter.py @@ -0,0 +1,88 @@ +"""A .csv file is not always comma separated. + +Excel writes the list separator of the machine's locale -- a semicolon across +most of Europe -- and a tab separated export is routinely saved as .csv. +Parsing either one with a comma does not fail: it yields a single column holding +the whole row, separators and all. +""" + +import io + +import pytest + +from markitdown import MarkItDown, StreamInfo + + +def _convert(content: bytes) -> str: + return ( + MarkItDown(enable_plugins=False) + .convert_stream( + io.BytesIO(content), + stream_info=StreamInfo(extension=".csv", charset="utf-8"), + ) + .markdown + ) + + +_EXPECTED = ( + "| Name | Region | Units |\n" + "| --- | --- | --- |\n" + "| Widget | EU | 12 |\n" + "| Gadget | US | 7 |" +) + + +@pytest.mark.parametrize("separator", [",", ";", "\t"]) +def test_the_separator_the_file_was_written_with_is_used(separator: str) -> None: + rows = ["Name;Region;Units", "Widget;EU;12", "Gadget;US;7"] + content = "\n".join(row.replace(";", separator) for row in rows) + "\n" + + assert _convert(content.encode("utf-8")) == _EXPECTED + + +def test_a_sep_directive_sets_the_separator_and_is_not_a_row() -> None: + """Excel honours a leading `sep=` line and does not show it.""" + content = b"sep=|\nName|Region|Units\nWidget|EU|12\nGadget|US|7\n" + + assert _convert(content) == _EXPECTED + + +def test_a_sep_directive_survives_a_bom() -> None: + content = "sep=;\nName;Region;Units\nWidget;EU;12\nGadget;US;7\n" + + assert _convert(content.encode("utf-8")) == _EXPECTED + + +def test_a_separator_inside_a_quoted_field_is_not_a_separator() -> None: + content = b'Name,Note\nWidget,"a; b; c"\nGadget,"d; e; f"\n' + + assert _convert(content) == ( + "| Name | Note |\n" + "| --- | --- |\n" + "| Widget | a; b; c |\n" + "| Gadget | d; e; f |" + ) + + +def test_a_single_column_file_stays_a_single_column() -> None: + """Guard: a separator that does not line up across the rows is not one.""" + content = b"Note\na; b\nc; d\n" + + assert _convert(content) == "| Note |\n| --- |\n| a; b |\n| c; d |" + + +def test_a_ragged_comma_file_is_unchanged() -> None: + """Guard: the existing padding behaviour for uneven rows still applies.""" + content = b"name,value\nAlice,1\n\nBob,2,extra\n" + + assert _convert(content) == ( + "| name | value | |\n" + "| --- | --- | --- |\n" + "| Alice | 1 | |\n" + "| | | |\n" + "| Bob | 2 | extra |" + ) + + +def test_an_empty_file_is_unchanged() -> None: + assert _convert(b"") == ""