diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py index 331b0712a..a957803b3 100644 --- a/packages/markitdown/src/markitdown/converters/_csv_converter.py +++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py @@ -19,6 +19,58 @@ _PIPE_ESCAPE_RE = re.compile(r"(? int: + """Columns per row under `delimiter`, or 0 when the rows disagree. + + A separator the file was not written with either does not occur at all (one + column) or occurs by accident, and then the rows do not line up. Requiring + the same count on every row is what keeps a comma inside a sentence from + being read as a separator. + """ + count = 0 + reader = csv.reader(io.StringIO(content, newline=""), delimiter=delimiter) + for index, row in enumerate(reader): + if index >= _DETECTION_SAMPLE_ROWS: + break + if not row: # a blank line says nothing about the separator + continue + if count and len(row) != count: + return 0 + count = len(row) + return count if count > 1 else 0 + + +def _detect_delimiter(content: str) -> str: + """The separator the file was written with, defaulting to a comma. + + Parsing a semicolon-separated export with a comma does not fail -- it yields + one column holding the whole row, separators and all. + """ + sample = content[:_DETECTION_SAMPLE_CHARS] + best_delimiter, best_columns = ",", 0 + for delimiter in _CANDIDATE_DELIMITERS: + columns = _consistent_column_count(sample, delimiter) + if columns > best_columns: + best_delimiter, best_columns = delimiter, columns + return best_delimiter + + def _escape_table_cell(value: str) -> str: r"""Escape a CSV value so it is safe inside a Markdown table cell. @@ -99,8 +151,16 @@ def convert( # it does not end up inside the first header cell. content = content.lstrip("\ufeff") + # A "sep=" line declares the separator and is not part of the table. + directive = _SEP_DIRECTIVE_RE.match(content) + if directive: + delimiter = directive.group(1) + content = content[directive.end() :] + else: + delimiter = _detect_delimiter(content) + # Parse CSV content - reader = csv.reader(io.StringIO(content, newline="")) + reader = csv.reader(io.StringIO(content, newline=""), delimiter=delimiter) rows = list(reader) _trim_outer_blank_rows(rows) diff --git a/packages/markitdown/tests/test_csv_delimiter.py b/packages/markitdown/tests/test_csv_delimiter.py new file mode 100644 index 000000000..342ed55cb --- /dev/null +++ b/packages/markitdown/tests/test_csv_delimiter.py @@ -0,0 +1,88 @@ +"""A .csv file is not always comma separated. + +Excel writes the list separator of the machine's locale -- a semicolon across +most of Europe -- and a tab separated export is routinely saved as .csv. +Parsing either one with a comma does not fail: it yields a single column holding +the whole row, separators and all. +""" + +import io + +import pytest + +from markitdown import MarkItDown, StreamInfo + + +def _convert(content: bytes) -> str: + return ( + MarkItDown(enable_plugins=False) + .convert_stream( + io.BytesIO(content), + stream_info=StreamInfo(extension=".csv", charset="utf-8"), + ) + .markdown + ) + + +_EXPECTED = ( + "| Name | Region | Units |\n" + "| --- | --- | --- |\n" + "| Widget | EU | 12 |\n" + "| Gadget | US | 7 |" +) + + +@pytest.mark.parametrize("separator", [",", ";", "\t"]) +def test_the_separator_the_file_was_written_with_is_used(separator: str) -> None: + rows = ["Name;Region;Units", "Widget;EU;12", "Gadget;US;7"] + content = "\n".join(row.replace(";", separator) for row in rows) + "\n" + + assert _convert(content.encode("utf-8")) == _EXPECTED + + +def test_a_sep_directive_sets_the_separator_and_is_not_a_row() -> None: + """Excel honours a leading `sep=` line and does not show it.""" + content = b"sep=|\nName|Region|Units\nWidget|EU|12\nGadget|US|7\n" + + assert _convert(content) == _EXPECTED + + +def test_a_sep_directive_survives_a_bom() -> None: + content = "sep=;\nName;Region;Units\nWidget;EU;12\nGadget;US;7\n" + + assert _convert(content.encode("utf-8")) == _EXPECTED + + +def test_a_separator_inside_a_quoted_field_is_not_a_separator() -> None: + content = b'Name,Note\nWidget,"a; b; c"\nGadget,"d; e; f"\n' + + assert _convert(content) == ( + "| Name | Note |\n" + "| --- | --- |\n" + "| Widget | a; b; c |\n" + "| Gadget | d; e; f |" + ) + + +def test_a_single_column_file_stays_a_single_column() -> None: + """Guard: a separator that does not line up across the rows is not one.""" + content = b"Note\na; b\nc; d\n" + + assert _convert(content) == "| Note |\n| --- |\n| a; b |\n| c; d |" + + +def test_a_ragged_comma_file_is_unchanged() -> None: + """Guard: the existing padding behaviour for uneven rows still applies.""" + content = b"name,value\nAlice,1\n\nBob,2,extra\n" + + assert _convert(content) == ( + "| name | value | |\n" + "| --- | --- | --- |\n" + "| Alice | 1 | |\n" + "| | | |\n" + "| Bob | 2 | extra |" + ) + + +def test_an_empty_file_is_unchanged() -> None: + assert _convert(b"") == ""