fix(csv): read the separator the file was written with

packages/markitdown/src/markitdown/converters/_csv_converter.py

@@ -19,6 +19,58 @@
 _PIPE_ESCAPE_RE = re.compile(r"(?<!\\)(\\*)\|")
 
 
+# The separators a spreadsheet actually writes into a file named ".csv".
+# Excel writes the list separator of the machine's locale, which is a semicolon
+# across most of Europe, and a tab-separated export is routinely saved as .csv.
+_CANDIDATE_DELIMITERS = (",", ";", "\t")
+
+# Some producers, Excel among them, write a "sep=" line ahead of the header to
+# declare the separator. Excel honours it and hides the line.
+_SEP_DIRECTIVE_RE = re.compile(r"^sep=(.)\r?\n", re.IGNORECASE)
+
+# How much of the file the detection looks at. A separator that holds for the
+# first rows holds for the file; reading all of a large export to decide would
+# not change the answer.
+_DETECTION_SAMPLE_CHARS = 64 * 1024
+_DETECTION_SAMPLE_ROWS = 20
+
+
+def _consistent_column_count(content: str, delimiter: str) -> int:
+    """Columns per row under `delimiter`, or 0 when the rows disagree.
+
+    A separator the file was not written with either does not occur at all (one
+    column) or occurs by accident, and then the rows do not line up. Requiring
+    the same count on every row is what keeps a comma inside a sentence from
+    being read as a separator.
+    """
+    count = 0
+    reader = csv.reader(io.StringIO(content, newline=""), delimiter=delimiter)
+    for index, row in enumerate(reader):
+        if index >= _DETECTION_SAMPLE_ROWS:
+            break
+        if not row:  # a blank line says nothing about the separator
+            continue
+        if count and len(row) != count:
+            return 0
+        count = len(row)
+    return count if count > 1 else 0
+
+
+def _detect_delimiter(content: str) -> str:
+    """The separator the file was written with, defaulting to a comma.
+
+    Parsing a semicolon-separated export with a comma does not fail -- it yields
+    one column holding the whole row, separators and all.
+    """
+    sample = content[:_DETECTION_SAMPLE_CHARS]
+    best_delimiter, best_columns = ",", 0
+    for delimiter in _CANDIDATE_DELIMITERS:
+        columns = _consistent_column_count(sample, delimiter)
+        if columns > best_columns:
+            best_delimiter, best_columns = delimiter, columns
+    return best_delimiter
+
+
 def _escape_table_cell(value: str) -> str:
     r"""Escape a CSV value so it is safe inside a Markdown table cell.
 
@@ -99,8 +151,16 @@ def convert(
         # it does not end up inside the first header cell.
         content = content.lstrip("\ufeff")
 
+        # A "sep=" line declares the separator and is not part of the table.
+        directive = _SEP_DIRECTIVE_RE.match(content)
+        if directive:
+            delimiter = directive.group(1)
+            content = content[directive.end() :]
+        else:
+            delimiter = _detect_delimiter(content)
+
         # Parse CSV content
-        reader = csv.reader(io.StringIO(content, newline=""))
+        reader = csv.reader(io.StringIO(content, newline=""), delimiter=delimiter)
         rows = list(reader)
         _trim_outer_blank_rows(rows)
 

packages/markitdown/tests/test_csv_delimiter.py

@@ -0,0 +1,88 @@
+"""A .csv file is not always comma separated.
+
+Excel writes the list separator of the machine's locale -- a semicolon across
+most of Europe -- and a tab separated export is routinely saved as .csv.
+Parsing either one with a comma does not fail: it yields a single column holding
+the whole row, separators and all.
+"""
+
+import io
+
+import pytest
+
+from markitdown import MarkItDown, StreamInfo
+
+
+def _convert(content: bytes) -> str:
+    return (
+        MarkItDown(enable_plugins=False)
+        .convert_stream(
+            io.BytesIO(content),
+            stream_info=StreamInfo(extension=".csv", charset="utf-8"),
+        )
+        .markdown
+    )
+
+
+_EXPECTED = (
+    "| Name | Region | Units |\n"
+    "| --- | --- | --- |\n"
+    "| Widget | EU | 12 |\n"
+    "| Gadget | US | 7 |"
+)
+
+
+@pytest.mark.parametrize("separator", [",", ";", "\t"])
+def test_the_separator_the_file_was_written_with_is_used(separator: str) -> None:
+    rows = ["Name;Region;Units", "Widget;EU;12", "Gadget;US;7"]
+    content = "\n".join(row.replace(";", separator) for row in rows) + "\n"
+
+    assert _convert(content.encode("utf-8")) == _EXPECTED
+
+
+def test_a_sep_directive_sets_the_separator_and_is_not_a_row() -> None:
+    """Excel honours a leading `sep=` line and does not show it."""
+    content = b"sep=|\nName|Region|Units\nWidget|EU|12\nGadget|US|7\n"
+
+    assert _convert(content) == _EXPECTED
+
+
+def test_a_sep_directive_survives_a_bom() -> None:
+    content = "sep=;\nName;Region;Units\nWidget;EU;12\nGadget;US;7\n"
+
+    assert _convert(content.encode("utf-8")) == _EXPECTED
+
+
+def test_a_separator_inside_a_quoted_field_is_not_a_separator() -> None:
+    content = b'Name,Note\nWidget,"a; b; c"\nGadget,"d; e; f"\n'
+
+    assert _convert(content) == (
+        "| Name | Note |\n"
+        "| --- | --- |\n"
+        "| Widget | a; b; c |\n"
+        "| Gadget | d; e; f |"
+    )
+
+
+def test_a_single_column_file_stays_a_single_column() -> None:
+    """Guard: a separator that does not line up across the rows is not one."""
+    content = b"Note\na; b\nc; d\n"
+
+    assert _convert(content) == "| Note |\n| --- |\n| a; b |\n| c; d |"
+
+
+def test_a_ragged_comma_file_is_unchanged() -> None:
+    """Guard: the existing padding behaviour for uneven rows still applies."""
+    content = b"name,value\nAlice,1\n\nBob,2,extra\n"
+
+    assert _convert(content) == (
+        "| name | value |  |\n"
+        "| --- | --- | --- |\n"
+        "| Alice | 1 |  |\n"
+        "|  |  |  |\n"
+        "| Bob | 2 | extra |"
+    )
+
+
+def test_an_empty_file_is_unchanged() -> None:
+    assert _convert(b"") == ""