fix(agents): read three-digit comma groups in currency columns as thousands

agents/anyplot/data/parse.py

@@ -94,7 +94,8 @@
 5. **Type each column**, see `_type_column`: dates, then booleans, integers,
    numbers (also with unambiguous thousands groups: `7.184.003`, `1'234'567`,
    `1 234,5`, `1,234,567`, see `_as_grouped_numbers`; a column of `1,234` shapes reads
-   as comma thousands only beside a comma-grouped column, `_commas_as_thousands`), then the
+   as comma thousands beside a comma-grouped column, or in a currency column of a file
+   that speaks for no decimal comma, `_commas_as_thousands`), then the
    same numbers behind one unit shared by every cell (`65 %`, `15,6 h`, `€ 5`, also
    with a no-break space; the unit leaves the cells), otherwise text. A whole number
    written without a fraction in a number column keeps that form (`87000` beside
@@ -359,6 +360,7 @@
 _NUMBER_PART = r"[+-]?(?:[0-9][0-9.,'’]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?"  # `'` and `’`: Swiss `CHF 1'200`
 _UNIT_SPACE = r"[^\S\r\n\t]?"
 _CURRENCY = r"[€$£]|CHF|EUR|USD|GBP"
+_CURRENCY_UNIT = re.compile(_CURRENCY)
 _TRAILING_UNIT = re.compile(
     rf"(?P<number>{_NUMBER_PART}){_UNIT_SPACE}(?P<unit>%|‰|°[CF]?|[€$£]|[A-Za-zµμ]{{1,4}}[²³]?\.?)"
 )
@@ -2759,8 +2761,9 @@ def _as_grouped_numbers(column: _Column, cells: list[str], present: list[str]) -
             continue
         bare = [cell.replace(mark, "").replace(",", ".") if cell else "" for cell in cells]
         if not (_as_numbers(column, bare) if decimal else _as_integers(column, bare)):
-            if not decimal:  # a whole number past int64: text keeps every digit, and says why
+            if not decimal:  # a whole number past int64: text keeps every digit, and says why; the mark still counts
                 column.notes.append(BIG_INTEGERS)
+                column.group_mark = mark
             return False
         column.decimal_comma = decimal == ","
         column.group_mark = mark
@@ -2775,8 +2778,8 @@ def _id_like(name: str) -> bool:
     return bool(words & ID_WORDS) or any(name.casefold().endswith(suffix) for suffix in ("_id", "-id", "id_"))
 
 
-def _commas_as_thousands(columns: list[_Column]) -> None:
-    """A column of `1,234` shapes reads by what the file's other columns settle, else keeps its decimal comma.
+def _commas_as_thousands(columns: list[_Column], *, semicolon: bool = False) -> None:
+    """A column of `1,234` shapes reads by what the file or its currency settles, else keeps its decimal comma.
 
     A column whose commas could be decimals or thousands (every cell a whole number or `1,234`,
     typed as decimal commas with the `AMBIGUOUS_COMMAS` note) is settled only by strong evidence
@@ -2786,12 +2789,21 @@ def _commas_as_thousands(columns: list[_Column]) -> None:
     * another column writes a decimal comma that is no such shape (`6,07`): the decimal comma
       stays and the note goes, so the root asks no question the file answers (live test of
       2026-10-15, `43,621` Mio. t beside `6,07` t);
+    * the column's own unit is a currency (`_CURRENCY`: `€ $ £ CHF EUR USD GBP`) and every
+      cell is a whole number or a three-digit comma group (`$1,200`, `1,200 USD`): a money
+      amount mostly has at most two decimals, so the column reads as whole numbers with comma
+      thousands, with a note (issue #12224). A file that speaks for a decimal comma keeps the
+      question instead (owner decision of 2026-10-22): `semicolon` (the file is delimited by
+      semicolons, as spreadsheets write it where the comma is the decimal mark), a column of
+      dot groups (`7.184.003`, also one kept as text past int64) or of `1.234` shapes, or a
+      comma column with a zero-led group (`0,125`), since a fuel price of `1,789 €` has three
+      decimals;
     * otherwise, another column groups its thousands with commas unambiguously (`1,234,567`,
       `1,234.5`, `_as_grouped_numbers`): the column reads as whole numbers with comma
       thousands, with a note.
 
-    Without either the decimal comma stays with its note. A column named as an id or a code
-    (`_id_like`) is never retyped.
+    Without any of these the decimal comma stays with its note. A column named as an id or a
+    code (`_id_like`) is never retyped.
     """
     ambiguous = [column for column in columns if column.decimal_comma and AMBIGUOUS_COMMAS in column.notes]
     if not ambiguous:
@@ -2800,15 +2812,43 @@ def _commas_as_thousands(columns: list[_Column]) -> None:
         for column in ambiguous:
             column.notes.remove(AMBIGUOUS_COMMAS)
         return
+    dot_groups = any(
+        column.group_mark == "."
+        or (column.dtype == "number" and not column.decimal_comma and not column.id_like and _dot_groups_only(column))
+        for column in columns
+    )
+    # `0,125` has the `1,234` shape but is no thousands group, so its comma is a decimal comma.
+    zero_led = any(not all(_COMMA_GROUP.fullmatch(cell) for cell in column.bare if cell) for column in ambiguous)
+    if not (semicolon or dot_groups or zero_led):
+        for column in ambiguous:
+            if column.unit is not None and _CURRENCY_UNIT.fullmatch(column.unit):
+                _retype_thousands_commas(
+                    column, "commas read as thousands separators, as a currency amount has at most two decimals"
+                )
     if not any(column.group_mark == "," for column in columns):
         return
     for column in ambiguous:
-        if column.id_like or not all(_COMMA_GROUP.fullmatch(cell) for cell in column.bare if cell):
-            continue
-        if _as_integers(column, [cell.replace(",", "") for cell in column.bare]):
-            column.decimal_comma = False
-            column.notes.remove(AMBIGUOUS_COMMAS)
-            column.notes.append("commas read as thousands separators, as in the file's other columns")
+        if AMBIGUOUS_COMMAS in column.notes:
+            _retype_thousands_commas(column, "commas read as thousands separators, as in the file's other columns")
+
+
+def _retype_thousands_commas(column: _Column, note: str) -> None:
+    """Retype a column of `1,234` shapes as whole numbers with comma thousands, with `note`.
+
+    A column named as an id or a code (`_id_like`) is never retyped. A whole number past int64
+    makes the column text with the `BIG_INTEGERS` note, so no digit is lost.
+    """
+    if column.id_like or not all(_COMMA_GROUP.fullmatch(cell) for cell in column.bare if cell):
+        return
+    column.decimal_comma = False
+    column.notes.remove(AMBIGUOUS_COMMAS)
+    if not _as_integers(column, [cell.replace(",", "") for cell in column.bare]):
+        # Every cell is a whole number (`_COMMA_GROUP`), one past int64: text keeps every digit, as rule 3 does.
+        column.dtype, column.cells, column.numbers, column.unit = "text", list(column.source), [], None
+        column.rounded = column.respelled = 0
+        column.notes.append(BIG_INTEGERS)
+        return
+    column.notes.append(note)
 
 
 def _split_units(present: list[str]) -> tuple[str, dict[str, str]] | None:
@@ -2852,10 +2892,7 @@ def _retype_thousands_dots(column: _Column, *, decimal_comma: bool) -> None:
     """In a file with decimal commas, or with a column of unambiguous dot groups (`7.184.003`), a dot-number
     column of `1.234` shapes holds thousands separators; `decimal_comma` says the file writes decimal commas.
     A column named as an id, a code or a version (`_id_like`) is never retyped."""
-    present = [cell for cell in column.bare if cell]
-    if column.id_like or not (
-        all(_THOUSANDS_DOT.fullmatch(cell) for cell in present) and any("." in cell for cell in present)
-    ):
+    if column.id_like or not _dot_groups_only(column):
         return
     cells = [cell.replace(".", "") for cell in column.bare]
     if not _as_integers(column, cells):
@@ -2868,6 +2905,12 @@ def _retype_thousands_dots(column: _Column, *, decimal_comma: bool) -> None:
     column.notes.append("dots read as thousands separators, as in the file's other columns")
 
 
+def _dot_groups_only(column: _Column) -> bool:
+    """Whether every cell is a whole number or a `1.234` shape (`_THOUSANDS_DOT`), with at least one dot."""
+    present = [cell for cell in column.bare if cell]
+    return all(_THOUSANDS_DOT.fullmatch(cell) for cell in present) and any("." in cell for cell in present)
+
+
 def _as_integers(column: _Column, cells: list[str]) -> bool:
     values = [int(cell) if cell else None for cell in cells]
     if any(value is not None and not _INT64_MIN <= value <= _INT64_MAX for value in values):
@@ -3286,7 +3329,7 @@ def _build(
             f"{_count(marked, 'cell')} of number or date columns matched the missing-value mark the notes declare "
             "and are empty"
         )
-    _commas_as_thousands(columns)
+    _commas_as_thousands(columns, semicolon=source_format == "semicolon")
     decimal_comma = any(column.decimal_comma for column in columns)
     if decimal_comma or any(column.group_mark == "." for column in columns):
         for column in columns:

agents/anyplot/data/warning_codes.py

@@ -179,7 +179,11 @@
             _COLUMN + r"commas read as decimal commas; remove them first if they separate thousands$",
             "decimal_commas_ambiguous",
         ),
-        (_COLUMN + r".* read as thousands separators(?:, as in the file's other columns)?$", "thousands_separators"),
+        (
+            _COLUMN
+            + r".* read as thousands separators(?:, as (?:in the file's other columns|a currency amount has at most two decimals))?$",
+            "thousands_separators",
+        ),
         (
             _COLUMN + r"times with the UTC offsets .* written in UTC \(\+00:00\); the instants are the same$",
             "utc_offsets_normalised",

changelog.d/currency-thousands-commas.md

@@ -0,0 +1,10 @@
+### Fixed
+
+- **Currency columns read three-digit comma groups as thousands.** A pasted
+  price column such as `"$1,200"`, `"$950"`, `"$12,500"` (also `1,200 USD`,
+  `€ 1,200`, `CHF 12,500`) read as 1.2, 950 and 12.5; it now reads as 1200,
+  950 and 12500 with a warning, since a money amount mostly has at most two
+  decimals. A cell with a one- or two-digit fraction (`$1,25`) keeps the
+  decimal comma, and a file that speaks for decimal commas keeps the question:
+  one delimited by semicolons, or with a column of dot groups or of `1.234`
+  shapes, so a fuel price of `1,789 €` stays 1.789. (#12228)

tests/unit/agents/data/test_parse.py

@@ -1859,6 +1859,124 @@ def test_one_comma_group_beside_a_comma_grouped_column_is_thousands(self) -> Non
         assert warned(parsed, "column 'population': commas read as thousands separators, as in the file's other")
         assert not warned(parsed, "commas read as decimal commas")
 
+    def test_a_currency_column_of_three_digit_comma_groups_is_thousands(self) -> None:
+        alone = parse_dataset('price\n"$1,200"\n"$950"\n"$12,500"\n')
+        beside = parse_dataset('price,rating\n"$1,200",4.5\n"$950",3.8\n"$12,500",4.9\n')
+
+        assert alone.column_dtypes == {"price ($)": "integer"}
+        assert alone.csv == "price ($)\n1200\n950\n12500\n"
+        assert alone.profile.decimal == "."
+        assert read_back(alone)["price ($)"].tolist() == [1200, 950, 12500]
+        assert warned(
+            alone, "column 'price ($)': commas read as thousands separators, as a currency amount has at most two"
+        )
+        assert not warned(alone, "commas read as decimal commas")
+        assert beside.csv == "price ($),rating\n1200,4.5\n950,3.8\n12500,4.9\n"
+        assert warned(
+            beside, "column 'price ($)': commas read as thousands separators, as a currency amount has at most two"
+        )
+        assert not warned(beside, "commas read as decimal commas")
+
+    @pytest.mark.parametrize(
+        ("cells", "unit"),
+        [
+            (['"1,200 USD"', '"950 USD"', '"12,500 USD"'], "USD"),
+            (['"€ 1,200"', '"€ 950"', '"€ 12,500"'], "€"),
+            (['"CHF 1,200"', '"CHF 950"', '"CHF 12,500"'], "CHF"),
+        ],
+    )
+    def test_other_currencies_read_three_digit_comma_groups_as_thousands(self, cells: list[str], unit: str) -> None:
+        rows = "".join(f"{cell},{rating}\n" for cell, rating in zip(cells, ["4.5", "3.8", "4.9"], strict=True))
+        parsed = parse_dataset("price,rating\n" + rows)
+        name = f"price ({unit})"
+
+        assert parsed.column_dtypes[name] == "integer"
+        assert read_back(parsed)[name].tolist() == [1200, 950, 12500]
+        assert read_back(parsed)["rating"].tolist() == [4.5, 3.8, 4.9]
+        assert warned(parsed, f"column '{name}': commas read as thousands separators, as a currency")
+        assert not warned(parsed, "commas read as decimal commas")
+
+    @pytest.mark.parametrize(
+        ("cells", "expected"),
+        [
+            (['"$1,25"', '"$3,50"'], [1.25, 3.5]),  # two-digit fractions: decimal commas
+            (['"$1,200"', '"$3,50"'], [1.2, 3.5]),  # a short fraction in the column keeps decimal commas
+            (['"$1,2000"', '"$950"'], [1.2, 950.0]),  # a four-digit group: not a thousands group
+            (['"$0,125"', '"$1,250"'], [0.125, 1.25]),  # a zero-leading group: not a thousands group
+        ],
+    )
+    def test_currency_commas_that_are_not_three_digit_groups_stay_decimal_commas(
+        self, cells: list[str], expected: list[float]
+    ) -> None:
+        parsed = parse_dataset("price\n" + "\n".join(cells) + "\n")
+
+        assert parsed.column_dtypes == {"price ($)": "number"}
+        assert parsed.profile.decimal == ","
+        assert read_back(parsed)["price ($)"].tolist() == expected
+        assert not warned(parsed, "commas read as thousands separators")
+
+    def test_a_semicolon_file_with_decimal_commas_is_unchanged(self) -> None:
+        parsed = parse_dataset("Preis;Menge\n3,50;1.200\n")
+
+        assert parsed.csv == "Preis,Menge\n3.5,1200\n"
+        assert parsed.profile.decimal == ","
+
+    @pytest.mark.parametrize(
+        ("text", "name", "expected", "asks"),
+        [
+            # a decimal-comma column settles the comma (a fuel receipt: `1,789 €` per litre)
+            (
+                "Datum;Preis;Liter\n01.10.2026;1,789 €;45,5\n02.10.2026;1,759 €;30,2\n",
+                "Preis (€)",
+                [1.789, 1.759],
+                False,
+            ),
+            ('a,b\n"$1,200","$1,25"\n"$950","$2,50"\n', "a ($)", [1.2, 950.0], False),
+            # a semicolon file is written where the comma is the decimal mark
+            ("Datum;Preis\n01.10.2026;1,789 €\n02.10.2026;1,759 €\n", "Preis (€)", [1.789, 1.759], True),
+            # a column of dot groups or `1.234` shapes speaks for dot thousands
+            ('price,population\n"€ 1,200","7.184.003"\n"€ 950","7.226.391"\n', "price (€)", [1.2, 950.0], True),
+            ('price,weight\n"$1,200",1.250\n"$950",2.500\n', "price ($)", [1.2, 950.0], True),
+            # dot groups kept as text past int64 still speak for dot thousands
+            (
+                'price,total\n"€ 1,789",12.345.678.901.234.567.890.123\n"€ 1,759",7.226.391\n',
+                "price (€)",
+                [1.789, 1.759],
+                True,
+            ),
+            # `0,125` is no thousands group, so its comma is a decimal comma
+            ("Preis\tLiter\n1,789 €\t0,125\n1,759 €\t0,250\n", "Preis (€)", [1.789, 1.759], True),
+        ],
+    )
+    def test_a_file_that_speaks_for_decimal_commas_keeps_them_in_a_currency_column(
+        self, text: str, name: str, expected: list[float], asks: bool
+    ) -> None:
+        parsed = parse_dataset(text)
+
+        assert parsed.column_dtypes[name] == "number"
+        assert read_back(parsed)[name].tolist() == expected
+        assert not warned(parsed, "as a currency amount")
+        assert warned(parsed, f"column '{name}': commas read as decimal commas") is asks
+
+    def test_an_id_or_oversized_currency_comma_column_is_not_retyped_as_integers(self) -> None:
+        code = parse_dataset('Code\n"$1,200"\n"$2,500"\n')
+        huge = parse_dataset('price\n"$1,200"\n"$12345678901234567890123"\n')
+
+        assert code.column_dtypes == {"Code ($)": "number"}
+        assert read_back(code)["Code ($)"].tolist() == [1.2, 2.5]
+        assert huge.column_dtypes == {"price": "text"}
+        assert huge.csv == 'price\n"$1,200"\n$12345678901234567890123\n'
+        assert f"column 'price': {parse_module.BIG_INTEGERS}" in huge.warnings
+        assert not warned(huge, "commas read as decimal commas")
+
+    def test_an_oversized_comma_column_beside_comma_groups_stays_text(self) -> None:
+        parsed = parse_dataset('a,b\n"1,234","1,234,567"\n9223372036854775808,"2,000"\n')
+
+        assert parsed.column_dtypes["a"] == "text"
+        assert read_back(parsed)["a"].tolist() == ["1,234", "9223372036854775808"]
+        assert f"column 'a': {parse_module.BIG_INTEGERS}" in parsed.warnings
+        assert not warned(parsed, "commas read as decimal commas")
+
     def test_one_dot_group_beside_a_dot_grouped_column_is_thousands(self) -> None:
         parsed = parse_dataset("a;b;c\nx;7.184.003;1.234\ny;7.226.391;2.500\nz;7.278.744;980\n")
 

tests/unit/agents/data/test_warning_codes.py

@@ -88,6 +88,9 @@
         "decimal_commas_ambiguous"
     ),
     "column 'x': dots read as thousands separators, as in the file's other columns": "thousands_separators",
+    "column 'x': commas read as thousands separators, as a currency amount has at most two decimals": (
+        "thousands_separators"
+    ),
     "column 'x': commas read as thousands separators": "thousands_separators",
     "column 't': times with the UTC offsets +01:00, +02:00 written in UTC (+00:00); the instants are the same": (
         "utc_offsets_normalised"