From 670aad16189bc8f6bea9e5b7b75df8731cb93dab Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:14:25 +0200 Subject: [PATCH 1/5] test(agents): add tests for currency columns with three-digit comma groups Step 1 of 3: add failing tests for currency columns whose commas sit in three-digit groups, plus negative cases for short fractions, four-digit groups, zero-leading groups, id-like columns, and integers beyond int64. Refs #12224 Co-Authored-By: Antigravity (Gemini 4 Argon) --- tests/unit/agents/data/test_parse.py | 66 ++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tests/unit/agents/data/test_parse.py b/tests/unit/agents/data/test_parse.py index 0f85253defa..b2fc0454bff 100644 --- a/tests/unit/agents/data/test_parse.py +++ b/tests/unit/agents/data/test_parse.py @@ -1859,6 +1859,72 @@ def test_one_comma_group_beside_a_comma_grouped_column_is_thousands(self) -> Non assert warned(parsed, "column 'population': commas read as thousands separators, as in the file's other") assert not warned(parsed, "commas read as decimal commas") + def test_a_currency_column_of_three_digit_comma_groups_is_thousands(self) -> None: + alone = parse_dataset('price\n"$1,200"\n"$950"\n"$12,500"\n') + beside = parse_dataset('price,rating\n"$1,200",4.5\n"$950",3.8\n"$12,500",4.9\n') + + assert alone.column_dtypes == {"price ($)": "integer"} + assert alone.csv == "price ($)\n1200\n950\n12500\n" + assert alone.profile.decimal == "." + assert read_back(alone)["price ($)"].tolist() == [1200, 950, 12500] + assert warned( + alone, "column 'price ($)': commas read as thousands separators, as a currency amount has at most two" + ) + assert not warned(alone, "commas read as decimal commas") + assert beside.csv == "price ($),rating\n1200,4.5\n950,3.8\n12500,4.9\n" + assert warned( + beside, "column 'price ($)': commas read as thousands separators, as a currency amount has at most two" + ) + assert not warned(beside, "commas read as decimal commas") + + @pytest.mark.parametrize( + ("cells", "unit"), + [ + (['"1,200 USD"', '"950 USD"', '"12,500 USD"'], "USD"), + (['"€ 1,200"', '"€ 950"', '"€ 12,500"'], "€"), + (['"CHF 1,200"', '"CHF 950"', '"CHF 12,500"'], "CHF"), + ], + ) + def test_other_currencies_read_three_digit_comma_groups_as_thousands(self, cells: list[str], unit: str) -> None: + rows = "".join(f"{cell},{rating}\n" for cell, rating in zip(cells, ["4.5", "3.8", "4.9"], strict=True)) + parsed = parse_dataset("price,rating\n" + rows) + name = f"price ({unit})" + + assert parsed.column_dtypes[name] == "integer" + assert read_back(parsed)[name].tolist() == [1200, 950, 12500] + assert warned(parsed, f"column '{name}': commas read as thousands separators, as a currency") + assert not warned(parsed, "commas read as decimal commas") + + @pytest.mark.parametrize( + ("cells", "expected"), + [ + (['"$1,25"', '"$3,50"'], [1.25, 3.5]), # two-digit fractions: decimal commas + (['"$1,200"', '"$3,50"'], [1.2, 3.5]), # a short fraction in the column keeps decimal commas + (['"$1,2000"', '"$950"'], [1.2, 950.0]), # a four-digit group: not a thousands group + (['"$0,125"', '"$1,250"'], [0.125, 1.25]), # a zero-leading group: not a thousands group + ], + ) + def test_currency_commas_that_are_not_three_digit_groups_stay_decimal_commas( + self, cells: list[str], expected: list[float] + ) -> None: + parsed = parse_dataset("price\n" + "\n".join(cells) + "\n") + + assert parsed.column_dtypes == {"price ($)": "number"} + assert parsed.profile.decimal == "," + assert read_back(parsed)["price ($)"].tolist() == expected + assert not warned(parsed, "commas read as thousands separators") + + def test_an_id_or_oversized_currency_comma_column_is_not_retyped_as_integers(self) -> None: + code = parse_dataset('Code\n"$1,200"\n"$2,500"\n') + huge = parse_dataset('price\n"$1,200"\n"$12345678901234567890123"\n') + + assert code.column_dtypes == {"Code ($)": "number"} + assert read_back(code)["Code ($)"].tolist() == [1.2, 2.5] + assert huge.column_dtypes == {"price": "text"} + assert huge.csv == 'price\n"$1,200"\n$12345678901234567890123\n' + assert f"column 'price': {parse_module.BIG_INTEGERS}" in huge.warnings + assert not warned(huge, "commas read as decimal commas") + def test_one_dot_group_beside_a_dot_grouped_column_is_thousands(self) -> None: parsed = parse_dataset("a;b;c\nx;7.184.003;1.234\ny;7.226.391;2.500\nz;7.278.744;980\n") From ae5a78cea050f77419f7f04829daef7600200900 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:27:19 +0200 Subject: [PATCH 2/5] fix(agents): read three-digit comma groups in currency columns as thousands Step 2 of 3: settle an AMBIGUOUS_COMMAS column whose unit matches _CURRENCY and whose bare cells all match _COMMA_GROUP as whole numbers with comma thousands. Refs #12224 Co-Authored-By: Antigravity (Gemini 4 Argon) --- agents/anyplot/data/parse.py | 34 +++++++++++++++----- agents/anyplot/data/warning_codes.py | 6 +++- tests/unit/agents/data/test_parse.py | 3 ++ tests/unit/agents/data/test_warning_codes.py | 3 ++ 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/agents/anyplot/data/parse.py b/agents/anyplot/data/parse.py index 27d38e95773..a9a83d56875 100644 --- a/agents/anyplot/data/parse.py +++ b/agents/anyplot/data/parse.py @@ -359,6 +359,7 @@ _NUMBER_PART = r"[+-]?(?:[0-9][0-9.,'’]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?" # `'` and `’`: Swiss `CHF 1'200` _UNIT_SPACE = r"[^\S\r\n\t]?" _CURRENCY = r"[€$£]|CHF|EUR|USD|GBP" +_CURRENCY_UNIT = re.compile(_CURRENCY) _TRAILING_UNIT = re.compile( rf"(?P{_NUMBER_PART}){_UNIT_SPACE}(?P%|‰|°[CF]?|[€$£]|[A-Za-zµμ]{{1,4}}[²³]?\.?)" ) @@ -2796,19 +2797,36 @@ def _commas_as_thousands(columns: list[_Column]) -> None: ambiguous = [column for column in columns if column.decimal_comma and AMBIGUOUS_COMMAS in column.notes] if not ambiguous: return + for column in ambiguous: + if column.unit is not None and _CURRENCY_UNIT.fullmatch(column.unit): + _retype_thousands_commas( + column, "commas read as thousands separators, as a currency amount has at most two decimals" + ) + rest = [column for column in ambiguous if AMBIGUOUS_COMMAS in column.notes] + if not rest: + return if any(column.decimal_comma and column not in ambiguous and column.group_mark != "," for column in columns): - for column in ambiguous: + for column in rest: column.notes.remove(AMBIGUOUS_COMMAS) return if not any(column.group_mark == "," for column in columns): return - for column in ambiguous: - if column.id_like or not all(_COMMA_GROUP.fullmatch(cell) for cell in column.bare if cell): - continue - if _as_integers(column, [cell.replace(",", "") for cell in column.bare]): - column.decimal_comma = False - column.notes.remove(AMBIGUOUS_COMMAS) - column.notes.append("commas read as thousands separators, as in the file's other columns") + for column in rest: + _retype_thousands_commas(column, "commas read as thousands separators, as in the file's other columns") + + +def _retype_thousands_commas(column: _Column, note: str) -> None: + if column.id_like or not all(_COMMA_GROUP.fullmatch(cell) for cell in column.bare if cell): + return + column.decimal_comma = False + column.notes.remove(AMBIGUOUS_COMMAS) + if not _as_integers(column, [cell.replace(",", "") for cell in column.bare]): + # Every cell is a whole number (`_COMMA_GROUP`), one past int64: text keeps every digit, as rule 3 does. + column.dtype, column.cells, column.numbers, column.unit = "text", list(column.source), [], None + column.rounded = column.respelled = 0 + column.notes.append(BIG_INTEGERS) + return + column.notes.append(note) def _split_units(present: list[str]) -> tuple[str, dict[str, str]] | None: diff --git a/agents/anyplot/data/warning_codes.py b/agents/anyplot/data/warning_codes.py index 0f5789dab20..617d9824658 100644 --- a/agents/anyplot/data/warning_codes.py +++ b/agents/anyplot/data/warning_codes.py @@ -179,7 +179,11 @@ _COLUMN + r"commas read as decimal commas; remove them first if they separate thousands$", "decimal_commas_ambiguous", ), - (_COLUMN + r".* read as thousands separators(?:, as in the file's other columns)?$", "thousands_separators"), + ( + _COLUMN + + r".* read as thousands separators(?:, as (?:in the file's other columns|a currency amount has at most two decimals))?$", + "thousands_separators", + ), ( _COLUMN + r"times with the UTC offsets .* written in UTC \(\+00:00\); the instants are the same$", "utc_offsets_normalised", diff --git a/tests/unit/agents/data/test_parse.py b/tests/unit/agents/data/test_parse.py index b2fc0454bff..a73a39ae279 100644 --- a/tests/unit/agents/data/test_parse.py +++ b/tests/unit/agents/data/test_parse.py @@ -1908,11 +1908,14 @@ def test_currency_commas_that_are_not_three_digit_groups_stay_decimal_commas( self, cells: list[str], expected: list[float] ) -> None: parsed = parse_dataset("price\n" + "\n".join(cells) + "\n") + semicolon = parse_dataset("Preis;Menge\n3,50;1.200\n") assert parsed.column_dtypes == {"price ($)": "number"} assert parsed.profile.decimal == "," assert read_back(parsed)["price ($)"].tolist() == expected assert not warned(parsed, "commas read as thousands separators") + assert semicolon.csv == "Preis,Menge\n3.5,1200\n" + assert semicolon.profile.decimal == "," def test_an_id_or_oversized_currency_comma_column_is_not_retyped_as_integers(self) -> None: code = parse_dataset('Code\n"$1,200"\n"$2,500"\n') diff --git a/tests/unit/agents/data/test_warning_codes.py b/tests/unit/agents/data/test_warning_codes.py index 4837eef6fae..b376ec75b33 100644 --- a/tests/unit/agents/data/test_warning_codes.py +++ b/tests/unit/agents/data/test_warning_codes.py @@ -88,6 +88,9 @@ "decimal_commas_ambiguous" ), "column 'x': dots read as thousands separators, as in the file's other columns": "thousands_separators", + "column 'x': commas read as thousands separators, as a currency amount has at most two decimals": ( + "thousands_separators" + ), "column 'x': commas read as thousands separators": "thousands_separators", "column 't': times with the UTC offsets +01:00, +02:00 written in UTC (+00:00); the instants are the same": ( "utc_offsets_normalised" From e0c1dc9159762ced7b6c5fea2d9819926219b610 Mon Sep 17 00:00:00 2001 From: Markus Neusinger <2921697+MarkusNeusinger@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:49:11 +0200 Subject: [PATCH 3/5] docs(agents): document currency thousands comma rule and add changelog Step 3 of 3: document the currency comma thousands rule in parse.py and docs/concepts/agent-network.md, and add changelog fragment. Refs #12224 Co-Authored-By: Antigravity (Gemini 4 Argon) --- agents/anyplot/data/parse.py | 13 +++++++++---- changelog.d/currency-thousands-commas.md | 12 ++++++++++++ docs/concepts/agent-network.md | 2 +- 3 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 changelog.d/currency-thousands-commas.md diff --git a/agents/anyplot/data/parse.py b/agents/anyplot/data/parse.py index a9a83d56875..05dd521eb6e 100644 --- a/agents/anyplot/data/parse.py +++ b/agents/anyplot/data/parse.py @@ -94,7 +94,8 @@ 5. **Type each column**, see `_type_column`: dates, then booleans, integers, numbers (also with unambiguous thousands groups: `7.184.003`, `1'234'567`, `1 234,5`, `1,234,567`, see `_as_grouped_numbers`; a column of `1,234` shapes reads - as comma thousands only beside a comma-grouped column, `_commas_as_thousands`), then the + as comma thousands in a currency column or beside a comma-grouped column, + `_commas_as_thousands`), then the same numbers behind one unit shared by every cell (`65 %`, `15,6 h`, `€ 5`, also with a no-break space; the unit leaves the cells), otherwise text. A whole number written without a fraction in a number column keeps that form (`87000` beside @@ -2777,13 +2778,17 @@ def _id_like(name: str) -> bool: def _commas_as_thousands(columns: list[_Column]) -> None: - """A column of `1,234` shapes reads by what the file's other columns settle, else keeps its decimal comma. + """A column of `1,234` shapes reads by its currency or what the file's other columns settle, else keeps its decimal comma. A column whose commas could be decimals or thousands (every cell a whole number or `1,234`, typed as decimal commas with the `AMBIGUOUS_COMMAS` note) is settled only by strong evidence (review of 2026-10-15: a dot number elsewhere is none, since `12.30` may be a time and a rating of `4.5` says nothing about a price of `1,250`): + * the column's own unit is a currency (`_CURRENCY`: `€ $ £ CHF EUR USD GBP`) and every + cell is a whole number or a three-digit comma group (`$1,200`, `1,200 USD`): a money + amount has at most two decimals, so the column reads as whole numbers with comma + thousands, with a note; * another column writes a decimal comma that is no such shape (`6,07`): the decimal comma stays and the note goes, so the root asks no question the file answers (live test of 2026-10-15, `43,621` Mio. t beside `6,07` t); @@ -2791,8 +2796,8 @@ def _commas_as_thousands(columns: list[_Column]) -> None: `1,234.5`, `_as_grouped_numbers`): the column reads as whole numbers with comma thousands, with a note. - Without either the decimal comma stays with its note. A column named as an id or a code - (`_id_like`) is never retyped. + Without any of these the decimal comma stays with its note. A column named as an id or a + code (`_id_like`) is never retyped. """ ambiguous = [column for column in columns if column.decimal_comma and AMBIGUOUS_COMMAS in column.notes] if not ambiguous: diff --git a/changelog.d/currency-thousands-commas.md b/changelog.d/currency-thousands-commas.md new file mode 100644 index 00000000000..df7a10aaa1d --- /dev/null +++ b/changelog.d/currency-thousands-commas.md @@ -0,0 +1,12 @@ +### Fixed + +- **Three-digit comma groups in currency columns read as thousands.** A column + whose unit is a currency (`€`, `$`, `£`, `CHF`, `EUR`, `USD`, `GBP`) and whose + comma cells all sit in three-digit groups (`"$1,200"`, `"$12,500"`, + `"1,200 USD"`) was previously typed as decimal commas (`1.2`, `12.5`) unless + another column carried two comma groups or a dot decimal behind a comma group. + Because a currency amount has at most two decimal places, `_commas_as_thousands` + in `agents/anyplot/data/parse.py` now retypes such a column as whole numbers + with comma thousands and announces the reading in a warning, while any cell + with a one- or two-digit fraction after a comma (`"$1,25"`, `"$3,50"`) keeps + the column on decimal commas. diff --git a/docs/concepts/agent-network.md b/docs/concepts/agent-network.md index 4fa683fb4e5..0abd65c1949 100644 --- a/docs/concepts/agent-network.md +++ b/docs/concepts/agent-network.md @@ -269,7 +269,7 @@ Schemas live in `agents/anyplot/schemas.py`: `ColumnProfile{name≤64, dtype, mi ### Parse -`agents/anyplot/data/parse.py` has no ADK import. Limits: UTF-8 at most 1 MB for a paste (200 KB until 2026-10-14) and 2 MB for a table the service computed (`MAX_COMPUTED_BYTES`, passed as `max_bytes` by every re-parse of the service's own CSV), a canonical CSV of at most 4 MiB, 50 columns, 20,000 rows and 200 characters per cell. It rejects NUL and control characters, strips the BOM, applies NFC, and strips zero-width and bidi control characters. Headers are canonicalised (NFC, control characters stripped, names over 64 characters renamed, duplicates suffixed, empty names become `column_N`, each rename a warning), and the canonical names are written into `data.csv`, which is the single source of column names. Formats: JSON records, a column map, an object of values or an array of `[category, value]` pairs (the last two become the columns `category`, the keys kept as text, and `value`, source format `json_pairs`; nesting depth at most 2, except the nested objects of records described below; a repeated key keeps its last value, with a warning), a JSON-stat 2.0 dataset (since 2026-10-14, source format `json_stat`; see below), JSON Lines (since 2026-10-16, live test round 6, source format `json_lines`: when the whole text is no JSON but its first line is one JSON object, every non-empty line must be one and they read as JSON records; a line that is not names its number and the JSON error; up to 10 lines above the first object, a title or an export's comments, become notes when every line from there on starts with `{`; a dropped file named `.jsonl` or `.ndjson` reads as JSON Lines first, so one record is one row, while a pasted single object of values stays category/value pairs, with a warning when its values mix text and numbers), or delimited text with the delimiter sniffed from `, ; \t |`. Since 2026-10-16 every refusal names its cause (owner rule: people paste reasonably clean data, but a refusal says what it fails on): the agents service answers a refused paste with `cause`, the parser's message, which the BFF passes on one line of at most 300 characters and the page shows after its own text. A table whose rows have more values than its header names every ragged line with its field count, grouped by count, the first five lines of a group and the rest counted, and is never repaired; empty fields past the header's last column are left out with a counted warning, and the warning about short rows names their lines. Text that starts like JSON and is none of the readers' shapes names the JSON (or, behind an assignment or with a call in it, the Python) error's line and column, counted from the paste's own first line, with a hint for a comma after each line of objects, a missing comma and unquoted keys; JSON that Python reads as names (`true`, `null`, an unquoted key) gets the JSON error. The page shows the cause in place of its own line. A damaged file the converter refuses as `corrupt` names its broken part from a closed set (owner decision of 2026-10-16: `zip`, `workbook`, `sheet`, `parquet`, `arrow`, `npy_header`, `npy_data`), carried from the sandbox's `status.json` through the renderer's `ConvertRun.damage`, the agents service's `damage` and the BFF to the page, never as an exception's text; the renderer leaves the key out while it is empty, so an agents service deployed before the field reads every other answer. Since 2026-10-15 (live test round 3) two more shapes read. A table aligned with spaces (source format `whitespace`: a weather or statistics office's `.txt`, such as MeteoSwiss' homogenised monthly series) splits at runs of two or more spaces, so a header such as `Max Temp` stays one name; a tab is no separator there. Spaces are tried only when none of the four real delimiters finds a table (a CSV or TSV whose text fields hold spaces stays CSV or TSV), and they need stricter evidence, because they also separate the words of prose, the items of a list (`Hund 7`, which stays with the transcriber) and the keys and values of a metadata block: at least two data rows, a header line right above the first data row that holds no value (two or more years excepted, a wide table's `Land 2020 2021`), and no key-value block (a header whose first cell, or rows whose every first cell, ends with a colon). Owner principle of the review of 2026-10-15: a silent misread is the worst failure, so a refusal beats a guess. Single spaces split a table only of numbers and dates (every row at the header's width, no text cell, no two neighbouring fields that read as one value such as `1 234` or `2024-01-01 12:00`), with a warning on every such read: a single space also sits inside a two-word name (`Anna Lee`), so a table with a text column goes to the transcriber. A field in matching quotes loses them. A short row of a fixed-width layout (a blank cell) has its fields placed under the header's columns by their character position, never padded at the end, also below the last full row until a note, a source line or a summary row (`Total`, left out like other summary rows); a field that overlaps no column or more than one, columns whose own ranges overlap, and a row that does not line up, also a value row right below the table, are `unparseable`. The metadata block above such a table becomes notes like any other left-out lines. A line of the paste that only declares the missing-value mark (`Missing values: NA`, `nodata = -9999`; a sentinel number, negative or at least 999 in size, or one to three symbols or letters, never an answer such as `none`) counts as missing already while the table is found, so a row of marks stays a row; it blanks the cells of number and date columns only, a number matching by value (`-999.0`), with one warning that counts them, while a text category `M` or `NA` (Namibia) stays (owner decision of 2026-10-15). JSON records may nest objects, which become columns named with dots (`country.value`, `indicator.id`) down to three levels; an object nested deeper, or a field holding a list of objects or lists, is left out with one warning that names it (owner rule: canonical data is one flat CSV). A field holding a list of values expands to long format (owner decision of 2026-10-15, one measurement curve per record with `x: [...]` and `y: [...]`): the lists of one record expand together into one row per value, paired by position (so independent lists of one length, such as `tags` and `scores`, are paired too; the warning names every expanded field), the other fields repeated; an empty list leaves its cells empty, non-empty lists of different lengths in one record are `unparseable` with a message that names the record, the fields and their lengths, and so is a field that is a list in one record and a single value in another. When no text or whole-number field has a different value in every record, compared as `data.csv` will hold it (a unique measurement such as a temperature never counts), a first column `record_id` numbers the records, or `record_id_2` and so on when a field already has that name. Above the 20,000-row cap each record keeps evenly spaced values with its first and last one, the smallest thinning that fits, with a warning that gives the counts and the remedy (fewer points for one record, fewer records or points for several), which the root passes on. Records may also come wrapped as data APIs answer: an array of metadata objects and exactly one list of records (the World Bank's `[{page, pages, per_page, total, ...}, [records]]`), or an object with exactly one list of records beside other fields (`{"data": [...], "meta": {...}}`, also under `results`, `records`, `items` or any other key); the metadata's text, number and boolean fields go to the notes as `: `, and two lists of records, or metadata without records, are `unparseable`, since the parser takes no table by guess. A list beside the records counts as their schema only when its objects hold nothing but descriptor keys (`name`, `type`, `format`, `description` and the like) and their names cover every field of the records; descriptors beside an empty list of records are `unparseable`. A schema with rows of cells (CKAN, Socrata, Frictionless, pandas `orient="split"`: column names under `fields`, `columns`, `schema.fields` or `meta.view.columns`, rows as lists) reads as one record per row; rows without names that fit them are `unparseable`, so the schema is never taken for the data. BigQuery's own `rows: [{"f": [{"v": ...}]}]` shape is not read (its cells are a list of objects, which is left out, so the paste is refused). JSON-stat is still recognised first. The dataset pre-filter (`instruction_like`) reads the table with its constant columns, which come back into `data.csv` when the notes are dropped. The root's prompt lists every control of the chat page (`.parse()`, `.confirm()`, `.undo()`, the binding dropdowns, the language chip and so on) and forbids naming any other, since it once told a user to pick "a separator that recognises spaces", a setting that does not exist; `tests/unit/agents/test_prompt_controls.py` checks that each control the prompt names is a label in the page's code. JSON-stat is what statistics offices' APIs answer (the Swiss BFS PxWeb API, Eurostat, Statistik Austria, Statistics Norway): an object with `"class": "dataset"`, `id`, `size`, `dimension` and `value`; a JSON-stat 1.x bundle of one dataset is read too, a bundle of several, a collection or a lone dimension is `unparseable`. It is the one shape nested deeper than 2 (about five levels), and the 50-level bracket count still applies. The parser checks it strictly before it builds a row (as many sizes as dimensions, the product of the sizes at most 20,000 and equal to the number of values, `value` as an array or as a sparse object of positions, every dimension with as many categories as its size, `category.index` as an array or an object of positions), and refuses any mismatch as `unparseable` with a message that names no data. It flattens the dataset into a long table (owner rule: long format, no wide pivot; `prepare_data` can pivot later): one column per dimension with more than one category, named by the dimension's label (else its id), its cells the category labels (else the codes) kept as text, never typed and never missing (`NA` for Namibia stays `NA`, a year code stays text; a label two categories share gets each one's code in parentheses), then one column `value`, the only typed column, whatever the dataset's unit (a dimension named `value` in any case becomes `