diff --git a/agents/anyplot/data/parse.py b/agents/anyplot/data/parse.py index 27d38e95773..faa1a9dda6b 100644 --- a/agents/anyplot/data/parse.py +++ b/agents/anyplot/data/parse.py @@ -94,7 +94,8 @@ 5. **Type each column**, see `_type_column`: dates, then booleans, integers, numbers (also with unambiguous thousands groups: `7.184.003`, `1'234'567`, `1 234,5`, `1,234,567`, see `_as_grouped_numbers`; a column of `1,234` shapes reads - as comma thousands only beside a comma-grouped column, `_commas_as_thousands`), then the + as comma thousands beside a comma-grouped column, or in a currency column of a file + that speaks for no decimal comma, `_commas_as_thousands`), then the same numbers behind one unit shared by every cell (`65 %`, `15,6 h`, `€ 5`, also with a no-break space; the unit leaves the cells), otherwise text. A whole number written without a fraction in a number column keeps that form (`87000` beside @@ -359,6 +360,7 @@ _NUMBER_PART = r"[+-]?(?:[0-9][0-9.,'’]*|\.[0-9]+)(?:[eE][+-]?[0-9]+)?" # `'` and `’`: Swiss `CHF 1'200` _UNIT_SPACE = r"[^\S\r\n\t]?" _CURRENCY = r"[€$£]|CHF|EUR|USD|GBP" +_CURRENCY_UNIT = re.compile(_CURRENCY) _TRAILING_UNIT = re.compile( rf"(?P{_NUMBER_PART}){_UNIT_SPACE}(?P%|‰|°[CF]?|[€$£]|[A-Za-zµμ]{{1,4}}[²³]?\.?)" ) @@ -2759,8 +2761,9 @@ def _as_grouped_numbers(column: _Column, cells: list[str], present: list[str]) - continue bare = [cell.replace(mark, "").replace(",", ".") if cell else "" for cell in cells] if not (_as_numbers(column, bare) if decimal else _as_integers(column, bare)): - if not decimal: # a whole number past int64: text keeps every digit, and says why + if not decimal: # a whole number past int64: text keeps every digit, and says why; the mark still counts column.notes.append(BIG_INTEGERS) + column.group_mark = mark return False column.decimal_comma = decimal == "," column.group_mark = mark @@ -2775,8 +2778,8 @@ def _id_like(name: str) -> bool: return bool(words & ID_WORDS) or any(name.casefold().endswith(suffix) for suffix in ("_id", "-id", "id_")) -def _commas_as_thousands(columns: list[_Column]) -> None: - """A column of `1,234` shapes reads by what the file's other columns settle, else keeps its decimal comma. +def _commas_as_thousands(columns: list[_Column], *, semicolon: bool = False) -> None: + """A column of `1,234` shapes reads by what the file or its currency settles, else keeps its decimal comma. A column whose commas could be decimals or thousands (every cell a whole number or `1,234`, typed as decimal commas with the `AMBIGUOUS_COMMAS` note) is settled only by strong evidence @@ -2786,12 +2789,21 @@ def _commas_as_thousands(columns: list[_Column]) -> None: * another column writes a decimal comma that is no such shape (`6,07`): the decimal comma stays and the note goes, so the root asks no question the file answers (live test of 2026-10-15, `43,621` Mio. t beside `6,07` t); + * the column's own unit is a currency (`_CURRENCY`: `€ $ £ CHF EUR USD GBP`) and every + cell is a whole number or a three-digit comma group (`$1,200`, `1,200 USD`): a money + amount mostly has at most two decimals, so the column reads as whole numbers with comma + thousands, with a note (issue #12224). A file that speaks for a decimal comma keeps the + question instead (owner decision of 2026-10-22): `semicolon` (the file is delimited by + semicolons, as spreadsheets write it where the comma is the decimal mark), a column of + dot groups (`7.184.003`, also one kept as text past int64) or of `1.234` shapes, or a + comma column with a zero-led group (`0,125`), since a fuel price of `1,789 €` has three + decimals; * otherwise, another column groups its thousands with commas unambiguously (`1,234,567`, `1,234.5`, `_as_grouped_numbers`): the column reads as whole numbers with comma thousands, with a note. - Without either the decimal comma stays with its note. A column named as an id or a code - (`_id_like`) is never retyped. + Without any of these the decimal comma stays with its note. A column named as an id or a + code (`_id_like`) is never retyped. """ ambiguous = [column for column in columns if column.decimal_comma and AMBIGUOUS_COMMAS in column.notes] if not ambiguous: @@ -2800,15 +2812,43 @@ def _commas_as_thousands(columns: list[_Column]) -> None: for column in ambiguous: column.notes.remove(AMBIGUOUS_COMMAS) return + dot_groups = any( + column.group_mark == "." + or (column.dtype == "number" and not column.decimal_comma and not column.id_like and _dot_groups_only(column)) + for column in columns + ) + # `0,125` has the `1,234` shape but is no thousands group, so its comma is a decimal comma. + zero_led = any(not all(_COMMA_GROUP.fullmatch(cell) for cell in column.bare if cell) for column in ambiguous) + if not (semicolon or dot_groups or zero_led): + for column in ambiguous: + if column.unit is not None and _CURRENCY_UNIT.fullmatch(column.unit): + _retype_thousands_commas( + column, "commas read as thousands separators, as a currency amount has at most two decimals" + ) if not any(column.group_mark == "," for column in columns): return for column in ambiguous: - if column.id_like or not all(_COMMA_GROUP.fullmatch(cell) for cell in column.bare if cell): - continue - if _as_integers(column, [cell.replace(",", "") for cell in column.bare]): - column.decimal_comma = False - column.notes.remove(AMBIGUOUS_COMMAS) - column.notes.append("commas read as thousands separators, as in the file's other columns") + if AMBIGUOUS_COMMAS in column.notes: + _retype_thousands_commas(column, "commas read as thousands separators, as in the file's other columns") + + +def _retype_thousands_commas(column: _Column, note: str) -> None: + """Retype a column of `1,234` shapes as whole numbers with comma thousands, with `note`. + + A column named as an id or a code (`_id_like`) is never retyped. A whole number past int64 + makes the column text with the `BIG_INTEGERS` note, so no digit is lost. + """ + if column.id_like or not all(_COMMA_GROUP.fullmatch(cell) for cell in column.bare if cell): + return + column.decimal_comma = False + column.notes.remove(AMBIGUOUS_COMMAS) + if not _as_integers(column, [cell.replace(",", "") for cell in column.bare]): + # Every cell is a whole number (`_COMMA_GROUP`), one past int64: text keeps every digit, as rule 3 does. + column.dtype, column.cells, column.numbers, column.unit = "text", list(column.source), [], None + column.rounded = column.respelled = 0 + column.notes.append(BIG_INTEGERS) + return + column.notes.append(note) def _split_units(present: list[str]) -> tuple[str, dict[str, str]] | None: @@ -2852,10 +2892,7 @@ def _retype_thousands_dots(column: _Column, *, decimal_comma: bool) -> None: """In a file with decimal commas, or with a column of unambiguous dot groups (`7.184.003`), a dot-number column of `1.234` shapes holds thousands separators; `decimal_comma` says the file writes decimal commas. A column named as an id, a code or a version (`_id_like`) is never retyped.""" - present = [cell for cell in column.bare if cell] - if column.id_like or not ( - all(_THOUSANDS_DOT.fullmatch(cell) for cell in present) and any("." in cell for cell in present) - ): + if column.id_like or not _dot_groups_only(column): return cells = [cell.replace(".", "") for cell in column.bare] if not _as_integers(column, cells): @@ -2868,6 +2905,12 @@ def _retype_thousands_dots(column: _Column, *, decimal_comma: bool) -> None: column.notes.append("dots read as thousands separators, as in the file's other columns") +def _dot_groups_only(column: _Column) -> bool: + """Whether every cell is a whole number or a `1.234` shape (`_THOUSANDS_DOT`), with at least one dot.""" + present = [cell for cell in column.bare if cell] + return all(_THOUSANDS_DOT.fullmatch(cell) for cell in present) and any("." in cell for cell in present) + + def _as_integers(column: _Column, cells: list[str]) -> bool: values = [int(cell) if cell else None for cell in cells] if any(value is not None and not _INT64_MIN <= value <= _INT64_MAX for value in values): @@ -3286,7 +3329,7 @@ def _build( f"{_count(marked, 'cell')} of number or date columns matched the missing-value mark the notes declare " "and are empty" ) - _commas_as_thousands(columns) + _commas_as_thousands(columns, semicolon=source_format == "semicolon") decimal_comma = any(column.decimal_comma for column in columns) if decimal_comma or any(column.group_mark == "." for column in columns): for column in columns: diff --git a/agents/anyplot/data/warning_codes.py b/agents/anyplot/data/warning_codes.py index 0f5789dab20..617d9824658 100644 --- a/agents/anyplot/data/warning_codes.py +++ b/agents/anyplot/data/warning_codes.py @@ -179,7 +179,11 @@ _COLUMN + r"commas read as decimal commas; remove them first if they separate thousands$", "decimal_commas_ambiguous", ), - (_COLUMN + r".* read as thousands separators(?:, as in the file's other columns)?$", "thousands_separators"), + ( + _COLUMN + + r".* read as thousands separators(?:, as (?:in the file's other columns|a currency amount has at most two decimals))?$", + "thousands_separators", + ), ( _COLUMN + r"times with the UTC offsets .* written in UTC \(\+00:00\); the instants are the same$", "utc_offsets_normalised", diff --git a/changelog.d/currency-thousands-commas.md b/changelog.d/currency-thousands-commas.md new file mode 100644 index 00000000000..ae90c7cf503 --- /dev/null +++ b/changelog.d/currency-thousands-commas.md @@ -0,0 +1,10 @@ +### Fixed + +- **Currency columns read three-digit comma groups as thousands.** A pasted + price column such as `"$1,200"`, `"$950"`, `"$12,500"` (also `1,200 USD`, + `€ 1,200`, `CHF 12,500`) read as 1.2, 950 and 12.5; it now reads as 1200, + 950 and 12500 with a warning, since a money amount mostly has at most two + decimals. A cell with a one- or two-digit fraction (`$1,25`) keeps the + decimal comma, and a file that speaks for decimal commas keeps the question: + one delimited by semicolons, or with a column of dot groups or of `1.234` + shapes, so a fuel price of `1,789 €` stays 1.789. (#12228) diff --git a/docs/concepts/agent-network.md b/docs/concepts/agent-network.md index 4fa683fb4e5..79661bff545 100644 --- a/docs/concepts/agent-network.md +++ b/docs/concepts/agent-network.md @@ -269,7 +269,7 @@ Schemas live in `agents/anyplot/schemas.py`: `ColumnProfile{name≤64, dtype, mi ### Parse -`agents/anyplot/data/parse.py` has no ADK import. Limits: UTF-8 at most 1 MB for a paste (200 KB until 2026-10-14) and 2 MB for a table the service computed (`MAX_COMPUTED_BYTES`, passed as `max_bytes` by every re-parse of the service's own CSV), a canonical CSV of at most 4 MiB, 50 columns, 20,000 rows and 200 characters per cell. It rejects NUL and control characters, strips the BOM, applies NFC, and strips zero-width and bidi control characters. Headers are canonicalised (NFC, control characters stripped, names over 64 characters renamed, duplicates suffixed, empty names become `column_N`, each rename a warning), and the canonical names are written into `data.csv`, which is the single source of column names. Formats: JSON records, a column map, an object of values or an array of `[category, value]` pairs (the last two become the columns `category`, the keys kept as text, and `value`, source format `json_pairs`; nesting depth at most 2, except the nested objects of records described below; a repeated key keeps its last value, with a warning), a JSON-stat 2.0 dataset (since 2026-10-14, source format `json_stat`; see below), JSON Lines (since 2026-10-16, live test round 6, source format `json_lines`: when the whole text is no JSON but its first line is one JSON object, every non-empty line must be one and they read as JSON records; a line that is not names its number and the JSON error; up to 10 lines above the first object, a title or an export's comments, become notes when every line from there on starts with `{`; a dropped file named `.jsonl` or `.ndjson` reads as JSON Lines first, so one record is one row, while a pasted single object of values stays category/value pairs, with a warning when its values mix text and numbers), or delimited text with the delimiter sniffed from `, ; \t |`. Since 2026-10-16 every refusal names its cause (owner rule: people paste reasonably clean data, but a refusal says what it fails on): the agents service answers a refused paste with `cause`, the parser's message, which the BFF passes on one line of at most 300 characters and the page shows after its own text. A table whose rows have more values than its header names every ragged line with its field count, grouped by count, the first five lines of a group and the rest counted, and is never repaired; empty fields past the header's last column are left out with a counted warning, and the warning about short rows names their lines. Text that starts like JSON and is none of the readers' shapes names the JSON (or, behind an assignment or with a call in it, the Python) error's line and column, counted from the paste's own first line, with a hint for a comma after each line of objects, a missing comma and unquoted keys; JSON that Python reads as names (`true`, `null`, an unquoted key) gets the JSON error. The page shows the cause in place of its own line. A damaged file the converter refuses as `corrupt` names its broken part from a closed set (owner decision of 2026-10-16: `zip`, `workbook`, `sheet`, `parquet`, `arrow`, `npy_header`, `npy_data`), carried from the sandbox's `status.json` through the renderer's `ConvertRun.damage`, the agents service's `damage` and the BFF to the page, never as an exception's text; the renderer leaves the key out while it is empty, so an agents service deployed before the field reads every other answer. Since 2026-10-15 (live test round 3) two more shapes read. A table aligned with spaces (source format `whitespace`: a weather or statistics office's `.txt`, such as MeteoSwiss' homogenised monthly series) splits at runs of two or more spaces, so a header such as `Max Temp` stays one name; a tab is no separator there. Spaces are tried only when none of the four real delimiters finds a table (a CSV or TSV whose text fields hold spaces stays CSV or TSV), and they need stricter evidence, because they also separate the words of prose, the items of a list (`Hund 7`, which stays with the transcriber) and the keys and values of a metadata block: at least two data rows, a header line right above the first data row that holds no value (two or more years excepted, a wide table's `Land 2020 2021`), and no key-value block (a header whose first cell, or rows whose every first cell, ends with a colon). Owner principle of the review of 2026-10-15: a silent misread is the worst failure, so a refusal beats a guess. Single spaces split a table only of numbers and dates (every row at the header's width, no text cell, no two neighbouring fields that read as one value such as `1 234` or `2024-01-01 12:00`), with a warning on every such read: a single space also sits inside a two-word name (`Anna Lee`), so a table with a text column goes to the transcriber. A field in matching quotes loses them. A short row of a fixed-width layout (a blank cell) has its fields placed under the header's columns by their character position, never padded at the end, also below the last full row until a note, a source line or a summary row (`Total`, left out like other summary rows); a field that overlaps no column or more than one, columns whose own ranges overlap, and a row that does not line up, also a value row right below the table, are `unparseable`. The metadata block above such a table becomes notes like any other left-out lines. A line of the paste that only declares the missing-value mark (`Missing values: NA`, `nodata = -9999`; a sentinel number, negative or at least 999 in size, or one to three symbols or letters, never an answer such as `none`) counts as missing already while the table is found, so a row of marks stays a row; it blanks the cells of number and date columns only, a number matching by value (`-999.0`), with one warning that counts them, while a text category `M` or `NA` (Namibia) stays (owner decision of 2026-10-15). JSON records may nest objects, which become columns named with dots (`country.value`, `indicator.id`) down to three levels; an object nested deeper, or a field holding a list of objects or lists, is left out with one warning that names it (owner rule: canonical data is one flat CSV). A field holding a list of values expands to long format (owner decision of 2026-10-15, one measurement curve per record with `x: [...]` and `y: [...]`): the lists of one record expand together into one row per value, paired by position (so independent lists of one length, such as `tags` and `scores`, are paired too; the warning names every expanded field), the other fields repeated; an empty list leaves its cells empty, non-empty lists of different lengths in one record are `unparseable` with a message that names the record, the fields and their lengths, and so is a field that is a list in one record and a single value in another. When no text or whole-number field has a different value in every record, compared as `data.csv` will hold it (a unique measurement such as a temperature never counts), a first column `record_id` numbers the records, or `record_id_2` and so on when a field already has that name. Above the 20,000-row cap each record keeps evenly spaced values with its first and last one, the smallest thinning that fits, with a warning that gives the counts and the remedy (fewer points for one record, fewer records or points for several), which the root passes on. Records may also come wrapped as data APIs answer: an array of metadata objects and exactly one list of records (the World Bank's `[{page, pages, per_page, total, ...}, [records]]`), or an object with exactly one list of records beside other fields (`{"data": [...], "meta": {...}}`, also under `results`, `records`, `items` or any other key); the metadata's text, number and boolean fields go to the notes as `: `, and two lists of records, or metadata without records, are `unparseable`, since the parser takes no table by guess. A list beside the records counts as their schema only when its objects hold nothing but descriptor keys (`name`, `type`, `format`, `description` and the like) and their names cover every field of the records; descriptors beside an empty list of records are `unparseable`. A schema with rows of cells (CKAN, Socrata, Frictionless, pandas `orient="split"`: column names under `fields`, `columns`, `schema.fields` or `meta.view.columns`, rows as lists) reads as one record per row; rows without names that fit them are `unparseable`, so the schema is never taken for the data. BigQuery's own `rows: [{"f": [{"v": ...}]}]` shape is not read (its cells are a list of objects, which is left out, so the paste is refused). JSON-stat is still recognised first. The dataset pre-filter (`instruction_like`) reads the table with its constant columns, which come back into `data.csv` when the notes are dropped. The root's prompt lists every control of the chat page (`.parse()`, `.confirm()`, `.undo()`, the binding dropdowns, the language chip and so on) and forbids naming any other, since it once told a user to pick "a separator that recognises spaces", a setting that does not exist; `tests/unit/agents/test_prompt_controls.py` checks that each control the prompt names is a label in the page's code. JSON-stat is what statistics offices' APIs answer (the Swiss BFS PxWeb API, Eurostat, Statistik Austria, Statistics Norway): an object with `"class": "dataset"`, `id`, `size`, `dimension` and `value`; a JSON-stat 1.x bundle of one dataset is read too, a bundle of several, a collection or a lone dimension is `unparseable`. It is the one shape nested deeper than 2 (about five levels), and the 50-level bracket count still applies. The parser checks it strictly before it builds a row (as many sizes as dimensions, the product of the sizes at most 20,000 and equal to the number of values, `value` as an array or as a sparse object of positions, every dimension with as many categories as its size, `category.index` as an array or an object of positions), and refuses any mismatch as `unparseable` with a message that names no data. It flattens the dataset into a long table (owner rule: long format, no wide pivot; `prepare_data` can pivot later): one column per dimension with more than one category, named by the dimension's label (else its id), its cells the category labels (else the codes) kept as text, never typed and never missing (`NA` for Namibia stays `NA`, a year code stays text; a label two categories share gets each one's code in parentheses), then one column `value`, the only typed column, whatever the dataset's unit (a dimension named `value` in any case becomes `