From fd98f46ed8e74e2242c3b2020933be9381fea33c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 28 Oct 2026 01:25:52 -0700 Subject: [PATCH] Count a word by box and country in the fielded local corpora name_usage.py --boxes adds a row for orcid-researchers and patentsview-inventors: the people whose given box and whose family box write the word, counted over the whole extract rather than the capped corpus, and the countries with the most of them, each as a share of that country's people (the records with no country last; --country CC adds a country, implying --boxes). PatentsView weighs a name by the inventors filing under it, ORCID one holder a record. The counts come from a second index, local_corpora.cached_box_index, built from a Source's new extract and boxes readers and cached as boxindex.json.gz, keyed to the extract's sha256 and a fingerprint of the reader and the fold. Building ORCID's took 2 min 19 s; a lookup with both cached costs about 5 s more than one without, so the rows are asked for. An extract or a cached index that cannot be read fails its row or is built over, never the run. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- tests/v2/test_local_corpora.py | 136 ++++++++++++++++++++++++ tests/v2/test_name_usage_tool.py | 100 +++++++++++++++++ tools/differential/README.md | 2 +- tools/differential/local_core.py | 11 +- tools/differential/local_corpora.py | 125 +++++++++++++++++++++- tools/differential/local_orcid.py | 14 +++ tools/differential/local_patentsview.py | 14 +++ tools/name_usage.py | 105 +++++++++++++++++- 9 files changed, 500 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 28e9b1b24..e5d422adb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -261,7 +261,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These ## Extension Patterns -**Adding a word to the vocabulary** — the steps are in `nameparser/vocab/README.md`, and the traps they have run into (other readings that clash, reading the real-name counts, the predicates that test a member's spelling, user-doc examples that use the word, recording the vetting on the word) are in `nameparser/vocab/AGENTS.md`, which loads when a session reads a vocabulary TOML file. First check the word's OTHER readings (`uv run python tools/vocab/query.py --word `), then its bearers (`uv run python tools/name_usage.py `). For a title that is also a name (rules/H9's batches, #913), `uv run python tools/vocab/word_report.py ... --base ` prints the H9 shape grid, the word's incidence in every fetched local corpus and its usage rows in one run, and `--rank` orders candidates before a batch is cut; `uv run python tools/differential/measure.py --base ` then runs every corpus check of the change, one after another, the local ones against the base. +**Adding a word to the vocabulary** — the steps are in `nameparser/vocab/README.md`, and the traps they have run into (other readings that clash, reading the real-name counts, the predicates that test a member's spelling, user-doc examples that use the word, recording the vetting on the word) are in `nameparser/vocab/AGENTS.md`, which loads when a session reads a vocabulary TOML file. First check the word's OTHER readings (`uv run python tools/vocab/query.py --word `), then its bearers (`uv run python tools/name_usage.py `, with `--boxes` adding rows for the fielded local corpora that count the word by box and country, and `--country CC` the country a pack is for). For a title that is also a name (rules/H9's batches, #913), `uv run python tools/vocab/word_report.py ... --base ` prints the H9 shape grid, the word's incidence in every fetched local corpus and its usage rows in one run, and `--rank` orders candidates before a batch is cut; `uv run python tools/differential/measure.py --base ` then runs every corpus check of the change, one after another, the local ones against the base. **Before adding a short/common word to `PARTICLES` globally**, test it mid-string against realistic 3-token names, not just check for English-word collisions: Korean/Vietnamese given names put a short syllable in the middle slot (`Park In Hwan`, `Nguyen To Nga`), and Western names put a bare initial there (`John V. Smith`). A word that looks safe ("nobody is named 'to'") can still swallow a real middle name/initial into the last name once it's a global prefix — confirmed regressions for `to`/`in`/`an`/`ten`/`then` and bare `v` this way (PR #191). diff --git a/tests/v2/test_local_corpora.py b/tests/v2/test_local_corpora.py index d895c251d..3d8cde1c3 100644 --- a/tests/v2/test_local_corpora.py +++ b/tests/v2/test_local_corpora.py @@ -4500,3 +4500,139 @@ def test_alike_against_a_base_counts_each_direction_apart() -> None: assert local.format_alike(nepal, {"0": True}) == ["1 of 1 pairs read alike"] with pytest.raises(SystemExit, match="other persons"): local.format_alike(nepal, {"0": True, "1": True}, {"0": True}) + + +# -- the box index ------------------------------------------------------------------ + +def _extract(path: Path, header: dict[str, object], lines: list[dict[str, object]]) -> Path: + """An extract written as the fetch writes one: a header line, then a + record a line, gzipped.""" + path.parent.mkdir(parents=True, exist_ok=True) + text = "".join(json.dumps(x) + "\n" for x in (header, *lines)) + path.write_bytes(gzip.compress(text.encode(), mtime=0)) + return path + + +def _pv_extract(*lines: dict[str, object]) -> Path: + return _extract(patentsview.extract_path(), {"release": patentsview.PV_RELEASE}, + list(lines)) + + +_PV_LINES = ( + {"given": "Ana María", "family": "del Río", "country": "ES", "people": 3}, + {"given": "Juan del", "family": "del Castillo del Río", "country": "MX", "people": 2}, + {"given": "Ann", "family": "Lee", "country": "US", "people": 10}, + {"given": "Bo", "family": "DEL.", "people": 1}) + + +def test_the_box_index_counts_people_by_box_and_country_once_a_record() -> None: + index = local.box_index(patentsview.pv_boxes(_pv_extract(*_PV_LINES))) + assert index.totals == {"ES": 3, "MX": 2, "US": 10, "": 1} + # 'del Castillo del Río' counts del once; 'DEL.' folds to it as the word + # index folds it; the country it is a share of is every record's. + assert index.family["del"] == {"ES": 3, "MX": 2, "": 1} + assert index.given["del"] == {"MX": 2} + assert index.given["maría"] == {"ES": 3} and "lee" not in index.given + + +def test_an_orcid_holder_weighs_one(_own_cache: Path) -> None: + path = _extract(orcid.extract_path(), {"file": orcid.ORCID_FILE}, [ + {"given": "Ann", "family": "Lee", "country": "NZ"}, {"given": "Ann"}]) + index = local.box_index(orcid.orcid_boxes(path)) + assert index.totals == {"NZ": 1, "": 1} and index.given["ann"] == {"NZ": 1, "": 1} + + +@pytest.mark.parametrize(("reader", "path", "header"), [ + ("orcid", "orcid", {"file": "ORCID_2024_10_summaries.tar.gz"}), + ("patentsview", "patentsview", {"release": "2024-12-31"})]) +def test_an_extract_of_another_release_is_refused_by_its_reader( + reader: str, path: str, header: dict[str, object]) -> None: + module = {"orcid": orcid, "patentsview": patentsview}[reader] + boxes = module.orcid_boxes if reader == "orcid" else module.pv_boxes + written = _extract(module.extract_path(), header, []) + with pytest.raises(ValueError, match="is not an extract of"): + list(boxes(written)) + + +def test_the_cached_box_index_is_read_back_under_its_key_and_rebuilt_off_it( + capsys: pytest.CaptureFixture[str]) -> None: + _pv_extract(*_PV_LINES) + built = local.cached_box_index(local.PV) + cache = local.directory(local.PV) / local.BOX_INDEX + assert "building the box index" in capsys.readouterr().err + assert local.cached_box_index(local.PV) == built + assert capsys.readouterr().err == "" # read back, not built again + # A refetched extract is other bytes: the index is built from them. + _pv_extract(*_PV_LINES[:2]) + assert local.cached_box_index(local.PV).totals == {"ES": 3, "MX": 2} + assert "rebuilding the box index" in capsys.readouterr().err + assert json.loads(gzip.decompress(cache.read_bytes()))["totals"] == {"ES": 3, "MX": 2} + + +@pytest.mark.parametrize("damage", [b"not gzip", gzip.compress(b"{", mtime=0), + gzip.compress(b'{"key": 1}', mtime=0)], + ids=["not gzip", "not JSON", "no counts"]) +def test_a_cached_box_index_that_cannot_be_read_is_built_over( + capsys: pytest.CaptureFixture[str], damage: bytes) -> None: + _pv_extract(*_PV_LINES) + cache = local.directory(local.PV) / local.BOX_INDEX + cache.write_bytes(damage) + assert local.cached_box_index(local.PV).totals["US"] == 10 + assert "rebuilding the box index" in capsys.readouterr().err + + +def test_a_box_index_without_its_extract_names_the_refresh() -> None: + local.directory(local.PV) + with pytest.raises(ValueError, match="is missing.*fetch patentsview-inventors --refresh"): + local.cached_box_index(local.PV) + + +def test_a_change_to_the_box_reader_moves_the_key(monkeypatch: pytest.MonkeyPatch) -> None: + before = local.box_fingerprint(local.PV) + monkeypatch.setitem(local.SOURCES, local.PV, dataclasses.replace( + local.SOURCES[local.PV], boxes=lambda path: iter(()))) + assert local.box_fingerprint(local.PV) != before + + +def test_the_box_fingerprint_lists_everything_box_index_calls() -> None: + # box_fingerprint's list is kept by hand, as _FOLD_CODE is: a helper + # box_index starts calling, left off it, leaves the key unmoved. + import inspect + module = vars(local) + listed = {"box_index", "words", "word_key", "_strippable", "_keyed", + *local._FOLD_CONSTANTS, "BoxIndex", "Counter"} + read: set[str] = set() + for name in ("box_index", "words", "word_key", "_keyed"): + read |= _globals_read(inspect.unwrap(module[name]).__code__) + called = {n for n in read & module.keys() + if not isinstance(module[n], types.ModuleType)} + assert called - listed == set() + + +@pytest.mark.parametrize("damage", ["truncated", "corrupt", "empty", "no people"]) +def test_an_extract_that_cannot_be_read_fails_as_a_value_error(damage: str) -> None: + # _attempt in name_usage catches ValueError and OSError: anything else + # gzip, json or the reader raised took every row of the run down. + path = _pv_extract(*_PV_LINES) + body = path.read_bytes() + if damage == "truncated": + path.write_bytes(body[: len(body) // 2]) + elif damage == "corrupt": + path.write_bytes(body[:12] + bytes(b ^ 0xFF for b in body[12:40]) + body[40:]) + elif damage == "empty": + path.write_bytes(gzip.compress(b"", mtime=0)) + else: + _pv_extract({"given": "Ann", "family": "Lee", "country": "US"}) + with pytest.raises(ValueError, match="could not be read.*--refresh"): + local.cached_box_index(local.PV) + + +def test_a_cached_box_index_whose_deflate_is_damaged_is_built_over( + capsys: pytest.CaptureFixture[str]) -> None: + _pv_extract(*_PV_LINES) + local.cached_box_index(local.PV) + cache = local.directory(local.PV) / local.BOX_INDEX + body = cache.read_bytes() + cache.write_bytes(body[:12] + bytes(b ^ 0xFF for b in body[12:40]) + body[40:]) + assert local.cached_box_index(local.PV).totals["US"] == 10 + assert "rebuilding the box index" in capsys.readouterr().err diff --git a/tests/v2/test_name_usage_tool.py b/tests/v2/test_name_usage_tool.py index 372a808d3..0b937d9d7 100644 --- a/tests/v2/test_name_usage_tool.py +++ b/tests/v2/test_name_usage_tool.py @@ -2295,3 +2295,103 @@ def test_a_damaged_local_corpus_fails_its_row_not_the_run( out = capsys.readouterr().out assert "ctgov-officials (local corpus)" in out and "failed; CorpusDamaged" in out assert "US Census 2010 surnames" in out + + +# -- box indexes ----------------------------------------------------------------- + +def _boxes() -> object: + corpora = tool._local_corpora() + return corpora.BoxIndex( + totals={"ES": 400, "MX": 100, "US": 5000, "": 9000}, + given={"del": {"MX": 30, "ES": 20, "": 50}}, + family={"del": {"ES": 60, "MX": 10, "US": 5, "": 80}}) + + +def test_a_box_row_counts_each_box_and_gives_each_country_its_share() -> None: + row = tool.box_usage("del", "patentsview-inventors", _boxes(), "", 2, []) + assert (row.given, row.family) == (100, 155) + # Ranked by people writing it, the records stating no country last: + # they would lead every list and say nothing of where a name is borne. + assert row.given_shares == [("MX", 30, 100), ("ES", 20, 400), ("--", 50, 9000)] + assert row.family_shares == [("ES", 60, 400), ("MX", 10, 100), ("--", 80, 9000)] + assert "14,500 people in 3 countries" in row.note + + +def test_a_country_asked_for_is_listed_outside_the_top() -> None: + row = tool.box_usage("del", "patentsview-inventors", _boxes(), "", 1, ["US", "PH"]) + assert row.family_shares == [("ES", 60, 400), ("US", 5, 5000), ("PH", 0, 0), + ("--", 80, 9000)] + rendered = tool.render("del", [row]) + assert "family box by country: ES 60 (15.00% of 400), US 5 (0.10% of 5,000), " \ + "PH 0 (- of 0), -- 80 (0.89% of 9,000)" in rendered + + +@pytest.mark.parametrize(("word", "says"), [ + ("de la", "a spaced word is not looked up"), + ("..", "nothing in it is left")]) +def test_a_box_row_answers_single_words_only(word: str, says: str) -> None: + row = tool.box_usage(word, "patentsview-inventors", _boxes(), "", 5, []) + assert row.status == "unrepresentable" and says in row.note + + +def test_a_box_row_not_fetched_or_failed_says_so() -> None: + missing = tool.box_usage("del", "patentsview-inventors", None, "", 5, []) + assert missing.status == "not downloaded" + assert "local_corpora.py fetch patentsview-inventors" in missing.note + failed = tool.box_usage("del", "patentsview-inventors", None, "ValueError: gone", 5, []) + assert (failed.status, failed.note) == ("failed", "ValueError: gone") + + +def test_only_the_sources_with_boxes_get_a_box_row( + monkeypatch: pytest.MonkeyPatch) -> None: + built: list[str] = [] + module = tool._local_corpora() + def built_for(source: str) -> object: + built.append(source) + return _boxes() + + monkeypatch.setattr(module, "cached_box_index", built_for) + locals_ = [(source, object(), "") for source in sorted(module.SOURCES)] + found = tool.load_box_indexes(locals_) # type: ignore[arg-type] + assert [s for s, _, _ in found] == built == sorted( + s for s, src in module.SOURCES.items() if src.boxes is not None) + assert {"orcid-researchers", "patentsview-inventors"} <= set(built) + + +def test_the_country_option_reaches_the_box_rows( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(tool, "load_locals", lambda: []) + monkeypatch.setattr(tool, "load_box_indexes", lambda locals_: [ + ("patentsview-inventors", _boxes(), "")]) + assert tool.main(["--no-wikidata", "--countries", "1", "--country", "us", "del"]) == 0 + out = capsys.readouterr().out + assert "patentsview-inventors (boxes by country)" in out + assert "family box by country: ES 60 (15.00% of 400), US 5 (0.10% of 5,000)" in out + + +def test_the_box_rows_are_built_only_when_asked_for( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + # Building one takes minutes and reading one seconds: a plain lookup + # must not pay for rows nobody asked for. + asked: list[object] = [] + monkeypatch.setattr(tool, "load_locals", lambda: []) + def loaded(locals_: object) -> list[tuple[str, object, str]]: + asked.append(1) + return [("patentsview-inventors", _boxes(), "")] + + monkeypatch.setattr(tool, "load_box_indexes", loaded) + assert tool.main(["--no-wikidata", "del"]) == 0 + assert asked == [] and "boxes by country" not in capsys.readouterr().out + assert tool.main(["--no-wikidata", "--boxes", "del"]) == 0 + assert asked == [1] and "boxes by country" in capsys.readouterr().out + + +def test_a_country_asked_for_twice_or_blank_is_listed_once( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(tool, "load_locals", lambda: []) + monkeypatch.setattr(tool, "load_box_indexes", lambda locals_: [ + ("patentsview-inventors", _boxes(), "")]) + assert tool.main(["--no-wikidata", "--countries", "1", "--country", "us", + "--country", "US", "--country", " ", "del"]) == 0 + line = next(l for l in capsys.readouterr().out.splitlines() if "family box" in l) + assert line.count("US 5") == 1 and line.count("--") == 1 diff --git a/tools/differential/README.md b/tools/differential/README.md index 62dcdbc76..1ebb4635f 100644 --- a/tools/differential/README.md +++ b/tools/differential/README.md @@ -288,7 +288,7 @@ Any snapshot reading of 5,000 entries or more, committed or local, parses in a w | `jrc-names` | the person entities of the EU Joint Research Centre's JRC-Names, each with the spellings of its name harvested from news text in many languages and scripts: a record is one distinct spelling, with its entity and its script. The file (`entities.gzip`, 14.8 MB) is pinned by sha256 and dated by its Last-Modified, 2024-03-15. Of its 1,317,586 spellings kept, 1,203,634 are Latin, 66,673 Cyrillic, 17,074 Arabic (7,833 persons, 5,347 of them spelled in Latin too), 13,039 Katakana and 8,016 Han, and 218,822 persons have two or more spellings, 25,415 of them in two scripts or more (a spelling whose letters are in two scripts, or none, not counted as one). The population for spelling and transliteration variants of one name, read by the `groups` view, and for Arabic-script names. Licensed under the JRC-Names EULA, personal and non-transferable, with no redistribution or derivative works, so its views' output and its snapshot stay in the cache like every local corpus's, and here the licence, not only the names being real people's, is why | titles and credentials, since the names are harvested from running text (`parse()` read a title in 3,011 of its 1,315,051 names and a suffix in 1,940, 2026-10-24); the ASCII comma, which none holds (1,510 Arabic spellings hold an Arabic comma, U+060C, which the parser reads as one); the Indic scripts (Devanagari 827 spellings, Bengali 419, the others fewer); people the news did not name | | `nepal-candidates` | every candidate in Nepal's local elections of 2079 BS (2022), from the Election Commission's results site (result.election.gov.np), one file per local body: a record per distinct pair of the Commission's romanization (`name`, the one `parse()` reads) and the name in Devanagari as filed (`devanagari`). The corpus and the word index hold both, so a snapshot and a `name_usage.py` lookup read both scripts; `tally`, which counts what follows a comma, reads the romanized name. Of 126,981 pairs fetched 2026-10-24 (152,960 candidates in 753 files, 26 of whose romanizations were a placeholder such as `- - -` and are dropped), the romanized names carry surnames written as initials (`B.K.`, `K.C.`, `G.C.`), honorific words that are titles in front (`Guru`; `Shekh` until #752's phase 2j made it a title only in a title phrase) and `Md.`, Muhammad's abbreviation, which its period makes a title by shape; the Devanagari ones carry vowel signs and the virama, combining marks the letter-shape tests do not count as letters (decisions/nfd-shapes), and the joiners ZWJ and ZWNJ. The population for romanized South Asian names and for Devanagari. The site states no licence, which the credit line says | case and the comma, since the Commission's romanization is one consistent hand; names a person typed themselves; credentials and generational words; the parliament and provincial candidates, whose files carry Devanagari only; names outside Nepal | -A live source has no fixed snapshot to pin, so the cached copy is the pin: `meta.json` holds the source's data date and the sha256 of `raw.jsonl` and `corpus.jsonl`, checked on every load, and every report, the `name_usage.py` row included, carries the source and that date. Quote a count with the date, and expect a later fetch to differ. A local snapshot records which fetch it was read from, and `snapshot.py --local` refuses to compare it with a corpus fetched since, which would report the names the source gained and lost as moves: after a `fetch --refresh`, record again from the base checkout. ClinicalTrials.gov's terms ask anyone redistributing its data to credit it, show NLM's processing date and say what was changed; keeping the names in the cache and citing counts with that header meets them. The FEC's bar on soliciting and commercial use is in the credit line of every `fec-individuals` report, so a count quoted from it carries the bar with it. A record there is a distinct name, not a contribution or a contributor: one donor gives many times (34,264,742 rows held 1,893,964 names in the file dated 2026-10-18), and counting contributions would weight a shape by how often its bearers give. `tools/name_usage.py` looks a word up in each corpus's word index, cached as `wordindex.json` beside it on first use and keyed to the records' sha256, to the code that cuts and folds a word and to the record keys it reads (`Source.corpus_keys`; a record whose two names write one word counts once), so a refetch, a change to the fold or a change to which of a record's names are read rebuilds it; a single-word lookup it answers parses neither the records nor the names. Measured 2026-10-22 with both corpora fetched, a single-word `name_usage.py --no-wikidata` run took 2.5s with their indexes cached (12.5s before the cache, 14.8s on the run that builds it). A spaced word (`Ph. D.`) still scans every name, which took 8.6s, nearly all of it the FEC corpus's. `fetch --refresh` re-downloads the FEC's zip rather than re-reading the cached one, replacing it only once the new one is whole, and a corpus fetched for another cycle than `FEC_CYCLE` is refused on load. The `fec-individuals` fetch refuses a file whose lines stop matching the 21 columns it reads, or whose names stop holding a comma, rather than read it as a smaller or a different corpus. A `ukec-individuals` record is a distinct name too, not a donation or a `DonorId`: one name often holds several IDs, which seem to come with reports rather than with people (15,214 names held 24,473 on 2026-10-22, the IDs counted in `meta.json`'s detail; 2,663 names held several at one `RegulatedEntityId` alone, counted from the export and not kept, though some of those may be two people of one name), so a count of IDs would weight a name by how often it was reported. Its fetch reads the register's total from the JSON search under the same filter before and after the export, and refuses an export holding another number of rows, or a register that changed in between: the export ignores `rows=`, so the total is what tells a cut or capped export from the whole register. It refuses an export whose last row is cut short (Python's `csv` returns a body cut inside a row as a short last row, which the row count would otherwise take for a whole one), and a header other than the 29 columns `UKEC_COLUMNS` names, and counts into `meta.json`'s detail the rows it skips: rows not 29 fields, rows from a donor that is not an individual (the filter asks the server to leave those out), and individuals with no name. Past `UKEC_MAX_BAD` of either of the first two, or `UKEC_MAX_NAMELESS` of the third (4 of 42,355 rows were nameless on 2026-10-22), it refuses the export. The data date is when the first search answered, in UTC: the register states no date of its own, so the counts above, dated by the day they were fetched (2026-10-22), carry the data date 2026-10-23T03:13:19Z in their header. `tw-donors` keeps one zip per report under `reports/`, whole, its masked ID, address and phone columns included, as the FEC's zip is kept: the platform's download answers a report that does not exist and a fault alike, with a 500 and an incident number, so the fetch reads which reports exist from the platform's listing and refuses a listed report it cannot download. It reads each file's columns by name, since a 2020 report lacks a column the newest ones write, and refuses a column it does not know, a category it does not know, a row whose serial column names another report (the zip is not the one asked for), a listing that changes during the fetch, and a report past `TW_MAX_REPORT_BAD` of whose rows it reads past. It reads back the rows a stray quote took into one field (169 rows in 12 records of three reports on 2026-10-22, 8 of them two rows joined by two stray quotes), counts donor cells that hold no name (匿名, a dash) or a masked one, and caches a downloaded zip only once its rows are read, so a refused zip is asked for again. A zip the platform rebuilt under the name it had is not seen: delete `reports/` to download every report again. A layout check written from one surveyed file meets every other file only at fetch time: `tw-donors`' first three fetches were each refused partway, by a report with no income writing one row of dashes, by a 2020 report with 24 columns where the surveyed one had 25, and by a row with no category. Each refusal cost every download again until the source cached its zips, so a new source that downloads many files caches each one before its checks run. `tse-donors` caches nothing but the corpus: each election's national member (`receitas_candidatos__BRASIL.csv`) is read from the TSE's CDN in HTTP Range requests, 209 MB for the two elections where their zips are 1.82 GB (2026-10-22), because the TSE regenerates the zips every few days and a cached one would be stale within the week. Every range request carries `If-Range` with the ETag the HEAD gave if it is strong, else its `Last-Modified`, and every part must state the `Content-Range` asked for, so a zip replaced during the read is refused rather than read across two versions; a HEAD that states no length, neither a strong ETag nor a `Last-Modified`, or `Accept-Ranges` other than `bytes` is refused before the read. A fetch took 69s on 2026-10-22. It refuses a zip without the national member, a header that is not 60 columns or lacks its three columns, a `DS_ORIGEM_RECEITA` value it does not know (a renamed category, or the file no longer in Latin-1), malformed rows or lines folded into a field past `TSE_MAX_BAD`, an election with too few rows from individuals (`TSE_MIN_INDIVIDUALS`: a member cut to its header, or the donations moved under another category) too many without a name or registry name (`TSE_MAX_MISSING`, counting a placeholder such as `#NULO` or `***` as none: a masked column) or with a number in one (`TSE_MAX_NUMBERED`: a numeric column moved under a name), too few distinct filed or registry names (`TSE_MIN_DISTINCT`: a column masked with one word), and a corpus fetched for other elections than `TSE_ELECTIONS`. The CPF in each row is unmasked in the file, and like every other column it is read past, never stored or printed; a pair either of whose names holds more than `TSE_MAX_DIGITS` digits is dropped and counted, since campaigns sometimes typed a number, a CPF among them, into the name field. A record is a pair, so the counts in its reports (`tally`, the `name_usage.py` row) are of pairs, and a filed name standing beside two registry names counts twice. +A live source has no fixed snapshot to pin, so the cached copy is the pin: `meta.json` holds the source's data date and the sha256 of `raw.jsonl` and `corpus.jsonl`, checked on every load, and every report, the `name_usage.py` row included, carries the source and that date. Quote a count with the date, and expect a later fetch to differ. A local snapshot records which fetch it was read from, and `snapshot.py --local` refuses to compare it with a corpus fetched since, which would report the names the source gained and lost as moves: after a `fetch --refresh`, record again from the base checkout. ClinicalTrials.gov's terms ask anyone redistributing its data to credit it, show NLM's processing date and say what was changed; keeping the names in the cache and citing counts with that header meets them. The FEC's bar on soliciting and commercial use is in the credit line of every `fec-individuals` report, so a count quoted from it carries the bar with it. A record there is a distinct name, not a contribution or a contributor: one donor gives many times (34,264,742 rows held 1,893,964 names in the file dated 2026-10-18), and counting contributions would weight a shape by how often its bearers give. `tools/name_usage.py` looks a word up in each corpus's word index, cached as `wordindex.json` beside it on first use and keyed to the records' sha256, to the code that cuts and folds a word and to the record keys it reads (`Source.corpus_keys`; a record whose two names write one word counts once), so a refetch, a change to the fold or a change to which of a record's names are read rebuilds it; a single-word lookup it answers parses neither the records nor the names. Measured 2026-10-22 with both corpora fetched, a single-word `name_usage.py --no-wikidata` run took 2.5s with their indexes cached (12.5s before the cache, 14.8s on the run that builds it). A spaced word (`Ph. D.`) still scans every name, which took 8.6s, nearly all of it the FEC corpus's. For `orcid-researchers` and `patentsview-inventors`, whose extracts keep every record's boxes and country uncapped, `name_usage.py --boxes` adds a row by box and country, read from a second index, `boxindex.json.gz` beside the corpus (`cached_box_index`): per word and country the people whose given box or family box writes it, each country shown as a share of its people, so a word can be weighed in the country a language pack is for (`--country CC`). It is keyed to the extract's sha256, taken from the bytes as they are read, and to a fingerprint of the reader and the fold, so a refetch rebuilds it. Building it took 2 min 19 s for ORCID's 24,990,909 records (3,780,309 words in 6,660,756 word-and-country cells, 41 MB cached) on 2026-10-28. With both indexes cached a lookup took 10.4 s with `--boxes` against 5.3 s without, the difference nearly all reading the two indexes (the 369 MB extract hashes in a fraction of a second once the system has it cached), which is why the rows are asked for rather than always built; an extract that cannot be read fails its own row, naming the refresh. `fetch --refresh` re-downloads the FEC's zip rather than re-reading the cached one, replacing it only once the new one is whole, and a corpus fetched for another cycle than `FEC_CYCLE` is refused on load. The `fec-individuals` fetch refuses a file whose lines stop matching the 21 columns it reads, or whose names stop holding a comma, rather than read it as a smaller or a different corpus. A `ukec-individuals` record is a distinct name too, not a donation or a `DonorId`: one name often holds several IDs, which seem to come with reports rather than with people (15,214 names held 24,473 on 2026-10-22, the IDs counted in `meta.json`'s detail; 2,663 names held several at one `RegulatedEntityId` alone, counted from the export and not kept, though some of those may be two people of one name), so a count of IDs would weight a name by how often it was reported. Its fetch reads the register's total from the JSON search under the same filter before and after the export, and refuses an export holding another number of rows, or a register that changed in between: the export ignores `rows=`, so the total is what tells a cut or capped export from the whole register. It refuses an export whose last row is cut short (Python's `csv` returns a body cut inside a row as a short last row, which the row count would otherwise take for a whole one), and a header other than the 29 columns `UKEC_COLUMNS` names, and counts into `meta.json`'s detail the rows it skips: rows not 29 fields, rows from a donor that is not an individual (the filter asks the server to leave those out), and individuals with no name. Past `UKEC_MAX_BAD` of either of the first two, or `UKEC_MAX_NAMELESS` of the third (4 of 42,355 rows were nameless on 2026-10-22), it refuses the export. The data date is when the first search answered, in UTC: the register states no date of its own, so the counts above, dated by the day they were fetched (2026-10-22), carry the data date 2026-10-23T03:13:19Z in their header. `tw-donors` keeps one zip per report under `reports/`, whole, its masked ID, address and phone columns included, as the FEC's zip is kept: the platform's download answers a report that does not exist and a fault alike, with a 500 and an incident number, so the fetch reads which reports exist from the platform's listing and refuses a listed report it cannot download. It reads each file's columns by name, since a 2020 report lacks a column the newest ones write, and refuses a column it does not know, a category it does not know, a row whose serial column names another report (the zip is not the one asked for), a listing that changes during the fetch, and a report past `TW_MAX_REPORT_BAD` of whose rows it reads past. It reads back the rows a stray quote took into one field (169 rows in 12 records of three reports on 2026-10-22, 8 of them two rows joined by two stray quotes), counts donor cells that hold no name (匿名, a dash) or a masked one, and caches a downloaded zip only once its rows are read, so a refused zip is asked for again. A zip the platform rebuilt under the name it had is not seen: delete `reports/` to download every report again. A layout check written from one surveyed file meets every other file only at fetch time: `tw-donors`' first three fetches were each refused partway, by a report with no income writing one row of dashes, by a 2020 report with 24 columns where the surveyed one had 25, and by a row with no category. Each refusal cost every download again until the source cached its zips, so a new source that downloads many files caches each one before its checks run. `tse-donors` caches nothing but the corpus: each election's national member (`receitas_candidatos__BRASIL.csv`) is read from the TSE's CDN in HTTP Range requests, 209 MB for the two elections where their zips are 1.82 GB (2026-10-22), because the TSE regenerates the zips every few days and a cached one would be stale within the week. Every range request carries `If-Range` with the ETag the HEAD gave if it is strong, else its `Last-Modified`, and every part must state the `Content-Range` asked for, so a zip replaced during the read is refused rather than read across two versions; a HEAD that states no length, neither a strong ETag nor a `Last-Modified`, or `Accept-Ranges` other than `bytes` is refused before the read. A fetch took 69s on 2026-10-22. It refuses a zip without the national member, a header that is not 60 columns or lacks its three columns, a `DS_ORIGEM_RECEITA` value it does not know (a renamed category, or the file no longer in Latin-1), malformed rows or lines folded into a field past `TSE_MAX_BAD`, an election with too few rows from individuals (`TSE_MIN_INDIVIDUALS`: a member cut to its header, or the donations moved under another category) too many without a name or registry name (`TSE_MAX_MISSING`, counting a placeholder such as `#NULO` or `***` as none: a masked column) or with a number in one (`TSE_MAX_NUMBERED`: a numeric column moved under a name), too few distinct filed or registry names (`TSE_MIN_DISTINCT`: a column masked with one word), and a corpus fetched for other elections than `TSE_ELECTIONS`. The CPF in each row is unmasked in the file, and like every other column it is read past, never stored or printed; a pair either of whose names holds more than `TSE_MAX_DIGITS` digits is dropped and counted, since campaigns sometimes typed a number, a CPF among them, into the name field. A record is a pair, so the counts in its reports (`tally`, the `name_usage.py` row) are of pairs, and a filed name standing beside two registry names counts twice. `psc-individuals` caches an extract of each of the snapshot's 33 parts rather than the zips: 168 MB against 2.44 GB on 2026-10-22, each extract holding only the records and fields the corpus keeps. A download that fails costs one part, and running the fetch again resumes, keeping the parts already read for the same day; the set replaces the cached one only once every part is read, and a part that disappears mid-fetch (the files are overwritten each morning) is refused naming the refresh. The fetch refuses a snapshot where more than one individual in 10,000 has no `name_elements`, or a display name that is not its four boxes joined by spaces, or where fewer than half of the lines are individuals (87% were on 2026-10-22; a kind Companies House renamed would read as another kind), and counts what it skips into `meta.json`'s detail. It reads the cached extracts only as one whole snapshot of one date, and an extract written by another version of the extraction (its file name carries the version) is neither resumed from nor read. Companies House's data-products guidance says it imposes no rules on how the register is used and leaves data protection to the user; the dataset's data.gov.uk entry gives its licence as "TBD", and gov.uk's footer licenses its own page text under the Open Government Licence. The credit line says the names are personal data. diff --git a/tools/differential/local_core.py b/tools/differential/local_core.py index 29710f057..c360097be 100644 --- a/tools/differential/local_core.py +++ b/tools/differential/local_core.py @@ -17,7 +17,7 @@ import urllib.error import urllib.request import zipfile -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Callable, Iterable, Iterator, Mapping from dataclasses import dataclass from pathlib import Path from typing import Any, TypeVar @@ -59,6 +59,15 @@ class Source: #: (nepal-candidates' "devanagari"). meta.json records them, and a #: corpus written under other keys is refused on load. corpus_keys: tuple[str, ...] = ("name",) + #: For a source whose cached extract holds every record uncapped, its + #: name in two boxes and a country (orcid-researchers, + #: patentsview-inventors): the extract's path, and a reader yielding + #: each record as (given box, family box, country, weight), the + #: weight being the people the record stands for. + #: local_corpora.box_index counts words by box and country from them, + #: for tools/name_usage.py's per-country rows. + extract: Callable[[], Path] | None = None + boxes: Callable[[Path], Iterator[tuple[str, str, str, int]]] | None = None #: Attempts per request, for every source that retries. diff --git a/tools/differential/local_corpora.py b/tools/differential/local_corpora.py index 008b51102..9e365b10d 100644 --- a/tools/differential/local_corpora.py +++ b/tools/differential/local_corpora.py @@ -58,6 +58,12 @@ record keys it reads (cached_index), so a single-word lookup parses neither raw.jsonl's records nor the names (load() still reads both files for their sha256). +- boxindex.json.gz -- for a source whose extract holds every record's + boxes and country (orcid-researchers, patentsview-inventors), its + words counted by box and country over the whole extract, uncapped, + for tools/name_usage.py's per-country rows; written on first use and + keyed to the extract's bytes and the code that counts them + (cached_box_index). A live source has no fixed snapshot to pin, so the cached copy IS the pin: a count quoted from it names the data date the header prints, and @@ -81,6 +87,7 @@ import functools import dataclasses import difflib +import gzip import hashlib import inspect import json @@ -90,6 +97,7 @@ import time import types import unicodedata +import zlib from collections import Counter from collections.abc import Callable, Iterable from dataclasses import dataclass, field @@ -113,8 +121,10 @@ from local_fec import check_fec, fetch_fec, renew_fec # noqa: E402 from local_jrc import JRC, check_jrc, fetch_jrc, renew_jrc # noqa: E402 from local_nepal import check_nepal, fetch_nepal # noqa: E402 -from local_orcid import ORCID, check_orcid, fetch_orcid, renew_orcid # noqa: E402 -from local_patentsview import PV, check_pv, fetch_pv, renew_pv # noqa: E402 +from local_orcid import (ORCID, check_orcid, extract_path as orcid_extract, # noqa: E402 + fetch_orcid, orcid_boxes, renew_orcid) +from local_patentsview import (PV, check_pv, extract_path as pv_extract, # noqa: E402 + fetch_pv, pv_boxes, renew_pv) from local_psc import PSC, check_psc, fetch_psc, renew_psc # noqa: E402 from local_tse import check_tse, fetch_tse # noqa: E402 from local_tw import fetch_tw # noqa: E402 @@ -122,6 +132,7 @@ RAW, CORPUS, META, SNAPSHOT = "raw.jsonl", "corpus.jsonl", "meta.json", "snapshot.jsonl" INDEX = "wordindex.json" +BOX_INDEX = "boxindex.json.gz" META_VERSION = 2 # 2: corpus.jsonl's sha256, and the source's detail nested #: The keys load() needs in meta.json, with their types. _META_KEYS = {"version": int, "source": str, "data_date": str, "records": int, @@ -230,11 +241,12 @@ def index(self) -> WordIndex: "(CC0 1.0; ORCID asks for credit and respect for the privacy " "of the people in it)", unit="records", fetch=fetch_orcid, renew=renew_orcid, check=check_orcid, - corpus_keys=("name", "credit")), + corpus_keys=("name", "credit"), extract=orcid_extract, boxes=orcid_boxes), PV: Source( credit="PatentsView granted-patent inventors, 1976-2025, from the USPTO " "Open Data Portal (CC BY 4.0; credit PatentsView)", - unit="names", fetch=fetch_pv, renew=renew_pv, check=check_pv), + unit="names", fetch=fetch_pv, renew=renew_pv, check=check_pv, + extract=pv_extract, boxes=pv_boxes), } @@ -1210,6 +1222,111 @@ def cached_index(local: Local) -> WordIndex: return index +@dataclass(frozen=True) +class BoxIndex: + """A fielded source's words by box and country, counted over its + whole extract rather than the capped corpus: per country the weight + of every record ("" for none), and per word key and country the + weight of the records whose given box, or family box, writes it (once + a record, however often the box repeats it). The weight is the people + a record stands for (Source.boxes): one ORCID holder, or the inventors + PatentsView counts under a name.""" + totals: dict[str, int] + given: dict[str, dict[str, int]] + family: dict[str, dict[str, int]] + + +def box_index(rows: Iterable[tuple[str, str, str, int]]) -> BoxIndex: + """The BoxIndex of `rows`, (given, family, country, weight) as a + Source.boxes reader yields them, each box cut and folded into words + as the word index cuts and folds a name (words).""" + totals: Counter[str] = Counter() + boxes: tuple[dict[str, Counter[str]], dict[str, Counter[str]]] = ({}, {}) + for given, family, country, weight in rows: + totals[country] += weight + for box, text in zip(boxes, (given, family)): + for key in set(words(text)): + box.setdefault(key, Counter())[country] += weight + return BoxIndex(totals=dict(totals), + given={k: dict(v) for k, v in boxes[0].items()}, + family={k: dict(v) for k, v in boxes[1].items()}) + + +def box_fingerprint(source: str) -> str: + """code_fingerprint of what turns `source`'s extract into its + BoxIndex: its Source.boxes reader, box_index and the fold the word + index shares (words, word_key, _strippable, _keyed, _FOLD_CONSTANTS), + and BoxIndex's fields.""" + reader = SOURCES[source].boxes + assert reader is not None + module = globals() + return code_fingerprint( + [reader, box_index, words, word_key, _strippable, _keyed], + [*(_stable(module[name]) for name in _FOLD_CONSTANTS), + repr([f.name for f in dataclasses.fields(BoxIndex)])]) + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as held: + while chunk := held.read(local_core.CHUNK): + digest.update(chunk) + return digest.hexdigest() + + +def cached_box_index(source: str) -> BoxIndex: + """`source`'s BoxIndex, read from boxindex.json.gz beside the corpus + where its key matches, else built from the cached extract and written + there. The key is the extract's sha256, taken from the bytes on disk + as they are read, and box_fingerprint(source), so a refetch or a + change to how a word is cut, folded or counted rebuilds it. A file + that is not an index, or not this one, is built over, with a line on + stderr: it is derived from the extract, never the extract itself. + ValueError naming the refresh where the extract is missing or cannot + be read.""" + found = SOURCES[source] + if found.extract is None or found.boxes is None: + raise ValueError(f"{source} keeps no extract with its names' boxes") + extract = found.extract() + if not extract.exists(): + raise ValueError(f"{extract} is missing{_again(source)}") + path = directory(source, create=False) / BOX_INDEX + key = f"{_file_sha256(extract)}:{box_fingerprint(source)}" + if path.exists(): + try: + data = json.loads(gzip.decompress(path.read_bytes())) + if (isinstance(data, dict) and data.get("key") == key + and _is_counts(data.get("totals")) + and all(isinstance(data.get(box), dict) + and all(_is_counts(v) for v in data[box].values()) + for box in ("given", "family"))): + return BoxIndex(totals=data["totals"], given=data["given"], + family=data["family"]) + why = "built from another extract or by another fold, or not an index" + except (OSError, ValueError, EOFError, zlib.error) as exc: + why = str(exc) + print(f"rebuilding the box index at {path}: {why}", file=sys.stderr) + else: + print(f"building the box index of {source} from {extract.name} ...", file=sys.stderr) + try: + index = box_index(found.boxes(extract)) + except (EOFError, zlib.error, KeyError, AttributeError, TypeError, RuntimeError, + ValueError) as exc: + # A truncated or damaged extract raises any of these from gzip, + # json or the reader (an empty one, StopIteration from inside a + # generator, as RuntimeError); one row fails, not the run. + raise ValueError(f"{extract} could not be read ({type(exc).__name__}: {exc})" + f"{_again(source)}") from None + if _file_sha256(extract) != key.split(":", 1)[0]: + raise ValueError(f"{extract} changed while it was read{_again(source)}") + body = json.dumps({"key": key, **dataclasses.asdict(index)}, ensure_ascii=False) + try: + corpus_build.write_atomic(path, gzip.compress(body.encode(), 6, mtime=0)) + except OSError as exc: + print(f"could not cache the box index at {path}: {exc}", file=sys.stderr) + return index + + def phrase_counts(local: Local, keys: list[str]) -> tuple[int, int]: """The records writing the words `keys` in a row (a spaced query, 'Ph. D.' as ['ph', 'd']), and those where such a run starts after diff --git a/tools/differential/local_orcid.py b/tools/differential/local_orcid.py index 1095b197b..edb697cc3 100644 --- a/tools/differential/local_orcid.py +++ b/tools/differential/local_orcid.py @@ -492,6 +492,20 @@ def read(lines: Iterator[str]) -> Iterator[dict[str, str]]: "countries_at_the_cap": sum(n == ORCID_CAP for n in by_country.values())}) +def orcid_boxes(path: Path) -> Iterator[tuple[str, str, str, int]]: + """Every record of the extract at `path` as (given, family, country, + 1): one holder each, uncapped, for local_corpora.box_index. The + extract was checked when it was fetched (_orcid_read); a file whose + header names another release is refused.""" + with gzip.open(path, "rt", encoding="utf-8", newline="\n") as lines: + if json.loads(next(lines)).get("file") != ORCID_FILE: + raise ValueError(f"{path} is not an extract of {ORCID_FILE}{_again(ORCID)}") + for line in lines: + record = json.loads(line) + yield (record.get("given", ""), record.get("family", ""), + record.get("country", ""), 1) + + def _kept() -> str: return f"{ORCID_FILE}, at most {ORCID_CAP:,} records a country" diff --git a/tools/differential/local_patentsview.py b/tools/differential/local_patentsview.py index 80c94d1d6..f10b01c4c 100644 --- a/tools/differential/local_patentsview.py +++ b/tools/differential/local_patentsview.py @@ -421,6 +421,20 @@ def read(lines: Iterator[str]) -> Iterator[dict[str, str]]: "countries_at_the_cap": sum(n == PV_CAP for n in by_country.values())}) +def pv_boxes(path: Path) -> Iterator[tuple[str, str, str, int]]: + """Every name of the extract at `path` as (given, family, country, + people), uncapped, for local_corpora.box_index. The extract was + checked when it was fetched (_pv_read); a file whose header names + another release is refused.""" + with gzip.open(path, "rt", encoding="utf-8", newline="\n") as lines: + if json.loads(next(lines)).get("release") != PV_RELEASE: + raise ValueError(f"{path} is not an extract of the {PV_RELEASE} release{_again(PV)}") + for line in lines: + record = json.loads(line) + yield (record.get("given", ""), record.get("family", ""), + record.get("country", ""), record["people"]) + + def _kept() -> str: return f"release {PV_RELEASE}, at most {PV_CAP:,} names a country" diff --git a/tools/name_usage.py b/tools/name_usage.py index 33b0e43b3..a3f05fa9b 100644 --- a/tools/name_usage.py +++ b/tools/name_usage.py @@ -19,6 +19,7 @@ uv run python tools/name_usage.py In Bay Bin # non-US given names uv run python tools/name_usage.py --fetch-nppes ma do # credentials too uv run python tools/name_usage.py --rank 50 --unlisted # top credentials + uv run python tools/name_usage.py --boxes del --country PH # by box, country A suffix that collides with a name is the same question from the other side: `ma`, `do` and `ba` are credentials and surnames both, and @@ -111,6 +112,18 @@ family, comma, given in capitals by a filer's form, so a count there is a word typed into a field rather than into a whole name. Not a census of anyone: it counts what each source holds. + Asked for with --boxes (or --country), a fielded source whose + extract keeps every record's boxes and country (orcid-researchers, + patentsview-inventors) adds a row "boxes by country": the people whose given box and whose family box write the + word, counted over the whole extract rather than the capped corpus, + and the --countries N countries with the most of them in each box, + each as a share of that country's people, the records stating no + country last; --country CC lists a country there too. PatentsView + counts the inventors filing under a name, ORCID one holder a record, + and a country is where they work or live, of researchers and US-patent + inventors, not of a country's population. A spaced word is not + looked up there. The first lookup builds each index (2 min 19 s for + ORCID's 25M records on 2026-10-28) and caches it beside the corpus. Both US sources spell names in ASCII letters only, without spaces or punctuation (O'Brien is OBRIEN), so a word is folded the same way @@ -153,7 +166,7 @@ from typing import TYPE_CHECKING, Any, Literal, TypeVar if TYPE_CHECKING: # loaded by path at run time, _local_corpora - from local_corpora import Local + from local_corpora import BoxIndex, Local CENSUS_URL = "https://www2.census.gov/topics/genealogy/2010surnames/names.zip" CENSUS_FILE = "census2010_surnames.zip" @@ -264,6 +277,11 @@ class Usage: #: (rare) to 13 (extremely common), highest first. Not a count, so #: `given` stays None beside it. given_bands: list[tuple[str, int]] = field(default_factory=list) + #: Per country, the people whose given or family box writes the word + #: and the people of that country, a fielded local corpus's box index + #: only: (country, people writing it, people in all). + given_shares: list[tuple[str, int, int]] = field(default_factory=list) + family_shares: list[tuple[str, int, int]] = field(default_factory=list) note: str = "" #: Where the word can end a name across the family/given boundary, #: the ends-with row only. @@ -1111,6 +1129,71 @@ def local_rows(word: str, locals_: list[tuple[str, Local | None, str]]) -> list[ for source, local, error in locals_] +def load_box_indexes(locals_: list[tuple[str, Local | None, str]] + ) -> list[tuple[str, BoxIndex | None, str]]: + """The box index of every fetched local corpus whose extract holds its + names' boxes and a country (local_corpora.cached_box_index): (source, + the index or None, the error that stopped it), built or read once a + run however many words are looked up. A corpus not fetched gets None + and no error; one that did not load, its load's error.""" + module = _local_corpora() + found: list[tuple[str, BoxIndex | None, str]] = [] + for source, local, error in locals_: + if module.SOURCES[source].boxes is None: + continue + if error or local is None: + found.append((source, None, error)) + continue + index, failed = _attempt(lambda source=source: module.cached_box_index(source)) + found.append((source, index, failed)) + return found + + +def box_usage(word: str, source: str, index: BoxIndex | None, error: str, top: int, + wanted: list[str]) -> Usage: + """One fielded local corpus's answer by box and country: the people + whose given box and whose family box write the word, over the whole + extract, and the `top` countries by people writing it in each box, + with `wanted` countries always among them and the records stating no + country last, each with the people of that country it is a share of.""" + module = _local_corpora() + usage = Usage(f"{source} (boxes by country)") + if error: + usage.status, usage.note = "failed", error + return usage + if index is None: + usage.status = "not downloaded" + usage.note = f"not fetched; {module.fetch_command(source)}" + return usage + keys = module.words(word) + if len(keys) != 1: + usage.status = "unrepresentable" + usage.note = ("nothing in it is left once periods and punctuation go" if not keys + else "a box index holds single words; a spaced word is not looked up") + return usage + total = sum(index.totals.values()) + + def shares(box: dict[str, dict[str, int]]) -> list[tuple[str, int, int]]: + found = box.get(keys[0], {}) + # Records with no country stated (most of ORCID's) would lead + # every list and say nothing of where a name is borne: they go + # last, after the countries ranked and those asked for. + ranked = sorted((c for c in found if c), key=lambda c: (-found[c], c))[:top] + ranked += [c for c in wanted if c not in ranked] + ranked += [""] if found.get("") else [] + return [(c or "--", found.get(c, 0), index.totals.get(c, 0)) for c in ranked] + + usage.given = sum(index.given.get(keys[0], {}).values()) + usage.family = sum(index.family.get(keys[0], {}).values()) + usage.given_shares = shares(index.given) + usage.family_shares = shares(index.family) + usage.note = _join(module.SOURCES[source].credit, + f"counted over the whole extract, {total:,} people in " + f"{sum(1 for c in index.totals if c):,} countries; " + "-- is no country stated") + return usage + + def _guarded_local_usage(word: str, source: str, local: Local | None) -> Usage: """local_usage, a corpus failing its own row: the records are read first here (building an uncached index, or scanning the names for a @@ -1810,6 +1893,11 @@ def render(word: str, usages: list[Usage]) -> str: if u.given_bands: lines.append(f" {'':<{width}} given band by country: " + ", ".join(f"{c} {b:X}" for c, b in u.given_bands)) + for role, shares in (("given", u.given_shares), ("family", u.family_shares)): + if shares: + lines.append(f" {'':<{width}} {role} box by country: " + ", ".join( + f"{c} {n:,} ({_share(n / of) if of else '-'} of {of:,})" + for c, n, of in shares)) for split in u.splits: example = "; ".join( split.family_examples @@ -1974,6 +2062,15 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--countries", type=_positive, default=5, metavar="N", help="countries to list per role (citizenship, " "or nam_dict's bands)") + parser.add_argument("--boxes", action="store_true", + help="add the fielded local corpora's rows by box and " + "country (orcid-researchers, patentsview-inventors), " + "read from an index each builds on first use " + "(minutes) and caches") + parser.add_argument("--country", action="append", default=[], metavar="CC", + help="a country to list in every by-country row even " + "outside the top N (an ISO code, repeatable; " + "implies --boxes)") parser.add_argument("--json", action="store_true") parser.add_argument("--refresh-index", action="store_true", help="fetch the CJK name index again") @@ -2007,6 +2104,8 @@ def main(argv: list[str] | None = None) -> int: lambda: load_index(download=True, refresh=args.refresh_index)) locals_ = load_locals() + wanted = list(dict.fromkeys(c.strip().upper() for c in args.country if c.strip())) + boxed = load_box_indexes(locals_) if args.boxes or wanted else [] report = {} for word in words: census_row, ssa_row = census_usage(word, census), ssa_usage(word, ssa) @@ -2018,7 +2117,9 @@ def main(argv: list[str] | None = None) -> int: if error: row.status, row.note = "failed", error usages = [census_row, ssa_row, namdict_row, nppes_row, - *local_rows(word, locals_)] + *local_rows(word, locals_), + *(box_usage(word, source, index, error, args.countries, wanted) + for source, index, error in boxed)] if not args.no_wikidata: usages.append(wikidata_usage(word, args.countries)) if is_cjk(word):