From 8b395d8b8bd49f873b48ce0499e26c42dbd9c4af Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 11 Oct 2026 20:17:55 -0700 Subject: [PATCH 1/5] Add tools/name_usage.py: count how often a word is borne as a given or family name Vocabulary membership turns on decisions.md#vocabulary-collisions: a word joins a positionally-claimed set only if it is not borne as a name in the position the claim acts on. That check was run from memory, and memory missed what the data shows at once -- shehu is an Albanian surname as well as a Hausa given name, and herra (a title since #606) is a US Census surname with about 640 bearers. The tool reports counts per source and role: US Census 2010 surnames (downloaded once into a cache outside the repo), SSA given names (a hand download, since ssa.gov refuses scripts), and live Wikidata given- and family-name statements with countries of citizenship, a lower bound. It reports; it does not decide. Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_name_usage_tool.py | 129 +++++++++++ tools/name_usage.py | 354 +++++++++++++++++++++++++++++++ 2 files changed, 483 insertions(+) create mode 100644 tests/v2/test_name_usage_tool.py create mode 100644 tools/name_usage.py diff --git a/tests/v2/test_name_usage_tool.py b/tests/v2/test_name_usage_tool.py new file mode 100644 index 00000000..0d307aa1 --- /dev/null +++ b/tests/v2/test_name_usage_tool.py @@ -0,0 +1,129 @@ +"""Offline checks on tools/name_usage.py, the name-collision lookup. + +The tool's sources are a downloaded file, a hand-downloaded file and a +live SPARQL endpoint, none of which a test may touch. What CAN go wrong +without them is the folding and the bookkeeping: a word that misses the +Census because it was folded differently from the Census's own spelling +reads as "under 100 bearers", which is exactly the false all-clear the +tool exists to prevent. These pin those pieces. + +tools/ is not a package, so the module is loaded by path, as +_differential_fixtures.py loads compare.py. +""" +import importlib.util +import io +import sys +import zipfile +from pathlib import Path +from types import ModuleType + +import pytest + +_PATH = Path(__file__).parents[2] / "tools" / "name_usage.py" + + +def _load() -> ModuleType: + spec = importlib.util.spec_from_file_location("name_usage", _PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + # Registered before it runs: @dataclass resolves the module's + # postponed annotations through sys.modules, and fails on a module + # loaded by path that is not in it. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +tool = _load() + +_CENSUS = """name,rank,count,prop100k,cum_prop100k,pctwhite,pctblack,pctapi,pctaian,pct2prace,pcthispanic +OBRIEN,258,118557,40.19,0,93.85,1.29,0.76,0.27,1.51,2.32 +SHEHU,39212,561,0.19,0,94.47,3.92,(S),0,0.89,(S) +""" + + +@pytest.mark.parametrize(("word", "key"), [ + ("O'Brien", "OBRIEN"), # apostrophe dropped, as the Census spells it + ("de la Cruz", "DELACRUZ"), # spaces dropped + ("Ó Murchú", "OMURCHU"), # diacritics folded away + ("关", ""), # nothing left: not representable +]) +def test_census_key_folds_a_word_the_way_the_census_spells_it(word: str, key: str) -> None: + assert tool.census_key(word) == key + + +def test_census_usage_finds_a_folded_surname_and_says_how() -> None: + usage = tool.census_usage("O'Brien", tool.read_census(_CENSUS)) + assert usage.family == 118557 + assert "looked up as OBRIEN" in usage.note + + +def test_census_usage_omits_suppressed_shares() -> None: + # (S) marks a share the Census suppressed for privacy; printing it as a + # percentage would invent a number. + note = tool.census_usage("shehu", tool.read_census(_CENSUS)).note + assert "asian/PI" not in note and "hispanic" not in note + assert "white 94.47%" in note + + +def test_census_usage_distinguishes_absent_from_unrepresentable() -> None: + table = tool.read_census(_CENSUS) + absent = tool.census_usage("marshal", table) + assert absent.family == 0 + unrepresentable = tool.census_usage("关", table) + assert unrepresentable.family is None # no answer, not a zero + + +def _ssa_zip(files: dict[str, str]) -> zipfile.ZipFile: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as zf: + for name, text in files.items(): + zf.writestr(name, text) + buffer.seek(0) + return zipfile.ZipFile(buffer) + + +def test_read_ssa_totals_each_name_by_sex_across_years() -> None: + totals = tool.read_ssa(_ssa_zip({ + "yob1990.txt": "Justice,F,40\nJustice,M,60\n", + "yob2000.txt": "Justice,M,100\nMarshal,M,7\n", + "NationalReadMe.pdf": "not data", + })) + assert totals["justice"] == {"F": 40, "M": 160} + usage = tool.ssa_usage("Justice", totals) + assert usage.given == 200 + + +def test_ssa_usage_reports_a_missing_download_without_a_count() -> None: + usage = tool.ssa_usage("justice", None) + assert usage.given is None and "not downloaded" in usage.note + + +def test_wikidata_query_matches_the_word_in_every_language_and_escapes_it() -> None: + query = tool.wikidata_query('Say "hi"', languages=("mul", "zh-hans")) + assert r'"Say \"hi\""@mul' in query + assert r'"Say \"hi\""@zh-hans' in query + + +def test_read_wikidata_keeps_stateless_people_out_of_the_country_list() -> None: + usage = tool.read_wikidata( + totals=[{"role": "family", "n": "180"}, {"role": "given", "n": "1"}], + by_country=[ + {"role": "family", "n": "148"}, # no country bound + {"role": "family", "country": "Albania", "n": "11"}, + {"role": "family", "country": "Nigeria", "n": "20"}, + {"role": "given", "country": "Nigeria", "n": "1"}, + ], + top=5) + assert (usage.given, usage.family) == (1, 180) + assert usage.family_countries == [("Nigeria", 20), ("Albania", 11)] + assert usage.family_stateless == 148 + + +def test_merge_sums_the_case_variants_per_key() -> None: + merged = tool._merge( + [{"role": "given", "n": "2"}, {"role": "given", "n": "3"}, + {"role": "family", "n": "1"}], + ("role",)) + assert sorted((row["role"], row["n"]) for row in merged) == [ + ("family", "1"), ("given", "5")] diff --git a/tools/name_usage.py b/tools/name_usage.py new file mode 100644 index 00000000..4e623de2 --- /dev/null +++ b/tools/name_usage.py @@ -0,0 +1,354 @@ +"""How often is a word borne as a given name, and as a family name? + +WHY THIS IS IN THE TREE. Vocabulary membership is decided by +decisions.md#vocabulary-collisions: a word may join a set the parser +claims by position (titles, never-given particles) only if it is not +borne as a name IN THE POSITION THE CLAIM ACTS ON. Until this tool, +that check was run from memory, and memory missed collisions the data +shows at once: `shehu` is an Albanian surname as well as a Hausa given +name, and `herra` (a title since #606) is borne by about 640 people in +the 2010 US Census. This answers the question with counts instead. + + uv run python tools/name_usage.py shehu greve marshal + uv run python tools/name_usage.py --no-wikidata knight # offline + uv run python tools/name_usage.py --json justice # machine-readable + +It reports; it does not decide. Read the counts against the rule's +position qualifier: given-name bearers threaten a title or a leading +particle in every culture, while family-name bearers threaten one only +where the family name is written first -- which is what Wikidata's +per-country breakdown is for. + +Sources, each answering for one role and one population: + +- US Census 2010 surnames (public domain): every surname with 100 or + more bearers, with rank and ethnic shares. Downloaded once into the + cache. Census names are ASCII with spaces and punctuation removed + (O'Brien is OBRIEN), so a word is folded the same way before lookup. +- US Social Security given names, 1880 onward (CC0 via data.gov), + counted by sex across all years. ssa.gov refuses scripted downloads, + so this one is fetched by hand -- the tool prints where to put it. +- Wikidata (CC0), queried live: people whose given name (P735) or + family name (P734) is an item labelled with the word, with their + countries of citizenship. A LOWER BOUND: many people have no P735 + statement at all (Shehu Shagari has none), so a small count proves + little and a zero proves nothing. + +Data is cached outside the repository, in $NAMEPARSER_TOOLS_CACHE or +~/.cache/nameparser-tools: third-party data stays out of the tree, and +the Census file alone is 9 MB. +""" +from __future__ import annotations + +import argparse +import csv +import io +import json +import os +import sys +import unicodedata +import urllib.parse +import urllib.request +import zipfile +from collections import Counter +from dataclasses import asdict, dataclass, field +from pathlib import Path + +CENSUS_URL = "https://www2.census.gov/topics/genealogy/2010surnames/names.zip" +CENSUS_MEMBER = "Names_2010Census.csv" +SSA_URL = "https://www.ssa.gov/oact/babynames/names.zip" +SSA_FILE = "ssa_names.zip" +WIKIDATA_ENDPOINT = "https://query.wikidata.org/sparql" +USER_AGENT = ("nameparser-name-usage/0.1 " + "(https://github.com/derek73/python-nameparser)") + +#: Label languages a name item is matched in. `mul` is Wikidata's +#: language-independent label, which most name items now carry; the +#: rest catch items labelled only in a language of their own. Matching +#: one exact literal per language keeps the query on the label index -- +#: a FILTER over every label would time out. +WIKIDATA_LANGUAGES = ( + "mul", "en", "de", "fr", "es", "it", "pt", "nl", "sv", "da", "nb", + "fi", "et", "lv", "lt", "pl", "cs", "sk", "sl", "hr", "sr", "bs", + "hu", "ro", "bg", "ru", "uk", "el", "tr", "az", "sq", "ga", "cy", + "is", "ar", "fa", "ur", "he", "hi", "bn", "ta", "te", "ja", "ko", + "vi", "th", "id", "ms", "tl", "sw", "ha", "yo", "ig", "am", "so", + # Chinese labels sit under script and region tags far more often + # than plain `zh`: the 关 surname item is labelled only zh-hans and + # zh-cn, so without these a simplified-script surname finds nothing. + "zh", "zh-hans", "zh-hant", "zh-cn", "zh-tw", "zh-hk", "yue", +) + + +@dataclass +class Usage: + """One source's answer for one word.""" + + source: str + given: int | None = None + family: int | None = None + #: Top countries of citizenship per role, Wikidata only. + given_countries: list[tuple[str, int]] = field(default_factory=list) + family_countries: list[tuple[str, int]] = field(default_factory=list) + #: People of that role with no citizenship statement, kept out of the + #: country lists: they are often the largest group and say nothing + #: about which culture bears the name. + given_stateless: int = 0 + family_stateless: int = 0 + note: str = "" + + +def cache_dir() -> Path: + path = Path(os.environ.get("NAMEPARSER_TOOLS_CACHE", + Path.home() / ".cache" / "nameparser-tools")) + path.mkdir(parents=True, exist_ok=True) + return path + + +# -- US Census surnames -------------------------------------------------- + +def census_key(word: str) -> str: + """Fold a word the way the Census list spells names: diacritics + dropped, then everything but ASCII letters removed, upper-cased.""" + ascii_word = unicodedata.normalize("NFKD", word).encode( + "ascii", "ignore").decode() + return "".join(ch for ch in ascii_word if ch.isalpha()).upper() + + +def read_census(csv_text: str) -> dict[str, dict[str, str]]: + return {row["name"]: row for row in csv.DictReader(io.StringIO(csv_text))} + + +def load_census() -> dict[str, dict[str, str]]: + path = cache_dir() / "census2010_surnames.zip" + if not path.exists(): + print(f"downloading {CENSUS_URL} ...", file=sys.stderr) + _download(CENSUS_URL, path) + with zipfile.ZipFile(path) as zf: + return read_census(zf.read(CENSUS_MEMBER).decode("utf-8")) + + +def census_usage(word: str, table: dict[str, dict[str, str]]) -> Usage: + key = census_key(word) + if not key: + return Usage("US Census 2010 surnames", + note="not representable in the Census's ASCII spelling") + row = table.get(key) + if row is None: + return Usage("US Census 2010 surnames", family=0, + note="under 100 bearers, or absent") + shares = ", ".join( + f"{label} {row[col]}%" for label, col in ( + ("white", "pctwhite"), ("black", "pctblack"), + ("asian/PI", "pctapi"), ("hispanic", "pcthispanic")) + if row[col] not in ("(S)", "")) + note = f"rank {int(row['rank']):,}; {shares}" + if key != word.upper(): + note += f"; looked up as {key}" + return Usage("US Census 2010 surnames", family=int(row["count"]), + note=note) + + +# -- US Social Security given names -------------------------------------- + +def read_ssa(zf: zipfile.ZipFile) -> dict[str, Counter[str]]: + """Total each given name's births by sex across every yobYYYY.txt.""" + totals: dict[str, Counter[str]] = {} + for member in zf.namelist(): + if not (member.startswith("yob") and member.endswith(".txt")): + continue + for line in zf.read(member).decode("utf-8").splitlines(): + if not line.strip(): + continue + name, sex, count = line.split(",") + totals.setdefault(name.lower(), Counter())[sex] += int(count) + return totals + + +def load_ssa() -> dict[str, Counter[str]] | None: + path = cache_dir() / SSA_FILE + if not path.exists(): + return None + with zipfile.ZipFile(path) as zf: + return read_ssa(zf) + + +def ssa_missing_message() -> str: + return (f"SSA given names not downloaded. ssa.gov refuses scripted " + f"downloads, so open {SSA_URL} in a browser and save it as " + f"{cache_dir() / SSA_FILE}") + + +def ssa_usage(word: str, table: dict[str, Counter[str]] | None) -> Usage: + source = "US SSA given names 1880-" + if table is None: + return Usage(source, note="not downloaded (see the end of the report)") + sexes = table.get(census_key(word).lower()) + if not sexes: + return Usage(source, given=0, note="under 5 births in any year, or absent") + split = ", ".join(f"{sex} {n:,}" for sex, n in sorted(sexes.items())) + return Usage(source, given=sum(sexes.values()), note=split) + + +# -- Wikidata ------------------------------------------------------------ + +def wikidata_query(word: str, languages: tuple[str, ...] = WIKIDATA_LANGUAGES, + ) -> str: + """People bearing `word` as given (P735) or family (P734) name, one + row per role and country of citizenship. A person with no P27 lands + under an empty country; one with two counts under both, so per-country + counts may sum past the role's total -- the total comes from the + distinct count, not from the sum.""" + literal = json.dumps(word, ensure_ascii=False) + values = " ".join(f"{literal}@{lang}" for lang in languages) + return f""" +SELECT ?role ?country (COUNT(DISTINCT ?p) AS ?n) WHERE {{ + VALUES ?label {{ {values} }} + ?item rdfs:label ?label . + {{ ?p wdt:P735 ?item . BIND("given" AS ?role) }} + UNION + {{ ?p wdt:P734 ?item . BIND("family" AS ?role) }} + OPTIONAL {{ ?p wdt:P27 ?c . ?c rdfs:label ?country . + FILTER(LANG(?country) = "en") }} +}} GROUP BY ?role ?country""" + + +def wikidata_totals_query(word: str, + languages: tuple[str, ...] = WIKIDATA_LANGUAGES, + ) -> str: + literal = json.dumps(word, ensure_ascii=False) + values = " ".join(f"{literal}@{lang}" for lang in languages) + return f""" +SELECT ?role (COUNT(DISTINCT ?p) AS ?n) WHERE {{ + VALUES ?label {{ {values} }} + ?item rdfs:label ?label . + {{ ?p wdt:P735 ?item . BIND("given" AS ?role) }} + UNION + {{ ?p wdt:P734 ?item . BIND("family" AS ?role) }} +}} GROUP BY ?role""" + + +def _bindings(response: dict) -> list[dict[str, str]]: # type: ignore[type-arg] + return [{key: cell["value"] for key, cell in row.items()} + for row in response["results"]["bindings"]] + + +def read_wikidata(totals: list[dict[str, str]], + by_country: list[dict[str, str]], top: int) -> Usage: + usage = Usage("Wikidata people (lower bound)", given=0, family=0) + for row in totals: + setattr(usage, row["role"], int(row["n"])) + for role in ("given", "family"): + rows = [row for row in by_country if row["role"] == role] + counts = sorted(((row["country"], int(row["n"])) + for row in rows if row.get("country")), + key=lambda item: -item[1]) + setattr(usage, f"{role}_countries", counts[:top]) + setattr(usage, f"{role}_stateless", + sum(int(row["n"]) for row in rows if not row.get("country"))) + return usage + + +def wikidata_usage(word: str, top: int) -> Usage: + forms = list(dict.fromkeys([word, word.capitalize()])) + totals: list[dict[str, str]] = [] + by_country: list[dict[str, str]] = [] + try: + for form in forms: + totals += _bindings(_sparql(wikidata_totals_query(form))) + by_country += _bindings(_sparql(wikidata_query(form))) + except OSError as exc: + return Usage("Wikidata people (lower bound)", note=f"query failed: {exc}") + return read_wikidata(_merge(totals, ("role",)), + _merge(by_country, ("role", "country")), top) + + +def _merge(rows: list[dict[str, str]], + keys: tuple[str, ...]) -> list[dict[str, str]]: + """Sum counts from the case variants' queries. A person whose name + item carries both spellings would count twice; name items carry one + canonical label, so in practice the variants find different items.""" + merged: dict[tuple[str, ...], int] = {} + for row in rows: + key = tuple(row.get(k, "") for k in keys) + merged[key] = merged.get(key, 0) + int(row["n"]) + return [dict(zip(keys, key), n=str(n)) for key, n in merged.items()] + + +def _sparql(query: str) -> dict: # type: ignore[type-arg] + url = WIKIDATA_ENDPOINT + "?" + urllib.parse.urlencode({"query": query}) + request = urllib.request.Request(url, headers={ + "Accept": "application/sparql-results+json", + "User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=90) as response: + return json.load(response) # type: ignore[no-any-return] + + +def _download(url: str, path: Path) -> None: + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=120) as response: + data = response.read() + tmp = path.with_suffix(".part") + tmp.write_bytes(data) + tmp.replace(path) + + +# -- output -------------------------------------------------------------- + +def _count(n: int | None) -> str: + return "-" if n is None else f"{n:,}" + + +def render(word: str, usages: list[Usage]) -> str: + lines = [word] + width = max(len(u.source) for u in usages) + lines.append(f" {'source':<{width}} {'given':>9} {'family':>9}") + for u in usages: + lines.append(f" {u.source:<{width}} {_count(u.given):>9} " + f"{_count(u.family):>9}") + if u.note: + lines.append(f" {'':<{width}} {u.note}") + for role, countries, stateless in ( + ("given", u.given_countries, u.given_stateless), + ("family", u.family_countries, u.family_stateless)): + if countries or stateless: + parts = [f"{c} {n}" for c, n in countries] + if stateless: + parts.append(f"{stateless} with no citizenship stated") + lines.append(f" {'':<{width}} {role} by citizenship: " + + ", ".join(parts)) + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Count how often words are borne as given and family names.") + parser.add_argument("words", nargs="+") + parser.add_argument("--no-wikidata", action="store_true", + help="offline: cached sources only") + parser.add_argument("--countries", type=int, default=5, metavar="N", + help="countries of citizenship to list per role") + parser.add_argument("--json", action="store_true") + args = parser.parse_args(argv) + + census = load_census() + ssa = load_ssa() + report = {} + for word in args.words: + usages = [census_usage(word, census), ssa_usage(word, ssa)] + if not args.no_wikidata: + usages.append(wikidata_usage(word, args.countries)) + report[word] = usages + if args.json: + print(json.dumps({w: [asdict(u) for u in us] for w, us in report.items()}, + ensure_ascii=False, indent=2)) + else: + print("\n\n".join(render(w, us) for w, us in report.items())) + if ssa is None: + sys.stdout.flush() # keep the notice after the report, not before it + print("\n" + ssa_missing_message(), file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 644a99c5ffc09281863fcd75d2e4e913d7174ffc Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 11 Oct 2026 20:18:15 -0700 Subject: [PATCH 2/5] AGENTS.md: point vocabulary collision checks at tools/name_usage.py Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d6097163..2011788f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -364,7 +364,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **Unknown-key attribute access on `TupleManager`/`RegexTupleManager` warns (1.4, #256) — 2.0 note: the warning became a hard `AttributeError` naming the known keys (shim `TupleManager`); the paragraph below describes the deleted v1 machinery and applies only when reading v1 history.** — a key not currently in the dict emits `DeprecationWarning` naming the miss and the known keys (`_warn_unknown_key`), before falling back to the same `None`/`EMPTY_REGEX` default as before; `.get()` stays silent. Dunder probes (`__deepcopy__`) still raise `AttributeError` outright, and single-underscore probes (`_repr_html_`, IPython's `_ipython_canary_method_should_not_exist_`, etc.) are excluded from the warning too — no real config key starts with `_`, so both guards just keep protocol/introspection probes from misfiring as "typo" warnings. This means internal parser code that reads `self.C.regexes.` unconditionally (e.g. `squash_bidi`'s `bidi`) now warns if a caller's custom `regexes` dict omits that key — a previously-silent partial-override pattern is on the same deprecation path as an actual typo. -**Adding a word to a config set** — first check the *other* sets for the same word (grep `nameparser/config/` or intersect the sets in a `python3 -c`). Real overlaps exist: `do`/`st`/`mc` ∈ `PARTICLES` ∩ `TITLES`/`SUFFIX_ACRONYMS`; `abd` = "ABD" ∈ `SUFFIX_ACRONYMS`; `abu` ∈ `PARTICLES` ∩ `BOUND_GIVEN_NAMES` (position-dependent: leading token → given-name join, mid-name → family-name join). Usually position-dependent and harmless, but can force a guard or an exclusion (the `last_base` all-particles guard; dropping `abd` from `BOUND_GIVEN_NAMES`). +**Adding a word to a config set** — first check the *other* sets for the same word (grep `nameparser/config/` or intersect the sets in a `python3 -c`). Real overlaps exist: `do`/`st`/`mc` ∈ `PARTICLES` ∩ `TITLES`/`SUFFIX_ACRONYMS`; `abd` = "ABD" ∈ `SUFFIX_ACRONYMS`; `abu` ∈ `PARTICLES` ∩ `BOUND_GIVEN_NAMES` (position-dependent: leading token → given-name join, mid-name → family-name join). Usually position-dependent and harmless, but can force a guard or an exclusion (the `last_base` all-particles guard; dropping `abd` from `BOUND_GIVEN_NAMES`). Then check the word against real names, not memory: `uv run python tools/name_usage.py ` counts its bearers as a given name and as a family name (US Census surnames, SSA given names, Wikidata with countries of citizenship). It reports and does not decide — read the counts against the position the set's claim acts on. **Before adding a short/common word to `PARTICLES` globally**, test it mid-string against realistic 3-token names, not just check for English-word collisions: Korean/Vietnamese given names put a short syllable in the middle slot (`Park In Hwan`, `Nguyen To Nga`), and Western names put a bare initial there (`John V. Smith`). A word that looks safe ("nobody is named 'to'") can still swallow a real middle name/initial into the last name once it's a global prefix — confirmed regressions for `to`/`in`/`an`/`ten`/`then` and bare `v` this way (PR #191). @@ -384,7 +384,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **`HumanName.C` is a property backed by `_C`, but pickles under the public key `'C'`** — `__init__`/direct assignment route through the `C` setter, which calls the shared `_validate_constants` staticmethod (also used by `__init__`) so an invalid value raises `TypeError` immediately instead of surfacing later as an unrelated `AttributeError` deep in parsing (#239). `__getstate__`/`__setstate__` deliberately translate `self._C` ↔ a `'C'` key in the pickled dict (with the usual `CONSTANTS`-singleton-becomes-`None` sentinel) rather than pickling `_C` directly, so the on-disk pickle format hasn't changed across this fix — don't "simplify" that translation away or old pickles/tests that hand-build a state dict with a `'C'` key will break. -**Titles permanently shadow first names — be conservative** — any word in `TITLES` is always consumed as a title and can never be parsed as a first name. `"Dean"` is the canonical example: it's a common academic title *and* a common given name, so it is intentionally absent from the default titles (see `docs/customize.rst` — users who need it add it via opt-in `Constants`). Before adding a word to `TITLES`, ask: "Could this plausibly be someone's given name in any culture?" If yes, don't add it globally; it belongs in caller-supplied `Constants` instead. This same caution applies to international honorifics — `Prince`, `Sheikh`, `Frau` are all first names in some contexts. It also applies to any prefix sub-set gated on "never a first name": obscure-looking foreign particles are surprisingly often real given names — `Von` (Von Miller), `Vander` (Brazilian, also the Arcane character). When unsure, exclude — a missing member just means that name isn't auto-handled, whereas a wrong member misparses a real person. The decided exception is a word borne as a name only RARELY in the leading position — mostly a surname written last (`graf`, `greve`, `knight`), a few with small given-name or surname-first counts (`herra`, `hrabia`): the title claim acts only in front, so it ships, and a name that begins with it (`Greve Anna`, `Herra Wijaya` → title) is the accepted cost. That is a judgment by incidence, made per word with the counts recorded in `decisions.md#salutation-titles` (rules.md#H1's Accepted clause states the cost), not a rule a sweep can apply: a word that commonly leads a real name (Vietnamese `Ông`, the given names `thiru`, `puan`, `marshal`) is still out. +**Titles permanently shadow first names — be conservative** — any word in `TITLES` is always consumed as a title and can never be parsed as a first name. `"Dean"` is the canonical example: it's a common academic title *and* a common given name, so it is intentionally absent from the default titles (see `docs/customize.rst` — users who need it add it via opt-in `Constants`). Before adding a word to `TITLES`, ask: "Could this plausibly be someone's given name in any culture?" If yes, don't add it globally; it belongs in caller-supplied `Constants` instead. This same caution applies to international honorifics — `Prince`, `Sheikh`, `Frau` are all first names in some contexts. It also applies to any prefix sub-set gated on "never a first name": obscure-looking foreign particles are surprisingly often real given names — `Von` (Von Miller), `Vander` (Brazilian, also the Arcane character). When unsure, exclude — a missing member just means that name isn't auto-handled, whereas a wrong member misparses a real person. The decided exception is a word borne as a name only RARELY in the leading position — mostly a surname written last (`graf`, `greve`, `knight`), a few with small given-name or surname-first counts (`herra`, `hrabia`): the title claim acts only in front, so it ships, and a name that begins with it (`Greve Anna`, `Herra Wijaya` → title) is the accepted cost. That is a judgment by incidence, made per word with the counts recorded in `decisions.md#salutation-titles` (rules.md#H1's Accepted clause states the cost), not a rule a sweep can apply: a word that commonly leads a real name (Vietnamese `Ông`, the given names `thiru`, `puan`, `marshal`) is still out. `tools/name_usage.py` produces the incidence counts this judgment needs; its Wikidata column is a lower bound (Shehu Shagari has no given-name statement), so a small count there proves little and a zero proves nothing. **The period-abbreviation title inference runs at the head of the GIVEN-NAME part, not the head of the name** — an unrecognized multi-letter word ending in a single trailing period (`_pieces._PERIOD_ABBREV`, a hand copy of the `period_abbreviation` regex, `{2,}` letters — it was assign's until #424 and group's until #439) is treated as a title in the leading title run, e.g. `"Insp. Jane Morse"` → `title='Insp.'`. "Leading" is per SEGMENT: `_peel_leading_titles` is called for NO_COMMA segment 0, SUFFIX_COMMA segment 0, and FAMILY_COMMA **segment 1**, so `"Morse, Det. Insp. Jane"` → `title='Det. Insp.'` and a lone `"Smith, Xyz."` → `title='Xyz.'` — long-standing, verified against 1.4.0, and the mechanism behind #296 (`"Smith, Jr."` read title in 1.4.0; the comma's credential read takes it as suffix `Jr.` now). The docs said "leading word" until 2026-08-01 and were wrong for every comma path. It does not mutate `C.titles`, so the periodless form (`"Insp"`) is unaffected elsewhere. The `{2,}` length requirement — not a separate initials check — is what excludes single-letter initials like `"J."`; the same word after the given name is left as a middle name. **The inference OUTRANKS vocabulary where it runs**: `"Esq. Smith"` → `title='Esq.'` even though `esq` is suffix-only vocabulary, because the shape rule fires before anything consults the suffix sets. **The INFERENCE still runs in one direction only, and that is what the two slots share and where they part** (rewritten 2026-09-08, #316): a period-marked word is claimed by SHAPE at the front and by VOCABULARY at the back, so an unlisted abbreviation opening a name is a title while an unlisted abbreviation ending one is a NAME word (`"John Smith Xyz."` → `family='Xyz.'`). A trailing period-marked word the vocabulary knows as a title now IS one (`"John Smith Prof."` → `title='Prof.'`, `rules.md#H5`) — the comma path always read it that way and the two agree now — and a trailing period-marked word the SUFFIX vocabulary knows is still a suffix, the suffix run being peeled first (`"John Smith Esq."` → `suffix='Esq.'`). A BARE trailing title word stays a name word (`"John Smith Sir"`, `"Mary Jane King"`), which is the doctrine line below. Meanwhile `period_joined_vocab` resolves INTERIOR-period tokens (`Lt.Gov.`, `Msc.Ed.`) to title-or-suffix by vocabulary, and `_extract._suffix_shaped` treats any period-final delimited content as not-a-nickname. Four sites, and the trailing one now resolves by two vocabularies in order rather than one; unifying the rest is open design work, not settled. (#109; see `docs/usage.rst` "Titles you didn't configure") From 088d44d91cb8c594ffbb1c9f93538fce30ed218e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 11 Oct 2026 20:30:51 -0700 Subject: [PATCH 3/5] Fix what the review of #651 found: false zeros and misread counts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five reviewers converged on one failure class -- a count that reads as 'nobody bears this name' when the tool simply failed to look: - The ASCII fold deleted letters NFKD leaves whole (ø, ß, Ł, Æ, Đ), so Sørensen was looked up as SRENSEN and reported 0 against a Census count of 28,069 (and Æsir looked up SIR). Transliterate them first, and treat any letter still without an ASCII spelling as unrepresentable rather than dropping it. - SSA reported 0 for a word it cannot spell (关); it now says unrepresentable, as the Census path already did. - A wrong or partial SSA zip (the state-level file, a re-zipped folder) read as empty, so every word reported 0. Members now match by basename in any case, and fewer than 100 year files is refused. - Case variants were queried separately and summed, counting twice a person reached through two spellings (de: 184 against 182). All spellings now go in one VALUES block; _merge is gone. Lowercase input cannot rebuild an internal capital (mcdonald finds 0 where McDonald finds 1,968), so the report lists the spellings tried and AGENTS.md says to pass names as written. - Every source's 0 meant 'below the listing threshold', not zero: Usage now carries a status, a floor (<100, <5/yr) and a lower_bound flag, rendered and in --json. - A country with no English label counted its people as stateless; it is now named by its Q-id. - A malformed Wikidata response crashed the run and lost every word; an unusable download stayed cached forever; --no-wikidata still downloaded. Each now reports, and the exit status is 2 when a source failed. The docstring also claimed herra was a collision memory had missed; it was shipped knowingly under decisions.md#salutation-titles with recorded counts. Corrected, with shehu as the example. Tests grow from 12 to 31, behind an autouse guard that fails any network call. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- tests/v2/test_name_usage_tool.py | 238 ++++++++++++++--- tools/name_usage.py | 438 +++++++++++++++++++++---------- 3 files changed, 496 insertions(+), 184 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2011788f..7a8c1a2a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -364,7 +364,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **Unknown-key attribute access on `TupleManager`/`RegexTupleManager` warns (1.4, #256) — 2.0 note: the warning became a hard `AttributeError` naming the known keys (shim `TupleManager`); the paragraph below describes the deleted v1 machinery and applies only when reading v1 history.** — a key not currently in the dict emits `DeprecationWarning` naming the miss and the known keys (`_warn_unknown_key`), before falling back to the same `None`/`EMPTY_REGEX` default as before; `.get()` stays silent. Dunder probes (`__deepcopy__`) still raise `AttributeError` outright, and single-underscore probes (`_repr_html_`, IPython's `_ipython_canary_method_should_not_exist_`, etc.) are excluded from the warning too — no real config key starts with `_`, so both guards just keep protocol/introspection probes from misfiring as "typo" warnings. This means internal parser code that reads `self.C.regexes.` unconditionally (e.g. `squash_bidi`'s `bidi`) now warns if a caller's custom `regexes` dict omits that key — a previously-silent partial-override pattern is on the same deprecation path as an actual typo. -**Adding a word to a config set** — first check the *other* sets for the same word (grep `nameparser/config/` or intersect the sets in a `python3 -c`). Real overlaps exist: `do`/`st`/`mc` ∈ `PARTICLES` ∩ `TITLES`/`SUFFIX_ACRONYMS`; `abd` = "ABD" ∈ `SUFFIX_ACRONYMS`; `abu` ∈ `PARTICLES` ∩ `BOUND_GIVEN_NAMES` (position-dependent: leading token → given-name join, mid-name → family-name join). Usually position-dependent and harmless, but can force a guard or an exclusion (the `last_base` all-particles guard; dropping `abd` from `BOUND_GIVEN_NAMES`). Then check the word against real names, not memory: `uv run python tools/name_usage.py ` counts its bearers as a given name and as a family name (US Census surnames, SSA given names, Wikidata with countries of citizenship). It reports and does not decide — read the counts against the position the set's claim acts on. +**Adding a word to a config set** — first check the *other* sets for the same word (grep `nameparser/config/` or intersect the sets in a `python3 -c`). Real overlaps exist: `do`/`st`/`mc` ∈ `PARTICLES` ∩ `TITLES`/`SUFFIX_ACRONYMS`; `abd` = "ABD" ∈ `SUFFIX_ACRONYMS`; `abu` ∈ `PARTICLES` ∩ `BOUND_GIVEN_NAMES` (position-dependent: leading token → given-name join, mid-name → family-name join). Usually position-dependent and harmless, but can force a guard or an exclusion (the `last_base` all-particles guard; dropping `abd` from `BOUND_GIVEN_NAMES`). Then check the word against real names, not memory: `uv run python tools/name_usage.py ` counts its bearers as a given name and as a family name (US Census surnames, SSA given names, Wikidata with countries of citizenship). Pass a name as it is written (`McDonald`): Wikidata labels match exactly, and no rule rebuilds an internal capital from `mcdonald`. It reports and does not decide — read the counts against the position the set's claim acts on. **Before adding a short/common word to `PARTICLES` globally**, test it mid-string against realistic 3-token names, not just check for English-word collisions: Korean/Vietnamese given names put a short syllable in the middle slot (`Park In Hwan`, `Nguyen To Nga`), and Western names put a bare initial there (`John V. Smith`). A word that looks safe ("nobody is named 'to'") can still swallow a real middle name/initial into the last name once it's a global prefix — confirmed regressions for `to`/`in`/`an`/`ten`/`then` and bare `v` this way (PR #191). @@ -384,7 +384,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **`HumanName.C` is a property backed by `_C`, but pickles under the public key `'C'`** — `__init__`/direct assignment route through the `C` setter, which calls the shared `_validate_constants` staticmethod (also used by `__init__`) so an invalid value raises `TypeError` immediately instead of surfacing later as an unrelated `AttributeError` deep in parsing (#239). `__getstate__`/`__setstate__` deliberately translate `self._C` ↔ a `'C'` key in the pickled dict (with the usual `CONSTANTS`-singleton-becomes-`None` sentinel) rather than pickling `_C` directly, so the on-disk pickle format hasn't changed across this fix — don't "simplify" that translation away or old pickles/tests that hand-build a state dict with a `'C'` key will break. -**Titles permanently shadow first names — be conservative** — any word in `TITLES` is always consumed as a title and can never be parsed as a first name. `"Dean"` is the canonical example: it's a common academic title *and* a common given name, so it is intentionally absent from the default titles (see `docs/customize.rst` — users who need it add it via opt-in `Constants`). Before adding a word to `TITLES`, ask: "Could this plausibly be someone's given name in any culture?" If yes, don't add it globally; it belongs in caller-supplied `Constants` instead. This same caution applies to international honorifics — `Prince`, `Sheikh`, `Frau` are all first names in some contexts. It also applies to any prefix sub-set gated on "never a first name": obscure-looking foreign particles are surprisingly often real given names — `Von` (Von Miller), `Vander` (Brazilian, also the Arcane character). When unsure, exclude — a missing member just means that name isn't auto-handled, whereas a wrong member misparses a real person. The decided exception is a word borne as a name only RARELY in the leading position — mostly a surname written last (`graf`, `greve`, `knight`), a few with small given-name or surname-first counts (`herra`, `hrabia`): the title claim acts only in front, so it ships, and a name that begins with it (`Greve Anna`, `Herra Wijaya` → title) is the accepted cost. That is a judgment by incidence, made per word with the counts recorded in `decisions.md#salutation-titles` (rules.md#H1's Accepted clause states the cost), not a rule a sweep can apply: a word that commonly leads a real name (Vietnamese `Ông`, the given names `thiru`, `puan`, `marshal`) is still out. `tools/name_usage.py` produces the incidence counts this judgment needs; its Wikidata column is a lower bound (Shehu Shagari has no given-name statement), so a small count there proves little and a zero proves nothing. +**Titles permanently shadow first names — be conservative** — any word in `TITLES` is always consumed as a title and can never be parsed as a first name. `"Dean"` is the canonical example: it's a common academic title *and* a common given name, so it is intentionally absent from the default titles (see `docs/customize.rst` — users who need it add it via opt-in `Constants`). Before adding a word to `TITLES`, ask: "Could this plausibly be someone's given name in any culture?" If yes, don't add it globally; it belongs in caller-supplied `Constants` instead. This same caution applies to international honorifics — `Prince`, `Sheikh`, `Frau` are all first names in some contexts. It also applies to any prefix sub-set gated on "never a first name": obscure-looking foreign particles are surprisingly often real given names — `Von` (Von Miller), `Vander` (Brazilian, also the Arcane character). When unsure, exclude — a missing member just means that name isn't auto-handled, whereas a wrong member misparses a real person. The decided exception is a word borne as a name only RARELY in the leading position — mostly a surname written last (`graf`, `greve`, `knight`), a few with small given-name or surname-first counts (`herra`, `hrabia`): the title claim acts only in front, so it ships, and a name that begins with it (`Greve Anna`, `Herra Wijaya` → title) is the accepted cost. That is a judgment by incidence, made per word with the counts recorded in `decisions.md#salutation-titles` (rules.md#H1's Accepted clause states the cost), not a rule a sweep can apply: a word that commonly leads a real name (Vietnamese `Ông`, the given names `thiru`, `puan`, `marshal`) is still out. `tools/name_usage.py` produces the counts this judgment starts from (role counts, plus Wikidata citizenship to tell where a family name is written first); its Wikidata column is a lower bound (Shehu Shagari has no given-name statement), so a small count there proves little and a zero proves nothing. **The period-abbreviation title inference runs at the head of the GIVEN-NAME part, not the head of the name** — an unrecognized multi-letter word ending in a single trailing period (`_pieces._PERIOD_ABBREV`, a hand copy of the `period_abbreviation` regex, `{2,}` letters — it was assign's until #424 and group's until #439) is treated as a title in the leading title run, e.g. `"Insp. Jane Morse"` → `title='Insp.'`. "Leading" is per SEGMENT: `_peel_leading_titles` is called for NO_COMMA segment 0, SUFFIX_COMMA segment 0, and FAMILY_COMMA **segment 1**, so `"Morse, Det. Insp. Jane"` → `title='Det. Insp.'` and a lone `"Smith, Xyz."` → `title='Xyz.'` — long-standing, verified against 1.4.0, and the mechanism behind #296 (`"Smith, Jr."` read title in 1.4.0; the comma's credential read takes it as suffix `Jr.` now). The docs said "leading word" until 2026-08-01 and were wrong for every comma path. It does not mutate `C.titles`, so the periodless form (`"Insp"`) is unaffected elsewhere. The `{2,}` length requirement — not a separate initials check — is what excludes single-letter initials like `"J."`; the same word after the given name is left as a middle name. **The inference OUTRANKS vocabulary where it runs**: `"Esq. Smith"` → `title='Esq.'` even though `esq` is suffix-only vocabulary, because the shape rule fires before anything consults the suffix sets. **The INFERENCE still runs in one direction only, and that is what the two slots share and where they part** (rewritten 2026-09-08, #316): a period-marked word is claimed by SHAPE at the front and by VOCABULARY at the back, so an unlisted abbreviation opening a name is a title while an unlisted abbreviation ending one is a NAME word (`"John Smith Xyz."` → `family='Xyz.'`). A trailing period-marked word the vocabulary knows as a title now IS one (`"John Smith Prof."` → `title='Prof.'`, `rules.md#H5`) — the comma path always read it that way and the two agree now — and a trailing period-marked word the SUFFIX vocabulary knows is still a suffix, the suffix run being peeled first (`"John Smith Esq."` → `suffix='Esq.'`). A BARE trailing title word stays a name word (`"John Smith Sir"`, `"Mary Jane King"`), which is the doctrine line below. Meanwhile `period_joined_vocab` resolves INTERIOR-period tokens (`Lt.Gov.`, `Msc.Ed.`) to title-or-suffix by vocabulary, and `_extract._suffix_shaped` treats any period-final delimited content as not-a-nickname. Four sites, and the trailing one now resolves by two vocabularies in order rather than one; unifying the rest is open design work, not settled. (#109; see `docs/usage.rst` "Titles you didn't configure") diff --git a/tests/v2/test_name_usage_tool.py b/tests/v2/test_name_usage_tool.py index 0d307aa1..ed47fd98 100644 --- a/tests/v2/test_name_usage_tool.py +++ b/tests/v2/test_name_usage_tool.py @@ -1,21 +1,27 @@ """Offline checks on tools/name_usage.py, the name-collision lookup. The tool's sources are a downloaded file, a hand-downloaded file and a -live SPARQL endpoint, none of which a test may touch. What CAN go wrong -without them is the folding and the bookkeeping: a word that misses the -Census because it was folded differently from the Census's own spelling -reads as "under 100 bearers", which is exactly the false all-clear the -tool exists to prevent. These pin those pieces. +live SPARQL endpoint, none of which a test may touch -- the autouse +guard below turns any network call into a failure. What CAN go wrong +without them is the folding and the bookkeeping, and the failure that +matters is a FALSE ZERO: a word that misses because it was folded into a +different name, or a source that read nothing, reports "not listed", +which is the all-clear the tool exists to withhold. These pin those +pieces, and each spelling row records what the first version got wrong. tools/ is not a package, so the module is loaded by path, as _differential_fixtures.py loads compare.py. """ import importlib.util import io +import json import sys +import urllib.request import zipfile +from collections import Counter from pathlib import Path from types import ModuleType +from typing import Any import pytest @@ -26,9 +32,9 @@ def _load() -> ModuleType: spec = importlib.util.spec_from_file_location("name_usage", _PATH) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) - # Registered before it runs: @dataclass resolves the module's - # postponed annotations through sys.modules, and fails on a module - # loaded by path that is not in it. + # Registered before it runs: @dataclass looks the module up in + # sys.modules to check string annotations for ClassVar/InitVar, and + # fails on a module loaded by path that is not in it. sys.modules[spec.name] = module spec.loader.exec_module(module) return module @@ -36,9 +42,29 @@ def _load() -> ModuleType: tool = _load() + +class _NetworkUsed(AssertionError): + pass + + +@pytest.fixture(autouse=True) +def _no_network(monkeypatch: pytest.MonkeyPatch) -> None: + def refuse(*args: object, **kwargs: object) -> None: + raise _NetworkUsed("a test reached the network") + monkeypatch.setattr(urllib.request, "urlopen", refuse) + + +def test_the_network_guard_bites() -> None: + # Negative control for the guard: without it, this call goes to + # query.wikidata.org. + with pytest.raises(_NetworkUsed): + tool._sparql("SELECT * WHERE {}") + + _CENSUS = """name,rank,count,prop100k,cum_prop100k,pctwhite,pctblack,pctapi,pctaian,pct2prace,pcthispanic OBRIEN,258,118557,40.19,0,93.85,1.29,0.76,0.27,1.51,2.32 SHEHU,39212,561,0.19,0,94.47,3.92,(S),0,0.89,(S) +SORENSEN,1101,28069,9.5,0,96.6,0.5,0.6,0.5,1.1,0.7 """ @@ -46,16 +72,26 @@ def _load() -> ModuleType: ("O'Brien", "OBRIEN"), # apostrophe dropped, as the Census spells it ("de la Cruz", "DELACRUZ"), # spaces dropped ("Ó Murchú", "OMURCHU"), # diacritics folded away - ("关", ""), # nothing left: not representable + # Letters NFKD leaves whole. The first version deleted them, looking + # up a DIFFERENT name: SRENSEN, STRAU, UKASZ, and SIR for Æsir. + ("Sørensen", "SORENSEN"), + ("Strauß", "STRAUSS"), + ("Łukasz", "LUKASZ"), + ("Æsir", "AESIR"), + ("Đorđević", "DORDEVIC"), + # No ASCII spelling, wholly or in part: no key at all, never a partial + # one (the first version looked up AHMED for the second). + ("关", ""), + ("Ahmed关", ""), ]) -def test_census_key_folds_a_word_the_way_the_census_spells_it(word: str, key: str) -> None: - assert tool.census_key(word) == key +def test_us_key_folds_a_word_the_way_the_us_lists_spell_it(word: str, key: str) -> None: + assert tool.us_key(word) == key def test_census_usage_finds_a_folded_surname_and_says_how() -> None: - usage = tool.census_usage("O'Brien", tool.read_census(_CENSUS)) - assert usage.family == 118557 - assert "looked up as OBRIEN" in usage.note + usage = tool.census_usage("Sørensen", tool.read_census(_CENSUS)) + assert usage.family == 28069 + assert "looked up as SORENSEN" in usage.note def test_census_usage_omits_suppressed_shares() -> None: @@ -66,15 +102,21 @@ def test_census_usage_omits_suppressed_shares() -> None: assert "white 94.47%" in note -def test_census_usage_distinguishes_absent_from_unrepresentable() -> None: +def test_census_usage_distinguishes_unlisted_from_unrepresentable() -> None: table = tool.read_census(_CENSUS) - absent = tool.census_usage("marshal", table) - assert absent.family == 0 + unlisted = tool.census_usage("Marshal", table) + assert (unlisted.status, unlisted.family) == ("ok", 0) unrepresentable = tool.census_usage("关", table) + assert unrepresentable.status == "unrepresentable" assert unrepresentable.family is None # no answer, not a zero -def _ssa_zip(files: dict[str, str]) -> zipfile.ZipFile: +def test_read_census_names_missing_columns() -> None: + with pytest.raises(ValueError, match="pctwhite"): + tool.read_census("name,rank,count\nSMITH,1,2442977\n") + + +def _zip(files: dict[str, str]) -> zipfile.ZipFile: buffer = io.BytesIO() with zipfile.ZipFile(buffer, "w") as zf: for name, text in files.items(): @@ -83,47 +125,157 @@ def _ssa_zip(files: dict[str, str]) -> zipfile.ZipFile: return zipfile.ZipFile(buffer) +def _ssa_files(**extra: str) -> dict[str, str]: + files = {f"yob{year}.txt": "Filler,F,5\n" for year in range(1880, 1990)} + files.update(extra) + return files + + def test_read_ssa_totals_each_name_by_sex_across_years() -> None: - totals = tool.read_ssa(_ssa_zip({ + totals = tool.read_ssa(_zip(_ssa_files(**{ "yob1990.txt": "Justice,F,40\nJustice,M,60\n", "yob2000.txt": "Justice,M,100\nMarshal,M,7\n", "NationalReadMe.pdf": "not data", - })) + }))) assert totals["justice"] == {"F": 40, "M": 160} - usage = tool.ssa_usage("Justice", totals) - assert usage.given == 200 + assert tool.ssa_usage("Justice", totals).given == 200 + + +def test_read_ssa_reads_a_re_zipped_folder_in_any_case() -> None: + files = {f"names/YOB{year}.TXT": "Justice,M,5\n" for year in range(1880, 1990)} + assert tool.read_ssa(_zip(files))["justice"] == {"M": 5 * 110} + + +@pytest.mark.parametrize("files", [ + {"AK.TXT": "AK,F,1910,Mary,14\n"}, # the state-level zip + {f"yob{y}.txt": "A,F,5\n" for y in range(1880, 1900)}, # a partial set +]) +def test_read_ssa_refuses_a_zip_that_is_not_the_national_file(files: dict[str, str]) -> None: + # The first version read such a zip as empty, so every word reported + # "not listed" -- a false zero for every name. + with pytest.raises(ValueError, match="yobYYYY.txt"): + tool.read_ssa(_zip(files)) + + +def test_ssa_usage_distinguishes_unlisted_from_unrepresentable() -> None: + table = {"justice": Counter(M=5)} + assert tool.ssa_usage("Zzyzx", table).given == 0 + unrepresentable = tool.ssa_usage("关", table) + assert unrepresentable.status == "unrepresentable" + assert unrepresentable.given is None def test_ssa_usage_reports_a_missing_download_without_a_count() -> None: usage = tool.ssa_usage("justice", None) - assert usage.given is None and "not downloaded" in usage.note + assert (usage.status, usage.given) == ("not downloaded", None) + + +def test_render_shows_no_answer_and_unlisted_differently_from_a_count() -> None: + text = tool.render("x", [ + tool.Usage("census", family=0, floor="<100"), + tool.Usage("ssa", status="not downloaded"), + tool.Usage("wikidata", given=0, family=12, lower_bound=True), + ]) + census, ssa, wikidata = (line.split() for line in text.splitlines()[2::1] + if line.split() and line.split()[0] in + ("census", "ssa", "wikidata")) + assert census[1:] == ["-", "<100"] + assert ssa[1:] == ["-", "-"] + assert wikidata[1:] == ["0+", "12+"] + + +def test_spellings_cover_lower_and_capitalized_forms_once() -> None: + assert tool.spellings("de la cruz") == ["de la cruz", "De La Cruz"] + assert tool.spellings("McDonald") == ["McDonald", "mcdonald", "Mcdonald"] -def test_wikidata_query_matches_the_word_in_every_language_and_escapes_it() -> None: - query = tool.wikidata_query('Say "hi"', languages=("mul", "zh-hans")) +def test_wikidata_queries_escape_the_word_in_every_language() -> None: + query = tool.wikidata_totals_query(['Say "hi"'], languages=("mul", "zh-hans")) assert r'"Say \"hi\""@mul' in query assert r'"Say \"hi\""@zh-hans' in query -def test_read_wikidata_keeps_stateless_people_out_of_the_country_list() -> None: - usage = tool.read_wikidata( - totals=[{"role": "family", "n": "180"}, {"role": "given", "n": "1"}], - by_country=[ - {"role": "family", "n": "148"}, # no country bound - {"role": "family", "country": "Albania", "n": "11"}, - {"role": "family", "country": "Nigeria", "n": "20"}, - {"role": "given", "country": "Nigeria", "n": "1"}, - ], - top=5) +def _stub_sparql(monkeypatch: pytest.MonkeyPatch, + responses: list[Any]) -> list[str]: + queries: list[str] = [] + + def fake(query: str) -> dict[str, Any]: + queries.append(query) + response = responses[len(queries) - 1] + if isinstance(response, Exception): + raise response + return {"results": {"bindings": [ + {k: {"value": v} for k, v in row.items()} for row in response]}} + monkeypatch.setattr(tool, "_sparql", fake) + return queries + + +def test_wikidata_usage_sends_every_spelling_in_one_query(monkeypatch: pytest.MonkeyPatch) -> None: + # The first version queried each spelling separately and summed the + # counts, counting twice a person reached through two spellings. + queries = _stub_sparql(monkeypatch, [ + [{"role": "family", "n": "180"}, {"role": "given", "n": "1"}], + [{"role": "family", "country": "Albania", "n": "11"}, + {"role": "family", "n": "148"}], + ]) + usage = tool.wikidata_usage("shehu", top=5) + assert len(queries) == 2 + assert '"shehu"@mul' in queries[0] and '"Shehu"@mul' in queries[0] assert (usage.given, usage.family) == (1, 180) - assert usage.family_countries == [("Nigeria", 20), ("Albania", 11)] + assert usage.family_countries == [("Albania", 11)] assert usage.family_stateless == 148 + assert "matched as shehu, Shehu" in usage.note + + +@pytest.mark.parametrize("failure", [ + OSError("timed out"), + json.JSONDecodeError("Expecting value", "", 0), # not an OSError +]) +def test_wikidata_usage_reports_a_failed_query_as_no_answer( + monkeypatch: pytest.MonkeyPatch, failure: Exception) -> None: + _stub_sparql(monkeypatch, [[{"role": "given", "n": "3"}], failure]) + usage = tool.wikidata_usage("shehu", top=5) + assert usage.status == "failed" + assert (usage.given, usage.family) == (None, None) + + +def test_read_wikidata_refuses_a_role_it_cannot_have_produced() -> None: + # A misspelled role must not fall through to a count of 0. + with pytest.raises(ValueError, match="givn"): + tool.read_wikidata([{"role": "givn", "n": "5"}], [], top=5) + + +def test_main_reports_json_offline_without_downloading( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + with zipfile.ZipFile(tmp_path / tool.CENSUS_FILE, "w") as zf: + zf.writestr(tool.CENSUS_MEMBER, _CENSUS) + status = tool.main(["--no-wikidata", "--json", "O'Brien", "关"]) + out, err = capsys.readouterr() + report = json.loads(out) + assert status == 0 + assert list(report) == ["O'Brien", "关"] + assert [u["source"] for u in report["关"]] == [ + "US Census 2010 surnames", "US SSA given names 1880-"] + assert report["O'Brien"][0]["family"] == 118557 + assert report["关"][0]["family"] is None # null, not 0 + assert "SSA given names not downloaded" in err + + +def test_main_does_not_download_under_no_wikidata( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + assert tool.main(["--no-wikidata", "--json", "knight"]) == 0 + census = json.loads(capsys.readouterr().out)["knight"][0] + assert (census["status"], census["family"]) == ("not downloaded", None) -def test_merge_sums_the_case_variants_per_key() -> None: - merged = tool._merge( - [{"role": "given", "n": "2"}, {"role": "given", "n": "3"}, - {"role": "family", "n": "1"}], - ("role",)) - assert sorted((row["role"], row["n"]) for row in merged) == [ - ("family", "1"), ("given", "5")] +def test_main_exits_2_when_a_source_failed( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + (tmp_path / tool.CENSUS_FILE).write_text("error page") + assert tool.main(["--no-wikidata", "knight"]) == 2 + assert "delete it and run again" in capsys.readouterr().out diff --git a/tools/name_usage.py b/tools/name_usage.py index 4e623de2..98ad3514 100644 --- a/tools/name_usage.py +++ b/tools/name_usage.py @@ -1,15 +1,17 @@ """How often is a word borne as a given name, and as a family name? -WHY THIS IS IN THE TREE. Vocabulary membership is decided by -decisions.md#vocabulary-collisions: a word may join a set the parser -claims by position (titles, never-given particles) only if it is not -borne as a name IN THE POSITION THE CLAIM ACTS ON. Until this tool, -that check was run from memory, and memory missed collisions the data -shows at once: `shehu` is an Albanian surname as well as a Hausa given -name, and `herra` (a title since #606) is borne by about 640 people in -the 2010 US Census. This answers the question with counts instead. +WHY THIS IS IN THE TREE. Vocabulary claims are weighed by +decisions.md#vocabulary-collisions: a word the parser claims by +position (titles, never-given particles) is safe to claim outright only +if it is not borne as a name IN THE POSITION THE CLAIM ACTS ON (C-i; +decisions.md#salutation-titles records the one TITLES-only exception, +decided on counts). That check was often run from memory, which misses +what the data shows at once: `shehu`, a title, is also an Albanian and +Nigerian surname, with 561 bearers in the 2010 US Census. This answers +the question with counts instead. uv run python tools/name_usage.py shehu greve marshal + uv run python tools/name_usage.py McDonald # pass names as written uv run python tools/name_usage.py --no-wikidata knight # offline uv run python tools/name_usage.py --json justice # machine-readable @@ -19,32 +21,46 @@ where the family name is written first -- which is what Wikidata's per-country breakdown is for. -Sources, each answering for one role and one population: +What a number means depends on its source, and the report says so: -- US Census 2010 surnames (public domain): every surname with 100 or - more bearers, with rank and ethnic shares. Downloaded once into the - cache. Census names are ASCII with spaces and punctuation removed - (O'Brien is OBRIEN), so a word is folded the same way before lookup. +- US Census 2010 surnames (public domain): surnames with 100 or more + bearers, with rank and ethnic shares. A surname under 100 is not + listed, so it reports "<100", never 0. Downloaded once into the + cache. - US Social Security given names, 1880 onward (CC0 via data.gov), - counted by sex across all years. ssa.gov refuses scripted downloads, - so this one is fetched by hand -- the tool prints where to put it. + summed by sex across all years. A name-sex pair is listed for a year + only with 5 or more births, so a count is a floor and an unlisted + name reports "<5/yr". ssa.gov refuses scripted downloads, so this + file is fetched by hand -- the tool says where to put it. - Wikidata (CC0), queried live: people whose given name (P735) or family name (P734) is an item labelled with the word, with their countries of citizenship. A LOWER BOUND: many people have no P735 statement at all (Shehu Shagari has none), so a small count proves - little and a zero proves nothing. + little and a zero proves nothing. Labels are matched exactly, in the + spellings the report lists, so a name with internal capitals is found + only when passed as written (`McDonald`, not `mcdonald`). + +Both US sources spell names in ASCII letters only, without spaces or +punctuation (O'Brien is OBRIEN), so a word is folded the same way +before lookup. A letter the fold cannot carry over (any non-Latin +script) makes the word unrepresentable there: no answer, never a zero. Data is cached outside the repository, in $NAMEPARSER_TOOLS_CACHE or -~/.cache/nameparser-tools: third-party data stays out of the tree, and -the Census file alone is 9 MB. +~/.cache/nameparser-tools, so third-party data stays out of the tree. + +Exit status is 2 when any source failed to answer (a query error, an +unusable download), so a script cannot mistake a partial report for a +complete one; a source that is merely not downloaded is not a failure. """ from __future__ import annotations import argparse import csv +import http.client import io import json import os +import re import sys import unicodedata import urllib.parse @@ -53,20 +69,28 @@ from collections import Counter from dataclasses import asdict, dataclass, field from pathlib import Path +from typing import Any, Literal CENSUS_URL = "https://www2.census.gov/topics/genealogy/2010surnames/names.zip" +CENSUS_FILE = "census2010_surnames.zip" CENSUS_MEMBER = "Names_2010Census.csv" +CENSUS_COLUMNS = {"name", "rank", "count", "pctwhite", "pctblack", "pctapi", + "pcthispanic"} SSA_URL = "https://www.ssa.gov/oact/babynames/names.zip" SSA_FILE = "ssa_names.zip" +#: The national file holds one yobYYYY.txt per year from 1880. Fewer than +#: this many is a partial or wrong download (the state-level zip holds +#: AK.TXT and so on, none of them yob files). +SSA_MIN_YEARS = 100 WIKIDATA_ENDPOINT = "https://query.wikidata.org/sparql" USER_AGENT = ("nameparser-name-usage/0.1 " "(https://github.com/derek73/python-nameparser)") #: Label languages a name item is matched in. `mul` is Wikidata's -#: language-independent label, which most name items now carry; the -#: rest catch items labelled only in a language of their own. Matching -#: one exact literal per language keeps the query on the label index -- -#: a FILTER over every label would time out. +#: language-independent label, which many name items carry; the rest +#: catch items labelled only in a language of their own. Matching one +#: exact literal per language keeps the query on the label index -- a +#: FILTER over every label times out. WIKIDATA_LANGUAGES = ( "mul", "en", "de", "fr", "es", "it", "pt", "nl", "sv", "da", "nb", "fi", "et", "lv", "lt", "pl", "cs", "sk", "sl", "hr", "sr", "bs", @@ -74,19 +98,40 @@ "is", "ar", "fa", "ur", "he", "hi", "bn", "ta", "te", "ja", "ko", "vi", "th", "id", "ms", "tl", "sw", "ha", "yo", "ig", "am", "so", # Chinese labels sit under script and region tags far more often - # than plain `zh`: the 关 surname item is labelled only zh-hans and - # zh-cn, so without these a simplified-script surname finds nothing. - "zh", "zh-hans", "zh-hant", "zh-cn", "zh-tw", "zh-hk", "yue", + # than plain `zh`: neither 关 surname item has a `zh` or `mul` label, + # so without these a simplified-script surname finds nothing. + "zh", "zh-hans", "zh-hant", "zh-cn", "zh-tw", "zh-hk", "zh-sg", "yue", ) +#: Letters NFKD leaves whole, so an ASCII fold would delete them and look +#: up a different name (Sørensen as SRENSEN). Spelled as English-language +#: records spell them. +_TRANSLITERATE = str.maketrans({ + "ß": "ss", "ẞ": "SS", "ø": "o", "Ø": "O", "æ": "ae", "Æ": "AE", + "œ": "oe", "Œ": "OE", "ł": "l", "Ł": "L", "đ": "d", "Đ": "D", + "ð": "d", "Ð": "D", "þ": "th", "Þ": "TH", "ı": "i", "ŋ": "ng", "Ŋ": "NG", +}) + +Status = Literal["ok", "unrepresentable", "not downloaded", "failed"] -@dataclass + +@dataclass(slots=True) class Usage: - """One source's answer for one word.""" + """One source's answer for one word. + + `given`/`family` are None when the source gave no answer for that + role -- it does not cover the role, or `status` says why it could + not answer. A number is a count; `floor` says what an unlisted name + reads as instead of 0 ("<100"), and `lower_bound` that the true + count may be higher. + """ source: str + status: Status = "ok" given: int | None = None family: int | None = None + floor: str = "" + lower_bound: bool = False #: Top countries of citizenship per role, Wikidata only. given_countries: list[tuple[str, int]] = field(default_factory=list) family_countries: list[tuple[str, int]] = field(default_factory=list) @@ -105,63 +150,121 @@ def cache_dir() -> Path: return path -# -- US Census surnames -------------------------------------------------- +# -- the US sources' spelling --------------------------------------------- -def census_key(word: str) -> str: - """Fold a word the way the Census list spells names: diacritics - dropped, then everything but ASCII letters removed, upper-cased.""" - ascii_word = unicodedata.normalize("NFKD", word).encode( - "ascii", "ignore").decode() - return "".join(ch for ch in ascii_word if ch.isalpha()).upper() +def us_key(word: str) -> str: + """Fold a word the way the Census and SSA lists spell names: ASCII + letters only, upper-cased. Empty when any letter of the word has no + ASCII spelling -- a lossy fold looks up a DIFFERENT name, which is + worse than no lookup.""" + folded = unicodedata.normalize("NFKD", word.translate(_TRANSLITERATE)) + letters = [] + for ch in folded: + if unicodedata.combining(ch): + continue + if ch.isascii(): + if ch.isalpha(): + letters.append(ch) + elif ch.isalpha(): + return "" + return "".join(letters).upper() -def read_census(csv_text: str) -> dict[str, dict[str, str]]: - return {row["name"]: row for row in csv.DictReader(io.StringIO(csv_text))} +def _lookup_note(word: str, key: str) -> str: + return "" if key == word.upper() else f"looked up as {key}" + + +def _join(*parts: str) -> str: + return "; ".join(p for p in parts if p) -def load_census() -> dict[str, dict[str, str]]: - path = cache_dir() / "census2010_surnames.zip" +# -- US Census surnames -------------------------------------------------- + +def read_census(csv_text: str) -> dict[str, dict[str, str]]: + reader = csv.DictReader(io.StringIO(csv_text)) + missing = CENSUS_COLUMNS - set(reader.fieldnames or ()) + if missing: + raise ValueError(f"Census file lacks columns {sorted(missing)}; " + f"has {reader.fieldnames}") + return {row["name"]: row for row in reader} + + +def load_census(download: bool) -> dict[str, dict[str, str]] | None: + """The cached Census table; None when it is not cached and `download` + is off. Raises ValueError naming the cache file when the cached file + is unusable.""" + path = cache_dir() / CENSUS_FILE if not path.exists(): + if not download: + return None print(f"downloading {CENSUS_URL} ...", file=sys.stderr) - _download(CENSUS_URL, path) - with zipfile.ZipFile(path) as zf: - return read_census(zf.read(CENSUS_MEMBER).decode("utf-8")) + _download(CENSUS_URL, path, required_member=CENSUS_MEMBER) + try: + with zipfile.ZipFile(path) as zf: + return read_census(zf.read(CENSUS_MEMBER).decode("utf-8")) + except (zipfile.BadZipFile, KeyError, ValueError) as exc: + raise ValueError(f"{path} is unusable ({exc}); delete it and run " + "again to re-download") from exc -def census_usage(word: str, table: dict[str, dict[str, str]]) -> Usage: - key = census_key(word) +def census_usage(word: str, table: dict[str, dict[str, str]] | None) -> Usage: + usage = Usage("US Census 2010 surnames", floor="<100") + if table is None: + usage.status = "not downloaded" + usage.note = "not cached, and --no-wikidata does not download" + return usage + key = us_key(word) if not key: - return Usage("US Census 2010 surnames", - note="not representable in the Census's ASCII spelling") + usage.status = "unrepresentable" + usage.note = "a letter has no ASCII spelling in the Census list" + return usage row = table.get(key) if row is None: - return Usage("US Census 2010 surnames", family=0, - note="under 100 bearers, or absent") + usage.family = 0 + usage.note = _join("not listed: under 100 bearers, or none", + _lookup_note(word, key)) + return usage shares = ", ".join( f"{label} {row[col]}%" for label, col in ( ("white", "pctwhite"), ("black", "pctblack"), ("asian/PI", "pctapi"), ("hispanic", "pcthispanic")) if row[col] not in ("(S)", "")) - note = f"rank {int(row['rank']):,}; {shares}" - if key != word.upper(): - note += f"; looked up as {key}" - return Usage("US Census 2010 surnames", family=int(row["count"]), - note=note) + usage.family = int(row["count"]) + usage.note = _join(f"rank {int(row['rank']):,}", shares, + _lookup_note(word, key)) + return usage # -- US Social Security given names -------------------------------------- -def read_ssa(zf: zipfile.ZipFile) -> dict[str, Counter[str]]: - """Total each given name's births by sex across every yobYYYY.txt.""" +_YOB = re.compile(r"yob(\d{4})\.txt", re.IGNORECASE) + + +def read_ssa(zf: zipfile.ZipFile, source: str = "the SSA zip", + ) -> dict[str, Counter[str]]: + """Total each given name's births by sex across every yobYYYY.txt, + matched by basename in any case so a re-zipped folder still reads. + Raises ValueError when the zip is not the national file.""" totals: dict[str, Counter[str]] = {} + years = 0 for member in zf.namelist(): - if not (member.startswith("yob") and member.endswith(".txt")): + if not _YOB.fullmatch(Path(member).name): continue - for line in zf.read(member).decode("utf-8").splitlines(): + years += 1 + for number, line in enumerate( + zf.read(member).decode("utf-8").splitlines(), 1): if not line.strip(): continue - name, sex, count = line.split(",") - totals.setdefault(name.lower(), Counter())[sex] += int(count) + try: + name, sex, count = line.split(",") + totals.setdefault(name.lower(), Counter())[sex] += int(count) + except ValueError as exc: + raise ValueError(f"{source}: {member} line {number} is not " + f"Name,Sex,Count: {line!r}") from exc + if years < SSA_MIN_YEARS: + raise ValueError( + f"{source} holds {years} yobYYYY.txt files; the national file " + f"from {SSA_URL} holds one per year since 1880") return totals @@ -169,8 +272,12 @@ def load_ssa() -> dict[str, Counter[str]] | None: path = cache_dir() / SSA_FILE if not path.exists(): return None - with zipfile.ZipFile(path) as zf: - return read_ssa(zf) + try: + with zipfile.ZipFile(path) as zf: + return read_ssa(zf, str(path)) + except zipfile.BadZipFile as exc: + raise ValueError(f"{path} is not a zip ({exc}); save {SSA_URL} " + "there again") from exc def ssa_missing_message() -> str: @@ -180,114 +287,153 @@ def ssa_missing_message() -> str: def ssa_usage(word: str, table: dict[str, Counter[str]] | None) -> Usage: - source = "US SSA given names 1880-" + usage = Usage("US SSA given names 1880-", floor="<5/yr", lower_bound=True) if table is None: - return Usage(source, note="not downloaded (see the end of the report)") - sexes = table.get(census_key(word).lower()) + usage.status = "not downloaded" + usage.note = "see the end of the report" + return usage + key = us_key(word) + if not key: + usage.status = "unrepresentable" + usage.note = "a letter has no ASCII spelling in the SSA list" + return usage + sexes = table.get(key.lower()) if not sexes: - return Usage(source, given=0, note="under 5 births in any year, or absent") - split = ", ".join(f"{sex} {n:,}" for sex, n in sorted(sexes.items())) - return Usage(source, given=sum(sexes.values()), note=split) + usage.given = 0 + usage.note = _join("not listed: under 5 births of either sex in " + "every year, or none", _lookup_note(word, key)) + return usage + usage.given = sum(sexes.values()) + usage.note = _join(", ".join(f"{sex} {n:,}" for sex, n in sorted(sexes.items())), + _lookup_note(word, key)) + return usage # -- Wikidata ------------------------------------------------------------ -def wikidata_query(word: str, languages: tuple[str, ...] = WIKIDATA_LANGUAGES, - ) -> str: - """People bearing `word` as given (P735) or family (P734) name, one - row per role and country of citizenship. A person with no P27 lands - under an empty country; one with two counts under both, so per-country - counts may sum past the role's total -- the total comes from the - distinct count, not from the sum.""" - literal = json.dumps(word, ensure_ascii=False) - values = " ".join(f"{literal}@{lang}" for lang in languages) +def spellings(word: str) -> list[str]: + """The label spellings a word is matched in: as written, lower-case, + and capitalized per word. A name with internal capitals (McDonald) + has no rule that reconstructs it, so it must be passed as written.""" + capped = " ".join(w[:1].upper() + w[1:].lower() for w in word.split(" ")) + return list(dict.fromkeys([word, word.lower(), capped, word.title()])) + + +def _values(forms: list[str], languages: tuple[str, ...]) -> str: + return " ".join(f"{json.dumps(form, ensure_ascii=False)}@{lang}" + for form in forms for lang in languages) + + +def _people(forms: list[str], languages: tuple[str, ...]) -> str: + # Every spelling in one VALUES block, so COUNT(DISTINCT ?p) counts a + # person reached through two spellings once. return f""" -SELECT ?role ?country (COUNT(DISTINCT ?p) AS ?n) WHERE {{ - VALUES ?label {{ {values} }} + VALUES ?label {{ {_values(forms, languages)} }} ?item rdfs:label ?label . {{ ?p wdt:P735 ?item . BIND("given" AS ?role) }} UNION - {{ ?p wdt:P734 ?item . BIND("family" AS ?role) }} - OPTIONAL {{ ?p wdt:P27 ?c . ?c rdfs:label ?country . - FILTER(LANG(?country) = "en") }} -}} GROUP BY ?role ?country""" + {{ ?p wdt:P734 ?item . BIND("family" AS ?role) }}""" -def wikidata_totals_query(word: str, +def wikidata_totals_query(forms: list[str], languages: tuple[str, ...] = WIKIDATA_LANGUAGES, ) -> str: - literal = json.dumps(word, ensure_ascii=False) - values = " ".join(f"{literal}@{lang}" for lang in languages) return f""" -SELECT ?role (COUNT(DISTINCT ?p) AS ?n) WHERE {{ - VALUES ?label {{ {values} }} - ?item rdfs:label ?label . - {{ ?p wdt:P735 ?item . BIND("given" AS ?role) }} - UNION - {{ ?p wdt:P734 ?item . BIND("family" AS ?role) }} +SELECT ?role (COUNT(DISTINCT ?p) AS ?n) WHERE {{{_people(forms, languages)} }} GROUP BY ?role""" -def _bindings(response: dict) -> list[dict[str, str]]: # type: ignore[type-arg] +def wikidata_countries_query(forms: list[str], + languages: tuple[str, ...] = WIKIDATA_LANGUAGES, + ) -> str: + """People per role and country of citizenship. A country with no + English label is named by its Q-id rather than dropped, so only a + person with no P27 at all lands in the empty country. A person with + two citizenships counts under both, so these rows may sum past the + role's total, which comes from wikidata_totals_query.""" + return f""" +SELECT ?role ?country (COUNT(DISTINCT ?p) AS ?n) WHERE {{{_people(forms, languages)} + OPTIONAL {{ ?p wdt:P27 ?c . + OPTIONAL {{ ?c rdfs:label ?cl . FILTER(LANG(?cl) = "en") }} }} + BIND(IF(BOUND(?c), COALESCE(?cl, STRAFTER(STR(?c), "entity/")), "") + AS ?country) +}} GROUP BY ?role ?country""" + + +def _bindings(response: dict[str, Any]) -> list[dict[str, str]]: return [{key: cell["value"] for key, cell in row.items()} for row in response["results"]["bindings"]] def read_wikidata(totals: list[dict[str, str]], by_country: list[dict[str, str]], top: int) -> Usage: - usage = Usage("Wikidata people (lower bound)", given=0, family=0) + """Raises ValueError on a role the queries cannot produce, rather + than letting it fall through to a count of 0.""" + roles = {"given", "family"} + counts = {role: 0 for role in roles} for row in totals: - setattr(usage, row["role"], int(row["n"])) - for role in ("given", "family"): - rows = [row for row in by_country if row["role"] == role] - counts = sorted(((row["country"], int(row["n"])) - for row in rows if row.get("country")), - key=lambda item: -item[1]) - setattr(usage, f"{role}_countries", counts[:top]) - setattr(usage, f"{role}_stateless", - sum(int(row["n"]) for row in rows if not row.get("country"))) - return usage + if row["role"] not in roles: + raise ValueError(f"unexpected role {row['role']!r}") + counts[row["role"]] = int(row["n"]) + countries: dict[str, list[tuple[str, int]]] = {role: [] for role in roles} + stateless = {role: 0 for role in roles} + for row in by_country: + if row["role"] not in roles: + raise ValueError(f"unexpected role {row['role']!r}") + if row.get("country"): + countries[row["role"]].append((row["country"], int(row["n"]))) + else: + stateless[row["role"]] += int(row["n"]) + for role in roles: + countries[role].sort(key=lambda item: -item[1]) + return Usage( + "Wikidata people", given=counts["given"], family=counts["family"], + lower_bound=True, + given_countries=countries["given"][:top], + family_countries=countries["family"][:top], + given_stateless=stateless["given"], + family_stateless=stateless["family"]) def wikidata_usage(word: str, top: int) -> Usage: - forms = list(dict.fromkeys([word, word.capitalize()])) - totals: list[dict[str, str]] = [] - by_country: list[dict[str, str]] = [] + forms = spellings(word) + matched = "matched as " + ", ".join(forms) try: - for form in forms: - totals += _bindings(_sparql(wikidata_totals_query(form))) - by_country += _bindings(_sparql(wikidata_query(form))) - except OSError as exc: - return Usage("Wikidata people (lower bound)", note=f"query failed: {exc}") - return read_wikidata(_merge(totals, ("role",)), - _merge(by_country, ("role", "country")), top) - - -def _merge(rows: list[dict[str, str]], - keys: tuple[str, ...]) -> list[dict[str, str]]: - """Sum counts from the case variants' queries. A person whose name - item carries both spellings would count twice; name items carry one - canonical label, so in practice the variants find different items.""" - merged: dict[tuple[str, ...], int] = {} - for row in rows: - key = tuple(row.get(k, "") for k in keys) - merged[key] = merged.get(key, 0) + int(row["n"]) - return [dict(zip(keys, key), n=str(n)) for key, n in merged.items()] - - -def _sparql(query: str) -> dict: # type: ignore[type-arg] + usage = read_wikidata( + _bindings(_sparql(wikidata_totals_query(forms))), + _bindings(_sparql(wikidata_countries_query(forms))), top) + except (OSError, ValueError, KeyError, http.client.HTTPException) as exc: + return Usage("Wikidata people", status="failed", lower_bound=True, + note=f"query failed: {type(exc).__name__}: {exc}") + zero = "a zero proves nothing" if not (usage.given or usage.family) else "" + usage.note = _join(matched, zero) + return usage + + +def _sparql(query: str) -> dict[str, Any]: url = WIKIDATA_ENDPOINT + "?" + urllib.parse.urlencode({"query": query}) request = urllib.request.Request(url, headers={ "Accept": "application/sparql-results+json", "User-Agent": USER_AGENT}) with urllib.request.urlopen(request, timeout=90) as response: - return json.load(response) # type: ignore[no-any-return] + result: dict[str, Any] = json.load(response) + return result -def _download(url: str, path: Path) -> None: +def _download(url: str, path: Path, required_member: str) -> None: + """Fetch a zip into the cache, refusing to cache anything that is not + a zip holding `required_member` -- an error page saved under the + cache name would otherwise break every later run.""" request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) with urllib.request.urlopen(request, timeout=120) as response: data = response.read() + try: + with zipfile.ZipFile(io.BytesIO(data)) as zf: + zf.getinfo(required_member) + except (zipfile.BadZipFile, KeyError) as exc: + raise ValueError(f"{url} did not return a zip holding " + f"{required_member} ({exc}); it began {data[:80]!r}" + ) from exc tmp = path.with_suffix(".part") tmp.write_bytes(data) tmp.replace(path) @@ -295,8 +441,12 @@ def _download(url: str, path: Path) -> None: # -- output -------------------------------------------------------------- -def _count(n: int | None) -> str: - return "-" if n is None else f"{n:,}" +def _count(n: int | None, usage: Usage) -> str: + if n is None: + return "-" + if n == 0 and usage.floor: + return usage.floor + return f"{n:,}+" if usage.lower_bound else f"{n:,}" def render(word: str, usages: list[Usage]) -> str: @@ -304,10 +454,11 @@ def render(word: str, usages: list[Usage]) -> str: width = max(len(u.source) for u in usages) lines.append(f" {'source':<{width}} {'given':>9} {'family':>9}") for u in usages: - lines.append(f" {u.source:<{width}} {_count(u.given):>9} " - f"{_count(u.family):>9}") - if u.note: - lines.append(f" {'':<{width}} {u.note}") + lines.append(f" {u.source:<{width}} {_count(u.given, u):>9} " + f"{_count(u.family, u):>9}") + note = u.note if u.status == "ok" else _join(u.status, u.note) + if note: + lines.append(f" {'':<{width}} {note}") for role, countries, stateless in ( ("given", u.given_countries, u.given_stateless), ("family", u.family_countries, u.family_stateless)): @@ -325,17 +476,25 @@ def main(argv: list[str] | None = None) -> int: description="Count how often words are borne as given and family names.") parser.add_argument("words", nargs="+") parser.add_argument("--no-wikidata", action="store_true", - help="offline: cached sources only") + help="offline: cached sources only, nothing downloaded") parser.add_argument("--countries", type=int, default=5, metavar="N", help="countries of citizenship to list per role") parser.add_argument("--json", action="store_true") args = parser.parse_args(argv) - census = load_census() + census_error = "" + try: + census = load_census(download=not args.no_wikidata) + except (OSError, ValueError) as exc: + census, census_error = None, f"{type(exc).__name__}: {exc}" ssa = load_ssa() + report = {} for word in args.words: - usages = [census_usage(word, census), ssa_usage(word, ssa)] + usage = census_usage(word, census) + if census_error: + usage.status, usage.note = "failed", census_error + usages = [usage, ssa_usage(word, ssa)] if not args.no_wikidata: usages.append(wikidata_usage(word, args.countries)) report[word] = usages @@ -347,7 +506,8 @@ def main(argv: list[str] | None = None) -> int: if ssa is None: sys.stdout.flush() # keep the notice after the report, not before it print("\n" + ssa_missing_message(), file=sys.stderr) - return 0 + failed = any(u.status == "failed" for us in report.values() for u in us) + return 2 if failed else 0 if __name__ == "__main__": From bd76515d858b358f0727adc6dc2e0d793d0ca866 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 11 Oct 2026 20:34:58 -0700 Subject: [PATCH 4/5] Fix what the second review of #651 found in the first fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The new SSA validation raised outside any try, so a state-level zip saved in place of the national file crashed the run, losing the Census answer and exiting 1. Both cached sources now load under the same handling: failed per word, the run continues, exit 2. - The strict fold refused modifier letters (category Lm, e.g. the okina in Kaʻahumanu) as letters with no ASCII spelling, a regression from the original fold. They are dropped like an apostrophe. - An 'unknown value' citizenship (a blank-node IRI) fell into the no-citizenship bucket; it is now named as unknown. - A truncated Census download (http.client.IncompleteRead, not an OSError) crashed the run; it is caught with the rest. Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_name_usage_tool.py | 23 +++++++++++++++++ tools/name_usage.py | 44 ++++++++++++++++++++++---------- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/tests/v2/test_name_usage_tool.py b/tests/v2/test_name_usage_tool.py index ed47fd98..1b950c76 100644 --- a/tests/v2/test_name_usage_tool.py +++ b/tests/v2/test_name_usage_tool.py @@ -79,6 +79,10 @@ def test_the_network_guard_bites() -> None: ("Łukasz", "LUKASZ"), ("Æsir", "AESIR"), ("Đorđević", "DORDEVIC"), + # A modifier letter (the ʻokina) is an apostrophe, dropped like + # O'Brien's. The second version refused it as a letter with no ASCII + # spelling, leaving the name with no answer. + ("Kaʻahumanu", "KAAHUMANU"), # No ASCII spelling, wholly or in part: no key at all, never a partial # one (the first version looked up AHMED for the second). ("关", ""), @@ -279,3 +283,22 @@ def test_main_exits_2_when_a_source_failed( (tmp_path / tool.CENSUS_FILE).write_text("error page") assert tool.main(["--no-wikidata", "knight"]) == 2 assert "delete it and run again" in capsys.readouterr().out + + +def test_main_reports_a_bad_ssa_zip_without_losing_the_other_sources( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + # The second version validated the SSA zip outside any try, so a + # state-level zip saved in its place crashed the run and lost the + # Census answer too. + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + with zipfile.ZipFile(tmp_path / tool.CENSUS_FILE, "w") as zf: + zf.writestr(tool.CENSUS_MEMBER, _CENSUS) + with zipfile.ZipFile(tmp_path / tool.SSA_FILE, "w") as zf: + zf.writestr("AK.TXT", "AK,F,1910,Mary,14\n") + assert tool.main(["--no-wikidata", "--json", "shehu"]) == 2 + out, err = capsys.readouterr() + census, ssa = json.loads(out)["shehu"] + assert census["family"] == 561 + assert (ssa["status"], ssa["given"]) == ("failed", None) + assert "not downloaded" not in err # the file exists; it is just wrong diff --git a/tools/name_usage.py b/tools/name_usage.py index 98ad3514..fc854dbe 100644 --- a/tools/name_usage.py +++ b/tools/name_usage.py @@ -160,7 +160,11 @@ def us_key(word: str) -> str: folded = unicodedata.normalize("NFKD", word.translate(_TRANSLITERATE)) letters = [] for ch in folded: - if unicodedata.combining(ch): + # Combining marks are accents NFKD split off; modifier letters + # (category Lm) are apostrophe-like marks such as the Hawaiian + # ʻokina in Kaʻahumanu. Both are dropped, as O'Brien's apostrophe + # is -- refusing them would leave such names with no answer. + if unicodedata.combining(ch) or unicodedata.category(ch) == "Lm": continue if ch.isascii(): if ch.isalpha(): @@ -347,15 +351,20 @@ def wikidata_countries_query(forms: list[str], languages: tuple[str, ...] = WIKIDATA_LANGUAGES, ) -> str: """People per role and country of citizenship. A country with no - English label is named by its Q-id rather than dropped, so only a - person with no P27 at all lands in the empty country. A person with - two citizenships counts under both, so these rows may sum past the - role's total, which comes from wikidata_totals_query.""" + English label is named by its Q-id rather than dropped, and an + "unknown value" citizenship (a blank-node IRI, not an item) is named + as such, so only a person with no P27 at all lands in the empty + country. A person with two citizenships counts under both, so these + rows may sum past the role's total, which comes from + wikidata_totals_query.""" return f""" SELECT ?role ?country (COUNT(DISTINCT ?p) AS ?n) WHERE {{{_people(forms, languages)} OPTIONAL {{ ?p wdt:P27 ?c . OPTIONAL {{ ?c rdfs:label ?cl . FILTER(LANG(?cl) = "en") }} }} - BIND(IF(BOUND(?c), COALESCE(?cl, STRAFTER(STR(?c), "entity/")), "") + BIND(IF(!BOUND(?c), "", + COALESCE(?cl, IF(STRSTARTS(STR(?c), STR(wd:)), + STRAFTER(STR(?c), STR(wd:)), + "unknown citizenship"))) AS ?country) }} GROUP BY ?role ?country""" @@ -482,19 +491,26 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--json", action="store_true") args = parser.parse_args(argv) - census_error = "" + # A source that cannot be loaded is reported per word as failed; it + # must not take the other sources' answers down with it. + failures = (OSError, ValueError, http.client.HTTPException) + census_error = ssa_error = "" try: census = load_census(download=not args.no_wikidata) - except (OSError, ValueError) as exc: + except failures as exc: census, census_error = None, f"{type(exc).__name__}: {exc}" - ssa = load_ssa() + try: + ssa = load_ssa() + except failures as exc: + ssa, ssa_error = None, f"{type(exc).__name__}: {exc}" report = {} for word in args.words: - usage = census_usage(word, census) - if census_error: - usage.status, usage.note = "failed", census_error - usages = [usage, ssa_usage(word, ssa)] + census_row, ssa_row = census_usage(word, census), ssa_usage(word, ssa) + for row, error in ((census_row, census_error), (ssa_row, ssa_error)): + if error: + row.status, row.note = "failed", error + usages = [census_row, ssa_row] if not args.no_wikidata: usages.append(wikidata_usage(word, args.countries)) report[word] = usages @@ -503,7 +519,7 @@ def main(argv: list[str] | None = None) -> int: ensure_ascii=False, indent=2)) else: print("\n\n".join(render(w, us) for w, us in report.items())) - if ssa is None: + if ssa is None and not ssa_error: sys.stdout.flush() # keep the notice after the report, not before it print("\n" + ssa_missing_message(), file=sys.stderr) failed = any(u.status == "failed" for us in report.values() for u in us) From 8cd678107d693ad9b4478781a4457a351c39b7ad Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 11 Oct 2026 20:36:16 -0700 Subject: [PATCH 5/5] Simplify tools/name_usage.py after review One loader helper in place of two identical try blocks in main(); the role check in read_wikidata runs once; a test reads its rows by name. No behavior change. Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_name_usage_tool.py | 11 ++++----- tools/name_usage.py | 40 +++++++++++++++++--------------- 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/tests/v2/test_name_usage_tool.py b/tests/v2/test_name_usage_tool.py index 1b950c76..23b68a95 100644 --- a/tests/v2/test_name_usage_tool.py +++ b/tests/v2/test_name_usage_tool.py @@ -180,12 +180,11 @@ def test_render_shows_no_answer_and_unlisted_differently_from_a_count() -> None: tool.Usage("ssa", status="not downloaded"), tool.Usage("wikidata", given=0, family=12, lower_bound=True), ]) - census, ssa, wikidata = (line.split() for line in text.splitlines()[2::1] - if line.split() and line.split()[0] in - ("census", "ssa", "wikidata")) - assert census[1:] == ["-", "<100"] - assert ssa[1:] == ["-", "-"] - assert wikidata[1:] == ["0+", "12+"] + rows = {cells[0]: cells[1:] + for cells in map(str.split, text.splitlines()[2:]) if cells} + assert rows["census"] == ["-", "<100"] + assert rows["ssa"] == ["-", "-"] + assert rows["wikidata"] == ["0+", "12+"] def test_spellings_cover_lower_and_capitalized_forms_once() -> None: diff --git a/tools/name_usage.py b/tools/name_usage.py index fc854dbe..dc8492ce 100644 --- a/tools/name_usage.py +++ b/tools/name_usage.py @@ -67,9 +67,10 @@ import urllib.request import zipfile from collections import Counter +from collections.abc import Callable from dataclasses import asdict, dataclass, field from pathlib import Path -from typing import Any, Literal +from typing import Any, Literal, TypeVar CENSUS_URL = "https://www2.census.gov/topics/genealogy/2010surnames/names.zip" CENSUS_FILE = "census2010_surnames.zip" @@ -113,6 +114,7 @@ }) Status = Literal["ok", "unrepresentable", "not downloaded", "failed"] +T = TypeVar("T") @dataclass(slots=True) @@ -379,16 +381,15 @@ def read_wikidata(totals: list[dict[str, str]], """Raises ValueError on a role the queries cannot produce, rather than letting it fall through to a count of 0.""" roles = {"given", "family"} - counts = {role: 0 for role in roles} - for row in totals: + for row in totals + by_country: if row["role"] not in roles: raise ValueError(f"unexpected role {row['role']!r}") + counts = {role: 0 for role in roles} + for row in totals: counts[row["role"]] = int(row["n"]) countries: dict[str, list[tuple[str, int]]] = {role: [] for role in roles} stateless = {role: 0 for role in roles} for row in by_country: - if row["role"] not in roles: - raise ValueError(f"unexpected role {row['role']!r}") if row.get("country"): countries[row["role"]].append((row["country"], int(row["n"]))) else: @@ -406,7 +407,6 @@ def read_wikidata(totals: list[dict[str, str]], def wikidata_usage(word: str, top: int) -> Usage: forms = spellings(word) - matched = "matched as " + ", ".join(forms) try: usage = read_wikidata( _bindings(_sparql(wikidata_totals_query(forms))), @@ -415,7 +415,7 @@ def wikidata_usage(word: str, top: int) -> Usage: return Usage("Wikidata people", status="failed", lower_bound=True, note=f"query failed: {type(exc).__name__}: {exc}") zero = "a zero proves nothing" if not (usage.given or usage.family) else "" - usage.note = _join(matched, zero) + usage.note = _join("matched as " + ", ".join(forms), zero) return usage @@ -450,6 +450,17 @@ def _download(url: str, path: Path, required_member: str) -> None: # -- output -------------------------------------------------------------- +def _attempt(load: Callable[[], T | None]) -> tuple[T | None, str]: + """Run a source's loader, returning its table (None when not + downloaded) and "" -- or None and the error it raised. A source that + cannot be loaded is reported per word as failed; it must not take the + other sources' answers down with it.""" + try: + return load(), "" + except (OSError, ValueError, http.client.HTTPException) as exc: + return None, f"{type(exc).__name__}: {exc}" + + def _count(n: int | None, usage: Usage) -> str: if n is None: return "-" @@ -491,18 +502,9 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--json", action="store_true") args = parser.parse_args(argv) - # A source that cannot be loaded is reported per word as failed; it - # must not take the other sources' answers down with it. - failures = (OSError, ValueError, http.client.HTTPException) - census_error = ssa_error = "" - try: - census = load_census(download=not args.no_wikidata) - except failures as exc: - census, census_error = None, f"{type(exc).__name__}: {exc}" - try: - ssa = load_ssa() - except failures as exc: - ssa, ssa_error = None, f"{type(exc).__name__}: {exc}" + census, census_error = _attempt( + lambda: load_census(download=not args.no_wikidata)) + ssa, ssa_error = _attempt(load_ssa) report = {} for word in args.words: