Skip to content

feat(verify): add real --category tablet support to smartphone backfill #79

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Seungpyo1007 merged 2 commits into from
Sep 23, 2026
Merged

feat(verify): add real --category tablet support to smartphone backfill #79

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f9eca5c
feat(verify): add real --category tablet support to smartphone backfill
Seungpyo1007 Sep 23, 2026
474531e
style: wrap long lines in render_summary for ruff
Seungpyo1007 Sep 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 79 additions & 12 deletions app/verify/wikipedia_smartphone_backfill.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,6 @@
CONTRADICT,
NOTFOUND,
Candidate,
WikipediaFetcher,
_heading_matches,
normalize_heading,
)
Expand Down Expand Up @@ -91,6 +90,12 @@
("asus", "ROG_Phone", "ROG Phone"),
)

TABLET_CROSSREF_PAGES: tuple[tuple[str, str, str], ...] = (
("apple", "List_of_iPad_models", "List of iPad models"),
("samsung", "Samsung_Galaxy_Tab", "Samsung Galaxy Tab"),
("google", "Google_Pixel_Tablet", "Google Pixel Tablet"),
)

_BRAND_TOKENS = (
"samsung",
"apple",
Expand Down Expand Up @@ -912,13 +917,13 @@ def append_wikipedia_source(path: Path, url: str) -> str:
return "written"


def smartphone_scan_root(data_root: Path) -> tuple[Path, Path]:
def smartphone_scan_root(data_root: Path, category: str = "smartphone") -> tuple[Path, Path]:
for cand in (data_root, data_root / "data"):
direct = cand / "smartphone"
direct = cand / category
if direct.is_dir():
parent = data_root.parent if data_root.name == "data" else data_root
return direct, parent
raise SystemExit(f"no data/smartphone directory under {data_root}")
raise SystemExit(f"no data/{category} directory under {data_root}")


def iter_smartphone_records(
Expand Down Expand Up @@ -1034,7 +1039,28 @@ def search(self, name: str) -> list[Candidate]:
if name in self._search_cache:
return self._search_cache[name]
self._pause()
res = WikipediaFetcher(timeout=self.timeout, limit=5).search(name)
if self._client is None:
self._client = httpx.Client(
timeout=self.timeout,
headers={"User-Agent": USER_AGENT},
follow_redirects=True,
)
try:
response = self._client.get(
"https://en.wikipedia.org/w/rest.php/v1/search/page",
params={"q": name, "limit": 5},
)
response.raise_for_status()
res = [
Candidate(
title=page["title"],
url=f"https://en.wikipedia.org/wiki/{quote(page['key'], safe='()_')}",
)
for page in response.json().get("pages", [])
if isinstance(page.get("title"), str) and isinstance(page.get("key"), str)
]
except (httpx.HTTPError, ValueError, KeyError, TypeError):
res = []
self._search_cache[name] = res
return res

Expand Down Expand Up @@ -1154,7 +1180,10 @@ def backfill(
search_fn: SearchFn | None = None,
records: list[tuple[str, dict[str, Any]]] | None = None,
cache_path: Path | None = None,
category: str = "smartphone",
) -> RunResult:
if category not in {"smartphone", "tablet"}:
raise ValueError(f"unsupported category: {category}")
writing = apply and not dry_run
html_cache = data_root / "data" / "_verify" / "cache" / "wikipedia_html"
polite = PoliteWiki(sleep_s=sleep_s, cache_dir=html_cache) if fetch_page is None else None
Expand All @@ -1169,7 +1198,9 @@ def backfill(
parsed_pages: set[str] = set()

# Pre-parse list pages
target_pages = pages if pages is not None else CROSSREF_PAGES
target_pages = pages if pages is not None else (
TABLET_CROSSREF_PAGES if category == "tablet" else CROSSREF_PAGES
)
for _mfg, page, _title in target_pages:
status, final, html = fetch(page)
_alive, reason = classify(f"https://en.wikipedia.org/wiki/{page}", status, final or None)
Expand Down Expand Up @@ -1226,9 +1257,31 @@ def consider_fallback(base: str, record_brand: str = "") -> None:
# Load candidate records
repo_root: Path | None = None
if records is None:
phone_dir, repo_root = smartphone_scan_root(data_root)
phone_dir, repo_root = smartphone_scan_root(data_root, category)
chosen = sample_diverse_records(phone_dir, repo_root, limit, exclude_paths=cached_paths)
eligible_count = 73465
if category == "tablet" and limit is not None and len(chosen) < limit:
remaining = sample_diverse_records(
phone_dir,
repo_root,
None,
exclude_paths=cached_paths | {rel for rel, _record in chosen},
)
chosen_models = {str(record.get("base_model_slug")) for _rel, record in chosen}
novel = [
(rel, record)
for rel, record in remaining
if str(record.get("base_model_slug")) not in chosen_models
]
chosen.extend(sample_diverse(novel, limit - len(chosen)))
if len(chosen) < limit:
picked = {rel for rel, _record in chosen}
chosen.extend(
sample_diverse(
[(rel, record) for rel, record in remaining if rel not in picked],
limit - len(chosen),
)
)
eligible_count = len(chosen)
else:
loaded = [
(rel, rec)
Expand Down Expand Up @@ -1272,6 +1325,14 @@ def maybe_write(rel: str, decision: object, url: object) -> None:
consider_fallback(phone.base, record_brand=rec_b)
hits = matching_rows(name, fetcher.rows, record_brand=rec_b)

if category == "tablet":
# A shared substring or a brand-omitted article is insufficient for
# the tablet batch: the article heading must name this exact model.
hits = [
row for row in hits
if normalize_heading(phone.base) == normalize_heading(row.model)
]

live = _liveness_for(hits[0].url if hits else None, page_liveness)
outcome = decide(record, hits, liveness=live if hits else "http-200")
entry = cache_entry(rel, outcome, record)
Expand All @@ -1294,7 +1355,9 @@ def _only(agreements: list[str], allowed: set[str]) -> bool:
return bool(agreements) and set(agreements) <= allowed


def render_summary(result: RunResult, *, dry_run: bool, sleep_s: float) -> str:
def render_summary(
result: RunResult, *, dry_run: bool, sleep_s: float, category: str = "smartphone"
) -> str:
counts = result.counts()
year_only = [
row
Expand All @@ -1307,7 +1370,9 @@ def render_summary(result: RunResult, *, dry_run: bool, sleep_s: float) -> str:
spec_conflicts = [row for row in result.rows if row.get("decision") == CONTRADICT]

lines = [
"# Wikipedia Smartphone backfill dry-run" if dry_run else "# Wikipedia Smartphone backfill",
f"# Wikipedia {category.title()} backfill dry-run"
if dry_run
else f"# Wikipedia {category.title()} backfill",
"",
f"- records processed: **{len(result.rows):,}** across **{len(result.brands)}** brands",
f"- total eligible in dataset: {result.eligible:,}",
Expand Down Expand Up @@ -1350,6 +1415,7 @@ def render_summary(result: RunResult, *, dry_run: bool, sleep_s: float) -> str:
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--data-root", type=Path, default=Path("."), help="TechAPI repository root")
parser.add_argument("--category", choices=("smartphone", "tablet"), default="smartphone")
parser.add_argument("--limit", type=int, default=300, help="Max records to process")
parser.add_argument(
"--sleep", type=float, default=MIN_SLEEP_S, help="Sleep between Wikipedia calls"
Expand All @@ -1368,7 +1434,7 @@ def main(argv: list[str] | None = None) -> int:

dry_run = not args.apply if args.apply else args.dry_run
cache_path = args.cache or (
args.data_root / "data" / "_verify" / "state" / "wikipedia_smartphone_cache.jsonl"
args.data_root / "data" / "_verify" / "state" / f"wikipedia_{args.category}_cache.jsonl"
)

result = backfill(
Expand All @@ -1379,8 +1445,9 @@ def main(argv: list[str] | None = None) -> int:
apply=args.apply,
max_fallback=args.max_fallback,
cache_path=cache_path,
category=args.category,
)
print(render_summary(result, dry_run=dry_run, sleep_s=args.sleep))
print(render_summary(result, dry_run=dry_run, sleep_s=args.sleep, category=args.category))
return 0


Expand Down
Loading