From 08800ea7acccbcfb06fae494b1c26b7d33fbc277 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Fri, 25 Sep 2026 14:38:01 +0900 Subject: [PATCH 1/4] Parse multi-row headers in the Wikipedia GPU ingest The GPU list pages put units in stacked header rows ("Memory / Size (MiB)", "Core clock (MHz)") and leave body cells as bare numbers, so the old single-row matcher dropped almost every row. Read the whole header block, apply the header unit, map code names to microarchitectures, keep the full bus string, file pre-Oct-2010 AMD cards under ATI, and skip IGP, dual-GPU and idle-TDP columns. Abbreviated month names now parse too. --- app/ingest/normalize.py | 32 +-- app/ingest/sources/wikipedia_gpu.py | 364 +++++++++++++++++------- app/ingest/sources/wikitable.py | 38 +++ tests/unit/test_ingest_wikipedia_gpu.py | 129 +++++---- 4 files changed, 391 insertions(+), 172 deletions(-) diff --git a/app/ingest/normalize.py b/app/ingest/normalize.py index d96e8d5..f3e89a8 100644 --- a/app/ingest/normalize.py +++ b/app/ingest/normalize.py @@ -32,30 +32,26 @@ _ISO_DATE_RE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})$") _NUMERIC_DATE_RE = re.compile(r"^(\d{4})/(\d{2})/(\d{2})$") +# Full month names and common abbreviations ("Mar", "Sept.") — GPU list tables +# write dates like "Mar 11, 2010" and "Dec 2024". +_MONTH_NAME = ( + r"(Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|June?|July?|" + r"Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\.?" +) _LONG_DATE_RE = re.compile( - r"\b(" - r"January|February|March|April|May|June|July|" - r"August|September|October|November|December" - r")\s+(\d{1,2}),?\s+(\d{4})\b", + r"\b" + _MONTH_NAME + r"\s+(\d{1,2}),?\s+(\d{4})\b", re.IGNORECASE, ) _SHORT_DATE_RE = re.compile( - r"\b(\d{1,2})\s+(" - r"January|February|March|April|May|June|July|" - r"August|September|October|November|December" - r")\s+(\d{4})\b", + r"\b(\d{1,2})\s+" + _MONTH_NAME + r"\s+(\d{4})\b", re.IGNORECASE, ) _QUARTER_RE = re.compile(r"\bQ([1-4])\s*'?(\d{2}|\d{4})\b", re.IGNORECASE) -_MONTH_YEAR_RE = re.compile( - r"\b(January|February|March|April|May|June|July|" - r"August|September|October|November|December)\s+(\d{4})\b", - re.IGNORECASE, -) +_MONTH_YEAR_RE = re.compile(r"\b" + _MONTH_NAME + r"\s+(\d{4})\b", re.IGNORECASE) _YEAR_ONLY_RE = re.compile(r"\b(19\d{2}|20\d{2})\b") _MONTHS = { - name.lower(): index + name.lower()[:3]: index for index, name in enumerate( [ "January", "February", "March", "April", "May", "June", @@ -151,17 +147,17 @@ def parse_date(text: str) -> date | None: return _safe_date(year, month, day) if (match := _LONG_DATE_RE.search(stripped)): month_name, day_str, year_str = match.group(1), match.group(2), match.group(3) - return _safe_date(int(year_str), _MONTHS[month_name.lower()], int(day_str)) + return _safe_date(int(year_str), _MONTHS[month_name.lower()[:3]], int(day_str)) if (match := _SHORT_DATE_RE.search(stripped)): day_str, month_name, year_str = match.group(1), match.group(2), match.group(3) - return _safe_date(int(year_str), _MONTHS[month_name.lower()], int(day_str)) + return _safe_date(int(year_str), _MONTHS[month_name.lower()[:3]], int(day_str)) if (match := _QUARTER_RE.search(stripped)): quarter = int(match.group(1)) year_raw = match.group(2) year = 2000 + int(year_raw) if len(year_raw) == 2 else int(year_raw) return _safe_date(year, (quarter - 1) * 3 + 1, 1) if (match := _MONTH_YEAR_RE.search(stripped)): - return _safe_date(int(match.group(2)), _MONTHS[match.group(1).lower()], 1) + return _safe_date(int(match.group(2)), _MONTHS[match.group(1).lower()[:3]], 1) if (match := _YEAR_ONLY_RE.search(stripped)): return _safe_date(int(match.group(1)), 1, 1) return None @@ -221,6 +217,8 @@ def guess_gpu_segment(name: str) -> str: "instinct", "mi300", "mi325", "mi350", "data center", "datacenter", "professional", "radeon pro", "rtx 6000", "rtx 5000", "rtx 4500", "rtx 4000", + # Workstation lines, matching where the curated records file them. + "firepro", "nvs ", "rtx pro", ) if any(token in lowered for token in enterprise_tokens): return "enterprise" diff --git a/app/ingest/sources/wikipedia_gpu.py b/app/ingest/sources/wikipedia_gpu.py index 463a8b4..94e1088 100644 --- a/app/ingest/sources/wikipedia_gpu.py +++ b/app/ingest/sources/wikipedia_gpu.py @@ -1,17 +1,29 @@ """Wikipedia GPU list pages → ``IngestCandidate`` rows. -The ``List_of__graphics_processing_units`` pages share the same -shape as the CPU lists (multiple ``wikitable``s, header-keyed columns) so -this source reuses the shared grid parser. Required GPU schema fields are -stricter than CPU: ``memory_gb``, ``memory_type``, ``memory_bus_bit``, -``base_clock_mhz``, ``boost_clock_mhz``, ``tdp_w``, ``pcie_version``. Many -list-page rows leave several of those blank — those rows surface as -incomplete candidates and stay out of the PR unless ``--include-drafts``. +The ``List_of__graphics_processing_units`` pages use multi-row +headers (``Memory`` spanning ``Size (MiB) | Bus type | Bus width (bit)``) and +put units in the header, leaving bare numbers in the cells ("Core clock +(MHz)" → ``350``). Columns are therefore classified from the joined header +label and the header's unit is applied to unit-less cells. + +Required GPU schema fields: ``architecture``, ``release_date``, +``memory_gb``, ``memory_type``, ``memory_bus_bit``, ``base_clock_mhz``, +``boost_clock_mhz``, ``tdp_w``, ``pcie_version``. Rows missing any of them +stay out of the PR unless ``--include-drafts``. + +``architecture`` follows the dataset's convention (microarchitecture such as +``Kepler`` / ``TeraScale 2``), not the per-chip code name the tables list; it +comes from an explicit Architecture column or from the code name via +:func:`architecture_from_codename`. Rows whose code name maps to nothing are +left without an architecture (incomplete) rather than guessed. """ from __future__ import annotations +import math +import re from collections.abc import Iterator +from datetime import date from pathlib import Path from bs4 import BeautifulSoup, Tag @@ -19,17 +31,9 @@ from app.coverage.normalize import slugify from app.coverage.sources.wikipedia import fetch_wikipedia_html -from ..normalize import ( - guess_gpu_segment, - parse_date, - parse_frequency_mhz, - parse_memory_bus_bit, - parse_memory_gb, - parse_pcie_version, - parse_tdp_w, -) +from ..normalize import guess_gpu_segment, parse_date from .base import IngestCandidate -from .wikitable import parse_table +from .wikitable import parse_table_header_block # ``Nvidia_Quadro`` redirects to ``Quadro``, whose AGP/PCI/PCIe and NVS tables # carry Quadro4-era boards the GeForce list does not. ATI Rage/Radeon legacy @@ -43,26 +47,182 @@ ("intel", "List_of_Intel_graphics_processing_units", "Intel Graphics"), ] -# Manufacturer keys are stored lowercase; these are their display forms used to -# synthesize ``name`` when the model string omits the brand. Plain ``.upper()`` -# mangles "intel" → "INTEL" (an ingest casing artifact). NVIDIA and AMD are -# genuinely all-caps so they get explicit entries rather than title-casing. -_BRAND_DISPLAY: dict[str, str] = {"nvidia": "NVIDIA", "amd": "AMD", "intel": "Intel"} - -# Same matching strategy as CPU but with GPU-specific keyword sets. -HEADER_RULES: dict[str, list[str]] = { - "model": ["model", "card", "name"], - "architecture": ["architecture", "codename", "code name", "chip"], - "release_date": ["released", "release", "launched", "launch", "date"], - "memory": ["memory", "vram"], - "memory_type": ["memory type", "mem type", "type"], - "memory_bus": ["bus", "interface"], - "base_clock": ["base", "core clock"], - "boost_clock": ["boost", "turbo", "max"], - "tdp": ["tdp", "tbp", "power"], - "process_node": ["process", "fab", "lithography"], - "pcie": ["pcie", "pci-e", "pci express"], -} +_BRAND_DISPLAY: dict[str, str] = {"nvidia": "NVIDIA", "amd": "AMD", "ati": "ATI", "intel": "Intel"} + +# The dataset keeps ATI-branded boards under ``ati``; AMD retired the ATI brand +# in August 2010 and the curated records switch to ``amd`` from October 2010. +_ATI_BRAND_END = date(2010, 10, 1) + +MEMORY_TYPES = ( + "GDDR7", "GDDR6X", "GDDR6", "GDDR5X", "GDDR5", "GDDR4", "GDDR3", "GDDR2", + "HBM3E", "HBM3", "HBM2E", "HBM2", "HBM", + "LPDDR5X", "LPDDR5", "LPDDR4X", "LPDDR4", + "DDR5", "DDR4", "DDR3", "DDR2", "DDR", "SDRAM", "SDR", +) +_MEMORY_TYPE_DISPLAY = {"HBM2E": "HBM2e", "HBM3E": "HBM3e"} + +_NUMBER_RE = re.compile(r"\d[\d,]*(?:\.\d+)?") +_RANGE_RE = re.compile(r"(\d[\d,]*(?:\.\d+)?)\s*(?:–|—|-|to)\s*(\d[\d,]*(?:\.\d+)?)") +_FOOTNOTE_RE = re.compile(r"\s*\[[^\]]{1,12}\]") +_TRAILING_PAREN_RE = re.compile(r"\s*\([^)]*\)\s*$") +_MULTI_GPU_RE = re.compile(r"\d\s*[×x]\s*\d") +_BUS_RE = re.compile(r"\b(PCIe|PCI-E|PCI Express|AGP|PCI|MXM(?:-[A-Z]+)?)\b[^/,;]*", re.IGNORECASE) +# Integrated/chipset graphics are out of scope for the discrete-GPU table. +_OUT_OF_SCOPE_RE = re.compile(r"\bIGP\b|integrated|on-?die|\bAPU\b|nForce", re.IGNORECASE) + +_KNOWN_ARCHITECTURES = ( + "Ada Lovelace", "Blackwell", "Hopper", "Ampere", "Turing", "Volta", "Pascal", + "Maxwell", "Kepler", "Fermi", "Tesla", "Curie", "Rankine", "Kelvin", "Celsius", + "TeraScale 3", "TeraScale 2", "TeraScale", "Ultra-Threaded SE", +) +_VERSIONED_ARCH_RE = re.compile(r"\b(RDNA|CDNA|GCN)\s*(\d)(?:\.(\d))?\b", re.IGNORECASE) + +_NVIDIA_CODENAME_ARCH: tuple[tuple[str, str], ...] = ( + (r"NV1\d", "Celsius"), + (r"NV2\d", "Kelvin"), + (r"NV3\d", "Rankine"), + (r"NV4\d|G7\d", "Curie"), + (r"G8\d|G9\d|GT2\d\d", "Tesla"), + (r"GF1\d\d", "Fermi"), + (r"GK\d", "Kepler"), + (r"GM\d", "Maxwell"), + (r"GP\d", "Pascal"), + (r"GV\d", "Volta"), + (r"TU\d", "Turing"), + (r"GA\d", "Ampere"), + (r"AD\d", "Ada Lovelace"), + (r"GH\d", "Hopper"), + (r"GB\d", "Blackwell"), +) +_ATI_CODENAME_ARCH: tuple[tuple[str, str], ...] = ( + (r"R2\d\d|RV2\d\d", "R200"), + (r"R3\d\d|RV3\d\d", "R300"), + (r"R4\d\d|RV4\d\d", "R400"), + (r"R5\d\d|RV5\d\d", "Ultra-Threaded SE"), + (r"R6\d\d|RV6\d\d|RV7\d\d", "TeraScale"), + (r"RV8\d\d|Cedar|Redwood|Juniper|Cypress|Hemlock|Caicos|Turks|Barts", "TeraScale 2"), + (r"Cayman|Antilles", "TeraScale 3"), +) + + +def architecture_from_codename(codename: str, manufacturer: str) -> str | None: + """Map a chip code name to the dataset's microarchitecture name. + + ``"NV34GL"`` → ``"Rankine"``; ``"2× G98-850"`` → ``"Tesla"``; + ``"Redwood XT GL (RV830)"`` → ``"TeraScale 2"``. Returns ``None`` when the + code name matches no known family — callers must not guess. + """ + text = codename.strip() + if not text: + return None + for known in _KNOWN_ARCHITECTURES: + if text.lower().startswith(known.lower()): + return known + if (versioned := _VERSIONED_ARCH_RE.search(text)) is not None: + family, major, minor = versioned.groups() + return f"{family.upper()} {major}.{minor or 0}" + token = re.sub(r"^\s*\d\s*[x×]\s*", "", text) + table = _NVIDIA_CODENAME_ARCH if manufacturer == "nvidia" else _ATI_CODENAME_ARCH + for pattern, arch in table: + if re.match(rf"(?:{pattern})", token, re.IGNORECASE) or re.search( + rf"\(({pattern})", token, re.IGNORECASE + ): + return arch + return None + + +def classify_column(label: str) -> str | None: + """Joined header label → field key (``None`` for columns we don't use).""" + lab = _FOOTNOTE_RE.sub("", label).lower() + if lab.startswith("model"): + return "model" + if lab.startswith("architecture"): + return "architecture" + if "code name" in lab or "codename" in lab: + return "codename" + if "bus interface" in lab: + return "bus" + if "launch" in lab or "release date" in lab: + return "release_date" + if "memory" in lab and "size" in lab: + return "memory_size" + if "memory" in lab and "type & width" in lab: + return "memory_type_width" + if "memory" in lab and "bus type" in lab: + return "memory_type" + if "memory" in lab and "bus width" in lab: + return "memory_bus" + if "boost" in lab and "clock" in lab: + return "boost_clock" + if "memory" not in lab and "shader" not in lab and ( + "core clock" in lab or "core / clock" in lab or "clock rate / core" in lab + or "base clock" in lab + ): + return "base_clock" + if "idle" in lab: + return None + if lab.startswith("tdp") or "board power" in lab: + return "tdp" + return None + + +def _unit(label: str) -> str: + lab = label.lower() + for unit in ("gib", "gb", "mib", "mb", "ghz", "mhz"): + if re.search(rf"\b{unit}\b", lab): + return unit + return "" + + +def _number(text: str) -> float | None: + match = _NUMBER_RE.search(text.replace(" ", "").replace("\xa0", " ")) + return float(match.group(0).replace(",", "")) if match else None + + +def _clock_mhz(text: str, unit: str) -> tuple[int | None, int | None]: + """Cell → ``(base, boost)`` MHz. ``"500–700"`` is a base–boost range.""" + scale = 1000 if unit == "ghz" or "ghz" in text.lower() else 1 + if (rng := _RANGE_RE.search(text)) is not None: + low = float(rng.group(1).replace(",", "")) * scale + high = float(rng.group(2).replace(",", "")) * scale + if low < high: + return round(low), round(high) + value = _number(text) + return (round(value * scale), None) if value is not None else (None, None) + + +def _memory_gb(text: str, unit: str) -> float | None: + value = _number(text) + if value is None: + return None + lowered = text.lower() + if "gb" in lowered or "gib" in lowered or unit in {"gb", "gib"}: + return value + if "mb" in lowered or "mib" in lowered or unit in {"mb", "mib"}: + return round(value / 1024, 4) + return None + + +def _memory_type(text: str) -> str | None: + upper = text.upper() + for kind in MEMORY_TYPES: + if re.search(rf"(? str | None: + """``"PCIe 2.0 ×16"`` → ``"PCIe 2.0 x16"``; ``"AGP 8×"`` → ``"AGP 8x"``.""" + match = _BUS_RE.search(_FOOTNOTE_RE.sub("", text)) + if not match: + return None + value = match.group(0).replace("×", "x").replace("PCI-E", "PCIe").replace("PCI Express", "PCIe") + # "AGP 4× PCI" lists alternative board variants; keep the first bus only. + first, *rest = re.split(r"\s(?=(?:PCIe|AGP|PCI|MXM)\b)", value, maxsplit=1) + value = first if rest else value + value = re.sub(r"\s+x\s*(\d+)", r" x\1", value) + value = re.sub(r"(\d)\s*x\b", r"\1x", value) + return " ".join(value.split()).strip(" -") or None class WikipediaGpuIngest: @@ -95,93 +255,101 @@ def _extract( soup = BeautifulSoup(html, "html.parser") source_url = f"https://en.wikipedia.org/wiki/{page}" for table in soup.select("table.wikitable"): - section_label = _nearest_section_label(table) or fallback_arch - for row in parse_table(table, HEADER_RULES): - model = row.cells.get("model", "") - slug = slugify(model, manufacturer=manufacturer) - if len(slug) < 4 or not any(ch.isdigit() for ch in slug): - continue - architecture = row.cells.get("architecture") or section_label - yield _build_candidate( - manufacturer=manufacturer, - architecture=architecture, - model=model, - slug=slug, - row=row.cells, - source_url=source_url, - ) - - -def _nearest_section_label(table: Tag) -> str | None: - for prev in table.find_all_previous(["h2", "h3", "h4"]): - text = prev.get_text(" ", strip=True) - if text and "edit" not in text.lower(): - return text.split("[")[0].strip() or None - return None + if not isinstance(table, Tag): + continue + labels, body = parse_table_header_block(table) + columns = {i: (classify_column(label), _unit(label)) for i, label in enumerate(labels)} + if not any(key == "model" for key, _ in columns.values()): + continue + for row in body: + cells: dict[str, tuple[str, str]] = {} + for i, text in enumerate(row): + key, unit = columns.get(i, (None, "")) + if key and text and key not in cells: + cells[key] = (text, unit) + candidate = _build_candidate(manufacturer, cells, source_url) + if candidate is not None: + yield candidate def _build_candidate( - *, - manufacturer: str, - architecture: str, - model: str, - slug: str, - row: dict[str, str], - source_url: str, -) -> IngestCandidate: - release_date = parse_date(row.get("release_date", "")) - memory_gb = parse_memory_gb(row.get("memory", "")) - memory_bus_bit = parse_memory_bus_bit(row.get("memory_bus", "")) - base_clock_mhz = parse_frequency_mhz(row.get("base_clock", "")) - boost_clock_mhz = parse_frequency_mhz(row.get("boost_clock", "")) - tdp_w = parse_tdp_w(row.get("tdp", "")) - pcie_version = parse_pcie_version(row.get("pcie", "")) - memory_type = (row.get("memory_type") or "").upper() or None + manufacturer: str, cells: dict[str, tuple[str, str]], source_url: str +) -> IngestCandidate | None: + def text(key: str) -> str: + return cells.get(key, ("", ""))[0] + + def unit(key: str) -> str: + return cells.get(key, ("", ""))[1] + + raw_model = _FOOTNOTE_RE.sub("", text("model")).strip() + model = _TRAILING_PAREN_RE.sub("", raw_model).strip() + # "Radeon DDR / Radeon 7200": two names for one board; the first is the + # launch name and is what distinguishes it (SDR vs DDR both became "7200"). + model = model.split(" / ")[0].strip() + bus_text = text("bus") + if not model or _OUT_OF_SCOPE_RE.search(model) or _OUT_OF_SCOPE_RE.search(bus_text): + return None + if _MULTI_GPU_RE.search(text("memory_size")): + # "2× 128": per-GPU values on dual-GPU boards don't fit one record. + return None + + release_date = parse_date(text("release_date")) + if manufacturer == "amd" and release_date is not None and release_date < _ATI_BRAND_END: + manufacturer = "ati" + slug = slugify(model, manufacturer=manufacturer) + if len(slug) < 4 or not any(ch.isdigit() for ch in slug): + return None + + base_clock, range_boost = _clock_mhz(text("base_clock"), unit("base_clock")) + boost_clock = _clock_mhz(text("boost_clock"), unit("boost_clock"))[0] or range_boost + if boost_clock is None and base_clock is not None and "boost_clock" not in cells: + # Pre-boost boards: the dataset stores boost == base (586/586 pre-2010). + boost_clock = base_clock + + type_width = text("memory_type_width") + memory_type = _memory_type(text("memory_type") or type_width) + width_match = re.search(r"(\d+)\s*-?\s*bit", type_width, re.IGNORECASE) + memory_bus = _number(text("memory_bus")) or ( + float(width_match.group(1)) if width_match else None + ) + tdp = _number(text("tdp")) + + architecture = architecture_from_codename(text("architecture"), manufacturer) or ( + architecture_from_codename(text("codename"), manufacturer) + ) - segment = guess_gpu_segment(model) brand = _BRAND_DISPLAY.get(manufacturer, manufacturer.title()) name = model if model.lower().startswith(manufacturer) else f"{brand} {model}" - record: dict[str, object | None] = { "slug": slug, "name": name, "manufacturer": manufacturer, "architecture": architecture, "release_date": release_date.isoformat() if release_date else None, - "memory_gb": memory_gb, + "memory_gb": _memory_gb(text("memory_size"), unit("memory_size")), "memory_type": memory_type, - "memory_bus_bit": memory_bus_bit, - "base_clock_mhz": base_clock_mhz, - "boost_clock_mhz": boost_clock_mhz, - "tdp_w": tdp_w, - "pcie_version": pcie_version, + "memory_bus_bit": int(memory_bus) if memory_bus else None, + "base_clock_mhz": base_clock, + "boost_clock_mhz": boost_clock, + "tdp_w": math.floor(tdp + 0.5) if tdp else None, + "pcie_version": normalize_bus_interface(bus_text), "msrp_usd": None, "verified": False, "source_urls": [source_url], } - required = ( - "architecture", - "release_date", - "memory_gb", - "memory_type", - "memory_bus_bit", - "base_clock_mhz", - "boost_clock_mhz", - "tdp_w", - "pcie_version", + "architecture", "release_date", "memory_gb", "memory_type", "memory_bus_bit", + "base_clock_mhz", "boost_clock_mhz", "tdp_w", "pcie_version", ) missing = tuple(field for field in required if record.get(field) in (None, "")) - year = release_date.year if release_date else "unknown" - output_path = Path("gpu") / manufacturer / str(year) / segment / f"{slug}.json" - + segment = guess_gpu_segment(model) return IngestCandidate( category="gpu", manufacturer=manufacturer, slug=slug, record=record, source_url=source_url, - output_path=output_path, + output_path=Path("gpu") / manufacturer / str(year) / segment / f"{slug}.json", missing_fields=missing, ) diff --git a/app/ingest/sources/wikitable.py b/app/ingest/sources/wikitable.py index 9119bca..c73dc75 100644 --- a/app/ingest/sources/wikitable.py +++ b/app/ingest/sources/wikitable.py @@ -91,6 +91,44 @@ def _table_to_grid(table: Tag) -> list[list[str]]: return grid +def parse_table_header_block(table: Tag) -> tuple[list[str], list[list[str]]]: + """Return ``(labels, data_rows)`` for tables with a multi-row ```` header. + + Consecutive leading all-```` rows form the header block. Each column's + label joins the distinct texts stacked above it with ``" / "``, so a + ``Memory`` cell spanning ``Size (MiB) | Bus type | Bus width (bit)`` + yields ``"Memory / Size (MiB)"`` etc. Body rows made only of ```` + (repeated headers, group captions) are skipped. Citation markers + (````) are dropped before reading text. + """ + # Plain ``2`` note markers too, not just ``sup.reference``. + for sup in table.select("sup"): + sup.decompose() + rows = [r for r in table.select("tr") if isinstance(r, Tag)] + is_header = [ + bool(cells) and all(c.name == "th" for c in cells) + for cells in (r.find_all(_HEADER_TAGS) for r in rows) + ] + grid = _table_to_grid(table) + block = 0 + while block < len(grid) and block < len(is_header) and is_header[block]: + block += 1 + if block == 0: + return [], [] + width = max(len(row) for row in grid[:block]) + labels: list[str] = [] + for col in range(width): + parts: list[str] = [] + for row in grid[:block]: + if col < len(row) and row[col] and row[col] not in parts: + parts.append(row[col]) + labels.append(" / ".join(parts)) + body = [ + grid[i] for i in range(block, len(grid)) if i >= len(is_header) or not is_header[i] + ] + return labels, body + + def _detect_headers( grid: list[list[str]], header_rules: dict[str, list[str]] ) -> tuple[int, dict[int, str]] | None: diff --git a/tests/unit/test_ingest_wikipedia_gpu.py b/tests/unit/test_ingest_wikipedia_gpu.py index d314d20..d3966da 100644 --- a/tests/unit/test_ingest_wikipedia_gpu.py +++ b/tests/unit/test_ingest_wikipedia_gpu.py @@ -2,79 +2,94 @@ from __future__ import annotations -from app.ingest.sources.wikipedia_gpu import WikipediaGpuIngest +from app.ingest.sources.wikipedia_gpu import ( + WikipediaGpuIngest, + architecture_from_codename, + normalize_bus_interface, +) +# Mirrors the real list pages: a two-row header whose units live in the +# header ("Size (MiB)", "Core clock (MHz)") while body cells are bare numbers. _HTML = """ -

GeForce RTX 40 series

- - - - - - - - - - + + + - - - - - - - - - - + - - - - - - - - - + + + + + + + + + + + + + + + + + +
ModelArchitectureLaunchedMemoryTypeBusCore clockBoostTDPPCIeModelLaunchCode nameBus interfaceCore clock (MHz)MemoryTDP (Watts)
GeForce RTX 4090Ada LovelaceOctober 12, 202224 GBGDDR6X384-bit2235 MHz2520 MHz450 WPCIe 4.0 x16Size (MiB)Bus typeBus width (bit)IdleMax.
GeForce RTX 4080 SuperJanuary 31, 202416 GBGDDR6X256-bit2295 MHz2550 MHz320 WPCIe 4.0 x16Radeon HD 5870 Eyefinity Edition2Mar 11, 2010Cypress XTPCIe 2.1 ×168502048GDDR525627228
Radeon HD 6970December 15, 2010Cayman XTPCIe 2.1 ×168802048GDDR525620250
Radeon HD 4200March 2, 2010RS880IGP500128DDR264115
Radeon HD 5970November 18, 20092× Hemlock XTPCIe 2.1 ×167252× 1024GDDR52× 25651294
""" -def test_extracts_rtx_4090_with_full_fields() -> None: - candidates = list( - WikipediaGpuIngest._extract( - _HTML, "nvidia", "List_of_Nvidia_graphics_processing_units", "NVIDIA GeForce" - ) +def _extract() -> dict[str, object]: + candidates = WikipediaGpuIngest._extract( + _HTML, "amd", "List_of_AMD_graphics_processing_units", "AMD Radeon" ) - by_slug = {c.slug: c for c in candidates} - assert "geforce-rtx-4090" in by_slug - rtx = by_slug["geforce-rtx-4090"] - assert rtx.is_complete - assert rtx.record["memory_gb"] == 24.0 - assert rtx.record["memory_type"] == "GDDR6X" - assert rtx.record["memory_bus_bit"] == 384 - assert rtx.record["base_clock_mhz"] == 2235 - assert rtx.record["boost_clock_mhz"] == 2520 - assert rtx.record["tdp_w"] == 450 - assert rtx.record["pcie_version"] == "4.0" - assert rtx.record["architecture"] == "Ada Lovelace" - assert rtx.record["release_date"] == "2022-10-12" - assert rtx.output_path.as_posix() == "gpu/nvidia/2022/consumer/geforce-rtx-4090.json" + return {c.slug: c for c in candidates} -def test_rowspan_carries_architecture_to_second_row() -> None: - candidates = list( - WikipediaGpuIngest._extract( - _HTML, "nvidia", "List_of_Nvidia_graphics_processing_units", "NVIDIA GeForce" - ) +def test_multirow_header_with_header_units() -> None: + card = _extract()["radeon-hd-5870-eyefinity-edition"] + assert card.is_complete # type: ignore[attr-defined] + assert card.record == card.record | { # type: ignore[attr-defined] + "architecture": "TeraScale 2", + "release_date": "2010-03-11", + "memory_gb": 2.0, + "memory_type": "GDDR5", + "memory_bus_bit": 256, + "base_clock_mhz": 850, + "boost_clock_mhz": 850, # pre-boost board: boost == base + "tdp_w": 228, # Max., not Idle + "pcie_version": "PCIe 2.1 x16", + } + # Launched before the Oct 2010 rebrand → filed under ATI. + assert card.output_path.as_posix() == ( # type: ignore[attr-defined] + "gpu/ati/2010/consumer/radeon-hd-5870-eyefinity-edition.json" ) - by_slug = {c.slug: c for c in candidates} - assert by_slug["geforce-rtx-4080-super"].record["architecture"] == "Ada Lovelace" + + +def test_post_rebrand_card_stays_amd() -> None: + card = _extract()["radeon-hd-6970"] + assert card.record["architecture"] == "TeraScale 3" # type: ignore[attr-defined] + assert card.output_path.as_posix().startswith("gpu/amd/2010/") # type: ignore[attr-defined] + + +def test_skips_igp_and_dual_gpu_rows() -> None: + slugs = _extract() + assert "radeon-hd-4200" not in slugs + assert "radeon-hd-5970" not in slugs + + +def test_codename_and_bus_helpers() -> None: + assert architecture_from_codename("NV34GL", "nvidia") == "Rankine" + assert architecture_from_codename("GK104", "nvidia") == "Kepler" + assert architecture_from_codename("RV620 PRO", "amd") == "TeraScale" + assert architecture_from_codename("Rage 4", "amd") is None + assert normalize_bus_interface("AGP 8×") == "AGP 8x" + assert normalize_bus_interface("AGP 4× PCI") == "AGP 4x" + assert normalize_bus_interface("IGP") is None From b0e675a1711e5f14f4d92742bc56015c552e6c9f Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Fri, 25 Sep 2026 14:46:16 +0900 Subject: [PATCH 2/4] Accept a comma after the month in day-first dates Wikipedia EPYC tables write "10 Oct, 2024", which fell through to the year-only fallback (2024-01-01). --- app/ingest/normalize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/ingest/normalize.py b/app/ingest/normalize.py index f3e89a8..f86a8d4 100644 --- a/app/ingest/normalize.py +++ b/app/ingest/normalize.py @@ -43,7 +43,7 @@ re.IGNORECASE, ) _SHORT_DATE_RE = re.compile( - r"\b(\d{1,2})\s+" + _MONTH_NAME + r"\s+(\d{4})\b", + r"\b(\d{1,2})\s+" + _MONTH_NAME + r",?\s+(\d{4})\b", re.IGNORECASE, ) _QUARTER_RE = re.compile(r"\bQ([1-4])\s*'?(\d{2}|\d{4})\b", re.IGNORECASE) From 6a45a7372bdf1ec74c85b9888756d83b3f654216 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Fri, 25 Sep 2026 14:55:30 +0900 Subject: [PATCH 3/4] Don't read stacked game/boost clock cells as base == boost RDNA and Polaris rows put game and boost clocks in one "Core / Clock" cell, and some list only the game clock. Leave base unknown when a cell stacks two clocks, and only copy base into boost for pre-2012 boards. --- app/ingest/sources/wikipedia_gpu.py | 15 ++++++++++++++- tests/unit/test_ingest_wikipedia_gpu.py | 25 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/app/ingest/sources/wikipedia_gpu.py b/app/ingest/sources/wikipedia_gpu.py index 94e1088..a3ae3cc 100644 --- a/app/ingest/sources/wikipedia_gpu.py +++ b/app/ingest/sources/wikipedia_gpu.py @@ -52,6 +52,8 @@ # The dataset keeps ATI-branded boards under ``ati``; AMD retired the ATI brand # in August 2010 and the curated records switch to ``amd`` from October 2010. _ATI_BRAND_END = date(2010, 10, 1) +# GPU Boost / PowerTune Boost arrived in 2012; before that one core clock is both. +_BOOST_ERA = date(2012, 1, 1) MEMORY_TYPES = ( "GDDR7", "GDDR6X", "GDDR6", "GDDR5X", "GDDR5", "GDDR4", "GDDR3", "GDDR2", @@ -301,9 +303,20 @@ def unit(key: str) -> str: return None base_clock, range_boost = _clock_mhz(text("base_clock"), unit("base_clock")) + unparenthesized = re.sub(r"\([^)]*\)", "", text("base_clock")) + if range_boost is None and len(re.findall(r"\d+", unparenthesized)) > 1: + # RDNA/Polaris "Core / Clock" cells stack game + boost ("1855 2495"): no base. + base_clock = None boost_clock = _clock_mhz(text("boost_clock"), unit("boost_clock"))[0] or range_boost - if boost_clock is None and base_clock is not None and "boost_clock" not in cells: + if ( + boost_clock is None + and base_clock is not None + and "boost_clock" not in cells + and release_date is not None + and release_date < _BOOST_ERA + ): # Pre-boost boards: the dataset stores boost == base (586/586 pre-2010). + # Later single-clock cells are often a game clock, so they stay unknown. boost_clock = base_clock type_width = text("memory_type_width") diff --git a/tests/unit/test_ingest_wikipedia_gpu.py b/tests/unit/test_ingest_wikipedia_gpu.py index d3966da..854c91d 100644 --- a/tests/unit/test_ingest_wikipedia_gpu.py +++ b/tests/unit/test_ingest_wikipedia_gpu.py @@ -93,3 +93,28 @@ def test_codename_and_bus_helpers() -> None: assert normalize_bus_interface("AGP 8×") == "AGP 8x" assert normalize_bus_interface("AGP 4× PCI") == "AGP 4x" assert normalize_bus_interface("IGP") is None + + +_RDNA_HTML = """ + + + + + + + + + + + + +
ModelLaunchArchitectureBus interfaceCoreMemoryTDP
Clock (MHz)ConfigSize (GB)Bus typeBus width (bit)
Radeon Pro W7900 Dual SlotJune 19, 2024RDNA 3PCIe 4.0 ×161855 24956144:384:19248GDDR6384295 W
Radeon RX 6300MJanuary 4, 2022RDNA 2PCIe 4.0 ×41512768:48:322GDDR63225 W
+""" + + +def test_game_clock_cells_do_not_become_base_and_boost() -> None: + page = "List_of_AMD_graphics_processing_units" + cards = {c.slug: c for c in WikipediaGpuIngest._extract(_RDNA_HTML, "amd", page, "")} + assert cards["radeon-pro-w7900-dual-slot"].record["base_clock_mhz"] is None + assert cards["radeon-rx-6300m"].record["boost_clock_mhz"] is None + assert not cards["radeon-rx-6300m"].is_complete From 22d3afe59f17f2673730a154d15740c7a9cab154 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Fri, 25 Sep 2026 15:05:13 +0900 Subject: [PATCH 4/4] Leave threads unknown when a table gives only a core count parse_cores_threads("16") returned (16, 16). The Atom and Xeon list pages have a bare "Cores" column, so Hyper-Threaded parts (Tunnel Creek, Centerton, Broadwell-EX...) came out with half their real thread count. Return (16, None) so such rows stay drafts instead of guessing. --- app/ingest/normalize.py | 9 ++++++--- tests/unit/test_ingest_normalize.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/app/ingest/normalize.py b/app/ingest/normalize.py index f86a8d4..41d90a9 100644 --- a/app/ingest/normalize.py +++ b/app/ingest/normalize.py @@ -121,15 +121,18 @@ def parse_int(text: str) -> int | None: def parse_cores_threads(text: str) -> tuple[int | None, int | None]: - """``"8 / 16"`` → ``(8, 16)``; ``"16"`` → ``(16, 16)`` (assumes SMT).""" + """``"8 / 16"`` → ``(8, 16)``; ``"16"`` → ``(16, None)``. + + A bare core count says nothing about SMT, so threads stay unknown rather + than guessed (Atom/Xeon tables list "Cores" only; many of those parts have HT). + """ if not text: return (None, None) nums = re.findall(r"\d+", text) if not nums: return (None, None) if len(nums) == 1: - cores = int(nums[0]) - return (cores, cores) + return (int(nums[0]), None) return (int(nums[0]), int(nums[1])) diff --git a/tests/unit/test_ingest_normalize.py b/tests/unit/test_ingest_normalize.py index 1fdb93a..8feef6f 100644 --- a/tests/unit/test_ingest_normalize.py +++ b/tests/unit/test_ingest_normalize.py @@ -86,7 +86,7 @@ def test_parse_cache_mb(text: str, expected: float | None) -> None: "text,expected", [ ("8 / 16", (8, 16)), - ("16", (16, 16)), + ("16", (16, None)), ("24/32", (24, 32)), ("", (None, None)), ],