fix(ingest): correct Wikipedia CPU parsing (fractional TDP, clock ranges, L2 vs L3, family rows)

app/ingest/normalize.py

@@ -8,6 +8,7 @@
 
 from __future__ import annotations
 
+import math
 import re
 from datetime import date
 
@@ -18,7 +19,12 @@
 _MEMORY_RE = re.compile(r"(\d+(?:\.\d+)?)\s*(GB|MB)\b", re.IGNORECASE)
 _BUS_RE = re.compile(r"(\d{2,4})\s*-?\s*bit\b", re.IGNORECASE)
 _PCIE_RE = re.compile(r"PCI[-\s]?[Ee]?\s*(?:Gen\s*)?(\d(?:\.\d)?)", re.IGNORECASE)
-_TDP_RE = re.compile(r"(\d{1,4})(?:\s*/\s*\d{1,4})?\s*W\b", re.IGNORECASE)
+_TDP_RE = re.compile(
+    r"(?<![\d.])(\d{1,4}(?:\.\d+)?)(?:\s*/\s*\d{1,4}(?:\.\d+)?)?\s*W\b", re.IGNORECASE
+)
+_FREQ_RANGE_RE = re.compile(
+    r"(\d+(?:\.\d+)?)\s*(?:–|—|-|to)\s*(\d+(?:\.\d+)?)\s*(GHz|MHz)\b", re.IGNORECASE
+)
 _RAM_RE = re.compile(r"(\d{1,3}(?:\.\d+)?)\s*(GB|MB)\b", re.IGNORECASE)
 _BATTERY_RE = re.compile(r"(\d{3,5})\s*m\s*A\s*h\b", re.IGNORECASE)
 _WEIGHT_RE = re.compile(r"(\d{1,3}(?:\.\d+)?)\s*g\b")
@@ -41,6 +47,11 @@
     re.IGNORECASE,
 )
 _QUARTER_RE = re.compile(r"\bQ([1-4])\s*'?(\d{2}|\d{4})\b", re.IGNORECASE)
+_MONTH_YEAR_RE = re.compile(
+    r"\b(January|February|March|April|May|June|July|"
+    r"August|September|October|November|December)\s+(\d{4})\b",
+    re.IGNORECASE,
+)
 _YEAR_ONLY_RE = re.compile(r"\b(19\d{2}|20\d{2})\b")
 
 _MONTHS = {
@@ -67,11 +78,24 @@ def parse_frequency_ghz(text: str) -> float | None:
 
 
 def parse_tdp_w(text: str) -> int | None:
-    """``"65 W"`` → ``65``; ``"65/95 W"`` → ``65`` (takes the lower bound)."""
+    """``"65 W"`` → ``65``; ``"65/95 W"`` → ``65``; ``"9.5 W"`` → ``10`` (half-up)."""
     if not text:
         return None
     match = _TDP_RE.search(text)
-    return int(match.group(1)) if match else None
+    return math.floor(float(match.group(1)) + 0.5) if match else None
+
+
+def parse_frequency_range_ghz(text: str) -> tuple[float, float] | None:
+    """``"1.7–2.0 GHz"`` → ``(1.7, 2.0)`` (base, boost); ``None`` if not a range."""
+    if not text:
+        return None
+    match = _FREQ_RANGE_RE.search(text)
+    if not match:
+        return None
+    low, high = float(match.group(1)), float(match.group(2))
+    if match.group(3).lower() == "mhz":
+        low, high = round(low / 1000, 3), round(high / 1000, 3)
+    return (low, high) if low < high else None
 
 
 def parse_cache_mb(text: str) -> float | None:
@@ -136,6 +160,8 @@ def parse_date(text: str) -> date | None:
         year_raw = match.group(2)
         year = 2000 + int(year_raw) if len(year_raw) == 2 else int(year_raw)
         return _safe_date(year, (quarter - 1) * 3 + 1, 1)
+    if (match := _MONTH_YEAR_RE.search(stripped)):
+        return _safe_date(int(match.group(2)), _MONTHS[match.group(1).lower()], 1)
     if (match := _YEAR_ONLY_RE.search(stripped)):
         return _safe_date(int(match.group(1)), 1, 1)
     return None
@@ -285,6 +311,17 @@ def guess_cpu_segment(name: str) -> str:
         return "server"
     if "threadripper" in lowered:
         return "hedt"
+    # Atom lines by Intel's own naming: C/P = microserver & network SoCs
+    # (Avoton, Rangeley, Denverton, Snow Ridge, Parker Ridge); Z/N/x3/x5/x7 =
+    # tablet/netbook/phone; D = desktop (Pineview); E/x6000 embedded fall
+    # through to desktop.
+    if (atom := re.search(r"\batom\s+([a-z]+)\d", lowered)) is not None:
+        line = atom.group(1)
+        if line in {"c", "p"}:
+            return "server"
+        if line in {"z", "n", "x"} and not re.search(r"\batom\s+x6\d{3}", lowered):
+            return "laptop"
+        return "desktop"
     # i7-13700K → desktop; i7-13700H → laptop. Look at the suffix on the model number.
     if re.search(r"\b\d{3,5}([a-z]{1,3})\b", lowered):
         match = re.search(r"\b\d{3,5}([a-z]{1,3})\b", lowered)

app/ingest/sources/wikipedia_cpu.py

@@ -11,6 +11,7 @@
 
 from __future__ import annotations
 
+import re
 from collections.abc import Iterator
 from pathlib import Path
 
@@ -25,6 +26,7 @@
     parse_cores_threads,
     parse_date,
     parse_frequency_ghz,
+    parse_frequency_range_ghz,
     parse_int,
     parse_tdp_w,
 )
@@ -43,6 +45,15 @@
     ("amd", "List_of_AMD_Threadripper_processors", "AMD Threadripper"),
 ]
 
+# Citation markers left in model cells ("7501 [ 32 ] [ 33 ]").
+_FOOTNOTE_RE = re.compile(r"\s*\[\s*[\w\s]{1,6}\]")
+# Rows naming only a family tier ("Ryzen 5", "Core i7", "Xeon 6") are group
+# headers, not SKUs; they pass the has-a-digit check but must never become records.
+_FAMILY_ONLY_RE = re.compile(
+    r"(?:ryzen|ryzen-pro|core|core-ultra|core-i|xeon|epyc|athlon|pentium|celeron|atom|opteron)"
+    r"-?\d{1,2}"
+)
+
 # Manufacturer keys are stored lowercase; these are their display forms used to
 # synthesize ``name`` when the model string omits the brand. Plain ``.upper()``
 # mangles "intel" → "INTEL" (an ingest casing artifact); AMD is genuinely
@@ -59,7 +70,9 @@
     "threads": ["threads", "thread"],
     "base_clock": ["base", "freq", "clock"],
     "boost_clock": ["boost", "turbo", "max"],
-    "l3_cache": ["l3", "cache"],
+    # Only an explicit L3 / Smart Cache column is L3. A bare "cache" match used
+    # to route "L2 cache" columns (e.g. every Atom table) into l3_cache_mb.
+    "l3_cache": ["l3", "smart cache"],
     "tdp": ["tdp", "power", "wattage"],
     "release_date": ["released", "release", "launched", "launch", "date"],
     "socket": ["socket"],
@@ -99,10 +112,12 @@ def _extract(
         for table in soup.select("table.wikitable"):
             section_label = _nearest_section_label(table) or fallback_arch
             for row in parse_table(table, HEADER_RULES):
-                model = row.cells.get("model", "")
+                model = _FOOTNOTE_RE.sub("", row.cells.get("model", "")).strip()
                 slug = slugify(model, manufacturer=manufacturer)
                 if len(slug) < 4 or not any(ch.isdigit() for ch in slug):
                     continue
+                if _FAMILY_ONLY_RE.fullmatch(slug):
+                    continue
                 architecture = row.cells.get("architecture") or section_label
                 yield _build_candidate(
                     manufacturer=manufacturer,
@@ -114,6 +129,22 @@ def _extract(
                 )
 
 
+_HEADING_NODE_RE = re.compile(r"\((\d+(?:\.\d+)?)\s*nm\)")
+
+
+def _clean_architecture(label: str) -> tuple[str, str | None]:
+    """``'" Denverton " (14 nm)'`` → ``("Denverton", "14 nm")``.
+
+    Section headings on the Atom/Core list pages quote the codename and carry
+    the node in parentheses; the raw heading text leaked both into
+    ``architecture``.
+    """
+    node_match = _HEADING_NODE_RE.search(label)
+    node = f"{node_match.group(1)} nm" if node_match else None
+    name = _HEADING_NODE_RE.sub("", label).replace('"', "").replace("“", "").replace("”", "")
+    return " ".join(name.split()) or label, node
+
+
 def _nearest_section_label(table: Tag) -> str | None:
     for prev in table.find_all_previous(["h2", "h3", "h4"]):
         text = prev.get_text(" ", strip=True)
@@ -139,10 +170,17 @@ def _build_candidate(
     release_date = parse_date(row.get("release_date", ""))
     base_clock = parse_frequency_ghz(row.get("base_clock", ""))
     boost_clock = parse_frequency_ghz(row.get("boost_clock", ""))
+    # "1.7–2.0 GHz" in a single frequency cell is base–boost; the plain parser
+    # would read only the number glued to the unit (2.0) as the base clock.
+    if (clock_range := parse_frequency_range_ghz(row.get("base_clock", ""))) is not None:
+        base_clock = clock_range[0]
+        boost_clock = boost_clock or clock_range[1]
     l3_cache = parse_cache_mb(row.get("l3_cache", ""))
     tdp = parse_tdp_w(row.get("tdp", ""))
     socket = row.get("socket") or None
     process_node = row.get("process_node") or None
+    architecture, node_from_heading = _clean_architecture(architecture)
+    process_node = process_node or node_from_heading
 
     segment = guess_cpu_segment(model)
     brand = _BRAND_DISPLAY.get(manufacturer, manufacturer.title())

tests/unit/test_ingest_normalize.py

@@ -12,6 +12,7 @@
     parse_cores_threads,
     parse_date,
     parse_frequency_ghz,
+    parse_frequency_range_ghz,
     parse_int,
     parse_tdp_w,
 )
@@ -38,12 +39,35 @@ def test_parse_frequency_ghz(text: str, expected: float | None) -> None:
         ("65/95 W", 65),
         ("125W", 125),
         ("none", None),
+        # Regression: "9.5 W" used to parse as 5 (the digits after the dot).
+        ("9.5 W", 10),
+        ("3.6 W", 4),
+        ("2.2/3 W", 2),
     ],
 )
 def test_parse_tdp_w(text: str, expected: int | None) -> None:
     assert parse_tdp_w(text) == expected
 
 
+@pytest.mark.parametrize(
+    "text,expected",
+    [
+        ("1.7–2.0 GHz", (1.7, 2.0)),
+        ("1.7-2.0 GHz", (1.7, 2.0)),
+        ("1600–2400 MHz", (1.6, 2.4)),
+        ("2.0 GHz", None),
+        ("2.0–1.7 GHz", None),
+        ("", None),
+    ],
+)
+def test_parse_frequency_range_ghz(text: str, expected: tuple[float, float] | None) -> None:
+    assert parse_frequency_range_ghz(text) == expected
+
+
+def test_parse_date_month_year_keeps_the_month() -> None:
+    assert parse_date("September 2013") == date(2013, 9, 1)
+
+
 @pytest.mark.parametrize(
     "text,expected",
     [
@@ -99,3 +123,19 @@ def test_guess_cpu_segment_classifies_common_naming() -> None:
     assert guess_cpu_segment("Intel Core i7-13700K") == "desktop"
     assert guess_cpu_segment("Intel Core i7-13700H") == "laptop"
     assert guess_cpu_segment("AMD Ryzen 9 7945HX") == "laptop"
+
+
+@pytest.mark.parametrize(
+    "name,expected",
+    [
+        ("Intel Atom C3950", "server"),
+        ("Intel Atom P5362", "server"),
+        ("Intel Atom Z3570", "laptop"),
+        ("Intel Atom x3-C3130", "laptop"),
+        ("Intel Atom N270", "laptop"),
+        ("Intel Atom D525", "desktop"),
+        ("Intel Atom x6414RE", "desktop"),
+    ],
+)
+def test_guess_cpu_segment_atom_lines(name: str, expected: str) -> None:
+    assert guess_cpu_segment(name) == expected

tests/unit/test_ingest_wikipedia_cpu.py

@@ -88,3 +88,73 @@ def test_filters_non_model_rows_lacking_a_slug() -> None:
     # short and gets filtered out.
     slugs = {c.slug for c in candidates}
     assert all("raptor" not in slug for slug in slugs)
+
+
+# Atom-style table rendered from {{cpulist}}: an "L2 cache" column, a
+# base–turbo range in one frequency cell, a fractional TDP, a family-tier
+# group row and a footnoted model name.
+_ATOM_HTML = """
+<html><body>
+<h3>" Avoton " (22 nm)</h3>
+<table class="wikitable">
+  <tr>
+    <th>Model</th><th>Cores</th><th>Frequency</th><th>L2 cache</th>
+    <th>TDP</th><th>Released</th><th>Socket</th>
+  </tr>
+  <tr>
+    <td>Atom C2530</td><td>4</td><td>1.7–2.0 GHz</td><td>2 × 1 MB</td>
+    <td>9 W</td><td>September 2013</td><td>FC-BGA 1283</td>
+  </tr>
+  <tr>
+    <td>Atom C2508</td><td>4</td><td>1.25 GHz</td><td>2 × 1 MB</td>
+    <td>9.5 W</td><td>March 2014</td><td>FC-BGA 1283</td>
+  </tr>
+  <tr>
+    <td>Atom 3 [ 12 ]</td><td>4</td><td>1.0 GHz</td><td>1 MB</td>
+    <td>5 W</td><td>2014</td><td>BGA</td>
+  </tr>
+  <tr>
+    <td>Atom C2338 [ 7 ]</td><td>2</td><td>1.7 GHz</td><td>1 MB</td>
+    <td>7 W</td><td>2014</td><td>BGA</td>
+  </tr>
+</table>
+</body></html>
+"""
+
+
+def _atom() -> dict[str, dict[str, object]]:
+    candidates = WikipediaCpuIngest._extract(
+        _ATOM_HTML, "intel", "List_of_Intel_Atom_processors", "Intel Atom"
+    )
+    return {c.slug: c.record for c in candidates}
+
+
+def test_l2_cache_column_is_not_written_as_l3() -> None:
+    assert _atom()["atom-c2530"]["l3_cache_mb"] is None
+
+
+def test_frequency_range_splits_into_base_and_boost() -> None:
+    record = _atom()["atom-c2530"]
+    assert record["base_clock_ghz"] == 1.7
+    assert record["boost_clock_ghz"] == 2.0
+
+
+def test_fractional_tdp_rounds_instead_of_dropping_the_integer_part() -> None:
+    assert _atom()["atom-c2508"]["tdp_w"] == 10
+
+
+def test_month_year_release_keeps_the_month() -> None:
+    assert _atom()["atom-c2530"]["release_date"] == "2013-09-01"
+
+
+def test_family_tier_rows_and_footnotes_are_handled() -> None:
+    records = _atom()
+    assert "atom-3" not in records
+    assert "atom-c2338" in records
+    assert records["atom-c2338"]["name"] == "Intel Atom C2338"
+
+
+def test_quoted_heading_is_cleaned_and_node_extracted() -> None:
+    record = _atom()["atom-c2530"]
+    assert record["architecture"] == "Avoton"
+    assert record["process_node"] == "22 nm"