diff --git a/app/ingest/normalize.py b/app/ingest/normalize.py index 048882f..d96e8d5 100644 --- a/app/ingest/normalize.py +++ b/app/ingest/normalize.py @@ -8,6 +8,7 @@ from __future__ import annotations +import math import re from datetime import date @@ -18,7 +19,12 @@ _MEMORY_RE = re.compile(r"(\d+(?:\.\d+)?)\s*(GB|MB)\b", re.IGNORECASE) _BUS_RE = re.compile(r"(\d{2,4})\s*-?\s*bit\b", re.IGNORECASE) _PCIE_RE = re.compile(r"PCI[-\s]?[Ee]?\s*(?:Gen\s*)?(\d(?:\.\d)?)", re.IGNORECASE) -_TDP_RE = re.compile(r"(\d{1,4})(?:\s*/\s*\d{1,4})?\s*W\b", re.IGNORECASE) +_TDP_RE = re.compile( + r"(? float | None: def parse_tdp_w(text: str) -> int | None: - """``"65 W"`` → ``65``; ``"65/95 W"`` → ``65`` (takes the lower bound).""" + """``"65 W"`` → ``65``; ``"65/95 W"`` → ``65``; ``"9.5 W"`` → ``10`` (half-up).""" if not text: return None match = _TDP_RE.search(text) - return int(match.group(1)) if match else None + return math.floor(float(match.group(1)) + 0.5) if match else None + + +def parse_frequency_range_ghz(text: str) -> tuple[float, float] | None: + """``"1.7–2.0 GHz"`` → ``(1.7, 2.0)`` (base, boost); ``None`` if not a range.""" + if not text: + return None + match = _FREQ_RANGE_RE.search(text) + if not match: + return None + low, high = float(match.group(1)), float(match.group(2)) + if match.group(3).lower() == "mhz": + low, high = round(low / 1000, 3), round(high / 1000, 3) + return (low, high) if low < high else None def parse_cache_mb(text: str) -> float | None: @@ -136,6 +160,8 @@ def parse_date(text: str) -> date | None: year_raw = match.group(2) year = 2000 + int(year_raw) if len(year_raw) == 2 else int(year_raw) return _safe_date(year, (quarter - 1) * 3 + 1, 1) + if (match := _MONTH_YEAR_RE.search(stripped)): + return _safe_date(int(match.group(2)), _MONTHS[match.group(1).lower()], 1) if (match := _YEAR_ONLY_RE.search(stripped)): return _safe_date(int(match.group(1)), 1, 1) return None @@ -285,6 +311,17 @@ def guess_cpu_segment(name: str) -> str: return "server" if "threadripper" in lowered: return "hedt" + # Atom lines by Intel's own naming: C/P = microserver & network SoCs + # (Avoton, Rangeley, Denverton, Snow Ridge, Parker Ridge); Z/N/x3/x5/x7 = + # tablet/netbook/phone; D = desktop (Pineview); E/x6000 embedded fall + # through to desktop. + if (atom := re.search(r"\batom\s+([a-z]+)\d", lowered)) is not None: + line = atom.group(1) + if line in {"c", "p"}: + return "server" + if line in {"z", "n", "x"} and not re.search(r"\batom\s+x6\d{3}", lowered): + return "laptop" + return "desktop" # i7-13700K → desktop; i7-13700H → laptop. Look at the suffix on the model number. if re.search(r"\b\d{3,5}([a-z]{1,3})\b", lowered): match = re.search(r"\b\d{3,5}([a-z]{1,3})\b", lowered) diff --git a/app/ingest/sources/wikipedia_cpu.py b/app/ingest/sources/wikipedia_cpu.py index 58330ec..185ca14 100644 --- a/app/ingest/sources/wikipedia_cpu.py +++ b/app/ingest/sources/wikipedia_cpu.py @@ -11,6 +11,7 @@ from __future__ import annotations +import re from collections.abc import Iterator from pathlib import Path @@ -25,6 +26,7 @@ parse_cores_threads, parse_date, parse_frequency_ghz, + parse_frequency_range_ghz, parse_int, parse_tdp_w, ) @@ -43,6 +45,15 @@ ("amd", "List_of_AMD_Threadripper_processors", "AMD Threadripper"), ] +# Citation markers left in model cells ("7501 [ 32 ] [ 33 ]"). +_FOOTNOTE_RE = re.compile(r"\s*\[\s*[\w\s]{1,6}\]") +# Rows naming only a family tier ("Ryzen 5", "Core i7", "Xeon 6") are group +# headers, not SKUs; they pass the has-a-digit check but must never become records. +_FAMILY_ONLY_RE = re.compile( + r"(?:ryzen|ryzen-pro|core|core-ultra|core-i|xeon|epyc|athlon|pentium|celeron|atom|opteron)" + r"-?\d{1,2}" +) + # Manufacturer keys are stored lowercase; these are their display forms used to # synthesize ``name`` when the model string omits the brand. Plain ``.upper()`` # mangles "intel" → "INTEL" (an ingest casing artifact); AMD is genuinely @@ -59,7 +70,9 @@ "threads": ["threads", "thread"], "base_clock": ["base", "freq", "clock"], "boost_clock": ["boost", "turbo", "max"], - "l3_cache": ["l3", "cache"], + # Only an explicit L3 / Smart Cache column is L3. A bare "cache" match used + # to route "L2 cache" columns (e.g. every Atom table) into l3_cache_mb. + "l3_cache": ["l3", "smart cache"], "tdp": ["tdp", "power", "wattage"], "release_date": ["released", "release", "launched", "launch", "date"], "socket": ["socket"], @@ -99,10 +112,12 @@ def _extract( for table in soup.select("table.wikitable"): section_label = _nearest_section_label(table) or fallback_arch for row in parse_table(table, HEADER_RULES): - model = row.cells.get("model", "") + model = _FOOTNOTE_RE.sub("", row.cells.get("model", "")).strip() slug = slugify(model, manufacturer=manufacturer) if len(slug) < 4 or not any(ch.isdigit() for ch in slug): continue + if _FAMILY_ONLY_RE.fullmatch(slug): + continue architecture = row.cells.get("architecture") or section_label yield _build_candidate( manufacturer=manufacturer, @@ -114,6 +129,22 @@ def _extract( ) +_HEADING_NODE_RE = re.compile(r"\((\d+(?:\.\d+)?)\s*nm\)") + + +def _clean_architecture(label: str) -> tuple[str, str | None]: + """``'" Denverton " (14 nm)'`` → ``("Denverton", "14 nm")``. + + Section headings on the Atom/Core list pages quote the codename and carry + the node in parentheses; the raw heading text leaked both into + ``architecture``. + """ + node_match = _HEADING_NODE_RE.search(label) + node = f"{node_match.group(1)} nm" if node_match else None + name = _HEADING_NODE_RE.sub("", label).replace('"', "").replace("“", "").replace("”", "") + return " ".join(name.split()) or label, node + + def _nearest_section_label(table: Tag) -> str | None: for prev in table.find_all_previous(["h2", "h3", "h4"]): text = prev.get_text(" ", strip=True) @@ -139,10 +170,17 @@ def _build_candidate( release_date = parse_date(row.get("release_date", "")) base_clock = parse_frequency_ghz(row.get("base_clock", "")) boost_clock = parse_frequency_ghz(row.get("boost_clock", "")) + # "1.7–2.0 GHz" in a single frequency cell is base–boost; the plain parser + # would read only the number glued to the unit (2.0) as the base clock. + if (clock_range := parse_frequency_range_ghz(row.get("base_clock", ""))) is not None: + base_clock = clock_range[0] + boost_clock = boost_clock or clock_range[1] l3_cache = parse_cache_mb(row.get("l3_cache", "")) tdp = parse_tdp_w(row.get("tdp", "")) socket = row.get("socket") or None process_node = row.get("process_node") or None + architecture, node_from_heading = _clean_architecture(architecture) + process_node = process_node or node_from_heading segment = guess_cpu_segment(model) brand = _BRAND_DISPLAY.get(manufacturer, manufacturer.title()) diff --git a/tests/unit/test_ingest_normalize.py b/tests/unit/test_ingest_normalize.py index eda7bff..1fdb93a 100644 --- a/tests/unit/test_ingest_normalize.py +++ b/tests/unit/test_ingest_normalize.py @@ -12,6 +12,7 @@ parse_cores_threads, parse_date, parse_frequency_ghz, + parse_frequency_range_ghz, parse_int, parse_tdp_w, ) @@ -38,12 +39,35 @@ def test_parse_frequency_ghz(text: str, expected: float | None) -> None: ("65/95 W", 65), ("125W", 125), ("none", None), + # Regression: "9.5 W" used to parse as 5 (the digits after the dot). + ("9.5 W", 10), + ("3.6 W", 4), + ("2.2/3 W", 2), ], ) def test_parse_tdp_w(text: str, expected: int | None) -> None: assert parse_tdp_w(text) == expected +@pytest.mark.parametrize( + "text,expected", + [ + ("1.7–2.0 GHz", (1.7, 2.0)), + ("1.7-2.0 GHz", (1.7, 2.0)), + ("1600–2400 MHz", (1.6, 2.4)), + ("2.0 GHz", None), + ("2.0–1.7 GHz", None), + ("", None), + ], +) +def test_parse_frequency_range_ghz(text: str, expected: tuple[float, float] | None) -> None: + assert parse_frequency_range_ghz(text) == expected + + +def test_parse_date_month_year_keeps_the_month() -> None: + assert parse_date("September 2013") == date(2013, 9, 1) + + @pytest.mark.parametrize( "text,expected", [ @@ -99,3 +123,19 @@ def test_guess_cpu_segment_classifies_common_naming() -> None: assert guess_cpu_segment("Intel Core i7-13700K") == "desktop" assert guess_cpu_segment("Intel Core i7-13700H") == "laptop" assert guess_cpu_segment("AMD Ryzen 9 7945HX") == "laptop" + + +@pytest.mark.parametrize( + "name,expected", + [ + ("Intel Atom C3950", "server"), + ("Intel Atom P5362", "server"), + ("Intel Atom Z3570", "laptop"), + ("Intel Atom x3-C3130", "laptop"), + ("Intel Atom N270", "laptop"), + ("Intel Atom D525", "desktop"), + ("Intel Atom x6414RE", "desktop"), + ], +) +def test_guess_cpu_segment_atom_lines(name: str, expected: str) -> None: + assert guess_cpu_segment(name) == expected diff --git a/tests/unit/test_ingest_wikipedia_cpu.py b/tests/unit/test_ingest_wikipedia_cpu.py index 14eb8e0..238f799 100644 --- a/tests/unit/test_ingest_wikipedia_cpu.py +++ b/tests/unit/test_ingest_wikipedia_cpu.py @@ -88,3 +88,73 @@ def test_filters_non_model_rows_lacking_a_slug() -> None: # short and gets filtered out. slugs = {c.slug for c in candidates} assert all("raptor" not in slug for slug in slugs) + + +# Atom-style table rendered from {{cpulist}}: an "L2 cache" column, a +# base–turbo range in one frequency cell, a fractional TDP, a family-tier +# group row and a footnoted model name. +_ATOM_HTML = """ + +

" Avoton " (22 nm)

+ + + + + + + + + + + + + + + + + + + + + +
ModelCoresFrequencyL2 cacheTDPReleasedSocket
Atom C253041.7–2.0 GHz2 × 1 MB9 WSeptember 2013FC-BGA 1283
Atom C250841.25 GHz2 × 1 MB9.5 WMarch 2014FC-BGA 1283
Atom 3 [ 12 ]41.0 GHz1 MB5 W2014BGA
Atom C2338 [ 7 ]21.7 GHz1 MB7 W2014BGA
+ +""" + + +def _atom() -> dict[str, dict[str, object]]: + candidates = WikipediaCpuIngest._extract( + _ATOM_HTML, "intel", "List_of_Intel_Atom_processors", "Intel Atom" + ) + return {c.slug: c.record for c in candidates} + + +def test_l2_cache_column_is_not_written_as_l3() -> None: + assert _atom()["atom-c2530"]["l3_cache_mb"] is None + + +def test_frequency_range_splits_into_base_and_boost() -> None: + record = _atom()["atom-c2530"] + assert record["base_clock_ghz"] == 1.7 + assert record["boost_clock_ghz"] == 2.0 + + +def test_fractional_tdp_rounds_instead_of_dropping_the_integer_part() -> None: + assert _atom()["atom-c2508"]["tdp_w"] == 10 + + +def test_month_year_release_keeps_the_month() -> None: + assert _atom()["atom-c2530"]["release_date"] == "2013-09-01" + + +def test_family_tier_rows_and_footnotes_are_handled() -> None: + records = _atom() + assert "atom-3" not in records + assert "atom-c2338" in records + assert records["atom-c2338"]["name"] == "Intel Atom C2338" + + +def test_quoted_heading_is_cleaned_and_node_extracted() -> None: + record = _atom()["atom-c2530"] + assert record["architecture"] == "Avoton" + assert record["process_node"] == "22 nm"