Skip to content

Commit 80ca850

Browse files
committed
Parse multi-row headers in the Wikipedia GPU ingest
The GPU list pages put units in stacked header rows ("Memory / Size (MiB)", "Core clock (MHz)") and leave body cells as bare numbers, so the old single-row matcher dropped almost every row. Read the whole header block, apply the header unit, map code names to microarchitectures, keep the full bus string, file pre-Oct-2010 AMD cards under ATI, and skip IGP, dual-GPU and idle-TDP columns. Abbreviated month names now parse too.
1 parent 269571f commit 80ca850

4 files changed

Lines changed: 391 additions & 172 deletions

File tree

‎app/ingest/normalize.py‎

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -32,30 +32,26 @@
3232

3333
_ISO_DATE_RE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})$")
3434
_NUMERIC_DATE_RE = re.compile(r"^(\d{4})/(\d{2})/(\d{2})$")
35+
# Full month names and common abbreviations ("Mar", "Sept.") — GPU list tables
36+
# write dates like "Mar 11, 2010" and "Dec 2024".
37+
_MONTH_NAME = (
38+
r"(Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|June?|July?|"
39+
r"Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\.?"
40+
)
3541
_LONG_DATE_RE = re.compile(
36-
r"\b("
37-
r"January|February|March|April|May|June|July|"
38-
r"August|September|October|November|December"
39-
r")\s+(\d{1,2}),?\s+(\d{4})\b",
42+
r"\b" + _MONTH_NAME + r"\s+(\d{1,2}),?\s+(\d{4})\b",
4043
re.IGNORECASE,
4144
)
4245
_SHORT_DATE_RE = re.compile(
43-
r"\b(\d{1,2})\s+("
44-
r"January|February|March|April|May|June|July|"
45-
r"August|September|October|November|December"
46-
r")\s+(\d{4})\b",
46+
r"\b(\d{1,2})\s+" + _MONTH_NAME + r"\s+(\d{4})\b",
4747
re.IGNORECASE,
4848
)
4949
_QUARTER_RE = re.compile(r"\bQ([1-4])\s*'?(\d{2}|\d{4})\b", re.IGNORECASE)
50-
_MONTH_YEAR_RE = re.compile(
51-
r"\b(January|February|March|April|May|June|July|"
52-
r"August|September|October|November|December)\s+(\d{4})\b",
53-
re.IGNORECASE,
54-
)
50+
_MONTH_YEAR_RE = re.compile(r"\b" + _MONTH_NAME + r"\s+(\d{4})\b", re.IGNORECASE)
5551
_YEAR_ONLY_RE = re.compile(r"\b(19\d{2}|20\d{2})\b")
5652

5753
_MONTHS = {
58-
name.lower(): index
54+
name.lower()[:3]: index
5955
for index, name in enumerate(
6056
[
6157
"January", "February", "March", "April", "May", "June",
@@ -151,17 +147,17 @@ def parse_date(text: str) -> date | None:
151147
return _safe_date(year, month, day)
152148
if (match := _LONG_DATE_RE.search(stripped)):
153149
month_name, day_str, year_str = match.group(1), match.group(2), match.group(3)
154-
return _safe_date(int(year_str), _MONTHS[month_name.lower()], int(day_str))
150+
return _safe_date(int(year_str), _MONTHS[month_name.lower()[:3]], int(day_str))
155151
if (match := _SHORT_DATE_RE.search(stripped)):
156152
day_str, month_name, year_str = match.group(1), match.group(2), match.group(3)
157-
return _safe_date(int(year_str), _MONTHS[month_name.lower()], int(day_str))
153+
return _safe_date(int(year_str), _MONTHS[month_name.lower()[:3]], int(day_str))
158154
if (match := _QUARTER_RE.search(stripped)):
159155
quarter = int(match.group(1))
160156
year_raw = match.group(2)
161157
year = 2000 + int(year_raw) if len(year_raw) == 2 else int(year_raw)
162158
return _safe_date(year, (quarter - 1) * 3 + 1, 1)
163159
if (match := _MONTH_YEAR_RE.search(stripped)):
164-
return _safe_date(int(match.group(2)), _MONTHS[match.group(1).lower()], 1)
160+
return _safe_date(int(match.group(2)), _MONTHS[match.group(1).lower()[:3]], 1)
165161
if (match := _YEAR_ONLY_RE.search(stripped)):
166162
return _safe_date(int(match.group(1)), 1, 1)
167163
return None
@@ -221,6 +217,8 @@ def guess_gpu_segment(name: str) -> str:
221217
"instinct", "mi300", "mi325", "mi350",
222218
"data center", "datacenter", "professional", "radeon pro",
223219
"rtx 6000", "rtx 5000", "rtx 4500", "rtx 4000",
220+
# Workstation lines, matching where the curated records file them.
221+
"firepro", "nvs ", "rtx pro",
224222
)
225223
if any(token in lowered for token in enterprise_tokens):
226224
return "enterprise"

0 commit comments

Comments
 (0)