Skip to content

Commit 0c5012b

Browse files
authored
Merge pull request #132 from GetTechAPI/fix/ingest-unambiguous-values
fix(ingest): write a value only when the cell states it unambiguously
2 parents ce09b14 + 458ad31 commit 0c5012b

20 files changed

Lines changed: 1353 additions & 148 deletions

‎app/ingest/cpu_identity.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,19 @@
1010

1111

1212
def cpu_key(value: str, manufacturer: str) -> str:
13+
# '6275 "Fangio"': a nickname is not part of the model number.
14+
value = re.sub(r'\s*"[^"]+"\s*$', "", value)
15+
# EPYC "7643 (P)" is 7643P.
16+
value = re.sub(r"(\d\w*)\s*\(\s*P\s*\)", r"\1P", value)
1317
tokens = slugify(value, manufacturer=manufacturer).split("-")
1418
if tokens and tokens[0] == manufacturer:
1519
tokens.pop(0)
20+
# "Pentium Gold G7400" is the same chip as "Pentium G7400", and the curated
21+
# "Opteron X3216 APU" the same as "Opteron X3216".
22+
if tokens[:1] == ["pentium"]:
23+
tokens = [token for token in tokens if token not in {"gold", "silver"}]
24+
if len(tokens) > 1 and tokens[-1] == "apu":
25+
tokens.pop()
1626
# Wikipedia uses both "1700X PRO" and "PRO 1700X" for the same SKU.
1727
if "pro" in tokens:
1828
tokens = [token for token in tokens if token != "pro"] + ["pro"]

‎app/ingest/gpu_identity.py‎

Lines changed: 46 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
33
One board is often listed under several names: ``GeForce3 Ti500`` and
44
``GeForce3 Ti 500``, ``RTX 6000 Ada Generation`` and ``RTX 6000 Ada``, a laptop
5-
part with and without ``Mobile``, one card per memory size. A candidate that
5+
part as ``Mobile`` or ``(Notebook)``, one card per memory size, an OEM card,
6+
``Tesla C2070`` and ``C2070 GPU Computing Module``. A laptop part is never the
7+
same board as the desktop card of the same name. A candidate that
68
matches a curated board under any of those spellings is skipped, never added
79
as a second record: a missed addition is cheap, a duplicate is not.
810
"""
@@ -15,12 +17,22 @@
1517
_QUALIFIERS = frozenset({
1618
"agp", "pci", "pcie", # bus
1719
"generation", "edition", # "Ada Generation", "Workstation Edition"
18-
"mobile", "laptop", "notebook", # laptop parts; Max-Q is its own power tier
1920
"engineering", "sample",
21+
"oem",
22+
"mxm", "mcm", # embedded module packages: "E9172 MXM"
23+
"cooling", # "Vega 64 Liquid Cooling"
2024
})
25+
# Laptop markers: kept in the family key as one tier word (Max-Q stays as is).
26+
_LAPTOP_TIER = frozenset({"mobile", "laptop", "notebook"})
27+
# "H100 GPU accelerator", "C2070 GPU Computing Module": data-center board names
28+
# that the curated records file as "Tesla C2070" and "H100".
29+
_DATACENTER_SUFFIX = frozenset({"gpu", "accelerator", "computing", "module"})
30+
# Form-factor suffixes of data-center boards: Tesla K20c/K20m/K20X, H200 NVL/SXM.
31+
_FORM_FACTORS = frozenset({"c", "d", "m", "s", "t", "x", "xm", "st", "nvl", "sxm", "pcie"})
2132
_MEMORY_RE = re.compile(r"\d+(?:gb|mb)")
22-
# Turing and later workstation boards dropped the Quadro brand ("Quadro T500" is "T500").
23-
_QUADRO_RENAMED_RE = re.compile(r"(?:t|a|rtx)\d*")
33+
# Boards that dropped the Quadro brand: Turing and later ("Quadro T500" is "T500"),
34+
# and the NVS line ("Quadro NVS 50" is "NVS 50").
35+
_QUADRO_RENAMED_RE = re.compile(r"(?:t|a|rtx)\d*|nvs")
2436

2537

2638
def _tokens(slug: str) -> list[str]:
@@ -49,23 +61,51 @@ def gpu_key(slug: str) -> str:
4961

5062

5163
def gpu_family_key(slug: str) -> str:
52-
"""Identity with variant words dropped: bus, memory size, revision, laptop
53-
markers, "Generation"/"(Workstation) Edition" and the retired Quadro brand."""
64+
"""Identity with variant words dropped: bus, memory size, revision, OEM,
65+
"Generation"/"(Workstation) Edition", the retired Quadro, Tesla and Radeon
66+
Instinct brands and data-center suffixes. The laptop tier stays (``-notebook``
67+
is ``-mobile``) except on a Tesla, where it names the MXM module."""
5468
tokens = _tokens(slug)
5569
if len(tokens) > 1 and tokens[0] == "quadro" and _QUADRO_RENAMED_RE.fullmatch(tokens[1]):
5670
tokens = tokens[1:]
71+
# "Radeon Instinct MI100" became "Instinct MI100".
72+
if tokens[:2] == ["radeon", "instinct"]:
73+
tokens = tokens[1:]
74+
tesla = len(tokens) > 1 and tokens[0] == "tesla"
75+
if tesla:
76+
tokens = tokens[1:]
77+
while len(tokens) > 1 and tokens[-1] in _DATACENTER_SUFFIX:
78+
tokens.pop()
5779
kept = [
5880
token
5981
for i, token in enumerate(tokens)
6082
if token not in _QUALIFIERS
83+
and token not in _LAPTOP_TIER
6184
and not _MEMORY_RE.fullmatch(token)
6285
and not re.fullmatch(r"rev\d*", token)
6386
# "RTX PRO 6000 Blackwell Workstation (Edition)" is the plain board.
6487
and not (token == "workstation" and tokens[i + 1 : i + 2] in ([], ["edition"]))
6588
]
89+
# The tier goes last: "RTX 500 Mobile Ada Generation" is "RTX 500 Ada Mobile".
90+
# A "Tesla M6 Mobile" is the MXM module of the M6, not a laptop chip.
91+
if not tesla and any(token in _LAPTOP_TIER for token in tokens):
92+
kept.append("mobile")
6693
return "".join(kept)
6794

6895

96+
def datacenter_family(slug: str) -> str | None:
97+
"""Family key of a data-center board name ("K20 GPU accelerator"), else ``None``."""
98+
tokens = _tokens(slug)
99+
if tokens[:1] == ["tesla"] or (tokens and tokens[-1] in _DATACENTER_SUFFIX):
100+
return gpu_family_key(slug)
101+
return None
102+
103+
104+
def form_factor_of(key: str, family: str) -> bool:
105+
"""Whether family key ``key`` is a form factor of ``family``: ``k20c`` of ``k20``."""
106+
return key.startswith(family) and key[len(family) :] in _FORM_FACTORS
107+
108+
69109
def spec_twins(candidate: dict[str, object], existing: dict[str, object]) -> bool:
70110
"""Two names where one extends the other (``HD 2350`` / ``HD 2350 PRO``) and
71111
the boards share core clock and TDP: likely the same board, so not a new one."""

‎app/ingest/normalize.py‎

Lines changed: 59 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@
2222
_TDP_RE = re.compile(
2323
r"(?<![\d.])(\d{1,4}(?:\.\d+)?)(?:\s*/\s*\d{1,4}(?:\.\d+)?)?\s*W\b", re.IGNORECASE
2424
)
25+
# "12–25 W": a cTDP range, which names no default TDP.
26+
_TDP_RANGE_RE = re.compile(
27+
r"(?<![\d.])\d{1,4}(?:\.\d+)?\s*(?:W\s*)?(?:–|—|−|-|to)\s*\d{1,4}(?:\.\d+)?\s*W\b",
28+
re.IGNORECASE,
29+
)
2530
_FREQ_RANGE_RE = re.compile(
2631
r"(\d+(?:\.\d+)?)\s*(?:–|—|-|to)\s*(\d+(?:\.\d+)?)\s*(GHz|MHz)\b", re.IGNORECASE
2732
)
@@ -47,7 +52,9 @@
4752
re.IGNORECASE,
4853
)
4954
_QUARTER_RE = re.compile(r"\bQ([1-4])\s*'?(\d{2}|\d{4})\b", re.IGNORECASE)
50-
_MONTH_YEAR_RE = re.compile(r"\b" + _MONTH_NAME + r"\s+(\d{4})\b", re.IGNORECASE)
55+
# "1H 2025" and "H2 2024".
56+
_HALF_YEAR_RE = re.compile(r"\b(?:([12])H|H([12]))\s*(\d{4})\b", re.IGNORECASE)
57+
_MONTH_YEAR_RE = re.compile(r"\b" + _MONTH_NAME + r",?\s+(\d{4})\b", re.IGNORECASE)
5158
_YEAR_ONLY_RE = re.compile(r"\b(19\d{2}|20\d{2})\b")
5259

5360
_MONTHS = {
@@ -74,10 +81,15 @@ def parse_frequency_ghz(text: str) -> float | None:
7481

7582

7683
def parse_tdp_w(text: str) -> int | None:
77-
"""``"65 W"`` → ``65``; ``"65/95 W"`` → ``65``; ``"9.5 W"`` → ``10`` (half-up)."""
84+
"""``"65 W"`` → ``65``; ``"65/95 W"`` → ``65``; ``"9.5 W"`` → ``10`` (half-up).
85+
86+
``"45 W (35–54 W)"`` → ``45``: the default precedes its cTDP range. A cell
87+
that is only a range (``"12–25 W"``) states no default, so it gives ``None``;
88+
neither end of the range is the part's TDP.
89+
"""
7890
if not text:
7991
return None
80-
match = _TDP_RE.search(text)
92+
match = _TDP_RE.search(_TDP_RANGE_RE.sub(" ", text))
8193
return math.floor(float(match.group(1)) + 0.5) if match else None
8294

8395

@@ -142,12 +154,29 @@ def parse_date(text: str) -> date | None:
142154
return period[0] if period else None
143155

144156

157+
# A quarter is the longest period whose first day still stands for the date.
158+
_LONGEST_DATED_PERIOD_DAYS = 92
159+
160+
161+
def release_date_of(period: tuple[date, date] | None) -> date | None:
162+
"""Release date a collector may write for ``period``, or ``None``.
163+
164+
A day, a month or a quarter gives its first day, as the curated records
165+
do. A half-year or a bare year gives ``None``: January 1 would be a guess.
166+
"""
167+
if period is None:
168+
return None
169+
first, last = period
170+
return first if (last - first).days < _LONGEST_DATED_PERIOD_DAYS else None
171+
172+
145173
def parse_date_period(text: str) -> tuple[date, date] | None:
146174
"""``(first, last)`` day of the period a date cell names.
147175
148176
A full date is a one-day period. ``"Q4 2026"`` is Oct 1 – Dec 31, ``"Dec 2024"``
149-
the whole month and a bare year the whole year, so a caller can tell whether
150-
the period had ended (the part had shipped) by a given day.
177+
the whole month, ``"1H 2025"`` Jan 1 – Jun 30 and a bare year the whole year,
178+
so a caller can tell whether the period had ended (the part had shipped) by a
179+
given day.
151180
"""
152181
if not text:
153182
return None
@@ -170,6 +199,9 @@ def parse_date_period(text: str) -> tuple[date, date] | None:
170199
year_raw = match.group(2)
171200
year = 2000 + int(year_raw) if len(year_raw) == 2 else int(year_raw)
172201
return _months(year, (quarter - 1) * 3 + 1, 3)
202+
if (match := _HALF_YEAR_RE.search(stripped)):
203+
half = int(match.group(1) or match.group(2))
204+
return _months(int(match.group(3)), 1 if half == 1 else 7, 6)
173205
if (match := _MONTH_YEAR_RE.search(stripped)):
174206
return _months(int(match.group(2)), _MONTHS[match.group(1).lower()[:3]], 1)
175207
if (match := _YEAR_ONLY_RE.search(stripped)):
@@ -220,6 +252,15 @@ def parse_memory_gb(text: str) -> float | None:
220252
return value if match.group(2).lower() == "gb" else round(value / 1024, 3)
221253

222254

255+
def mib_to_gb(mib: float) -> float | None:
256+
"""MiB → GB as the dataset stores it (binary): ``32`` → ``0.03125``.
257+
258+
Unrounded, so power-of-two sizes stay exact. 1 MiB and less gives ``None``:
259+
0.0009765625 is under the validator's 0.001 GB floor.
260+
"""
261+
return mib / 1024 if mib > 1 else None
262+
263+
223264
def parse_memory_bus_bit(text: str) -> int | None:
224265
"""``"384-bit"`` → ``384``; ``"128 bit"`` → ``128``."""
225266
if not text:
@@ -326,6 +367,17 @@ def guess_os(text: str, *, brand: str = "") -> str | None:
326367
return None
327368

328369

370+
def normalize_socket(text: str) -> str | None:
371+
"""Socket in the model doc's form: ``"LGA 1151"`` → ``"LGA1151"``,
372+
``"Socket AM4"`` → ``"AM4"``, ``"F"`` → ``"F (1207)"``."""
373+
value = " ".join(text.split())
374+
value = re.sub(r"^Socket\s+", "", value, flags=re.IGNORECASE)
375+
value = re.sub(r"^(FC)?LGA\s+(?=\d)", r"\1LGA", value, flags=re.IGNORECASE)
376+
if value == "F":
377+
value = "F (1207)"
378+
return value or None
379+
380+
329381
def guess_cpu_segment(name: str) -> str:
330382
"""Heuristic CPU segment classifier.
331383
@@ -353,6 +405,7 @@ def guess_cpu_segment(name: str) -> str:
353405
suffix = match.group(1) if match else ""
354406
if any(letter in suffix for letter in ("h", "u", "y", "p", "m")) and "k" not in suffix:
355407
return "laptop"
356-
if suffix in {"x", "xe"}:
408+
# Core X/XE parts are HEDT; Ryzen X parts are ordinary desktop chips.
409+
if suffix in {"x", "xe"} and re.search(r"\bcore\b", lowered):
357410
return "hedt"
358411
return "desktop"

0 commit comments

Comments
 (0)