Skip to content

Commit a7a858c

Browse files
Seungpyo1007Seungpyo1007
committed
feat: cover all dataset categories and actual PR diffs
Refs GetTechAPI/TechAPI#1
1 parent 3db7c60 commit a7a858c

13 files changed

Lines changed: 377 additions & 103 deletions

‎.github/workflows/techapi-pr-validation-comment.yml‎

Lines changed: 41 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@ on:
2121
type: string
2222
required: false
2323
default: ""
24+
base_ref:
25+
description: "TechAPI PR base branch (resolved from PR if omitted)"
26+
required: false
27+
base_sha:
28+
description: "TechAPI PR base SHA (resolved from PR if omitted)"
29+
required: false
2430

2531
permissions:
2632
contents: read
@@ -35,6 +41,8 @@ jobs:
3541
env:
3642
TECHAPI_COMMENT_TOKEN: ${{ secrets.TECHENGINEBOT_TOKEN || secrets.TECHAPI_TOKEN }}
3743
TECHAPI_PR_NUMBER: ${{ github.event.client_payload.pr_number || inputs.pr_number }}
44+
TECHAPI_BASE_REF: ${{ github.event.client_payload.base_ref || inputs.base_ref }}
45+
TECHAPI_BASE_SHA: ${{ github.event.client_payload.base_sha || inputs.base_sha }}
3846
TECHAPI_HEAD_SHA: ${{ github.event.client_payload.head_sha || inputs.head_sha }}
3947
TECHAPI_HEAD_REF: ${{ github.event.client_payload.head_ref || '' }}
4048
TECHAPI_PR_URL: ${{ github.event.client_payload.pr_url || inputs.pr_url }}
@@ -45,6 +53,20 @@ jobs:
4553
- name: Checkout TechEngine
4654
uses: actions/checkout@v4
4755

56+
- name: Resolve actual PR base
57+
env:
58+
GH_TOKEN: ${{ env.TECHAPI_COMMENT_TOKEN || github.token }}
59+
shell: bash
60+
run: |
61+
set -euo pipefail
62+
if [ -z "$TECHAPI_BASE_REF" ] || [ -z "$TECHAPI_BASE_SHA" ]; then
63+
gh api "repos/GetTechAPI/TechAPI/pulls/$TECHAPI_PR_NUMBER" > pr-base.json
64+
TECHAPI_BASE_REF=$(jq -r '.base.ref' pr-base.json)
65+
TECHAPI_BASE_SHA=$(jq -r '.base.sha' pr-base.json)
66+
fi
67+
echo "TECHAPI_BASE_REF=$TECHAPI_BASE_REF" >> "$GITHUB_ENV"
68+
echo "TECHAPI_BASE_SHA=$TECHAPI_BASE_SHA" >> "$GITHUB_ENV"
69+
4870
- name: Checkout TechAPI PR head
4971
uses: actions/checkout@v4
5072
with:
@@ -57,16 +79,18 @@ jobs:
5779
uses: actions/checkout@v4
5880
with:
5981
repository: GetTechAPI/TechAPI
60-
ref: develop
82+
ref: ${{ env.TECHAPI_BASE_SHA }}
6183
path: TechAPI-base
6284
fetch-depth: 0
6385

6486
- name: Pin PR base to the merge base
6587
shell: bash
6688
run: |
67-
git -C TechAPI fetch --no-tags origin develop
68-
base_sha="$(git -C TechAPI merge-base origin/develop HEAD)"
89+
git -C TechAPI fetch --no-tags origin "$TECHAPI_BASE_SHA"
90+
base_sha="$(git -C TechAPI merge-base "$TECHAPI_BASE_SHA" HEAD)"
91+
git -C TechAPI-base fetch --no-tags origin "$base_sha"
6992
git -C TechAPI-base checkout --detach "$base_sha"
93+
echo "TECHAPI_DIFF_BASE=$base_sha" >> "$GITHUB_ENV"
7094
7195
- uses: actions/setup-python@v6
7296
with:
@@ -165,15 +189,20 @@ jobs:
165189
PY
166190
)
167191
192+
python -m app.dump_check --base "$TECHAPI_BASE_SHA" --base-ref "$TECHAPI_BASE_REF" > dump-check.log 2>&1
193+
dump_status=$?
194+
cat dump-check.log >> validation.log
195+
python -m app.dump_check --base "$TECHAPI_BASE_SHA" --base-ref "$TECHAPI_BASE_REF" --scope-only > scope.txt
168196
sed -i '/_status=/d' validation.log
169197
170198
status="success"
171-
if [ "${app_status:-1}" != "0" ] || [ "${integrity_status:-1}" != "0" ]; then
199+
if [ "${app_status:-1}" != "0" ] || [ "${integrity_status:-1}" != "0" ] || [ "${dump_status:-1}" != "0" ]; then
172200
status="failure"
173201
fi
174202
175203
echo "status=$status" >> "$GITHUB_OUTPUT"
176204
echo "app_status=${app_status:-1}" >> "$GITHUB_OUTPUT"
205+
echo "dump_status=${dump_status:-1}" >> "$GITHUB_OUTPUT"
177206
echo "integrity_status=${integrity_status:-1}" >> "$GITHUB_OUTPUT"
178207
179208
- name: Build TechAPI homepage
@@ -210,16 +239,8 @@ jobs:
210239
211240
HEAD = Path("TechAPI/data")
212241
BASE = Path("TechAPI-base/data")
213-
CATEGORIES = (
214-
"brand",
215-
"soc",
216-
"smartphone",
217-
"tablet",
218-
"watch",
219-
"pda",
220-
"gpu",
221-
"cpu",
222-
)
242+
import os
243+
from app.categories import CATEGORIES
223244
MAX_WARNINGS = 20
224245
225246
def load_json(path: Path) -> dict[str, Any]:
@@ -250,7 +271,7 @@ jobs:
250271
"diff",
251272
"--name-status",
252273
"--no-renames",
253-
"origin/develop...HEAD",
274+
f"{os.environ['TECHAPI_BASE_SHA']}...HEAD",
254275
"--",
255276
"data",
256277
],
@@ -668,6 +689,8 @@ jobs:
668689
echo "<!-- techengine-pr-validation -->"
669690
echo "## TechEngine change review: ${result}"
670691
echo
692+
cat scope.txt
693+
echo
671694
echo "- PR: #${TECHAPI_PR_NUMBER}"
672695
echo "- Ref: \`${TECHAPI_HEAD_REF:-detached}\`"
673696
echo "- Commit: \`${short_sha}\`"
@@ -677,6 +700,7 @@ jobs:
677700
echo "| Check | Result |"
678701
echo "| --- | --- |"
679702
echo "| \`python -m app.validate\` | $([ "${{ steps.validate.outputs.app_status }}" = "0" ] && echo PASS || echo FAIL) |"
703+
echo "| Changed dump JSON and manifest/index counts | $([ "${{ steps.validate.outputs.dump_status }}" = "0" ] && echo PASS || echo FAIL) |"
680704
echo "| New hard integrity anomalies vs PR base | $([ "${{ steps.validate.outputs.integrity_status }}" = "0" ] && echo PASS || echo FAIL) |"
681705
if [ "${site_changed}" = "true" ]; then
682706
echo "| \`cd TechAPI/site && npm ci && npm run build\` | $([ "${site_build_status}" = "0" ] && echo PASS || echo FAIL) |"
@@ -693,6 +717,8 @@ jobs:
693717
echo "<!-- techengine-pr-validation-stats -->"
694718
echo "## TechEngine validation stats: ${result}"
695719
echo
720+
cat scope.txt
721+
echo
696722
echo "- PR: #${TECHAPI_PR_NUMBER}"
697723
echo "- Ref: \`${TECHAPI_HEAD_REF:-detached}\`"
698724
echo "- Commit: \`${short_sha}\`"

‎.github/workflows/techapi-verify-comment.yml‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,12 @@ on:
1515
head_sha:
1616
description: "TechAPI commit SHA to verify"
1717
required: true
18+
base_ref:
19+
description: "TechAPI PR base branch (resolved from PR if omitted)"
20+
required: false
21+
base_sha:
22+
description: "TechAPI PR base SHA (resolved from PR if omitted)"
23+
required: false
1824

1925
permissions:
2026
contents: read
@@ -30,6 +36,8 @@ jobs:
3036
PYTHONIOENCODING: utf-8
3137
TECHAPI_COMMENT_TOKEN: ${{ secrets.TECHENGINEBOT_TOKEN || secrets.TECHAPI_TOKEN }}
3238
TECHAPI_PR_NUMBER: ${{ github.event.client_payload.pr_number || inputs.pr_number }}
39+
TECHAPI_BASE_REF: ${{ github.event.client_payload.base_ref || inputs.base_ref }}
40+
TECHAPI_BASE_SHA: ${{ github.event.client_payload.base_sha || inputs.base_sha }}
3341
TECHAPI_HEAD_SHA: ${{ github.event.client_payload.head_sha || inputs.head_sha }}
3442
REQUESTED_BY: ${{ github.event.client_payload.requested_by || github.actor }}
3543
TECHAPI_COMMENT_ID: ${{ github.event.client_payload.comment_id }}
@@ -59,6 +67,20 @@ jobs:
5967
- name: Checkout TechEngine
6068
uses: actions/checkout@v4
6169

70+
- name: Resolve actual PR base
71+
env:
72+
GH_TOKEN: ${{ env.TECHAPI_COMMENT_TOKEN || github.token }}
73+
shell: bash
74+
run: |
75+
set -euo pipefail
76+
if [ -z "$TECHAPI_BASE_REF" ] || [ -z "$TECHAPI_BASE_SHA" ]; then
77+
gh api "repos/GetTechAPI/TechAPI/pulls/$TECHAPI_PR_NUMBER" > pr-base.json
78+
TECHAPI_BASE_REF=$(jq -r '.base.ref' pr-base.json)
79+
TECHAPI_BASE_SHA=$(jq -r '.base.sha' pr-base.json)
80+
fi
81+
echo "TECHAPI_BASE_REF=$TECHAPI_BASE_REF" >> "$GITHUB_ENV"
82+
echo "TECHAPI_BASE_SHA=$TECHAPI_BASE_SHA" >> "$GITHUB_ENV"
83+
6284
- name: Checkout TechAPI PR head
6385
uses: actions/checkout@v4
6486
with:
@@ -83,19 +105,21 @@ jobs:
83105
env:
84106
TECHAPI_DATA_DIR: ${{ github.workspace }}/TechAPI/data
85107
run: |
86-
git -C TechAPI fetch origin main --depth=1 || true
108+
git -C TechAPI fetch --no-tags origin "$TECHAPI_BASE_SHA"
87109
{
88110
echo 'report<<VERIFY_EOF'
111+
python -m app.dump_check --base "$TECHAPI_BASE_SHA" --base-ref "$TECHAPI_BASE_REF" --scope-only
112+
echo ""
89113
if [ "${MODE}" = "tier0" ]; then
90114
echo "### Changed records in this PR"
91115
echo ""
92-
python -m app.verify score --changed --no-cache --format md || echo "_app.verify unavailable on this ref._"
116+
python -m app.verify score --base "$TECHAPI_BASE_SHA" --changed --no-cache --format md || echo "_app.verify unavailable on this ref._"
93117
echo ""
94118
echo "### Full-dataset baseline"
95119
echo ""
96120
python -m app.verify score --no-cache --format md || true
97121
else
98-
python -m app.verify pr || echo "_app.verify unavailable on this ref._"
122+
python -m app.verify pr --base "$TECHAPI_BASE_SHA" || echo "_app.verify unavailable on this ref._"
99123
fi
100124
echo VERIFY_EOF
101125
} >> "$GITHUB_OUTPUT"

‎.github/workflows/verify-network.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,8 @@ jobs:
5959
- name: Install TechEngine
6060
run: pip install -e .
6161

62-
- name: Tier 0 score (writes scores cache)
63-
run: python -m app.verify score
62+
- name: Tier 0 score (recomputed; no committed cache)
63+
run: python -m app.verify score --no-cache
6464

6565
- name: Tier 1 source-URL liveness
6666
run: python -m app.verify check-urls --max ${{ github.event.inputs.max_urls || '2000' }}

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,6 @@ coverage.xml
5454

5555
# Local Wikipedia SoC backfill cache and reports (never commit)
5656
.wikipedia-soc-backfill/
57+
58+
# Recomputable verifier cache
59+
/data/_verify/state/

‎app/categories.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
"""Canonical seed categories and their public collection names."""
2+
3+
CATEGORIES: tuple[str, ...] = (
4+
"brand", "soc", "smartphone", "tablet", "watch", "pda", "gpu", "cpu",
5+
"laptop", "monitor", "software", "website",
6+
)
7+
COLLECTIONS = dict(zip(CATEGORIES, (
8+
"brands", "socs", "smartphones", "tablets", "watches", "pdas", "gpus", "cpus",
9+
"laptops", "monitors", "software", "websites",
10+
), strict=True))

‎app/dump.py‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,23 +17,12 @@
1717

1818
from fastapi.testclient import TestClient
1919

20+
from app.categories import COLLECTIONS as CATEGORY_COLLECTIONS
21+
2022
OUTPUT_DIR = Path(__file__).resolve().parent.parent / "dump"
2123

2224
# Collections that expose list + detail endpoints.
23-
COLLECTIONS = [
24-
"brands",
25-
"socs",
26-
"smartphones",
27-
"tablets",
28-
"watches",
29-
"pdas",
30-
"gpus",
31-
"cpus",
32-
"laptops",
33-
"monitors",
34-
"software",
35-
"websites",
36-
]
25+
COLLECTIONS = list(CATEGORY_COLLECTIONS.values())
3726
# Collections with a /score sub-resource (§8) and a `scored` manifest count.
3827
SCORED = {"smartphones", "cpus", "gpus", "socs"}
3928
PAGE_LIMIT = 100 # API max page size (§7.3)

‎app/dump_check.py‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
"""Lightweight validation of a changed static dump against seed record counts."""
2+
3+
from __future__ import annotations
4+
5+
import argparse
6+
import json
7+
import subprocess
8+
from pathlib import Path
9+
from typing import Any
10+
11+
from app.categories import CATEGORIES, COLLECTIONS
12+
13+
14+
def check_dump(repo: Path, base: str) -> list[str]:
15+
changed = subprocess.run(
16+
["git", "diff", "--name-only", "--diff-filter=ACMR", f"{base}...HEAD",
17+
"--", "site/public/v1/"],
18+
cwd=repo, text=True, capture_output=True, check=True,
19+
).stdout.splitlines()
20+
touched = subprocess.run(
21+
["git", "diff", "--name-only", f"{base}...HEAD", "--", "site/public/v1/"],
22+
cwd=repo, text=True, capture_output=True, check=True,
23+
).stdout.splitlines()
24+
if not touched:
25+
return []
26+
errors: list[str] = []
27+
28+
def read(path: Path) -> Any:
29+
try:
30+
return json.loads(path.read_text(encoding="utf-8-sig"))
31+
except (OSError, ValueError) as exc:
32+
errors.append(f"{path.relative_to(repo).as_posix()}: {exc}")
33+
return None
34+
35+
for rel in changed:
36+
if rel.endswith(".json"):
37+
read(repo / rel)
38+
root = repo / "site/public/v1"
39+
manifest = read(root / "index.json")
40+
collections = manifest.get("collections", {}) if isinstance(manifest, dict) else {}
41+
if not isinstance(collections, dict):
42+
errors.append("manifest collections must be an object")
43+
collections = {}
44+
for category in CATEGORIES:
45+
resource = COLLECTIONS[category]
46+
count = sum(1 for p in (repo / "data" / category).rglob("*.json")
47+
if not p.name.startswith("_"))
48+
entry = collections.get(resource)
49+
index = read(root / resource / "index.json")
50+
if not isinstance(entry, dict) or entry.get("count") != count:
51+
errors.append(f"{resource}: manifest count must equal {count}")
52+
if (not isinstance(index, dict) or index.get("count") != count
53+
or not isinstance(index.get("results"), list)
54+
or len(index["results"]) != count):
55+
errors.append(f"{resource}: index count/results must equal {count}")
56+
return errors
57+
58+
59+
def scope(repo: Path, base_ref: str, base_sha: str) -> str:
60+
count = sum(1 for category in CATEGORIES
61+
for p in (repo / "data" / category).rglob("*.json")
62+
if not p.name.startswith("_"))
63+
return f"12/12 categories ? {count:,} records ? diff base {base_ref}@{base_sha[:7]}"
64+
65+
66+
def main() -> int:
67+
parser = argparse.ArgumentParser(description=__doc__)
68+
parser.add_argument("--repo", type=Path, default=Path("TechAPI"))
69+
parser.add_argument("--base", required=True)
70+
parser.add_argument("--base-ref", default="unknown")
71+
parser.add_argument("--scope-only", action="store_true")
72+
args = parser.parse_args()
73+
print(scope(args.repo, args.base_ref, args.base))
74+
if args.scope_only:
75+
return 0
76+
errors = check_dump(args.repo, args.base)
77+
for error in errors:
78+
print(error)
79+
print(f"Dump JSON/count check: {'FAIL' if errors else 'PASS (or no dump changes)'}")
80+
return int(bool(errors))
81+
82+
83+
if __name__ == "__main__":
84+
raise SystemExit(main())

0 commit comments

Comments
 (0)