feat: analyze all dataset categories and actual PR diffs

.github/workflows/techapi-pr-validation-comment.yml

@@ -21,6 +21,12 @@ on:
         type: string
         required: false
         default: ""
+      base_ref:
+        description: "TechAPI PR base branch (resolved from PR if omitted)"
+        required: false
+      base_sha:
+        description: "TechAPI PR base SHA (resolved from PR if omitted)"
+        required: false
 
 permissions:
   contents: read
@@ -35,6 +41,8 @@ jobs:
     env:
       TECHAPI_COMMENT_TOKEN: ${{ secrets.TECHENGINEBOT_TOKEN || secrets.TECHAPI_TOKEN }}
       TECHAPI_PR_NUMBER: ${{ github.event.client_payload.pr_number || inputs.pr_number }}
+      TECHAPI_BASE_REF: ${{ github.event.client_payload.base_ref || inputs.base_ref }}
+      TECHAPI_BASE_SHA: ${{ github.event.client_payload.base_sha || inputs.base_sha }}
       TECHAPI_HEAD_SHA: ${{ github.event.client_payload.head_sha || inputs.head_sha }}
       TECHAPI_HEAD_REF: ${{ github.event.client_payload.head_ref || '' }}
       TECHAPI_PR_URL: ${{ github.event.client_payload.pr_url || inputs.pr_url }}
@@ -45,6 +53,20 @@ jobs:
       - name: Checkout TechEngine
         uses: actions/checkout@v4
 
+      - name: Resolve actual PR base
+        env:
+          GH_TOKEN: ${{ env.TECHAPI_COMMENT_TOKEN || github.token }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [ -z "$TECHAPI_BASE_REF" ] || [ -z "$TECHAPI_BASE_SHA" ]; then
+            gh api "repos/GetTechAPI/TechAPI/pulls/$TECHAPI_PR_NUMBER" > pr-base.json
+            TECHAPI_BASE_REF=$(jq -r '.base.ref' pr-base.json)
+            TECHAPI_BASE_SHA=$(jq -r '.base.sha' pr-base.json)
+          fi
+          echo "TECHAPI_BASE_REF=$TECHAPI_BASE_REF" >> "$GITHUB_ENV"
+          echo "TECHAPI_BASE_SHA=$TECHAPI_BASE_SHA" >> "$GITHUB_ENV"
+
       - name: Checkout TechAPI PR head
         uses: actions/checkout@v4
         with:
@@ -57,16 +79,18 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: GetTechAPI/TechAPI
-          ref: develop
+          ref: ${{ env.TECHAPI_BASE_SHA }}
           path: TechAPI-base
           fetch-depth: 0
 
       - name: Pin PR base to the merge base
         shell: bash
         run: |
-          git -C TechAPI fetch --no-tags origin develop
-          base_sha="$(git -C TechAPI merge-base origin/develop HEAD)"
+          git -C TechAPI fetch --no-tags origin "$TECHAPI_BASE_SHA"
+          base_sha="$(git -C TechAPI merge-base "$TECHAPI_BASE_SHA" HEAD)"
+          git -C TechAPI-base fetch --no-tags origin "$base_sha"
           git -C TechAPI-base checkout --detach "$base_sha"
+          echo "TECHAPI_DIFF_BASE=$base_sha" >> "$GITHUB_ENV"
 
       - uses: actions/setup-python@v6
         with:
@@ -165,15 +189,20 @@ jobs:
           PY
           )
 
+          python -m app.dump_check --base "$TECHAPI_BASE_SHA" --base-ref "$TECHAPI_BASE_REF" > dump-check.log 2>&1
+          dump_status=$?
+          cat dump-check.log >> validation.log
+          python -m app.dump_check --base "$TECHAPI_BASE_SHA" --base-ref "$TECHAPI_BASE_REF" --scope-only > scope.txt
           sed -i '/_status=/d' validation.log
 
           status="success"
-          if [ "${app_status:-1}" != "0" ] || [ "${integrity_status:-1}" != "0" ]; then
+          if [ "${app_status:-1}" != "0" ] || [ "${integrity_status:-1}" != "0" ] || [ "${dump_status:-1}" != "0" ]; then
             status="failure"
           fi
 
           echo "status=$status" >> "$GITHUB_OUTPUT"
           echo "app_status=${app_status:-1}" >> "$GITHUB_OUTPUT"
+          echo "dump_status=${dump_status:-1}" >> "$GITHUB_OUTPUT"
           echo "integrity_status=${integrity_status:-1}" >> "$GITHUB_OUTPUT"
 
       - name: Build TechAPI homepage
@@ -210,16 +239,8 @@ jobs:
 
           HEAD = Path("TechAPI/data")
           BASE = Path("TechAPI-base/data")
-          CATEGORIES = (
-              "brand",
-              "soc",
-              "smartphone",
-              "tablet",
-              "watch",
-              "pda",
-              "gpu",
-              "cpu",
-          )
+          import os
+          from app.categories import CATEGORIES
           MAX_WARNINGS = 20
 
           def load_json(path: Path) -> dict[str, Any]:
@@ -250,7 +271,7 @@ jobs:
                       "diff",
                       "--name-status",
                       "--no-renames",
-                      "origin/develop...HEAD",
+                      f"{os.environ['TECHAPI_BASE_SHA']}...HEAD",
                       "--",
                       "data",
                   ],
@@ -668,6 +689,8 @@ jobs:
             echo "<!-- techengine-pr-validation -->"
             echo "## TechEngine change review: ${result}"
             echo
+            cat scope.txt
+            echo
             echo "- PR: #${TECHAPI_PR_NUMBER}"
             echo "- Ref: \`${TECHAPI_HEAD_REF:-detached}\`"
             echo "- Commit: \`${short_sha}\`"
@@ -677,6 +700,7 @@ jobs:
             echo "| Check | Result |"
             echo "| --- | --- |"
             echo "| \`python -m app.validate\` | $([ "${{ steps.validate.outputs.app_status }}" = "0" ] && echo PASS || echo FAIL) |"
+            echo "| Changed dump JSON and manifest/index counts | $([ "${{ steps.validate.outputs.dump_status }}" = "0" ] && echo PASS || echo FAIL) |"
             echo "| New hard integrity anomalies vs PR base | $([ "${{ steps.validate.outputs.integrity_status }}" = "0" ] && echo PASS || echo FAIL) |"
             if [ "${site_changed}" = "true" ]; then
               echo "| \`cd TechAPI/site && npm ci && npm run build\` | $([ "${site_build_status}" = "0" ] && echo PASS || echo FAIL) |"
@@ -693,6 +717,8 @@ jobs:
             echo "<!-- techengine-pr-validation-stats -->"
             echo "## TechEngine validation stats: ${result}"
             echo
+            cat scope.txt
+            echo
             echo "- PR: #${TECHAPI_PR_NUMBER}"
             echo "- Ref: \`${TECHAPI_HEAD_REF:-detached}\`"
             echo "- Commit: \`${short_sha}\`"

.github/workflows/techapi-verify-comment.yml

@@ -15,6 +15,12 @@ on:
       head_sha:
         description: "TechAPI commit SHA to verify"
         required: true
+      base_ref:
+        description: "TechAPI PR base branch (resolved from PR if omitted)"
+        required: false
+      base_sha:
+        description: "TechAPI PR base SHA (resolved from PR if omitted)"
+        required: false
 
 permissions:
   contents: read
@@ -30,6 +36,8 @@ jobs:
       PYTHONIOENCODING: utf-8
       TECHAPI_COMMENT_TOKEN: ${{ secrets.TECHENGINEBOT_TOKEN || secrets.TECHAPI_TOKEN }}
       TECHAPI_PR_NUMBER: ${{ github.event.client_payload.pr_number || inputs.pr_number }}
+      TECHAPI_BASE_REF: ${{ github.event.client_payload.base_ref || inputs.base_ref }}
+      TECHAPI_BASE_SHA: ${{ github.event.client_payload.base_sha || inputs.base_sha }}
       TECHAPI_HEAD_SHA: ${{ github.event.client_payload.head_sha || inputs.head_sha }}
       REQUESTED_BY: ${{ github.event.client_payload.requested_by || github.actor }}
       TECHAPI_COMMENT_ID: ${{ github.event.client_payload.comment_id }}
@@ -59,6 +67,20 @@ jobs:
       - name: Checkout TechEngine
         uses: actions/checkout@v4
 
+      - name: Resolve actual PR base
+        env:
+          GH_TOKEN: ${{ env.TECHAPI_COMMENT_TOKEN || github.token }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [ -z "$TECHAPI_BASE_REF" ] || [ -z "$TECHAPI_BASE_SHA" ]; then
+            gh api "repos/GetTechAPI/TechAPI/pulls/$TECHAPI_PR_NUMBER" > pr-base.json
+            TECHAPI_BASE_REF=$(jq -r '.base.ref' pr-base.json)
+            TECHAPI_BASE_SHA=$(jq -r '.base.sha' pr-base.json)
+          fi
+          echo "TECHAPI_BASE_REF=$TECHAPI_BASE_REF" >> "$GITHUB_ENV"
+          echo "TECHAPI_BASE_SHA=$TECHAPI_BASE_SHA" >> "$GITHUB_ENV"
+
       - name: Checkout TechAPI PR head
         uses: actions/checkout@v4
         with:
@@ -83,19 +105,21 @@ jobs:
         env:
           TECHAPI_DATA_DIR: ${{ github.workspace }}/TechAPI/data
         run: |
-          git -C TechAPI fetch origin main --depth=1 || true
+          git -C TechAPI fetch --no-tags origin "$TECHAPI_BASE_SHA"
           {
             echo 'report<<VERIFY_EOF'
+            python -m app.dump_check --base "$TECHAPI_BASE_SHA" --base-ref "$TECHAPI_BASE_REF" --scope-only
+            echo ""
             if [ "${MODE}" = "tier0" ]; then
               echo "### Changed records in this PR"
               echo ""
-              python -m app.verify score --changed --no-cache --format md || echo "_app.verify unavailable on this ref._"
+              python -m app.verify score --base "$TECHAPI_BASE_SHA" --changed --no-cache --format md || echo "_app.verify unavailable on this ref._"
               echo ""
               echo "### Full-dataset baseline"
               echo ""
               python -m app.verify score --no-cache --format md || true
             else
-              python -m app.verify pr || echo "_app.verify unavailable on this ref._"
+              python -m app.verify pr --base "$TECHAPI_BASE_SHA" || echo "_app.verify unavailable on this ref._"
             fi
             echo VERIFY_EOF
           } >> "$GITHUB_OUTPUT"

.github/workflows/verify-network.yml

@@ -59,8 +59,8 @@ jobs:
       - name: Install TechEngine
         run: pip install -e .
 
-      - name: Tier 0 score (writes scores cache)
-        run: python -m app.verify score
+      - name: Tier 0 score (recomputed; no committed cache)
+        run: python -m app.verify score --no-cache
 
       - name: Tier 1 source-URL liveness
         run: python -m app.verify check-urls --max ${{ github.event.inputs.max_urls || '2000' }}

.gitignore

@@ -54,3 +54,6 @@ coverage.xml
 
 # Local Wikipedia SoC backfill cache and reports (never commit)
 .wikipedia-soc-backfill/
+
+# Recomputable verifier cache
+/data/_verify/state/

app/categories.py

@@ -0,0 +1,10 @@
+"""Canonical seed categories and their public collection names."""
+
+CATEGORIES: tuple[str, ...] = (
+    "brand", "soc", "smartphone", "tablet", "watch", "pda", "gpu", "cpu",
+    "laptop", "monitor", "software", "website",
+)
+COLLECTIONS = dict(zip(CATEGORIES, (
+    "brands", "socs", "smartphones", "tablets", "watches", "pdas", "gpus", "cpus",
+    "laptops", "monitors", "software", "websites",
+), strict=True))

app/dump.py

@@ -17,23 +17,12 @@
 
 from fastapi.testclient import TestClient
 
+from app.categories import COLLECTIONS as CATEGORY_COLLECTIONS
+
 OUTPUT_DIR = Path(__file__).resolve().parent.parent / "dump"
 
 # Collections that expose list + detail endpoints.
-COLLECTIONS = [
-    "brands",
-    "socs",
-    "smartphones",
-    "tablets",
-    "watches",
-    "pdas",
-    "gpus",
-    "cpus",
-    "laptops",
-    "monitors",
-    "software",
-    "websites",
-]
+COLLECTIONS = list(CATEGORY_COLLECTIONS.values())
 # Collections with a /score sub-resource (§8) and a `scored` manifest count.
 SCORED = {"smartphones", "cpus", "gpus", "socs"}
 PAGE_LIMIT = 100  # API max page size (§7.3)

app/dump_check.py

@@ -0,0 +1,84 @@
+"""Lightweight validation of a changed static dump against seed record counts."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+from pathlib import Path
+from typing import Any
+
+from app.categories import CATEGORIES, COLLECTIONS
+
+
+def check_dump(repo: Path, base: str) -> list[str]:
+    changed = subprocess.run(
+        ["git", "diff", "--name-only", "--diff-filter=ACMR", f"{base}...HEAD",
+         "--", "site/public/v1/"],
+        cwd=repo, text=True, capture_output=True, check=True,
+    ).stdout.splitlines()
+    touched = subprocess.run(
+        ["git", "diff", "--name-only", f"{base}...HEAD", "--", "site/public/v1/"],
+        cwd=repo, text=True, capture_output=True, check=True,
+    ).stdout.splitlines()
+    if not touched:
+        return []
+    errors: list[str] = []
+
+    def read(path: Path) -> Any:
+        try:
+            return json.loads(path.read_text(encoding="utf-8-sig"))
+        except (OSError, ValueError) as exc:
+            errors.append(f"{path.relative_to(repo).as_posix()}: {exc}")
+            return None
+
+    for rel in changed:
+        if rel.endswith(".json"):
+            read(repo / rel)
+    root = repo / "site/public/v1"
+    manifest = read(root / "index.json")
+    collections = manifest.get("collections", {}) if isinstance(manifest, dict) else {}
+    if not isinstance(collections, dict):
+        errors.append("manifest collections must be an object")
+        collections = {}
+    for category in CATEGORIES:
+        resource = COLLECTIONS[category]
+        count = sum(1 for p in (repo / "data" / category).rglob("*.json")
+                    if not p.name.startswith("_"))
+        entry = collections.get(resource)
+        index = read(root / resource / "index.json")
+        if not isinstance(entry, dict) or entry.get("count") != count:
+            errors.append(f"{resource}: manifest count must equal {count}")
+        if (not isinstance(index, dict) or index.get("count") != count
+                or not isinstance(index.get("results"), list)
+                or len(index["results"]) != count):
+            errors.append(f"{resource}: index count/results must equal {count}")
+    return errors
+
+
+def scope(repo: Path, base_ref: str, base_sha: str) -> str:
+    count = sum(1 for category in CATEGORIES
+                for p in (repo / "data" / category).rglob("*.json")
+                if not p.name.startswith("_"))
+    return f"12/12 categories ? {count:,} records ? diff base {base_ref}@{base_sha[:7]}"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--repo", type=Path, default=Path("TechAPI"))
+    parser.add_argument("--base", required=True)
+    parser.add_argument("--base-ref", default="unknown")
+    parser.add_argument("--scope-only", action="store_true")
+    args = parser.parse_args()
+    print(scope(args.repo, args.base_ref, args.base))
+    if args.scope_only:
+        return 0
+    errors = check_dump(args.repo, args.base)
+    for error in errors:
+        print(error)
+    print(f"Dump JSON/count check: {'FAIL' if errors else 'PASS (or no dump changes)'}")
+    return int(bool(errors))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())