From 64ddb751ab2cd6dfb285574f8a8151aaebba7cde Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Thu, 29 Oct 2026 12:14:20 +0900 Subject: [PATCH] ci: validate only changed records on pull requests Every PR validated all 962,384 records, a run that took 117 minutes on Windows. PRs now check just the records they add or modify; pushes to develop and main still run the full check. The validator also enforces the bucket rule the README already states (data/game//.json). All existing records pass it, and it is what keeps a scoped run from missing a duplicate slug: a duplicate would have to land on the same path. --- .github/workflows/validate-data.yml | 22 +++++++++++++++++-- README.md | 7 +++++- app/validate.py | 34 +++++++++++++++++++++++++---- tests/test_validate.py | 15 +++++++++++++ 4 files changed, 71 insertions(+), 7 deletions(-) diff --git a/.github/workflows/validate-data.yml b/.github/workflows/validate-data.yml index 8c3601b6190..cc497cfedde 100644 --- a/.github/workflows/validate-data.yml +++ b/.github/workflows/validate-data.yml @@ -1,20 +1,38 @@ name: validate-data +# GITHUB_TOKEN only: no personal access tokens, no bot account, no writes. on: pull_request: push: branches: [develop, main] +permissions: + contents: read + +concurrency: + group: validate-data-${{ github.ref }} + cancel-in-progress: true + jobs: validate: runs-on: ubuntu-latest - timeout-minutes: 90 # 962k files; measure a real run before trimming + # PRs check only the records they change; a full run over 962k files is + # for pushes to develop/main. Measure a real full run before trimming. + timeout-minutes: ${{ github.event_name == 'pull_request' && 15 || 90 }} steps: - uses: actions/checkout@v4 + with: + # A PR checks out its merge commit; depth 2 brings the base parent + # (HEAD^1), which is all the diff needs. + fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 1 }} - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Validate game catalog + - name: Validate changed records (PR) + if: github.event_name == 'pull_request' + run: python -m app.validate --changed-since HEAD^1 + - name: Validate game catalog (full) + if: github.event_name != 'pull_request' run: python -m app.validate - name: Tests run: | diff --git a/README.md b/README.md index 8149eb84aa3..913b03a6c65 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,15 @@ follow TechEngine's game model: `release_date`, `rating` (0-5), `rating_count`, ## Self-check ```bash -python -m app.validate +python -m app.validate # every record (~2h on Windows) +python -m app.validate --changed-since develop # only records your branch touched python -m pytest -q ``` +CI runs the scoped check on pull requests and the full check on pushes to +`develop` and `main`. Workflows use `GITHUB_TOKEN` only — no personal access +tokens, bot accounts, schedules or writes back to any repository. + ## Site `python site/build.py` writes two files that the TechAPI homepage reads so the diff --git a/app/validate.py b/app/validate.py index 7692772e15e..83f7f6ab5c5 100644 --- a/app/validate.py +++ b/app/validate.py @@ -5,12 +5,15 @@ a list first. ~1M records will not fit in a runner's memory otherwise. Run with: python -m app.validate + or: python -m app.validate --changed-since (PRs: changed records only) """ from __future__ import annotations +import argparse import json import re +import subprocess import sys from pathlib import Path from typing import Any @@ -51,12 +54,17 @@ def _check(rel: str, rec: dict[str, Any], errors: list[str]) -> None: errors.append(f"{rel}: {field} {value!r} outside {low}-{high}") -def validate(data_dir: Path = DATA) -> list[str]: +def validate(data_dir: Path = DATA, paths: list[Path] | None = None) -> list[str]: + """Check every record, or only `paths` when given. + + A scoped run still catches duplicate slugs: the bucket check pins each slug + to exactly one path, so a duplicate would have to overwrite that file. + """ errors: list[str] = [] seen: dict[str, str] = {} count = 0 - for path in sorted(data_dir.rglob("*.json")): + for path in sorted(data_dir.rglob("*.json")) if paths is None else sorted(paths): rel = path.relative_to(data_dir.parent).as_posix() count += 1 try: @@ -78,18 +86,36 @@ def validate(data_dir: Path = DATA) -> list[str]: seen[slug] = rel if path.stem != slug: errors.append(f"{rel}: filename does not match slug '{slug}'") + if path.parent.name != slug[:2]: + errors.append(f"{rel}: bucket must be '{slug[:2]}' (first two slug characters)") if len(errors) >= MAX_ERRORS: errors.append(f"... stopped after {MAX_ERRORS} errors") break - if count == 0: + if count == 0 and paths is None: errors.append(f"no game records found under {data_dir}") return errors +def changed_records(base: str) -> list[Path]: + """Records added or modified since `base` (deletions need no checking).""" + out = subprocess.run( + ["git", "diff", "--name-only", "--diff-filter=AMR", f"{base}...HEAD", "--", "data/game"], + cwd=ROOT, check=True, capture_output=True, text=True, + ).stdout + return [ROOT / line for line in out.splitlines() if line.endswith(".json")] + + def main() -> int: - errors = validate() + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--changed-since", metavar="SHA", help="validate only records changed since SHA") + args = parser.parse_args() + + paths = changed_records(args.changed_since) if args.changed_since else None + if paths is not None: + print(f"validating {len(paths)} changed record(s)") + errors = validate(paths=paths) for error in errors: print(error) if errors: diff --git a/tests/test_validate.py b/tests/test_validate.py index e4c70d3390b..96be522485a 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -66,3 +66,18 @@ def test_duplicate_slug(tmp_path): other.mkdir(parents=True, exist_ok=True) (other / "the-witcher-3-wild-hunt.json").write_text(json.dumps(GOOD), encoding="utf-8") assert any("duplicate slug" in e for e in validate(data)) + + +def test_wrong_bucket(tmp_path): + data = tmp_path / "game" / "zz" + data.mkdir(parents=True) + (data / "the-witcher-3-wild-hunt.json").write_text(json.dumps(GOOD), encoding="utf-8") + assert any("bucket" in e for e in validate(tmp_path / "game")) + + +def test_scoped_run_checks_only_given_paths(tmp_path): + data = _write(tmp_path, GOOD) + _write(tmp_path, {**GOOD, "slug": "the-bad-one", "rating": 9}) + good = data / "th" / "the-witcher-3-wild-hunt.json" + assert validate(data, paths=[good]) == [] + assert validate(data, paths=[]) == []