From 2603441d005d23a6b3857b8710c47450f3e5e208 Mon Sep 17 00:00:00 2001 From: Seungpyo1007 Date: Wed, 23 Sep 2026 16:46:36 +0900 Subject: [PATCH] fix(ci): retry refresh PR creation and compare validation to base --- .../techapi-pr-validation-comment.yml | 57 +++++++++++++------ .github/workflows/weekly-refresh.yml | 23 ++++++-- integrity_check.py | 15 ++++- 3 files changed, 69 insertions(+), 26 deletions(-) diff --git a/.github/workflows/techapi-pr-validation-comment.yml b/.github/workflows/techapi-pr-validation-comment.yml index 9f92401..d7cffed 100644 --- a/.github/workflows/techapi-pr-validation-comment.yml +++ b/.github/workflows/techapi-pr-validation-comment.yml @@ -51,13 +51,22 @@ jobs: repository: GetTechAPI/TechAPI ref: ${{ env.TECHAPI_HEAD_SHA }} path: TechAPI + fetch-depth: 0 - - name: Checkout TechAPI main + - name: Checkout TechAPI PR base uses: actions/checkout@v4 with: repository: GetTechAPI/TechAPI - ref: main - path: TechAPI-main + ref: develop + path: TechAPI-base + fetch-depth: 0 + + - name: Pin PR base to the merge base + shell: bash + run: | + git -C TechAPI fetch --no-tags origin develop + base_sha="$(git -C TechAPI merge-base origin/develop HEAD)" + git -C TechAPI-base checkout --detach "$base_sha" - uses: actions/setup-python@v6 with: @@ -100,7 +109,7 @@ jobs: return files head = rel_site_files(Path("TechAPI")) - base = rel_site_files(Path("TechAPI-main")) + base = rel_site_files(Path("TechAPI-base")) added = sorted(set(head) - set(base)) deleted = sorted(set(base) - set(head)) modified = sorted(key for key in set(head) & set(base) if digest(head[key]) != digest(base[key])) @@ -135,11 +144,26 @@ jobs: { echo - echo "## integrity_check.py --strict" - python integrity_check.py TechAPI/data --strict - echo "integrity_status=$?" + echo "## integrity_check.py (PR head compared with PR base)" + python integrity_check.py TechAPI/data --hard-report head-hard.json + echo "head_integrity_scan_status=$?" + python integrity_check.py TechAPI-base/data --hard-report base-hard.json > baseline-integrity.log + echo "base_integrity_scan_status=$?" } >> validation.log 2>&1 - integrity_status=$(grep "integrity_status=" validation.log | tail -n 1 | cut -d= -f2) + integrity_status=$(python - <<'PY' + import json + from pathlib import Path + + head = set(json.loads(Path("head-hard.json").read_text(encoding="utf-8"))) + base = set(json.loads(Path("base-hard.json").read_text(encoding="utf-8"))) + introduced = sorted(head - base) + with Path("validation.log").open("a", encoding="utf-8") as log: + log.write(f"\nBaseline hard anomalies: {len(base)}; PR head: {len(head)}; introduced: {len(introduced)}\n") + for anomaly in introduced: + log.write(f"NEW HARD ANOMALY: {anomaly}\n") + print("1" if introduced else "0") + PY + ) sed -i '/_status=/d' validation.log @@ -173,7 +197,6 @@ jobs: - name: Build data quality summary shell: bash run: | - git -C TechAPI fetch --no-tags --depth=1 origin main python - <<'PY' from __future__ import annotations @@ -186,7 +209,7 @@ jobs: from typing import Any HEAD = Path("TechAPI/data") - BASE = Path("TechAPI-main/data") + BASE = Path("TechAPI-base/data") CATEGORIES = ( "brand", "soc", @@ -227,8 +250,7 @@ jobs: "diff", "--name-status", "--no-renames", - "FETCH_HEAD", - "HEAD", + "origin/develop...HEAD", "--", "data", ], @@ -469,7 +491,7 @@ jobs: site_lines.append("## Changed site") site_lines.append("") head_site = rel_site_files(Path("TechAPI")) - base_site = rel_site_files(Path("TechAPI-main")) + base_site = rel_site_files(Path("TechAPI-base")) site_added = sorted(set(head_site) - set(base_site)) site_deleted = sorted(set(base_site) - set(head_site)) site_modified = sorted( @@ -573,12 +595,13 @@ jobs: continue if current_section and line.startswith(" "): section_counts[current_section] += 1 - if any(token in line for token in ("DUP ", "slug!=file", " > ")): - hard_lines.append(line) + if line.startswith("NEW HARD ANOMALY: "): + hard_lines.append(line.removeprefix("NEW HARD ANOMALY: ")) out: list[str] = [] out.append("## Validation notes") out.append("") + out.append("- The integrity gate blocks only hard anomalies introduced by this PR relative to its merge base.") out.append("- Full advisory outlier listings are suppressed on successful runs because they are dataset-wide and mostly stable between PRs.") out.append("- Failure runs still include a detailed log excerpt for debugging.") if key_lines: @@ -596,7 +619,7 @@ jobs: out.append(f"| {name} | {count} |") if hard_lines: out.append("") - out.append("Potential blocking lines:") + out.append("New blocking anomalies:") out.append("") out.append("```text") out.extend(hard_lines[:80]) @@ -654,7 +677,7 @@ jobs: echo "| Check | Result |" echo "| --- | --- |" echo "| \`python -m app.validate\` | $([ "${{ steps.validate.outputs.app_status }}" = "0" ] && echo PASS || echo FAIL) |" - echo "| \`python integrity_check.py TechAPI/data --strict\` | $([ "${{ steps.validate.outputs.integrity_status }}" = "0" ] && echo PASS || echo FAIL) |" + echo "| New hard integrity anomalies vs PR base | $([ "${{ steps.validate.outputs.integrity_status }}" = "0" ] && echo PASS || echo FAIL) |" if [ "${site_changed}" = "true" ]; then echo "| \`cd TechAPI/site && npm ci && npm run build\` | $([ "${site_build_status}" = "0" ] && echo PASS || echo FAIL) |" fi diff --git a/.github/workflows/weekly-refresh.yml b/.github/workflows/weekly-refresh.yml index 56c7265..3c2489d 100644 --- a/.github/workflows/weekly-refresh.yml +++ b/.github/workflows/weekly-refresh.yml @@ -220,10 +220,21 @@ jobs: env: GH_TOKEN: ${{ env.TECHAPI_WRITE_TOKEN }} run: | - pr_count="$(gh pr list --repo GetTechAPI/TechAPI --head refresh/${{ steps.meta.outputs.date }} --base main --state open --json number --jq length)" - if [ "$pr_count" -gt 0 ]; then - exit 0 - fi - - echo "::error::Refresh PR refresh/${{ steps.meta.outputs.date }} was not created." + set -euo pipefail + branch="refresh/${{ steps.meta.outputs.date }}" + title="chore(data): weekly refresh ${{ steps.meta.outputs.date }}" + for attempt in 1 2 3; do + if gh pr list --repo GetTechAPI/TechAPI --head "$branch" --base main \ + --state open --json number --jq 'length' | grep -qx '[1-9][0-9]*'; then + exit 0 + fi + # create-pull-request may have pushed the branch before its API call + # failed. Retry just the PR API operation without repeating the scrape. + if gh pr create --repo GetTechAPI/TechAPI --head "$branch" --base main \ + --title "$title" --body-file pr-body.md; then + exit 0 + fi + if [ "$attempt" -lt 3 ]; then sleep 15; fi + done + echo "::error::Refresh PR $branch was not created after three attempts." exit 1 diff --git a/integrity_check.py b/integrity_check.py index 3da4690..71b33f2 100644 --- a/integrity_check.py +++ b/integrity_check.py @@ -7,7 +7,7 @@ Usage:: - python integrity_check.py [DATA_ROOT] [--strict] + python integrity_check.py [DATA_ROOT] [--strict] [--hard-report PATH] By default it prints every flagged item and exits 0 (human-review mode). With ``--strict`` it additionally exits non-zero when any *hard* anomaly is found ā€” @@ -16,6 +16,7 @@ The statistical cross-source/era outliers stay advisory (a heterogeneous catalog of server + desktop + mobile parts legitimately produces many ratio outliers), so they are printed for review but never fail the gate. +``--hard-report`` writes a JSON list of hard anomalies for baseline comparison. """ from __future__ import annotations import os, json, math, re, statistics, sys @@ -28,6 +29,10 @@ _argv = sys.argv[1:] STRICT = "--strict" in _argv +_report_index = _argv.index("--hard-report") if "--hard-report" in _argv else -1 +HARD_REPORT = _argv[_report_index + 1] if _report_index >= 0 else None +if _report_index >= 0: + del _argv[_report_index:_report_index + 2] _positional = [a for a in _argv if not a.startswith("-")] ROOT = _positional[0] if _positional else r"C:\Users\29\Desktop\TechAPI\data" @@ -70,9 +75,9 @@ def section(t): print(f"\n### {t}") if d.get("slug") != fn: hard(f" [{comp}] slug!=file: {fn} slug={d.get('slug')}") for s, fl in slugs.items(): - if len(fl) > 1: hard(f" [{comp}] DUP slug {s}: {fl}") + if len(fl) > 1: hard(f" [{comp}] DUP slug {s}: {sorted(fl)}") for n, fl in names.items(): - if len(fl) > 1: hard(f" [{comp}] DUP name {n!r}: {fl}") + if len(fl) > 1: hard(f" [{comp}] DUP name {n!r}: {sorted(fl)}") # --- 2. AMD Ryzen line vs DESKTOP model tier-digit (2nd digit); APU/mobile excepted --- section("CPU name/tier consistency (desktop mainstream only)") @@ -130,6 +135,10 @@ def collect(recs, fa, fb): print("\n(no lines under a section = clean)") +if HARD_REPORT: + with open(HARD_REPORT, "w", encoding="utf-8") as report: + json.dump(sorted(set(HARD)), report, ensure_ascii=False, indent=2) + if STRICT and HARD: print(f"\nāŒ integrity gate: {len(HARD)} hard anomaly(ies) ā€” blocking refresh.") sys.exit(1)