verify-promote.yml

name: verify-promote

# Reusable, manual change generator (ADR-017). Only TechAPI's collect.yml calls
# it, from workflow_dispatch, with a read-only token. The run commits locally,
# gates the result and uploads the commit as collect.patch; a maintainer opens
# the PR from their own account with TechAPI's .github/scripts/open-collect-pr.sh.
# Nothing here pushes, opens a PR, posts a comment, runs on a schedule, or uses
# a personal or bot-account token. Crawlers send the honest TechEngine-*
# User-Agent and keep their --sleep and limits.
#
# Task: network verification (Tier 1 source-URL liveness, Wikidata, Tier 2
# cross-reference) and `verified` promotion. Dry-run unless `apply` is true;
# with apply, the diff-scope guard allows nothing but `verified` toggles and
# the ledger under data/_verify/.
on:
  workflow_call:
    inputs:
      apply:
        description: "Flip verified->true and upload the commit as a patch (else dry-run)"
        type: boolean
        default: false
      max_urls:
        description: "Frontier records to URL-check"
        type: string
        default: "2000"
      max_wikidata:
        description: "Maximum Wikidata QIDs to batch-check"
        type: string
        default: "20000"
      max_crossref:
        description: "Records to cross-reference"
        type: string
        default: "500"

permissions:
  contents: read

concurrency:
  group: verify-network
  cancel-in-progress: false

jobs:
  promote:
    runs-on: ubuntu-24.04
    timeout-minutes: 240
    env:
      PYTHONIOENCODING: utf-8
      MAX_URLS: ${{ inputs.max_urls }}
      MAX_WIKIDATA: ${{ inputs.max_wikidata }}
      MAX_CROSSREF: ${{ inputs.max_crossref }}
      BRANCH: collect/promote
      TITLE: "data(verify): promote records to verified"
      TECHAPI_DATA_DIR: ${{ github.workspace }}/TechAPI/data
    steps:
      # The TechEngine commit TechAPI pins as its submodule is the engine that
      # runs here, so collection, PR checks and the deploy share one engine.
      - name: Checkout TechAPI develop with its pinned TechEngine
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: GetTechAPI/TechAPI
          ref: develop
          path: TechAPI
          submodules: true
          # The Tier 0 diff compares the new commit with develop's tip only.
          fetch-depth: 1
          # Nothing in this job pushes.
          persist-credentials: false

      - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0
        with:
          python-version: "3.12"

      - name: Install the pinned TechEngine
        run: pip install -e ./TechAPI/TechEngine

      - name: Restore source URL cache
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: TechAPI/data/_verify/state/url_cache.jsonl
          key: verify-url-cache-${{ github.run_id }}-${{ github.run_attempt }}
          restore-keys: verify-url-cache-

      - name: Tier 0 score (recomputed; no committed cache)
        working-directory: TechAPI/TechEngine
        run: python -m app.verify score --no-cache

      - name: Tier 1 source-URL liveness
        working-directory: TechAPI/TechEngine
        run: python -m app.verify check-urls --max "$MAX_URLS"

      - name: Batched Wikidata source liveness
        working-directory: TechAPI/TechEngine
        run: python -m app.verify check-wikidata --max "$MAX_WIKIDATA"

      - name: Tier 2 external cross-reference
        working-directory: TechAPI/TechEngine
        run: python -m app.verify crossref --max "$MAX_CROSSREF"

      - name: Tier 3 promote (dry-run)
        working-directory: TechAPI/TechEngine
        run: python -m app.verify promote | tee "$GITHUB_WORKSPACE/promote-dry-run.md" >> "$GITHUB_STEP_SUMMARY"

      - name: Upload promotion output
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: promote-output
          path: promote-dry-run.md
          if-no-files-found: ignore

      - name: Tier 3 promote (apply)
        if: inputs.apply
        working-directory: TechAPI/TechEngine
        run: python -m app.verify promote --apply

      # Guard: the only tracked changes in TechAPI may be `verified` toggles in
      # data/**.json (the ledger under data/_verify/ is expected to change too).
      - name: Guard diff scope
        if: inputs.apply
        run: |
          python - <<'PY'
          import subprocess, sys
          out = subprocess.run(
              ["git", "-C", "TechAPI", "diff", "--unified=0", "--",
               "data/", ":(exclude)data/_verify/**"],
              capture_output=True, text=True).stdout
          bad = []
          for line in out.splitlines():
              if line.startswith(("+++", "---", "@@", "diff ", "index ")):
                  continue
              if line.startswith(("+", "-")) and line[1:].strip():
                  body = line[1:].strip().rstrip(",")
                  if body not in ('"verified": true', '"verified": false'):
                      bad.append(line)
          if bad:
              print("Unexpected non-verified changes:")
              print("\n".join(bad[:50]))
              sys.exit(1)
          print("diff scope OK: only verified toggles")
          PY

      # Commit locally first: the Tier 0 report scores the records this commit
      # changes against develop.
      - name: Commit data changes
        id: commit
        if: inputs.apply
        working-directory: TechAPI
        run: |
          set -euo pipefail
          git add data/
          if git diff --cached --quiet; then
            echo "No data changes; no patch."
            echo "changed=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          echo "base=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git switch -C "$BRANCH"
          git commit -q -m "$TITLE"
          echo "changed=true" >> "$GITHUB_OUTPUT"

      # Gate: the result goes in the PR body. Any failure stops the run before
      # the patch is uploaded.
      - name: Gate (app.validate + integrity_check --strict)
        if: steps.commit.outputs.changed == 'true'
        working-directory: TechAPI/TechEngine
        run: |
          set -euo pipefail
          python -m app.validate
          python integrity_check.py "$TECHAPI_DATA_DIR" --strict

      - name: Tier 0 report for the changed records
        if: steps.commit.outputs.changed == 'true'
        working-directory: TechAPI/TechEngine
        env:
          BASE: ${{ steps.commit.outputs.base }}
        run: |
          python -m app.verify score --base "$BASE" --changed --no-cache --format md \
            > "$GITHUB_WORKSPACE/verify-report.md" 2>&1 \
            || echo "_Tier 0 report unavailable._" > "$GITHUB_WORKSPACE/verify-report.md"

      # The body carries scraped text, which must not mention anyone or
      # cross-reference an issue: drop "@", break "#123"/"GH-123" and
      # URLs with a zero-width space. Colours such as #3fb950 are untouched.
      - name: Build PR body
        if: steps.commit.outputs.changed == 'true'
        run: |
          zw=$(printf '\342\200\213')
          ref="s/(#|[Gg][Hh]-)([0-9]+)(\$|[^0-9A-Za-z_])/\1$zw\2\3/g"
          scrub() { tr -d '@' | sed -E -e "$ref" -e "$ref" -e "s#://#:$zw//#g" -e "s/www\./www$zw./g"; }
          {
            echo "Closes #296"
            echo
            echo "Generated by TechEngine \`$GITHUB_WORKFLOW\` (run ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})."
            echo "Add the \`final-check\` label for the final pass."
            echo
            echo "## Gate"
            echo "- \`app.validate\`: **passed**"
            echo "- \`integrity_check.py --strict\`: **passed**"
            echo
            echo "## Tier 0 verification (changed records)"
            echo
            scrub < verify-report.md
            echo
            echo "## Promotion run"
            echo
            echo '```'
            scrub < promote-dry-run.md
            echo '```'
          } > pr-body.full.md
          # GitHub rejects a PR body over 65,536 characters.
          python - <<'PY'
          from pathlib import Path
          body = Path("pr-body.full.md").read_text(encoding="utf-8", errors="replace")
          limit = 60000
          if len(body) > limit:
              note = "\n\n_Truncated; the full summaries are in the run's artifact._\n"
              body = body[: limit - len(note)] + note
          Path("pr-body.md").write_text(body, encoding="utf-8")
          PY

      # The token is read-only, so the commit leaves the run as a patch. A
      # maintainer opens the PR from their own account with TechAPI's
      # .github/scripts/open-collect-pr.sh <run-id>, so the PR's checks run.
      - name: Export the commit as collect.patch
        if: steps.commit.outputs.changed == 'true'
        env:
          BASE: ${{ steps.commit.outputs.base }}
        run: |
          set -euo pipefail
          outside=$(git -C TechAPI diff --no-renames --name-only "$BASE" HEAD | grep -v '^data/' || true)
          if [ -n "$outside" ]; then
            echo "::error::The commit touches files outside data/:"
            echo "$outside"
            exit 1
          fi
          git -C TechAPI format-patch -1 --base="$BASE" --stdout HEAD > collect.patch
          python - <<'PY'
          import json
          import os
          meta = {
              "branch": os.environ["BRANCH"],
              "title": os.environ["TITLE"],
              "draft": os.environ.get("DRAFT") == "true",
              "base": os.environ["BASE"],
          }
          with open("pr.json", "w", encoding="utf-8") as f:
              json.dump(meta, f, indent=2)
              f.write("\n")
          PY
          cat pr-body.md >> "$GITHUB_STEP_SUMMARY"

      - name: Upload collect.patch
        if: steps.commit.outputs.changed == 'true'
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: collect-promote
          path: |
            collect.patch
            pr-body.md
            pr.json
          if-no-files-found: error
          retention-days: 14