eval-quality.yml

name: Eval quality gate

on:
  push:
    branches: [main]
  pull_request:

jobs:
  eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: npm

      - run: npm ci

      # 1 — Run your evals; emit eval-report/v1 JSON to .evals_output/
      - name: Run evals
        run: npm run eval

      # 2 — Generate HTML dashboard + machine-readable summaries
      - name: Generate eval dashboard
        run: npx @icodenet/eval-dashboards report --reporter=html --reporter=json-summary

      # 2b — Clear previous machine outputs to avoid stale heartbeat/result reuse
      - name: Clear stale check machine outputs
        run: |
          rm -f eval-report/check-result.json \
            eval-report/check-heartbeat.json \
            eval-report/check-result.junit.xml \
            eval-report/check-result.sarif.json \
            eval-report/check-annotations.json

      # 3 — Enforce gates and produce CI-native machine outputs
      - name: Check eval gates
        id: check
        continue-on-error: true
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        run: |
          # --json-out is required for follow-up gate-result checks.
          notify_flags=()
          if [ -n "${SLACK_WEBHOOK_URL:-}" ]; then
            notify_flags+=(
              --notify=slack
              --notify-webhook="$SLACK_WEBHOOK_URL"
              --notify-report-link="https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/eval-report/index.html"
            )
          fi

          npx @icodenet/eval-dashboards check \
            --min-pass-rate=0.9 \
            --max-new-failures=0 \
            --zero-critical \
            --json-out=eval-report/check-result.json \
            --heartbeat-out=eval-report/check-heartbeat.json \
            --junit-out=eval-report/check-result.junit.xml \
            --sarif-out=eval-report/check-result.sarif.json \
            --github-annotations-out=eval-report/check-annotations.json \
            "${notify_flags[@]}"

      # 3b — Emit GitHub log annotations from adapter JSON
      - name: Emit GitHub annotations
        if: always() && hashFiles('eval-report/check-annotations.json') != ''
        run: |
          jq -r '.[] | @base64' eval-report/check-annotations.json | while read -r item; do
            json=$(echo "$item" | base64 --decode)
            level=$(echo "$json" | jq -r '.level')
            title=$(echo "$json" | jq -r '.title')
            message=$(echo "$json" | jq -r '.message')
            message=${message//'%'/'%25'}
            message=${message//$'\r'/'%0D'}
            message=${message//$'\n'/'%0A'}
            if [ "$level" = "error" ]; then
              echo "::error title=$title::$message"
            else
              echo "::warning title=$title::$message"
            fi
          done

      # 3c — Fail fast when check was skipped or errored (heartbeat status)
      - name: Fail on skipped/errored check run status
        if: always() && hashFiles('eval-report/check-heartbeat.json') != ''
        run: |
          status=$(jq -r '.gateRunStatus' eval-report/check-heartbeat.json)
          exit_code=$(jq -r '.exitCode' eval-report/check-heartbeat.json)
          if [ "$status" != "ran" ]; then
            echo "Eval check did not run cleanly (status=$status). See eval-report/check-heartbeat.json"
            exit 1
          fi
          if [ "$exit_code" != "0" ] && [ "$exit_code" != "1" ]; then
            echo "Eval check errored (exitCode=$exit_code). See eval-report/check-heartbeat.json"
            exit 1
          fi
          if [ "$exit_code" = "1" ]; then
            echo "Eval gates failed. See eval-report/check-result.json and eval-report/check-heartbeat.json"
            exit 1
          fi

      # 3d — Fail if heartbeat is missing (guard against fail-open monitoring)
      - name: Fail on missing check heartbeat
        if: always() && hashFiles('eval-report/check-heartbeat.json') == ''
        run: |
          echo "Eval check heartbeat output missing: eval-report/check-heartbeat.json"
          exit 1

      # 3e — Fail if check result JSON is missing
      - name: Fail on missing check result
        if: always() && hashFiles('eval-report/check-result.json') == ''
        run: |
          echo "Eval check result output missing: eval-report/check-result.json"
          exit 1

      # 3f — Fail job explicitly when gates failed
      - name: Fail on gate outcome
        if: always() && hashFiles('eval-report/check-result.json') != ''
        run: |
          passed=$(jq -r '.passed' eval-report/check-result.json)
          if [ "$passed" != "true" ]; then
            echo "Eval gates failed. See eval-report/check-result.json"
            exit 1
          fi

      # 4 — Publish dashboard to GitHub Pages (main branch only)
      - name: Publish to GitHub Pages
        if: github.ref == 'refs/heads/main'
        run: |
          npx @icodenet/eval-dashboards publish \
            --target=github-pages \
            --repo=${{ github.repository }} \
            --branch=eval-results
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      # 5 — Upload dashboard as a CI artifact (every run)
      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: eval-dashboard-${{ github.run_id }}
          path: eval-report/
          retention-days: 30