feat(evals): Record and replay evals with the Roach service

.github/workflows/evals-behavioral.yml

@@ -2,7 +2,6 @@ name: Evals
 
 permissions:
   contents: read
-  # The eval-recordings-mode action reads the commits and runs of the PR.
   pull-requests: read
   actions: read
   checks: write
@@ -13,9 +12,7 @@ on:
     types: [opened, reopened, synchronize, labeled]
 
 concurrency:
-  # A recordings commit must not cancel the runs that are still recording,
-  # so each bot that pushes has its own group.
-  group: evals-behavioral-${{ github.event.pull_request.number || github.ref }}-${{ github.event.sender.type == 'Bot' && github.event.sender.login || 'user' }}
+  group: evals-behavioral-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 env:
@@ -27,7 +24,6 @@ jobs:
     runs-on: blacksmith-4vcpu-ubuntu-2404
     outputs:
       should_run: ${{ steps.decision.outputs.should_run }}
-      mode: ${{ steps.mode.outputs.mode || 'auto' }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -49,10 +45,6 @@ jobs:
               - 'packages/junior-evals/evals/skills/**'
               - '.github/workflows/evals-behavioral.yml'
               - '.github/actions/setup-node-pnpm/**'
-              - '.github/actions/collect-eval-recordings/**'
-              - '.github/actions/commit-eval-recordings/**'
-              - '.github/actions/eval-recordings-mode/**'
-              - 'packages/junior-evals/recordings/**'
               - 'pnpm-lock.yaml'
               - 'pnpm-workspace.yaml'
               - 'packages/junior-evals/src/**'
@@ -101,11 +93,6 @@ jobs:
             echo "- will_run: $should_run"
             echo "- minimum_pass_rate: $EVAL_MIN_PASS_RATE"
           } >> "$GITHUB_STEP_SUMMARY"
-      - id: mode
-        if: steps.decision.outputs.should_run == 'true'
-        uses: ./.github/actions/eval-recordings-mode
-        with:
-          suite: behavioral
 
   behavioral:
     name: behavioral / shard ${{ matrix.shard }}/4
@@ -175,14 +162,9 @@ jobs:
         env:
           VITEST_EVALS_OUTPUT_FILE: behavioral-results-${{ matrix.shard }}.json
           VITEST_EVALS_REPORT_LEVEL: info
-          EVAL_RECORDING_MISSES_DIR: ${{ runner.temp }}/eval-recording-misses
-          VITEST_EVALS_REPLAY_MODE: ${{ needs.select.outputs.mode }}
+          # The write token of Roach. With it, the run records new requests.
+          ROACH_TOKEN: ${{ secrets.ROACH_TOKEN }}
         run: packages/junior-evals/scripts/network-jail.sh pnpm --filter @sentry/junior-evals evals:behavioral --shard=${{ matrix.shard }}/4
-      - name: Collect new recordings
-        if: always() && !cancelled()
-        uses: ./.github/actions/collect-eval-recordings
-        with:
-          name: eval-recordings-behavioral-${{ matrix.shard }}
       - name: Require behavioral eval results
         id: results
         if: steps.run.conclusion != 'skipped'
@@ -255,25 +237,3 @@ jobs:
         env:
           SENTRY_EVALS_API_KEY: ${{ secrets.SENTRY_EVALS_API_KEY }}
         run: node packages/junior-evals/scripts/report.mjs behavioral eval-results/behavioral-results-*.json
-
-  recordings:
-    name: behavioral / commit recordings
-    needs: [select, behavioral]
-    # Only for a branch of this repository: the token of a fork run is
-    # read-only. Strict replay runs write no recordings.
-    if: ${{ always() && !cancelled() && needs.select.outputs.should_run == 'true' && needs.select.outputs.mode == 'auto' && github.event.pull_request.head.repo.full_name == github.repository }}
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    # GITHUB_TOKEN pushes the commit. It expires when the job ends, and it
-    # can do only what a person who can push a branch here can already do.
-    # The job holds no other secret, so it can run the action from the pull
-    # request.
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-          sparse-checkout: .github/actions
-      - uses: ./.github/actions/commit-eval-recordings
-        with:
-          suite: behavioral