diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml index 300eb60..5c536c6 100644 --- a/.github/workflows/eval-refresh.yml +++ b/.github/workflows/eval-refresh.yml @@ -303,6 +303,14 @@ jobs: # on 13 August scored outpost-001 as six agent failures rather than # skipping it, which is a false result published against named vendors. OUTPOST_API_KEY: ${{ secrets.OUTPOST_API_KEY }} + # Synthetic AWS credentials a scenario's workspace note carries as + # `${SEED_*}` placeholders. Not real, and not committed: a key realistic + # enough to convince an agent is realistic enough to trip push + # protection. Absent, `benchmark-outpost-004` is skipped rather than + # failed — an agent handed a literal placeholder is right to refuse, and + # scoring that is the 13 August mistake in a new form. + SEED_ACME_SQS_ACCESS_KEY: ${{ secrets.SEED_ACME_SQS_ACCESS_KEY }} + SEED_ACME_SQS_SECRET_KEY: ${{ secrets.SEED_ACME_SQS_SECRET_KEY }} # For `gh run cancel` when a provider stops answering. Needs # `actions: write` at the workflow level, which `publish-results` did # not previously require. @@ -354,6 +362,8 @@ jobs: echo "HOOKDECK_API_KEY=${HOOKDECK_API_KEY}" echo "HOOKDECK_WEBHOOK_SECRET=${HOOKDECK_WEBHOOK_SECRET}" echo "OUTPOST_API_KEY=${OUTPOST_API_KEY}" + echo "SEED_ACME_SQS_ACCESS_KEY=${SEED_ACME_SQS_ACCESS_KEY}" + echo "SEED_ACME_SQS_SECRET_KEY=${SEED_ACME_SQS_SECRET_KEY}" } > .env - name: Run evals diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index a052d50..ad80c5d 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -108,7 +108,50 @@ function unmetRequirements(ev: EvalManifest): string[] { const available: Record = { outpost: Boolean(process.env.OUTPOST_API_KEY), }; - return (ev.metadata.requires ?? []).filter((r) => !available[r]); + const unmet = (ev.metadata.requires ?? []).filter((r) => !available[r]); + + // A `${SEED_*}` placeholder the environment cannot fill is an unmet + // requirement, not a task. + // + // `local/` files leave an unset variable as written, so the agent is handed + // the literal `${SEED_ACME_SQS_ACCESS_KEY}`. That reads as an obvious + // placeholder, and an agent that notices is right to stop rather than + // configure delivery that will silently fail — measured on 24 August, where + // exactly that scored 0/1 while three agents that did not notice scored 6/6. + // + // Scoring it is the 13 August mistake again: a missing credential became six + // agent failures against named vendors, and the fix then was to skip rather + // than score. The same reasoning applies to a credential the scenario + // supplies itself. + return [...unmet, ...unfilledSeedPlaceholders(ev)]; +} + +/** `SEED_*` names a scenario's workspace asks for and the environment lacks. */ +function unfilledSeedPlaceholders(ev: EvalManifest): string[] { + if (!ev.localDir || !existsSync(ev.localDir)) return []; + const missing = new Set(); + + const walk = (dir: string) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) { + walk(path); + continue; + } + let text: string; + try { + text = readFileSync(path, 'utf8'); + } catch { + continue; // binary; nothing to expand + } + for (const [, name] of text.matchAll(/\$\{(SEED_[A-Z0-9_]+)\}/g)) { + if (!process.env[name]) missing.add(name); + } + } + }; + walk(ev.localDir); + + return [...missing]; } type ToolsSkill = { name: string; description: string; body: string };