From a31c7aa1427f6e2776260dd9e09fb0e75e9cf9e1 Mon Sep 17 00:00:00 2001
From: Phil Leggetter <phil@leggetter.co.uk>
Date: Mon, 24 Aug 2026 21:05:18 +0100
Subject: [PATCH] Skip a scenario whose seed credentials are missing, rather
 than failing it

`local/` files leave an unset `${VAR}` as written, so an agent is handed the
literal `${SEED_ACME_SQS_ACCESS_KEY}`. That reads as an obvious placeholder,
and an agent that notices is right to stop rather than configure delivery
that will silently fail. Measured on 24 August: exactly that scored 0/1
while three agents that did not notice scored 6/6.

Scoring it is the 13 August mistake again. A missing OUTPOST_API_KEY then
became six agent failures against named vendors, and the fix was to skip
rather than score. The same reasoning applies to a credential the scenario
supplies itself: a missing row is recoverable, a wrong row is not.

`unmetRequirements` now also reports any `SEED_*` placeholder a scenario's
workspace asks for and the environment cannot fill. Verified: without the
two variables, outpost-004 skips all six cells naming both; outpost-005,
which uses none, still plans its six.

The workflow passes the two secrets through so the scenario runs once they
exist. Until then CI skips it, which is why the weekly cron does not need
pausing.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Nt2Zgjw7STjrnFXYKRRVAA
---
 .github/workflows/eval-refresh.yml | 10 +++++++
 apps/framework/harness/run-eval.ts | 45 +++++++++++++++++++++++++++++-
 2 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/eval-refresh.yml b/.github/workflows/eval-refresh.yml
index 300eb60..5c536c6 100644
--- a/.github/workflows/eval-refresh.yml
+++ b/.github/workflows/eval-refresh.yml
@@ -303,6 +303,14 @@ jobs:
       # on 13 August scored outpost-001 as six agent failures rather than
       # skipping it, which is a false result published against named vendors.
       OUTPOST_API_KEY: ${{ secrets.OUTPOST_API_KEY }}
+      # Synthetic AWS credentials a scenario's workspace note carries as
+      # `${SEED_*}` placeholders. Not real, and not committed: a key realistic
+      # enough to convince an agent is realistic enough to trip push
+      # protection. Absent, `benchmark-outpost-004` is skipped rather than
+      # failed — an agent handed a literal placeholder is right to refuse, and
+      # scoring that is the 13 August mistake in a new form.
+      SEED_ACME_SQS_ACCESS_KEY: ${{ secrets.SEED_ACME_SQS_ACCESS_KEY }}
+      SEED_ACME_SQS_SECRET_KEY: ${{ secrets.SEED_ACME_SQS_SECRET_KEY }}
       # For `gh run cancel` when a provider stops answering. Needs
       # `actions: write` at the workflow level, which `publish-results` did
       # not previously require.
@@ -354,6 +362,8 @@ jobs:
             echo "HOOKDECK_API_KEY=${HOOKDECK_API_KEY}"
             echo "HOOKDECK_WEBHOOK_SECRET=${HOOKDECK_WEBHOOK_SECRET}"
             echo "OUTPOST_API_KEY=${OUTPOST_API_KEY}"
+            echo "SEED_ACME_SQS_ACCESS_KEY=${SEED_ACME_SQS_ACCESS_KEY}"
+            echo "SEED_ACME_SQS_SECRET_KEY=${SEED_ACME_SQS_SECRET_KEY}"
           } > .env
 
       - name: Run evals
diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts
index a052d50..ad80c5d 100644
--- a/apps/framework/harness/run-eval.ts
+++ b/apps/framework/harness/run-eval.ts
@@ -108,7 +108,50 @@ function unmetRequirements(ev: EvalManifest): string[] {
   const available: Record<string, boolean> = {
     outpost: Boolean(process.env.OUTPOST_API_KEY),
   };
-  return (ev.metadata.requires ?? []).filter((r) => !available[r]);
+  const unmet = (ev.metadata.requires ?? []).filter((r) => !available[r]);
+
+  // A `${SEED_*}` placeholder the environment cannot fill is an unmet
+  // requirement, not a task.
+  //
+  // `local/` files leave an unset variable as written, so the agent is handed
+  // the literal `${SEED_ACME_SQS_ACCESS_KEY}`. That reads as an obvious
+  // placeholder, and an agent that notices is right to stop rather than
+  // configure delivery that will silently fail — measured on 24 August, where
+  // exactly that scored 0/1 while three agents that did not notice scored 6/6.
+  //
+  // Scoring it is the 13 August mistake again: a missing credential became six
+  // agent failures against named vendors, and the fix then was to skip rather
+  // than score. The same reasoning applies to a credential the scenario
+  // supplies itself.
+  return [...unmet, ...unfilledSeedPlaceholders(ev)];
+}
+
+/** `SEED_*` names a scenario's workspace asks for and the environment lacks. */
+function unfilledSeedPlaceholders(ev: EvalManifest): string[] {
+  if (!ev.localDir || !existsSync(ev.localDir)) return [];
+  const missing = new Set<string>();
+
+  const walk = (dir: string) => {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      const path = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        walk(path);
+        continue;
+      }
+      let text: string;
+      try {
+        text = readFileSync(path, 'utf8');
+      } catch {
+        continue; // binary; nothing to expand
+      }
+      for (const [, name] of text.matchAll(/\$\{(SEED_[A-Z0-9_]+)\}/g)) {
+        if (!process.env[name]) missing.add(name);
+      }
+    }
+  };
+  walk(ev.localDir);
+
+  return [...missing];
 }
 
 type ToolsSkill = { name: string; description: string; body: string };