outpost-004: stop shipping AWS's documented example credentials

.env.example

@@ -35,3 +35,7 @@ OUTPOST_API_KEY=
 # Vercel AI Gateway. The opencode harness routes through this. Not needed until
 # secondary models go on the scoreboard.
 AI_GATEWAY_API_KEY=
+
+# Synthetic AWS credentials for benchmark-outpost-004 (any AKIA-shaped value works).
+SEED_ACME_SQS_ACCESS_KEY=
+SEED_ACME_SQS_SECRET_KEY=

evals/benchmark-outpost-004-queue-destination/EVAL.ts

@@ -30,6 +30,17 @@ import type {
  * `422 "config.queue_url failed pattern validation"`. So the API itself covers
  * the part a live queue would add least to.
  *
+ * The credentials in the workspace note are synthetic, and that is not
+ * cosmetic. They were AWS's published documentation example pair
+ * (`AKIAIOSFODNN7EXAMPLE`), which appears verbatim in most AWS tutorials ever
+ * written. On 24 August `claude-code-sonnet-5` recognised them, worked out the
+ * entire correct solution — `aws_sqs`, `config.queue_url`, credentials as a
+ * separate object, narrowing the webhook to `retries` — and then **stopped**,
+ * because configuring delivery with placeholder keys would fail silently. It
+ * scored 0/1. Three other agents did not notice and passed. The scenario was
+ * rewarding not checking your inputs and penalising checking them, which is the
+ * inversion AGENTS.md calls the worst thing a scorer can do.
+ *
  * The trap is in the workspace note rather than the API. Acme want *orders* on
  * the queue and everything else unchanged, so deleting the webhook destination
  * — the obvious way to "stop sending their orders to the old endpoint" — also

evals/benchmark-outpost-004-queue-destination/SOLUTION.ts

@@ -34,9 +34,13 @@ export default async function solve(ctx: ToolEvalContext): Promise<void> {
     // The queue URL is config; the key pair is credentials. Keeping them
     // separate is the whole shape difference between this and a webhook.
     config: { queue_url: QUEUE_URL },
+    // From the environment, matching the `${VAR}` placeholders the workspace
+    // note carries. A literal AKIA-shaped key in the repo is blocked by GitHub
+    // push protection — a credential realistic enough to convince an agent is
+    // realistic enough to look like a leak.
     credentials: {
-      key: 'AKIAIOSFODNN7EXAMPLE',
-      secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
+      key: process.env.SEED_ACME_SQS_ACCESS_KEY ?? '',
+      secret: process.env.SEED_ACME_SQS_SECRET_KEY ?? '',
     },
   });
 

evals/benchmark-outpost-004-queue-destination/local/INFRA.md

@@ -14,8 +14,8 @@ Sent over by their platform team on the 14th:
 ```
 Queue URL:  https://sqs.eu-west-1.amazonaws.com/402319887654/acme-order-events
 Region:     eu-west-1
-Access key: AKIAIOSFODNN7EXAMPLE
-Secret key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+Access key: ${SEED_ACME_SQS_ACCESS_KEY}
+Secret key: ${SEED_ACME_SQS_SECRET_KEY}
 ```
 
 They only want `orders` on the queue. Anything else we send them today should

packages/hookdeck/src/runtime.ts

@@ -177,6 +177,24 @@ export function hookdeckRuntime(options: HookdeckRuntimeOptions): EvalRuntime {
               .filter((p): p is string => Boolean(p)),
           ].join('\n\n'),
           sandboxEnv: {
+            // Anything a scenario opted into by naming it `SEED_*`.
+            //
+            // `local/` files expand `${VAR}` from *this* object, not from the
+            // host environment — so a scenario that needs a credential of its
+            // own had no way to supply one without editing this file. Measured:
+            // `outpost-004` put `${ACME_SQS_ACCESS_KEY}` in its workspace note,
+            // the variable was set in `.env`, and the agent was handed the
+            // literal placeholder and stopped.
+            //
+            // A prefix rather than a passthrough of the whole environment: a
+            // workspace file saying `${OPENAI_API_KEY}` should not be able to
+            // help itself to one. Opting in by name keeps the blast radius to
+            // variables somebody chose to expose.
+            ...Object.fromEntries(
+              Object.entries(process.env).flatMap(([k, v]) =>
+                k.startsWith('SEED_') && v ? [[k, v] as const] : []
+              )
+            ),
             HOOKDECK_API_KEY: project.apiKey,
             // The secret Hookdeck signs forwarded requests with, so a handler
             // an agent writes can verify `x-hookdeck-signature`. It is