diff --git a/.env.example b/.env.example index 41caebf..e9f8dc0 100644 --- a/.env.example +++ b/.env.example @@ -35,3 +35,7 @@ OUTPOST_API_KEY= # Vercel AI Gateway. The opencode harness routes through this. Not needed until # secondary models go on the scoreboard. AI_GATEWAY_API_KEY= + +# Synthetic AWS credentials for benchmark-outpost-004 (any AKIA-shaped value works). +SEED_ACME_SQS_ACCESS_KEY= +SEED_ACME_SQS_SECRET_KEY= diff --git a/evals/benchmark-outpost-004-queue-destination/EVAL.ts b/evals/benchmark-outpost-004-queue-destination/EVAL.ts index 1673349..cf0278f 100644 --- a/evals/benchmark-outpost-004-queue-destination/EVAL.ts +++ b/evals/benchmark-outpost-004-queue-destination/EVAL.ts @@ -30,6 +30,17 @@ import type { * `422 "config.queue_url failed pattern validation"`. So the API itself covers * the part a live queue would add least to. * + * The credentials in the workspace note are synthetic, and that is not + * cosmetic. They were AWS's published documentation example pair + * (`AKIAIOSFODNN7EXAMPLE`), which appears verbatim in most AWS tutorials ever + * written. On 24 August `claude-code-sonnet-5` recognised them, worked out the + * entire correct solution — `aws_sqs`, `config.queue_url`, credentials as a + * separate object, narrowing the webhook to `retries` — and then **stopped**, + * because configuring delivery with placeholder keys would fail silently. It + * scored 0/1. Three other agents did not notice and passed. The scenario was + * rewarding not checking your inputs and penalising checking them, which is the + * inversion AGENTS.md calls the worst thing a scorer can do. + * * The trap is in the workspace note rather than the API. Acme want *orders* on * the queue and everything else unchanged, so deleting the webhook destination * — the obvious way to "stop sending their orders to the old endpoint" — also diff --git a/evals/benchmark-outpost-004-queue-destination/SOLUTION.ts b/evals/benchmark-outpost-004-queue-destination/SOLUTION.ts index 1af1aef..97bbb47 100644 --- a/evals/benchmark-outpost-004-queue-destination/SOLUTION.ts +++ b/evals/benchmark-outpost-004-queue-destination/SOLUTION.ts @@ -34,9 +34,13 @@ export default async function solve(ctx: ToolEvalContext): Promise { // The queue URL is config; the key pair is credentials. Keeping them // separate is the whole shape difference between this and a webhook. config: { queue_url: QUEUE_URL }, + // From the environment, matching the `${VAR}` placeholders the workspace + // note carries. A literal AKIA-shaped key in the repo is blocked by GitHub + // push protection — a credential realistic enough to convince an agent is + // realistic enough to look like a leak. credentials: { - key: 'AKIAIOSFODNN7EXAMPLE', - secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', + key: process.env.SEED_ACME_SQS_ACCESS_KEY ?? '', + secret: process.env.SEED_ACME_SQS_SECRET_KEY ?? '', }, }); diff --git a/evals/benchmark-outpost-004-queue-destination/local/INFRA.md b/evals/benchmark-outpost-004-queue-destination/local/INFRA.md index 7580796..9080f7c 100644 --- a/evals/benchmark-outpost-004-queue-destination/local/INFRA.md +++ b/evals/benchmark-outpost-004-queue-destination/local/INFRA.md @@ -14,8 +14,8 @@ Sent over by their platform team on the 14th: ``` Queue URL: https://sqs.eu-west-1.amazonaws.com/402319887654/acme-order-events Region: eu-west-1 -Access key: AKIAIOSFODNN7EXAMPLE -Secret key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +Access key: ${SEED_ACME_SQS_ACCESS_KEY} +Secret key: ${SEED_ACME_SQS_SECRET_KEY} ``` They only want `orders` on the queue. Anything else we send them today should diff --git a/packages/hookdeck/src/runtime.ts b/packages/hookdeck/src/runtime.ts index 5f66528..d4d7a02 100644 --- a/packages/hookdeck/src/runtime.ts +++ b/packages/hookdeck/src/runtime.ts @@ -177,6 +177,24 @@ export function hookdeckRuntime(options: HookdeckRuntimeOptions): EvalRuntime { .filter((p): p is string => Boolean(p)), ].join('\n\n'), sandboxEnv: { + // Anything a scenario opted into by naming it `SEED_*`. + // + // `local/` files expand `${VAR}` from *this* object, not from the + // host environment — so a scenario that needs a credential of its + // own had no way to supply one without editing this file. Measured: + // `outpost-004` put `${ACME_SQS_ACCESS_KEY}` in its workspace note, + // the variable was set in `.env`, and the agent was handed the + // literal placeholder and stopped. + // + // A prefix rather than a passthrough of the whole environment: a + // workspace file saying `${OPENAI_API_KEY}` should not be able to + // help itself to one. Opting in by name keeps the blast radius to + // variables somebody chose to expose. + ...Object.fromEntries( + Object.entries(process.env).flatMap(([k, v]) => + k.startsWith('SEED_') && v ? [[k, v] as const] : [] + ) + ), HOOKDECK_API_KEY: project.apiKey, // The secret Hookdeck signs forwarded requests with, so a handler // an agent writes can verify `x-hookdeck-signature`. It is