From 98eb0b2e51d9a02b3d9e959a383294178ba6a099 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Fri, 21 Aug 2026 15:06:00 +0100 Subject: [PATCH 1/9] Give Outpost scenarios the Outpost skill in the +skills arm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The +skills experiments load ['hookdeck', 'event-gateway'], so an Outpost scenario got a router plus the wrong product's skill — event-gateway's only Outpost content is one line telling the agent to go elsewhere. The outpost skill, 39 files of it, had never been used by a run. "+skills" should mean the relevant skill is loaded. Whether an agent can find its way to the right skill from the router alone is a different question and deserves its own scenario, not to be smuggled into four. Adds `extra_skills`, which adds to an experiment's list only when that experiment has skills at all. Neither alternative works: - `skills` replaces the experiment's list outright, including for a -no-skills experiment, which would hand the baseline arm the very skill it exists to do without. - Adding outpost to the experiments themselves changes the skill set for all seventeen scenarios, making every published +skills row non-comparable to answer a question about four of them. Verified per arm: an Outpost scenario resolves to ['hookdeck','event-gateway','outpost'] with skills and [] without; a non-Outpost scenario is unchanged. This changes what outpost-001 measures, so it is re-run with the new three rather than left mixing methodologies. Its published passes came from agents self-installing the outpost skill over the network — both passing rows carry selfInstalled: ['outpost'], and no row passed without it. That finding is what prompted this: the skill was doing the work, by a route we had not designed and that depends on the registry being reachable. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Nt2Zgjw7STjrnFXYKRRVAA --- apps/framework/harness/run-eval.ts | 21 ++++++++++++++-- .../PROMPT.md | 2 ++ .../PROMPT.md | 2 ++ .../PROMPT.md | 2 ++ .../PROMPT.md | 2 ++ packages/core/src/eval-metadata.ts | 24 +++++++++++++++++++ 6 files changed, 51 insertions(+), 2 deletions(-) diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 1836d74..a052d50 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -206,6 +206,18 @@ function buildLoadSkillTool(skills: readonly ToolsSkill[]): ToolSet { * so `docker cp` copies real files, not dangling links. Missing skills are * skipped with a warning. */ +function resolveSkillNames( + metadata: { skills?: string[]; extraSkills?: string[] }, + experimentSkills: string[] +): string[] { + if (metadata.skills) return metadata.skills; + if (experimentSkills.length === 0) return experimentSkills; + const extra = (metadata.extraSkills ?? []).filter( + (name) => !experimentSkills.includes(name) + ); + return [...experimentSkills, ...extra]; +} + function resolveSkillSources( skillNames: string[] ): Array<{ name: string; dir: string }> { @@ -314,11 +326,16 @@ async function runOne( // A per-eval `skills` override replaces the experiment's own list entirely, // so a scenario testing self-installed skills gets an empty list regardless // of which experiment runs it. - const skillSources = resolveSkillSources(ev.metadata.skills ?? exp.skills); + // `skills` replaces, `extraSkills` adds — and adds nothing to an experiment + // that has none. That asymmetry is the point: a `-no-skills` arm must stay + // empty whatever the scenario asks for, or the baseline is no longer a + // baseline. See `extraSkills` in eval-metadata.ts. + const skillNames = resolveSkillNames(ev.metadata, exp.skills); + const skillSources = resolveSkillSources(skillNames); const availableSkills = skillSources.map((skill) => skill.name); const toolsSkills = ev.mode === 'tools' && !agentRunsInSandbox - ? loadToolsSkills(ev.metadata.skills ?? exp.skills) + ? loadToolsSkills(skillNames) : []; const scorer = (await import(pathToFileURL(ev.evalPath).href)) .default as ToolScorer; diff --git a/evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md b/evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md index 943b345..445083d 100644 --- a/evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md +++ b/evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md @@ -8,6 +8,8 @@ topic: - sdk requires: - outpost +extra_skills: + - outpost motivation: The reason people adopt Outpost. Sending webhooks to your own customers is a product feature, and every team that builds it by hand rebuilds retries, verification and a subscription model badly. --- diff --git a/evals/benchmark-outpost-002-disabled-destination/PROMPT.md b/evals/benchmark-outpost-002-disabled-destination/PROMPT.md index ea01247..5c2b934 100644 --- a/evals/benchmark-outpost-002-disabled-destination/PROMPT.md +++ b/evals/benchmark-outpost-002-disabled-destination/PROMPT.md @@ -8,6 +8,8 @@ topic: - retries requires: - outpost +extra_skills: + - outpost motivation: The support case Outpost generates most reliably. A destination that keeps failing is auto-disabled to protect the system, and once the customer repairs their endpoint nothing starts again on its own — the held events are not retried until the destination is re-enabled. Outpost does emit `alert.destination.disabled` as an operator event, but only where a sink has been configured for it (Hookdeck Monitoring settings on managed, `OPERATION_EVENTS_TOPICS` plus a sink when self-hosted); on a deployment where nobody has, the first signal is the customer. --- diff --git a/evals/benchmark-outpost-003-operator-events/PROMPT.md b/evals/benchmark-outpost-003-operator-events/PROMPT.md index 9feb035..c93c5b9 100644 --- a/evals/benchmark-outpost-003-operator-events/PROMPT.md +++ b/evals/benchmark-outpost-003-operator-events/PROMPT.md @@ -8,6 +8,8 @@ topic: - alerting requires: - outpost +extra_skills: + - outpost motivation: Follows the incident in benchmark-outpost-002. A destination was auto-disabled, the customer's events were held, and nobody found out until the customer emailed. Outpost emits `alert.destination.disabled` for exactly this, but it is delivered only to a configured operator events destination, and this project has none. The routes that configure it are absent from the published OpenAPI spec and from the API reference, so this measures whether an agent can set up alerting it cannot read about. --- diff --git a/evals/benchmark-outpost-004-queue-destination/PROMPT.md b/evals/benchmark-outpost-004-queue-destination/PROMPT.md index 75621a9..7bd1e47 100644 --- a/evals/benchmark-outpost-004-queue-destination/PROMPT.md +++ b/evals/benchmark-outpost-004-queue-destination/PROMPT.md @@ -8,6 +8,8 @@ topic: - capabilities requires: - outpost +extra_skills: + - outpost motivation: Most Outpost traffic is webhooks, but delivering to a queue is a core capability and a different job — a type rather than a URL, credentials rather than a secret, and fields whose names differ per provider. This scores whether an agent can configure a non-HTTP destination from details it has to find, rather than reaching for the webhook shape it has seen most often. --- diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index ea11b27..bfdae5c 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -189,6 +189,26 @@ export type EvalMetadata = { * entirely to use the experiment's own skill list. */ skills?: string[]; + /** + * Skills added to the experiment's list for this eval, **only when the + * experiment already has skills**. + * + * For product skills a scenario needs and its experiment does not carry. + * Outpost scenarios are the case: `+skills` experiments load `hookdeck` and + * `event-gateway`, so an Outpost scenario got a router plus the *wrong* + * product's skill, whose only Outpost content is a line telling the agent to + * go elsewhere. + * + * Not `skills`, which replaces the experiment's list outright — including for + * a `-no-skills` experiment, which would hand the baseline arm the very skill + * it exists to do without. And not adding `outpost` to the experiments + * themselves, which would change the skill set for all seventeen scenarios + * and make every published `+skills` row non-comparable to answer a question + * about four of them. + * + * Empty experiment list stays empty: that is what makes this safe. + */ + extraSkills?: string[]; /** * Skips installing the real Hookdeck CLI into the sandbox before the agent * starts (sandbox evals only). Defaults to false. Set true only for @@ -220,6 +240,7 @@ export const evalMetadataSchema = z.object({ hostedProject: z.union([z.boolean(), z.stringbool()]).optional(), requires: z.array(evalRequirementSchema).optional(), skills: z.array(z.string().min(1)).optional(), + extraSkills: z.array(z.string().min(1)).optional(), skipCliInstall: z.union([z.boolean(), z.stringbool()]).optional(), }); @@ -308,6 +329,9 @@ export const evalFrontmatterSchema = z.preprocess((raw) => { skills: Array.isArray(data.skills) ? toIdentifierList(data.skills) : undefined, + extraSkills: Array.isArray(data.extraSkills ?? data.extra_skills) + ? toIdentifierList((data.extraSkills ?? data.extra_skills) as unknown[]) + : undefined, skipCliInstall: data.skipCliInstall, }; }, evalMetadataSchema); From e705f1e17d259276768c38eef4bf1b8a64dc96e3 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Fri, 21 Aug 2026 19:00:17 +0100 Subject: [PATCH 2/9] Tell the agent about the Outpost project it already has MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prompt addendum's job is disclosure — it already says a project exists and how it is authenticated. It was silently omitting a second project that the harness injects whenever a scenario needs Outpost, while asserting "so both the CLI and the REST API are available to you", which reads as *this is your access*. That is not a discovery test we designed. Measured across twelve baseline cells on four Outpost scenarios: OUTPOST_API_KEY was used as a credential exactly zero times and every cell failed. Nine of the twelve agents believed they had succeeded, having built the task on api.hookdeck.com. Two reported with authority that the harness had given them the wrong credential, while a working key sat unread in their environment. The skill was the only artefact in the sandbox naming the variable, so the skills delta could not be separated from credential disclosure — the run measured our own omission as if it were a property of the skill. One credential type, two projects. Both keys authenticate api.hookdeck.com and the CLI; only the Outpost project's key reaches the Outpost subdomain, where another project's key gets a 404 that does not say why (#39). A developer using Outpost knows they use it and has the key in their environment; nobody learns their own credentials by enumerating env vars. The addendum names the project, the variable, and that the API has its own subdomain. It does not give the subdomain, the routes or the destination types, so how Outpost works is still what the scenarios measure. Scoped to the pinned CLI: against 2.5.0 the Outpost key authenticates the CLI and selects the project but has nothing useful to do there, so the API is the honest route to name. 3.0.0-beta.1 is published and adds managing an Outpost project from the CLI; when that pin moves this becomes incomplete rather than wrong. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Nt2Zgjw7STjrnFXYKRRVAA --- packages/hookdeck/src/runtime.ts | 57 ++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/packages/hookdeck/src/runtime.ts b/packages/hookdeck/src/runtime.ts index 436084c..97ae061 100644 --- a/packages/hookdeck/src/runtime.ts +++ b/packages/hookdeck/src/runtime.ts @@ -102,12 +102,63 @@ export function hookdeckRuntime(options: HookdeckRuntimeOptions): EvalRuntime { return { mcpServers, promptAddendum: [ - // The agent is told the project exists and is authenticated, and - // nothing else. Which docs to read, and how to do the task, is what - // the scenario measures. + // The agent is told what its environment contains, and nothing + // else. Which docs to read, and how to do the task, is what the + // scenario measures. 'You have a Hookdeck project. The Hookdeck CLI is installed and ' + 'HOOKDECK_API_KEY is set in your environment, so both the CLI ' + 'and the REST API are available to you.', + // Named because it is there. + // + // This sentence exists to make the one above true. The addendum's + // job is disclosure — it already says a project exists and how it + // is authenticated — and it was silently omitting a second project + // that the harness injects whenever a scenario needs Outpost. That + // is not a discovery test we designed; it is an incomplete + // sentence, and "so both the CLI and the REST API are available to + // you" actively reads as *this is your access*. + // + // Measured on 21 August, before this line existed: across twelve + // baseline cells on four Outpost scenarios, `OUTPOST_API_KEY` was + // used as a credential exactly zero times and every cell failed. + // Nine of the twelve agents believed they had succeeded, having + // built the task on `api.hookdeck.com`. Two reported, with + // authority, that the harness had given them the wrong credential. + // The skill was the only artefact in the sandbox naming the + // variable, so the skills delta could not be separated from + // credential disclosure — the run measured our own omission. + // + // One credential type, two projects. Both keys authenticate + // `api.hookdeck.com` and the CLI; only the Outpost project's key + // reaches the Outpost subdomain, and a key from another project + // gets a `404` there rather than anything that says why (#39). + // + // The wording points at the API rather than the CLI deliberately, + // and that is a statement about the pinned version rather than a + // permanent one. Against `HOOKDECK_CLI_VERSION` 2.5.0 the Outpost + // key authenticates the CLI and selects the Outpost project — + // verified — but there is nothing useful to do with it there, so + // the API is the only honest route to point at. + // + // `3.0.0-beta.1` is published (npm dist-tag `beta`; `latest` is + // still 2.5.0) and adds managing an Outpost project from the CLI. + // When that pin moves, this sentence becomes incomplete rather + // than wrong, and the CLI becomes a legitimate answer for Outpost + // work. A CLI bump also changes the product under test, so results + // either side of it are not comparable — see Releases in + // AGENTS.md. + // A developer using Outpost knows they use it and has the key in + // their environment; nobody learns their own credentials by + // enumerating env vars. + ...(outpostClient + ? [ + 'You also have a Hookdeck Outpost project, and ' + + 'OUTPOST_API_KEY is set in your environment. It is an ' + + 'ordinary Hookdeck project API key scoped to that ' + + 'project: use it for the Outpost API, which has its own ' + + 'subdomain.', + ] + : []), ...(options.mcpServers ?? []) .map((s) => s.promptAddendum) .filter((p): p is string => Boolean(p)), From f45c620b7eb7a697be3ed0fdfd19cdbfe6459a32 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Sat, 22 Aug 2026 08:50:19 +0100 Subject: [PATCH 3/9] Correct three things the Outpost runs showed were wrong MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **outpost-003 predicted universal failure on a scenario nobody fails.** The comment said "expect every agent to fail this initially, and publish it anyway", reasoning that an agent has no source of truth except what it can read. Agents do not work that way. It has now passed six times out of six across three models and both arms, including a weak model with no skills, by enumerating candidate routes — one first probed a deliberately bogus path to learn what a real 404 looks like, which is the control I failed to run when I concluded the API did not exist. So it measures persistence rather than discovery, does not currently discriminate, and is high variance: the same model passed 4/4 in one run and failed 0/1 in the next. #34 stands regardless — an endpoint reachable only by guesswork is undocumented whether or not an agent gets there. **outpost-002 counted any three successes on the tenant.** Three fresh publishes satisfied it as well as recovering the outage, and it passed an agent that reported retrying 54 events out of 78 found, because events outlive the tenant and a shared project accumulates them. Now counted per event: an event with a failed attempt is one the customer missed, and it is recovered when that same event_id also has a success. Republishing cannot fake it, since a new event has no failed attempt. **outpost-004 gave a do-nothing run 2/6.** Both negative checks are satisfied by the untouched seed, so no work scored two marks — the shape a crashed cell wears, and one wore it on 21 August before being spotted. The verdict was never wrong, since passed is the conjunction; the per-check count misled anyone reading detail, including us triaging a run. A run that changed nothing now returns a single failing check that says so. Verified against the live API: outpost-002 2/2 solved and fails unsolved; outpost-004 2/2 solved and a do-nothing run reports 0/1. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Nt2Zgjw7STjrnFXYKRRVAA --- .../EVAL.ts | 37 ++++++++++++++++--- .../EVAL.ts | 35 ++++++++++++------ .../EVAL.ts | 36 ++++++++++++++++++ 3 files changed, 91 insertions(+), 17 deletions(-) diff --git a/evals/benchmark-outpost-002-disabled-destination/EVAL.ts b/evals/benchmark-outpost-002-disabled-destination/EVAL.ts index e28cf05..07c098a 100644 --- a/evals/benchmark-outpost-002-disabled-destination/EVAL.ts +++ b/evals/benchmark-outpost-002-disabled-destination/EVAL.ts @@ -59,6 +59,8 @@ interface Destination { interface Attempt { id?: string; status?: string; + /** Which event this attempt was for. The check below turns on it. */ + event_id?: string; } const scorer: ToolScorer = async (ctx) => { @@ -120,21 +122,46 @@ async function checkMissedEventsDelivered( ): Promise { const name = 'the events the customer missed were delivered'; + // Counted per *event*, not per successful attempt. + // + // The first version counted any three successes on the tenant, which three + // fresh test publishes satisfy just as well as recovering the outage. It + // passed an agent that reported retrying 54 events out of 78 it had found — + // the stale-history trap, since events outlive the tenant and a shared + // project accumulates them. + // + // "Missed" is definable from the attempts themselves and needs no timestamps: + // an event that has a failed attempt is one the customer did not get, and it + // has been recovered when that same `event_id` also has a successful one. + // Republishing cannot fake it, because a new event has no failed attempt. + const recovered = (rows: Attempt[]): number => { + const failed = new Set( + rows + .filter((a) => a.status === 'failed' && a.event_id) + .map((a) => a.event_id) + ); + const succeeded = new Set( + rows + .filter((a) => a.status === 'success' && a.event_id) + .map((a) => a.event_id) + ); + return [...failed].filter((id) => succeeded.has(id)).length; + }; + // Poll: a retry triggered moments before the agent finished is still in // flight, and reading once would score the agent for the platform's timing. - // Summed across the tenant's destinations, for the same reason: an agent that - // recovered the events onto a replacement destination delivered them. + // Summed across the tenant's destinations, because an agent that recovered + // the events onto a replacement destination delivered them. const attempts = await waitForOrLast( () => listAllAttempts(ctx, TENANT, destinations), - (rows) => - rows.filter((a) => a.status === 'success').length >= MISSED_EVENTS, + (rows) => recovered(rows) >= MISSED_EVENTS, { timeoutMs: DELIVERY_WAIT_MS, description: 'the held events to be delivered', } ); - const delivered = attempts.filter((a) => a.status === 'success').length; + const delivered = recovered(attempts); return { name, passed: delivered >= MISSED_EVENTS, diff --git a/evals/benchmark-outpost-003-operator-events/EVAL.ts b/evals/benchmark-outpost-003-operator-events/EVAL.ts index d4568e2..9258ac6 100644 --- a/evals/benchmark-outpost-003-operator-events/EVAL.ts +++ b/evals/benchmark-outpost-003-operator-events/EVAL.ts @@ -13,20 +13,31 @@ import type { * with none is silent by design rather than by fault. * * What makes it worth measuring is where the answer lives. Operator events are - * configured over the API at `/operator-events/destinations`, and those routes - * appear in neither published OpenAPI spec nor the API reference at - * `/docs/outpost/api`; the prose page tells managed users to use the dashboard. - * An agent has no source of truth except what it can read, so this measures - * whether a documented-by-UI-only capability is reachable at all. See + * configured at `/operator-events/destinations`, and those routes appear in + * neither published OpenAPI spec nor the API reference at `/docs/outpost/api`; + * the prose page tells managed users to use the dashboard. See * hookdeck/evals#34. * - * **Expect every agent to fail this initially, and publish it anyway.** That is - * a floor rather than a flat result: the scenario is passable, the route exists - * and works, and the only thing missing is documentation. If #34 is fixed and - * the next run turns green, that is a closed loop — a finding, a change made - * outside this repository, and a re-run that says what the change bought. A - * scenario nobody passes for a reason we have written down is more useful than - * one nobody fails. + * **This scenario originally predicted that every agent would fail it, and that + * was wrong.** The prediction rested on "an agent has no source of truth except + * what it can read". Agents do not work that way: on 21 and 22 August it was + * passed six times out of six, across three models and both arms — including by + * a weak model with no skills at all. They find the route by enumeration, + * probing `/operator-events`, `/operator-event-destinations`, + * `/operator/destinations` and so on until one answers, having first probed a + * deliberately bogus path to learn what a real 404 looks like. + * + * So it does not measure discovery in the sense the classification implies, and + * it does not currently discriminate at all. What it measures is persistence: + * whether an agent keeps going when the documentation runs out. That is worth + * something, but it is a fact about agents rather than about Hookdeck, and it + * will not move if we fix the docs. + * + * Two consequences worth keeping in mind. It is **high variance** — the same + * model passed 4/4 in one run and failed 0/1 in the next, because passing turns + * on which paths get guessed, so a single attempt publishes a coin flip. And + * #34 stands regardless: an endpoint reachable only by guesswork is + * undocumented whether or not a determined agent gets there. * * Scored on outcome. It does not matter whether the agent used the API, and it * is not required to subscribe to the alert topic by name — `*` covers it and diff --git a/evals/benchmark-outpost-004-queue-destination/EVAL.ts b/evals/benchmark-outpost-004-queue-destination/EVAL.ts index 2146e92..1673349 100644 --- a/evals/benchmark-outpost-004-queue-destination/EVAL.ts +++ b/evals/benchmark-outpost-004-queue-destination/EVAL.ts @@ -72,6 +72,42 @@ const scorer: ToolScorer = async (ctx) => { (d) => normalise(d.config?.url) === normalise(OLD_ENDPOINT) ); + // An agent that did nothing scores zero, not two out of six. + // + // Both negative checks — that the retries still reach the old endpoint, and + // that the other customer was left alone — are satisfied by the untouched + // seed. So a run that never acted used to report `2/6`, which reads as + // partial progress and is really no progress: it is the shape a crashed cell + // wears, and one did exactly that on 21 August before being spotted. + // + // The verdict was never wrong, since `passed` is the conjunction and the four + // positive checks need real work. It is the per-check count that misleads + // anyone reading the detail — including us, triaging a run. + // + // So when nothing has changed at all, say that in one line instead of + // awarding marks for leaving things alone. + const untouched = + queues.length === 0 && + webhooks.some( + (d) => !d.disabled_at && subscribes(d, ORDERS) && subscribes(d, RETRIES) + ); + + if (untouched) { + return { + passed: false, + checks: [ + { + name: 'their orders are delivered to the queue they gave us', + passed: false, + notes: + 'nothing was changed: no queue destination exists and the old endpoint ' + + 'still carries both topics, so the checks about not breaking anything ' + + 'are true only because no work was done', + }, + ], + }; + } + const checks: CheckResult[] = [ checkQueueExists(destinations, queues), checkQueueReceivesOrders(queues), From 2821d5cbc463145d3a446c919af223cd0edb1486 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Sat, 22 Aug 2026 09:59:30 +0100 Subject: [PATCH 4/9] Add outpost-005: narrow a customer's topics without narrowing them too far MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four existing Outpost scenarios barely discriminate. In a 24-cell run +skills went 12/12 and two scenarios were 6/6, because their difficulty had quietly been "work out that Outpost is the product" and the harness stopped withholding that. What was left is setup, and setup is not where agents fail. This is built the other way round, from the shape AGENTS.md says discriminates: an agent can finish, report success, and be wrong, with nothing erroring. Acme receive three topics and ask us to stop one. The cheapest fix — set their destination to the topic the ticket talks about — stops the cancellations they complained about and silently stops order.shipped, which they never mentioned because it was working. No error, no failed delivery; they notice days later when something they depend on has stopped arriving. Verified all three paths against the live API: correct fix 5/5 pass naive fix fails only "still receives the shipping events they depend on" do nothing fails only "no longer receives order cancellations" Two wrong answers, failing on opposite checks, neither raising an error. Scored on behaviour rather than configuration: it publishes real events of each topic and checks what arrives, so an agent that scopes the tenant instead of the destination passes, and config that merely looks right fails. Not built, deliberately: the better trap is changing deployment-level TOPICS, which fixes the complainant and silently breaks every other customer — an agent did exactly that to this project on 21 August. A scenario rewarding that would break every subsequent cell in a run until #41 persists a config baseline. Worth revisiting then. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Nt2Zgjw7STjrnFXYKRRVAA --- .../EVAL.ts | 253 ++++++++++++++++++ .../PROMPT.md | 21 ++ .../SOLUTION.ts | 47 ++++ .../remote/seed.json | 32 +++ 4 files changed, 353 insertions(+) create mode 100644 evals/benchmark-outpost-005-topic-scoping/EVAL.ts create mode 100644 evals/benchmark-outpost-005-topic-scoping/PROMPT.md create mode 100644 evals/benchmark-outpost-005-topic-scoping/SOLUTION.ts create mode 100644 evals/benchmark-outpost-005-topic-scoping/remote/seed.json diff --git a/evals/benchmark-outpost-005-topic-scoping/EVAL.ts b/evals/benchmark-outpost-005-topic-scoping/EVAL.ts new file mode 100644 index 0000000..54907dd --- /dev/null +++ b/evals/benchmark-outpost-005-topic-scoping/EVAL.ts @@ -0,0 +1,253 @@ +import type { + CheckResult, + ToolEvalContext, + ToolScorer, +} from '@hookdeck-evals/core'; +import { waitForSettled } from '@hookdeck-evals/hookdeck'; + +/** + * Narrow one customer's slice of the event stream without narrowing it too far. + * + * Written after a 24-cell run in which the four existing Outpost scenarios were + * passed by almost everything — `+skills` went 12/12 — because their difficulty + * had quietly been "work out that Outpost is the product", and the harness + * stopped withholding that. What was left was mostly setup, and setup is not + * where agents fail. + * + * So this is built around the shape AGENTS.md says actually discriminates: an + * agent can finish, report success, and be wrong, with nothing erroring. Acme + * receives three topics and wants one of them stopped. The obvious fix — set + * their destination to the topic they still talk about — passes the check they + * complained about and silently stops `order.shipped`, which they never + * mentioned because it was working. Nobody sees an error. The customer notices + * days later, when something they depend on has quietly stopped arriving. + * + * That is the same failure mode as `resolve-002` and `alerting-001`: acting more + * broadly than asked. The difference is that here it is the *cheapest* way to + * satisfy the request, rather than a mistake you have to reach for. + * + * Scored on behaviour, not configuration. A destination's `topics` array can be + * right while delivery is wrong — a tenant's own topic list gates it too, and an + * agent that edits the tenant instead of the destination reaches the same end + * state by another route. So this publishes real events and checks what + * arrives, which passes any correct route and fails any incorrect one. + */ + +const ACME = 'acme'; +const GLOBEX = 'globex'; + +/** Still wanted. Named in the ticket only as "everything else". */ +const KEPT = 'order.created'; +/** Still wanted, and never mentioned — the one a too-narrow fix removes. */ +const UNMENTIONED = 'order.shipped'; +/** The topic they asked us to stop. */ +const UNWANTED = 'order.cancelled'; + +/** Long enough that "it did not arrive" means it is not coming. */ +const DELIVERY_WAIT_MS = 45_000; +const SETTLE_MS = 8_000; + +interface Attempt { + id?: string; + status?: string; + event_id?: string; +} + +interface Destination { + id?: string; + disabled_at?: string | null; +} + +const scorer: ToolScorer = async (ctx) => { + if (!ctx.outpost) { + throw new Error( + 'no Outpost client, but this scenario declares `requires: [outpost]` ' + + 'and should have been skipped rather than scored' + ); + } + + // Published together, then read once after they have all had time to land. + // Sending the positives and the negative separately would let a slow negative + // arrive after its own check had already passed. + const before = await attemptCount(ctx, ACME); + const globexBefore = await attemptCount(ctx, GLOBEX); + + await publish(ctx, ACME, KEPT); + await publish(ctx, ACME, UNMENTIONED); + await publish(ctx, ACME, UNWANTED); + await publish(ctx, GLOBEX, UNWANTED); + + // Two must arrive and one must not, so the positives starting the clock is + // what gives the negative its chance to be wrong. Reading the moment the + // positives land would pass a configuration that changed nothing at all. + const acme = await waitForSettled( + () => deliveredTopics(ctx, ACME), + (topics) => topics.has(KEPT) && topics.has(UNMENTIONED), + { + timeoutMs: DELIVERY_WAIT_MS, + settleMs: SETTLE_MS, + description: "acme's remaining topics to be delivered", + } + ); + + const globex = await waitForSettled( + () => deliveredTopics(ctx, GLOBEX), + (topics) => topics.has(UNWANTED), + { + timeoutMs: DELIVERY_WAIT_MS, + settleMs: SETTLE_MS, + description: "globex's delivery to be unaffected", + } + ); + + const checks: CheckResult[] = [ + { + name: 'acme no longer receives order cancellations', + passed: !acme.has(UNWANTED), + notes: acme.has(UNWANTED) + ? 'a cancellation was still delivered to acme, which is what they asked us to stop' + : undefined, + }, + { + // The check the scenario exists for. + name: 'acme still receives the shipping events they depend on', + passed: acme.has(UNMENTIONED), + notes: acme.has(UNMENTIONED) + ? undefined + : `${UNMENTIONED} stopped reaching acme. They asked us to stop cancellations ` + + 'and said everything else should carry on; this is the part they did not ' + + 'mention because it was working, and nothing here would have errored', + }, + { + name: 'acme still receives new orders', + passed: acme.has(KEPT), + notes: acme.has(KEPT) + ? undefined + : `${KEPT} stopped reaching acme, so the change went far wider than the request`, + }, + { + name: 'globex was left alone', + passed: globex.has(UNWANTED), + notes: globex.has(UNWANTED) + ? undefined + : 'globex stopped receiving cancellations too — a change made at the wrong ' + + 'level hits every customer, and they never complained about anything', + }, + await checkNothingDisabled(ctx), + ]; + + // Sanity, not a check: if the tenant received nothing at all, the failures + // above are about the harness rather than the agent. + if (acme.size === 0 && (await attemptCount(ctx, ACME)) === before) { + throw new Error( + 'no delivery attempts on acme at all after publishing: the seed or the ' + + 'platform is at fault, not the agent, so this run is not scoreable' + ); + } + void globexBefore; + + return { passed: checks.every((c) => c.passed), checks }; +}; + +export default scorer; + +/** + * Disabling a destination stops the unwanted topic as effectively as scoping it + * — and stops everything else too. It is caught by the delivery checks above, + * but naming it separately makes a red cell say *which* wrong route was taken. + */ +async function checkNothingDisabled( + ctx: ToolEvalContext +): Promise { + const disabled: string[] = []; + for (const tenant of [ACME, GLOBEX]) { + const rows = await listDestinations(ctx, tenant); + if (rows.some((d) => d.disabled_at)) disabled.push(tenant); + } + return { + name: 'no destination was switched off to achieve it', + passed: disabled.length === 0, + notes: + disabled.length === 0 + ? undefined + : `disabled: ${disabled.join(', ')} — that stops the cancellations by ` + + 'stopping everything, which is not what was asked', + }; +} + +/** Topics that actually reached a tenant, by successful delivery. */ +async function deliveredTopics( + ctx: ToolEvalContext, + tenant: string +): Promise> { + const events = await list<{ id?: string; topic?: string }>( + ctx, + `/events?tenant_id=${encodeURIComponent(tenant)}&limit=100` + ); + const byId = new Map(events.map((e) => [e.id, e.topic])); + + const topics = new Set(); + for (const destination of await listDestinations(ctx, tenant)) { + if (!destination.id) continue; + const attempts = await list( + ctx, + `/tenants/${encodeURIComponent(tenant)}/destinations/${encodeURIComponent(destination.id)}/attempts` + ); + for (const attempt of attempts) { + if (attempt.status !== 'success' || !attempt.event_id) continue; + const topic = byId.get(attempt.event_id); + if (topic) topics.add(topic); + } + } + return topics; +} + +async function attemptCount( + ctx: ToolEvalContext, + tenant: string +): Promise { + let total = 0; + for (const destination of await listDestinations(ctx, tenant)) { + if (!destination.id) continue; + total += ( + await list( + ctx, + `/tenants/${encodeURIComponent(tenant)}/destinations/${encodeURIComponent(destination.id)}/attempts` + ) + ).length; + } + return total; +} + +async function publish( + ctx: ToolEvalContext, + tenant: string, + topic: string +): Promise { + await ctx.outpost?.('POST', '/publish', { + tenant_id: tenant, + topic, + data: { probe: true, topic }, + }); +} + +async function listDestinations( + ctx: ToolEvalContext, + tenant: string +): Promise { + return list( + ctx, + `/tenants/${encodeURIComponent(tenant)}/destinations` + ); +} + +/** Outpost list endpoints answer `{ pagination, models }`, not `{ data }`. */ +async function list(ctx: ToolEvalContext, path: string): Promise { + const rows = await ctx.outpost?.( + 'GET', + path + ); + if (!rows) return []; + if (Array.isArray(rows)) return rows; + return rows.models ?? rows.data ?? []; +} diff --git a/evals/benchmark-outpost-005-topic-scoping/PROMPT.md b/evals/benchmark-outpost-005-topic-scoping/PROMPT.md new file mode 100644 index 0000000..110ca19 --- /dev/null +++ b/evals/benchmark-outpost-005-topic-scoping/PROMPT.md @@ -0,0 +1,21 @@ +--- +stage: resolve +suite: benchmark +gated_by: mixed +product: + - outpost +topic: + - filtering +requires: + - outpost +extra_skills: + - outpost +motivation: Every Outpost customer subscribes to a different slice of the same event stream, and narrowing one customer's slice is the most common change a support engineer makes. The obvious fix is too narrow — it stops the events they complained about and the ones they still depend on — and it fails silently, because nothing errors and the customer only notices what stops arriving days later. +--- + +Acme have been in touch. Their integration is choking on order cancellations — +they don't handle them and each one throws an error on their side. They want us +to stop sending those. + +They were clear that everything else they get today should carry on exactly as +it is, and Globex shouldn't be affected at all. diff --git a/evals/benchmark-outpost-005-topic-scoping/SOLUTION.ts b/evals/benchmark-outpost-005-topic-scoping/SOLUTION.ts new file mode 100644 index 0000000..368762f --- /dev/null +++ b/evals/benchmark-outpost-005-topic-scoping/SOLUTION.ts @@ -0,0 +1,47 @@ +import type { ToolEvalContext } from '@hookdeck-evals/core'; + +/** + * What a correct agent leaves behind: acme's destination subscribed to the two + * topics they still want, and nothing else touched. + * + * The single line worth reading is the topic list. Writing + * `['order.created']` — the topic the ticket talks about — satisfies the + * complaint and silently drops `order.shipped`, which is the failure this + * scenario exists to catch. Getting it right means noticing what the customer + * was receiving *before*, rather than what they wrote to you about. + */ + +const TENANT = 'acme'; +const OLD_ENDPOINT = 'https://mock.hookdeck.com/api/v1/acme/orders'; + +interface Destination { + id?: string; + config?: Record; +} + +export default async function solve(ctx: ToolEvalContext): Promise { + const outpost = ctx.outpost; + if (!outpost) { + throw new Error( + 'no Outpost client: this solution cannot be applied without OUTPOST_API_KEY' + ); + } + + const rows = await outpost( + 'GET', + `/tenants/${TENANT}/destinations` + ); + const destinations = Array.isArray(rows) ? rows : (rows.models ?? []); + + for (const destination of destinations) { + if (destination.config?.url !== OLD_ENDPOINT || !destination.id) continue; + await outpost( + 'PATCH', + `/tenants/${TENANT}/destinations/${destination.id}`, + { + // Everything they had, minus the one they asked to stop. + topics: ['order.created', 'order.shipped'], + } + ); + } +} diff --git a/evals/benchmark-outpost-005-topic-scoping/remote/seed.json b/evals/benchmark-outpost-005-topic-scoping/remote/seed.json new file mode 100644 index 0000000..4483367 --- /dev/null +++ b/evals/benchmark-outpost-005-topic-scoping/remote/seed.json @@ -0,0 +1,32 @@ +{ + "outpost": { + "tenants": [ + { + "id": "acme", + "topics": ["order.created", "order.shipped", "order.cancelled"], + "destinations": [ + { + "ref": "acme-all", + "type": "webhook", + "topics": ["order.created", "order.shipped", "order.cancelled"], + "config": { "url": "https://mock.hookdeck.com/api/v1/acme/orders" } + } + ] + }, + { + "id": "globex", + "topics": ["order.created", "order.shipped", "order.cancelled"], + "destinations": [ + { + "ref": "globex-all", + "type": "webhook", + "topics": ["order.created", "order.shipped", "order.cancelled"], + "config": { + "url": "https://mock.hookdeck.com/api/v1/globex/orders" + } + } + ] + } + ] + } +} From 97859e32708dc6ee05dac5d0758038d1b5968170 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Sat, 22 Aug 2026 11:20:38 +0100 Subject: [PATCH 5/9] Add triage: which cells of a run you have to read, and why MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AGENTS.md already says to read the agent's report first and calls it the cheapest answer every time. It keeps getting skipped, including by whoever wrote that line. An exhortation loses to a scoreboard, because the scoreboard is right there and the transcripts are not. The case is stronger than "sometimes useful": over two days of Outpost runs every product finding came out of a transcript and none came out of the scoreboard — an API answering 404 where 401 was meant, a skill that never names the credential, docs whose env vars do not exist, and a harness omission of ours that failed twelve baseline cells and read as a skills result. A pass rate cannot express any of those, and cannot separate "the agent could not" from "we misled it". So this turns the convention into a list, from fields the harness already records: unclean exit, failed-while-claiming-success, a declared credential never referenced, a skill offered and never opened, a baseline that self-installed a product skill, and a cell whose only green checks are the ones an idle agent satisfies. Three false positives were removed before it was worth trusting, because a triage tool that cries wolf returns us to nobody reading transcripts: - required credentials now come from each scenario's `requires`, not a blanket flag, which had faulted the ElevenLabs and Stripe scenarios for not using an Outpost key - self-install only flags a *product* skill in a *baseline* arm; a provider skill like stripe-webhooks is legitimate and AGENTS.md says so - "only negatives passed" now requires every green check to be negative, not any: an agent that re-enabled a destination but never recovered the held events did half the job, and calling that idle misdescribes it Against the current results it flags two cells: the one that produced findings #39 and #40, and a failed capability question answered with no tool calls — which is that scenario's whole point. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Nt2Zgjw7STjrnFXYKRRVAA --- AGENTS.md | 28 ++++ apps/framework/package.json | 1 + apps/framework/scripts/triage.ts | 257 +++++++++++++++++++++++++++++++ 3 files changed, 286 insertions(+) create mode 100644 apps/framework/scripts/triage.ts diff --git a/AGENTS.md b/AGENTS.md index 100f3fe..8a01b35 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -331,6 +331,34 @@ agent runs inside a container, which needs an API key the same way CI does. ## Conventions +**Every product finding this benchmark has produced came from a transcript. +None came from the scoreboard.** Two days of Outpost runs produced four: an API +answering `404` where `401` was meant, a skill that never names the credential, +a docs page whose environment variables do not exist, and a harness omission of +our own that failed twelve baseline cells and read as a skills result. A pass +rate cannot express any of them. It cannot even separate "the agent could not" +from "we misled it" — which is the difference between a to-do for the product +and a bug in our instrument. + +So the scoreboard says *which* cells to read and never *what happened*. Run + +```bash +pnpm --filter @hookdeck-evals/framework triage +``` + +after a run. It reads what the harness already records and flags the cells where +the number is not the whole story: an unclean exit (a killed container arrives +with a plausible partial score, and one was written as a `2/6` agent failure), a +run that failed while its own report claims success (nine of twelve baselines in +one run, every one of them confidently building on the wrong product), a +declared credential the agent never referenced, a skill offered and never +opened, a baseline that self-installed a product skill, and a cell whose only +green checks are the ones an idle agent satisfies. + +Nothing flagged is not the same as nothing to learn. It means no cell tripped a +signal the script knows about, and the signals it knows about are the ones that +have already cost us something. + **When a scorer disagrees with an agent, read the agent's report first.** It has been the cheapest answer every time and it keeps getting skipped. It named the credential it could not fetch, and the variable name it looked for diff --git a/apps/framework/package.json b/apps/framework/package.json index 788c661..34a41f3 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -14,6 +14,7 @@ "score-only": "node --env-file=../../.env --import tsx/esm scripts/score-only.ts", "compare-snapshots": "node --import tsx/esm scripts/compare-snapshots.ts", "report-results": "node --import tsx/esm scripts/report-results.ts", + "triage": "node --import tsx/esm scripts/triage.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" }, diff --git a/apps/framework/scripts/triage.ts b/apps/framework/scripts/triage.ts new file mode 100644 index 0000000..d787d91 --- /dev/null +++ b/apps/framework/scripts/triage.ts @@ -0,0 +1,257 @@ +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { discoverEvals, EVALS_ROOT } from '../lib/discovery.js'; + +/** + * Which cells of a run you have to read, and why. + * + * AGENTS.md already says to read the agent's report first, and calls it the + * cheapest answer every time — and it keeps getting skipped, including by + * whoever wrote that line. An exhortation loses to a scoreboard, because the + * scoreboard is right there and the transcripts are not. + * + * The case for reading them is stronger than "sometimes useful". Over two days + * of Outpost runs, **every** product finding came out of a transcript and none + * came out of the scoreboard: an API answering `404` where `401` was meant, a + * skill that never names the credential, docs whose environment variables do + * not exist, and — worst — a harness omission that made our own baseline fail + * twelve cells and read as a skills result. A pass rate cannot express any of + * those. It cannot even distinguish "the agent could not" from "we lied to it". + * + * So this turns the convention into a list. It reads what the harness already + * records and flags the cells where the number on the board is not the whole + * story. It does not judge the run; it tells you where to look. + * + * ```bash + * pnpm --filter @hookdeck-evals/framework triage + * pnpm --filter @hookdeck-evals/framework triage --require OUTPOST_API_KEY + * ``` + */ + +const RUNS_DIR = join(EVALS_ROOT, '.eval-runs'); + +interface Check { + name?: string; + passed?: boolean; +} + +interface Row { + experiment: string; + eval: string; + passed?: boolean; + checks?: Check[]; + stoppedReason?: string; + agentReport?: string; + toolCalls?: unknown[]; + skills?: { + available?: string[]; + loaded?: string[]; + selfInstalled?: string[]; + }; +} + +interface Flag { + label: string; + detail: string; +} + +/** + * Capabilities a scenario declares, mapped to the variable that carries them. + * + * Derived from each scenario's `requires`, not from a flag on the command line. + * A blanket `--require OUTPOST_API_KEY` flagged the ElevenLabs and Stripe + * scenarios for not using an Outpost credential they have no business touching, + * and a triage tool that cries wolf gets ignored — which returns us to nobody + * reading transcripts, the thing this exists to fix. + */ +const CAPABILITY_ENV: Record = { outpost: 'OUTPOST_API_KEY' }; + +/** + * Skills whose self-installation invalidates a baseline. + * + * Only these. AGENTS.md draws the line deliberately: a *product* skill pulled + * into a `-no-skills` run means that row is no longer a baseline, while a + * *provider* skill like `stripe-webhooks` is legitimate — documenting a third + * party's signature format was never Hookdeck's job, and we ship those skills + * for exactly this. Flagging every self-install buried the real case in noise. + */ +const PRODUCT_SKILLS = new Set(['hookdeck', 'event-gateway', 'outpost']); + +/** Phrases an agent uses when it believes it finished. */ +const SUCCESS_CLAIM = + /\b(everything (is )?(set up|working|verified)|successfully|all set|is now (set up|configured|working)|done!|completed successfully|verified end-to-end)\b/i; + +/** + * A clean stop. Anything else means the process died, and a dead run must not + * be read as an agent's answer — the guard in `run-eval.ts` only rejects an + * *empty* transcript, so a container killed mid-flight arrives here with 15 + * tool calls and a plausible partial score. One did, on 21 August, and was + * written as a `2/6` agent failure. + */ +const CLEAN_STOP = new Set(['stop', 'end_turn', 'complete']); + +function flagsFor(row: Row, requiredEnv: string[]): Flag[] { + const flags: Flag[] = []; + const blob = JSON.stringify(row); + const checks = row.checks ?? []; + const failed = checks.filter((c) => c.passed === false); + + if (row.stoppedReason && !CLEAN_STOP.has(row.stoppedReason)) { + flags.push({ + label: 'UNCLEAN EXIT', + detail: `stoppedReason=${row.stoppedReason} — the process did not finish, so this is not the agent's answer`, + }); + } + + // The most valuable signal in the whole file. An agent that fails while + // reporting success has done something coherent and wrong, which is where + // findings come from — nine of twelve baselines did exactly this in one run, + // having built the task on the wrong product. + if (row.passed === false && SUCCESS_CLAIM.test(row.agentReport ?? '')) { + flags.push({ + label: 'CLAIMED SUCCESS', + detail: 'failed while reporting the task complete — read this one first', + }); + } + + // A credential the scenario needs that the agent never referenced usually + // means it never found the thing, and that is often our fault rather than + // the model's: the variable may be injected and unannounced. + for (const name of requiredEnv) { + if (row.passed === false && !blob.includes(name)) { + flags.push({ + label: 'CREDENTIAL UNUSED', + detail: `${name} never appears — did the agent know it existed?`, + }); + } + } + + const available = row.skills?.available ?? []; + const loaded = row.skills?.loaded ?? []; + if (row.passed === false && available.length > 0 && loaded.length === 0) { + flags.push({ + label: 'SKILL NOT OPENED', + detail: `offered ${available.join(', ')} and loaded none — this measures selection, not content`, + }); + } + + const smuggled = (row.skills?.selfInstalled ?? []).filter((n) => + PRODUCT_SKILLS.has(n) + ); + if (smuggled.length > 0 && row.experiment.endsWith('-no-skills')) { + flags.push({ + label: 'BASELINE COMPROMISED', + detail: `fetched product skill ${smuggled.join(', ')} at run time — exclude this row from any skills delta`, + }); + } + + // Passing only negative checks is a do-nothing run wearing a partial score. + if ( + row.passed === false && + failed.length > 0 && + failed.length < checks.length + ) { + // *Every* green check must be a negative one, not merely some of them. + // Matching on any was wrong: a cell where the agent re-enabled a + // destination but never recovered the held events has one real positive + // green and one negative, and flagging it as idle misdescribes an agent + // that did half the job. The signal being hunted here is the run that did + // nothing and still scored. + const passedNames = checks.filter((c) => c.passed).map((c) => c.name ?? ''); + const isNegative = (n: string) => + /left alone|unchanged|untouched|no longer|not .*(disabled|removed)|was not/i.test( + n + ); + if (passedNames.length > 0 && passedNames.every(isNegative)) { + flags.push({ + label: 'ONLY NEGATIVES PASSED', + detail: `${checks.length - failed.length}/${checks.length} green, and they are the checks an idle agent satisfies`, + }); + } + } + + if (row.passed === false && (row.toolCalls?.length ?? 0) === 0) { + flags.push({ label: 'NO TOOL CALLS', detail: 'scored without acting' }); + } + + return flags; +} + +function main() { + const args = process.argv.slice(2); + void args; + + // What each scenario declares it needs, by eval id. + const needs = new Map(); + for (const ev of discoverEvals()) { + const requires = + (ev.metadata as { requires?: string[] } | undefined)?.requires ?? []; + const env = requires + .map((r) => CAPABILITY_ENV[r]) + .filter((v): v is string => Boolean(v)); + if (env.length > 0) needs.set(ev.id, env); + } + + if (!existsSync(RUNS_DIR)) throw new Error(`no ${RUNS_DIR}`); + + const rows: Array<{ row: Row; flags: Flag[] }> = []; + for (const dir of readdirSync(RUNS_DIR)) { + const experimentDir = join(RUNS_DIR, dir); + let files: string[] = []; + try { + files = readdirSync(experimentDir).filter((f) => f.endsWith('.json')); + } catch { + continue; + } + for (const file of files) { + const row = JSON.parse( + readFileSync(join(experimentDir, file), 'utf8') + ) as Row; + const flags = flagsFor(row, needs.get(row.eval) ?? []); + if (flags.length > 0) rows.push({ row, flags }); + } + } + + const total = readdirSync(RUNS_DIR).reduce((n, d) => { + try { + return ( + n + + readdirSync(join(RUNS_DIR, d)).filter((f) => f.endsWith('.json')).length + ); + } catch { + return n; + } + }, 0); + + console.log(`${total} cell(s) on disk; ${rows.length} worth reading.\n`); + if (rows.length === 0) { + console.log( + ' Nothing flagged. That is not the same as nothing to learn —' + ); + console.log(' it means no cell tripped a signal this script knows about.'); + return; + } + + // Claimed-success first: it is the one that has produced findings. + rows.sort((a, b) => { + const rank = (f: Flag[]) => + f.some((x) => x.label === 'CLAIMED SUCCESS') + ? 0 + : f.some((x) => x.label === 'UNCLEAN EXIT') + ? 1 + : 2; + return rank(a.flags) - rank(b.flags); + }); + + for (const { row, flags } of rows) { + console.log(` ${row.eval} x ${row.experiment}`); + for (const flag of flags) { + console.log(` ${flag.label}: ${flag.detail}`); + } + console.log( + ` → .eval-runs/${row.experiment}/${row.eval}.json (agentReport, toolCalls)\n` + ); + } +} + +main(); From 9b67fedf2bafafe9183adda05e4337877d8d3d22 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 24 Aug 2026 09:31:24 +0100 Subject: [PATCH 6/9] Fix outpost-005: it could not report its own worst failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three faults, found by auditing the scorers rather than by a run. **It threw on the failures it exists to catch.** The sanity guard fired whenever acme received nothing — which is exactly what deleting the destination, disabling it, or emptying its topics produces. The checks array describing those failures was built and then discarded, the throw propagated, and no result row was written at all. checkNothingDisabled existed to name a route the scorer then threw away. Now only a platform fault throws: acme has an enabled destination subscribed to the topic and still received nothing. If nothing arrived because no destination subscribes any more, the agent did that and the checks say so. Verified: disable now scores 2/5 naming the switched-off destination, delete scores 1/5, and empty topics is rejected by the API. **A vanished tenant crashed the read.** Deleting a tenant's last destination deletes the tenant, so every later read 404s. `list` now treats 404 as an empty read — and only 404, because swallowing everything reports a confident agent failure for a platform blip. **The collateral check failed on the runs where it mattered.** Globex shared one publish with acme, so its event was 45 seconds old whenever acme's wait ran to timeout — precisely when the agent had broken acme. Measured: globex passed with a correct fix and failed twice on the delete and disable paths, while delivering fine when tested alone. The exact interaction was never pinned down and this does not guess at it; globex now gets its own publish and its own window, so acme's timing cannot decide whether globex looks untouched. Accusing an agent of breaking a customer it never touched is the worst false failure here. Also: the header justified behaviour-scoring with a lever that does not exist — it claimed an agent could edit a tenant's topic list. Tenant topics are read-only and derived from destinations, verified live: a PUT with topics returns [], and adding a destination on order.shipped makes it ["order.shipped"]. The tenants[].topics field in every seed here is inert. The real reason is per-destination filters, which config-shaped scoring would miss. Polling widened from 1s to 3s: two back-to-back waits at one second each issue well over a hundred requests per cell, and waitFor treats a failed probe as "not ready", so a rate-limited read is indistinguishable from nothing arriving. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Nt2Zgjw7STjrnFXYKRRVAA --- apps/framework/t5b.ts | 36 +++++ .../EVAL.ts | 130 +++++++++++++----- 2 files changed, 128 insertions(+), 38 deletions(-) create mode 100644 apps/framework/t5b.ts diff --git a/apps/framework/t5b.ts b/apps/framework/t5b.ts new file mode 100644 index 0000000..082d0d7 --- /dev/null +++ b/apps/framework/t5b.ts @@ -0,0 +1,36 @@ +import { + discoverEvals, + loadExperiments, + readSessionSeedArgs, +} from './lib/discovery.js'; +const ev = discoverEvals().find( + (e) => e.id === 'benchmark-outpost-005-topic-scoping' +)!; +const runtime = ((await loadExperiments())[0].config as any).runtime; +const un = (r: any) => (Array.isArray(r) ? r : (r?.models ?? [])); +const scorer = (await import(ev.scorerPath ?? `${ev.dir}/EVAL.ts`)).default; +for (const mode of ['delete', 'correct'] as const) { + const s = await runtime.startSession(readSessionSeedArgs(ev)); + const ctx = s.scoringContext; + try { + for (const d of un( + await ctx.outpost('GET', '/tenants/acme/destinations') + )) { + if (mode === 'delete') + await ctx.outpost('DELETE', `/tenants/acme/destinations/${d.id}`); + else + await ctx.outpost('PATCH', `/tenants/acme/destinations/${d.id}`, { + topics: ['order.created', 'order.shipped'], + }); + } + const r = await scorer(ctx); + console.log( + `${mode}: passed=${r.passed} (${r.checks.filter((c: any) => c.passed).length}/${r.checks.length})` + ); + for (const c of r.checks) if (!c.passed) console.log(' FAIL', c.name); + } catch (e: any) { + console.log(`${mode}: THREW -> ${String(e.message).slice(0, 80)}`); + } finally { + await s.close(); + } +} diff --git a/evals/benchmark-outpost-005-topic-scoping/EVAL.ts b/evals/benchmark-outpost-005-topic-scoping/EVAL.ts index 54907dd..4b84596 100644 --- a/evals/benchmark-outpost-005-topic-scoping/EVAL.ts +++ b/evals/benchmark-outpost-005-topic-scoping/EVAL.ts @@ -26,11 +26,19 @@ import { waitForSettled } from '@hookdeck-evals/hookdeck'; * broadly than asked. The difference is that here it is the *cheapest* way to * satisfy the request, rather than a mistake you have to reach for. * - * Scored on behaviour, not configuration. A destination's `topics` array can be - * right while delivery is wrong — a tenant's own topic list gates it too, and an - * agent that edits the tenant instead of the destination reaches the same end - * state by another route. So this publishes real events and checks what - * arrives, which passes any correct route and fails any incorrect one. + * Scored on behaviour, not configuration, and the reason matters because the + * first version of this comment gave a fictional one. It claimed a tenant's own + * topic list gates delivery and that an agent could edit that instead — + * `Tenant.topics` is read-only and derived from its destinations, verified + * live: a `PUT` with topics returns `[]`, and adding a destination on + * `order.shipped` makes it `["order.shipped"]`. (The `tenants[].topics` field + * in every seed in this repo is therefore inert.) + * + * The real reason is Outpost's per-destination `filter`, which can suppress an + * event that the `topics` array admits. A scorer reading configuration would + * pass a destination whose topics look right and whose filter drops everything. + * Publishing real events and checking what arrives arrives at the truth + * whichever lever the agent used. */ const ACME = 'acme'; @@ -46,6 +54,18 @@ const UNWANTED = 'order.cancelled'; /** Long enough that "it did not arrive" means it is not coming. */ const DELIVERY_WAIT_MS = 45_000; const SETTLE_MS = 8_000; +/** + * Deliberately slower than the 1s default. + * + * Each poll costs one events read plus one attempts read per destination, and + * two waits run back to back. At 1s that is well over a hundred requests for a + * single cell, and `waitFor` treats a failed probe as "not ready yet" — so a + * rate-limited read is indistinguishable from nothing having arrived, and + * surfaces as a false failure on whichever check is being measured. Observed + * once on the collateral check, which is the worst place for it: it accuses an + * agent of breaking a customer it never touched. + */ +const POLL_INTERVAL_MS = 3_000; interface Attempt { id?: string; @@ -55,6 +75,7 @@ interface Attempt { interface Destination { id?: string; + topics?: string[]; disabled_at?: string | null; } @@ -69,13 +90,9 @@ const scorer: ToolScorer = async (ctx) => { // Published together, then read once after they have all had time to land. // Sending the positives and the negative separately would let a slow negative // arrive after its own check had already passed. - const before = await attemptCount(ctx, ACME); - const globexBefore = await attemptCount(ctx, GLOBEX); - await publish(ctx, ACME, KEPT); await publish(ctx, ACME, UNMENTIONED); await publish(ctx, ACME, UNWANTED); - await publish(ctx, GLOBEX, UNWANTED); // Two must arrive and one must not, so the positives starting the clock is // what gives the negative its chance to be wrong. Reading the moment the @@ -86,16 +103,34 @@ const scorer: ToolScorer = async (ctx) => { { timeoutMs: DELIVERY_WAIT_MS, settleMs: SETTLE_MS, + intervalMs: POLL_INTERVAL_MS, description: "acme's remaining topics to be delivered", } ); + // Globex is probed *after* acme's wait, not alongside it. + // + // Sharing one publish meant globex's event was already 45 seconds old by the + // time it was measured whenever acme's wait ran to timeout — which is exactly + // the case where the agent broke acme, so the collateral check failed on the + // runs where it mattered most. Measured: with a correct fix acme resolves + // fast and globex passed 5/5; on the delete and disable paths globex failed + // twice, while delivering perfectly well when tested on its own. + // + // The precise interaction was never pinned down, and this does not attempt + // to. It removes the coupling instead: each tenant gets its own publish and + // its own window, so how long acme takes cannot decide whether globex looks + // untouched. Accusing an agent of breaking a customer it never touched is the + // worst false failure this scorer could produce. + await publish(ctx, GLOBEX, UNWANTED); + const globex = await waitForSettled( () => deliveredTopics(ctx, GLOBEX), (topics) => topics.has(UNWANTED), { timeoutMs: DELIVERY_WAIT_MS, settleMs: SETTLE_MS, + intervalMs: POLL_INTERVAL_MS, description: "globex's delivery to be unaffected", } ); @@ -136,15 +171,37 @@ const scorer: ToolScorer = async (ctx) => { await checkNothingDisabled(ctx), ]; - // Sanity, not a check: if the tenant received nothing at all, the failures - // above are about the harness rather than the agent. - if (acme.size === 0 && (await attemptCount(ctx, ACME)) === before) { - throw new Error( - 'no delivery attempts on acme at all after publishing: the seed or the ' + - 'platform is at fault, not the agent, so this run is not scoreable' + // A platform fault throws; an agent fault is scored. Telling them apart needs + // the configuration, not the delivery count. + // + // The first version threw whenever acme received nothing — which is exactly + // what deleting the destination, disabling it, or setting `topics: []` + // produces. Those are the worst things an agent can do here, and the throw + // discarded the `checks` array that described them, so no result row was + // written at all: the scenario structurally could not report its own most + // severe failures. `checkNothingDisabled` existed to name a route the scorer + // then threw away. + // + // So only throw when the configuration says acme *should* have received + // something and nothing arrived. That is the seed or the platform. If nothing + // arrived because no enabled destination subscribes to the topic any more, + // the agent did that, and the checks above already say so. + if (acme.size === 0) { + const live = (await listDestinations(ctx, ACME)).filter( + (d) => !d.disabled_at ); + const shouldHaveArrived = live.some((d) => { + const topics = d.topics ?? []; + return topics.includes('*') || topics.includes(KEPT); + }); + if (shouldHaveArrived) { + throw new Error( + `acme has an enabled destination subscribed to ${KEPT} and received nothing ` + + 'after publishing: the seed or the platform is at fault, not the agent, ' + + 'so this run is not scoreable' + ); + } } - void globexBefore; return { passed: checks.every((c) => c.passed), checks }; }; @@ -202,23 +259,6 @@ async function deliveredTopics( return topics; } -async function attemptCount( - ctx: ToolEvalContext, - tenant: string -): Promise { - let total = 0; - for (const destination of await listDestinations(ctx, tenant)) { - if (!destination.id) continue; - total += ( - await list( - ctx, - `/tenants/${encodeURIComponent(tenant)}/destinations/${encodeURIComponent(destination.id)}/attempts` - ) - ).length; - } - return total; -} - async function publish( ctx: ToolEvalContext, tenant: string, @@ -241,12 +281,26 @@ async function listDestinations( ); } -/** Outpost list endpoints answer `{ pagination, models }`, not `{ data }`. */ +/** + * Outpost list endpoints answer `{ pagination, models }`, not `{ data }`. + * + * A `404` is an empty read rather than a failure, and only a `404`. Deleting a + * tenant's last destination deletes the tenant with it, so an agent that + * removes the destination — one of the wrong routes this scenario scores — + * makes every subsequent read 404. Letting that propagate threw the whole cell + * away and wrote no result, which is the same defect the throw above was just + * repaired for. Anything else still propagates, because a scorer that treats + * every error as "nothing there" reports a confident agent failure for a + * platform blip. + */ async function list(ctx: ToolEvalContext, path: string): Promise { - const rows = await ctx.outpost?.( - 'GET', - path - ); + let rows: T[] | { models?: T[]; data?: T[] } | undefined; + try { + rows = await ctx.outpost?.('GET', path); + } catch (error) { + if (String((error as Error).message).includes('404')) return []; + throw error; + } if (!rows) return []; if (Array.isArray(rows)) return rows; return rows.models ?? rows.data ?? []; From 38398afae3c82d6fccd02f16130d254c86a5a35f Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 24 Aug 2026 09:37:01 +0100 Subject: [PATCH 7/9] Fix outpost-001: it could be passed entirely by a leftover tenant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was the only Outpost scenario with no seed, so nothing removed `acme` before the agent ran — and every other Outpost scenario seeds a tenant called `acme` carrying a destination on an order-ish topic. Tenant cleanup runs on release inside a catch-and-ignore and is skipped when a run is killed, both of which have happened. A survivor satisfied all three checks with no agent action and published green against a named vendor. project-source.ts argues the residual leak is tolerable because it makes this scenario's first check pass visibly. That has it backwards: it makes every check pass, and nothing about the row looks wrong. Fixed in the seed, with a new `deleteTenants`, because the state has to be absent rather than distinguishable. Scoping by `created_at >= acquiredAt` was tried first and fails the correct answer: tenant create is idempotent, so an agent that properly PUTs an existing id gets the original timestamp back and is scored as inheriting a leftover. Measured — the tenant read two minutes older than the lease about to score it. Two more faults in the same file: - "an order event reaches the customer" counted attempts of any status, so a destination pointed at an invented hostname passed while the customer received nothing. Now counts successful attempts. outpost-004's header claims delivery is "already proven by outpost-001"; it was not. - listDestinations read `{ data }` alone — the trap this file's own comments warn about twice — surviving only because that endpoint is unpaged and returns a bare array. The residual, called out in the code rather than fixed quietly: the ticket never says where the customer's endpoint is, so any reachable URL passes. One agent stood up a localtunnel inside its own sandbox, delivered to itself, passed, and offered to "swap the temporary receiver URL for the real customer endpoint next". Fixing that means giving the ticket an endpoint, which changes a published scenario. Adds the SOLUTION.ts this scenario never had, which is why score-only could only ever exercise its failing path — and how the created_at mistake was caught before it shipped. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Nt2Zgjw7STjrnFXYKRRVAA --- .../EVAL.ts | 68 +++++++++++++++++-- .../SOLUTION.ts | 35 ++++++++++ .../remote/seed.json | 5 ++ packages/hookdeck/src/seed.ts | 22 ++++++ 4 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 evals/benchmark-outpost-001-customer-subscriptions/SOLUTION.ts create mode 100644 evals/benchmark-outpost-001-customer-subscriptions/remote/seed.json diff --git a/evals/benchmark-outpost-001-customer-subscriptions/EVAL.ts b/evals/benchmark-outpost-001-customer-subscriptions/EVAL.ts index 5c2cc07..24872d0 100644 --- a/evals/benchmark-outpost-001-customer-subscriptions/EVAL.ts +++ b/evals/benchmark-outpost-001-customer-subscriptions/EVAL.ts @@ -45,6 +45,19 @@ const scorer: ToolScorer = async (ctx) => { ); } + // A leftover `acme` used to satisfy every check here with no agent action. + // This was the only Outpost scenario with no seed, and every other one seeds + // a tenant called `acme` with a destination on an order-ish topic; tenant + // cleanup runs on release inside a `catch`-and-ignore and is skipped when a + // run is killed, both of which have happened. The row published green. + // + // The fix is in the seed — `deleteTenants: ["acme", "globex"]` — because the + // state has to be *absent*, not merely distinguishable. Comparing the + // tenant's `created_at` against the lease was tried first and does not work: + // tenant create is idempotent, so an agent that correctly `PUT`s an existing + // id gets the original timestamp back and is scored as having inherited + // someone else's work. Measured on 24 August, the tenant read two minutes + // older than the lease about to score it. const tenants = await listTenants(ctx); const tenant = tenants.find((t) => /acme/i.test(String(t.id ?? ''))); @@ -110,7 +123,23 @@ async function checkOrderEventDelivered( }; } - const before = await attemptCount(ctx, tenantId, destinations); + // Successful attempts, not attempts. + // + // Counting any attempt made this check "Outpost tried", which is not what its + // name claims. A destination pointed at a hostname the agent invented records + // an attempt and fails to deliver, and the customer receives nothing. The + // header of `outpost-004` asserts "delivery is already proven against webhook + // destinations by outpost-001" — it was not. + // + // **This does not close the whole hole, and the remaining half is a scenario + // problem rather than a scorer one.** The ticket never says where the + // customer's endpoint is, so any reachable URL satisfies it: one agent stood + // up a localtunnel *inside its own sandbox*, delivered to itself, passed, and + // offered to "swap the temporary receiver URL for the real customer endpoint + // next". That receiver died with the container. Fixing it means giving the + // ticket an endpoint to deliver to, which changes a published scenario, so it + // is called out here rather than done quietly. + const before = await successCount(ctx, tenantId, destinations); await ctx.outpost?.('POST', '/publish', { tenant_id: tenantId, topic, @@ -120,7 +149,7 @@ async function checkOrderEventDelivered( // attempt count rather than sleeping and reading once. A single positive // assertion, so the first observation that satisfies it is the answer. const after = await waitForOrLast( - () => attemptCount(ctx, tenantId, destinations), + () => successCount(ctx, tenantId, destinations), (count) => count > before, { timeoutMs: DELIVERY_WAIT_MS, @@ -161,6 +190,28 @@ function orderTopic( return undefined; } +/** Attempts that actually delivered. */ +async function successCount( + ctx: ToolEvalContext, + tenantId: string, + destinations: Record[] +): Promise { + let total = 0; + for (const destination of destinations) { + const id = String(destination.id ?? ''); + if (!id) continue; + const body = await ctx.outpost?.< + { status?: string }[] | { models?: { status?: string }[] } + >( + 'GET', + `/tenants/${encodeURIComponent(tenantId)}/destinations/${encodeURIComponent(id)}/attempts` + ); + const rows = Array.isArray(body) ? body : (body?.models ?? []); + total += rows.filter((a) => a.status === 'success').length; + } + return total; +} + async function attemptCount( ctx: ToolEvalContext, tenantId: string, @@ -202,8 +253,17 @@ async function listDestinations( ctx: ToolEvalContext, tenantId: string ): Promise[]> { + // `models ?? data`, matching every sibling scorer and the client. This file + // read `data` alone — the exact trap its own comments warn about twice, which + // survives only because this endpoint happens to be unpaged and returns a + // bare array. The day it gains an envelope, every agent is told the tenant + // has nowhere to deliver to. const body = await ctx.outpost?.< - Record[] | { data?: Record[] } + | Record[] + | { + models?: Record[]; + data?: Record[]; + } >('GET', `/tenants/${encodeURIComponent(tenantId)}/destinations`); - return Array.isArray(body) ? body : (body?.data ?? []); + return Array.isArray(body) ? body : (body?.models ?? body?.data ?? []); } diff --git a/evals/benchmark-outpost-001-customer-subscriptions/SOLUTION.ts b/evals/benchmark-outpost-001-customer-subscriptions/SOLUTION.ts new file mode 100644 index 0000000..5853316 --- /dev/null +++ b/evals/benchmark-outpost-001-customer-subscriptions/SOLUTION.ts @@ -0,0 +1,35 @@ +import type { ToolEvalContext } from '@hookdeck-evals/core'; + +/** + * What a correct agent leaves behind: a tenant for the customer, a destination + * subscribed to order events, and a delivery that actually arrives. + * + * Added late, and for a specific reason. This scenario had no solution, so + * `score-only` could only ever exercise its failing path — which is how it kept + * a check that passed on a *leftover* tenant and another that counted a + * delivery *attempt* rather than a delivery. Both were repaired on 24 August, + * and the repair for the first compares the tenant's `created_at` against the + * lease. That comparison is worth testing before trusting: clock skew between + * this machine and Outpost would reject a tenant an agent had just made. + */ + +const TENANT = 'acme'; + +export default async function solve(ctx: ToolEvalContext): Promise { + const outpost = ctx.outpost; + if (!outpost) { + throw new Error( + 'no Outpost client: this solution cannot be applied without OUTPOST_API_KEY' + ); + } + + await outpost('PUT', `/tenants/${TENANT}`, {}); + + await outpost('POST', `/tenants/${TENANT}/destinations`, { + type: 'webhook', + topics: ['order.created'], + // A reachable endpoint, because the check requires the event to arrive + // rather than merely to be attempted. + config: { url: 'https://mock.hookdeck.com/api/v1/acme/orders' }, + }); +} diff --git a/evals/benchmark-outpost-001-customer-subscriptions/remote/seed.json b/evals/benchmark-outpost-001-customer-subscriptions/remote/seed.json new file mode 100644 index 0000000..ba798db --- /dev/null +++ b/evals/benchmark-outpost-001-customer-subscriptions/remote/seed.json @@ -0,0 +1,5 @@ +{ + "outpost": { + "deleteTenants": ["acme", "globex"] + } +} diff --git a/packages/hookdeck/src/seed.ts b/packages/hookdeck/src/seed.ts index 9df8a66..c6a5f0b 100644 --- a/packages/hookdeck/src/seed.ts +++ b/packages/hookdeck/src/seed.ts @@ -84,6 +84,22 @@ export interface OutpostSeed { * so this removes them for the run rather than for good. */ clearOperatorEventDestinations?: boolean; + /** + * Tenants to delete before the agent runs, and not recreate. + * + * For scenarios that score an agent for *creating* something. `outpost-001` + * asks for a tenant to be set up and had no seed at all, so a tenant left by + * any other Outpost scenario — they all use `acme` — satisfied every check + * with no agent action, and the row published green. + * + * Scoping the check by `created_at` instead does not work: tenant create is + * idempotent, so an agent that correctly `PUT`s an existing id gets the + * original timestamp back and is scored as having inherited a leftover. + * Measured on 24 August — the tenant read two minutes older than the lease + * that was about to score it. The state has to be absent, not merely + * distinguishable. + */ + deleteTenants?: string[]; tenants?: { id: string; topics?: string[]; @@ -352,6 +368,12 @@ export async function applyOutpostSeed( } } + for (const id of seed.deleteTenants ?? []) { + await outpost('DELETE', `/tenants/${encodeURIComponent(id)}`).catch( + () => undefined + ); + } + if (seed.clearOperatorEventDestinations) { await clearOperatorEventDestinations(outpost); } From 9f7dd10603b4df49d21064f36fa101637e76c128 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 24 Aug 2026 14:06:07 +0100 Subject: [PATCH 8/9] Set the trap outpost-005 always claimed, and stop the addendum leaking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **The addendum was gated on the machine, not the scenario.** `outpostClient` is built whenever OUTPOST_API_KEY is set, which is always, so every Event Gateway scenario was being told about an Outpost project it has no use for. That is noise in fourteen prompts to answer a question about five, and it would have made every published cell non-comparable rather than just the Outpost ones — which is the difference between a patch release and a full re-run. Now gated on the scenario's seed declaring Outpost state; verified that outpost-002 sees the sentence and filtering-001 does not. **outpost-005's ticket never set its trap.** The scorer described the failure as "set their destination to the topic they still talk about", but the ticket named no topic acme wanted — only the one to remove — so narrowing was not a candidate action and the wrong answer took more work than the right one. It also said "everything else they get today should carry on exactly as it is, and Globex shouldn't be affected at all", which states four of the five checks and replaces the silence the scenario depends on with an instruction to enumerate and preserve. All six agents passed. The ticket now names order confirmations as working and relied upon, says nothing about shipping events, and does not mention Globex. Verified: the naive fix — scoping to the named keeper — scores 4/5, failing only "acme still receives the shipping events they depend on". The lesson is in the header: verifying that a scorer rejects a hand-written wrong answer proves the scorer works, not that agents make that mistake. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Nt2Zgjw7STjrnFXYKRRVAA --- apps/framework/v5.ts | 18 ++++++++++++++++++ .../EVAL.ts | 11 +++++++++++ .../PROMPT.md | 9 ++++----- packages/hookdeck/src/runtime.ts | 17 +++++++++++++++-- 4 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 apps/framework/v5.ts diff --git a/apps/framework/v5.ts b/apps/framework/v5.ts new file mode 100644 index 0000000..6f0d94f --- /dev/null +++ b/apps/framework/v5.ts @@ -0,0 +1,18 @@ +import { discoverEvals, loadExperiments, readSessionSeedArgs } from './lib/discovery.js'; +import { pathToFileURL } from 'node:url'; +const ev = discoverEvals().find((e) => e.id === 'benchmark-outpost-005-topic-scoping')!; +const runtime = (((await loadExperiments())[0]).config as any).runtime; +const un = (r: any) => Array.isArray(r) ? r : (r?.models ?? []); +const scorer = (await import(ev.scorerPath ?? `${ev.dir}/EVAL.ts`)).default; +for (const [label, topics] of [['naive (only the named keeper)', ['order.created']], ['correct', ['order.created','order.shipped']]] as const) { + const s = await runtime.startSession(readSessionSeedArgs(ev)); + const ctx = s.scoringContext; + try { + for (const d of un(await ctx.outpost('GET', '/tenants/acme/destinations'))) { + await ctx.outpost('PATCH', `/tenants/acme/destinations/${d.id}`, { topics }); + } + const r = await scorer(ctx); + console.log(`${label}: passed=${r.passed} (${r.checks.filter((c:any)=>c.passed).length}/${r.checks.length})`); + for (const c of r.checks) if (!c.passed) console.log(' FAIL', c.name); + } finally { await s.close(); } +} diff --git a/evals/benchmark-outpost-005-topic-scoping/EVAL.ts b/evals/benchmark-outpost-005-topic-scoping/EVAL.ts index 4b84596..f04371b 100644 --- a/evals/benchmark-outpost-005-topic-scoping/EVAL.ts +++ b/evals/benchmark-outpost-005-topic-scoping/EVAL.ts @@ -22,6 +22,17 @@ import { waitForSettled } from '@hookdeck-evals/hookdeck'; * mentioned because it was working. Nobody sees an error. The customer notices * days later, when something they depend on has quietly stopped arriving. * + * The ticket had to be rewritten to make that true. Its first version added + * "everything else they get today should carry on exactly as it is, and Globex + * shouldn't be affected at all" — which states four of the five checks, and + * replaces the silence the scenario depends on with an instruction to enumerate + * and preserve. It also never named a topic acme *wanted*, so "narrow to the + * one they talk about" was not a candidate action and the wrong answer took + * more work than the right one. All six agents passed. The trap was described + * here and never set in the prompt, which is a thing to check for: verifying a + * scorer rejects a hand-written wrong answer proves the scorer works, not that + * agents make that mistake. + * * That is the same failure mode as `resolve-002` and `alerting-001`: acting more * broadly than asked. The difference is that here it is the *cheapest* way to * satisfy the request, rather than a mistake you have to reach for. diff --git a/evals/benchmark-outpost-005-topic-scoping/PROMPT.md b/evals/benchmark-outpost-005-topic-scoping/PROMPT.md index 110ca19..ed452cc 100644 --- a/evals/benchmark-outpost-005-topic-scoping/PROMPT.md +++ b/evals/benchmark-outpost-005-topic-scoping/PROMPT.md @@ -10,12 +10,11 @@ requires: - outpost extra_skills: - outpost -motivation: Every Outpost customer subscribes to a different slice of the same event stream, and narrowing one customer's slice is the most common change a support engineer makes. The obvious fix is too narrow — it stops the events they complained about and the ones they still depend on — and it fails silently, because nothing errors and the customer only notices what stops arriving days later. +motivation: Every Outpost customer subscribes to a different slice of the same event stream, and narrowing one customer's slice is the most common change a support engineer makes. The ticket names the topic to stop and one topic to keep, so scoping to the named keeper is the obvious fix — and it silently drops a third topic nobody mentioned, because it was working. Nothing errors; the customer notices days later. --- Acme have been in touch. Their integration is choking on order cancellations — -they don't handle them and each one throws an error on their side. They want us -to stop sending those. +they don't handle them and each one throws an error on their side. Their order +confirmations are working fine and they rely on those. -They were clear that everything else they get today should carry on exactly as -it is, and Globex shouldn't be affected at all. +Stop the cancellations. diff --git a/packages/hookdeck/src/runtime.ts b/packages/hookdeck/src/runtime.ts index 97ae061..5f66528 100644 --- a/packages/hookdeck/src/runtime.ts +++ b/packages/hookdeck/src/runtime.ts @@ -56,6 +56,19 @@ export function hookdeckRuntime(options: HookdeckRuntimeOptions): EvalRuntime { // as skipped rather than failing inside a check on a machine that has // no Outpost project. const outpostKey = process.env.OUTPOST_API_KEY; + // Does *this scenario* involve Outpost, as opposed to this machine + // merely having a key? + // + // Gating the addendum on the client existing was wrong: the client is + // built whenever OUTPOST_API_KEY is set, which is always, so every + // Event Gateway scenario was being told about an Outpost project it has + // no use for — noise in fourteen prompts to answer a question about + // five, and it would have made every published cell non-comparable + // rather than just the Outpost ones. + const scenarioSeed = args.remoteDir + ? readSeed(args.remoteDir) + : undefined; + const scenarioUsesOutpost = Boolean(scenarioSeed?.outpost); const outpostClient = outpostKey ? new OutpostClient({ apiKey: outpostKey }) : undefined; @@ -68,7 +81,7 @@ export function hookdeckRuntime(options: HookdeckRuntimeOptions): EvalRuntime { // here — so arriving without a client means the requirement is missing, // not that the machine is simply unconfigured. if (args.remoteDir) { - const seed = readSeed(args.remoteDir); + const seed = scenarioSeed; if (seed?.outpost) { if (!outpostClient) { throw new Error( @@ -150,7 +163,7 @@ export function hookdeckRuntime(options: HookdeckRuntimeOptions): EvalRuntime { // A developer using Outpost knows they use it and has the key in // their environment; nobody learns their own credentials by // enumerating env vars. - ...(outpostClient + ...(outpostClient && scenarioUsesOutpost ? [ 'You also have a Hookdeck Outpost project, and ' + 'OUTPOST_API_KEY is set in your environment. It is an ' + From 5e99dc0277c8497f1d70807cd52e4dfa2fe8ea10 Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 24 Aug 2026 14:07:45 +0100 Subject: [PATCH 9/9] Remove two probe scripts committed by accident `v5.ts` and `t5b.ts` are throwaway verification scripts. They have to live inside apps/framework to resolve its imports, so they are not covered by the temp-directory convention, and both were swept in by `git add -A` after the command that would have deleted them timed out. CI caught the second one on formatting; the first had been sitting in the branch for two commits because it happened to be formatted correctly. Scratch files are now `*.probe.ts` and gitignored, so the next one cannot be added by accident rather than relying on remembering to delete it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Nt2Zgjw7STjrnFXYKRRVAA --- .gitignore | 6 ++++++ apps/framework/t5b.ts | 36 ------------------------------------ apps/framework/v5.ts | 18 ------------------ 3 files changed, 6 insertions(+), 54 deletions(-) delete mode 100644 apps/framework/t5b.ts delete mode 100644 apps/framework/v5.ts diff --git a/.gitignore b/.gitignore index 4ab9a96..21f0052 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,9 @@ dist/ # Agent worktrees. Real git repos; never part of this one. .claude/worktrees/ + +# Throwaway probe scripts. They have to live inside a workspace to resolve its +# imports, so they cannot go in a temp directory — and two were committed by a +# `git add -A` after the command that would have deleted them timed out. Name +# scratch files `*.probe.ts` and they cannot be added by accident. +*.probe.ts diff --git a/apps/framework/t5b.ts b/apps/framework/t5b.ts deleted file mode 100644 index 082d0d7..0000000 --- a/apps/framework/t5b.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { - discoverEvals, - loadExperiments, - readSessionSeedArgs, -} from './lib/discovery.js'; -const ev = discoverEvals().find( - (e) => e.id === 'benchmark-outpost-005-topic-scoping' -)!; -const runtime = ((await loadExperiments())[0].config as any).runtime; -const un = (r: any) => (Array.isArray(r) ? r : (r?.models ?? [])); -const scorer = (await import(ev.scorerPath ?? `${ev.dir}/EVAL.ts`)).default; -for (const mode of ['delete', 'correct'] as const) { - const s = await runtime.startSession(readSessionSeedArgs(ev)); - const ctx = s.scoringContext; - try { - for (const d of un( - await ctx.outpost('GET', '/tenants/acme/destinations') - )) { - if (mode === 'delete') - await ctx.outpost('DELETE', `/tenants/acme/destinations/${d.id}`); - else - await ctx.outpost('PATCH', `/tenants/acme/destinations/${d.id}`, { - topics: ['order.created', 'order.shipped'], - }); - } - const r = await scorer(ctx); - console.log( - `${mode}: passed=${r.passed} (${r.checks.filter((c: any) => c.passed).length}/${r.checks.length})` - ); - for (const c of r.checks) if (!c.passed) console.log(' FAIL', c.name); - } catch (e: any) { - console.log(`${mode}: THREW -> ${String(e.message).slice(0, 80)}`); - } finally { - await s.close(); - } -} diff --git a/apps/framework/v5.ts b/apps/framework/v5.ts deleted file mode 100644 index 6f0d94f..0000000 --- a/apps/framework/v5.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { discoverEvals, loadExperiments, readSessionSeedArgs } from './lib/discovery.js'; -import { pathToFileURL } from 'node:url'; -const ev = discoverEvals().find((e) => e.id === 'benchmark-outpost-005-topic-scoping')!; -const runtime = (((await loadExperiments())[0]).config as any).runtime; -const un = (r: any) => Array.isArray(r) ? r : (r?.models ?? []); -const scorer = (await import(ev.scorerPath ?? `${ev.dir}/EVAL.ts`)).default; -for (const [label, topics] of [['naive (only the named keeper)', ['order.created']], ['correct', ['order.created','order.shipped']]] as const) { - const s = await runtime.startSession(readSessionSeedArgs(ev)); - const ctx = s.scoringContext; - try { - for (const d of un(await ctx.outpost('GET', '/tenants/acme/destinations'))) { - await ctx.outpost('PATCH', `/tenants/acme/destinations/${d.id}`, { topics }); - } - const r = await scorer(ctx); - console.log(`${label}: passed=${r.passed} (${r.checks.filter((c:any)=>c.passed).length}/${r.checks.length})`); - for (const c of r.checks) if (!c.passed) console.log(' FAIL', c.name); - } finally { await s.close(); } -}