diff --git a/.gitignore b/.gitignore index 4ab9a96..21f0052 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,9 @@ dist/ # Agent worktrees. Real git repos; never part of this one. .claude/worktrees/ + +# Throwaway probe scripts. They have to live inside a workspace to resolve its +# imports, so they cannot go in a temp directory — and two were committed by a +# `git add -A` after the command that would have deleted them timed out. Name +# scratch files `*.probe.ts` and they cannot be added by accident. +*.probe.ts diff --git a/AGENTS.md b/AGENTS.md index 100f3fe..8a01b35 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -331,6 +331,34 @@ agent runs inside a container, which needs an API key the same way CI does. ## Conventions +**Every product finding this benchmark has produced came from a transcript. +None came from the scoreboard.** Two days of Outpost runs produced four: an API +answering `404` where `401` was meant, a skill that never names the credential, +a docs page whose environment variables do not exist, and a harness omission of +our own that failed twelve baseline cells and read as a skills result. A pass +rate cannot express any of them. It cannot even separate "the agent could not" +from "we misled it" — which is the difference between a to-do for the product +and a bug in our instrument. + +So the scoreboard says *which* cells to read and never *what happened*. Run + +```bash +pnpm --filter @hookdeck-evals/framework triage +``` + +after a run. It reads what the harness already records and flags the cells where +the number is not the whole story: an unclean exit (a killed container arrives +with a plausible partial score, and one was written as a `2/6` agent failure), a +run that failed while its own report claims success (nine of twelve baselines in +one run, every one of them confidently building on the wrong product), a +declared credential the agent never referenced, a skill offered and never +opened, a baseline that self-installed a product skill, and a cell whose only +green checks are the ones an idle agent satisfies. + +Nothing flagged is not the same as nothing to learn. It means no cell tripped a +signal the script knows about, and the signals it knows about are the ones that +have already cost us something. + **When a scorer disagrees with an agent, read the agent's report first.** It has been the cheapest answer every time and it keeps getting skipped. It named the credential it could not fetch, and the variable name it looked for diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index 1836d74..a052d50 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -206,6 +206,18 @@ function buildLoadSkillTool(skills: readonly ToolsSkill[]): ToolSet { * so `docker cp` copies real files, not dangling links. Missing skills are * skipped with a warning. */ +function resolveSkillNames( + metadata: { skills?: string[]; extraSkills?: string[] }, + experimentSkills: string[] +): string[] { + if (metadata.skills) return metadata.skills; + if (experimentSkills.length === 0) return experimentSkills; + const extra = (metadata.extraSkills ?? []).filter( + (name) => !experimentSkills.includes(name) + ); + return [...experimentSkills, ...extra]; +} + function resolveSkillSources( skillNames: string[] ): Array<{ name: string; dir: string }> { @@ -314,11 +326,16 @@ async function runOne( // A per-eval `skills` override replaces the experiment's own list entirely, // so a scenario testing self-installed skills gets an empty list regardless // of which experiment runs it. - const skillSources = resolveSkillSources(ev.metadata.skills ?? exp.skills); + // `skills` replaces, `extraSkills` adds — and adds nothing to an experiment + // that has none. That asymmetry is the point: a `-no-skills` arm must stay + // empty whatever the scenario asks for, or the baseline is no longer a + // baseline. See `extraSkills` in eval-metadata.ts. + const skillNames = resolveSkillNames(ev.metadata, exp.skills); + const skillSources = resolveSkillSources(skillNames); const availableSkills = skillSources.map((skill) => skill.name); const toolsSkills = ev.mode === 'tools' && !agentRunsInSandbox - ? loadToolsSkills(ev.metadata.skills ?? exp.skills) + ? loadToolsSkills(skillNames) : []; const scorer = (await import(pathToFileURL(ev.evalPath).href)) .default as ToolScorer; diff --git a/apps/framework/package.json b/apps/framework/package.json index 788c661..34a41f3 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -14,6 +14,7 @@ "score-only": "node --env-file=../../.env --import tsx/esm scripts/score-only.ts", "compare-snapshots": "node --import tsx/esm scripts/compare-snapshots.ts", "report-results": "node --import tsx/esm scripts/report-results.ts", + "triage": "node --import tsx/esm scripts/triage.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" }, diff --git a/apps/framework/scripts/triage.ts b/apps/framework/scripts/triage.ts new file mode 100644 index 0000000..d787d91 --- /dev/null +++ b/apps/framework/scripts/triage.ts @@ -0,0 +1,257 @@ +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { discoverEvals, EVALS_ROOT } from '../lib/discovery.js'; + +/** + * Which cells of a run you have to read, and why. + * + * AGENTS.md already says to read the agent's report first, and calls it the + * cheapest answer every time — and it keeps getting skipped, including by + * whoever wrote that line. An exhortation loses to a scoreboard, because the + * scoreboard is right there and the transcripts are not. + * + * The case for reading them is stronger than "sometimes useful". Over two days + * of Outpost runs, **every** product finding came out of a transcript and none + * came out of the scoreboard: an API answering `404` where `401` was meant, a + * skill that never names the credential, docs whose environment variables do + * not exist, and — worst — a harness omission that made our own baseline fail + * twelve cells and read as a skills result. A pass rate cannot express any of + * those. It cannot even distinguish "the agent could not" from "we lied to it". + * + * So this turns the convention into a list. It reads what the harness already + * records and flags the cells where the number on the board is not the whole + * story. It does not judge the run; it tells you where to look. + * + * ```bash + * pnpm --filter @hookdeck-evals/framework triage + * pnpm --filter @hookdeck-evals/framework triage --require OUTPOST_API_KEY + * ``` + */ + +const RUNS_DIR = join(EVALS_ROOT, '.eval-runs'); + +interface Check { + name?: string; + passed?: boolean; +} + +interface Row { + experiment: string; + eval: string; + passed?: boolean; + checks?: Check[]; + stoppedReason?: string; + agentReport?: string; + toolCalls?: unknown[]; + skills?: { + available?: string[]; + loaded?: string[]; + selfInstalled?: string[]; + }; +} + +interface Flag { + label: string; + detail: string; +} + +/** + * Capabilities a scenario declares, mapped to the variable that carries them. + * + * Derived from each scenario's `requires`, not from a flag on the command line. + * A blanket `--require OUTPOST_API_KEY` flagged the ElevenLabs and Stripe + * scenarios for not using an Outpost credential they have no business touching, + * and a triage tool that cries wolf gets ignored — which returns us to nobody + * reading transcripts, the thing this exists to fix. + */ +const CAPABILITY_ENV: Record = { outpost: 'OUTPOST_API_KEY' }; + +/** + * Skills whose self-installation invalidates a baseline. + * + * Only these. AGENTS.md draws the line deliberately: a *product* skill pulled + * into a `-no-skills` run means that row is no longer a baseline, while a + * *provider* skill like `stripe-webhooks` is legitimate — documenting a third + * party's signature format was never Hookdeck's job, and we ship those skills + * for exactly this. Flagging every self-install buried the real case in noise. + */ +const PRODUCT_SKILLS = new Set(['hookdeck', 'event-gateway', 'outpost']); + +/** Phrases an agent uses when it believes it finished. */ +const SUCCESS_CLAIM = + /\b(everything (is )?(set up|working|verified)|successfully|all set|is now (set up|configured|working)|done!|completed successfully|verified end-to-end)\b/i; + +/** + * A clean stop. Anything else means the process died, and a dead run must not + * be read as an agent's answer — the guard in `run-eval.ts` only rejects an + * *empty* transcript, so a container killed mid-flight arrives here with 15 + * tool calls and a plausible partial score. One did, on 21 August, and was + * written as a `2/6` agent failure. + */ +const CLEAN_STOP = new Set(['stop', 'end_turn', 'complete']); + +function flagsFor(row: Row, requiredEnv: string[]): Flag[] { + const flags: Flag[] = []; + const blob = JSON.stringify(row); + const checks = row.checks ?? []; + const failed = checks.filter((c) => c.passed === false); + + if (row.stoppedReason && !CLEAN_STOP.has(row.stoppedReason)) { + flags.push({ + label: 'UNCLEAN EXIT', + detail: `stoppedReason=${row.stoppedReason} — the process did not finish, so this is not the agent's answer`, + }); + } + + // The most valuable signal in the whole file. An agent that fails while + // reporting success has done something coherent and wrong, which is where + // findings come from — nine of twelve baselines did exactly this in one run, + // having built the task on the wrong product. + if (row.passed === false && SUCCESS_CLAIM.test(row.agentReport ?? '')) { + flags.push({ + label: 'CLAIMED SUCCESS', + detail: 'failed while reporting the task complete — read this one first', + }); + } + + // A credential the scenario needs that the agent never referenced usually + // means it never found the thing, and that is often our fault rather than + // the model's: the variable may be injected and unannounced. + for (const name of requiredEnv) { + if (row.passed === false && !blob.includes(name)) { + flags.push({ + label: 'CREDENTIAL UNUSED', + detail: `${name} never appears — did the agent know it existed?`, + }); + } + } + + const available = row.skills?.available ?? []; + const loaded = row.skills?.loaded ?? []; + if (row.passed === false && available.length > 0 && loaded.length === 0) { + flags.push({ + label: 'SKILL NOT OPENED', + detail: `offered ${available.join(', ')} and loaded none — this measures selection, not content`, + }); + } + + const smuggled = (row.skills?.selfInstalled ?? []).filter((n) => + PRODUCT_SKILLS.has(n) + ); + if (smuggled.length > 0 && row.experiment.endsWith('-no-skills')) { + flags.push({ + label: 'BASELINE COMPROMISED', + detail: `fetched product skill ${smuggled.join(', ')} at run time — exclude this row from any skills delta`, + }); + } + + // Passing only negative checks is a do-nothing run wearing a partial score. + if ( + row.passed === false && + failed.length > 0 && + failed.length < checks.length + ) { + // *Every* green check must be a negative one, not merely some of them. + // Matching on any was wrong: a cell where the agent re-enabled a + // destination but never recovered the held events has one real positive + // green and one negative, and flagging it as idle misdescribes an agent + // that did half the job. The signal being hunted here is the run that did + // nothing and still scored. + const passedNames = checks.filter((c) => c.passed).map((c) => c.name ?? ''); + const isNegative = (n: string) => + /left alone|unchanged|untouched|no longer|not .*(disabled|removed)|was not/i.test( + n + ); + if (passedNames.length > 0 && passedNames.every(isNegative)) { + flags.push({ + label: 'ONLY NEGATIVES PASSED', + detail: `${checks.length - failed.length}/${checks.length} green, and they are the checks an idle agent satisfies`, + }); + } + } + + if (row.passed === false && (row.toolCalls?.length ?? 0) === 0) { + flags.push({ label: 'NO TOOL CALLS', detail: 'scored without acting' }); + } + + return flags; +} + +function main() { + const args = process.argv.slice(2); + void args; + + // What each scenario declares it needs, by eval id. + const needs = new Map(); + for (const ev of discoverEvals()) { + const requires = + (ev.metadata as { requires?: string[] } | undefined)?.requires ?? []; + const env = requires + .map((r) => CAPABILITY_ENV[r]) + .filter((v): v is string => Boolean(v)); + if (env.length > 0) needs.set(ev.id, env); + } + + if (!existsSync(RUNS_DIR)) throw new Error(`no ${RUNS_DIR}`); + + const rows: Array<{ row: Row; flags: Flag[] }> = []; + for (const dir of readdirSync(RUNS_DIR)) { + const experimentDir = join(RUNS_DIR, dir); + let files: string[] = []; + try { + files = readdirSync(experimentDir).filter((f) => f.endsWith('.json')); + } catch { + continue; + } + for (const file of files) { + const row = JSON.parse( + readFileSync(join(experimentDir, file), 'utf8') + ) as Row; + const flags = flagsFor(row, needs.get(row.eval) ?? []); + if (flags.length > 0) rows.push({ row, flags }); + } + } + + const total = readdirSync(RUNS_DIR).reduce((n, d) => { + try { + return ( + n + + readdirSync(join(RUNS_DIR, d)).filter((f) => f.endsWith('.json')).length + ); + } catch { + return n; + } + }, 0); + + console.log(`${total} cell(s) on disk; ${rows.length} worth reading.\n`); + if (rows.length === 0) { + console.log( + ' Nothing flagged. That is not the same as nothing to learn —' + ); + console.log(' it means no cell tripped a signal this script knows about.'); + return; + } + + // Claimed-success first: it is the one that has produced findings. + rows.sort((a, b) => { + const rank = (f: Flag[]) => + f.some((x) => x.label === 'CLAIMED SUCCESS') + ? 0 + : f.some((x) => x.label === 'UNCLEAN EXIT') + ? 1 + : 2; + return rank(a.flags) - rank(b.flags); + }); + + for (const { row, flags } of rows) { + console.log(` ${row.eval} x ${row.experiment}`); + for (const flag of flags) { + console.log(` ${flag.label}: ${flag.detail}`); + } + console.log( + ` → .eval-runs/${row.experiment}/${row.eval}.json (agentReport, toolCalls)\n` + ); + } +} + +main(); diff --git a/evals/benchmark-outpost-001-customer-subscriptions/EVAL.ts b/evals/benchmark-outpost-001-customer-subscriptions/EVAL.ts index 5c2cc07..24872d0 100644 --- a/evals/benchmark-outpost-001-customer-subscriptions/EVAL.ts +++ b/evals/benchmark-outpost-001-customer-subscriptions/EVAL.ts @@ -45,6 +45,19 @@ const scorer: ToolScorer = async (ctx) => { ); } + // A leftover `acme` used to satisfy every check here with no agent action. + // This was the only Outpost scenario with no seed, and every other one seeds + // a tenant called `acme` with a destination on an order-ish topic; tenant + // cleanup runs on release inside a `catch`-and-ignore and is skipped when a + // run is killed, both of which have happened. The row published green. + // + // The fix is in the seed — `deleteTenants: ["acme", "globex"]` — because the + // state has to be *absent*, not merely distinguishable. Comparing the + // tenant's `created_at` against the lease was tried first and does not work: + // tenant create is idempotent, so an agent that correctly `PUT`s an existing + // id gets the original timestamp back and is scored as having inherited + // someone else's work. Measured on 24 August, the tenant read two minutes + // older than the lease about to score it. const tenants = await listTenants(ctx); const tenant = tenants.find((t) => /acme/i.test(String(t.id ?? ''))); @@ -110,7 +123,23 @@ async function checkOrderEventDelivered( }; } - const before = await attemptCount(ctx, tenantId, destinations); + // Successful attempts, not attempts. + // + // Counting any attempt made this check "Outpost tried", which is not what its + // name claims. A destination pointed at a hostname the agent invented records + // an attempt and fails to deliver, and the customer receives nothing. The + // header of `outpost-004` asserts "delivery is already proven against webhook + // destinations by outpost-001" — it was not. + // + // **This does not close the whole hole, and the remaining half is a scenario + // problem rather than a scorer one.** The ticket never says where the + // customer's endpoint is, so any reachable URL satisfies it: one agent stood + // up a localtunnel *inside its own sandbox*, delivered to itself, passed, and + // offered to "swap the temporary receiver URL for the real customer endpoint + // next". That receiver died with the container. Fixing it means giving the + // ticket an endpoint to deliver to, which changes a published scenario, so it + // is called out here rather than done quietly. + const before = await successCount(ctx, tenantId, destinations); await ctx.outpost?.('POST', '/publish', { tenant_id: tenantId, topic, @@ -120,7 +149,7 @@ async function checkOrderEventDelivered( // attempt count rather than sleeping and reading once. A single positive // assertion, so the first observation that satisfies it is the answer. const after = await waitForOrLast( - () => attemptCount(ctx, tenantId, destinations), + () => successCount(ctx, tenantId, destinations), (count) => count > before, { timeoutMs: DELIVERY_WAIT_MS, @@ -161,6 +190,28 @@ function orderTopic( return undefined; } +/** Attempts that actually delivered. */ +async function successCount( + ctx: ToolEvalContext, + tenantId: string, + destinations: Record[] +): Promise { + let total = 0; + for (const destination of destinations) { + const id = String(destination.id ?? ''); + if (!id) continue; + const body = await ctx.outpost?.< + { status?: string }[] | { models?: { status?: string }[] } + >( + 'GET', + `/tenants/${encodeURIComponent(tenantId)}/destinations/${encodeURIComponent(id)}/attempts` + ); + const rows = Array.isArray(body) ? body : (body?.models ?? []); + total += rows.filter((a) => a.status === 'success').length; + } + return total; +} + async function attemptCount( ctx: ToolEvalContext, tenantId: string, @@ -202,8 +253,17 @@ async function listDestinations( ctx: ToolEvalContext, tenantId: string ): Promise[]> { + // `models ?? data`, matching every sibling scorer and the client. This file + // read `data` alone — the exact trap its own comments warn about twice, which + // survives only because this endpoint happens to be unpaged and returns a + // bare array. The day it gains an envelope, every agent is told the tenant + // has nowhere to deliver to. const body = await ctx.outpost?.< - Record[] | { data?: Record[] } + | Record[] + | { + models?: Record[]; + data?: Record[]; + } >('GET', `/tenants/${encodeURIComponent(tenantId)}/destinations`); - return Array.isArray(body) ? body : (body?.data ?? []); + return Array.isArray(body) ? body : (body?.models ?? body?.data ?? []); } diff --git a/evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md b/evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md index 943b345..445083d 100644 --- a/evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md +++ b/evals/benchmark-outpost-001-customer-subscriptions/PROMPT.md @@ -8,6 +8,8 @@ topic: - sdk requires: - outpost +extra_skills: + - outpost motivation: The reason people adopt Outpost. Sending webhooks to your own customers is a product feature, and every team that builds it by hand rebuilds retries, verification and a subscription model badly. --- diff --git a/evals/benchmark-outpost-001-customer-subscriptions/SOLUTION.ts b/evals/benchmark-outpost-001-customer-subscriptions/SOLUTION.ts new file mode 100644 index 0000000..5853316 --- /dev/null +++ b/evals/benchmark-outpost-001-customer-subscriptions/SOLUTION.ts @@ -0,0 +1,35 @@ +import type { ToolEvalContext } from '@hookdeck-evals/core'; + +/** + * What a correct agent leaves behind: a tenant for the customer, a destination + * subscribed to order events, and a delivery that actually arrives. + * + * Added late, and for a specific reason. This scenario had no solution, so + * `score-only` could only ever exercise its failing path — which is how it kept + * a check that passed on a *leftover* tenant and another that counted a + * delivery *attempt* rather than a delivery. Both were repaired on 24 August, + * and the repair for the first compares the tenant's `created_at` against the + * lease. That comparison is worth testing before trusting: clock skew between + * this machine and Outpost would reject a tenant an agent had just made. + */ + +const TENANT = 'acme'; + +export default async function solve(ctx: ToolEvalContext): Promise { + const outpost = ctx.outpost; + if (!outpost) { + throw new Error( + 'no Outpost client: this solution cannot be applied without OUTPOST_API_KEY' + ); + } + + await outpost('PUT', `/tenants/${TENANT}`, {}); + + await outpost('POST', `/tenants/${TENANT}/destinations`, { + type: 'webhook', + topics: ['order.created'], + // A reachable endpoint, because the check requires the event to arrive + // rather than merely to be attempted. + config: { url: 'https://mock.hookdeck.com/api/v1/acme/orders' }, + }); +} diff --git a/evals/benchmark-outpost-001-customer-subscriptions/remote/seed.json b/evals/benchmark-outpost-001-customer-subscriptions/remote/seed.json new file mode 100644 index 0000000..ba798db --- /dev/null +++ b/evals/benchmark-outpost-001-customer-subscriptions/remote/seed.json @@ -0,0 +1,5 @@ +{ + "outpost": { + "deleteTenants": ["acme", "globex"] + } +} diff --git a/evals/benchmark-outpost-002-disabled-destination/EVAL.ts b/evals/benchmark-outpost-002-disabled-destination/EVAL.ts index e28cf05..07c098a 100644 --- a/evals/benchmark-outpost-002-disabled-destination/EVAL.ts +++ b/evals/benchmark-outpost-002-disabled-destination/EVAL.ts @@ -59,6 +59,8 @@ interface Destination { interface Attempt { id?: string; status?: string; + /** Which event this attempt was for. The check below turns on it. */ + event_id?: string; } const scorer: ToolScorer = async (ctx) => { @@ -120,21 +122,46 @@ async function checkMissedEventsDelivered( ): Promise { const name = 'the events the customer missed were delivered'; + // Counted per *event*, not per successful attempt. + // + // The first version counted any three successes on the tenant, which three + // fresh test publishes satisfy just as well as recovering the outage. It + // passed an agent that reported retrying 54 events out of 78 it had found — + // the stale-history trap, since events outlive the tenant and a shared + // project accumulates them. + // + // "Missed" is definable from the attempts themselves and needs no timestamps: + // an event that has a failed attempt is one the customer did not get, and it + // has been recovered when that same `event_id` also has a successful one. + // Republishing cannot fake it, because a new event has no failed attempt. + const recovered = (rows: Attempt[]): number => { + const failed = new Set( + rows + .filter((a) => a.status === 'failed' && a.event_id) + .map((a) => a.event_id) + ); + const succeeded = new Set( + rows + .filter((a) => a.status === 'success' && a.event_id) + .map((a) => a.event_id) + ); + return [...failed].filter((id) => succeeded.has(id)).length; + }; + // Poll: a retry triggered moments before the agent finished is still in // flight, and reading once would score the agent for the platform's timing. - // Summed across the tenant's destinations, for the same reason: an agent that - // recovered the events onto a replacement destination delivered them. + // Summed across the tenant's destinations, because an agent that recovered + // the events onto a replacement destination delivered them. const attempts = await waitForOrLast( () => listAllAttempts(ctx, TENANT, destinations), - (rows) => - rows.filter((a) => a.status === 'success').length >= MISSED_EVENTS, + (rows) => recovered(rows) >= MISSED_EVENTS, { timeoutMs: DELIVERY_WAIT_MS, description: 'the held events to be delivered', } ); - const delivered = attempts.filter((a) => a.status === 'success').length; + const delivered = recovered(attempts); return { name, passed: delivered >= MISSED_EVENTS, diff --git a/evals/benchmark-outpost-002-disabled-destination/PROMPT.md b/evals/benchmark-outpost-002-disabled-destination/PROMPT.md index ea01247..5c2b934 100644 --- a/evals/benchmark-outpost-002-disabled-destination/PROMPT.md +++ b/evals/benchmark-outpost-002-disabled-destination/PROMPT.md @@ -8,6 +8,8 @@ topic: - retries requires: - outpost +extra_skills: + - outpost motivation: The support case Outpost generates most reliably. A destination that keeps failing is auto-disabled to protect the system, and once the customer repairs their endpoint nothing starts again on its own — the held events are not retried until the destination is re-enabled. Outpost does emit `alert.destination.disabled` as an operator event, but only where a sink has been configured for it (Hookdeck Monitoring settings on managed, `OPERATION_EVENTS_TOPICS` plus a sink when self-hosted); on a deployment where nobody has, the first signal is the customer. --- diff --git a/evals/benchmark-outpost-003-operator-events/EVAL.ts b/evals/benchmark-outpost-003-operator-events/EVAL.ts index d4568e2..9258ac6 100644 --- a/evals/benchmark-outpost-003-operator-events/EVAL.ts +++ b/evals/benchmark-outpost-003-operator-events/EVAL.ts @@ -13,20 +13,31 @@ import type { * with none is silent by design rather than by fault. * * What makes it worth measuring is where the answer lives. Operator events are - * configured over the API at `/operator-events/destinations`, and those routes - * appear in neither published OpenAPI spec nor the API reference at - * `/docs/outpost/api`; the prose page tells managed users to use the dashboard. - * An agent has no source of truth except what it can read, so this measures - * whether a documented-by-UI-only capability is reachable at all. See + * configured at `/operator-events/destinations`, and those routes appear in + * neither published OpenAPI spec nor the API reference at `/docs/outpost/api`; + * the prose page tells managed users to use the dashboard. See * hookdeck/evals#34. * - * **Expect every agent to fail this initially, and publish it anyway.** That is - * a floor rather than a flat result: the scenario is passable, the route exists - * and works, and the only thing missing is documentation. If #34 is fixed and - * the next run turns green, that is a closed loop — a finding, a change made - * outside this repository, and a re-run that says what the change bought. A - * scenario nobody passes for a reason we have written down is more useful than - * one nobody fails. + * **This scenario originally predicted that every agent would fail it, and that + * was wrong.** The prediction rested on "an agent has no source of truth except + * what it can read". Agents do not work that way: on 21 and 22 August it was + * passed six times out of six, across three models and both arms — including by + * a weak model with no skills at all. They find the route by enumeration, + * probing `/operator-events`, `/operator-event-destinations`, + * `/operator/destinations` and so on until one answers, having first probed a + * deliberately bogus path to learn what a real 404 looks like. + * + * So it does not measure discovery in the sense the classification implies, and + * it does not currently discriminate at all. What it measures is persistence: + * whether an agent keeps going when the documentation runs out. That is worth + * something, but it is a fact about agents rather than about Hookdeck, and it + * will not move if we fix the docs. + * + * Two consequences worth keeping in mind. It is **high variance** — the same + * model passed 4/4 in one run and failed 0/1 in the next, because passing turns + * on which paths get guessed, so a single attempt publishes a coin flip. And + * #34 stands regardless: an endpoint reachable only by guesswork is + * undocumented whether or not a determined agent gets there. * * Scored on outcome. It does not matter whether the agent used the API, and it * is not required to subscribe to the alert topic by name — `*` covers it and diff --git a/evals/benchmark-outpost-003-operator-events/PROMPT.md b/evals/benchmark-outpost-003-operator-events/PROMPT.md index 9feb035..c93c5b9 100644 --- a/evals/benchmark-outpost-003-operator-events/PROMPT.md +++ b/evals/benchmark-outpost-003-operator-events/PROMPT.md @@ -8,6 +8,8 @@ topic: - alerting requires: - outpost +extra_skills: + - outpost motivation: Follows the incident in benchmark-outpost-002. A destination was auto-disabled, the customer's events were held, and nobody found out until the customer emailed. Outpost emits `alert.destination.disabled` for exactly this, but it is delivered only to a configured operator events destination, and this project has none. The routes that configure it are absent from the published OpenAPI spec and from the API reference, so this measures whether an agent can set up alerting it cannot read about. --- diff --git a/evals/benchmark-outpost-004-queue-destination/EVAL.ts b/evals/benchmark-outpost-004-queue-destination/EVAL.ts index 2146e92..1673349 100644 --- a/evals/benchmark-outpost-004-queue-destination/EVAL.ts +++ b/evals/benchmark-outpost-004-queue-destination/EVAL.ts @@ -72,6 +72,42 @@ const scorer: ToolScorer = async (ctx) => { (d) => normalise(d.config?.url) === normalise(OLD_ENDPOINT) ); + // An agent that did nothing scores zero, not two out of six. + // + // Both negative checks — that the retries still reach the old endpoint, and + // that the other customer was left alone — are satisfied by the untouched + // seed. So a run that never acted used to report `2/6`, which reads as + // partial progress and is really no progress: it is the shape a crashed cell + // wears, and one did exactly that on 21 August before being spotted. + // + // The verdict was never wrong, since `passed` is the conjunction and the four + // positive checks need real work. It is the per-check count that misleads + // anyone reading the detail — including us, triaging a run. + // + // So when nothing has changed at all, say that in one line instead of + // awarding marks for leaving things alone. + const untouched = + queues.length === 0 && + webhooks.some( + (d) => !d.disabled_at && subscribes(d, ORDERS) && subscribes(d, RETRIES) + ); + + if (untouched) { + return { + passed: false, + checks: [ + { + name: 'their orders are delivered to the queue they gave us', + passed: false, + notes: + 'nothing was changed: no queue destination exists and the old endpoint ' + + 'still carries both topics, so the checks about not breaking anything ' + + 'are true only because no work was done', + }, + ], + }; + } + const checks: CheckResult[] = [ checkQueueExists(destinations, queues), checkQueueReceivesOrders(queues), diff --git a/evals/benchmark-outpost-004-queue-destination/PROMPT.md b/evals/benchmark-outpost-004-queue-destination/PROMPT.md index 75621a9..7bd1e47 100644 --- a/evals/benchmark-outpost-004-queue-destination/PROMPT.md +++ b/evals/benchmark-outpost-004-queue-destination/PROMPT.md @@ -8,6 +8,8 @@ topic: - capabilities requires: - outpost +extra_skills: + - outpost motivation: Most Outpost traffic is webhooks, but delivering to a queue is a core capability and a different job — a type rather than a URL, credentials rather than a secret, and fields whose names differ per provider. This scores whether an agent can configure a non-HTTP destination from details it has to find, rather than reaching for the webhook shape it has seen most often. --- diff --git a/evals/benchmark-outpost-005-topic-scoping/EVAL.ts b/evals/benchmark-outpost-005-topic-scoping/EVAL.ts new file mode 100644 index 0000000..f04371b --- /dev/null +++ b/evals/benchmark-outpost-005-topic-scoping/EVAL.ts @@ -0,0 +1,318 @@ +import type { + CheckResult, + ToolEvalContext, + ToolScorer, +} from '@hookdeck-evals/core'; +import { waitForSettled } from '@hookdeck-evals/hookdeck'; + +/** + * Narrow one customer's slice of the event stream without narrowing it too far. + * + * Written after a 24-cell run in which the four existing Outpost scenarios were + * passed by almost everything — `+skills` went 12/12 — because their difficulty + * had quietly been "work out that Outpost is the product", and the harness + * stopped withholding that. What was left was mostly setup, and setup is not + * where agents fail. + * + * So this is built around the shape AGENTS.md says actually discriminates: an + * agent can finish, report success, and be wrong, with nothing erroring. Acme + * receives three topics and wants one of them stopped. The obvious fix — set + * their destination to the topic they still talk about — passes the check they + * complained about and silently stops `order.shipped`, which they never + * mentioned because it was working. Nobody sees an error. The customer notices + * days later, when something they depend on has quietly stopped arriving. + * + * The ticket had to be rewritten to make that true. Its first version added + * "everything else they get today should carry on exactly as it is, and Globex + * shouldn't be affected at all" — which states four of the five checks, and + * replaces the silence the scenario depends on with an instruction to enumerate + * and preserve. It also never named a topic acme *wanted*, so "narrow to the + * one they talk about" was not a candidate action and the wrong answer took + * more work than the right one. All six agents passed. The trap was described + * here and never set in the prompt, which is a thing to check for: verifying a + * scorer rejects a hand-written wrong answer proves the scorer works, not that + * agents make that mistake. + * + * That is the same failure mode as `resolve-002` and `alerting-001`: acting more + * broadly than asked. The difference is that here it is the *cheapest* way to + * satisfy the request, rather than a mistake you have to reach for. + * + * Scored on behaviour, not configuration, and the reason matters because the + * first version of this comment gave a fictional one. It claimed a tenant's own + * topic list gates delivery and that an agent could edit that instead — + * `Tenant.topics` is read-only and derived from its destinations, verified + * live: a `PUT` with topics returns `[]`, and adding a destination on + * `order.shipped` makes it `["order.shipped"]`. (The `tenants[].topics` field + * in every seed in this repo is therefore inert.) + * + * The real reason is Outpost's per-destination `filter`, which can suppress an + * event that the `topics` array admits. A scorer reading configuration would + * pass a destination whose topics look right and whose filter drops everything. + * Publishing real events and checking what arrives arrives at the truth + * whichever lever the agent used. + */ + +const ACME = 'acme'; +const GLOBEX = 'globex'; + +/** Still wanted. Named in the ticket only as "everything else". */ +const KEPT = 'order.created'; +/** Still wanted, and never mentioned — the one a too-narrow fix removes. */ +const UNMENTIONED = 'order.shipped'; +/** The topic they asked us to stop. */ +const UNWANTED = 'order.cancelled'; + +/** Long enough that "it did not arrive" means it is not coming. */ +const DELIVERY_WAIT_MS = 45_000; +const SETTLE_MS = 8_000; +/** + * Deliberately slower than the 1s default. + * + * Each poll costs one events read plus one attempts read per destination, and + * two waits run back to back. At 1s that is well over a hundred requests for a + * single cell, and `waitFor` treats a failed probe as "not ready yet" — so a + * rate-limited read is indistinguishable from nothing having arrived, and + * surfaces as a false failure on whichever check is being measured. Observed + * once on the collateral check, which is the worst place for it: it accuses an + * agent of breaking a customer it never touched. + */ +const POLL_INTERVAL_MS = 3_000; + +interface Attempt { + id?: string; + status?: string; + event_id?: string; +} + +interface Destination { + id?: string; + topics?: string[]; + disabled_at?: string | null; +} + +const scorer: ToolScorer = async (ctx) => { + if (!ctx.outpost) { + throw new Error( + 'no Outpost client, but this scenario declares `requires: [outpost]` ' + + 'and should have been skipped rather than scored' + ); + } + + // Published together, then read once after they have all had time to land. + // Sending the positives and the negative separately would let a slow negative + // arrive after its own check had already passed. + await publish(ctx, ACME, KEPT); + await publish(ctx, ACME, UNMENTIONED); + await publish(ctx, ACME, UNWANTED); + + // Two must arrive and one must not, so the positives starting the clock is + // what gives the negative its chance to be wrong. Reading the moment the + // positives land would pass a configuration that changed nothing at all. + const acme = await waitForSettled( + () => deliveredTopics(ctx, ACME), + (topics) => topics.has(KEPT) && topics.has(UNMENTIONED), + { + timeoutMs: DELIVERY_WAIT_MS, + settleMs: SETTLE_MS, + intervalMs: POLL_INTERVAL_MS, + description: "acme's remaining topics to be delivered", + } + ); + + // Globex is probed *after* acme's wait, not alongside it. + // + // Sharing one publish meant globex's event was already 45 seconds old by the + // time it was measured whenever acme's wait ran to timeout — which is exactly + // the case where the agent broke acme, so the collateral check failed on the + // runs where it mattered most. Measured: with a correct fix acme resolves + // fast and globex passed 5/5; on the delete and disable paths globex failed + // twice, while delivering perfectly well when tested on its own. + // + // The precise interaction was never pinned down, and this does not attempt + // to. It removes the coupling instead: each tenant gets its own publish and + // its own window, so how long acme takes cannot decide whether globex looks + // untouched. Accusing an agent of breaking a customer it never touched is the + // worst false failure this scorer could produce. + await publish(ctx, GLOBEX, UNWANTED); + + const globex = await waitForSettled( + () => deliveredTopics(ctx, GLOBEX), + (topics) => topics.has(UNWANTED), + { + timeoutMs: DELIVERY_WAIT_MS, + settleMs: SETTLE_MS, + intervalMs: POLL_INTERVAL_MS, + description: "globex's delivery to be unaffected", + } + ); + + const checks: CheckResult[] = [ + { + name: 'acme no longer receives order cancellations', + passed: !acme.has(UNWANTED), + notes: acme.has(UNWANTED) + ? 'a cancellation was still delivered to acme, which is what they asked us to stop' + : undefined, + }, + { + // The check the scenario exists for. + name: 'acme still receives the shipping events they depend on', + passed: acme.has(UNMENTIONED), + notes: acme.has(UNMENTIONED) + ? undefined + : `${UNMENTIONED} stopped reaching acme. They asked us to stop cancellations ` + + 'and said everything else should carry on; this is the part they did not ' + + 'mention because it was working, and nothing here would have errored', + }, + { + name: 'acme still receives new orders', + passed: acme.has(KEPT), + notes: acme.has(KEPT) + ? undefined + : `${KEPT} stopped reaching acme, so the change went far wider than the request`, + }, + { + name: 'globex was left alone', + passed: globex.has(UNWANTED), + notes: globex.has(UNWANTED) + ? undefined + : 'globex stopped receiving cancellations too — a change made at the wrong ' + + 'level hits every customer, and they never complained about anything', + }, + await checkNothingDisabled(ctx), + ]; + + // A platform fault throws; an agent fault is scored. Telling them apart needs + // the configuration, not the delivery count. + // + // The first version threw whenever acme received nothing — which is exactly + // what deleting the destination, disabling it, or setting `topics: []` + // produces. Those are the worst things an agent can do here, and the throw + // discarded the `checks` array that described them, so no result row was + // written at all: the scenario structurally could not report its own most + // severe failures. `checkNothingDisabled` existed to name a route the scorer + // then threw away. + // + // So only throw when the configuration says acme *should* have received + // something and nothing arrived. That is the seed or the platform. If nothing + // arrived because no enabled destination subscribes to the topic any more, + // the agent did that, and the checks above already say so. + if (acme.size === 0) { + const live = (await listDestinations(ctx, ACME)).filter( + (d) => !d.disabled_at + ); + const shouldHaveArrived = live.some((d) => { + const topics = d.topics ?? []; + return topics.includes('*') || topics.includes(KEPT); + }); + if (shouldHaveArrived) { + throw new Error( + `acme has an enabled destination subscribed to ${KEPT} and received nothing ` + + 'after publishing: the seed or the platform is at fault, not the agent, ' + + 'so this run is not scoreable' + ); + } + } + + return { passed: checks.every((c) => c.passed), checks }; +}; + +export default scorer; + +/** + * Disabling a destination stops the unwanted topic as effectively as scoping it + * — and stops everything else too. It is caught by the delivery checks above, + * but naming it separately makes a red cell say *which* wrong route was taken. + */ +async function checkNothingDisabled( + ctx: ToolEvalContext +): Promise { + const disabled: string[] = []; + for (const tenant of [ACME, GLOBEX]) { + const rows = await listDestinations(ctx, tenant); + if (rows.some((d) => d.disabled_at)) disabled.push(tenant); + } + return { + name: 'no destination was switched off to achieve it', + passed: disabled.length === 0, + notes: + disabled.length === 0 + ? undefined + : `disabled: ${disabled.join(', ')} — that stops the cancellations by ` + + 'stopping everything, which is not what was asked', + }; +} + +/** Topics that actually reached a tenant, by successful delivery. */ +async function deliveredTopics( + ctx: ToolEvalContext, + tenant: string +): Promise> { + const events = await list<{ id?: string; topic?: string }>( + ctx, + `/events?tenant_id=${encodeURIComponent(tenant)}&limit=100` + ); + const byId = new Map(events.map((e) => [e.id, e.topic])); + + const topics = new Set(); + for (const destination of await listDestinations(ctx, tenant)) { + if (!destination.id) continue; + const attempts = await list( + ctx, + `/tenants/${encodeURIComponent(tenant)}/destinations/${encodeURIComponent(destination.id)}/attempts` + ); + for (const attempt of attempts) { + if (attempt.status !== 'success' || !attempt.event_id) continue; + const topic = byId.get(attempt.event_id); + if (topic) topics.add(topic); + } + } + return topics; +} + +async function publish( + ctx: ToolEvalContext, + tenant: string, + topic: string +): Promise { + await ctx.outpost?.('POST', '/publish', { + tenant_id: tenant, + topic, + data: { probe: true, topic }, + }); +} + +async function listDestinations( + ctx: ToolEvalContext, + tenant: string +): Promise { + return list( + ctx, + `/tenants/${encodeURIComponent(tenant)}/destinations` + ); +} + +/** + * Outpost list endpoints answer `{ pagination, models }`, not `{ data }`. + * + * A `404` is an empty read rather than a failure, and only a `404`. Deleting a + * tenant's last destination deletes the tenant with it, so an agent that + * removes the destination — one of the wrong routes this scenario scores — + * makes every subsequent read 404. Letting that propagate threw the whole cell + * away and wrote no result, which is the same defect the throw above was just + * repaired for. Anything else still propagates, because a scorer that treats + * every error as "nothing there" reports a confident agent failure for a + * platform blip. + */ +async function list(ctx: ToolEvalContext, path: string): Promise { + let rows: T[] | { models?: T[]; data?: T[] } | undefined; + try { + rows = await ctx.outpost?.('GET', path); + } catch (error) { + if (String((error as Error).message).includes('404')) return []; + throw error; + } + if (!rows) return []; + if (Array.isArray(rows)) return rows; + return rows.models ?? rows.data ?? []; +} diff --git a/evals/benchmark-outpost-005-topic-scoping/PROMPT.md b/evals/benchmark-outpost-005-topic-scoping/PROMPT.md new file mode 100644 index 0000000..ed452cc --- /dev/null +++ b/evals/benchmark-outpost-005-topic-scoping/PROMPT.md @@ -0,0 +1,20 @@ +--- +stage: resolve +suite: benchmark +gated_by: mixed +product: + - outpost +topic: + - filtering +requires: + - outpost +extra_skills: + - outpost +motivation: Every Outpost customer subscribes to a different slice of the same event stream, and narrowing one customer's slice is the most common change a support engineer makes. The ticket names the topic to stop and one topic to keep, so scoping to the named keeper is the obvious fix — and it silently drops a third topic nobody mentioned, because it was working. Nothing errors; the customer notices days later. +--- + +Acme have been in touch. Their integration is choking on order cancellations — +they don't handle them and each one throws an error on their side. Their order +confirmations are working fine and they rely on those. + +Stop the cancellations. diff --git a/evals/benchmark-outpost-005-topic-scoping/SOLUTION.ts b/evals/benchmark-outpost-005-topic-scoping/SOLUTION.ts new file mode 100644 index 0000000..368762f --- /dev/null +++ b/evals/benchmark-outpost-005-topic-scoping/SOLUTION.ts @@ -0,0 +1,47 @@ +import type { ToolEvalContext } from '@hookdeck-evals/core'; + +/** + * What a correct agent leaves behind: acme's destination subscribed to the two + * topics they still want, and nothing else touched. + * + * The single line worth reading is the topic list. Writing + * `['order.created']` — the topic the ticket talks about — satisfies the + * complaint and silently drops `order.shipped`, which is the failure this + * scenario exists to catch. Getting it right means noticing what the customer + * was receiving *before*, rather than what they wrote to you about. + */ + +const TENANT = 'acme'; +const OLD_ENDPOINT = 'https://mock.hookdeck.com/api/v1/acme/orders'; + +interface Destination { + id?: string; + config?: Record; +} + +export default async function solve(ctx: ToolEvalContext): Promise { + const outpost = ctx.outpost; + if (!outpost) { + throw new Error( + 'no Outpost client: this solution cannot be applied without OUTPOST_API_KEY' + ); + } + + const rows = await outpost( + 'GET', + `/tenants/${TENANT}/destinations` + ); + const destinations = Array.isArray(rows) ? rows : (rows.models ?? []); + + for (const destination of destinations) { + if (destination.config?.url !== OLD_ENDPOINT || !destination.id) continue; + await outpost( + 'PATCH', + `/tenants/${TENANT}/destinations/${destination.id}`, + { + // Everything they had, minus the one they asked to stop. + topics: ['order.created', 'order.shipped'], + } + ); + } +} diff --git a/evals/benchmark-outpost-005-topic-scoping/remote/seed.json b/evals/benchmark-outpost-005-topic-scoping/remote/seed.json new file mode 100644 index 0000000..4483367 --- /dev/null +++ b/evals/benchmark-outpost-005-topic-scoping/remote/seed.json @@ -0,0 +1,32 @@ +{ + "outpost": { + "tenants": [ + { + "id": "acme", + "topics": ["order.created", "order.shipped", "order.cancelled"], + "destinations": [ + { + "ref": "acme-all", + "type": "webhook", + "topics": ["order.created", "order.shipped", "order.cancelled"], + "config": { "url": "https://mock.hookdeck.com/api/v1/acme/orders" } + } + ] + }, + { + "id": "globex", + "topics": ["order.created", "order.shipped", "order.cancelled"], + "destinations": [ + { + "ref": "globex-all", + "type": "webhook", + "topics": ["order.created", "order.shipped", "order.cancelled"], + "config": { + "url": "https://mock.hookdeck.com/api/v1/globex/orders" + } + } + ] + } + ] + } +} diff --git a/packages/core/src/eval-metadata.ts b/packages/core/src/eval-metadata.ts index ea11b27..bfdae5c 100644 --- a/packages/core/src/eval-metadata.ts +++ b/packages/core/src/eval-metadata.ts @@ -189,6 +189,26 @@ export type EvalMetadata = { * entirely to use the experiment's own skill list. */ skills?: string[]; + /** + * Skills added to the experiment's list for this eval, **only when the + * experiment already has skills**. + * + * For product skills a scenario needs and its experiment does not carry. + * Outpost scenarios are the case: `+skills` experiments load `hookdeck` and + * `event-gateway`, so an Outpost scenario got a router plus the *wrong* + * product's skill, whose only Outpost content is a line telling the agent to + * go elsewhere. + * + * Not `skills`, which replaces the experiment's list outright — including for + * a `-no-skills` experiment, which would hand the baseline arm the very skill + * it exists to do without. And not adding `outpost` to the experiments + * themselves, which would change the skill set for all seventeen scenarios + * and make every published `+skills` row non-comparable to answer a question + * about four of them. + * + * Empty experiment list stays empty: that is what makes this safe. + */ + extraSkills?: string[]; /** * Skips installing the real Hookdeck CLI into the sandbox before the agent * starts (sandbox evals only). Defaults to false. Set true only for @@ -220,6 +240,7 @@ export const evalMetadataSchema = z.object({ hostedProject: z.union([z.boolean(), z.stringbool()]).optional(), requires: z.array(evalRequirementSchema).optional(), skills: z.array(z.string().min(1)).optional(), + extraSkills: z.array(z.string().min(1)).optional(), skipCliInstall: z.union([z.boolean(), z.stringbool()]).optional(), }); @@ -308,6 +329,9 @@ export const evalFrontmatterSchema = z.preprocess((raw) => { skills: Array.isArray(data.skills) ? toIdentifierList(data.skills) : undefined, + extraSkills: Array.isArray(data.extraSkills ?? data.extra_skills) + ? toIdentifierList((data.extraSkills ?? data.extra_skills) as unknown[]) + : undefined, skipCliInstall: data.skipCliInstall, }; }, evalMetadataSchema); diff --git a/packages/hookdeck/src/runtime.ts b/packages/hookdeck/src/runtime.ts index 436084c..5f66528 100644 --- a/packages/hookdeck/src/runtime.ts +++ b/packages/hookdeck/src/runtime.ts @@ -56,6 +56,19 @@ export function hookdeckRuntime(options: HookdeckRuntimeOptions): EvalRuntime { // as skipped rather than failing inside a check on a machine that has // no Outpost project. const outpostKey = process.env.OUTPOST_API_KEY; + // Does *this scenario* involve Outpost, as opposed to this machine + // merely having a key? + // + // Gating the addendum on the client existing was wrong: the client is + // built whenever OUTPOST_API_KEY is set, which is always, so every + // Event Gateway scenario was being told about an Outpost project it has + // no use for — noise in fourteen prompts to answer a question about + // five, and it would have made every published cell non-comparable + // rather than just the Outpost ones. + const scenarioSeed = args.remoteDir + ? readSeed(args.remoteDir) + : undefined; + const scenarioUsesOutpost = Boolean(scenarioSeed?.outpost); const outpostClient = outpostKey ? new OutpostClient({ apiKey: outpostKey }) : undefined; @@ -68,7 +81,7 @@ export function hookdeckRuntime(options: HookdeckRuntimeOptions): EvalRuntime { // here — so arriving without a client means the requirement is missing, // not that the machine is simply unconfigured. if (args.remoteDir) { - const seed = readSeed(args.remoteDir); + const seed = scenarioSeed; if (seed?.outpost) { if (!outpostClient) { throw new Error( @@ -102,12 +115,63 @@ export function hookdeckRuntime(options: HookdeckRuntimeOptions): EvalRuntime { return { mcpServers, promptAddendum: [ - // The agent is told the project exists and is authenticated, and - // nothing else. Which docs to read, and how to do the task, is what - // the scenario measures. + // The agent is told what its environment contains, and nothing + // else. Which docs to read, and how to do the task, is what the + // scenario measures. 'You have a Hookdeck project. The Hookdeck CLI is installed and ' + 'HOOKDECK_API_KEY is set in your environment, so both the CLI ' + 'and the REST API are available to you.', + // Named because it is there. + // + // This sentence exists to make the one above true. The addendum's + // job is disclosure — it already says a project exists and how it + // is authenticated — and it was silently omitting a second project + // that the harness injects whenever a scenario needs Outpost. That + // is not a discovery test we designed; it is an incomplete + // sentence, and "so both the CLI and the REST API are available to + // you" actively reads as *this is your access*. + // + // Measured on 21 August, before this line existed: across twelve + // baseline cells on four Outpost scenarios, `OUTPOST_API_KEY` was + // used as a credential exactly zero times and every cell failed. + // Nine of the twelve agents believed they had succeeded, having + // built the task on `api.hookdeck.com`. Two reported, with + // authority, that the harness had given them the wrong credential. + // The skill was the only artefact in the sandbox naming the + // variable, so the skills delta could not be separated from + // credential disclosure — the run measured our own omission. + // + // One credential type, two projects. Both keys authenticate + // `api.hookdeck.com` and the CLI; only the Outpost project's key + // reaches the Outpost subdomain, and a key from another project + // gets a `404` there rather than anything that says why (#39). + // + // The wording points at the API rather than the CLI deliberately, + // and that is a statement about the pinned version rather than a + // permanent one. Against `HOOKDECK_CLI_VERSION` 2.5.0 the Outpost + // key authenticates the CLI and selects the Outpost project — + // verified — but there is nothing useful to do with it there, so + // the API is the only honest route to point at. + // + // `3.0.0-beta.1` is published (npm dist-tag `beta`; `latest` is + // still 2.5.0) and adds managing an Outpost project from the CLI. + // When that pin moves, this sentence becomes incomplete rather + // than wrong, and the CLI becomes a legitimate answer for Outpost + // work. A CLI bump also changes the product under test, so results + // either side of it are not comparable — see Releases in + // AGENTS.md. + // A developer using Outpost knows they use it and has the key in + // their environment; nobody learns their own credentials by + // enumerating env vars. + ...(outpostClient && scenarioUsesOutpost + ? [ + 'You also have a Hookdeck Outpost project, and ' + + 'OUTPOST_API_KEY is set in your environment. It is an ' + + 'ordinary Hookdeck project API key scoped to that ' + + 'project: use it for the Outpost API, which has its own ' + + 'subdomain.', + ] + : []), ...(options.mcpServers ?? []) .map((s) => s.promptAddendum) .filter((p): p is string => Boolean(p)), diff --git a/packages/hookdeck/src/seed.ts b/packages/hookdeck/src/seed.ts index 9df8a66..c6a5f0b 100644 --- a/packages/hookdeck/src/seed.ts +++ b/packages/hookdeck/src/seed.ts @@ -84,6 +84,22 @@ export interface OutpostSeed { * so this removes them for the run rather than for good. */ clearOperatorEventDestinations?: boolean; + /** + * Tenants to delete before the agent runs, and not recreate. + * + * For scenarios that score an agent for *creating* something. `outpost-001` + * asks for a tenant to be set up and had no seed at all, so a tenant left by + * any other Outpost scenario — they all use `acme` — satisfied every check + * with no agent action, and the row published green. + * + * Scoping the check by `created_at` instead does not work: tenant create is + * idempotent, so an agent that correctly `PUT`s an existing id gets the + * original timestamp back and is scored as having inherited a leftover. + * Measured on 24 August — the tenant read two minutes older than the lease + * that was about to score it. The state has to be absent, not merely + * distinguishable. + */ + deleteTenants?: string[]; tenants?: { id: string; topics?: string[]; @@ -352,6 +368,12 @@ export async function applyOutpostSeed( } } + for (const id of seed.deleteTenants ?? []) { + await outpost('DELETE', `/tenants/${encodeURIComponent(id)}`).catch( + () => undefined + ); + } + if (seed.clearOperatorEventDestinations) { await clearOperatorEventDestinations(outpost); }