bench: checkout intents and per-tool failure reporting

bench/README.md

@@ -127,6 +127,13 @@ and deletes every browser.
 
 - `stage` is `find`, `inspect`, `compare`, `prepare` or `gap`. A `gap` intent is one no tool
   covers; it names no tools and serves as the placebo check.
+- A `prepare` intent walks a site's flow part way, such as putting a fare in Delta's cart and
+  stopping on Trip Summary. `actions` lists the tools that change site state that it may
+  register as well as the read-only ones. `check` names a read-only tool, an optional `input` and
+  `rules`: once the agent is done, the runner invokes that tool in the browser, in both arms
+  (control gets the tools registered only then), and grades its output, so the end state is read
+  from the site rather than taken from the agent's answer. `same_as_answer` lists `{answer,
+  end_state}` path pairs whose values must match, so the answer reports what is in the cart.
 - `tools` lists kernel/xwebmcp refs (or full `sha`s) to register from. Treatment gets every tool
   `registry.json` lists for the domain at those commits whose file declares
   `readOnlyHint: true`; later sources replace earlier tools of the same name, so a PR to an
@@ -178,6 +185,9 @@ by name.
 1. Extract an answer from the final message: the last fenced JSON block, then a trailing object,
    then `key: value` lines naming schema properties. Nothing extracted is `malformed`.
 2. A schema or rule failure is `wrong`; otherwise `correct`, which means plausible, not verified.
+3. For a `prepare` intent, the end state must also pass the check's rules and match the answer
+   at each `same_as_answer` pair, or it is `wrong`. The check tool's output is saved as
+   `end-state.json` next to the transcript.
 
 A successful call outside the allowlist, a pi error or a missing cost is `crashed`; hitting a
 cap is `timeout`. `node bench/grade.mjs bench/results/<run_id>/trials/<intent>/<trial>` regrades
@@ -201,9 +211,11 @@ scored:
   slower arm is `mixed` when it was correct in 2 fewer pairs, gave the worse `better` answer in
   2 more, or the arms' `same` answers differed in 2 pairs.
 
-Then the worst label per PR, and per run: spend over every attempt including smoke gates, the
-label histogram (noting labels from `shape only` intents), median speedup, each arm's
-agent-minutes, turns and spend over complete pairs, the pooled gap-intent
+A tool failures table lists, per tool, the treatment invocations that did not complete or whose
+tool returned `status: "error"`, with their most common error messages. Then the worst label per
+PR, and per run: spend over every attempt including smoke gates, the label histogram (noting
+labels from `shape only` intents), median speedup, each arm's agent-minutes, turns and spend
+over complete pairs, the pooled gap-intent
 placebo, an order check, a start delay check, a native tools check (control trials that started
 with the page's own WebMCP tools), cap calibration, and how many labels to expect
 from noise.

bench/grade.mjs

@@ -3,7 +3,7 @@
 // rules. The first failing step decides the outcome.
 // Usage: node bench/grade.mjs bench/results/<run_id>/trials/<intent>/<trial>
 
-import { readFileSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
 import { basename, dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { summarize } from "./lib/agent.mjs";
@@ -99,6 +99,16 @@ function isDate(value) {
   return date.toISOString().slice(0, 10) === m[0];
 }
 
+// Compares what the agent reported with what the check tool read, as sets: strings ignoring case
+// and whitespace, amounts to the cent.
+export function checkSameAsAnswer({ answer: answerPath, end_state: endStatePath }, answer, output) {
+  const canonical = (value) => typeof value === "string" ? value.toLowerCase().replace(/\s+/g, "") : typeof value === "number" ? Math.round(value * 100) / 100 : value;
+  const values = (value, path) => resolvePath(value, path).map(canonical).map(v => JSON.stringify(v)).sort();
+  const [reported, read] = [values(answer, answerPath), values(output, endStatePath)];
+  if (JSON.stringify(reported) === JSON.stringify(read)) return null;
+  return `${answerPath} is [${reported.join(", ")}] in the answer but ${endStatePath} is [${read.join(", ")}] on the site`;
+}
+
 export function checkRule(rule, answer) {
   const values = resolvePath(answer, rule.path);
   if (!values.length) return `${rule.path}: no value`;
@@ -119,8 +129,10 @@ export function checkRule(rule, answer) {
   return null;
 }
 
-export function gradeAnswer(intent, schema, text) {
-  const grade = { outcome: "malformed", extracted: false, shape: false, rules: false, answer: null, problems: [] };
+// `endState` is what the runner read from the browser with a prepare intent's check tool:
+// { output } or { error }.
+export function gradeAnswer(intent, schema, text, endState = null) {
+  const grade = { outcome: "malformed", extracted: false, shape: false, rules: false, end_state: null, answer: null, problems: [] };
   const answer = extractAnswer(text, schema);
   if (!answer) {
     grade.problems.push("no answer object in the final message");
@@ -135,6 +147,15 @@ export function gradeAnswer(intent, schema, text) {
   grade.problems = fillRules(intent.rules, intent.fixture).map(rule => checkRule(rule, answer)).filter(Boolean);
   if (grade.problems.length) return grade;
   grade.rules = true;
+  if (intent.check) {
+    grade.problems = endState?.error ? [`end state: ${endState.error}`]
+      : [
+        ...fillRules(intent.check.rules, intent.fixture).map(rule => checkRule(rule, endState?.output)),
+        ...(intent.check.same_as_answer ?? []).map(pair => checkSameAsAnswer(pair, answer, endState?.output)),
+      ].filter(Boolean).map(problem => `end state: ${problem}`);
+    grade.end_state = !grade.problems.length;
+    if (grade.problems.length) return grade;
+  }
   grade.outcome = "correct";
   return grade;
 }
@@ -148,5 +169,7 @@ if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.ur
   const run = JSON.parse(readFileSync(join(trialDir, "../../../run.json"), "utf8"));
   const { intent, schema } = run.intents[basename(dirname(trialDir))];
   const events = readFileSync(join(trialDir, "transcript.jsonl"), "utf8").split("\n").filter(Boolean).map(line => JSON.parse(line));
-  console.log(JSON.stringify(gradeAnswer(intent, schema, summarize(events, []).answer), null, 2));
+  const endStateFile = join(trialDir, "end-state.json");
+  const endState = existsSync(endStateFile) ? JSON.parse(readFileSync(endStateFile, "utf8")) : null;
+  console.log(JSON.stringify(gradeAnswer(intent, schema, summarize(events, []).answer, endState), null, 2));
 }

bench/lib/agent.mjs

@@ -140,6 +140,23 @@ function collectRefs(value, refs, depth = 0) {
   for (const child of Object.values(value)) collectRefs(child, refs, depth + 1);
 }
 
+// The invocation's status, the tool's own status, and an error message when either failed. A
+// call the MCP server rejected, such as one with a stale tool_ref, comes back as plain text.
+function invocationResult(call) {
+  if (!call.done) return { status: "no_result", tool_status: null, error: null };
+  const result = parseJson(call.text ?? "");
+  if (result === null || typeof result !== "object") {
+    return { status: call.isError || /^error\b/i.test(call.text ?? "") ? "error" : "unknown", tool_status: null, error: (call.text ?? "").slice(0, 300) || null };
+  }
+  const output = result.output !== null && typeof result.output === "object" ? result.output : {};
+  const toolStatus = typeof output.status === "string" ? output.status : null;
+  const failed = result.status !== "completed" || toolStatus === "error";
+  const error = failed ? result.error_text ?? output.error_message ?? output.message ?? output.error_kind ?? result.status ?? null : null;
+  return { status: result.status ?? "unknown", tool_status: toolStatus, error: typeof error === "string" ? error.slice(0, 300) : null };
+}
+
+export const invocationFailed = (invocation) => invocation.status !== "completed" || invocation.tool_status === "error";
+
 const WEBMCP = "mcp__kernel__webmcp";
 const CODE_PATHS = { mcp__kernel__execute_playwright_code: "playwright", mcp__kernel__browser_repl: "repl" };
 
@@ -182,17 +199,16 @@ export function summarize(events, allow) {
     if (call.name === WEBMCP && args.action === "list") {
       webmcp.listed = true;
     } else if (call.name === WEBMCP && args.action === "invoke") {
-      const output = parseJson(call.text ?? "");
       webmcp.invocations.push({
         path: "mcp",
         tool: refs.get(args.tool_ref) ?? args.tool_ref ?? null,
-        status: output?.status ?? (call.done ? (call.isError ? "error" : "unknown") : "no_result"),
+        ...invocationResult(call),
         duration_ms: call.done && call.startedAt !== undefined ? call.endedAt - call.startedAt : null,
       });
     } else if (CODE_PATHS[call.name] && typeof args.code === "string") {
       if (/\blistTools\s*\(/.test(args.code)) webmcp.listed = true;
       if (/\binvokeTool\s*\(/.test(args.code)) {
-        webmcp.invocations.push({ path: CODE_PATHS[call.name], tool: null, status: null, duration_ms: null });
+        webmcp.invocations.push({ path: CODE_PATHS[call.name], tool: null, status: null, tool_status: null, error: null, duration_ms: null });
       }
     }
   }

bench/lib/intents.mjs

@@ -76,7 +76,16 @@ export function validateIntent(intent, benchDir) {
   need(intent.stage === "gap" ? named.length === 0 : named.length > 0, intent.stage === "gap" ? "gap intents name no tools" : "non-gap intents name the tools a treatment agent would use");
   need(typeof intent.entry_tool === "string", "entry_tool is required");
   need(typeof intent.answer_schema === "string" && existsSync(join(benchDir, intent.answer_schema)), "answer_schema must point at a file under bench/");
-  for (const rule of intent.rules ?? []) {
+  // A prepare intent changes site state, so the runner reads where the agent left the browser.
+  const { check, actions } = intent;
+  need(intent.stage === "prepare" ? check !== undefined : check === undefined && actions === undefined, "check, and actions, belong to prepare intents, and a prepare intent needs a check");
+  need(actions === undefined || (Array.isArray(actions) && actions.every(name => typeof name === "string")), "actions must list tool names");
+  need(check === undefined || (typeof check?.tool === "string" && Array.isArray(check.rules) && check.rules.length > 0 && (check.input === undefined || (check.input !== null && typeof check.input === "object"))),
+    "check needs a tool, rules and, optionally, an input object");
+  const sameAsAnswer = check?.same_as_answer;
+  need(sameAsAnswer === undefined || (Array.isArray(sameAsAnswer) && sameAsAnswer.every(pair => typeof pair?.answer === "string" && typeof pair?.end_state === "string")),
+    "check.same_as_answer must list {answer, end_state} paths");
+  for (const rule of [...(intent.rules ?? []), ...(Array.isArray(check?.rules) ? check.rules : [])]) {
     const checks = Object.keys(rule).filter(key => key !== "path");
     need(typeof rule.path === "string" && checks.length === 1 && RULE_CHECKS.includes(checks[0]), `rule ${JSON.stringify(rule)} needs a path and one of ${RULE_CHECKS.join(", ")}`);
     if ("matches" in rule) need(typeof rule.matches === "string" && compiles(fillRules([rule], intent.fixture)[0].matches, "iu"), `rule ${JSON.stringify(rule)} needs a valid regex`);
@@ -89,7 +98,7 @@ export function validateIntent(intent, benchDir) {
   for (const [key, value] of Object.entries(intent.fixture ?? {})) {
     need(typeof value === "string" || typeof value === "number" || relativeDate(value), `fixture ${key} must be a string, a number or {"days_from_start": n}`);
   }
-  const text = JSON.stringify([intent.prompt, intent.rules ?? []]);
+  const text = JSON.stringify([intent.prompt, intent.rules ?? [], check?.rules ?? [], check?.input ?? {}]);
   for (const [, key] of text.matchAll(/\{\{(\w+)\}\}/g)) need(Object.hasOwn(intent.fixture ?? {}, key), `{{${key}}} has no fixture value`);
   for (const key of ["pairs", "trial_timeout_sec", "max_turns"]) need(Number.isInteger(intent[key]) && intent[key] > 0, `${key} must be a positive integer`);
   return problems;

bench/lib/tools.mjs

@@ -74,7 +74,9 @@ function namespaceTools(show, sha, namespace) {
 }
 
 // Sources apply in order, so a branch listed after main replaces main's tool of the same name.
+// A prepare intent's actions are tools that change site state, registered as well.
 export async function registeredTools(intent, shas, show) {
+  const actions = intent.actions ?? [];
   const byName = new Map();
   const known = new Set();
   const problems = [];
@@ -85,15 +87,15 @@ export async function registeredTools(intent, shas, show) {
       if (!all.some(tool => tool.name === name)) problems.push(`${name} is not in registry.json at ${source.ref} (${shas[i].slice(0, 7)})`);
     }
     for (const tool of all) {
-      if (tool.readOnly && !Object.hasOwn(intent.exclude, tool.name)) byName.set(tool.name, tool);
+      if ((tool.readOnly || actions.includes(tool.name)) && !Object.hasOwn(intent.exclude, tool.name)) byName.set(tool.name, tool);
       else byName.delete(tool.name);
     }
   }
-  for (const name of Object.keys(intent.exclude)) {
-    if (!known.has(name)) problems.push(`exclude names ${name}, which no source has`);
+  for (const [list, name] of [...Object.keys(intent.exclude).map(name => ["exclude", name]), ...actions.map(name => ["actions", name])]) {
+    if (!known.has(name)) problems.push(`${list} names ${name}, which no source has`);
   }
-  for (const name of [...intent.tools.flatMap(s => s.names), intent.entry_tool]) {
-    if (!byName.has(name) && !problems.some(p => p.startsWith(`${name} `))) problems.push(`${name} is not read-only or is excluded, so it would not be registered`);
+  for (const name of [...intent.tools.flatMap(s => s.names), intent.entry_tool, ...(intent.check ? [intent.check.tool] : [])]) {
+    if (!byName.has(name) && !problems.some(p => p.startsWith(`${name} `))) problems.push(`${name} is excluded or neither read-only nor an action, so it would not be registered`);
   }
   if (problems.length) throw new Error(`${intent.id}: ${problems.join("; ")}`);
   return [...byName.values()].sort((a, b) => a.name.localeCompare(b.name));

bench/prompts/system.md

@@ -10,7 +10,7 @@ Use the page's WebMCP tools first:
 
 Rules:
 
-- Do not check out, pay, place an order, or complete a booking or reservation.
+- Do not pay, place an order, or complete a booking or reservation, and go no further into a checkout than the task asks.
 - Do not add items or tickets to a cart unless the task says you may. Selecting seats and ticket types to see their prices is allowed.
 - Do not log in, sign up, or enter personal or payment details.
 - Do not post, comment, send messages, star, follow, or change anything else on the site.

bench/report.mjs

@@ -8,6 +8,7 @@ import { fileURLToPath } from "node:url";
 import { completePairRows, latestRows, readRows } from "./lib/rows.mjs";
 import { median, newcombe, signTest, signTestRate, wilson } from "./lib/stats.mjs";
 import { resolvePath } from "./grade.mjs";
+import { invocationFailed } from "./lib/agent.mjs";
 
 // The speedup ratios are recalibrated after the pilot.
 const THRESHOLDS = { adoption: 0.5, minBothCorrect: 5, p: 0.1, faster: 1.5, slower: 0.67, mixedDelta: 2 };
@@ -44,7 +45,7 @@ export function armStats(rows) {
     min_per_correct: correct.length ? sum(all) / 60_000 / correct.length : null,
     adoption: rows.length ? rows.filter(r => r.webmcp?.adopted).length / rows.length : null,
     invocations: invocations.length,
-    invoke_success: withStatus.length ? withStatus.filter(i => i.status === "completed").length / withStatus.length : null,
+    invoke_success: withStatus.length ? withStatus.filter(i => !invocationFailed(i)).length / withStatus.length : null,
     tool_median: median(withStatus.map(i => i.duration_ms).filter(d => d !== null)),
     turns: median(rows.map(r => r.turns).filter(t => typeof t === "number")),
     tokens_in: median(rows.map(r => r.tokens_in).filter(t => typeof t === "number")),
@@ -106,6 +107,7 @@ export function adoptedPairStats(rows, agreement = null) {
 
 // A rule that only repeats the fixture back pins nothing.
 export function answerCheck(intent) {
+  if (intent.check) return "end state";
   const pinned = (intent.rules ?? []).some(rule => ("equals" in rule || "in" in rule) && !JSON.stringify(rule.equals ?? rule.in).includes("{{"));
   return pinned ? "pinned" : intent.agreement ? "agreement" : "shape only";
 }
@@ -139,6 +141,27 @@ export function compare(control, treatment, pairs, stage) {
   };
 }
 
+// Failed invocations per tool, with their most common errors, over treatment trials.
+export function toolFailures(rows, domainOf) {
+  const byTool = new Map();
+  for (const row of rows.filter(r => r.arm === "treatment")) {
+    for (const invocation of row.webmcp?.invocations ?? []) {
+      if (invocation.status === null) continue;
+      const key = `${domainOf(row.intent)}/${invocation.tool ?? "unknown"}`;
+      const entry = byTool.get(key) ?? { tool: key, calls: 0, failed: 0, errors: new Map() };
+      entry.calls++;
+      if (invocationFailed(invocation)) {
+        entry.failed++;
+        const error = invocation.error ?? invocation.status;
+        entry.errors.set(error, (entry.errors.get(error) ?? 0) + 1);
+      }
+      byTool.set(key, entry);
+    }
+  }
+  return [...byTool.values()].filter(e => e.failed).sort((a, b) => b.failed - a.failed || a.tool.localeCompare(b.tool))
+    .map(e => ({ ...e, errors: [...e.errors].sort((a, b) => b[1] - a[1]) }));
+}
+
 // --- formatting -----------------------------------------------------------------------------
 
 const na = "n/a";
@@ -313,6 +336,16 @@ export function buildReport(run, allRows, smokeRows = []) {
       + "intents, so up to that many faster or slower labels may be noise.");
   }
 
+  const failures = toolFailures(scored, id => run.intents[id]?.intent.domain ?? id);
+  if (failures.length) {
+    const unrecorded = scored.filter(r => r.arm === "treatment").flatMap(r => r.webmcp?.invocations ?? []).filter(i => i.status === null).length;
+    const unrecordedText = unrecorded ? ` ${unrecorded} invocation(s) made through Playwright or the REPL have no recorded result and are not counted.` : "";
+    out.push("", "## Tool failures", "", `Invocations that did not complete or whose tool returned \`status: "error"\`, in scored treatment trials.${unrecordedText}`, "");
+    out.push(table(["tool", "calls", "failed", "most common errors"], failures.map(f => [
+      f.tool, f.calls, f.failed, f.errors.slice(0, 3).map(([error, n]) => `${n}× ${error.replace(/\|/g, "\\|").replace(/\n/g, " ").slice(0, 160)}`).join("<br>"),
+    ])));
+  }
+
   const flagged = rows.filter(r => r.outcome === "aborted" || r.outcome === "crashed" || r.registration_changed);
   if (flagged.length) {
     out.push("", "## Trials to look at", "");