From e5c377cfaa1593ccf184c79d4c0c0f16222e68bd Mon Sep 17 00:00:00 2001 From: Matthew Podwysocki Date: Mon, 12 Oct 2026 09:58:19 -0400 Subject: [PATCH 1/3] feat(evals): add tool-selection eval for description routing regressions Runs 17 prompts against Claude and grades which tools it calls against expected trajectories, with all descriptions loaded and again behind the API's tool search. The tool surface comes from a real McpServer read back over MCP; tool results are canned per case, so no Mapbox calls are made. EVAL_SERVER_ROOT compares another checkout (e.g. a PR worktree). Runner and lite report builder are vendored from the claude-api skill's eval scaffold. Adds @anthropic-ai/sdk and tsx as dev dependencies. Co-Authored-By: Claude Opus 5.5 --- .claude/hillclimb/tool-selection/_state.json | 36 ++ .gitignore | 5 + CHANGELOG.md | 4 + evals/tool-selection/README.md | 60 ++ evals/tool-selection/build-report-lite.mjs | 488 +++++++++++++++ evals/tool-selection/cases.json | 253 ++++++++ evals/tool-selection/harness.ts | 492 +++++++++++++++ evals/tool-selection/rescore.ts | 49 ++ evals/tool-selection/run-eval.mjs | 591 +++++++++++++++++++ package-lock.json | 583 ++++++++++++++++++ package.json | 6 +- 11 files changed, 2566 insertions(+), 1 deletion(-) create mode 100644 .claude/hillclimb/tool-selection/_state.json create mode 100644 evals/tool-selection/README.md create mode 100644 evals/tool-selection/build-report-lite.mjs create mode 100644 evals/tool-selection/cases.json create mode 100644 evals/tool-selection/harness.ts create mode 100644 evals/tool-selection/rescore.ts create mode 100644 evals/tool-selection/run-eval.mjs diff --git a/.claude/hillclimb/tool-selection/_state.json b/.claude/hillclimb/tool-selection/_state.json new file mode 100644 index 00000000..e819e0dd --- /dev/null +++ b/.claude/hillclimb/tool-selection/_state.json @@ -0,0 +1,36 @@ +{ + "flow": "tool-selection", + "metrics": [ + { + "id": "pass", + "label": "Pass", + "kind": "binary" + }, + { + "id": "first_ok", + "label": "First tool ok", + "kind": "binary" + }, + { + "id": "no_forbidden", + "label": "No forbidden", + "kind": "binary" + }, + { + "id": "ref_ok", + "label": "Ref passed", + "kind": "binary" + } + ], + "perf_fields": [ + "tool_calls", + "tool_searches", + "latency_s", + "usage" + ], + "harness_paths": [ + "evals/tool-selection/cases.json", + "evals/tool-selection/harness.ts" + ], + "harness_sha": "31d09b8e2336c02342caeed5edbd9a5ddfc6d4760662d16fcdc217c38e780a6f" +} diff --git a/.gitignore b/.gitignore index 7fb42f77..d3fe1840 100644 --- a/.gitignore +++ b/.gitignore @@ -155,3 +155,8 @@ test-results.xml # Build artifacts *.tsbuildinfo PR_DESCRIPTION.md + +# Tool-selection eval output (state and case files are committed) +.claude/hillclimb/*/*/ +.claude/hillclimb/*/report.html +.claude/hillclimb/*/trajectory/ diff --git a/CHANGELOG.md b/CHANGELOG.md index db463284..55350b45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## Unreleased +### Other Features + +- **Tool-selection eval for catching routing regressions from tool descriptions.** `npm run eval:tools` runs 17 prompts against Claude (Opus 5.5 by default) and grades which tools it calls against the expected ones: map only when asked, `static_map_image_tool` only for image files, `ground_location_tool` vs `reverse_geocode_tool`, brand vs category search, and whether web search stays available for place questions. The tool list and server instructions come from a real `McpServer` built the way the hosted deployment builds it and read back over MCP. Each prompt runs with all descriptions loaded and again with the tools behind the API's tool search, which approximates hosts like Claude Desktop that load tools on demand. Tool results are canned per case, so Mapbox APIs are never called. A grader check also confirms that every `render_map_tool` call passes the map ref unchanged. `EVAL_SERVER_ROOT` points the harness at another checkout, so two branches can be compared, and `npm run eval:tools:report` builds an HTML report with per-case transcripts. It calls the Claude API and costs about $7 per 102-row run, so it is not part of `npm test`. Adds `@anthropic-ai/sdk` and `tsx` as dev dependencies. See `evals/tool-selection/README.md`. + ## 0.15.1 - 2026-10-09 ### New Features diff --git a/evals/tool-selection/README.md b/evals/tool-selection/README.md new file mode 100644 index 00000000..cb7b51fb --- /dev/null +++ b/evals/tool-selection/README.md @@ -0,0 +1,60 @@ +# Tool-selection eval + +Measures which tools Claude picks for a prompt, given this server's tool descriptions and server instructions. Use it to catch routing regressions when you change a description, the server instructions, or the render hint that data tools append to their results. + +It is not part of `npm test`: it calls the Claude API and costs money. Mapbox APIs are never called, since every tool result is a canned stub. + +## How it works + +- **Tool surface:** `harness.ts` builds a real `McpServer` the same way the hosted deployment does (`publishJsonSchema2020`, `installTo`, `buildServerInstructions`) and reads `tools/list` and the server instructions back over an in-memory MCP connection. +- **Host simulation:** two host tools sit next to the Mapbox tools. `web_search` and `places_map_display_v0` let us see whether our descriptions crowd out other sources. +- **Two modes per case:** + - `eager`: every tool description is loaded up front. + - `deferred`: Mapbox tools are marked `defer_loading` behind the API's tool search. This approximates hosts like Claude Desktop that only load tools on demand, where the server instructions do most of the routing. +- **Conversation:** up to 5 model calls per case. Tool calls get stub results, and data tools get the server's real render hint appended. +- **Grading:** programmatic, against each case's `expect` in `cases.json`: + - `first`: acceptable first tool + - `must`: tools that must all be called + - `any_of`: at least one of these must be called + - `must_not`: none may be called; `*mapbox*` means any Mapbox tool +- **Metrics:** `pass` (all checks hold), `first_ok` and `no_forbidden`. + +## Running + +The runner reads `ANTHROPIC_API_KEY` from the environment. + +```bash +# Baseline: the current checkout +npm run eval:tools -- --variant baseline --reps 3 + +# Compare another branch (e.g. a PR) from a worktree +git worktree add --detach ../mcp-server-pr origin/ +ln -s "$PWD/node_modules" ../mcp-server-pr/node_modules +EVAL_SERVER_ROOT=../mcp-server-pr npm run eval:tools -- --variant v1 --reps 3 + +# Build the HTML report (baseline vs v1, per case) +npm run eval:tools:report +open .claude/hillclimb/tool-selection/report.html +``` + +- **Options:** `--model` (default `claude-opus-5-5`), `--reps`, `--concurrency` (default 8) and `--timeout-s` (default 600). +- **Resume:** runs pick up where they left off. Each `(case, rep)` is skipped once it has a row in `results.jsonl`. +- **Failures:** failed attempts go to `errors.jsonl` instead. +- **Variants:** a v1 run should note what changed in `.claude/hillclimb/tool-selection/v1/change.md`, so the report can explain the difference. + +**First run:** the runner refuses to start until you approve the harness. Review `harness.ts` and `cases.json`, then add `--approve-harness` once. It records a hash of the runner, the lockfile and those two files in `_state.json`. Any later edit to them needs another approval. + +## Adding cases + +Add an entry to `cases.json` with a `why` that names the description or instruction it exercises. + +Give it `stubs` for every tool the prompt might reasonably call, written about the place in the prompt. A canned answer about the wrong place makes Claude retry or switch tools, and that gets scored as a routing failure. Calls without a stub return "No results" and are counted in the `unstubbed` column. A non-zero count means the case needs another stub, not that the model misrouted. + +The grader throws if a case names a tool that doesn't exist, so a renamed tool can't turn a `must_not` into a silent pass. The noise floor for a pass rate is roughly `1/sqrt(rows x reps)`. At 34 rows x 3 reps that's about ±10 points overall, and much wider for any single case. + +## Files + +- `harness.ts`: tool surface, host tools, stubs, conversation loop, grader +- `cases.json`: prompts and expected trajectories +- `run-eval.mjs`: runner, vendored from the claude-api skill's eval scaffold (resume, backoff, wall-clock ceiling, served-model check, failure sidecar) +- `build-report-lite.mjs`: static report builder, vendored from the same skill diff --git a/evals/tool-selection/build-report-lite.mjs b/evals/tool-selection/build-report-lite.mjs new file mode 100644 index 00000000..ba1c1ba0 --- /dev/null +++ b/evals/tool-selection/build-report-lite.mjs @@ -0,0 +1,488 @@ +#!/usr/bin/env node +// Lite report builder for the build-eval / hillclimb loop: a single static +// report.html with no vendored runtime, fonts, or markdown engine. +// +// node build-report-lite.mjs .claude/hillclimb// +// +// Reads the same on-disk layout as build-report.mjs (baseline/, v/, +// results.jsonl, errors.jsonl, change.md, summary.json, traces/, _state.json) +// and derives the same numbers: per-variant mean of the primary metric over +// status-ok rows, per-case means, split (train/val/test) from _state.json, +// failed-attempt counts from errors.jsonl. It also writes +// trajectory/scores.tsv exactly as the full builder does, so hillclimb's +// per-round bookkeeping is identical whichever builder ran. +// +// What it deliberately does not do: render markdown, inline transcripts or +// attachments, or execute anything from the data. Every string from disk is +// HTML-escaped; traces are linked by relative path, not embedded. Sorting is +// a few lines of inline script over the rendered table only. +// +// Runs under node or bun. Node builtins only. + +import { closeSync, constants as FS, fstatSync, ftruncateSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from 'node:fs'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; + +const isNum = x => typeof x === 'number' && Number.isFinite(x); +// readText/readJSON refuse symlinks, like the full builder's lib/adapter.mjs: +// the flow dir is model-influenced, and a prompt-injected agent could plant +// `summary.json -> ~/.ssh/id_rsa`; the user later runs this builder (outside +// any sandbox) and the target lands in the shareable report.html. Open +// O_NOFOLLOW and fstat the fd (not lstat-then-read) so a concurrent writer +// can't swap in a symlink between the check and the read. +// O_NOFOLLOW is only trustworthy on POSIX: Node leaves it undefined on +// Windows and Bun defines it there with a meaningless nonzero value that +// makes every open fail, so the flag is applied by platform, and +// Windows falls back to an lstat refusal - a check-then-use window, but the +// alternative is no symlink check at all. +const WIN = process.platform === 'win32'; +const NOFOLLOW = WIN ? 0 : FS.O_NOFOLLOW; +// A guard that cannot tell must refuse: only "no such entry" reads as absent; +// any other lstat failure (EACCES, ENAMETOOLONG, ...) is rethrown, never "no". +const isSymlink = p => { + try { return lstatSync(p).isSymbolicLink(); } catch (e) { if (e?.code === 'ENOENT') return false; throw e; } +}; +// Files skipped for having a second hard link, so build() can say so as well +// as reporting them as missing or empty. +const skippedHardLinks = []; +const readText = p => { + let fd; + try { + if (WIN && isSymlink(p)) return ''; + fd = openSync(p, FS.O_RDONLY | NOFOLLOW); + const st = fstatSync(fd); + // O_NOFOLLOW and lstat cannot see a hard link (a second name for a file + // outside the flow dir); nothing in a flow dir needs one, so read nothing. + if (!st.isFile()) return ''; + if (st.nlink > 1) { skippedHardLinks.push(p); return ''; } + return readFileSync(fd, 'utf8'); + } catch { return ''; } + finally { if (fd !== undefined) closeSync(fd); } +}; +// Output writes get the same discipline as reads: the flow dir is +// model-influenced, so `report.html -> ~/.bashrc` planted there must not be +// followed. POSIX opens O_NOFOLLOW (a symlink fails with ELOOP); Windows +// lstat-refuses first. A symlinked parent dir is refused the same way. +// The immediate-parent lstat can't see a symlink higher up the path, so +// writes are also bound to the flow root resolved ONCE at build entry +// (resolveOutputRoot below). +// Re-resolving the caller-named path here would self-anchor: root and +// target would resolve through the same planted link and the check would +// pass vacuously. +function assertContained(dir, root) { + if (root == null) throw new Error('refusing to write: output root not resolved'); + const dirReal = realpathSync(dir); + if (dirReal !== root && !dirReal.startsWith(root + (WIN ? '\\' : '/'))) + throw new Error('refusing to write outside ' + root + ': ' + dir); +} +// Refuse a symlink at the flow path itself, then pin its resolution for the +// whole build. Takes the path AS THE CALLER NAMED IT: a RELATIVE root is +// lstat-walked component by component from the cwd first (a pre-planted +// ancestor link would otherwise relocate the anchor itself), and a trailing +// separator is stripped (lstat("link/") follows the final symlink). +// Residual: an ancestor link above an ABSOLUTE root stays the caller's own +// trust decision (it can be legitimate - macOS /tmp); a swap after entry +// fails assertContained when the check sees it. Residual, all platforms: the +// check and the open are separate path lookups (Node's sync fs has no +// openat-style call), so a directory swapped for a symlink in between is +// still followed. This stops a planted link, not a writer racing the build. +function resolveOutputRoot(root) { + // Only Windows treats `\` as a separator; on POSIX it is a filename byte, so + // splitting on it would walk prefixes that are not real path components. + root = String(root).replace(WIN ? /(.)[\\/]+$/ : /(.)\/+$/, '$1'); + const segments = root.split(WIN ? /[\\/]/ : '/'); + // A `.`/`..` segment (e.g. a trailing `/.`) makes the leaf lstat below + // follow a planted link at the flow root, and the absolute branch skips the + // ancestor walk. Refuse dot segments outright. + if (segments.some(seg => seg === '.' || seg === '..')) + throw new Error('refusing to write into ' + root + ": '.'/'..' path segment"); + if (!isAbsolute(root)) { + let walk = ''; + for (const part of segments.filter(Boolean).slice(0, -1)) { + walk = walk ? join(walk, part) : part; + if (isSymlink(walk)) throw new Error('refusing to write into ' + root + ': ancestor ' + walk + ' is a symlink'); + } + } + if (isSymlink(root)) throw new Error('refusing to write into ' + root + ': it is a symlink'); + return realpathSync(root); +} +function writeNoFollow(p, data, root) { + if (isSymlink(dirname(p))) throw new Error('refusing to write through symlinked directory: ' + dirname(p)); + assertContained(dirname(p), root); + if (WIN && isSymlink(p)) throw new Error('refusing to write through symlink: ' + p); + // Opened without O_TRUNC and truncated only after the checks, so a refused + // file keeps its bytes. + const fd = openSync(p, FS.O_WRONLY | FS.O_CREAT | NOFOLLOW, 0o644); + try { + const st = fstatSync(fd); + if (!st.isFile()) throw new Error('refusing to write to non-regular file: ' + p); + if (st.nlink > 1) throw new Error('refusing to write to ' + p + ': it has a second hard link (another name for the same file); replace it with a plain copy if it is yours'); + ftruncateSync(fd, 0); + // writeFileSync on the fd loops until every byte lands (a bare writeSync + // is one write(2) that may return short on ENOSPC and silently truncate). + writeFileSync(fd, data); + } finally { closeSync(fd); } +} +function mkdirNoFollow(dir, root) { + if (isSymlink(dir)) throw new Error('refusing to use symlinked directory: ' + dir); + mkdirSync(dir, { recursive: true }); + // Check after creating: mkdirSync(recursive) follows symlinked ancestors. + assertContained(dir, root); +} +const readJSON = (p, dflt) => { try { return JSON.parse(readText(p)); } catch { return dflt; } }; +// Warnings/errors interpolate model-influenced bytes (directory names, +// JSON.parse messages). Strip escape sequences and control characters before +// they reach the terminal, so a planted +// OSC/CSI can't retitle the terminal or forge output lines. +const ESC_SEQ = /\x1b\[[0-?]*[ -\/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)?|\x1b[@-_]/g; +const CONTROL = /[\x00-\x1f\x7f-\x9f]/g; +const termSafe = s => String(s).replace(ESC_SEQ, '').replace(CONTROL, ''); +const eprint = (...a) => console.error(...a.map(termSafe)); +const esc = s => String(s ?? '').replace(/[&<>"']/g, c => + ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }[c])); +const fmt = x => (x == null ? '' : x.toFixed(3)); +// TSV id cells come from untrusted results.jsonl: strip the separators that +// would forge rows/columns in trajectory/scores.tsv, and neutralize a leading +// formula trigger so spreadsheet apps don't execute '=WEBSERVICE(...)' on +// open. Mirrored in the full builder's lib/adapter.mjs so both emit +// byte-identical scores.tsv. +const tsvCell = s => { + const flat = String(s).replace(/[\t\n\r]/g, ' '); + return /^[=+\-@]/.test(flat) ? "'" + flat : flat; +}; + +// Variant directories: exactly `baseline` or `v`, numeric order. +function discoverVariants(flow) { + const names = readdirSync(flow, { withFileTypes: true }) + .filter(e => e.isDirectory() && /^(baseline|v\d+)$/.test(e.name)) + .map(e => e.name); + const rank = n => (n === 'baseline' ? -1 : +n.slice(1)); + return names.sort((a, b) => rank(a) - rank(b)); +} + +// Mirrors the full adapter: an object of numeric/boolean grades, a bare +// boolean, or a bare number (-> {score}). +function coerceScores(g) { + if (g && typeof g === 'object' && !Array.isArray(g)) { + const out = {}; + for (const [k, v] of Object.entries(g)) if (isNum(v) || typeof v === 'boolean') out[k] = +v; + return out; + } + if (typeof g === 'boolean') return { score: g ? 1 : 0 }; + if (isNum(g)) return { score: g }; + return {}; +} + +function loadRows(p, warnings, dir) { + // Keyed by data-supplied ids: null-prototype so '__proto__' is a key, not a crash. + const byId = Object.create(null); + readText(p).split('\n').forEach((line, i) => { + line = line.trim(); + if (!line) return; + let r; + try { r = JSON.parse(line); } catch { warnings.push(dir + '/results.jsonl:' + (i + 1) + ': malformed JSON, skipped'); return; } + const pid = String(r.prompt_id ?? r.id ?? r.case_id ?? ''); + if (!pid) { warnings.push(dir + '/results.jsonl:' + (i + 1) + ': no prompt_id, skipped'); return; } + (byId[pid] ||= []).push(r); + }); + return byId; +} + +function countErrors(vdir) { + let total = 0; + const byClass = Object.create(null); + for (const line of readText(join(vdir, 'errors.jsonl')).split('\n')) { + if (!line.trim()) continue; + let e; try { e = JSON.parse(line); } catch { continue; } + total++; + const c = String(e?.failure_class ?? 'error'); + byClass[c] = (byClass[c] || 0) + 1; + } + return { total, byClass }; +} + +// Inferred metrics, mirroring the full builder's inferMetrics: any +// explanation for a key -> judge, all-0/1 values -> binary, else float; +// first-seen order. +function inferMetrics(rows, variants) { + const seen = new Map(); + for (const v of variants) for (const reps of Object.values(rows[v])) for (const r of reps) { + const expl = r.explanation && typeof r.explanation === 'object' ? r.explanation : {}; + for (const [k, val] of Object.entries(coerceScores(r.grade))) { + const m = seen.get(k) || { binary: true, judge: false }; + if (val !== 0 && val !== 1) m.binary = false; + if (k in expl) m.judge = true; + seen.set(k, m); + } + } + return [...seen.entries()].map(([id, s]) => ({ id, kind: s.judge ? 'judge' : s.binary ? 'binary' : 'float' })); +} + +function build(flowArg) { + const flow = resolve(flowArg); + if (!statSync(flow, { throwIfNoEntry: false })?.isDirectory()) throw new Error('not a directory: ' + flow); + // Pinned from the path as the caller named it, so the relative-ancestor + // walk in resolveOutputRoot applies (resolve() would skip it). + const flowRoot = resolveOutputRoot(flowArg); + const variants = discoverVariants(flow); + if (!variants.length) throw new Error('no variant directories (baseline/, v1/, ...) under ' + flow); + const warnings = []; + // Same warning the full builder gives: a mis-named variant dir is the usual + // reason the header shows one variant fewer than expected. + const nonVariant = new Set(['trajectory', 'attachments', 'refs', 'ref', 'inputs', 'out', 'target', '__pycache__']); + for (const e of readdirSync(flow, { withFileTypes: true })) + if (e.isDirectory() && !variants.includes(e.name) && !nonVariant.has(e.name) + && !e.name.startsWith('.') && !e.name.startsWith('_')) + warnings.push("ignored directory '" + e.name + "/' - variant dirs must be named 'baseline' or 'v'; put the descriptive name in change.md's first line instead"); + const state = readJSON(join(flow, '_state.json'), {}) || {}; + + const splitOf = Object.create(null); + for (const sp of ['train', 'val', 'test']) { + const ids = state[sp + '_ids']; + if (ids == null) continue; + if (!Array.isArray(ids)) { warnings.push('_state.json ' + sp + '_ids is not a list - ignored'); continue; } + for (const pid of ids) splitOf[String(pid)] = sp; + } + + const rows = Object.create(null); + for (const v of variants) rows[v] = loadRows(join(flow, v, 'results.jsonl'), warnings, v); + + // Trace links are emitted only for a real traces/ directory that resolves + // inside the pinned flow root: the per-file lstat below refuses a symlinked + // trace but silently resolves a symlinked traces/ (or variant) directory, + // which would link files from wherever it points into the report. + const traceDirOk = Object.create(null); + for (const v of variants) { + const tdir = join(flow, v, 'traces'); + try { + if (isSymlink(join(flow, v)) || isSymlink(tdir)) { + warnings.push(v + '/traces is reached through a symlink - trace links omitted'); + traceDirOk[v] = false; + } else { + const real = realpathSync(tdir); + traceDirOk[v] = real.startsWith(flowRoot + (WIN ? '\\' : '/')); + } + } catch { traceDirOk[v] = false; } + } + + // Same as the full builder: drop non-baseline variants with zero result + // rows (a vN/ holding only change.md while the run warms up would render a + // blank column), and refuse to build when the first variant itself has none + // rather than overwrite a good trajectory/scores.tsv with a header-only file. + for (let i = variants.length - 1; i > 0; i--) { + const v = variants[i]; + if (Object.keys(rows[v]).length) continue; + warnings.push(v + ': zero result rows - dropping (run not started or results.jsonl missing/empty)'); + variants.splice(i, 1); + } + if (!Object.keys(rows[variants[0]]).length) + throw new Error(variants[0] + '/ has no result rows under ' + flow + ' - nothing to report' + + (skippedHardLinks.length ? ' (not read, second hard link: ' + skippedHardLinks.map(p => p.slice(flow.length + 1)).join(', ') + ' - replace with plain copies if they are yours)' : '')); + + const header = Object.create(null); + for (const v of variants) { + const vdir = join(flow, v); + const change = readText(join(vdir, 'change.md')).split('\n').find(l => l.trim()) || ''; + const summary = readJSON(join(vdir, 'summary.json'), {}) || {}; + // Model: summary.json wins, else the most common row.model (what the app called). + const tally = Object.create(null); + for (const reps of Object.values(rows[v])) for (const r of reps) if (typeof r.model === 'string') tally[r.model] = (tally[r.model] || 0) + 1; + const rowModel = Object.entries(tally).sort((a, b) => b[1] - a[1])[0]?.[0] || ''; + header[v] = { change: change.replace(/^#+\s*/, ''), model: summary.model || rowModel, errors: countErrors(vdir) }; + } + + // Metrics: _state.json `metrics` (or the legacy `criteria` key) wins, + // keeping each declared `kind`; else inferred from grade keys. Primary is + // the first declared-or-inferred binary metric, else the first - the same + // selection the full builder's lib/adapter.mjs makes, so both builders + // report the same metric's numbers on the same flow. + const metricsCfg = state.metrics || state.criteria; + const declared = (Array.isArray(metricsCfg) ? metricsCfg : []) + .map(m => (typeof m === 'string' ? { id: m } : { id: m?.id, kind: m?.kind })) + .filter(m => m.id); + const metrics = declared.length ? declared : inferMetrics(rows, variants); + if (!metrics.length) metrics.push({ id: 'score', kind: 'float' }); + const primary = (metrics.find(m => m.kind === 'binary') || metrics[0]).id; + + // Cases: union of ids across variants, in first-seen order. + const ids = []; + const seenId = new Set(); + for (const v of variants) for (const pid of Object.keys(rows[v])) if (!seenId.has(pid)) { seenId.add(pid); ids.push(pid); } + + const okReps = reps => reps.filter(r => r.status == null || r.status === 'ok'); + const caseMean = (reps, m) => { + const vals = okReps(reps).map(r => coerceScores(r.grade)[m]).filter(isNum); + return vals.length ? vals.reduce((s, x) => s + x, 0) / vals.length : null; + }; + const cases = ids.map(pid => { + const first = variants.map(v => rows[v][pid]?.[0]).find(Boolean) || {}; + const tags = Array.isArray(first.tags) ? first.tags.map(String) : []; + const per = {}; + for (const v of variants) { + const reps = rows[v][pid] || []; + per[v] = { + mean: caseMean(reps, primary), + reps: reps.length, + truncated: reps.filter(r => r.status === 'truncated').length, + traces: reps.map((r, k) => { + // Link only to files directly inside this variant's traces/ dir: + // ids and reps come from results.jsonl (data, not trusted), so the + // composed path must not resolve anywhere else, and a symlinked + // trace must not become a link out of the tree. Keep the rep + // number so the link label matches the file it points at even when + // earlier reps have no trace. For rep 0 also accept the flat + // traces/.json a single-rep runner may write (the full builder + // reads it as rep 0 too). + // The href is resolved by a browser with URL semantics, not the + // filesystem's: allowlist the id to the scaffold's path-safe + // charset (so `#`, `?`, `\\`, `%` and separators never reach the + // link) and require an integer rep, on top of the traceDirOk + // resolved-path check above and the lexical one below. + if (!traceDirOk[v]) return null; + const rep = r.rep ?? k; + if (!/^[A-Za-z0-9_.-]+$/.test(pid) || /^\.+$/.test(pid)) return null; + if (!Number.isInteger(+rep) || +rep < 0) return null; + const stems = +rep === 0 ? [pid + '_rep' + rep, pid] : [pid + '_rep' + rep]; + for (const stem of stems) { + const rel = v + '/traces/' + stem + '.json'; + const abs = resolve(flow, rel); + if (dirname(abs) !== join(flow, v, 'traces')) return null; + // A hard-linked trace is no more ours than a symlinked one. + const ls = lstatSync(abs, { throwIfNoEntry: false }); + if (ls?.isFile() && !(ls.nlink > 1)) return { rep, rel }; + } + return null; + }).filter(Boolean), + }; + } + return { id: pid, split: splitOf[pid] || '', tags, prompt: String(first.prompt ?? ''), per }; + }); + + // Per-variant aggregate: mean over cases that have a value, per split. + const agg = {}; + for (const v of variants) { + const all = cases.map(c => c.per[v].mean).filter(isNum); + const test = cases.filter(c => c.split === 'test').map(c => c.per[v].mean).filter(isNum); + const m = xs => (xs.length ? xs.reduce((s, x) => s + x, 0) / xs.length : null); + agg[v] = { all: m(all), n: all.length, test: m(test), nTest: test.length, + truncated: cases.reduce((s, c) => s + c.per[v].truncated, 0) }; + } + + // trajectory/scores.tsv - same shape as the full builder. + mkdirNoFollow(join(flow, 'trajectory'), flowRoot); + const tsv = [['id', 'split', ...variants].join('\t')]; + for (const c of cases) tsv.push([tsvCell(c.id), tsvCell(c.split || 'all'), ...variants.map(v => fmt(c.per[v].mean))].join('\t')); + writeNoFollow(join(flow, 'trajectory', 'scores.tsv'), tsv.join('\n') + '\n', flowRoot); + + // _state.json's `best` is {round, test_score}; map the round to a variant + // name the way the full builder does (0 = baseline), and drop it when the + // named variant isn't on disk. + let best = null; + if (Number.isInteger(state.best?.round)) { + best = state.best.round === 0 ? 'baseline' : 'v' + state.best.round; + if (!variants.includes(best)) best = null; + } + + for (const p of new Set(skippedHardLinks)) + warnings.push(p.slice(flow.length + 1) + ' has a second hard link - not read; replace it with a plain copy if it is yours'); + const html = render({ flow, variants, header, agg, cases, primary, metrics: metrics.map(m => m.id), warnings, best }); + writeNoFollow(join(flow, 'report.html'), html, flowRoot); + return { variants: variants.length, cases: cases.length, warnings }; +} + +function render({ flow, variants, header, agg, cases, primary, metrics, warnings, best }) { + const hasSplit = cases.some(c => c.split); + const hasTest = variants.some(v => agg[v].nTest > 0); + const th = (label, key) => '' + esc(label) + ''; + + const variantRows = variants.map(v => { + const h = header[v], a = agg[v]; + const err = h.errors.total + ? h.errors.total + ' (' + Object.entries(h.errors.byClass).map(([k, n]) => esc(k) + ' ' + n).join(', ') + ')' + : '0'; + return '' + esc(v) + (best === v ? ' best' : '') + '' + + '' + esc(h.change) + '' + esc(h.model) + '' + + '' + fmt(a.all) + '' + a.n + '' + + (hasTest ? '' + fmt(a.test) + '' + a.nTest + '' : '') + + '' + a.truncated + '' + err + ''; + }).join('\n'); + + const caseRows = cases.map(c => { + const cells = variants.map(v => { + const p = c.per[v]; + const links = p.traces.map(t => 'rep' + esc(t.rep) + '').join(' '); + return '' + fmt(p.mean) + + (p.truncated ? ' ' + p.truncated + ' truncated' : '') + + (links ? '
' + links + '
' : '') + ''; + }).join(''); + const prompt = c.prompt.length > 600 ? c.prompt.slice(0, 600) + '...' : c.prompt; + return '' + esc(c.id) + '' + + (hasSplit ? '' + esc(c.split) + '' : '') + + '' + c.tags.map(t => '' + esc(t) + '').join(' ') + '' + + cells + + '
' + esc(prompt.split('\n')[0].slice(0, 80)) + '
' + esc(prompt) + '
'; + }).join('\n'); + + const warn = warnings.length + ? '

Warnings

    ' + warnings.map(w => '
  • ' + esc(w) + '
    • ').join('') + '
' + : ''; + + return '\n' + + '' + + '' + esc(flow.split(/[\\/]/).pop()) + ' - eval report' + + '' + + '

' + esc(flow.split(/[\\/]/).pop()) + '

' + + '

' + variants.length + ' variant' + (variants.length === 1 ? '' : 's') + ' · ' + cases.length + ' cases' + + ' · primary metric ' + esc(primary) + '' + + (metrics.length > 1 ? ' (also: ' + metrics.filter(m => m !== primary).map(esc).join(', ') + ')' : '') + + ' · generated ' + new Date().toISOString().slice(0, 19).replace('T', ' ') + ' UTC' + + ' · lite report (no transcripts inlined; click rep links)

' + + '

Variants

' + th('variant', 's') + th('change', 's') + th('model', 's') + + th('mean ' + primary, 'n') + th('cases', 'n') + + (hasTest ? th('test mean', 'n') + th('test cases', 'n') : '') + + th('truncated', 'n') + th('failed attempts', 's') + '' + variantRows + '
' + + '

Cases

Click a column header to sort. Per-case values are the mean of ' + esc(primary) + ' over status-ok reps.

' + + '' + th('id', 's') + (hasSplit ? th('split', 's') : '') + th('tags', 's') + + variants.map(v => th(v, 'n')).join('') + th('prompt', 's') + '' + caseRows + '
' + + warn + + '\n'; +} + +const CSS = [ + ':root{color-scheme:light dark;--fg:#1a1a1a;--bg:#fff;--muted:#666;--line:#ddd;--pill:#eee;--warn:#b45309;--best:#ecfdf5}', + '@media(prefers-color-scheme:dark){:root{--fg:#e6e6e6;--bg:#121212;--muted:#9a9a9a;--line:#333;--pill:#2a2a2a;--warn:#f59e0b;--best:#0f2a1f}}', + 'body{font:14px/1.45 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;color:var(--fg);background:var(--bg);margin:0;padding:24px;max-width:1400px}', + 'h1{font-size:22px;margin:0 0 4px}h2{font-size:16px;margin:28px 0 8px}.meta,.hint{color:var(--muted);margin:0 0 8px}', + 'table{border-collapse:collapse;width:100%}th,td{border-bottom:1px solid var(--line);padding:6px 8px;text-align:left;vertical-align:top}', + 'th{cursor:pointer;user-select:none;white-space:nowrap}th.asc:after{content:" \\25B4"}th.desc:after{content:" \\25BE"}', + 'td.num{text-align:right;font-variant-numeric:tabular-nums;white-space:nowrap}td.id{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;white-space:nowrap}', + '.pill{display:inline-block;background:var(--pill);border-radius:10px;padding:0 8px;font-size:12px}.pill.warn{color:var(--warn)}', + 'tr.best td{background:var(--best)}.links{font-size:12px}.links a{margin-right:6px}', + 'td.prompt{max-width:520px}details summary{cursor:pointer;color:var(--muted)}pre{white-space:pre-wrap;font-size:12px;margin:6px 0 0}', + '.warnings{color:var(--warn)}code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace}', +].join(''); + +const SORT_JS = [ + 'document.querySelectorAll("th[data-k]").forEach(function(th){th.addEventListener("click",function(){', + 'var t=th.closest("table"),tb=t.tBodies[0],i=Array.prototype.indexOf.call(th.parentNode.children,th);', + 'var asc=!th.classList.contains("asc");t.querySelectorAll("th").forEach(function(h){h.classList.remove("asc","desc")});', + 'th.classList.add(asc?"asc":"desc");var num=th.dataset.k==="n";', + 'var rows=Array.prototype.slice.call(tb.rows);rows.sort(function(a,b){var x=a.cells[i],y=b.cells[i];', + 'var u=num?parseFloat("v" in x.dataset?x.dataset.v:x.textContent):x.textContent.trim().toLowerCase();', + 'var w=num?parseFloat("v" in y.dataset?y.dataset.v:y.textContent):y.textContent.trim().toLowerCase();', + 'if(num){if(isNaN(u))u=-Infinity;if(isNaN(w))w=-Infinity}return (uw?1:0)*(asc?1:-1)});', + 'rows.forEach(function(r){tb.appendChild(r)})})});', +].join(''); + +const arg = process.argv[2]; +if (!arg || arg === '-h' || arg === '--help') { + eprint('usage: node build-report-lite.mjs (e.g. .claude/hillclimb//)'); + process.exit(arg ? 0 : 2); +} +try { + const { variants, cases, warnings } = build(arg); + for (const w of warnings) eprint('warning: ' + w); + eprint('wrote ' + join(resolve(arg), 'report.html') + ' (' + variants + ' variants, ' + cases + ' cases) and trajectory/scores.tsv'); +} catch (e) { + eprint('build-report-lite: ' + (e?.message || e)); + process.exit(1); +} diff --git a/evals/tool-selection/cases.json b/evals/tool-selection/cases.json new file mode 100644 index 00000000..c682387b --- /dev/null +++ b/evals/tool-selection/cases.json @@ -0,0 +1,253 @@ +[ + { + "id": "address-no-map", + "category": "no-unrequested-map", + "prompt": "What's the address of the post office closest to 2400 Larimer St in Denver?", + "why": "Connector review: render_map_tool must not draw a map nobody asked for. Needs live data, so Claude can't answer from memory.", + "expect": { + "any_of": ["search_and_geocode_tool", "category_search_tool"], + "must_not": [ + "render_map_tool", + "static_map_image_tool", + "places_map_display_v0" + ] + }, + "stubs": { + "search_and_geocode_tool": "1. 2400 Larimer St, Denver, CO 80205 (39.7569, -104.9862)\n2. USPS Post Office, 2160 Market St, Denver, CO 80205 (39.7537, -104.9925), 0.6 km from 2400 Larimer St", + "category_search_tool": "1. USPS Post Office, 2160 Market St, Denver, CO 80205 (39.7537, -104.9925), 0.6 km\n2. USPS Post Office, 3600 Downing St, Denver, CO 80205 (39.7672, -104.9732), 1.6 km", + "place_details_tool": "USPS Post Office, 2160 Market St, Denver, CO 80205. Hours: Mon-Fri 9am-5pm.", + "web_search": "1. \"Post office locations near Five Points, Denver\" usps.com: 2160 Market St, Denver, CO 80205." + } + }, + { + "id": "drive-time-no-map", + "category": "no-unrequested-map", + "prompt": "How long does it take to drive from Boulder to Denver Union Station?", + "why": "A route lookup returns a render hint; a text question should still get a text answer.", + "expect": { + "must": ["directions_tool"], + "must_not": [ + "render_map_tool", + "static_map_image_tool", + "places_map_display_v0" + ] + }, + "stubs": { + "search_and_geocode_tool": "1. Boulder, Colorado, United States (40.0150, -105.2705)\n2. Denver Union Station, 1701 Wynkoop St, Denver, CO 80202 (39.7531, -105.0000)", + "directions_tool": "Route 1: 45.2 km, 37 min via US-36 E (current traffic). Route 2: 49.8 km, 44 min via CO-93 S and I-70 E." + } + }, + { + "id": "museums-no-map", + "category": "no-unrequested-map", + "prompt": "What museums are near the National Mall in Washington, DC?", + "why": "Generic place type goes to category search; no map requested.", + "expect": { + "must": ["category_search_tool"], + "must_not": [ + "render_map_tool", + "static_map_image_tool", + "places_map_display_v0" + ] + }, + "stubs": { + "category_search_tool": "1. Smithsonian National Air and Space Museum, 600 Independence Ave SW (38.8882, -77.0199)\n2. National Museum of Natural History, 10th St & Constitution Ave NW (38.8913, -77.0261)\n3. National Gallery of Art, Constitution Ave NW (38.8913, -77.0200)\n4. National Museum of American History, 1300 Constitution Ave NW (38.8912, -77.0300)\n5. Hirshhorn Museum, Independence Ave & 7th St SW (38.8882, -77.0230)", + "search_and_geocode_tool": "1. National Mall, Washington, DC, United States (38.8896, -77.0230)" + } + }, + { + "id": "route-on-map", + "category": "map-requested", + "prompt": "Show me the driving route from the Golden Gate Bridge to Stanford University on a map.", + "why": "Map requested: route data then render_map_tool, not static or the host map.", + "expect": { + "must": ["directions_tool", "render_map_tool"], + "must_not": ["static_map_image_tool", "places_map_display_v0"] + }, + "stubs": { + "search_and_geocode_tool": "1. Golden Gate Bridge, San Francisco, CA (37.8199, -122.4783)\n2. Stanford University, 450 Jane Stanford Way, Stanford, CA 94305 (37.4275, -122.1697)", + "directions_tool": "Route 1: 62.1 km, 52 min via US-101 S and I-280 S. Route 2: 58.4 km, 57 min via US-101 S." + } + }, + { + "id": "coffee-on-map", + "category": "map-requested", + "prompt": "Find coffee shops near Union Square in San Francisco and put them on a map.", + "why": "Map requested after a category search.", + "expect": { + "must": ["category_search_tool", "render_map_tool"], + "must_not": ["static_map_image_tool", "places_map_display_v0"] + }, + "stubs": { + "search_and_geocode_tool": "1. Union Square, San Francisco, CA 94108 (37.7880, -122.4075)", + "category_search_tool": "1. Blue Bottle Coffee, 66 Mint St (37.7825, -122.4079), 0.6 km\n2. Sightglass Coffee, 270 7th St (37.7767, -122.4083), 1.3 km\n3. Philz Coffee, 201 Berry St (37.7766, -122.3921), 1.8 km\n4. Peet's Coffee, 291 Geary St (37.7873, -122.4079), 0.1 km\n5. Saint Frank Coffee, 2340 Polk St (37.7983, -122.4220), 1.9 km" + } + }, + { + "id": "landmark-on-map", + "category": "map-requested", + "prompt": "Show me where the Space Needle is on a map.", + "why": "Single-place map request; the host map tool is the tempting alternative.", + "expect": { + "must": ["render_map_tool"], + "must_not": ["static_map_image_tool", "places_map_display_v0"] + }, + "stubs": { + "search_and_geocode_tool": "1. Space Needle, 400 Broad St, Seattle, WA 98109, United States (47.6205, -122.3493)", + "place_details_tool": "Space Needle, 400 Broad St, Seattle, WA 98109. Category: observation tower, landmark." + } + }, + { + "id": "walk-isochrone-map", + "category": "map-requested", + "prompt": "Show me everywhere I can walk to within 15 minutes of Times Square.", + "why": "Reachability plus an implied map.", + "expect": { + "must": ["isochrone_tool", "render_map_tool"], + "must_not": ["static_map_image_tool", "places_map_display_v0"] + }, + "stubs": { + "search_and_geocode_tool": "1. Times Square, Manhattan, New York, NY 10036 (40.7580, -73.9855)", + "isochrone_tool": "Walking isochrone from Times Square: 15 min contour covers about 2.3 sq km, reaching roughly from 34th St to 57th St and from 5th Ave to 10th Ave." + } + }, + { + "id": "static-png", + "category": "static-image", + "prompt": "Generate a static PNG map image of downtown Denver at 800x600 that I can paste into a slide.", + "why": "Server instructions: static_map_image_tool only when a static image file is explicitly requested.", + "expect": { + "must": ["static_map_image_tool"], + "must_not": ["render_map_tool", "places_map_display_v0"] + }, + "stubs": { + "search_and_geocode_tool": "1. Downtown Denver, Denver, CO (39.7447, -104.9918)" + } + }, + { + "id": "coords-lunch", + "category": "coordinates", + "prompt": "I'm at 37.7955, -122.3937. Any good lunch spots within walking distance?", + "why": "ground_location_tool already includes reverse geocoding; no separate reverse_geocode_tool call.", + "expect": { + "first": ["ground_location_tool"], + "must_not": ["reverse_geocode_tool"] + }, + "stubs": { + "ground_location_tool": "Place: Ferry Building, Embarcadero, San Francisco, CA 94111. Nearby restaurants (walking): Hog Island Oyster Co. (90 m), Gott's Roadside (120 m), Slanted Door (110 m), Boulettes Larder (150 m), Sens Restaurant (260 m). A 15 min walk reaches the Financial District and North Beach.", + "category_search_tool": "1. Hog Island Oyster Co., 1 Ferry Building (37.7956, -122.3934), 90 m\n2. Gott's Roadside, 1 Ferry Building (37.7953, -122.3938), 120 m\n3. Slanted Door, 1 Ferry Building (37.7956, -122.3932), 110 m", + "web_search": "1. \"Best lunch at the Ferry Building\" sfchronicle.com: Hog Island Oyster Co., Gott's Roadside, Slanted Door..." + } + }, + { + "id": "coords-neighborhood", + "category": "coordinates", + "prompt": "What neighborhood is 40.7411, -73.9897 in?", + "why": "Neighborhood context: ground_location_tool covers it, and reverse_geocode_tool with a neighborhood type is also a fine first call. No map requested.", + "expect": { + "first": ["ground_location_tool", "reverse_geocode_tool"], + "must_not": ["render_map_tool", "places_map_display_v0"] + }, + "stubs": { + "ground_location_tool": "Place: Flatiron District, Manhattan, New York, NY 10010. Near the Flatiron Building and Madison Square Park. Nearby: Eataly (100 m), Madison Square Park (150 m).", + "reverse_geocode_tool": "175 5th Ave, Flatiron District, Manhattan, New York, NY 10010, United States" + } + }, + { + "id": "coords-address", + "category": "coordinates", + "prompt": "What's the street address at 38.8977, -77.0365?", + "why": "reverse_geocode_tool description: use when you only need an address.", + "expect": { + "first": ["reverse_geocode_tool"], + "must_not": ["ground_location_tool"] + }, + "stubs": { + "reverse_geocode_tool": "1600 Pennsylvania Avenue NW, Washington, DC 20500, United States", + "ground_location_tool": "Place: 1600 Pennsylvania Avenue NW, Washington, DC 20500 (the White House). Nearby: Lafayette Square (200 m), The Ellipse (300 m)." + } + }, + { + "id": "brand-search", + "category": "search-routing", + "prompt": "Find the nearest Blue Bottle Coffee to 1 Market St, San Francisco.", + "why": "Named brand goes to search_and_geocode_tool, not category search.", + "expect": { + "must": ["search_and_geocode_tool"], + "must_not": ["category_search_tool"] + }, + "stubs": { + "search_and_geocode_tool": "1. Blue Bottle Coffee, 1 Ferry Building #7, San Francisco, CA 94111 (37.7956, -122.3935), 0.2 km from 1 Market St\n2. Blue Bottle Coffee, 66 Mint St, San Francisco, CA 94103 (37.7825, -122.4079), 1.7 km\n3. 1 Market St, San Francisco, CA 94105 (37.7941, -122.3947)", + "category_search_tool": "1. Blue Bottle Coffee, 1 Ferry Building #7 (37.7956, -122.3935), 0.2 km\n2. Peet's Coffee, 1 Ferry Building (37.7955, -122.3937), 0.2 km", + "directions_tool": "Route 1: 0.3 km, 4 min walking via The Embarcadero.", + "web_search": "1. \"Blue Bottle Coffee Ferry Building\" bluebottlecoffee.com: 1 Ferry Building #7, San Francisco, CA 94111." + } + }, + { + "id": "visit-order", + "category": "search-routing", + "prompt": "Starting at the Ferry Building, what's the fastest order to drive to Coit Tower and the Painted Ladies?", + "why": "Multi-stop ordering goes to optimization or matrix; no map requested.", + "expect": { + "any_of": ["optimization_tool", "matrix_tool"], + "must_not": ["render_map_tool", "places_map_display_v0"] + }, + "stubs": { + "search_and_geocode_tool": "1. Ferry Building, 1 Ferry Building, San Francisco, CA (37.7955, -122.3937)\n2. Coit Tower, 1 Telegraph Hill Blvd, San Francisco, CA (37.8024, -122.4058)\n3. Painted Ladies, 710-720 Steiner St, San Francisco, CA (37.7763, -122.4328)", + "matrix_tool": "Driving durations (min): Ferry Building->Coit Tower 9, Ferry Building->Painted Ladies 15, Coit Tower->Painted Ladies 18, Painted Ladies->Coit Tower 19, Coit Tower->Ferry Building 8, Painted Ladies->Ferry Building 16.", + "optimization_tool": "Optimized order: Ferry Building -> Coit Tower -> Painted Ladies. Total 27 min, 8.9 km.", + "directions_tool": "Route 1: Ferry Building -> Coit Tower -> Painted Ladies, 8.9 km, 27 min." + } + }, + { + "id": "straight-line", + "category": "search-routing", + "prompt": "How far is it in a straight line from (48.8584, 2.2945) to (48.8606, 2.3376)?", + "why": "Offline distance: no routing call and no map. Computing it without a tool is fine.", + "expect": { + "must_not": ["directions_tool", "render_map_tool"] + }, + "stubs": { + "distance_tool": "3.16 km" + } + }, + { + "id": "reviews-web", + "category": "other-sources", + "prompt": "Is Tartine Bakery in San Francisco any good? What do people say about it?", + "why": "Connector review: Mapbox must not keep other sources away from a place question. Reviews need the web.", + "expect": { + "must": ["web_search"], + "must_not": ["render_map_tool", "places_map_display_v0"] + }, + "stubs": { + "web_search": "1. \"Tartine Bakery\" yelp.com: 4.3 stars from 8,000+ reviews. Known for morning buns and country bread; long lines on weekends.\n2. \"Is Tartine still worth it?\" sfgate.com: The bread is still excellent; pastries are pricey.", + "search_and_geocode_tool": "1. Tartine Bakery, 600 Guerrero St, San Francisco, CA 94110 (37.7614, -122.4241)", + "place_details_tool": "Tartine Bakery, 600 Guerrero St, San Francisco, CA 94110. Category: bakery, cafe. Hours: 8am-5pm daily." + } + }, + { + "id": "history-web", + "category": "other-sources", + "prompt": "Search for the history of the Ferry Building in San Francisco. When was it built and who designed it?", + "why": "Explicit search request about a place; web search must stay available.", + "expect": { + "must": ["web_search"], + "must_not": ["render_map_tool", "places_map_display_v0"] + }, + "stubs": { + "web_search": "1. \"Ferry Building history\" sfport.com: Completed in 1898, designed by architect A. Page Brown; the clock tower is modeled on the Giralda in Seville. Restored and reopened as a marketplace in 2003.", + "search_and_geocode_tool": "1. Ferry Building, 1 Ferry Building, San Francisco, CA 94111 (37.7955, -122.3937)" + } + }, + { + "id": "non-geo", + "category": "other-sources", + "prompt": "Write a haiku about autumn leaves.", + "why": "Unrelated request; no Mapbox tool should fire.", + "expect": { + "must_not": ["*mapbox*"] + }, + "stubs": {} + } +] diff --git a/evals/tool-selection/harness.ts b/evals/tool-selection/harness.ts new file mode 100644 index 00000000..41c5b396 --- /dev/null +++ b/evals/tool-selection/harness.ts @@ -0,0 +1,492 @@ +// Copyright (c) Mapbox, Inc. +// Licensed under the MIT License. + +/** + * Tool-selection eval harness: which tools does Claude pick for a prompt, + * given this server's tool descriptions and server instructions? + * + * The tool surface comes from a real McpServer built the same way the hosted + * deployment builds it (publishJsonSchema2020 + installTo + server + * instructions) and read back over MCP `tools/list`, so the eval sees exactly + * what a host sees. EVAL_SERVER_ROOT points at the checkout whose src/ to + * load, which lets one harness compare two branches (e.g. main vs a PR + * worktree). Tool calls are answered with canned stubs; no Mapbox API calls + * are made. + */ + +import Anthropic from '@anthropic-ai/sdk'; +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; + +const here = dirname(fileURLToPath(import.meta.url)); +const SERVER_ROOT = resolve( + process.env.EVAL_SERVER_ROOT ?? resolve(here, '../..') +); +const MAX_MODEL_CALLS = 5; +const FAKE_REF = 'mapbox://temp/map-payload-eval-0001'; + +// Distractors standing in for tools a host provides alongside the connector. +// They let the eval see whether our descriptions crowd out other sources, +// which is the connector-review concern. +const HOST_TOOLS: Anthropic.Tool[] = [ + { + name: 'web_search', + description: + 'Search the web for current information. Returns result titles, URLs and snippets.', + input_schema: { + type: 'object', + properties: { query: { type: 'string', description: 'Search query' } }, + required: ['query'] + } + }, + { + name: 'places_map_display_v0', + description: + 'Display one or more places as pins on a map in the chat. Takes place names or coordinates.', + input_schema: { + type: 'object', + properties: { + places: { + type: 'array', + items: { type: 'string' }, + description: 'Place names or "lat,lng" strings to pin' + } + }, + required: ['places'] + } + } +]; +const HOST_TOOL_NAMES = new Set(HOST_TOOLS.map((t) => t.name)); + +export type Mode = 'eager' | 'deferred'; + +export interface Expectation { + /** Acceptable names for the first client tool called. */ + first?: string[]; + /** Every one of these must be called at some point. */ + must?: string[]; + /** At least one of these must be called. */ + any_of?: string[]; + /** None of these may be called. "*mapbox*" means any Mapbox tool. */ + must_not?: string[]; +} + +export interface EvalCase { + id: string; + prompt: string; + tags: string[]; + expect: Expectation; + /** Canned results keyed by tool name, written to match this prompt. */ + stubs: Record; + meta?: Record; +} + +interface BaseCase { + id: string; + prompt: string; + category: string; + why: string; + expect: Expectation; + stubs?: Record; +} + +interface ServerSurface { + tools: Anthropic.Tool[]; + instructions: string | undefined; + names: Set; + renderHint: (ref: string) => string; +} + +let surfacePromise: Promise | undefined; + +async function importFromRoot(rel: string): Promise { + return (await import(pathToFileURL(resolve(SERVER_ROOT, rel)).href)) as T; +} + +async function buildSurface(): Promise { + const registry = await importFromRoot<{ + getCoreTools(): readonly { + name: string; + installTo(s: McpServer): unknown; + }[]; + }>('src/tools/toolRegistry.ts'); + const { publishJsonSchema2020 } = await importFromRoot<{ + publishJsonSchema2020(s: McpServer): void; + }>('src/utils/jsonSchema2020.ts'); + const { buildServerInstructions } = await importFromRoot<{ + buildServerInstructions(names: readonly string[]): string | undefined; + }>('src/utils/serverInstructions.ts'); + const { renderHint } = await importFromRoot<{ + renderHint(ref: string): string; + }>('src/utils/storeMapPayload.ts'); + + const coreTools = registry.getCoreTools(); + const server = new McpServer( + { name: 'mapbox-eval', version: '0.0.0' }, + { + instructions: buildServerInstructions(coreTools.map((t) => t.name)), + capabilities: { tools: {} } + } + ); + publishJsonSchema2020(server); + coreTools.forEach((t) => t.installTo(server)); + + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + const client = new Client({ name: 'tool-selection-eval', version: '0.0.0' }); + await Promise.all([ + server.connect(serverTransport), + client.connect(clientTransport) + ]); + const { tools } = await client.listTools(); + const instructions = client.getInstructions(); + await client.close(); + + const apiTools: Anthropic.Tool[] = tools + .map((t) => ({ + name: t.name, + description: t.description ?? '', + input_schema: t.inputSchema as Anthropic.Tool.InputSchema + })) + .sort((a, b) => a.name.localeCompare(b.name)); // stable order keeps the prompt cache warm + return { + tools: apiTools, + instructions, + names: new Set(apiTools.map((t) => t.name)), + renderHint + }; +} + +export function loadSurface(): Promise { + surfacePromise ??= buildSurface(); + return surfacePromise; +} + +/** Each base case runs once per mode, so one row = (case, mode). */ +export function loadCases(): EvalCase[] { + const base = JSON.parse( + readFileSync(resolve(here, 'cases.json'), 'utf8') + ) as BaseCase[]; + const modes: Mode[] = ['eager', 'deferred']; + return base.flatMap((c) => + modes.map((mode) => ({ + id: `${c.id}__${mode}`, + prompt: c.prompt, + tags: [mode, c.category], + expect: c.expect, + stubs: c.stubs ?? {}, + meta: { base_id: c.id, mode, why: c.why } + })) + ); +} + +function systemPrompt(instructions: string | undefined, mode: Mode): string { + const lines = [ + "You are Claude, a helpful assistant in a chat app. The user has connected the Mapbox MCP server; its tools are available to you alongside the app's built-in tools (web_search, places_map_display_v0)." + ]; + if (mode === 'deferred') { + lines.push( + 'Mapbox tools are not loaded up front. Use tool search to find and load them when you need one.' + ); + } + if (instructions) { + lines.push( + '', + '', + instructions, + '' + ); + } + return lines.join('\n'); +} + +// Results that don't depend on the query. Everything else is stubbed per case +// in cases.json: a canned answer about the wrong place makes Claude retry or +// switch tools, which would be scored as a routing failure. +const COMMON_STUBS: Record = { + render_map_tool: 'Map rendered for the user.', + static_map_image_tool: 'Static map image generated (PNG).', + places_map_display_v0: 'Map with pins displayed to the user.' +}; +const UNSTUBBED = 'No results. (This tool returned nothing for this request.)'; + +/** + * Canned result for a tool call. Tools that return map payloads on the real + * server get the server's own render hint appended, so a change to that hint + * text shows up in the eval. Calls the case has no stub for get an empty + * result and are flagged so the run can report them. + */ +function stubResult( + name: string, + c: EvalCase, + surface: ServerSurface +): { text: string; stubbed: boolean } { + const base = c.stubs[name] ?? COMMON_STUBS[name]; + if (base === undefined) return { text: UNSTUBBED, stubbed: false }; + return { + text: HINTED_TOOLS.has(name) ? base + surface.renderHint(FAKE_REF) : base, + stubbed: true + }; +} + +const HINTED_TOOLS = new Set([ + 'search_and_geocode_tool', + 'category_search_tool', + 'directions_tool', + 'isochrone_tool', + 'ground_location_tool', + 'map_matching_tool', + 'optimization_tool', + 'union_tool', + 'intersect_tool', + 'difference_tool' +]); + +export interface RunCtx { + model?: string; +} + +export interface RunOutput { + output: string; + transcript: Array>; + model: string; + usage: Record; + stop_reason: string | null; + status?: string; + tool_calls: string[]; + tool_searches: string[]; + /** Tool calls the case had no stub for; non-empty means cases.json needs one. */ + unstubbed: string[]; +} + +const client = new Anthropic(); + +function addUsage(total: Record, u: Anthropic.Usage): void { + for (const k of [ + 'input_tokens', + 'output_tokens', + 'cache_read_input_tokens', + 'cache_creation_input_tokens' + ] as const) { + total[k] = (total[k] ?? 0) + (u[k] ?? 0); + } +} + +export async function runCase(c: EvalCase, ctx: RunCtx): Promise { + const surface = await loadSurface(); + const mode = (c.meta?.mode as Mode) ?? 'eager'; + const model = ctx.model ?? 'claude-opus-5-5'; + const system = systemPrompt(surface.instructions, mode); + + const tools: Anthropic.ToolUnion[] = + mode === 'deferred' + ? [ + { + type: 'tool_search_tool_bm25_20251119', + name: 'tool_search_tool_bm25' + }, + ...HOST_TOOLS, + ...surface.tools.map((t) => ({ ...t, defer_loading: true })) + ] + : [...HOST_TOOLS, ...surface.tools]; + + const messages: Anthropic.MessageParam[] = [ + { role: 'user', content: c.prompt } + ]; + const transcript: Array> = [ + { role: 'system', content: system }, + { role: 'user', content: c.prompt } + ]; + const usage: Record = {}; + const toolCalls: string[] = []; + const toolSearches: string[] = []; + const unstubbed: string[] = []; + let servedModel = model; + let stopReason: string | null = null; + + for (let call = 0; call < MAX_MODEL_CALLS; call++) { + const res = await client.messages.create({ + model, + max_tokens: 16000, + // Caches the conversation so far; in deferred mode the tool definitions + // that search loads live in the messages, past the system breakpoint. + cache_control: { type: 'ephemeral' }, + system: [ + { type: 'text', text: system, cache_control: { type: 'ephemeral' } } + ], + tools, + messages + }); + servedModel = res.model; + stopReason = res.stop_reason; + addUsage(usage, res.usage); + + const results: Anthropic.ToolResultBlockParam[] = []; + for (const block of res.content) { + if (block.type === 'text' && block.text.trim()) { + transcript.push({ role: 'assistant', content: block.text }); + } else if (block.type === 'server_tool_use') { + const query = JSON.stringify(block.input); + toolSearches.push(query); + transcript.push({ + role: 'tool_call', + name: block.name, + content: query + }); + } else if (block.type === 'tool_search_tool_result') { + const found = + block.content.type === 'tool_search_tool_search_result' + ? block.content.tool_references.map((r) => r.tool_name).join(', ') + : `error: ${block.content.error_code}`; + transcript.push({ role: 'tool_result', content: `found: ${found}` }); + } else if (block.type === 'tool_use') { + toolCalls.push(block.name); + const { text: result, stubbed } = stubResult(block.name, c, surface); + if (!stubbed) unstubbed.push(block.name); + transcript.push({ + role: 'tool_call', + name: block.name, + content: JSON.stringify(block.input, null, 2) + }); + transcript.push({ role: 'tool_result', content: result }); + results.push({ + type: 'tool_result', + tool_use_id: block.id, + content: result + }); + } + } + + // Keep the assistant turn unchanged (thinking and tool-search blocks included). + messages.push({ role: 'assistant', content: res.content }); + if (res.stop_reason === 'pause_turn') continue; + if (res.stop_reason !== 'tool_use' || results.length === 0) break; + messages.push({ role: 'user', content: results }); + } + + return { + output: toolCalls.join(' > '), + transcript, + model: servedModel, + usage, + stop_reason: stopReason, + status: + stopReason === 'max_tokens' + ? 'truncated' + : stopReason === 'refusal' + ? 'refusal' + : 'ok', + tool_calls: toolCalls, + tool_searches: toolSearches, + unstubbed + }; +} + +/** + * Did every render_map_tool call pass the map ref unchanged? Data tools return + * a ref plus a hint telling Claude to pass it as-is; before the hint existed, + * Claude invented refs. Once a data tool has returned a ref, a render call must + * include it, and no render call may carry any other mapbox:// ref. Returns + * null when render_map_tool wasn't called. + */ +function checkRefs( + transcript: RunOutput['transcript'] +): { ok: boolean; why: string } | null { + let refAvailable = false; + let renders = 0; + for (const turn of transcript) { + if (turn.role !== 'tool_call') continue; + const name = String(turn.name); + if (HINTED_TOOLS.has(name)) refAvailable = true; + if (name !== 'render_map_tool') continue; + renders++; + let input: { payload_refs?: unknown }; + try { + input = JSON.parse(String(turn.content)) as { payload_refs?: unknown }; + } catch { + return { ok: false, why: 'unparseable render_map_tool input' }; + } + const refs = Array.isArray(input.payload_refs) + ? input.payload_refs.map(String) + : []; + const invented = refs.filter((r) => r !== FAKE_REF); + if (invented.length) { + return { ok: false, why: `invented ref ${invented[0]}` }; + } + if (refAvailable && !refs.includes(FAKE_REF)) { + return { + ok: false, + why: 'ref available but not passed (hand-built map)' + }; + } + } + return renders ? { ok: true, why: 'ref passed unchanged' } : null; +} + +export async function gradeCase( + c: EvalCase, + run: Pick +): Promise<{ + grade: Record; + explanation: Record; +}> { + const surface = await loadSurface(); + const called = new Set(run.tool_calls); + const expand = (names: string[] = []): string[] => + names.flatMap((n) => (n === '*mapbox*' ? [...surface.names] : [n])); + for (const n of [ + ...expand(c.expect.first), + ...expand(c.expect.must), + ...expand(c.expect.any_of), + ...expand(c.expect.must_not) + ]) { + // A renamed or removed tool would otherwise turn a must_not into a silent pass. + if (!surface.names.has(n) && !HOST_TOOL_NAMES.has(n)) { + throw new Error(`case ${c.id} references unknown tool ${n}`); + } + } + + const problems: string[] = []; + const first = run.tool_calls[0]; + const firstOk = + !c.expect.first || + (first !== undefined && expand(c.expect.first).includes(first)); + if (!firstOk) { + problems.push( + `first tool was ${first ?? 'none'}, expected one of ${c.expect.first?.join('/')}` + ); + } + const missing = expand(c.expect.must).filter((n) => !called.has(n)); + if (missing.length) problems.push(`missing: ${missing.join(', ')}`); + const anyOf = expand(c.expect.any_of); + if (anyOf.length && !anyOf.some((n) => called.has(n))) { + problems.push(`none of: ${c.expect.any_of?.join(', ')}`); + } + const forbidden = expand(c.expect.must_not).filter((n) => called.has(n)); + if (forbidden.length) + problems.push(`called forbidden: ${forbidden.join(', ')}`); + + const pass = problems.length === 0 ? 1 : 0; + const trajectory = run.tool_calls.join(' > ') || '(no tools)'; + // Rows without a render call score 1 on ref_ok so the metric's failures + // are only real ref mistakes. + const refs = checkRefs(run.transcript); + return { + grade: { + pass, + first_ok: firstOk ? 1 : 0, + no_forbidden: forbidden.length ? 0 : 1, + ref_ok: refs && !refs.ok ? 0 : 1 + }, + explanation: { + pass: pass + ? `ok: ${trajectory}` + : `${problems.join('; ')} | ${trajectory}`, + ref_ok: refs ? refs.why : 'no render_map_tool call' + } + }; +} diff --git a/evals/tool-selection/rescore.ts b/evals/tool-selection/rescore.ts new file mode 100644 index 00000000..180abd9f --- /dev/null +++ b/evals/tool-selection/rescore.ts @@ -0,0 +1,49 @@ +// Copyright (c) Mapbox, Inc. +// Licensed under the MIT License. + +/** + * Re-grade existing results from their saved transcripts, without calling the + * API. Use after changing the grader or a case's `expect` so old rows are + * scored the same way as new ones. + * + * npx tsx evals/tool-selection/rescore.ts baseline v1 + */ + +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { gradeCase, loadCases } from './harness.ts'; + +const FLOW = '.claude/hillclimb/tool-selection'; + +const cases = new Map(loadCases().map((c) => [c.id, c])); + +for (const variant of process.argv.slice(2)) { + const path = join(FLOW, variant, 'results.jsonl'); + const rows = readFileSync(path, 'utf8') + .split('\n') + .filter((l) => l.trim()) + .map((l) => JSON.parse(l) as Record); + + let changed = 0; + for (const row of rows) { + const c = cases.get(String(row.prompt_id)); + if (!c) throw new Error(`${variant}: no case for ${row.prompt_id}`); + const transcript = JSON.parse( + readFileSync( + join(FLOW, variant, 'traces', `${row.prompt_id}_rep${row.rep}.json`), + 'utf8' + ) + ) as Array>; + const trajectory = String(row.trajectory ?? ''); + const g = await gradeCase(c, { + tool_calls: trajectory ? trajectory.split(' > ') : [], + transcript + }); + if (JSON.stringify(g.grade) !== JSON.stringify(row.grade)) changed++; + row.grade = g.grade; + row.explanation = g.explanation; + } + writeFileSync(path, rows.map((r) => JSON.stringify(r)).join('\n') + '\n'); + console.log(`${variant}: ${rows.length} rows, ${changed} grades changed`); +} +process.exit(0); diff --git a/evals/tool-selection/run-eval.mjs b/evals/tool-selection/run-eval.mjs new file mode 100644 index 00000000..051d8819 --- /dev/null +++ b/evals/tool-selection/run-eval.mjs @@ -0,0 +1,591 @@ +#!/usr/bin/env node +// Runner scaffold for the build-eval / hillclimb loop. Copy this into the +// user's repo and fill in loadCases / runCase / gradeCase below - the I/O +// shape, file naming, resume, and CLI surface are already hillclimb-ready +// so adding v2, v3, ... is `--variant v3`, not a refactor. +// +// node run-eval.mjs --flow .claude/hillclimb/ --variant baseline --reps 2 +// +// Structural properties this encodes (so you don't have to remember them): +// - parameterized by --variant / --model / --reps (no hardcoded A/B pair) +// - rep-aware filenames + resume (traces/_rep.json) +// - reads _state.json, never writes it (loop state belongs to the orchestrator) - +// the ONE exception is --approve-harness recording `harness_sha` (see below) +// - refuses to run when the harness (this file + _state.json.harness_paths) has +// changed since the sha a human last approved with --approve-harness, so a +// round that edits the runner cannot execute unreviewed under a standing +// session allowlist +// - pairwise graders judge against frozen baseline/ref/.* on disk +// - writes rows as cases complete (crash-safe) +// - jittered exponential backoff on transient 429/overloaded/5xx errors +// - hard per-case wall-clock ceiling (--timeout-s; stream keepalives don't reset it) +// - served-model assertion (response model must match --model; documented alias->snapshot +// shapes tolerated: 'foo-latest'/'foo-0'/'foo' -> 'foo-20250101' / 'foo@20250101' / 'foo-2025-01-01') +// - failed attempts land in errors.jsonl with a failure class and, when the call +// completed, the billed model/usage (never in results.jsonl) +// - row ids, trace filenames, and frozen refs share one path-safe id +// (original id kept in meta.original_id when sanitization changed it) + +import { createHash } from 'node:crypto'; +import { closeSync, constants as FS, existsSync, fstatSync, ftruncateSync, lstatSync, mkdirSync, openSync, readFileSync, realpathSync, writeFileSync, writeSync } from 'node:fs'; +import { dirname, isAbsolute, join, relative, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +// Every output write refuses symlinks: the flow dir is model-influenced, and a +// prompt-injected round can plant `results.jsonl -> ~/.bashrc` where the next +// unattended run would append. POSIX opens O_NOFOLLOW (a symlink fails with +// ELOOP); Windows - where Node leaves O_NOFOLLOW undefined and Bun defines a +// meaningless value - lstat-refuses first. Symlinked parent dirs are +// refused the same way. Same discipline as the report builders' reads. +const WIN = process.platform === 'win32'; +const NOFOLLOW = WIN ? 0 : FS.O_NOFOLLOW; +// A guard that cannot tell must refuse: only "no such entry" reads as absent; +// any other lstat failure (EACCES, ENAMETOOLONG, ...) is rethrown, never "no". +const lstatOrNull = p => { try { return lstatSync(p); } catch (e) { if (e?.code === 'ENOENT') return null; throw e; } }; +const isSymlink = p => lstatOrNull(p)?.isSymbolicLink() === true; +// Stderr lines interpolate model-influenced bytes (case ids, error text that +// can echo model output, JSON.parse messages). Strip escape sequences and +// control characters, as build-report-lite.mjs's eprint does, so a planted +// OSC/CSI can't retitle the terminal or forge output lines. +const ESC_SEQ = /\x1b\[[0-?]*[ -\/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)?|\x1b[@-_]/g; +const CONTROL = /[\x00-\x1f\x7f-\x9f]/g; +const termSafe = s => String(s).replace(ESC_SEQ, '').replace(CONTROL, ''); +const eprint = (...a) => console.error(...a.map(termSafe)); +// The leaf checks above can't see a symlink on an INTERMEDIATE component +// (lstat and open both resolve those silently), so every open is also bound +// to the flow root: main() captures realpathSync(flow) once, and any path +// whose resolved parent leaves it - e.g. `vdir` or the flow dir itself +// replaced by a directory symlink - is refused when the check sees it. +// Residual, all platforms: the check and the open are separate path lookups +// (Node's sync fs has no openat-style call), so a directory swapped for a +// symlink in between is still followed. This stops a planted link, not a +// writer racing the run. +let flowRealRoot = null; +function assertInFlow(dir, what) { + if (flowRealRoot == null) throw new Error(`refusing to ${what}: flow root not resolved yet`); + const dirReal = realpathSync(dir); + if (dirReal !== flowRealRoot && !dirReal.startsWith(flowRealRoot + (WIN ? '\\' : '/'))) + throw new Error(`refusing to ${what}: ${dir} resolves outside the flow directory`); +} +function openNoFollow(p, flags) { + if (isSymlink(dirname(p))) throw new Error(`refusing to open through symlinked directory: ${dirname(p)}`); + assertInFlow(dirname(p), 'open'); + if (WIN && isSymlink(p)) throw new Error(`refusing to open through symlink: ${p}`); + const fd = openSync(p, flags | NOFOLLOW, 0o644); + try { + const st = fstatSync(fd); + if (!st.isFile()) throw new Error(`refusing to use non-regular file: ${p}`); + // O_NOFOLLOW and lstat cannot see a hard link: a second name for a file + // outside the flow dir opens as an ordinary regular file. Nothing the + // runner creates has more than one link, so refuse any that does. + if (st.nlink > 1) throw new Error(`refusing to use ${p}: it has a second hard link (another name for the same file); replace it with a plain copy if it is yours`); + } catch (e) { closeSync(fd); throw e; } + return fd; +} +// writeFileSync on the fd loops until every byte lands (a bare writeSync is +// one write(2) that may return short on ENOSPC and silently truncate a +// results row or trace). +// Opened without O_TRUNC and truncated only after openNoFollow's checks, so a +// refused file keeps its bytes. +function writeFileNoFollow(p, data) { + const fd = openNoFollow(p, FS.O_WRONLY | FS.O_CREAT); + try { ftruncateSync(fd, 0); writeFileSync(fd, data); } finally { closeSync(fd); } +} +// POSIX appends atomically under O_APPEND with no position. On Windows, Bun +// writes an O_APPEND handle at offset 0 unless given a position, so there the +// write starts at the current size and re-issues any short write. +function appendFileNoFollow(p, data) { + const fd = openNoFollow(p, FS.O_WRONLY | FS.O_CREAT | FS.O_APPEND); + try { + if (!WIN) { writeFileSync(fd, data); return; } + const buf = Buffer.from(data); + const start = fstatSync(fd).size; + for (let off = 0; off < buf.length;) { + const n = writeSync(fd, buf, off, buf.length - off, start + off); + if (n <= 0) throw new Error(`append to ${p} made no progress`); + off += n; + } + } finally { closeSync(fd); } +} +// Reads of the frozen pairwise refs get the same discipline as writes (same +// open guard): the flow dir is model-influenced, so `baseline/ref/ -> +// ~/.ssh/id_rsa` planted after the startup preflight must not be read into +// the judge prompt. lexists probes with lstat so a planted symlink still +// counts as "present" at the freeze guard (never overwritten - or followed). +const lexists = p => lstatOrNull(p) != null; +function readFileNoFollow(p) { + const fd = openNoFollow(p, FS.O_RDONLY); + try { return readFileSync(fd, 'utf8'); } finally { closeSync(fd); } +} +// null when the file is absent; any other failure (a planted link included) throws. +function readIfPresent(p) { + try { return readFileNoFollow(p); } catch (e) { if (e?.code === 'ENOENT') return null; throw e; } +} +function mkdirNoFollow(dir) { + if (isSymlink(dir)) throw new Error(`refusing to use symlinked directory: ${dir}`); + mkdirSync(dir, { recursive: true }); + // Check after creating: mkdirSync(recursive) follows symlinked ancestors, + // so a dir minted through one resolves outside the flow root and is refused + // here before any file lands in it. + assertInFlow(dir, 'create directory'); +} +// Frozen pairwise refs may carry an extension; reader and freeze-guard probe +// the same list so a suffixed ref never gets an extensionless shadow. +const REF_EXTS = ['', '.html', '.txt', '.json']; + +// --- fill these in ---------------------------------------------------------- +// Vendored from the claude-api skill's runner-scaffold.mjs. Everything below +// the "harness" line is the scaffold unchanged except the --model default. +// Run through tsx so the TypeScript harness can be imported: +// npx tsx evals/tool-selection/run-eval.mjs --variant baseline --reps 3 + +import * as harness from './harness.ts'; + +async function loadCases() { + return harness.loadCases(); +} + +async function runCase(input, ctx) { + return harness.runCase(input, ctx); +} + +// Programmatic grading against each case's expected tool trajectory; `ref` +// is unused because there is no pairwise judge. +async function gradeCase(input, run, ref, ctx) { + return harness.gradeCase(input, run); +} + +function perfFrom(run) { + return { + tool_calls: run.tool_calls.length, + tool_searches: run.tool_searches.length, + unstubbed: run.unstubbed.length, + trajectory: run.tool_calls.join(' > ') + }; +} + +// --- harness (you usually won't need to touch below this line) -------------- + +function parseArgs(argv) { + const a = { flow: '.claude/hillclimb/tool-selection', variant: 'baseline', + model: 'claude-opus-5-5', reps: 1, concurrency: 8, timeoutS: 600, + approveHarness: false }; + // A flag at the end of argv would otherwise consume undefined - which for + // --model equals the default and silently disables the served-model check. + const val = (i) => { if (argv[i] === undefined) { eprint(`missing value for ${argv[i - 1]}`); usage(); process.exit(2); } return argv[i]; }; + for (let i = 0; i < argv.length; i++) { + const k = argv[i]; + if (k === '--flow') a.flow = val(++i); + else if (k === '--variant') a.variant = val(++i); + else if (k === '--model') a.model = val(++i); + else if (k === '--reps') a.reps = +val(++i); + else if (k === '--concurrency') a.concurrency = +val(++i); + else if (k === '--timeout-s') a.timeoutS = +val(++i); + else if (k === '--approve-harness') a.approveHarness = true; + else if (k === '-h' || k === '--help') { usage(); process.exit(0); } + else { eprint(`unknown argument: ${k}`); usage(); process.exit(2); } + } + if (!/^(baseline|v[1-9]\d*)$/.test(a.variant)) { + // The report only reads directories named 'baseline' or 'v' - any other + // name runs to completion but spends the pass into a directory the Summary, + // trajectory, and budget arithmetic never see. + eprint(`--variant must be 'baseline' or 'v', got '${a.variant}'`); + usage(); process.exit(2); + } + if (!Number.isFinite(a.timeoutS) || a.timeoutS < 0 + || a.timeoutS * 1000 > 2147483647 // setTimeout clamps >2^31-1 ms to 1 ms - the ceiling would fire instantly + || !Number.isInteger(a.reps) || a.reps < 1 + || !Number.isInteger(a.concurrency) || a.concurrency < 1) { usage(); process.exit(2); } + return a; +} +function usage() { + eprint('usage: node run-eval.mjs --flow DIR --variant ID [--model ID] [--reps N] [--concurrency N] [--timeout-s N (0 = no ceiling)] [--approve-harness]'); +} + +// Harness integrity gate. The hillclimb loop gets this runner command +// allowlisted for the session and then runs rounds unattended, while the +// per-round change (proposed by an analyzer fed untrusted transcripts) may +// legitimately edit harness code. Without this gate a round that rewrites the +// runner would execute attacker-chosen code on the next unattended run under +// the user's one-time approval. So: sha256 over this file, any lockfile +// beside it or in the directory the runner is invoked from (dependency drift +// is harness drift too; with neither, a note says so), plus +// every path in `_state.json.harness_paths` (relative to the directory the +// runner is invoked from, i.e. the repo root); compare to +// `_state.json.harness_sha`; refuse on absent/mismatch unless a human passes +// --approve-harness, which records the new sha. That write is the one +// sanctioned exception to "never write _state.json". +// +// Scope, stated plainly: this is a change detector, not a security boundary. +// The sha, the path list and this file all live where the loop agent can +// write, so an agent that is already prompt-injected can forge the record or +// pass --approve-harness itself. What it catches is the common case - a +// round's harness edit reaching an unattended run unreviewed - and what +// actually bounds an unattended run is the permission allowlist the user +// grants the runner command (scope it to this exact command line, not a +// prefix). Installed dependencies beyond the lockfile are outside the digest. +function checkHarness(statePath, st, approve) { + const self = fileURLToPath(import.meta.url); + const listed = Array.isArray(st.harness_paths) ? st.harness_paths.map(String) : []; + const lockfiles = [...new Set([dirname(self), process.cwd()].flatMap(d => + ['package-lock.json', 'bun.lock', 'bun.lockb', 'yarn.lock', 'pnpm-lock.yaml'].map(f => join(d, f))))] + .filter(f => existsSync(f)); + const paths = [...new Set([self, ...lockfiles, ...listed.map(p => resolve(p))])].sort(); + const h = createHash('sha256'); + const hashed = []; + for (const p of paths) { + let buf; + try { buf = readFileSync(p); } + catch (e) { + if (p === self) throw e; + eprint(`warning: harness path '${relative(process.cwd(), p)}' not readable (${e?.code || 'error'}) - skipped`); + continue; + } + h.update(relative(process.cwd(), p)).update('\0').update(buf).update('\0'); + hashed.push(relative(process.cwd(), p)); + } + const sha = h.digest('hex'); + if (st.harness_sha === sha) return; + // Said only here, where a person is about to approve or is being refused. + if (!lockfiles.length) eprint('note: no lockfile beside the runner or in the current directory - dependency changes are outside the harness sha'); + if (approve) { + st.harness_sha = sha; + writeFileNoFollow(statePath, JSON.stringify(st, null, 2) + '\n'); + eprint(`harness approved: sha256 ${sha.slice(0, 12)} over ${hashed.length} file(s) recorded in ${statePath}`); + return; + } + if (st.harness_sha == null) { + eprint(`no approved harness sha in ${statePath} (computed ${sha.slice(0, 12)} over: ${hashed.join(', ')}).`); + eprint('Review the harness, then run once with --approve-harness to record it.'); + } else { + eprint(`harness changed since last approved run (files: ${hashed.join(', ')}); ` + + `approved ${String(st.harness_sha).slice(0, 12)}, now ${sha.slice(0, 12)}.`); + eprint('Re-run with --approve-harness after reviewing the diff.'); + } + process.exit(2); +} + +// Transient provider errors (429 / overloaded / 5xx) retry with jittered +// exponential backoff - a zero-delay retry loop multiplies cost invisibly +// under rate limits and can turn one transient 429 into a torn-down batch. +// The attempt count lands in the row's meta (or the errors sidecar) so retry +// churn is visible in the data, not just the bill. +async function withBackoff(fn, retry, deadline = Infinity, tries = 5) { + for (let attempt = 0; ; attempt++) { + // Checked before every attempt, not just before sleeps: once the case's + // ceiling has passed, an abandoned chain must not issue another call + // (e.g. a judge call after the app call consumed the whole ceiling). + if (Date.now() >= deadline) { + const e = new Error('wall-clock ceiling exceeded before attempt'); + e.failure_class = 'timeout'; + throw e; + } + try { return await fn(); } catch (e) { + const status = e?.status ?? e?.response?.status; + const transient = status === 429 || status === 529 || (status >= 500 && status < 600) + || /overloaded|rate.?limit/i.test(String(e?.message ?? '')); + if (!transient || attempt >= tries - 1) throw e; + const delay = Math.min(60_000, 1000 * 2 ** attempt) * (0.5 + Math.random()); + // Never start a retry that would outlive the case's wall-clock ceiling - + // otherwise an abandoned chain keeps issuing API calls after the case failed. + if (Date.now() + delay >= deadline) throw e; + retry.count++; + await new Promise(r => setTimeout(r, delay)); + } + } +} + +// Hard per-case wall-clock ceiling, independent of stream liveness - a hung +// SSE stream can emit keepalives forever, defeating inactivity-based timers. +// The underlying call may keep running; the case fails and the slot is freed. +function withTimeout(promise, seconds, label) { + if (!(seconds > 0)) return promise; + let timer; + const ceiling = new Promise((_, reject) => { + timer = setTimeout(() => { + const e = new Error(`${label}: exceeded ${seconds}s wall-clock ceiling`); + e.failure_class = 'timeout'; + reject(e); + }, seconds * 1000); + }); + return Promise.race([promise, ceiling]).finally(() => clearTimeout(timer)); +} + +// Case ids appear in file paths AND as the row/file join key the report uses, +// so rows, trace filenames, and frozen refs all carry the same path-safe id. +// When sanitization changes the id, a short content hash keeps distinct ids +// distinct ('case/1' vs 'case_1'); the original rides in meta.original_id. +function pathSafeId(id) { + const raw = String(id); + const cleaned = raw.replace(/[^\w.-]/g, '_'); + // Idempotent by construction: anything already path-safe and within the + // length bound - including this function's own truncated+suffixed output - + // passes through unchanged. Long ids (URLs, prompt text as id) truncate to + // 120 chars plus an 8-hex hash of the full original, so they fail here, not + // at the trace write after the spend, and distinct ids stay distinct. + if (cleaned === raw && raw.length <= 129) return raw; + return `${cleaned.slice(0, 120)}-${createHash('sha256').update(raw).digest('hex').slice(0, 8)}`; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + // lstat("link/") follows the final symlink, so a trailing separator on + // --flow would blind every leaf isSymlink check below - strip it first. + // Only Windows treats `\` as a separator; on POSIX it is a filename byte, so + // splitting on it would walk prefixes that are not real path components. + args.flow = args.flow.replace(WIN ? /(.)[\\/]+$/ : /(.)\/+$/, '$1'); + const flowSegments = args.flow.split(WIN ? /[\\/]/ : '/'); + // A `.`/`..` segment (e.g. a trailing `/.`) makes isSymlink(args.flow) below + // resolve a different final component than the named dir - following a + // planted link at the flow root - while join() collapses it and the absolute + // branch skips the ancestor walk. Refuse dot segments outright + // (absolute --flow stays supported). + if (flowSegments.some(seg => seg === '.' || seg === '..')) { + eprint(`refusing to run: --flow must not contain '.' or '..' segments, got '${args.flow}'`); + process.exit(2); + } + const vdir = join(args.flow, args.variant); + // Preflight every output path before the first model call: a planted + // symlink would otherwise fail each case after its (billed) run. + for (const p of [args.flow, join(args.flow, 'baseline'), vdir, join(vdir, 'traces'), + join(vdir, 'results.jsonl'), join(vdir, 'errors.jsonl'), + join(vdir, 'progress.txt'), join(args.flow, 'baseline', 'ref'), join(args.flow, '_state.json')]) + if (isSymlink(p)) { eprint(`refusing to run: ${p} is a symlink (the flow dir must hold regular files)`); process.exit(2); } + // A relative --flow (the documented `.claude/hillclimb/` layout) is + // also lstat-walked component by component from the cwd: a pre-planted + // link at an ancestor (`.claude/hillclimb -> elsewhere`) would otherwise + // relocate the root capture below - the containment anchor itself - to the + // attacker's target. An absolute --flow is the caller's own trust decision + // and is not walked (an absolute ancestor link can be legitimate: /tmp on + // macOS). + if (!isAbsolute(args.flow)) { + let walk = ''; + for (const part of flowSegments.filter(Boolean).slice(0, -1)) { + walk = walk ? join(walk, part) : part; + if (isSymlink(walk)) { eprint(`refusing to run: ${walk} is a symlink (ancestor of --flow)`); process.exit(2); } + } + } + // Every later open/mkdir is bound to this resolved root (see assertInFlow): + // create the flow dir when fresh (the preflight above refused a link at it + // and, for a relative path, at every ancestor), then capture where it + // really resolves. + mkdirSync(args.flow, { recursive: true }); + flowRealRoot = realpathSync(args.flow); + mkdirNoFollow(join(vdir, 'traces')); + // _state.json is READ-ONLY here. The orchestrator owns it. Absent is fine + // (a baseline-only run has no loop state yet), but present-and-unparsable + // must not let the id-space gate below pass vacuously over a corrupt file. + const statePath = join(args.flow, '_state.json'); + let st = {}; + // Read through the no-follow opener like every other flow-dir file; the + // parse message is not echoed (it can quote the file's first bytes). + const stateText = readIfPresent(statePath); + if (stateText != null) { + try { st = JSON.parse(stateText) || {}; } + catch { eprint(`${statePath} exists but is not valid JSON - fix it before spending a pass`); process.exit(2); } + } + checkHarness(statePath, st, args.approveHarness); + const ctx = { ...args, state: st }; + + // Resume: which (id, rep) pairs already have a row? + const resultsPath = join(vdir, 'results.jsonl'); + const done = new Set(); + for (const ln of (readIfPresent(resultsPath) ?? '').split('\n')) { + if (!ln.trim()) continue; + try { const r = JSON.parse(ln); done.add(`${r.prompt_id}\0${r.rep}`); } catch {} + } + // Rows key on the path-safe id (see pathSafeId), so resume must too. + + const cases = await loadCases(); + // Validate the id space before spending anything: duplicate path-safe ids - + // including case-insensitive twins, which macOS/Windows filesystems collapse - + // would silently overwrite traces and frozen refs; and a _state.json split id + // that matches no case would silently shrink the scored denominator. + const seen = new Map(); + for (const c of cases) { + const k = pathSafeId(c.id).toLowerCase(); + if (seen.has(k)) { + eprint(`duplicate case id after sanitization: '${c.id}' collides with '${seen.get(k)}'`); + process.exit(2); + } + seen.set(k, c.id); + } + const safeIds = new Set(cases.map(c => pathSafeId(c.id))); + for (const k of ['train_ids', 'val_ids', 'test_ids']) + if (st[k] != null && !Array.isArray(st[k])) { eprint(`_state.json ${k} must be a list of ids`); process.exit(2); } + for (const sid of [...(st.train_ids ?? []), ...(st.val_ids ?? []), ...(st.test_ids ?? [])]) { + const s = String(sid); // the adapter joins with String() on both sides - numeric ids are fine + if (safeIds.has(s)) continue; // matches a loaded case - definitionally valid + if (s !== pathSafeId(s)) { + // Can never match a row: rows key on path-safe ids. This is the silent + // shrunken-denominator bug - fail before anything is spent. + eprint(`_state.json split id '${s}' is not a path-safe id - record split ids exactly as they appear in results.jsonl's prompt_id`); + process.exit(2); + } + // Well-formed but absent is legitimate (a trimmed top-K subset run) - note it, don't fail. + eprint(`note: split id '${s}' matches no loaded case (expected for a trimmed subset run)`); + } + const refDir = join(args.flow, 'baseline', 'ref'); + const tasks = []; + for (const c of cases) for (let rep = 0; rep < args.reps; rep++) { + if (done.has(`${pathSafeId(c.id)}\0${rep}`)) continue; + tasks.push({ c, rep }); + } + eprint(`[${args.variant}] ${tasks.length} of ${cases.length * args.reps} (id,rep) to run`); + + let i = 0, ok = 0, fail = 0; + const errorsPath = join(vdir, 'errors.jsonl'); + // A hard crash (power loss, ENOSPC) can leave a torn final line with no + // trailing newline; the next append would merge two rows into one permanently + // unparseable line. Isolate any fragment before appending anything. + for (const p of [resultsPath, errorsPath]) { + const tail = readIfPresent(p); + if (tail && !tail.endsWith('\n')) appendFileNoFollow(p, '\n'); + } + async function worker() { + while (i < tasks.length) { + const { c, rep } = tasks[i++]; + const safeId = pathSafeId(c.id); + const t0 = Date.now(); + let lastRun = null; // survives into the catch - billed spend on a failed attempt + let rowWritten = false; // set once the results row lands - the attempt is scored + const deadline = args.timeoutS > 0 ? t0 + args.timeoutS * 1000 : Infinity; + const appRetry = { count: 0 }, judgeRetry = { count: 0 }; + try { + // One ceiling over the whole case - app call, identity check, and grading - + // so a hung judge stream can't hold the slot either. + const { run, g, latency_s } = await withTimeout((async () => { + let tAttempt = t0; + const run = await withBackoff(() => { tAttempt = Date.now(); return runCase(c, ctx); }, + appRetry, deadline); + lastRun = run; + // latency_s = the final app attempt only; backoff sleeps, failed + // attempts, and judge time are excluded (retry counts are in meta). + const latency_s = (Date.now() - tAttempt) / 1000; + // Serving identity: fail loudly when the response was served by a model + // other than the one requested. Accept exact match or a documented + // alias->snapshot resolution - 'foo-latest'/'foo-0'/'foo' served as + // 'foo-20250101', 'foo@20250101', or 'foo-2025-01-01'. Anything else - + // another snapshot of the requested pin, a sibling model, or the bare + // base id ('foo-latest' served as 'foo', an unversioned echo that can + // hide snapshot drift across rounds) - fails the attempt. Non-Anthropic + // id schemes (e.g. Bedrock's 'anthropic.claude-...-v1:0') need their own + // rule here. + if (ctx.model && run.model && run.model !== ctx.model) { + const base = ctx.model.replace(/-latest$|-0$/, ''); + const rest = String(run.model).startsWith(base) + ? String(run.model).slice(base.length) : null; + if (!(rest != null && /^[-@](\d{8}|\d{4}-\d{2}-\d{2})$/.test(rest))) { + const e = new Error(`served model ${run.model} != requested ${ctx.model}`); + e.failure_class = 'serving_substitution'; + throw e; + } + } + // Frozen pairwise reference (never regenerated): baseline/ref/.* + let ref = null; + if (args.variant !== 'baseline') { + const p = join(refDir, safeId); + // A planted symlink throws (ELOOP) rather than feeding the judge + // its target; the case then fails loudly instead of leaking. + for (const ext of REF_EXTS) { + try { ref = readFileNoFollow(p + ext); break; } + catch (e) { if (e?.code !== 'ENOENT') throw e; } + } + } + const g = await withBackoff(() => gradeCase(c, run, ref, ctx), judgeRetry, deadline); + return { run, g, latency_s }; + })(), args.timeoutS, `${c.id} rep${rep}`); + const row = { + prompt_id: safeId, rep, prompt: c.prompt ?? c.input ?? c.id, + tags: c.tags, attachments: c.attachments, + meta: safeId !== String(c.id) || appRetry.count || judgeRetry.count + ? { ...(c.meta ?? {}), + ...(safeId !== String(c.id) ? { original_id: String(c.id) } : {}), + ...(appRetry.count ? { retries: appRetry.count } : {}), + ...(judgeRetry.count ? { judge_retries: judgeRetry.count } : {}) } + : c.meta, + model: run.model, usage: run.usage, stop_reason: run.stop_reason, + // The report keys on `status`, not stop_reason: a clipped answer is + // counted and shown but kept out of the means. runCase may set + // run.status to override the max_tokens rule. + status: run.status ?? (run.stop_reason === 'max_tokens' ? 'truncated' : 'ok'), + judge_model: g.judge_model ?? run.judge_model, + judge_usage: g.judge_usage ?? run.judge_usage, + latency_s, ...perfFrom(run), + grade: g.grade, explanation: g.explanation, + }; + appendFileNoFollow(resultsPath, JSON.stringify(row) + '\n'); + rowWritten = true; // past this point the attempt is scored - a later throw (trace write, ref freeze) must not also append an error row + if (run.transcript) + writeFileNoFollow(join(vdir, 'traces', `${safeId}_rep${rep}.json`), + JSON.stringify(run.transcript, null, 2)); + // For pairwise: on the baseline run, freeze the reference output once. + if (args.variant === 'baseline' && run.output != null + && !REF_EXTS.some(ext => lexists(join(refDir, safeId) + ext))) { + mkdirNoFollow(refDir); + writeFileNoFollow(join(refDir, safeId), + typeof run.output === 'string' ? run.output : JSON.stringify(run.output)); + } + ok++; + } catch (e) { + fail++; + if (rowWritten) { + // The attempt scored; only a post-row write (trace, ref) failed. An error + // row here would double-count the billed usage under the budget rule. + eprint(` [${args.variant}] ${c.id} rep${rep} scored, but a post-row write failed: ${e?.message || e}`); + continue; + } + // Failed attempts are data too - but they must not occupy the (case, rep) + // slot in results.jsonl, or resume would never re-run them. + appendFileNoFollow(errorsPath, JSON.stringify({ + prompt_id: safeId, rep, + ...(safeId !== String(c.id) ? { original_id: String(c.id) } : {}), + failure_class: e?.failure_class ?? 'error', + error: String(e?.message || e), + retries: appRetry.count, judge_retries: judgeRetry.count, + // Billed-but-failed spend stays countable: when the app call completed + // before the failure (e.g. a served-model mismatch, a judge-stage + // ceiling), carry its identity and usage on the error row. + model: lastRun?.model, usage: lastRun?.usage, + judge_model: e?.judge_model ?? lastRun?.judge_model, + judge_usage: e?.judge_usage ?? lastRun?.judge_usage, + latency_s: (Date.now() - t0) / 1000, + }) + '\n'); + eprint(` [${args.variant}] ${c.id} rep${rep} FAILED: ${e?.message || e}`); + } + } + } + // One progress line every 30s (and to /progress.txt) so "how far along + // is it?" is answerable from the background shell's output or one file read, + // without the orchestrator parsing results.jsonl mid-write. ETA is a plain + // rate extrapolation from this pass. + const t0 = Date.now(); + const progress = () => { + const done = ok + fail, total = tasks.length; + const el = (Date.now() - t0) / 1000; + const eta = done ? Math.round((el / done) * (total - done)) : null; + const line = `[${args.variant}] ${done}/${total} done (${ok} ok, ${fail} failed), ` + + `${Math.round(el)}s elapsed` + (eta != null ? `, ~${eta}s left` : ''); + eprint(line); + try { writeFileNoFollow(join(vdir, 'progress.txt'), line + '\n'); } catch {} + }; + const tick = setInterval(progress, 30_000); + workersStarted = true; + await Promise.all(Array.from({ length: Math.max(1, args.concurrency) }, worker)); + clearInterval(tick); progress(); + eprint(`[${args.variant}] done - ${ok} ok, ${fail} failed -> ${resultsPath}`); + process.exit(fail ? 1 : 0); +} + +// Anything main() throws prints as one sanitized line, not a raw stack. Before +// the workers start it is a refusal (a planted link at _state.json or +// results.jsonl, an lstat that fails, an error from loadCases) and exits 2 like +// the preflight refusals. After they start, only a failed errors.jsonl append +// gets here; rows may already be on disk, so say that and exit 1. +let workersStarted = false; +main().catch(e => { + const m = String(e?.message || e); + if (workersStarted) { eprint('stopped mid-run (rows already written are kept; re-run to resume): ' + m); process.exit(1); } + eprint(m.startsWith('refusing to ') ? m : 'refusing to run: ' + m); + process.exit(2); +}); diff --git a/package-lock.json b/package-lock.json index d68c45c0..78642a7d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "mcp-server": "dist/esm/index.js" }, "devDependencies": { + "@anthropic-ai/sdk": "^0.133.0", "@eslint/js": "^9.27.0", "@types/node": "^22.0.0", "@typescript-eslint/eslint-plugin": "^8.58.2", @@ -46,6 +47,7 @@ "plop": "^4.0.1", "prettier": "^3.0.0", "tshy": "^4.1.1", + "tsx": "^4.23.15", "typescript": "^6.0.2", "typescript-eslint": "^8.58.2", "vitest": "^4.1.4" @@ -54,6 +56,28 @@ "node": ">=22" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.133.0", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.133.0.tgz", + "integrity": "sha512-jlqUN8p8ESTJj+GJnA4KjzbEfF1waEoGzRT8jwytlYGFFgv+/DUQOxlYGvYw10UzhD1kwLqYzxvA3FrTB2YWuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1", + "standardwebhooks": "^1.0.0" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, "node_modules/@babel/code-frame": { "version": "7.29.0", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", @@ -266,6 +290,16 @@ "node": ">=6.0.0" } }, + "node_modules/@babel/runtime": { + "version": "7.29.10", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.10.tgz", + "integrity": "sha512-NpugTzbKgGrNe45Ui32cN8vCFFMxVtP/9rtfwv0nP5FRDAjvYBxJcLVpYA9mr1jxP6cnIT7x9LUzuDTbWYYNRA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/template": { "version": "7.28.6", "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.28.6.tgz", @@ -977,6 +1011,448 @@ "tslib": "^2.4.0" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.9.1", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz", @@ -3216,6 +3692,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@stablelib/base64": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", + "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -7221,6 +7704,48 @@ "node": ">= 0.4" } }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -7865,6 +8390,13 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-sha256": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", + "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", + "dev": true, + "license": "Unlicense" + }, "node_modules/fast-uri": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz", @@ -9207,6 +9739,20 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -11637,6 +12183,17 @@ "dev": true, "license": "MIT" }, + "node_modules/standardwebhooks": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.1.1.tgz", + "integrity": "sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@stablelib/base64": "^1.0.0", + "fast-sha256": "^1.3.0" + } + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -11975,6 +12532,13 @@ "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "license": "MIT" }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "dev": true, + "license": "MIT" + }, "node_modules/ts-api-utils": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", @@ -12061,6 +12625,25 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/tsx": { + "version": "4.23.15", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.15.tgz", + "integrity": "sha512-Yiex1Ovn8z2xPpOWckIiysV1SSyRMY9BkLF++q0yKiDxCqRhosKfMg3janKkiLBwZ5c/YryloKwGZcrEmtwxKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index a54dd02a..faa1678c 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,9 @@ "test": "vitest", "tracing:jaeger:start": "docker run --rm -d --name jaeger -p 16686:16686 -p 14250:14250 -p 4317:4317 -p 4318:4318 jaegertracing/all-in-one:latest", "tracing:jaeger:stop": "docker stop jaeger", - "tracing:verify": "node -e \"console.log('🔍 Verifying tracing setup with Jaeger...\\n1. Copy .env.example to .env and add your MAPBOX_ACCESS_TOKEN\\n2. Start Jaeger: npm run tracing:jaeger:start\\n3. Run server: npm run inspect:build\\n4. Check traces at: http://localhost:16686')\"" + "tracing:verify": "node -e \"console.log('🔍 Verifying tracing setup with Jaeger...\\n1. Copy .env.example to .env and add your MAPBOX_ACCESS_TOKEN\\n2. Start Jaeger: npm run tracing:jaeger:start\\n3. Run server: npm run inspect:build\\n4. Check traces at: http://localhost:16686')\"", + "eval:tools": "tsx evals/tool-selection/run-eval.mjs", + "eval:tools:report": "node evals/tool-selection/build-report-lite.mjs .claude/hillclimb/tool-selection" }, "lint-staged": { "*.{js,jsx,ts,tsx}": "eslint --fix", @@ -36,6 +38,7 @@ "license": "MIT", "homepage": "https://github.com/mapbox/mcp-server#readme", "devDependencies": { + "@anthropic-ai/sdk": "^0.133.0", "@eslint/js": "^9.27.0", "@types/node": "^22.0.0", "@typescript-eslint/eslint-plugin": "^8.58.2", @@ -53,6 +56,7 @@ "plop": "^4.0.1", "prettier": "^3.0.0", "tshy": "^4.1.1", + "tsx": "^4.23.15", "typescript": "^6.0.2", "typescript-eslint": "^8.58.2", "vitest": "^4.1.4" From 2354ced6c6a6a95773d2572b663225be973ed7b0 Mon Sep 17 00:00:00 2001 From: Matthew Podwysocki Date: Mon, 12 Oct 2026 09:59:13 -0400 Subject: [PATCH 2/3] docs(changelog): add PR number for tool-selection eval Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55350b45..aa643fff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Other Features -- **Tool-selection eval for catching routing regressions from tool descriptions.** `npm run eval:tools` runs 17 prompts against Claude (Opus 5.5 by default) and grades which tools it calls against the expected ones: map only when asked, `static_map_image_tool` only for image files, `ground_location_tool` vs `reverse_geocode_tool`, brand vs category search, and whether web search stays available for place questions. The tool list and server instructions come from a real `McpServer` built the way the hosted deployment builds it and read back over MCP. Each prompt runs with all descriptions loaded and again with the tools behind the API's tool search, which approximates hosts like Claude Desktop that load tools on demand. Tool results are canned per case, so Mapbox APIs are never called. A grader check also confirms that every `render_map_tool` call passes the map ref unchanged. `EVAL_SERVER_ROOT` points the harness at another checkout, so two branches can be compared, and `npm run eval:tools:report` builds an HTML report with per-case transcripts. It calls the Claude API and costs about $7 per 102-row run, so it is not part of `npm test`. Adds `@anthropic-ai/sdk` and `tsx` as dev dependencies. See `evals/tool-selection/README.md`. +- **Tool-selection eval for catching routing regressions from tool descriptions (#274).** `npm run eval:tools` runs 17 prompts against Claude (Opus 5.5 by default) and grades which tools it calls against the expected ones: map only when asked, `static_map_image_tool` only for image files, `ground_location_tool` vs `reverse_geocode_tool`, brand vs category search, and whether web search stays available for place questions. The tool list and server instructions come from a real `McpServer` built the way the hosted deployment builds it and read back over MCP. Each prompt runs with all descriptions loaded and again with the tools behind the API's tool search, which approximates hosts like Claude Desktop that load tools on demand. Tool results are canned per case, so Mapbox APIs are never called. A grader check also confirms that every `render_map_tool` call passes the map ref unchanged. `EVAL_SERVER_ROOT` points the harness at another checkout, so two branches can be compared, and `npm run eval:tools:report` builds an HTML report with per-case transcripts. It calls the Claude API and costs about $7 per 102-row run, so it is not part of `npm test`. Adds `@anthropic-ai/sdk` and `tsx` as dev dependencies. See `evals/tool-selection/README.md`. ## 0.15.1 - 2026-10-09 From 809a6e337a6e5723c285cd27d3c48c35d98b62f2 Mon Sep 17 00:00:00 2001 From: Matthew Podwysocki Date: Tue, 13 Oct 2026 12:59:17 -0400 Subject: [PATCH 3/3] feat(evals): add OpenAI provider, site-building cases and token check - EVAL_PROVIDER=openai runs the same cases through the Responses API, with server-side tool search for deferred mode (eval:tools:openai). - Each stubbed tool result gets its own ref, so an invented or hand-built render_map_tool call is caught by ref_ok. - Two site-building cases add a write_file host tool and expect export_map_html_tool (#275); needs_tools lets them fail cleanly on servers without it. - New no_token_ask metric flags asking the user for a Mapbox token or writing a placeholder token into a file. - EVAL_MODES selects eager/deferred; rescore reads EVAL_FLOW. Co-Authored-By: Claude Opus 5.5 --- .../tool-selection-openai/_state.json | 41 ++ .claude/hillclimb/tool-selection/_state.json | 7 +- CHANGELOG.md | 2 +- evals/tool-selection/README.md | 35 +- evals/tool-selection/cases.json | 32 ++ evals/tool-selection/harness.ts | 350 ++++++++++++++++-- evals/tool-selection/rescore.ts | 3 +- package-lock.json | 39 ++ package.json | 5 +- 9 files changed, 479 insertions(+), 35 deletions(-) create mode 100644 .claude/hillclimb/tool-selection-openai/_state.json diff --git a/.claude/hillclimb/tool-selection-openai/_state.json b/.claude/hillclimb/tool-selection-openai/_state.json new file mode 100644 index 00000000..74d7168a --- /dev/null +++ b/.claude/hillclimb/tool-selection-openai/_state.json @@ -0,0 +1,41 @@ +{ + "flow": "tool-selection-openai", + "metrics": [ + { + "id": "pass", + "label": "Pass", + "kind": "binary" + }, + { + "id": "first_ok", + "label": "First tool ok", + "kind": "binary" + }, + { + "id": "no_forbidden", + "label": "No forbidden", + "kind": "binary" + }, + { + "id": "ref_ok", + "label": "Ref passed", + "kind": "binary" + }, + { + "id": "no_token_ask", + "label": "No token ask", + "kind": "binary" + } + ], + "perf_fields": [ + "tool_calls", + "tool_searches", + "latency_s", + "usage" + ], + "harness_paths": [ + "evals/tool-selection/cases.json", + "evals/tool-selection/harness.ts" + ], + "harness_sha": "831b60ff5233625f5c7e02bc9a66498463a2ae1fd84231a879cff69b715a05e6" +} diff --git a/.claude/hillclimb/tool-selection/_state.json b/.claude/hillclimb/tool-selection/_state.json index e819e0dd..67f4186e 100644 --- a/.claude/hillclimb/tool-selection/_state.json +++ b/.claude/hillclimb/tool-selection/_state.json @@ -20,6 +20,11 @@ "id": "ref_ok", "label": "Ref passed", "kind": "binary" + }, + { + "id": "no_token_ask", + "label": "No token ask", + "kind": "binary" } ], "perf_fields": [ @@ -32,5 +37,5 @@ "evals/tool-selection/cases.json", "evals/tool-selection/harness.ts" ], - "harness_sha": "31d09b8e2336c02342caeed5edbd9a5ddfc6d4760662d16fcdc217c38e780a6f" + "harness_sha": "831b60ff5233625f5c7e02bc9a66498463a2ae1fd84231a879cff69b715a05e6" } diff --git a/CHANGELOG.md b/CHANGELOG.md index aa643fff..6e155353 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Other Features -- **Tool-selection eval for catching routing regressions from tool descriptions (#274).** `npm run eval:tools` runs 17 prompts against Claude (Opus 5.5 by default) and grades which tools it calls against the expected ones: map only when asked, `static_map_image_tool` only for image files, `ground_location_tool` vs `reverse_geocode_tool`, brand vs category search, and whether web search stays available for place questions. The tool list and server instructions come from a real `McpServer` built the way the hosted deployment builds it and read back over MCP. Each prompt runs with all descriptions loaded and again with the tools behind the API's tool search, which approximates hosts like Claude Desktop that load tools on demand. Tool results are canned per case, so Mapbox APIs are never called. A grader check also confirms that every `render_map_tool` call passes the map ref unchanged. `EVAL_SERVER_ROOT` points the harness at another checkout, so two branches can be compared, and `npm run eval:tools:report` builds an HTML report with per-case transcripts. It calls the Claude API and costs about $7 per 102-row run, so it is not part of `npm test`. Adds `@anthropic-ai/sdk` and `tsx` as dev dependencies. See `evals/tool-selection/README.md`. +- **Tool-selection eval for catching routing regressions from tool descriptions (#274).** `npm run eval:tools` runs 19 prompts against Claude (Opus 5.5 by default) and grades which tools it calls against the expected ones: map only when asked, `static_map_image_tool` only for image files, `ground_location_tool` vs `reverse_geocode_tool`, brand vs category search, and whether web search stays available for place questions. The tool list and server instructions come from a real `McpServer` built the way the hosted deployment builds it and read back over MCP. Each prompt runs with all descriptions loaded and again with the tools behind the API's tool search, which approximates hosts like Claude Desktop that load tools on demand. Tool results are canned per case, so Mapbox APIs are never called. A grader check also confirms that every `render_map_tool` call passes the map ref unchanged, with a distinct ref per tool result so an invented or hand-built map is caught. Two site-building cases give the agent a `write_file` host tool and check that it neither asks the user for a Mapbox token nor leaves a placeholder token in the page; they expect `export_map_html_tool` (#275) and fail on servers without it. `EVAL_SERVER_ROOT` points the harness at another checkout, so two branches can be compared, and `npm run eval:tools:report` builds an HTML report with per-case transcripts. It calls the Claude API and costs about $8 per 114-row run, so it is not part of `npm test`. `EVAL_PROVIDER=openai` (`npm run eval:tools:openai -- --model `) runs the same cases against an OpenAI model through the Responses API, including OpenAI's server-side tool search for the deferred mode, so description changes can be checked for ChatGPT-style regressions too. Adds `@anthropic-ai/sdk`, `openai` and `tsx` as dev dependencies. See `evals/tool-selection/README.md`. ## 0.15.1 - 2026-10-09 diff --git a/evals/tool-selection/README.md b/evals/tool-selection/README.md index cb7b51fb..0c37b94d 100644 --- a/evals/tool-selection/README.md +++ b/evals/tool-selection/README.md @@ -7,17 +7,20 @@ It is not part of `npm test`: it calls the Claude API and costs money. Mapbox AP ## How it works - **Tool surface:** `harness.ts` builds a real `McpServer` the same way the hosted deployment does (`publishJsonSchema2020`, `installTo`, `buildServerInstructions`) and reads `tools/list` and the server instructions back over an in-memory MCP connection. -- **Host simulation:** two host tools sit next to the Mapbox tools. `web_search` and `places_map_display_v0` let us see whether our descriptions crowd out other sources. +- **Host simulation:** two host tools sit next to the Mapbox tools. `web_search` and `places_map_display_v0` let us see whether our descriptions crowd out other sources. A case can add `write_file` through `host_tools`, for agents building a page or file. - **Two modes per case:** - `eager`: every tool description is loaded up front. - `deferred`: Mapbox tools are marked `defer_loading` behind the API's tool search. This approximates hosts like Claude Desktop that only load tools on demand, where the server instructions do most of the routing. -- **Conversation:** up to 5 model calls per case. Tool calls get stub results, and data tools get the server's real render hint appended. +- **Conversation:** up to 5 model calls per case. Tool calls get stub results, and data tools get the server's real render hint appended, with a distinct ref per result. - **Grading:** programmatic, against each case's `expect` in `cases.json`: - `first`: acceptable first tool - `must`: tools that must all be called - `any_of`: at least one of these must be called - `must_not`: none may be called; `*mapbox*` means any Mapbox tool -- **Metrics:** `pass` (all checks hold), `first_ok` and `no_forbidden`. +- **Metrics:** + - `pass` (all checks hold), `first_ok` and `no_forbidden` + - `ref_ok`: every `render_map_tool` call passes refs it was actually given, unchanged, and doesn't hand-build a map when a ref was available + - `no_token_ask`: the agent neither asks the user for a Mapbox token nor writes a placeholder token (`YOUR_MAPBOX_...`, `pk...`) into a file ## Running @@ -44,13 +47,37 @@ open .claude/hillclimb/tool-selection/report.html **First run:** the runner refuses to start until you approve the harness. Review `harness.ts` and `cases.json`, then add `--approve-harness` once. It records a hash of the runner, the lockfile and those two files in `_state.json`. Any later edit to them needs another approval. +## OpenAI models + +The same cases, stubs and grader can run against an OpenAI model through the Responses API, to check that a description change doesn't regress routing for ChatGPT-style hosts. Results go to a separate flow, `.claude/hillclimb/tool-selection-openai`, so comparisons never mix providers. + +```bash +npm run eval:tools:openai -- --variant baseline --reps 3 --model +EVAL_SERVER_ROOT=../mcp-server-pr npm run eval:tools:openai -- --variant v1 --reps 3 --model +npm run eval:tools:report:openai +``` + +- **Key:** the runner reads `OPENAI_API_KEY`. +- **`--model` is required:** the runner's default is a Claude model, so an OpenAI run without `--model` refuses to start. +- **Which model:** + - `chat-latest` is the closest proxy for ChatGPT, but it only supports tools loaded up front (tool search returns a 400). Run it with `EVAL_MODES=eager`. It also reports itself as `chat-latest`, not a dated snapshot, so run the two branches back to back to keep the comparison on one model. + - `gpt-6.1-sol` supports both modes and stands in for API-based hosts. (Checked 2026-10-12.) +- **Deferred mode:** uses OpenAI's server-executed tool search (`{type: "tool_search", execution: "server"}`) with `defer_loading` on the Mapbox functions. +- **Turns:** chained with `previous_response_id`. +- **Schemas:** passed with `strict: false`, so the 2020-12 schemas aren't rewritten for strict mode. +- **Usage:** recorded in the same shape as Claude's. Cached tokens go to `cache_read_input_tokens`. The report has no OpenAI prices, so cost isn't computed. + +**What this does and doesn't measure.** It measures how an OpenAI model routes given our descriptions, server instructions and render hint. It doesn't reproduce ChatGPT itself. ChatGPT's system prompt, how it exposes connector tools, whether it passes MCP server instructions to the model, and its own built-in tools are not public. Treat a regression here as a strong signal, and a pass as "no regression at the model level". + ## Adding cases Add an entry to `cases.json` with a `why` that names the description or instruction it exercises. Give it `stubs` for every tool the prompt might reasonably call, written about the place in the prompt. A canned answer about the wrong place makes Claude retry or switch tools, and that gets scored as a routing failure. Calls without a stub return "No results" and are counted in the `unstubbed` column. A non-zero count means the case needs another stub, not that the model misrouted. -The grader throws if a case names a tool that doesn't exist, so a renamed tool can't turn a `must_not` into a silent pass. The noise floor for a pass rate is roughly `1/sqrt(rows x reps)`. At 34 rows x 3 reps that's about ±10 points overall, and much wider for any single case. +The grader throws if a case names a tool that doesn't exist, so a renamed tool can't turn a `must_not` into a silent pass. A case can list tools in `needs_tools` instead: on a server without one of them, the case fails with "needs X, which this server doesn't have" rather than throwing. The site-building cases use this for `export_map_html_tool` (#275), so on main they show the baseline behavior instead of crashing the run. + +The stubs return canned output, so a stubbed `export_map_html_tool` result is a placeholder page. An agent that notices sometimes writes its own page or offers to rebuild it with a token, which `no_token_ask` flags. Read those transcripts before counting them as regressions. The noise floor for a pass rate is roughly `1/sqrt(rows x reps)`. At 34 rows x 3 reps that's about ±10 points overall, and much wider for any single case. ## Files diff --git a/evals/tool-selection/cases.json b/evals/tool-selection/cases.json index c682387b..fbd757e6 100644 --- a/evals/tool-selection/cases.json +++ b/evals/tool-selection/cases.json @@ -249,5 +249,37 @@ "must_not": ["*mapbox*"] }, "stubs": {} + }, + { + "id": "site-walksheds", + "category": "site-building", + "prompt": "I'm building a website for our coffee shops. Create index.html with an interactive map showing the areas within a 10-minute walk of our two shops: 1 Main St, Brooklyn and 200 Kent Ave, Brooklyn.", + "why": "Field report (ChatGPT/Codex building a site): the agent asked the user for a Mapbox token because render_map_tool only renders in chat. The page should get its map from export_map_html_tool without a token request.", + "host_tools": ["write_file"], + "needs_tools": ["export_map_html_tool"], + "expect": { + "must": ["isochrone_tool", "export_map_html_tool", "write_file"], + "must_not": ["static_map_image_tool", "places_map_display_v0"] + }, + "stubs": { + "search_and_geocode_tool": "1. 1 Main St, Brooklyn, NY 11201 (40.7033, -73.9903)\n2. 200 Kent Ave, Brooklyn, NY 11249 (40.7170, -73.9640)", + "isochrone_tool": "Walking isochrones (10 min): 1 Main St covers about 1.1 sq km (DUMBO, Vinegar Hill, Brooklyn Heights north); 200 Kent Ave covers about 1.2 sq km (Williamsburg waterfront to Bedford Ave)." + } + }, + { + "id": "site-route", + "category": "site-building", + "prompt": "Make a simple web page (index.html) for our team offsite showing the driving route from Denver Union Station to Red Rocks Amphitheatre on an interactive map.", + "why": "Same field report as site-walksheds, with a route instead of areas.", + "host_tools": ["write_file"], + "needs_tools": ["export_map_html_tool"], + "expect": { + "must": ["directions_tool", "export_map_html_tool", "write_file"], + "must_not": ["static_map_image_tool", "places_map_display_v0"] + }, + "stubs": { + "search_and_geocode_tool": "1. Denver Union Station, 1701 Wynkoop St, Denver, CO 80202 (39.7531, -105.0000)\n2. Red Rocks Amphitheatre, 18300 W Alameda Pkwy, Morrison, CO 80465 (39.6654, -105.2057)", + "directions_tool": "Route 1: 25.9 km, 27 min via US-6 W and I-70 W. Route 2: 28.4 km, 31 min via US-285 S and CO-470 N." + } } ] diff --git a/evals/tool-selection/harness.ts b/evals/tool-selection/harness.ts index 41c5b396..1c6460d6 100644 --- a/evals/tool-selection/harness.ts +++ b/evals/tool-selection/harness.ts @@ -15,6 +15,7 @@ */ import Anthropic from '@anthropic-ai/sdk'; +import OpenAI from 'openai'; import { readFileSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -27,7 +28,10 @@ const SERVER_ROOT = resolve( process.env.EVAL_SERVER_ROOT ?? resolve(here, '../..') ); const MAX_MODEL_CALLS = 5; -const FAKE_REF = 'mapbox://temp/map-payload-eval-0001'; +// Each stubbed data result gets its own ref, as on the real server; sharing +// one ref across results confuses models about which dataset it points to. +const REF_PREFIX = 'mapbox://temp/map-payload-eval-'; +const REF_PATTERN = /mapbox:\/\/temp\/map-payload-eval-\d+/g; // Distractors standing in for tools a host provides alongside the connector. // They let the eval see whether our descriptions crowd out other sources, @@ -60,9 +64,48 @@ const HOST_TOOLS: Anthropic.Tool[] = [ } } ]; -const HOST_TOOL_NAMES = new Set(HOST_TOOLS.map((t) => t.name)); +// Host tools only some cases get (via `host_tools` in cases.json), so adding +// one never changes what the other cases see. +const EXTRA_HOST_TOOLS: Record = { + write_file: { + name: 'write_file', + description: + "Write a file in the user's project, e.g. a page of the site you are building. Creates or overwrites the file.", + input_schema: { + type: 'object', + properties: { + path: { type: 'string', description: 'File path, e.g. index.html' }, + content: { type: 'string', description: 'Full file contents' } + }, + required: ['path', 'content'] + } + } +}; +const HOST_TOOL_NAMES = new Set([ + ...HOST_TOOLS.map((t) => t.name), + ...Object.keys(EXTRA_HOST_TOOLS) +]); + +function hostToolsFor(c: EvalCase): Anthropic.Tool[] { + return [ + ...HOST_TOOLS, + ...c.host_tools.map((name) => { + const tool = EXTRA_HOST_TOOLS[name]; + if (!tool) throw new Error(`case ${c.id}: unknown host tool ${name}`); + return tool; + }) + ]; +} export type Mode = 'eager' | 'deferred'; +type Provider = 'anthropic' | 'openai'; + +// Which model family to test. Each provider should write to its own --flow +// directory so baseline/vN comparisons stay within one provider. +const PROVIDER = (process.env.EVAL_PROVIDER ?? 'anthropic') as Provider; +if (PROVIDER !== 'anthropic' && PROVIDER !== 'openai') { + throw new Error(`EVAL_PROVIDER must be anthropic or openai, got ${PROVIDER}`); +} export interface Expectation { /** Acceptable names for the first client tool called. */ @@ -82,6 +125,14 @@ export interface EvalCase { expect: Expectation; /** Canned results keyed by tool name, written to match this prompt. */ stubs: Record; + /** Extra host tools for this case, from EXTRA_HOST_TOOLS. */ + host_tools: string[]; + /** + * Mapbox tools the case needs that a given server may not have yet. The + * case still runs on such a server, to record what the model does instead, + * but it can't pass. + */ + needs_tools: string[]; meta?: Record; } @@ -92,6 +143,8 @@ interface BaseCase { why: string; expect: Expectation; stubs?: Record; + host_tools?: string[]; + needs_tools?: string[]; } interface ServerSurface { @@ -168,10 +221,23 @@ export function loadSurface(): Promise { /** Each base case runs once per mode, so one row = (case, mode). */ export function loadCases(): EvalCase[] { + // The runner defaults --model to a Claude id; fail before spending anything. + if (PROVIDER === 'openai' && !process.argv.includes('--model')) { + throw new Error('EVAL_PROVIDER=openai needs --model '); + } const base = JSON.parse( readFileSync(resolve(here, 'cases.json'), 'utf8') ) as BaseCase[]; - const modes: Mode[] = ['eager', 'deferred']; + // EVAL_MODES=eager skips tool search, for models that don't support it + // (e.g. OpenAI's chat-latest). + const modes = (process.env.EVAL_MODES ?? 'eager,deferred') + .split(',') + .map((m) => m.trim()) as Mode[]; + for (const m of modes) { + if (m !== 'eager' && m !== 'deferred') { + throw new Error(`EVAL_MODES entries must be eager or deferred, got ${m}`); + } + } return base.flatMap((c) => modes.map((mode) => ({ id: `${c.id}__${mode}`, @@ -179,14 +245,24 @@ export function loadCases(): EvalCase[] { tags: [mode, c.category], expect: c.expect, stubs: c.stubs ?? {}, + host_tools: c.host_tools ?? [], + needs_tools: c.needs_tools ?? [], meta: { base_id: c.id, mode, why: c.why } })) ); } -function systemPrompt(instructions: string | undefined, mode: Mode): string { +function systemPrompt( + instructions: string | undefined, + mode: Mode, + hostTools: Anthropic.Tool[] +): string { + const assistant = + PROVIDER === 'anthropic' + ? 'You are Claude, a helpful' + : 'You are a helpful'; const lines = [ - "You are Claude, a helpful assistant in a chat app. The user has connected the Mapbox MCP server; its tools are available to you alongside the app's built-in tools (web_search, places_map_display_v0)." + `${assistant} assistant in a chat app. The user has connected the Mapbox MCP server; its tools are available to you alongside the app's built-in tools (${hostTools.map((t) => t.name).join(', ')}).` ]; if (mode === 'deferred') { lines.push( @@ -210,7 +286,10 @@ function systemPrompt(instructions: string | undefined, mode: Mode): string { const COMMON_STUBS: Record = { render_map_tool: 'Map rendered for the user.', static_map_image_tool: 'Static map image generated (PNG).', - places_map_display_v0: 'Map with pins displayed to the user.' + places_map_display_v0: 'Map with pins displayed to the user.', + write_file: 'File written.', + export_map_html_tool: + 'Map
' }; const UNSTUBBED = 'No results. (This tool returned nothing for this request.)'; @@ -223,12 +302,13 @@ const UNSTUBBED = 'No results. (This tool returned nothing for this request.)'; function stubResult( name: string, c: EvalCase, - surface: ServerSurface + surface: ServerSurface, + nextRef: () => string ): { text: string; stubbed: boolean } { const base = c.stubs[name] ?? COMMON_STUBS[name]; if (base === undefined) return { text: UNSTUBBED, stubbed: false }; return { - text: HINTED_TOOLS.has(name) ? base + surface.renderHint(FAKE_REF) : base, + text: HINTED_TOOLS.has(name) ? base + surface.renderHint(nextRef()) : base, stubbed: true }; } @@ -263,7 +343,9 @@ export interface RunOutput { unstubbed: string[]; } -const client = new Anthropic(); +// Created on first use so a run only needs the key for its own provider. +let anthropic: Anthropic | undefined; +let openai: OpenAI | undefined; function addUsage(total: Record, u: Anthropic.Usage): void { for (const k of [ @@ -276,11 +358,18 @@ function addUsage(total: Record, u: Anthropic.Usage): void { } } -export async function runCase(c: EvalCase, ctx: RunCtx): Promise { +export function runCase(c: EvalCase, ctx: RunCtx): Promise { + return PROVIDER === 'openai' ? runOpenAI(c, ctx) : runClaude(c, ctx); +} + +async function runClaude(c: EvalCase, ctx: RunCtx): Promise { + anthropic ??= new Anthropic(); + const client = anthropic; const surface = await loadSurface(); const mode = (c.meta?.mode as Mode) ?? 'eager'; const model = ctx.model ?? 'claude-opus-5-5'; - const system = systemPrompt(surface.instructions, mode); + const hostTools = hostToolsFor(c); + const system = systemPrompt(surface.instructions, mode, hostTools); const tools: Anthropic.ToolUnion[] = mode === 'deferred' @@ -289,10 +378,10 @@ export async function runCase(c: EvalCase, ctx: RunCtx): Promise { type: 'tool_search_tool_bm25_20251119', name: 'tool_search_tool_bm25' }, - ...HOST_TOOLS, + ...hostTools, ...surface.tools.map((t) => ({ ...t, defer_loading: true })) ] - : [...HOST_TOOLS, ...surface.tools]; + : [...hostTools, ...surface.tools]; const messages: Anthropic.MessageParam[] = [ { role: 'user', content: c.prompt } @@ -305,6 +394,8 @@ export async function runCase(c: EvalCase, ctx: RunCtx): Promise { const toolCalls: string[] = []; const toolSearches: string[] = []; const unstubbed: string[] = []; + let refCount = 0; + const nextRef = () => `${REF_PREFIX}${String(++refCount).padStart(4, '0')}`; let servedModel = model; let stopReason: string | null = null; @@ -345,7 +436,12 @@ export async function runCase(c: EvalCase, ctx: RunCtx): Promise { transcript.push({ role: 'tool_result', content: `found: ${found}` }); } else if (block.type === 'tool_use') { toolCalls.push(block.name); - const { text: result, stubbed } = stubResult(block.name, c, surface); + const { text: result, stubbed } = stubResult( + block.name, + c, + surface, + nextRef + ); if (!stubbed) unstubbed.push(block.name); transcript.push({ role: 'tool_call', @@ -386,23 +482,186 @@ export async function runCase(c: EvalCase, ctx: RunCtx): Promise { }; } +function addOpenAIUsage( + total: Record, + u: OpenAI.Responses.ResponseUsage | undefined +): void { + if (!u) return; + const cached = u.input_tokens_details?.cached_tokens ?? 0; + const add = (k: string, n: number) => (total[k] = (total[k] ?? 0) + n); + // Same shape as Claude usage: input_tokens excludes cache reads. + add('input_tokens', u.input_tokens - cached); + add('cache_read_input_tokens', cached); + add( + 'cache_creation_input_tokens', + u.input_tokens_details?.cache_write_tokens ?? 0 + ); + add('output_tokens', u.output_tokens); +} + +/** + * Same conversation as runClaude, on OpenAI's Responses API. Deferred mode + * uses OpenAI's server-executed tool search with defer_loading, the closest + * equivalent of Anthropic's. Turns are chained with previous_response_id, so + * reasoning items carry over without being resent. + */ +async function runOpenAI(c: EvalCase, ctx: RunCtx): Promise { + openai ??= new OpenAI(); + const client = openai; + const surface = await loadSurface(); + const mode = (c.meta?.mode as Mode) ?? 'eager'; + if (!ctx.model) throw new Error('missing --model'); + const model = ctx.model; + const hostTools = hostToolsFor(c); + const system = systemPrompt(surface.instructions, mode, hostTools); + + const asFunction = ( + t: Anthropic.Tool, + deferred: boolean + ): OpenAI.Responses.FunctionTool => { + // Drop the $schema marker; the rest of the 2020-12 schema is passed as-is. + const parameters: Record = { ...t.input_schema }; + delete parameters.$schema; + return { + type: 'function', + name: t.name, + description: t.description, + parameters, + strict: false, + ...(deferred ? { defer_loading: true } : {}) + }; + }; + const tools: OpenAI.Responses.Tool[] = [ + ...(mode === 'deferred' + ? [{ type: 'tool_search', execution: 'server' } as const] + : []), + ...hostTools.map((t) => asFunction(t, false)), + ...surface.tools.map((t) => asFunction(t, mode === 'deferred')) + ]; + + const transcript: Array> = [ + { role: 'system', content: system }, + { role: 'user', content: c.prompt } + ]; + const usage: Record = {}; + const toolCalls: string[] = []; + const toolSearches: string[] = []; + const unstubbed: string[] = []; + let refCount = 0; + const nextRef = () => `${REF_PREFIX}${String(++refCount).padStart(4, '0')}`; + let servedModel = model; + let stopReason: string | null = null; + let status = 'ok'; + let input: OpenAI.Responses.ResponseInput = [ + { role: 'user', content: c.prompt } + ]; + let previous: string | undefined; + + for (let call = 0; call < MAX_MODEL_CALLS; call++) { + const res = await client.responses.create({ + model, + instructions: system, + input, + tools, + previous_response_id: previous, + max_output_tokens: 16000, + prompt_cache_key: `mapbox-tool-selection-${mode}` + }); + servedModel = res.model; + addOpenAIUsage(usage, res.usage); + stopReason = res.incomplete_details?.reason ?? res.status ?? null; + if (res.incomplete_details?.reason === 'max_output_tokens') { + status = 'truncated'; + } else if (res.incomplete_details?.reason === 'content_filter') { + status = 'refusal'; + } + + const outputs: OpenAI.Responses.ResponseInputItem.FunctionCallOutput[] = []; + for (const item of res.output) { + if (item.type === 'message') { + for (const part of item.content) { + if (part.type === 'refusal') status = 'refusal'; + const text = part.type === 'output_text' ? part.text : part.refusal; + if (text.trim()) + transcript.push({ role: 'assistant', content: text }); + } + } else if (item.type === 'tool_search_call') { + const query = JSON.stringify(item.arguments); + toolSearches.push(query); + transcript.push({ + role: 'tool_call', + name: 'tool_search', + content: query + }); + } else if (item.type === 'tool_search_output') { + const found = item.tools + .map((t) => ('name' in t && t.name ? t.name : t.type)) + .join(', '); + transcript.push({ role: 'tool_result', content: `found: ${found}` }); + } else if (item.type === 'function_call') { + toolCalls.push(item.name); + const { text: result, stubbed } = stubResult( + item.name, + c, + surface, + nextRef + ); + if (!stubbed) unstubbed.push(item.name); + let args: string = item.arguments; + try { + args = JSON.stringify(JSON.parse(item.arguments), null, 2); + } catch { + // keep the raw string; the ref check reports it as unparseable + } + transcript.push({ role: 'tool_call', name: item.name, content: args }); + transcript.push({ role: 'tool_result', content: result }); + outputs.push({ + type: 'function_call_output', + call_id: item.call_id, + output: result + }); + } + } + + if (status !== 'ok' || outputs.length === 0) break; + previous = res.id; + input = outputs; + } + + return { + output: toolCalls.join(' > '), + transcript, + model: servedModel, + usage, + stop_reason: stopReason, + status, + tool_calls: toolCalls, + tool_searches: toolSearches, + unstubbed + }; +} + /** * Did every render_map_tool call pass the map ref unchanged? Data tools return * a ref plus a hint telling Claude to pass it as-is; before the hint existed, - * Claude invented refs. Once a data tool has returned a ref, a render call must - * include it, and no render call may carry any other mapbox:// ref. Returns - * null when render_map_tool wasn't called. + * Claude invented refs. Once a tool result has handed out a ref, a render call + * must pass at least one ref, and every ref it passes must be one a tool + * result handed out. Refs are read from the transcript, so old rows can be + * re-scored. Returns null when render_map_tool wasn't called. */ function checkRefs( transcript: RunOutput['transcript'] ): { ok: boolean; why: string } | null { - let refAvailable = false; + const issued = new Set(); let renders = 0; for (const turn of transcript) { - if (turn.role !== 'tool_call') continue; - const name = String(turn.name); - if (HINTED_TOOLS.has(name)) refAvailable = true; - if (name !== 'render_map_tool') continue; + if (turn.role === 'tool_result') { + for (const ref of String(turn.content).match(REF_PATTERN) ?? []) { + issued.add(ref); + } + continue; + } + if (turn.role !== 'tool_call' || turn.name !== 'render_map_tool') continue; renders++; let input: { payload_refs?: unknown }; try { @@ -413,11 +672,11 @@ function checkRefs( const refs = Array.isArray(input.payload_refs) ? input.payload_refs.map(String) : []; - const invented = refs.filter((r) => r !== FAKE_REF); + const invented = refs.filter((r) => !issued.has(r)); if (invented.length) { return { ok: false, why: `invented ref ${invented[0]}` }; } - if (refAvailable && !refs.includes(FAKE_REF)) { + if (issued.size && refs.length === 0) { return { ok: false, why: 'ref available but not passed (hand-built map)' @@ -427,6 +686,32 @@ function checkRefs( return renders ? { ok: true, why: 'ref passed unchanged' } : null; } +// The symptom from the field: an agent building a page asks the user for a +// Mapbox token, or writes one in as a placeholder. Heuristic, so the +// explanation quotes what matched. +const TOKEN_ASK = + /\b(provide|enter|paste|share|send|add|need)\b[^.?!\n]{0,40}\b(your|a|an)\b[^.?!\n]{0,20}\b(mapbox\s+)?(public\s+|access\s+)?token\b/i; +const TOKEN_PLACEHOLDER = + /YOUR_MAPBOX[A-Z_]*|<\s*your[^>]*token[^>]*>|pk\.\.\.|ACCESS_TOKEN_HERE|REPLACE_WITH[A-Z_]*TOKEN|accessToken\s*=\s*(['"`])\1/i; + +function checkTokenAsk( + transcript: RunOutput['transcript'] +): { ok: boolean; why: string } | null { + let relevant = false; + for (const turn of transcript) { + const text = String(turn.content); + if (turn.role === 'assistant') { + const m = text.match(TOKEN_ASK); + if (m) return { ok: false, why: `asked for a token: "${m[0]}"` }; + } else if (turn.role === 'tool_call' && turn.name === 'write_file') { + relevant = true; + const m = text.match(TOKEN_PLACEHOLDER); + if (m) return { ok: false, why: `placeholder token in file: ${m[0]}` }; + } + } + return relevant ? { ok: true, why: 'no token request or placeholder' } : null; +} + export async function gradeCase( c: EvalCase, run: Pick @@ -445,12 +730,20 @@ export async function gradeCase( ...expand(c.expect.must_not) ]) { // A renamed or removed tool would otherwise turn a must_not into a silent pass. - if (!surface.names.has(n) && !HOST_TOOL_NAMES.has(n)) { + if ( + !surface.names.has(n) && + !HOST_TOOL_NAMES.has(n) && + !c.needs_tools.includes(n) + ) { throw new Error(`case ${c.id} references unknown tool ${n}`); } } const problems: string[] = []; + const absent = c.needs_tools.filter((n) => !surface.names.has(n)); + if (absent.length) { + problems.push(`needs ${absent.join(', ')}, which this server doesn't have`); + } const first = run.tool_calls[0]; const firstOk = !c.expect.first || @@ -475,18 +768,21 @@ export async function gradeCase( // Rows without a render call score 1 on ref_ok so the metric's failures // are only real ref mistakes. const refs = checkRefs(run.transcript); + const token = checkTokenAsk(run.transcript); return { grade: { pass, first_ok: firstOk ? 1 : 0, no_forbidden: forbidden.length ? 0 : 1, - ref_ok: refs && !refs.ok ? 0 : 1 + ref_ok: refs && !refs.ok ? 0 : 1, + no_token_ask: token && !token.ok ? 0 : 1 }, explanation: { pass: pass ? `ok: ${trajectory}` : `${problems.join('; ')} | ${trajectory}`, - ref_ok: refs ? refs.why : 'no render_map_tool call' + ref_ok: refs ? refs.why : 'no render_map_tool call', + no_token_ask: token ? token.why : 'no page written' } }; } diff --git a/evals/tool-selection/rescore.ts b/evals/tool-selection/rescore.ts index 180abd9f..0f800d29 100644 --- a/evals/tool-selection/rescore.ts +++ b/evals/tool-selection/rescore.ts @@ -7,13 +7,14 @@ * scored the same way as new ones. * * npx tsx evals/tool-selection/rescore.ts baseline v1 + * EVAL_FLOW=.claude/hillclimb/tool-selection-openai npx tsx evals/tool-selection/rescore.ts baseline */ import { readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { gradeCase, loadCases } from './harness.ts'; -const FLOW = '.claude/hillclimb/tool-selection'; +const FLOW = process.env.EVAL_FLOW ?? '.claude/hillclimb/tool-selection'; const cases = new Map(loadCases().map((c) => [c.id, c])); diff --git a/package-lock.json b/package-lock.json index 78642a7d..146d1833 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,6 +44,7 @@ "globals": "^16.3.0", "husky": "^9.0.0", "lint-staged": "^16.1.0", + "openai": "^7.32.0", "plop": "^4.0.1", "prettier": "^3.0.0", "tshy": "^4.1.1", @@ -10819,6 +10820,44 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/openai": { + "version": "7.32.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-7.32.0.tgz", + "integrity": "sha512-avqxu/jl1gsVBCBgFNPr9sTh+wZppQphuhhFy1zEvAJxEaJ4t0ery6hZdOT9kws/EcNTARoQcZs2IT3bliE1SA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=22.0.0" + }, + "peerDependencies": { + "@aws-sdk/credential-provider-node": ">=3.972.0 <4", + "@smithy/hash-node": ">=4.3.0 <5", + "@smithy/signature-v4": ">=5.4.0 <6", + "undici": ">=5 <9", + "ws": "^8.21.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-provider-node": { + "optional": true + }, + "@smithy/hash-node": { + "optional": true + }, + "@smithy/signature-v4": { + "optional": true + }, + "undici": { + "optional": true + }, + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", diff --git a/package.json b/package.json index faa1678c..7338ded9 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,9 @@ "tracing:jaeger:stop": "docker stop jaeger", "tracing:verify": "node -e \"console.log('🔍 Verifying tracing setup with Jaeger...\\n1. Copy .env.example to .env and add your MAPBOX_ACCESS_TOKEN\\n2. Start Jaeger: npm run tracing:jaeger:start\\n3. Run server: npm run inspect:build\\n4. Check traces at: http://localhost:16686')\"", "eval:tools": "tsx evals/tool-selection/run-eval.mjs", - "eval:tools:report": "node evals/tool-selection/build-report-lite.mjs .claude/hillclimb/tool-selection" + "eval:tools:report": "node evals/tool-selection/build-report-lite.mjs .claude/hillclimb/tool-selection", + "eval:tools:openai": "EVAL_PROVIDER=openai tsx evals/tool-selection/run-eval.mjs --flow .claude/hillclimb/tool-selection-openai", + "eval:tools:report:openai": "node evals/tool-selection/build-report-lite.mjs .claude/hillclimb/tool-selection-openai" }, "lint-staged": { "*.{js,jsx,ts,tsx}": "eslint --fix", @@ -53,6 +55,7 @@ "globals": "^16.3.0", "husky": "^9.0.0", "lint-staged": "^16.1.0", + "openai": "^7.32.0", "plop": "^4.0.1", "prettier": "^3.0.0", "tshy": "^4.1.1",