From 6f33ed6030592ae210e4330ab98612bc4a0c68da Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 01:49:05 -0700 Subject: [PATCH 01/16] feat!: run evals with one runEvaluation call --- README.md | 34 ++-- e2e/evaluate.e2e.test.ts | 298 ++++++++++++++++++----------- e2e/jev.e2e.test.ts | 55 +++--- e2e/workflow.e2e.test.ts | 25 +-- examples/local-jev.ts | 11 +- examples/local-loop.ts | 11 +- examples/ports/battle.ts | 11 +- examples/ports/factuality.ts | 11 +- examples/ports/faithfulness.ts | 11 +- examples/ports/keyword-coverage.ts | 11 +- examples/ports/mtbench-score.ts | 11 +- package.json | 2 +- src/calibrate.ts | 2 +- src/{evaluate.ts => evaluation.ts} | 129 ++++++------- src/index.ts | 9 +- src/reports.ts | 42 ++-- src/runs.ts | 9 +- src/types.ts | 6 +- 18 files changed, 401 insertions(+), 287 deletions(-) rename src/{evaluate.ts => evaluation.ts} (68%) diff --git a/README.md b/README.md index 2396211..392c03d 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,13 @@ Bun only (>=1.2.0). ## Example ```ts -import { evaluate, mentions, under } from "@corbits/evals"; +import { mentions, runEvaluation, under } from "@corbits/evals"; -const report = await evaluate("refund-agent", { - run: refundAgent, // an @intx/agent Agent, workflow(), or (input) => output - cases: [{ name: "duplicate charge", input: "I was charged twice" }], - checks: { +const report = await runEvaluation({ + name: "refund-agent", + task: refundAgent, // an @intx/agent Agent, workflow(), or (input) => output + data: [{ name: "duplicate charge", input: "I was charged twice" }], + scorers: { refunds: mentions(["refund"]), readyToSend: "Is this reply ready to send to the customer?", fast: under({ ms: 5_000 }), @@ -118,12 +119,13 @@ compared runs and defaults to the input. ## Passing ```ts -await evaluate("judge", { - run, - cases, // a case can say { must: ["tests"], skip: ["docs"] } - checks, +await runEvaluation({ + name: "judge", + task: run, + data, // a case can say { must: ["tests"], skip: ["docs"] } + scorers, compare: "evals/judge.last.json", // read: measure drops against it - save: "evals/judge.last.json", // write: store this run, pass or fail + save: "evals/judge.last.json", // write: store this experiment, pass or fail allow: { tests: { miss: 1, drop: 0 }, docs: { miss: 5, drop: 2 }, @@ -141,16 +143,16 @@ Per check, `allow` sets (default 0): A case's `must` checks must pass whatever `allow` says; its `skip` checks are not run or counted. -`evaluate` prints each check's passes, misses and drops, and rejects with -`EvalFailure` naming the limit each failing check broke and its failed +`runEvaluation` prints each scorer's passes, misses and drops, and rejects with +`EvalFailure` naming the limit each failing scorer broke and its failed cases; `.report` holds every result. -A stored run is each case × check pass and value — plus a Jev check's +An experiment is each case × scorer pass and value — plus a Jev scorer's probability distribution and confidence — and `about`, no times. `compare` only reads; a missing file skips drops. `save` only writes, on -every run. One path for both compares with the last run, then replaces -it. `compare` also takes a report from an earlier `evaluate` of the same -eval, with no file: `evaluate(name, { ...options, compare: before })`. +every run. One path for both compares with the last experiment, then replaces +it. `compare` also takes an experiment from an earlier `runEvaluation` of the same +evaluation, with no file: `runEvaluation({ ...spec, compare: before.experiment })`. Flips print as `fixed` / `broke`, new and gone cases as `added` / `removed`. `quiet: true` skips the summary. diff --git a/e2e/evaluate.e2e.test.ts b/e2e/evaluate.e2e.test.ts index e8e8ae1..3a58acb 100644 --- a/e2e/evaluate.e2e.test.ts +++ b/e2e/evaluate.e2e.test.ts @@ -7,7 +7,7 @@ import { type } from "arktype"; import { avoids, EvalFailure, - evaluate, + runEvaluation, fitsSchema, matches, mentions, @@ -30,14 +30,15 @@ async function failure(pending: Promise): Promise { if (cause instanceof EvalFailure) return cause; throw cause; } - throw new Error("expected evaluate to fail"); + throw new Error("expected runEvaluation to fail"); } describe("runs", () => { test("a function run with the default check", async () => { - const report = await evaluate("upper", { - run: upper, - cases: [ + const report = await runEvaluation({ + name: "upper", + task: upper, + data: [ { input: "hi", expect: "HI" }, { name: "yo", input: "yo", expect: "YO" }, ], @@ -56,15 +57,20 @@ describe("runs", () => { return { type: "reply" as const, reply: `re: ${content}` }; }, }; - const report = await evaluate("agent", { - run: agent, - cases: [{ input: "refund", expect: "re: refund" }], + const report = await runEvaluation({ + name: "agent", + task: agent, + data: [{ input: "refund", expect: "re: refund" }], }); expect(report.ok).toBe(true); expect(seen[0]).toBeInstanceOf(AbortSignal); const suspended = { send: async () => ({ type: "suspended" as const }) }; const err = await failure( - evaluate("agent-suspended", { run: suspended, cases: ["x"].map((input) => ({ input })) }), + runEvaluation({ + name: "agent-suspended", + task: suspended, + data: ["x"].map((input) => ({ input })), + }), ); expect(err.report.cases[0]?.error).toContain('did not reply (got "suspended")'); }); @@ -72,15 +78,16 @@ describe("runs", () => { test("a raw workflow definition is refused with the fix", async () => { const definition = { id: "w", triggers: [], steps: {}, stepOrder: [] }; await expect( - evaluate("raw", { run: definition as never, cases: [{ input: "x" }] }), + runEvaluation({ name: "raw", task: definition as never, data: [{ input: "x" }] }), ).rejects.toThrow("@corbits/evals/workflow"); }); test("objects stay raw and match deeply", async () => { - const report = await evaluate("objects", { - run: (input: { amount: number }) => ({ decision: "refund", amount: input.amount }), - cases: [{ input: { amount: 5 }, expect: { decision: "refund", amount: 5 } }], - checks: { + const report = await runEvaluation({ + name: "objects", + task: (input: { amount: number }) => ({ decision: "refund", amount: input.amount }), + data: [{ input: { amount: 5 }, expect: { decision: "refund", amount: 5 } }], + scorers: { whole: matches(), decision: matches("decision"), amount: matches((v: { amount: number }) => v.amount), @@ -97,11 +104,11 @@ describe("runs", () => { path, '{"name":"a","input":"hi","expect":"HI"}\n\n{"input":"yo","expect":"YO"}\n', ); - const report = await evaluate("jsonl", { run: upper, cases: path }); + const report = await runEvaluation({ name: "jsonl", task: upper, data: path }); expect(report.cases.map((c) => c.name)).toEqual(["a", "yo"]); expect(report.passed).toBe(2); await Bun.write(path, '{"input":"x"}\n{"name":"b"}\n'); - await expect(evaluate("jsonl-bad", { run: upper, cases: path })).rejects.toThrow( + await expect(runEvaluation({ name: "jsonl-bad", task: upper, data: path })).rejects.toThrow( `${path}:2: missing "input"`, ); }); @@ -109,10 +116,11 @@ describe("runs", () => { describe("checks", () => { test("code checks return booleans or 0..1 numbers", async () => { - const report = await evaluate("code", { - run: upper, - cases: [{ input: "hi" }], - checks: { short: (r) => r.output.length < 5, half: () => 0.5, slow: async () => true }, + const report = await runEvaluation({ + name: "code", + task: upper, + data: [{ input: "hi" }], + scorers: { short: (r) => r.output.length < 5, half: () => 0.5, slow: async () => true }, allow: { half: 1 }, }); const checks = report.cases[0]?.checks; @@ -131,10 +139,11 @@ describe("checks", () => { ], }), }; - const report = await evaluate("helpers", { - run: steps, - cases: [{ input: "order 42" }], - checks: { + const report = await runEvaluation({ + name: "helpers", + task: steps, + data: [{ input: "order 42" }], + scorers: { mentionsAll: mentions(["refund", "ORDER 42"]), mentionsSome: mentions(["refund", "credit"]), mentionsHalf: mentions(["refund", "credit"], { atLeast: 0.5 }), @@ -178,16 +187,19 @@ describe("checks", () => { }); test("matches fails a case without expect", async () => { - const err = await failure(evaluate("no-expect", { run: upper, cases: [{ input: "hi" }] })); + const err = await failure( + runEvaluation({ name: "no-expect", task: upper, data: [{ input: "hi" }] }), + ); expect(err.report.cases[0]?.checks["matches"]?.explanation).toBe("case has no expect"); }); test("a throwing, hanging or malformed check fails itself closed, not its siblings", async () => { const err = await failure( - evaluate("bad-checks", { - run: upper, - cases: [{ input: "hi", expect: "HI" }], - checks: { + runEvaluation({ + name: "bad-checks", + task: upper, + data: [{ input: "hi", expect: "HI" }], + scorers: { exact: matches(), boom: () => { throw new Error("kaput"); @@ -209,8 +221,9 @@ describe("checks", () => { test("a run error or timeout fails every check and aborts the run", async () => { let aborted = false; const err = await failure( - evaluate("bad-runs", { - run: (input: string, signal) => { + runEvaluation({ + name: "bad-runs", + task: (input: string, signal) => { if (input === "boom") throw new Error("kaput"); return new Promise((_, reject) => { signal.addEventListener("abort", () => { @@ -219,8 +232,8 @@ describe("checks", () => { }); }); }, - cases: [{ input: "boom" }, { input: "hang" }], - checks: { fine: () => true }, + data: [{ input: "boom" }, { input: "hang" }], + scorers: { fine: () => true }, timeout: 30, }), ); @@ -236,10 +249,11 @@ describe("gates", () => { test("repeats fail a check when any attempt fails and report the deciding one", async () => { let calls = 0; const err = await failure( - evaluate("flaky", { - run: upper, - cases: [{ input: "hi" }], - checks: { + runEvaluation({ + name: "flaky", + task: upper, + data: [{ input: "hi" }], + scorers: { flaky: () => { calls += 1; return calls === 1 @@ -259,25 +273,31 @@ describe("gates", () => { }); test("bad options are rejected up front", async () => { - const base = { run: upper, cases: [{ input: "a", expect: "A" }] }; - await expect(evaluate("o", { ...base, concurrency: Number.NaN })).rejects.toThrow( + const base = { task: upper, data: [{ input: "a", expect: "A" }] }; + await expect(runEvaluation({ name: "o", ...base, concurrency: Number.NaN })).rejects.toThrow( "concurrency", ); - await expect(evaluate("o", { ...base, repeats: 0 })).rejects.toThrow("repeats"); - await expect(evaluate("o", { ...base, timeout: "soon" as never })).rejects.toThrow("timeout"); - await expect(evaluate("o", { ...base, checks: {} })).rejects.toThrow("no checks"); - await expect(evaluate("o", { ...base, cases: [] })).rejects.toThrow("no cases"); + await expect(runEvaluation({ name: "o", ...base, repeats: 0 })).rejects.toThrow("repeats"); + await expect(runEvaluation({ name: "o", ...base, timeout: "soon" as never })).rejects.toThrow( + "timeout", + ); + await expect(runEvaluation({ name: "o", ...base, scorers: {} })).rejects.toThrow("no scorers"); + await expect(runEvaluation({ name: "o", ...base, data: [] })).rejects.toThrow("no data"); await expect( - evaluate("o", { ...base, cases: [...base.cases, { input: "a" }] }), + runEvaluation({ name: "o", ...base, data: [...base.data, { input: "a" }] }), ).rejects.toThrow('duplicate case "a"'); }); test("markdown escapes cells", async () => { const err = await failure( - evaluate("md", { run: upper, cases: [{ name: "a|b\nc", input: "", expect: "no" }] }), + runEvaluation({ + name: "md", + task: upper, + data: [{ name: "a|b\nc", input: "", expect: "no" }], + }), ); const md = err.report.markdown(); - expect(md).toContain("# eval: md"); + expect(md).toContain("# evaluation: md"); expect(md).toContain("| matches | 0/1 | 1 (0 allowed) | FAIL |"); expect(md).toContain("| a\\|b c | FAIL"); expect(md).not.toContain(""); @@ -293,19 +313,33 @@ describe("passing", () => { const checks = { exact: matches(), short: (r: { output: string }) => r.output.length === 1 }; test("each check may miss as many cases as allowed", async () => { - const strict = await failure(evaluate("strict", { run: upper, cases, checks })); + const strict = await failure( + runEvaluation({ name: "strict", task: upper, data: cases, scorers: checks }), + ); expect(strict.message).toContain("exact: 1 missed, 0 allowed"); expect(strict.message).toContain('c / exact: expected "nope", got "C"'); expect(strict.report.checks["exact"]).toMatchObject({ passed: 2, counted: 3, missed: 1 }); expect(strict.report.checks["short"]?.ok).toBe(true); - const report = await evaluate("lenient", { run: upper, cases, checks, allow: { exact: 1 } }); + const report = await runEvaluation({ + name: "lenient", + task: upper, + data: cases, + scorers: checks, + allow: { exact: 1 }, + }); expect(report.ok).toBe(true); expect(report.checks["exact"]?.allow).toEqual({ miss: 1, drop: 0 }); const long = (input: string) => input.toUpperCase().repeat(2); const over = await failure( - evaluate("over", { run: long, cases, checks, allow: { exact: { miss: 3 }, short: 2 } }), + runEvaluation({ + name: "over", + task: long, + data: cases, + scorers: checks, + allow: { exact: { miss: 3 }, short: 2 }, + }), ); expect(over.message).toContain("short: 3 missed, 2 allowed"); expect(over.message).not.toContain("exact:"); @@ -315,10 +349,11 @@ describe("passing", () => { test("must fails a check despite its allowance; skip leaves a case out", async () => { const must = await failure( - evaluate("must", { - run: upper, - cases: cases.map((c) => (c.name === "c" ? { ...c, must: ["exact"] } : c)), - checks, + runEvaluation({ + name: "must", + task: upper, + data: cases.map((c) => (c.name === "c" ? { ...c, must: ["exact"] } : c)), + scorers: checks, allow: { exact: 1 }, }), ); @@ -326,10 +361,11 @@ describe("passing", () => { expect(must.report.checks["exact"]).toMatchObject({ missed: 1, mustFailed: ["c"], ok: false }); const seen: string[] = []; - const report = await evaluate("skip", { - run: upper, - cases: cases.map((c) => (c.name === "c" ? { ...c, skip: ["exact"] } : c)), - checks: { + const report = await runEvaluation({ + name: "skip", + task: upper, + data: cases.map((c) => (c.name === "c" ? { ...c, skip: ["exact"] } : c)), + scorers: { exact: (r) => { seen.push(r.input); return r.output === r.expect; @@ -348,38 +384,44 @@ describe("passing", () => { path, '{"name":"a","input":"a","expect":"A","must":["exact"]}\n{"name":"c","input":"c","expect":"nope","skip":["exact"]}\n', ); - const report = await evaluate("jsonl-must", { run: upper, cases: path, checks }); + const report = await runEvaluation({ + name: "jsonl-must", + task: upper, + data: path, + scorers: checks, + }); expect(report.checks["exact"]).toMatchObject({ passed: 1, counted: 1 }); await Bun.write(path, '{"input":"a","skip":"exact"}\n'); - await expect(evaluate("jsonl-must", { run: upper, cases: path, checks })).rejects.toThrow( - `${path}:1: "skip" must be an array of check names`, - ); + await expect( + runEvaluation({ name: "jsonl-must", task: upper, data: path, scorers: checks }), + ).rejects.toThrow(`${path}:1: "skip" must be an array of check names`); }); test("bad allow, must and skip names are rejected", async () => { - const base = { run: upper, cases, checks }; - await expect(evaluate("o", { ...base, allow: { exakt: 1 } })).rejects.toThrow( - 'allow names unknown check "exakt"', + const base = { task: upper, data: cases, scorers: checks }; + await expect(runEvaluation({ name: "o", ...base, allow: { exakt: 1 } })).rejects.toThrow( + 'allow names unknown scorer "exakt"', ); - await expect(evaluate("o", { ...base, allow: { exact: -1 } })).rejects.toThrow( + await expect(runEvaluation({ name: "o", ...base, allow: { exact: -1 } })).rejects.toThrow( "allow.exact must be a whole number >= 0", ); - await expect(evaluate("o", { ...base, allow: { exact: { drop: 0.5 } } })).rejects.toThrow( - "allow.exact.drop must be a whole number >= 0", - ); await expect( - evaluate("o", { ...base, allow: { exact: { misses: 1 } as never } }), + runEvaluation({ name: "o", ...base, allow: { exact: { drop: 0.5 } } }), + ).rejects.toThrow("allow.exact.drop must be a whole number >= 0"); + await expect( + runEvaluation({ name: "o", ...base, allow: { exact: { misses: 1 } as never } }), ).rejects.toThrow('allow.exact has unknown "misses"'); await expect( - evaluate("o", { ...base, cases: [{ name: "a", input: "a", must: ["nope"] }] }), - ).rejects.toThrow('case "a" must names unknown check "nope"'); + runEvaluation({ name: "o", ...base, data: [{ name: "a", input: "a", must: ["nope"] }] }), + ).rejects.toThrow('case "a" must names unknown scorer "nope"'); await expect( - evaluate("o", { ...base, cases: [{ name: "a", input: "a", skip: ["nope"] }] }), - ).rejects.toThrow('case "a" skip names unknown check "nope"'); + runEvaluation({ name: "o", ...base, data: [{ name: "a", input: "a", skip: ["nope"] }] }), + ).rejects.toThrow('case "a" skip names unknown scorer "nope"'); await expect( - evaluate("o", { + runEvaluation({ + name: "o", ...base, - cases: [{ name: "a", input: "a", must: ["exact"], skip: ["exact"] }], + data: [{ name: "a", input: "a", must: ["exact"], skip: ["exact"] }], }), ).rejects.toThrow('case "a" both must and skip "exact"'); }); @@ -408,15 +450,22 @@ describe("compare and save", () => { } test("without either nothing is read or written", async () => { - await touchesNoFile(() => evaluate("upper", { run: upper, cases, allow })); + await touchesNoFile(() => runEvaluation({ name: "upper", task: upper, data: cases, allow })); }); test("save writes the run, passing or failing", async () => { const path = join(await tempDir(), "upper.json"); - await evaluate("upper", { run: upper, cases, allow, save: path, about: { model: "m-1" } }); + await runEvaluation({ + name: "upper", + task: upper, + data: cases, + allow, + save: path, + about: { model: "m-1" }, + }); expect(JSON.parse(await Bun.file(path).text())).toEqual({ - format: 1, - eval: "upper", + format: 2, + name: "upper", about: { model: "m-1" }, cases: { a: { matches: { pass: true, value: 1 } }, @@ -425,7 +474,7 @@ describe("compare and save", () => { }, }); - await failure(evaluate("upper", { run: upper, cases, save: path })); + await failure(runEvaluation({ name: "upper", task: upper, data: cases, save: path })); expect(JSON.parse(await Bun.file(path).text()).about).toBeUndefined(); }); @@ -433,7 +482,13 @@ describe("compare and save", () => { const path = join(await tempDir(), "missing.json"); const log = spyOn(console, "log").mockImplementation(() => {}); try { - const report = await evaluate("upper", { run: swap, cases, allow, compare: path }); + const report = await runEvaluation({ + name: "upper", + task: swap, + data: cases, + allow, + compare: path, + }); expect(report.checks["matches"]?.drop).toBeUndefined(); expect(log).toHaveBeenCalledWith(` no compare run at ${path}`); } finally { @@ -441,19 +496,26 @@ describe("compare and save", () => { } expect(await Bun.file(path).exists()).toBe(false); - await evaluate("upper", { run: upper, cases, allow, save: path }); + await runEvaluation({ name: "upper", task: upper, data: cases, allow, save: path }); const stored = await Bun.file(path).text(); - await evaluate("upper", { run: upper, cases, allow, compare: path }); - await failure(evaluate("upper", { run: swap, cases, allow, compare: path })); + await runEvaluation({ name: "upper", task: upper, data: cases, allow, compare: path }); + await failure(runEvaluation({ name: "upper", task: swap, data: cases, allow, compare: path })); expect(await Bun.file(path).text()).toBe(stored); }); test("one path for both compares, then overwrites", async () => { const path = join(await tempDir(), "upper.last.json"); - await evaluate("upper", { run: upper, cases, allow, compare: path, save: path }); + await runEvaluation({ + name: "upper", + task: upper, + data: cases, + allow, + compare: path, + save: path, + }); const err = await failure( - evaluate("upper", { run: swap, cases, allow, compare: path, save: path }), + runEvaluation({ name: "upper", task: swap, data: cases, allow, compare: path, save: path }), ); expect(err.report.changes).toEqual([ { case: "b", check: "matches", change: "broke" }, @@ -464,32 +526,54 @@ describe("compare and save", () => { expect(err.message).toContain("b / matches"); expect(JSON.parse(await Bun.file(path).text()).cases.b.matches.pass).toBe(false); - const again = await evaluate("upper", { run: swap, cases, allow, compare: path, save: path }); + const again = await runEvaluation({ + name: "upper", + task: swap, + data: cases, + allow, + compare: path, + save: path, + }); expect(again.checks["matches"]?.drop).toBe(0); }); test("an earlier report gates drops with no file", async () => { - const before = await evaluate("upper", { run: upper, cases, allow }); + const before = await runEvaluation({ name: "upper", task: upper, data: cases, allow }); await touchesNoFile(async () => { - const err = await failure(evaluate("upper", { run: swap, cases, allow, compare: before })); + const err = await failure( + runEvaluation({ + name: "upper", + task: swap, + data: cases, + allow, + compare: before.experiment, + }), + ); expect(err.report.checks["matches"]).toMatchObject({ drop: 1, ok: false }); - const within = await evaluate("upper", { - run: swap, - cases, + const within = await runEvaluation({ + name: "upper", + task: swap, + data: cases, allow: { matches: { miss: 1, drop: 1 } }, - compare: before, + compare: before.experiment, }); expect(within.ok).toBe(true); }); - await expect(evaluate("other", { run: upper, cases, allow, compare: before })).rejects.toThrow( - 'compare is a report of eval "upper"', - ); + await expect( + runEvaluation({ name: "other", task: upper, data: cases, allow, compare: before.experiment }), + ).rejects.toThrow('compare is an experiment of "upper"'); }); test("added and removed cases are reported, not counted as drops", async () => { - const before = await evaluate("upper", { run: upper, cases, allow }); + const before = await runEvaluation({ name: "upper", task: upper, data: cases, allow }); const changed = [...cases.slice(1), { name: "d", input: "d", expect: "nope" }]; - const report = await evaluate("upper", { run: upper, cases: changed, allow, compare: before }); + const report = await runEvaluation({ + name: "upper", + task: upper, + data: changed, + allow, + compare: before.experiment, + }); expect(report.added).toEqual(["d"]); expect(report.removed).toEqual(["a"]); expect(report.checks["matches"]?.drop).toBe(0); @@ -497,16 +581,16 @@ describe("compare and save", () => { test("refuses another eval's run and malformed files; reads 0.1.0 files", async () => { const path = join(await tempDir(), "b.json"); - await evaluate("one", { run: upper, cases, allow, save: path }); - await expect(evaluate("two", { run: upper, cases, allow, compare: path })).rejects.toThrow( - 'stored run of eval "one", not "two"', - ); + await runEvaluation({ name: "one", task: upper, data: cases, allow, save: path }); + await expect( + runEvaluation({ name: "two", task: upper, data: cases, allow, compare: path }), + ).rejects.toThrow('stored experiment "one", not "two"'); const old = { ...JSON.parse(await Bun.file(path).text()), passRate: 0.667 }; await Bun.write(path, JSON.stringify(old)); - await evaluate("one", { run: upper, cases, allow, compare: path }); - await Bun.write(path, JSON.stringify({ format: 2 })); - await expect(evaluate("one", { run: upper, cases, allow, compare: path })).rejects.toThrow( - "unknown format 2", - ); + await runEvaluation({ name: "one", task: upper, data: cases, allow, compare: path }); + await Bun.write(path, JSON.stringify({ format: 1, name: "one", cases: {} })); + await expect( + runEvaluation({ name: "one", task: upper, data: cases, allow, compare: path }), + ).rejects.toThrow("expected 2"); }); }); diff --git a/e2e/jev.e2e.test.ts b/e2e/jev.e2e.test.ts index 29eb7c0..4926f72 100644 --- a/e2e/jev.e2e.test.ts +++ b/e2e/jev.e2e.test.ts @@ -7,9 +7,9 @@ import type { EvaluateDeps } from "@corbits/system-one"; import { ask, EvalFailure, - evaluate, + runEvaluation, jev, - type Check, + type Scorer, type Escalation, type Judge, type Outcome, @@ -44,13 +44,14 @@ function stubJev(answer: unknown) { return { bodies, judge: jev({ apiKey: "test-key", deps: stubDeps(fetch) }) }; } -async function judged(check: Check, judge: Judge) { +async function judged(check: Scorer, judge: Judge) { let report: Report; try { - report = await evaluate("jev", { - run: (input: string) => `reply to ${input}`, - cases: [{ input: "refund?", expect: "yes" }], - checks: { judged: check }, + report = await runEvaluation({ + name: "jev", + task: (input: string) => `reply to ${input}`, + data: [{ input: "refund?", expect: "yes" }], + scorers: { judged: check }, judge, }); } catch (cause) { @@ -60,7 +61,7 @@ async function judged(check: Check, judge: Judge) { return { outcome: report.cases[0]?.checks["judged"] }; } -async function answered(answer: unknown, check: Check) { +async function answered(answer: unknown, check: Scorer) { const { bodies, judge } = stubJev(answer); return { ...(await judged(check, judge)), bodies }; } @@ -116,10 +117,11 @@ describe("sentence checks", () => { describe("batching", () => { test("a case's Jev checks answer in one call and map back by name", async () => { const { bodies, judge } = stubJev({ type: "noul", noul: 0.9 }); - const report = await evaluate("jev", { - run: (input: string) => `reply to ${input}`, - cases: [{ input: "refund?" }], - checks: { ready: "Ready to send?", polite: "Is the tone polite?" }, + const report = await runEvaluation({ + name: "jev", + task: (input: string) => `reply to ${input}`, + data: [{ input: "refund?" }], + scorers: { ready: "Ready to send?", polite: "Is the tone polite?" }, judge, }); expect(bodies).toHaveLength(1); @@ -142,10 +144,11 @@ describe("batching", () => { }) as unknown as EvaluateDeps["fetch"]; let report: Report; try { - await evaluate("jev", { - run: (input: string) => input, - cases: [{ input: "a" }], - checks: { ready: "Ready to send?", polite: "Is the tone polite?" }, + await runEvaluation({ + name: "jev", + task: (input: string) => input, + data: [{ input: "a" }], + scorers: { ready: "Ready to send?", polite: "Is the tone polite?" }, judge: jev({ apiKey: "test-key", deps: stubDeps(failing) }), }); throw new Error("expected failure"); @@ -162,10 +165,11 @@ describe("batching", () => { test("a case that skips one Jev check keeps it out of the call", async () => { const { bodies, judge } = stubJev({ type: "noul", noul: 0.9 }); - await evaluate("jev", { - run: (input: string) => input, - cases: [{ input: "a", skip: ["polite"] }], - checks: { ready: "Ready to send?", polite: "Is the tone polite?" }, + await runEvaluation({ + name: "jev", + task: (input: string) => input, + data: [{ input: "a", skip: ["polite"] }], + scorers: { ready: "Ready to send?", polite: "Is the tone polite?" }, judge, }); expect(Object.keys(bodies[0]?.["questions"] as object)).toEqual(["ready"]); @@ -240,7 +244,7 @@ describe("thresholds", () => { await expect(judged({ question, yes: 0.2, no: 0.5 }, judge)).rejects.toThrow( "no must not exceed yes", ); - // Even a hand-built ask (bypassing ask()) is validated at evaluate time. + // Even a hand-built ask (bypassing ask()) is validated at runEvaluation time. await expect( judged({ question: "Ready?", options: { a: "a" }, pass: ["a"], sure: 0 }, judge), ).rejects.toThrow("sure must be in (0, 1]"); @@ -514,10 +518,11 @@ describe("stored reports", () => { probabilities: { ready: 0.8, revise: 0.1, escalate: 0.1 }, confidence: 0.8, }; - await evaluate("jev", { - run: (input: string) => `reply to ${input}`, - cases: [{ input: "refund?" }], - checks: { + await runEvaluation({ + name: "jev", + task: (input: string) => `reply to ${input}`, + data: [{ input: "refund?" }], + scorers: { judged: ask("Ready to send?", { options: { ready: "send", revise: "work", escalate: "human" }, pass: "ready", diff --git a/e2e/workflow.e2e.test.ts b/e2e/workflow.e2e.test.ts index b5782a6..00724b0 100644 --- a/e2e/workflow.e2e.test.ts +++ b/e2e/workflow.e2e.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from "bun:test"; import { defineAgent } from "@intx/agent"; import { defineWorkflow, step, type StepInvoker } from "@intx/workflow"; -import { EvalFailure, evaluate, matches, ranSteps, under } from "../src/index.js"; +import { EvalFailure, runEvaluation, matches, ranSteps, under } from "../src/index.js"; import { workflow } from "../src/workflow.js"; function agent(id: string) { @@ -36,10 +36,11 @@ const invokeStep: StepInvoker = async ({ agent: a, input }) => { describe("workflow runs", () => { test("output defaults to the last step and every step is recorded", async () => { - const report = await evaluate("workflow", { - run: workflow(pipeline, { invokeStep }), - cases: [{ input: "refund me", expect: "DRAFT: REFUND ME" }], - checks: { + const report = await runEvaluation({ + name: "workflow", + task: workflow(pipeline, { invokeStep }), + data: [{ input: "refund me", expect: "DRAFT: REFUND ME" }], + scorers: { exact: matches(), order: (r) => r.steps.map((s) => s.name).join(">") === "drafter>reviewer", steps: ranSteps(["drafter", "reviewer"]), @@ -59,13 +60,14 @@ describe("workflow runs", () => { }); test("output and trigger pick object values", async () => { - const report = await evaluate("workflow-object", { - run: workflow(pipeline, { + const report = await runEvaluation({ + name: "workflow-object", + task: workflow(pipeline, { invokeStep, trigger: (input: { order: number }) => `order ${input.order}`, output: (run) => ({ draft: run.outputs["draft"], review: run.outputs["review"] }), }), - cases: [ + data: [ { input: { order: 42 }, expect: { draft: "draft: order 42", review: "DRAFT: ORDER 42" } }, ], }); @@ -74,14 +76,15 @@ describe("workflow runs", () => { test("a failing step fails the case closed", async () => { try { - await evaluate("workflow-fail", { - run: workflow(pipeline, { + await runEvaluation({ + name: "workflow-fail", + task: workflow(pipeline, { invokeStep: async ({ agent: a }) => { if (a.id === "reviewer") throw new Error("reviewer down"); return { output: "draft" }; }, }), - cases: [{ input: "x", expect: "X" }], + data: [{ input: "x", expect: "X" }], }); throw new Error("expected a failure"); } catch (cause) { diff --git a/examples/local-jev.ts b/examples/local-jev.ts index ffe8775..d9949b2 100644 --- a/examples/local-jev.ts +++ b/examples/local-jev.ts @@ -1,6 +1,6 @@ // Local Jev judge over canned support drafts (good, invented-compensation, // refusal). Run: bun examples/local-jev.ts -import { ask, evaluate } from "../src/index.js"; +import { ask, runEvaluation } from "../src/index.js"; import { localJev } from "./ollama.js"; const DRAFTS: Record = { @@ -12,10 +12,11 @@ const DRAFTS: Record = { escalate: "I can't help with that. Goodbye.", }; -const report = await evaluate("local-jev-support", { - run: (id: string) => DRAFTS[id] ?? "", - cases: Object.keys(DRAFTS).map((id) => ({ input: id })), - checks: { +const report = await runEvaluation({ + name: "local-jev-support", + task: (id: string) => DRAFTS[id] ?? "", + data: Object.keys(DRAFTS).map((id) => ({ input: id })), + scorers: { ready: ask( "Classify the draft reply. Ready means accurate, answers the request, invents nothing. Unsafe means it invents refunds, credits, or policies.", { diff --git a/examples/local-loop.ts b/examples/local-loop.ts index c5e5be0..ed9a04b 100644 --- a/examples/local-loop.ts +++ b/examples/local-loop.ts @@ -1,19 +1,20 @@ // Full local loop: qwen2.5:7b writes replies, clef-flash:9b grades them. // Run: bun examples/local-loop.ts -import { ask, evaluate } from "../src/index.js"; +import { ask, runEvaluation } from "../src/index.js"; import { localJev, ollamaGenerate } from "./ollama.js"; const SYSTEM = `You are a support agent. Rules: never promise refunds, credits, or compensation beyond what the context states. Verified context is ground truth. Reply in 2 sentences max.`; type Ticket = { message: string; verified: string }; -const report = await evaluate("local-loop-support", { - run: (ticket: Ticket) => +const report = await runEvaluation({ + name: "local-loop-support", + task: (ticket: Ticket) => ollamaGenerate( "qwen2.5:7b", `${SYSTEM}\n\nVerified context: ${ticket.verified}\nCustomer: ${ticket.message}`, ), - cases: [ + data: [ { name: "dup-charge", input: { @@ -36,7 +37,7 @@ const report = await evaluate("local-loop-support", { }, }, ], - checks: { + scorers: { ready: ask( "Classify the reply against the verified context. Ready = accurate, answers the request, invents nothing beyond context. Revise otherwise.", { options: { ready: "sendable", revise: "inaccurate or invents" }, pass: "ready" }, diff --git a/examples/ports/battle.ts b/examples/ports/battle.ts index fc8b52f..3557175 100644 --- a/examples/ports/battle.ts +++ b/examples/ports/battle.ts @@ -1,6 +1,6 @@ // Pairwise beats absolute 1-10 for small judges. // Run: bun examples/ports/battle.ts -import { ask, evaluate } from "../../src/index.js"; +import { ask, runEvaluation } from "../../src/index.js"; import { localJev } from "../ollama.js"; const QUESTION = "What is the capital of France?"; @@ -21,13 +21,14 @@ const PAIRS: [string, string][] = [ ]; // Jev sees the case input, so each pair goes in as { question, A, B }. -const report = await evaluate("port-battle", { - run: (pair: { question: string; A: string; B: string }) => pair, - cases: PAIRS.map(([a, b]) => ({ +const report = await runEvaluation({ + name: "port-battle", + task: (pair: { question: string; A: string; B: string }) => pair, + data: PAIRS.map(([a, b]) => ({ name: `${a}-vs-${b}`, input: { question: QUESTION, A: ANSWERS[a] ?? "", B: ANSWERS[b] ?? "" }, })), - checks: { + scorers: { battle: ask("Which answer better serves the question? Correctness first, then completeness.", { options: { A: "answer A is better", diff --git a/examples/ports/factuality.ts b/examples/ports/factuality.ts index 1c14f0d..c448467 100644 --- a/examples/ports/factuality.ts +++ b/examples/ports/factuality.ts @@ -1,7 +1,7 @@ // ABCDE factuality rubric: A, B, C and E count as factual, D does not. // Agreement vs ground truth is the metric; exact match is the baseline. // Run: bun examples/ports/factuality.ts -import { ask, evaluate, matches } from "../../src/index.js"; +import { ask, runEvaluation, matches } from "../../src/index.js"; import { localJev } from "../ollama.js"; const CRITERIA = { @@ -74,10 +74,11 @@ const CASES: { id: string; question: string; expert: string; submission: string; }, ]; -const report = await evaluate("port-factuality", { - run: (question: string) => CASES.find((c) => c.question === question)?.submission ?? "", - cases: CASES.map((c) => ({ name: c.id, input: c.question, expect: c.expert })), - checks: { +const report = await runEvaluation({ + name: "port-factuality", + task: (question: string) => CASES.find((c) => c.question === question)?.submission ?? "", + data: CASES.map((c) => ({ name: c.id, input: c.question, expect: c.expert })), + scorers: { baseline_exact: matches(), jev_factuality: ask( "Compare the factual content of the output with the expert answer in expect. Ignore style, grammar, punctuation. Pick one option.", diff --git a/examples/ports/faithfulness.ts b/examples/ports/faithfulness.ts index 75c8487..1c379a6 100644 --- a/examples/ports/faithfulness.ts +++ b/examples/ports/faithfulness.ts @@ -1,7 +1,7 @@ // Claim-checking (extract claims, verify each against context) as one // Jev choice. Keyword coverage passes the vocab-reuse trap; the judge // must catch it. Run: bun examples/ports/faithfulness.ts -import { ask, evaluate } from "../../src/index.js"; +import { ask, runEvaluation } from "../../src/index.js"; import { localJev } from "../ollama.js"; import { coverage } from "../words.js"; @@ -53,10 +53,11 @@ const CASES: { id: string; context: string; answer: string; truth: Truth }[] = [ ]; // The context goes in as the input, so Jev sees it next to the answer. -const report = await evaluate("port-faithfulness", { - run: (context: string) => CASES.find((c) => c.context === context)?.answer ?? "", - cases: CASES.map((c) => ({ name: c.id, input: c.context })), - checks: { +const report = await runEvaluation({ + name: "port-faithfulness", + task: (context: string) => CASES.find((c) => c.context === context)?.answer ?? "", + data: CASES.map((c) => ({ name: c.id, input: c.context })), + scorers: { baseline_coverage: (r) => coverage(r.input, r.output) >= 0.5, jev_faithfulness: ask( "Is every claim in the output grounded in the context given as input? Pick one option.", diff --git a/examples/ports/keyword-coverage.ts b/examples/ports/keyword-coverage.ts index 2c016db..f87f75a 100644 --- a/examples/ports/keyword-coverage.ts +++ b/examples/ports/keyword-coverage.ts @@ -1,6 +1,6 @@ // Keyword overlap: exact match fails every paraphrase; coverage passes // them. Run: bun examples/ports/keyword-coverage.ts -import { evaluate, matches } from "../../src/index.js"; +import { runEvaluation, matches } from "../../src/index.js"; import { coverage } from "../words.js"; const CASES: { id: string; reference: string; output: string; truth: boolean }[] = [ @@ -36,10 +36,11 @@ const CASES: { id: string; reference: string; output: string; truth: boolean }[] }, ]; -const report = await evaluate("port-keyword-coverage", { - run: (id: string) => CASES.find((c) => c.id === id)?.output ?? "", - cases: CASES.map((c) => ({ input: c.id, expect: c.reference })), - checks: { +const report = await runEvaluation({ + name: "port-keyword-coverage", + task: (id: string) => CASES.find((c) => c.id === id)?.output ?? "", + data: CASES.map((c) => ({ input: c.id, expect: c.reference })), + scorers: { baseline_exact: matches(), port_coverage: (r) => coverage(r.expect ?? "", r.output) >= 0.5, }, diff --git a/examples/ports/mtbench-score.ts b/examples/ports/mtbench-score.ts index f3bfd17..fece94e 100644 --- a/examples/ports/mtbench-score.ts +++ b/examples/ports/mtbench-score.ts @@ -1,6 +1,6 @@ // 1-10 helpfulness as Jev levels. Rank order of four answers is the // ground truth. Run: bun examples/ports/mtbench-score.ts -import { ask, evaluate } from "../../src/index.js"; +import { ask, runEvaluation } from "../../src/index.js"; import { localJev } from "../ollama.js"; const LEVELS = [ @@ -25,11 +25,12 @@ const CASES: { id: string; rank: number; answer: string }[] = [ { id: "wrong", rank: 4, answer: "The capital of France is Lyon, a lovely city." }, ]; -const report = await evaluate("port-mtbench-score", { - run: (input: { question: string; id: string }) => +const report = await runEvaluation({ + name: "port-mtbench-score", + task: (input: { question: string; id: string }) => CASES.find((c) => c.id === input.id)?.answer ?? "", - cases: CASES.map((c) => ({ name: c.id, input: { question: QUESTION, id: c.id } })), - checks: { + data: CASES.map((c) => ({ name: c.id, input: { question: QUESTION, id: c.id } })), + scorers: { jev_helpfulness: ask( "Rate the output's helpfulness and correctness for the question. A bare fragment with no explanation caps at 5. A complete correct answer with supporting detail scores 8+. A wrong answer scores 1-2.", { levels: LEVELS, pass: "6: good" }, diff --git a/package.json b/package.json index 33ccb92..4cc752a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@corbits/evals", "version": "0.1.2", - "description": "Evals for agents and workflows in one evaluate call: code, Jev and helper checks over JSONL or inline cases, gated per check on allowed misses and drops.", + "description": "Evals for agents and workflows in one runEvaluation call: code, Jev and helper scorers over JSONL or inline data, gated per scorer on allowed misses and drops.", "keywords": [ "agents", "corbits", diff --git a/src/calibrate.ts b/src/calibrate.ts index 34872ad..3afed93 100644 --- a/src/calibrate.ts +++ b/src/calibrate.ts @@ -1,6 +1,6 @@ import type { Decision } from "@corbits/system-one"; -import { caseName } from "./evaluate.js"; +import { caseName } from "./evaluation.js"; import { DEFAULTS, decideBatch, toOutcome, type Ask, type Judge, type Sentence } from "./jev.js"; import { mapPool, runOnce, type RunSpec } from "./runner.js"; import { toTraced, type Runnable } from "./runs.js"; diff --git a/src/evaluate.ts b/src/evaluation.ts similarity index 68% rename from src/evaluate.ts rename to src/evaluation.ts index ec6b7b0..e88d040 100644 --- a/src/evaluate.ts +++ b/src/evaluation.ts @@ -8,7 +8,7 @@ import { toMarkdown, toStored, toSummary, - type Stored, + type Experiment, } from "./reports.js"; import { runCases, type NamedCase } from "./runner.js"; import { toTraced, type AgentLike, type Runnable } from "./runs.js"; @@ -16,7 +16,7 @@ import { toMs, type Duration } from "./timeout.js"; import type { Case, CaseResult, CheckFn, CheckResult, Report, Run } from "./types.js"; /** Code, a yes/no question for Jev, or `ask(...)`. */ -export type Check = +export type Scorer = | CheckFn | string | Ask @@ -25,34 +25,35 @@ export type Check = /** Cases a check may miss, or `{ miss, drop }`; both default to 0. */ export type Allow = number | { miss?: number; drop?: number }; -export type EvaluateOptions = { - run: Runnable; +export type Evaluation = { + name: string; + task: Runnable; /** Case objects, or the path of a JSONL file of them. */ - cases: readonly Case[] | string; + data: readonly Case[] | string; /** Defaults to `{ matches: matches() }`. */ - checks?: Record>; - /** Per check name. Unlisted checks may miss no case and drop nothing. */ + scorers?: Record>; + /** Per scorer name. Unlisted scorers may miss no case and drop nothing. */ allow?: Record; /** - * A stored run's path or a previous report to measure drops against. + * An experiment file's path, or an experiment, to measure drops against. * Never written; a missing file means drops are not gated. */ - compare?: string | Report; - /** Path to store this run at, pass or fail. */ + compare?: string | Experiment; + /** Path to store this experiment at, pass or fail. */ save?: string; repeats?: number; concurrency?: number; - /** Bounds each run and each check. A run that times out is aborted via its signal. */ + /** Bounds each task and each scorer. A task that times out is aborted via its signal. */ timeout?: Duration; - /** Stamped into the stored run, e.g. `{ model }`. */ + /** Stamped into the experiment, e.g. `{ model }`. */ about?: Record; - /** Jev for string and `ask` checks. Defaults to `jev()`. */ + /** Jev for string and `ask` scorers. Defaults to `jev()`. */ judge?: Judge; /** Skip the console summary. */ quiet?: boolean; }; -/** Thrown by `evaluate` when the gate fails; `report` holds every result. */ +/** Thrown by `runEvaluation` when the gate fails; `report` holds every result. */ export class EvalFailure extends Error { readonly report: Report; constructor(message: string, report: Report) { @@ -65,7 +66,7 @@ export class EvalFailure extends Error { function positiveInt(name: string, key: string, raw: number | undefined, fallback: number): number { if (raw === undefined) return fallback; if (!Number.isInteger(raw) || raw < 1) { - throw new Error(`eval "${name}": ${key} must be a whole number >= 1`); + throw new Error(`evaluation "${name}": ${key} must be a whole number >= 1`); } return raw; } @@ -73,7 +74,7 @@ function positiveInt(name: string, key: string, raw: number | undefined, fallbac function count(name: string, key: string, raw: unknown): number { if (raw === undefined) return 0; if (typeof raw !== "number" || !Number.isInteger(raw) || raw < 0) { - throw new Error(`eval "${name}": ${key} must be a whole number >= 0`); + throw new Error(`evaluation "${name}": ${key} must be a whole number >= 0`); } return raw; } @@ -87,7 +88,7 @@ function toAllowances( ): Record { const unknown = Object.keys(allow).find((key) => !checks.includes(key)); if (unknown !== undefined) { - throw new Error(`eval "${name}": allow names unknown check "${unknown}"`); + throw new Error(`evaluation "${name}": allow names unknown scorer "${unknown}"`); } return Object.fromEntries( checks.map((check) => { @@ -97,7 +98,7 @@ function toAllowances( } const extra = Object.keys(raw).find((key) => key !== "miss" && key !== "drop"); if (extra !== undefined) { - throw new Error(`eval "${name}": allow.${check} has unknown "${extra}"`); + throw new Error(`evaluation "${name}": allow.${check} has unknown "${extra}"`); } const { miss, drop } = raw as { miss?: unknown; drop?: unknown }; return [ @@ -125,21 +126,23 @@ function named( cases: readonly Case[], checks: readonly string[], ): Named[] { - if (cases.length === 0) throw new Error(`eval "${name}" has no cases`); + if (cases.length === 0) throw new Error(`evaluation "${name}" has no data`); const seen = new Set(); return cases.map((evalCase) => { const key = caseName(evalCase); - if (seen.has(key)) throw new Error(`eval "${name}" has duplicate case "${key}"`); + if (seen.has(key)) throw new Error(`evaluation "${name}" has duplicate case "${key}"`); seen.add(key); for (const field of ["must", "skip"] as const) { const unknown = evalCase[field]?.find((check) => !checks.includes(check)); if (unknown !== undefined) { - throw new Error(`eval "${name}": case "${key}" ${field} names unknown check "${unknown}"`); + throw new Error( + `evaluation "${name}": case "${key}" ${field} names unknown scorer "${unknown}"`, + ); } } const both = evalCase.must?.find((check) => evalCase.skip?.includes(check)); if (both !== undefined) { - throw new Error(`eval "${name}": case "${key}" both must and skip "${both}"`); + throw new Error(`evaluation "${name}": case "${key}" both must and skip "${both}"`); } return { ...evalCase, name: key }; }); @@ -147,13 +150,13 @@ function named( function toCheckParts( name: string, - checks: Record>, + checks: Record>, ): { code: Record) => unknown>; jev: Record; } { const entries = Object.entries(checks); - if (entries.length === 0) throw new Error(`eval "${name}" has no checks`); + if (entries.length === 0) throw new Error(`evaluation "${name}" has no scorers`); const code: Record) => unknown> = {}; const jev: Record = {}; for (const [key, check] of entries) { @@ -163,7 +166,7 @@ function toCheckParts( } const valid = typeof check === "string" ? check.trim() !== "" : typeof check?.question === "string"; - if (!valid) throw new Error(`eval "${name}": check "${key}" is not a check`); + if (!valid) throw new Error(`evaluation "${name}": check "${key}" is not a scorer`); checkThresholds(check as Ask | Sentence); jev[key] = check as Ask | Sentence | string; } @@ -174,7 +177,7 @@ function score( results: readonly CaseResult[], cases: readonly Named[], allowances: Record, - stored: Stored | undefined, + stored: Experiment | undefined, ): { checks: Record; reasons: string[] } { const must = new Map(cases.map((c) => [c.name, c.must ?? []])); const checks: Record = {}; @@ -217,20 +220,20 @@ function score( async function toCompared( name: string, - compare: string | Report | undefined, -): Promise { + compare: string | Experiment | undefined, +): Promise { if (compare === undefined) return undefined; if (typeof compare !== "string") { if (compare.name !== name) { - throw new Error(`eval "${name}": compare is a report of eval "${compare.name}"`); + throw new Error(`evaluation "${name}": compare is an experiment of "${compare.name}"`); } - return fromReport(compare); + return compare; } const file = Bun.file(compare); if (!(await file.exists())) return undefined; const stored = parseStored(await file.text(), compare); - if (stored.eval !== name) { - throw new Error(`${compare}: stored run of eval "${stored.eval}", not "${name}"`); + if (stored.name !== name) { + throw new Error(`${compare}: stored experiment "${stored.name}", not "${name}"`); } return stored; } @@ -253,41 +256,35 @@ function failures(cases: readonly CaseResult[], checks: Record( - name: string, - options: EvaluateOptions & { run: AgentLike }, +export async function runEvaluation( + spec: Evaluation & { task: AgentLike }, ): Promise>; -export async function evaluate( - name: string, - options: EvaluateOptions, +export async function runEvaluation( + spec: Evaluation, ): Promise>; -export async function evaluate( - name: string, - options: EvaluateOptions, -): Promise> { - if (name.trim() === "") throw new Error("evaluate needs a name"); - const repeats = positiveInt(name, "repeats", options.repeats, 1); - const concurrency = positiveInt(name, "concurrency", options.concurrency, 4); - const timeoutMs = options.timeout === undefined ? undefined : toMs(options.timeout, "timeout"); - let judge: Judge | undefined = options.judge; - const defined = options.checks ?? { matches: matches() as Check }; +export async function runEvaluation(spec: Evaluation): Promise> { + const name = spec.name; + if (typeof name !== "string" || name.trim() === "") throw new Error("runEvaluation needs a name"); + const repeats = positiveInt(name, "repeats", spec.repeats, 1); + const concurrency = positiveInt(name, "concurrency", spec.concurrency, 4); + const timeoutMs = spec.timeout === undefined ? undefined : toMs(spec.timeout, "timeout"); + let judge: Judge | undefined = spec.judge; + const defined = spec.scorers ?? { matches: matches() as Scorer }; const { code, jev: jevChecks } = toCheckParts(name, defined); const checkNames = Object.keys(defined); const jevGroup = Object.keys(jevChecks).length === 0 ? undefined : { judge: (judge ??= jev()), checks: jevChecks }; - const allowances = toAllowances(name, options.allow ?? {}, checkNames); + const allowances = toAllowances(name, spec.allow ?? {}, checkNames); const cases = named( name, - typeof options.cases === "string" - ? ((await fromJsonl(options.cases)) as Case[]) - : options.cases, + typeof spec.data === "string" ? ((await fromJsonl(spec.data)) as Case[]) : spec.data, checkNames, ); - const run = toTraced(options.run); + const run = toTraced(spec.task, "task"); - const stored = await toCompared(name, options.compare); + const stored = await toCompared(name, spec.compare); const results = await runCases({ cases, @@ -299,7 +296,7 @@ export async function evaluate( ...(timeoutMs === undefined ? {} : { timeoutMs }), }); const scored = score(results, cases, allowances, stored); - const data: Omit, "markdown"> = { + const data: Omit, "markdown" | "experiment"> = { name, ok: scored.reasons.length === 0, total: results.length, @@ -307,7 +304,7 @@ export async function evaluate( checks: scored.checks, cases: results, }; - if (options.about !== undefined) data.about = { ...options.about }; + if (spec.about !== undefined) data.about = { ...spec.about }; if (stored !== undefined) { const before = stored.cases; const now = new Set(results.map((c) => c.name)); @@ -315,20 +312,24 @@ export async function evaluate( data.added = results.filter((c) => before[c.name] === undefined).map((c) => c.name); data.removed = Object.keys(before).filter((c) => !now.has(c)); } - const report: Report = { ...data, markdown: () => toMarkdown(data) }; - const log = options.quiet === true ? () => {} : console.log; + const report: Report = { + ...data, + experiment: fromReport(data), + markdown: () => toMarkdown(data), + }; + const log = spec.quiet === true ? () => {} : console.log; log(toSummary(data)); - if (typeof options.compare === "string" && stored === undefined) { - log(` no compare run at ${options.compare}`); + if (typeof spec.compare === "string" && stored === undefined) { + log(` no compare run at ${spec.compare}`); } - if (options.save !== undefined) { - await Bun.write(options.save, toStored(data)); - log(` wrote ${options.save}`); + if (spec.save !== undefined) { + await Bun.write(spec.save, toStored(data)); + log(` wrote ${spec.save}`); } if (!report.ok) { throw new EvalFailure( [ - `eval "${name}" failed: ${scored.reasons.join("; ")}`, + `evaluation "${name}" failed: ${scored.reasons.join("; ")}`, ...failures(results, scored.checks), ].join("\n"), report, diff --git a/src/index.ts b/src/index.ts index dd86892..efe1428 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,7 +31,14 @@ export { type CalibrationRow, type Labeled, } from "./calibrate.js"; -export { EvalFailure, evaluate, type Allow, type Check, type EvaluateOptions } from "./evaluate.js"; +export { + EvalFailure, + runEvaluation, + type Allow, + type Evaluation, + type Scorer, +} from "./evaluation.js"; +export type { Experiment, ExperimentRow } from "./reports.js"; export { ask, jev, diff --git a/src/reports.ts b/src/reports.ts index ad05f51..f4c47ec 100644 --- a/src/reports.ts +++ b/src/reports.ts @@ -2,32 +2,32 @@ import type { CaseResult, Change, CheckResult, Outcome, Report } from "./types.j type Row = Record; -/** A report without its methods. */ -export type ReportData = Omit; +/** A report without its methods or its experiment. */ +export type ReportData = Omit; -/** Per case × check pass and value, without times, so it diffs cleanly in git. */ -export type Stored = { - eval: string; +/** Per case × scorer pass and value, without times, so it diffs cleanly in git. */ +export type Experiment = { + name: string; about?: Record; - cases: Record>; + cases: Record>; }; -/** One check's stored answer: the outcome plus a Jev distribution when present. */ -export type StoredRow = { +/** One scorer's stored answer: the outcome plus a Jev distribution when present. */ +export type ExperimentRow = { pass: boolean; value: number; probabilities?: Record; confidence?: number; }; -const STORED_FORMAT = 1; +const STORED_FORMAT = 2; function round(n: number): number { return Math.round(n * 1000) / 1000; } -export function fromReport(report: ReportData): Stored { - const cases: Stored["cases"] = {}; +export function fromReport(report: ReportData): Experiment { + const cases: Experiment["cases"] = {}; for (const c of report.cases) { cases[c.name] = Object.fromEntries( Object.entries(c.checks).map(([name, o]) => [ @@ -48,7 +48,7 @@ export function fromReport(report: ReportData): Stored { ); } return { - eval: report.name, + name: report.name, ...(report.about === undefined ? {} : { about: report.about }), cases, }; @@ -62,7 +62,7 @@ function isObject(value: unknown): value is Row { return typeof value === "object" && value !== null && !Array.isArray(value); } -function stored(raw: unknown, where: string): StoredRow { +function stored(raw: unknown, where: string): ExperimentRow { if (!isObject(raw) || typeof raw["pass"] !== "boolean") { throw new Error(`${where}: "pass" must be a boolean`); } @@ -70,7 +70,7 @@ function stored(raw: unknown, where: string): StoredRow { if (typeof value !== "number" || !Number.isFinite(value)) { throw new Error(`${where}: "value" must be a finite number`); } - const row: StoredRow = { pass: raw["pass"], value }; + const row: ExperimentRow = { pass: raw["pass"], value }; const probabilities = raw["probabilities"]; if (probabilities !== undefined) { if (!isObject(probabilities)) { @@ -93,7 +93,7 @@ function stored(raw: unknown, where: string): StoredRow { return row; } -export function parseStored(text: string, source: string): Stored { +export function parseStored(text: string, source: string): Experiment { let raw: unknown; try { raw = JSON.parse(text); @@ -106,12 +106,12 @@ export function parseStored(text: string, source: string): Stored { `${source}: unknown format ${JSON.stringify(raw["format"])}, expected ${STORED_FORMAT}`, ); } - const name = raw["eval"]; + const name = raw["name"]; if (typeof name !== "string" || name === "") { - throw new Error(`${source}: "eval" must be a non-empty string`); + throw new Error(`${source}: "name" must be a non-empty string`); } if (!isObject(raw["cases"])) throw new Error(`${source}: "cases" must be an object`); - const cases: Stored["cases"] = {}; + const cases: Experiment["cases"] = {}; for (const [caseName, checks] of Object.entries(raw["cases"])) { if (!isObject(checks)) throw new Error(`${source}: case "${caseName}" must be an object`); cases[caseName] = Object.fromEntries( @@ -121,7 +121,7 @@ export function parseStored(text: string, source: string): Stored { ]), ); } - const run: Stored = { eval: name, cases }; + const run: Experiment = { name, cases }; const about = raw["about"]; if (about !== undefined) { if (!isObject(about)) throw new Error(`${source}: "about" must be an object`); @@ -131,7 +131,7 @@ export function parseStored(text: string, source: string): Stored { } /** Case × check pass flips. Cases or checks on one side only are not changes. */ -export function flips(stored: Stored, cases: readonly CaseResult[]): Change[] { +export function flips(stored: Experiment, cases: readonly CaseResult[]): Change[] { const changes: Change[] = []; for (const c of cases) { const before = stored.cases[c.name]; @@ -166,7 +166,7 @@ export function toMarkdown(report: ReportData): string { const names = Object.keys(report.checks); const comparing = report.changes !== undefined; const lines = [ - `# eval: ${cell(report.name)}`, + `# evaluation: ${cell(report.name)}`, ``, `- cases: ${report.total}, passed: ${report.passed}`, `- result: ${report.ok ? "ok" : "FAILED"}`, diff --git a/src/runs.ts b/src/runs.ts index 08c5f42..f446d98 100644 --- a/src/runs.ts +++ b/src/runs.ts @@ -22,7 +22,7 @@ export type Traced = { export type Runnable = RunFn | Traced | AgentLike; -const NOT_RUNNABLE = "run must be a function, an @intx/agent Agent, or workflow()"; +const NOT_RUNNABLE = "must be a function, an @intx/agent Agent, or workflow()"; function isWorkflowDefinition(value: object): boolean { return "stepOrder" in value && "steps" in value && "triggers" in value; @@ -34,12 +34,13 @@ function isWorkflowDefinition(value: object): boolean { */ export function toTraced( run: Runnable, + noun = "run", ): (input: I, signal: AbortSignal) => Promise> { if (typeof run === "function") { return async (input, signal) => ({ output: await run(input, signal), steps: [] }); } if (typeof run !== "object" || run === null) { - throw new Error(NOT_RUNNABLE); + throw new Error(`${noun} ${NOT_RUNNABLE}`); } if (TRACE in run) return run[TRACE]; if ("send" in run && typeof run.send === "function") { @@ -52,7 +53,7 @@ export function toTraced( }; } if (isWorkflowDefinition(run)) { - throw new Error('run: wrap a workflow with workflow() from "@corbits/evals/workflow"'); + throw new Error(`${noun}: wrap a workflow with workflow() from "@corbits/evals/workflow"`); } - throw new Error(NOT_RUNNABLE); + throw new Error(`${noun} ${NOT_RUNNABLE}`); } diff --git a/src/types.ts b/src/types.ts index 632ec8d..756c72b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,3 +1,5 @@ +import type { Experiment } from "./reports.js"; + export type Step = { name: string; kind: "llm" | "tool" | "step"; @@ -90,7 +92,9 @@ export type Report = { changes?: Change[]; /** Cases only in this run. Present when comparing. */ added?: string[]; - /** Cases only in the compared run. Present when comparing. */ + /** Cases only in the compared experiment. Present when comparing. */ removed?: string[]; + /** The persisted experiment for this run. */ + experiment: Experiment; markdown(): string; }; From 3bfef54e79bec68896d837518d5ef1e4ce7cacfd Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 01:52:46 -0700 Subject: [PATCH 02/16] feat!: name the case ground truth expectedOutput JSONL rows that still say expect or expected are rejected. --- README.md | 34 ++++++++++++++-------------- e2e/evaluate.e2e.test.ts | 36 +++++++++++++++--------------- e2e/jev.e2e.test.ts | 4 ++-- e2e/workflow.e2e.test.ts | 9 +++++--- examples/ports/factuality.ts | 4 ++-- examples/ports/keyword-coverage.ts | 4 ++-- src/calibrate.ts | 4 ++-- src/checks.ts | 11 ++++----- src/datasets.test.ts | 11 +++++++++ src/datasets.ts | 5 ++++- src/jev.ts | 2 +- src/runner.ts | 11 ++++++--- src/types.ts | 6 ++--- 13 files changed, 82 insertions(+), 59 deletions(-) create mode 100644 src/datasets.test.ts diff --git a/README.md b/README.md index 392c03d..eb1b167 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ console.log(report.markdown()); ## Checks `checks` maps a name to a check. Without `checks`, one `matches` check -compares output with `expect`. Every check gets the run -`r = { input, output, expect, steps, ms }`; `output` is whatever the run +compares output with `expectedOutput`. Every check gets the run +`r = { input, output, expectedOutput, steps, ms }`; `output` is whatever the run returned. - **Code**: `(r) => boolean | number`, sync or async. A number is 0..1 @@ -47,18 +47,18 @@ returned. `unsure (p=…)` in between. `{ question, yes, no }` sets its own bars. - **Helpers**: -| helper | passes when | -| ----------------------------- | -------------------------------------------------------------------------- | -| `matches(field?)` | output deep-equals expect, or the same field (key or accessor) of each | -| `fitsSchema(schema)` | output satisfies a Standard Schema (arktype, zod, valibot) | -| `mentions(words)` | output contains every word, case-insensitive; value is the fraction found | -| `avoids(words)` | output contains none of the words | -| `ranSteps(names)` | `tool` and `step` names match `names`, order ignored; value is their F1 | -| `stepsAtMost(n)` | the run took at most `n` steps | -| `under({ ms })` | the run finished within `ms` | -| `ask(q, { options, pass })` | Jev picks an option named in `pass` | -| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt` | -| `ask(q, { levels, pass })` | Jev grades at level `pass` or above (levels lowest first) | +| helper | passes when | +| ----------------------------- | ------------------------------------------------------------------------------ | +| `matches(field?)` | output deep-equals expectedOutput, or the same field (key or accessor) of each | +| `fitsSchema(schema)` | output satisfies a Standard Schema (arktype, zod, valibot) | +| `mentions(words)` | output contains every word, case-insensitive; value is the fraction found | +| `avoids(words)` | output contains none of the words | +| `ranSteps(names)` | `tool` and `step` names match `names`, order ignored; value is their F1 | +| `stepsAtMost(n)` | the run took at most `n` steps | +| `under({ ms })` | the run finished within `ms` | +| `ask(q, { options, pass })` | Jev picks an option named in `pass` | +| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt` | +| `ask(q, { levels, pass })` | Jev grades at level `pass` or above (levels lowest first) | `mentions` and `avoids` also take `(r) => words` for per-case words. `mentions` and `ranSteps` take `{ atLeast }` to pass on partial credit. @@ -68,7 +68,7 @@ returned. A check that throws, hangs past `timeout`, or returns anything else fails itself closed; its siblings still run. Jev transport failures fail closed. -Jev sees `{ input, output, expect, steps }`, and a case's string and +Jev sees `{ input, output, expectedOutput, steps }`, and a case's string and `ask` checks answer in one call. Configure it with `judge: jev({ endpoint, model, apiKey, timeout })`; `endpoint` is `"official"` (default), `"gateway"` or a URL. Without an `apiKey` the @@ -112,7 +112,7 @@ attempt passes. `concurrency` (default 4) bounds cases in flight; ## Cases -An array of `{ name?, input, expect?, must?, skip? }`, or the path of a +An array of `{ name?, input, expectedOutput?, must?, skip? }`, or the path of a JSONL file of the same objects. `name` keys the case in reports and compared runs and defaults to the input. @@ -165,7 +165,7 @@ verdicts agree, land unsure, or confidently disagree at each threshold: ```ts const table = await calibrate("refund-agent", { run: refundAgent, - cases: labeledCases, // { input, expect?, labels: { readyToSend: true } } + cases: labeledCases, // { input, expectedOutput?, labels: { readyToSend: true } } checks: { readyToSend: "Is this reply ready to send to the customer?" }, judge, }); diff --git a/e2e/evaluate.e2e.test.ts b/e2e/evaluate.e2e.test.ts index 3a58acb..be27513 100644 --- a/e2e/evaluate.e2e.test.ts +++ b/e2e/evaluate.e2e.test.ts @@ -39,8 +39,8 @@ describe("runs", () => { name: "upper", task: upper, data: [ - { input: "hi", expect: "HI" }, - { name: "yo", input: "yo", expect: "YO" }, + { input: "hi", expectedOutput: "HI" }, + { name: "yo", input: "yo", expectedOutput: "YO" }, ], }); expect(report.ok).toBe(true); @@ -60,7 +60,7 @@ describe("runs", () => { const report = await runEvaluation({ name: "agent", task: agent, - data: [{ input: "refund", expect: "re: refund" }], + data: [{ input: "refund", expectedOutput: "re: refund" }], }); expect(report.ok).toBe(true); expect(seen[0]).toBeInstanceOf(AbortSignal); @@ -86,7 +86,7 @@ describe("runs", () => { const report = await runEvaluation({ name: "objects", task: (input: { amount: number }) => ({ decision: "refund", amount: input.amount }), - data: [{ input: { amount: 5 }, expect: { decision: "refund", amount: 5 } }], + data: [{ input: { amount: 5 }, expectedOutput: { decision: "refund", amount: 5 } }], scorers: { whole: matches(), decision: matches("decision"), @@ -102,7 +102,7 @@ describe("runs", () => { const path = join(dir, "cases.jsonl"); await Bun.write( path, - '{"name":"a","input":"hi","expect":"HI"}\n\n{"input":"yo","expect":"YO"}\n', + '{"name":"a","input":"hi","expectedOutput":"HI"}\n\n{"input":"yo","expectedOutput":"YO"}\n', ); const report = await runEvaluation({ name: "jsonl", task: upper, data: path }); expect(report.cases.map((c) => c.name)).toEqual(["a", "yo"]); @@ -190,7 +190,7 @@ describe("checks", () => { const err = await failure( runEvaluation({ name: "no-expect", task: upper, data: [{ input: "hi" }] }), ); - expect(err.report.cases[0]?.checks["matches"]?.explanation).toBe("case has no expect"); + expect(err.report.cases[0]?.checks["matches"]?.explanation).toBe("case has no expectedOutput"); }); test("a throwing, hanging or malformed check fails itself closed, not its siblings", async () => { @@ -198,7 +198,7 @@ describe("checks", () => { runEvaluation({ name: "bad-checks", task: upper, - data: [{ input: "hi", expect: "HI" }], + data: [{ input: "hi", expectedOutput: "HI" }], scorers: { exact: matches(), boom: () => { @@ -273,7 +273,7 @@ describe("gates", () => { }); test("bad options are rejected up front", async () => { - const base = { task: upper, data: [{ input: "a", expect: "A" }] }; + const base = { task: upper, data: [{ input: "a", expectedOutput: "A" }] }; await expect(runEvaluation({ name: "o", ...base, concurrency: Number.NaN })).rejects.toThrow( "concurrency", ); @@ -293,7 +293,7 @@ describe("gates", () => { runEvaluation({ name: "md", task: upper, - data: [{ name: "a|b\nc", input: "", expect: "no" }], + data: [{ name: "a|b\nc", input: "", expectedOutput: "no" }], }), ); const md = err.report.markdown(); @@ -306,9 +306,9 @@ describe("gates", () => { describe("passing", () => { const cases = [ - { name: "a", input: "a", expect: "A" }, - { name: "b", input: "b", expect: "B" }, - { name: "c", input: "c", expect: "nope" }, + { name: "a", input: "a", expectedOutput: "A" }, + { name: "b", input: "b", expectedOutput: "B" }, + { name: "c", input: "c", expectedOutput: "nope" }, ]; const checks = { exact: matches(), short: (r: { output: string }) => r.output.length === 1 }; @@ -368,7 +368,7 @@ describe("passing", () => { scorers: { exact: (r) => { seen.push(r.input); - return r.output === r.expect; + return r.output === r.expectedOutput; }, }, }); @@ -382,7 +382,7 @@ describe("passing", () => { const path = join(await tempDir(), "cases.jsonl"); await Bun.write( path, - '{"name":"a","input":"a","expect":"A","must":["exact"]}\n{"name":"c","input":"c","expect":"nope","skip":["exact"]}\n', + '{"name":"a","input":"a","expectedOutput":"A","must":["exact"]}\n{"name":"c","input":"c","expectedOutput":"nope","skip":["exact"]}\n', ); const report = await runEvaluation({ name: "jsonl-must", @@ -429,9 +429,9 @@ describe("passing", () => { describe("compare and save", () => { const cases = [ - { name: "a", input: "a", expect: "A" }, - { name: "b", input: "b", expect: "B" }, - { name: "c", input: "c", expect: "nope" }, + { name: "a", input: "a", expectedOutput: "A" }, + { name: "b", input: "b", expectedOutput: "B" }, + { name: "c", input: "c", expectedOutput: "nope" }, ]; const allow = { matches: { miss: 2 } }; const swap = (input: string) => (input === "b" ? "?" : input === "c" ? "nope" : "A"); @@ -566,7 +566,7 @@ describe("compare and save", () => { test("added and removed cases are reported, not counted as drops", async () => { const before = await runEvaluation({ name: "upper", task: upper, data: cases, allow }); - const changed = [...cases.slice(1), { name: "d", input: "d", expect: "nope" }]; + const changed = [...cases.slice(1), { name: "d", input: "d", expectedOutput: "nope" }]; const report = await runEvaluation({ name: "upper", task: upper, diff --git a/e2e/jev.e2e.test.ts b/e2e/jev.e2e.test.ts index 4926f72..a37ab9b 100644 --- a/e2e/jev.e2e.test.ts +++ b/e2e/jev.e2e.test.ts @@ -50,7 +50,7 @@ async function judged(check: Scorer, judge: Judge) { report = await runEvaluation({ name: "jev", task: (input: string) => `reply to ${input}`, - data: [{ input: "refund?", expect: "yes" }], + data: [{ input: "refund?", expectedOutput: "yes" }], scorers: { judged: check }, judge, }); @@ -81,7 +81,7 @@ describe("sentence checks", () => { expect(bodies[0]?.["state"]).toEqual({ input: "refund?", output: "reply to refund?", - expect: "yes", + expectedOutput: "yes", }); expect(bodies[0]?.["questions"]).toEqual({ judged: { type: "noul", instructions: question } }); }); diff --git a/e2e/workflow.e2e.test.ts b/e2e/workflow.e2e.test.ts index 00724b0..060de90 100644 --- a/e2e/workflow.e2e.test.ts +++ b/e2e/workflow.e2e.test.ts @@ -39,7 +39,7 @@ describe("workflow runs", () => { const report = await runEvaluation({ name: "workflow", task: workflow(pipeline, { invokeStep }), - data: [{ input: "refund me", expect: "DRAFT: REFUND ME" }], + data: [{ input: "refund me", expectedOutput: "DRAFT: REFUND ME" }], scorers: { exact: matches(), order: (r) => r.steps.map((s) => s.name).join(">") === "drafter>reviewer", @@ -68,7 +68,10 @@ describe("workflow runs", () => { output: (run) => ({ draft: run.outputs["draft"], review: run.outputs["review"] }), }), data: [ - { input: { order: 42 }, expect: { draft: "draft: order 42", review: "DRAFT: ORDER 42" } }, + { + input: { order: 42 }, + expectedOutput: { draft: "draft: order 42", review: "DRAFT: ORDER 42" }, + }, ], }); expect(report.ok).toBe(true); @@ -84,7 +87,7 @@ describe("workflow runs", () => { return { output: "draft" }; }, }), - data: [{ input: "x", expect: "X" }], + data: [{ input: "x", expectedOutput: "X" }], }); throw new Error("expected a failure"); } catch (cause) { diff --git a/examples/ports/factuality.ts b/examples/ports/factuality.ts index c448467..011b3a6 100644 --- a/examples/ports/factuality.ts +++ b/examples/ports/factuality.ts @@ -77,11 +77,11 @@ const CASES: { id: string; question: string; expert: string; submission: string; const report = await runEvaluation({ name: "port-factuality", task: (question: string) => CASES.find((c) => c.question === question)?.submission ?? "", - data: CASES.map((c) => ({ name: c.id, input: c.question, expect: c.expert })), + data: CASES.map((c) => ({ name: c.id, input: c.question, expectedOutput: c.expert })), scorers: { baseline_exact: matches(), jev_factuality: ask( - "Compare the factual content of the output with the expert answer in expect. Ignore style, grammar, punctuation. Pick one option.", + "Compare the factual content of the output with the expert answer in expectedOutput. Ignore style, grammar, punctuation. Pick one option.", { options: CRITERIA, pass: ["A", "B", "C", "E"] }, ), }, diff --git a/examples/ports/keyword-coverage.ts b/examples/ports/keyword-coverage.ts index f87f75a..c0a0426 100644 --- a/examples/ports/keyword-coverage.ts +++ b/examples/ports/keyword-coverage.ts @@ -39,10 +39,10 @@ const CASES: { id: string; reference: string; output: string; truth: boolean }[] const report = await runEvaluation({ name: "port-keyword-coverage", task: (id: string) => CASES.find((c) => c.id === id)?.output ?? "", - data: CASES.map((c) => ({ input: c.id, expect: c.reference })), + data: CASES.map((c) => ({ input: c.id, expectedOutput: c.reference })), scorers: { baseline_exact: matches(), - port_coverage: (r) => coverage(r.expect ?? "", r.output) >= 0.5, + port_coverage: (r) => coverage(r.expectedOutput ?? "", r.output) >= 0.5, }, allow: { baseline_exact: CASES.length, port_coverage: CASES.length }, }); diff --git a/src/calibrate.ts b/src/calibrate.ts index 3afed93..6453f9d 100644 --- a/src/calibrate.ts +++ b/src/calibrate.ts @@ -11,7 +11,7 @@ import type { Outcome } from "./types.js"; export type Labeled = { name?: string; input: I; - expect?: E; + expectedOutput?: E; /** The expected verdict per check name. */ labels: Readonly>; }; @@ -115,7 +115,7 @@ export async function calibrate( const evalCase = { name: key, input: labeled.input, - ...(labeled.expect === undefined ? {} : { expect: labeled.expect }), + ...(labeled.expectedOutput === undefined ? {} : { expectedOutput: labeled.expectedOutput }), }; const runResult = await runOnce(spec, evalCase); const decisions = await decideBatch(options.judge, options.checks, runResult); diff --git a/src/checks.ts b/src/checks.ts index c1b0b3e..c13bc41 100644 --- a/src/checks.ts +++ b/src/checks.ts @@ -27,7 +27,7 @@ export function toOutcome(name: string, raw: unknown): Outcome { }; } -/** Output deep-equals expect, or one field of each (a key or an accessor). */ +/** Output deep-equals expectedOutput, or one field of each (a key or an accessor). */ export function matches( field?: string | ((value: T) => unknown), ): CheckFn { @@ -38,10 +38,11 @@ export function matches( ? (value as Record)[field] : undefined; }; - return ({ output, expect }) => { - if (expect === undefined) return { pass: false, value: 0, explanation: "case has no expect" }; + return ({ output, expectedOutput }) => { + if (expectedOutput === undefined) + return { pass: false, value: 0, explanation: "case has no expectedOutput" }; const got = get(output); - const want = get(expect); + const want = get(expectedOutput); if (Bun.deepEquals(got, want)) return true; return { pass: false, @@ -73,7 +74,7 @@ export function fitsSchema(schema: StandardSchema): CheckFn { }; } -/** Fixed words, or words picked per case (e.g. from `expect`). */ +/** Fixed words, or words picked per case (e.g. from `expectedOutput`). */ export type Words = | readonly string[] | ((run: Run) => readonly string[]); diff --git a/src/datasets.test.ts b/src/datasets.test.ts new file mode 100644 index 0000000..1ef9596 --- /dev/null +++ b/src/datasets.test.ts @@ -0,0 +1,11 @@ +import { expect, test } from "bun:test"; + +import { parseJsonl } from "./datasets.js"; + +test("parseJsonl loads expectedOutput and rejects expect or expected", () => { + expect(parseJsonl('{"input":"hi","expectedOutput":"HI"}\n')).toEqual([ + { input: "hi", expectedOutput: "HI" }, + ]); + expect(() => parseJsonl('{"input":"hi","expect":"HI"}\n')).toThrow("expectedOutput"); + expect(() => parseJsonl('{"input":"hi","expected":"HI"}\n')).toThrow("expectedOutput"); +}); diff --git a/src/datasets.ts b/src/datasets.ts index 9fc5aa7..057c08b 100644 --- a/src/datasets.ts +++ b/src/datasets.ts @@ -21,7 +21,10 @@ function toCase(row: Record, where: string): Case { if (!("input" in row)) throw new Error(`${where}: missing "input"`); const evalCase: Case = { input: row["input"] }; if (name !== undefined) evalCase.name = name; - if (row["expect"] !== undefined) evalCase.expect = row["expect"]; + if (row["expect"] !== undefined || row["expected"] !== undefined) { + throw new Error(`${where}: use "expectedOutput", not "expect" or "expected"`); + } + if (row["expectedOutput"] !== undefined) evalCase.expectedOutput = row["expectedOutput"]; const must = checkNames(row, "must", where); if (must !== undefined) evalCase.must = must; const skip = checkNames(row, "skip", where); diff --git a/src/jev.ts b/src/jev.ts index 7147092..d07f4f4 100644 --- a/src/jev.ts +++ b/src/jev.ts @@ -328,7 +328,7 @@ export function toOutcome( function toState(run: Run): Record { const state: Record = { input: run.input, output: run.output }; - if (run.expect !== undefined) state["expect"] = run.expect; + if (run.expectedOutput !== undefined) state["expectedOutput"] = run.expectedOutput; if (run.steps.length > 0) state["steps"] = run.steps; return state; } diff --git a/src/runner.ts b/src/runner.ts index 27d690e..798eff8 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -4,7 +4,12 @@ import type { Trace } from "./runs.js"; import { withTimeout } from "./timeout.js"; import type { CaseResult, Outcome, Run } from "./types.js"; -export type NamedCase = { name: string; input: I; expect?: E; skip?: readonly string[] }; +export type NamedCase = { + name: string; + input: I; + expectedOutput?: E; + skip?: readonly string[]; +}; /** String and `ask` checks Jev answers in one call per attempt. */ export type JevGroup = { judge: Judge; checks: Record }; @@ -49,7 +54,7 @@ export async function runOnce( steps: trace.steps, ms: Date.now() - started, }; - if (evalCase.expect !== undefined) run.expect = evalCase.expect; + if (evalCase.expectedOutput !== undefined) run.expectedOutput = evalCase.expectedOutput; return run; } finally { clearTimeout(timer); @@ -168,7 +173,7 @@ async function runCase( pass: error === undefined && Object.values(checks).every((o) => o.pass), checks, }; - if (evalCase.expect !== undefined) result.expect = evalCase.expect; + if (evalCase.expectedOutput !== undefined) result.expectedOutput = evalCase.expectedOutput; if (last !== undefined) result.output = last.output; if (error !== undefined) result.error = error; return result; diff --git a/src/types.ts b/src/types.ts index 756c72b..10079c3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -12,7 +12,7 @@ export type Case = { /** Keys the case in reports and compared runs. Defaults to the input. */ name?: string; input: I; - expect?: E; + expectedOutput?: E; /** Checks that must pass on this case, whatever `allow` says. */ must?: readonly string[]; /** Checks this case is not run or counted for. */ @@ -23,7 +23,7 @@ export type Case = { export type Run = { input: I; output: O; - expect?: E; + expectedOutput?: E; steps: Step[]; ms: number; }; @@ -51,7 +51,7 @@ export type CheckFn = ( export type CaseResult = { name: string; input: I; - expect?: E; + expectedOutput?: E; output?: O; steps: Step[]; /** Whole case: every attempt's run plus checks. */ From 014d4f469f20771e6afbbd809c2c0c4e0e77a258 Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 02:00:03 -0700 Subject: [PATCH 03/16] feat!: run the judge as one workflow step jev() keeps thresholds and escalation only. judgeModel, exported from the workflow entry, forces a single step whose input is the case payload. --- README.md | 21 ++-- e2e/calibrate.e2e.test.ts | 65 ++++++------- e2e/jev.e2e.test.ts | 199 +++++++++++++++++++++++++++++--------- examples/ollama.ts | 61 ++++++++++-- src/calibrate.ts | 3 +- src/evaluation.ts | 13 ++- src/jev.ts | 128 ++++++++++++++---------- src/workflow.ts | 119 +++++++++++++++++++++++ 8 files changed, 457 insertions(+), 152 deletions(-) diff --git a/README.md b/README.md index eb1b167..de6bfb8 100644 --- a/README.md +++ b/README.md @@ -69,20 +69,22 @@ A check that throws, hangs past `timeout`, or returns anything else fails itself closed; its siblings still run. Jev transport failures fail closed. Jev sees `{ input, output, expectedOutput, steps }`, and a case's string and -`ask` checks answer in one call. Configure it with -`judge: jev({ endpoint, model, apiKey, timeout })`; `endpoint` is -`"official"` (default), `"gateway"` or a URL. Without an `apiKey` the -key comes from the environment as `@corbits/system-one` reads it. +`ask` checks answer in one call. The model is not configured on `jev()`. +Build it with `judgeModel({ model, invokeStep, thresholds, escalate, timeout })` +from `@corbits/evals/workflow`. `model` is one workflow step (or a one-step +workflow). `invokeStep` runs that step; its input is `{ state, questions }` +and its output is `{ decisions }` or `{ fallback: true, reason }`. +`jev({ thresholds, escalate })` only sets bars and an escalation agent. Thresholds: a yes/no check passes at p(yes) >= `yes` and fails at p(yes) <= `no` (0.7 / 0.3); `ask` fails as unsure when its top probability is under `sure` (0.6). Retune them per judge with -`jev({ thresholds: { yes, no, sure } })` or per check as above. -Thresholds are tuned to a judge version, so pin `model` — e.g. -`jev({ model: "jev-1.13.0" })` — when they must stay stable. +`judgeModel({ thresholds: { yes, no, sure }, ... })` or per check as above. +Thresholds are tuned to a judge version, so pin the step agent's model +when they must stay stable. Escalation: `sure` doubles as the escalation gate. Give the judge an -agent — `jev({ escalate: agent })`, anything with +agent — `escalate` on `jev` or `judgeModel`, anything with `send(prompt)` returning `{ type: "reply", reply }` or `{ type: "suspended" }`, e.g. an `@intx/agent` agent from a `@corbits/*` adapter — and every unsure decision is re-asked with the exact answer @@ -104,7 +106,8 @@ space cannot absorb — those fail closed by design. definition is refused. Each step lands in `r.steps`. `output` picks the result from the finished run (default: the last listed step's output), `trigger` maps the input to the trigger payload. Without them, type the - run as `workflow(...)`. + run as `workflow(...)`. The same entry exports `judgeModel` for + string and `ask` scorers. A code-only eval needs no judge. `repeats` runs each case several times; a check passes only if every attempt passes. `concurrency` (default 4) bounds cases in flight; diff --git a/e2e/calibrate.e2e.test.ts b/e2e/calibrate.e2e.test.ts index 7f95eb6..86a7ce3 100644 --- a/e2e/calibrate.e2e.test.ts +++ b/e2e/calibrate.e2e.test.ts @@ -1,48 +1,43 @@ import { describe, expect, test } from "bun:test"; -import type { EvaluateDeps } from "@corbits/system-one"; +import { defineAgent } from "@intx/agent"; +import { step, type StepInvoker } from "@intx/workflow"; -import { ask, calibrate, jev } from "../src/index.js"; - -function stubDeps(fetch: EvaluateDeps["fetch"]): EvaluateDeps { - return { - fetch, - scheduler: { - setTimeout: ((fn: () => void, ms: number) => { - const t = setTimeout(fn, ms); - return () => clearTimeout(t); - }) as EvaluateDeps["scheduler"]["setTimeout"], - now: () => Date.now(), - }, - }; -} +import { ask, calibrate } from "../src/index.js"; +import { judgeModel } from "../src/workflow.js"; /** Answers yes/no questions with the input's probability; choice questions pick ready/revise by it. */ function stubCalibration(byInput: Record) { - const fetch = (async (_url: string, init: RequestInit) => { - const body = JSON.parse(String(init.body)) as { + const invokeStep: StepInvoker = async (request) => { + const body = request.input as { state: { input: string }; questions: Record; }; const p = byInput[body.state.input] ?? 0.5; - const answers = Object.fromEntries( - Object.entries(body.questions).map(([id, q]) => [ - id, - q.type === "choice" - ? { - type: "choice", - choice: p >= 0.5 ? "ready" : "revise", - probabilities: { ready: p, revise: Math.round((1 - p) * 100) / 100 }, - confidence: p, - } - : { type: "noul", noul: p }, - ]), + const decisions = Object.entries(body.questions).map(([id, q]) => + q.type === "choice" + ? { + id, + type: "choice", + choice: p >= 0.5 ? "ready" : "revise", + probabilities: { ready: p, revise: Math.round((1 - p) * 100) / 100 }, + confidence: p, + } + : { id, type: "noul", noul: p }, ); - return new Response(JSON.stringify({ model: "jev-test", answers }), { - status: 200, - headers: { "content-type": "application/json" }, - }); - }) as unknown as EvaluateDeps["fetch"]; - return jev({ apiKey: "test-key", deps: stubDeps(fetch) }); + return { output: { decisions } }; + }; + return judgeModel({ + model: step({ + agent: defineAgent({ + id: "jev", + systemPrompt: "judge", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "stub", model: "stub" }] }, + }), + }), + invokeStep, + }); } describe("calibrate", () => { diff --git a/e2e/jev.e2e.test.ts b/e2e/jev.e2e.test.ts index a37ab9b..12c8dcd 100644 --- a/e2e/jev.e2e.test.ts +++ b/e2e/jev.e2e.test.ts @@ -2,7 +2,8 @@ import { describe, expect, test } from "bun:test"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import type { EvaluateDeps } from "@corbits/system-one"; +import { defineAgent } from "@intx/agent"; +import { defineWorkflow, step, type StepInvoker, type WorkflowDefinition } from "@intx/workflow"; import { ask, @@ -15,33 +16,37 @@ import { type Outcome, type Report, } from "../src/index.js"; - -function stubDeps(fetch: EvaluateDeps["fetch"]): EvaluateDeps { - return { - fetch, - scheduler: { - setTimeout: ((fn: () => void, ms: number) => { - const t = setTimeout(fn, ms); - return () => clearTimeout(t); - }) as EvaluateDeps["scheduler"]["setTimeout"], - now: () => Date.now(), - }, - }; +import { judgeModel } from "../src/workflow.js"; + +function judgeAgent(sources: readonly { provider: string; model: string }[]) { + return defineAgent({ + id: "jev", + systemPrompt: "judge", + tools: [], + capabilities: [], + inference: { sources }, + }); } -/** Answers every question on a POST with `answer` and records the request bodies. */ -function stubJev(answer: unknown) { +/** Answers every question with `answer` and records the step request. */ +function stubJev( + answer: unknown, + sources: readonly { provider: string; model: string }[] = [{ provider: "stub", model: "stub" }], +) { const bodies: Record[] = []; - const fetch = (async (_url: string, init: RequestInit) => { - const body = JSON.parse(String(init.body)) as { questions: Record }; - bodies.push(body); - const answers = Object.fromEntries(Object.keys(body.questions).map((id) => [id, answer])); - return new Response(JSON.stringify({ model: "jev-test", answers }), { - status: 200, - headers: { "content-type": "application/json" }, - }); - }) as unknown as EvaluateDeps["fetch"]; - return { bodies, judge: jev({ apiKey: "test-key", deps: stubDeps(fetch) }) }; + const seen: { provider: string; model: string }[][] = []; + const invokeStep: StepInvoker = async (request) => { + const input = request.input as { questions: Record }; + bodies.push(input); + seen.push(request.agent.inference.sources.map((source) => ({ ...source }))); + const decisions = Object.keys(input.questions).map((id) => ({ id, ...(answer as object) })); + return { output: { decisions } }; + }; + return { + bodies, + seen, + judge: judgeModel({ model: step({ agent: judgeAgent(sources) }), invokeStep }), + }; } async function judged(check: Scorer, judge: Judge) { @@ -101,16 +106,20 @@ describe("sentence checks", () => { }); }); - test("transport failure fails closed", async () => { - const failing = (async () => { + test("a thrown invoke fails closed", async () => { + const failing: StepInvoker = async () => { throw new TypeError("boom"); - }) as unknown as EvaluateDeps["fetch"]; + }; const { outcome } = await judged( question, - jev({ apiKey: "test-key", deps: stubDeps(failing) }), + judgeModel({ + model: step({ agent: judgeAgent([{ provider: "stub", model: "stub" }]) }), + invokeStep: failing, + }), ); expect(outcome).toMatchObject({ pass: false, value: 0 }); expect(outcome?.explanation).toContain("jev fallback"); + expect(outcome?.explanation).toContain("boom"); }); }); @@ -138,10 +147,10 @@ describe("batching", () => { }); }); - test("a transport failure fails every Jev check in the batch", async () => { - const failing = (async () => { + test("a thrown invoke fails every Jev check in the batch", async () => { + const failing: StepInvoker = async () => { throw new TypeError("boom"); - }) as unknown as EvaluateDeps["fetch"]; + }; let report: Report; try { await runEvaluation({ @@ -149,7 +158,10 @@ describe("batching", () => { task: (input: string) => input, data: [{ input: "a" }], scorers: { ready: "Ready to send?", polite: "Is the tone polite?" }, - judge: jev({ apiKey: "test-key", deps: stubDeps(failing) }), + judge: judgeModel({ + model: step({ agent: judgeAgent([{ provider: "stub", model: "stub" }]) }), + invokeStep: failing, + }), }); throw new Error("expected failure"); } catch (cause) { @@ -180,24 +192,21 @@ describe("thresholds", () => { const question = "Is this reply ready to send to the customer?"; const noul = (p: number) => ({ type: "noul", noul: p }); - function flatFetch(answer: unknown) { - return (async (_url: string, init: RequestInit) => { - const body = JSON.parse(String(init.body)) as { questions: Record }; - const answers = Object.fromEntries(Object.keys(body.questions).map((id) => [id, answer])); - return new Response(JSON.stringify({ model: "jev-test", answers }), { - status: 200, - headers: { "content-type": "application/json" }, - }); - }) as unknown as EvaluateDeps["fetch"]; + function answering(answer: unknown): StepInvoker { + return async (request) => { + const questions = (request.input as { questions: Record }).questions; + const decisions = Object.keys(questions).map((id) => ({ id, ...(answer as object) })); + return { output: { decisions } }; + }; } test("judge thresholds retune every check's bars", async () => { const { judge } = stubJev(noul(0.75)); expect((await judged(question, judge)).outcome?.pass).toBe(true); - const strict = jev({ - apiKey: "test-key", + const strict = judgeModel({ + model: step({ agent: judgeAgent([{ provider: "stub", model: "stub" }]) }), thresholds: { yes: 0.8 }, - deps: stubDeps(flatFetch(noul(0.75))), + invokeStep: answering(noul(0.75)), }); expect((await judged(question, strict)).outcome).toMatchObject({ pass: false, @@ -545,3 +554,103 @@ describe("stored reports", () => { } }); }); + +describe("judgeModel", () => { + const agent = judgeAgent([{ provider: "stub", model: "stub" }]); + const invokeStep: StepInvoker = async () => ({ output: { decisions: [] } }); + + test("swapping inference sources does not change the judge payload", async () => { + const alpha = [{ provider: "alpha", model: "m1" }]; + const beta = [{ provider: "beta", model: "m2" }]; + const a = stubJev({ type: "noul", noul: 0.9 }, alpha); + const b = stubJev({ type: "noul", noul: 0.9 }, beta); + const question = "Is this reply ready to send to the customer?"; + await judged(question, a.judge); + await judged(question, b.judge); + expect(a.bodies).toHaveLength(1); + expect(a.bodies[0]).toEqual(b.bodies[0]); + expect(a.seen[0]).toEqual(alpha); + expect(b.seen[0]).toEqual(beta); + expect(a.seen[0]).not.toEqual(b.seen[0]); + }); + + test("string and ask checks throw before cases without judge.invoke", async () => { + await expect( + runEvaluation({ + name: "missing-judge", + task: (input: string) => input, + data: "/no/such/cases.jsonl", + scorers: { ready: "Ready?" }, + }), + ).rejects.toThrow("judge.invoke"); + await expect( + runEvaluation({ + name: "thresholds-only", + task: (input: string) => input, + data: "/no/such/cases.jsonl", + scorers: { ready: ask("Ready?", { options: { a: "a" }, pass: "a" }) }, + judge: jev({ thresholds: { yes: 0.9 } }), + }), + ).rejects.toThrow("judge.invoke"); + const report = await runEvaluation({ + name: "code-only", + task: (input: string) => input, + data: [{ input: "a", expectedOutput: "a" }], + }); + expect(report.ok).toBe(true); + }); + + test("rejects a model that is not one plain step", () => { + expect(() => judgeModel({ model: step({ agent, triggers: 2 }), invokeStep })).toThrow( + "triggers must be 1", + ); + expect(() => + judgeModel({ + model: step({ agent, input: { from: "trigger.payload" } }), + invokeStep, + }), + ).toThrow("must not set input"); + expect(() => + judgeModel({ + model: step({ agent, inference: { literal: null } }), + invokeStep, + }), + ).toThrow("must not set inference"); + const two = defineWorkflow({ + id: "two", + trigger: { type: "manual" }, + steps: { + a: step({ agent }), + b: step({ agent, after: ["a"] }), + }, + }); + expect(() => judgeModel({ model: two, invokeStep })).toThrow("exactly one step"); + const empty = { + id: "empty", + triggers: [{ type: "manual" }], + steps: {}, + stepOrder: [], + } as WorkflowDefinition; + expect(() => judgeModel({ model: empty, invokeStep })).toThrow("no steps"); + const gated = { + id: "gated", + triggers: [{ type: "manual" }], + steps: { g: { id: "g", kind: "gate" } }, + stepOrder: ["g"], + } as unknown as WorkflowDefinition; + expect(() => judgeModel({ model: gated, invokeStep })).toThrow("must be a step"); + const twoTriggers = { + id: "twice", + triggers: [{ type: "manual" }, { type: "manual" }], + steps: { only: { id: "only", kind: "step", agent } }, + stepOrder: ["only"], + } as unknown as WorkflowDefinition; + expect(() => judgeModel({ model: twoTriggers, invokeStep })).toThrow("exactly one trigger"); + expect(() => + judgeModel({ + model: step({ agent }), + invokeStep: undefined as unknown as StepInvoker, + }), + ).toThrow("needs invokeStep"); + }); +}); diff --git a/examples/ollama.ts b/examples/ollama.ts index 3db5511..f3738c4 100644 --- a/examples/ollama.ts +++ b/examples/ollama.ts @@ -1,12 +1,59 @@ -// Shared local-Ollama wiring for the examples. The dummy key works -// around @corbits/system-one fail-closing on keyless custom endpoints. -import { jev } from "../src/index.js"; +// Shared local-Ollama wiring for the examples. The judge is a workflow step; +// this file's invoker turns the step input into a Jev reply. No provider SDK. +import { defineAgent } from "@intx/agent"; +import { step } from "@intx/workflow"; -export const localJev = jev({ - endpoint: "http://localhost:11434/v1/systemone", - model: "clef-flash:9b", - apiKey: "ollama-local", +import { judgeModel } from "../src/workflow.js"; + +const JUDGE_URL = "http://localhost:11434/api/chat"; + +export const localJev = judgeModel({ + model: step({ + agent: defineAgent({ + id: "local-jev", + systemPrompt: "Answer the Jev questions in the input.", + tools: [], + capabilities: [], + inference: { sources: [{ provider: "ollama", model: "clef-flash:9b" }] }, + }), + }), timeout: "180s", + invokeStep: async ({ input, agent }) => { + const model = agent.inference.sources[0]?.model ?? "clef-flash:9b"; + const res = await fetch(JUDGE_URL, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model, + stream: false, + messages: [ + { + role: "user", + content: `Reply with JSON {"decisions":[...]} for this Jev payload:\n${JSON.stringify(input)}`, + }, + ], + }), + }); + if (!res.ok) { + return { output: { fallback: true, reason: "http-error", detail: `HTTP ${res.status}` } }; + } + const data = (await res.json()) as { message?: { content?: unknown } }; + const content = data.message?.content; + if (typeof content !== "string") { + return { output: { fallback: true, reason: "parse-error", detail: "no reply text" } }; + } + try { + return { output: JSON.parse(content) as unknown }; + } catch (cause) { + return { + output: { + fallback: true, + reason: "parse-error", + detail: cause instanceof Error ? cause.message : String(cause), + }, + }; + } + }, }); export async function ollamaGenerate(model: string, prompt: string): Promise { diff --git a/src/calibrate.ts b/src/calibrate.ts index 6453f9d..f55211f 100644 --- a/src/calibrate.ts +++ b/src/calibrate.ts @@ -22,7 +22,7 @@ export type CalibrateOptions = { checks: Record; judge: Judge; concurrency?: number; - /** Bounds each case run. The Jev call is bounded by `judge.timeout`. */ + /** Bounds each case run. The judge step is bounded by its own timeout. */ timeout?: Duration; }; @@ -140,7 +140,6 @@ export async function calibrate( const rows: CalibrationRow[] = []; for (const threshold of GRID) { const judge: Judge = { - config: {}, thresholds: isQuestion ? { yes: base.yes, no: base.no, sure: threshold } : { yes: threshold, no: r3(1 - threshold), sure: base.sure }, diff --git a/src/evaluation.ts b/src/evaluation.ts index e88d040..26666aa 100644 --- a/src/evaluation.ts +++ b/src/evaluation.ts @@ -1,6 +1,6 @@ import { matches } from "./checks.js"; import { fromJsonl } from "./datasets.js"; -import { checkThresholds, jev, type Ask, type Judge, type Sentence } from "./jev.js"; +import { checkThresholds, type Ask, type Judge, type Sentence } from "./jev.js"; import { flips, fromReport, @@ -47,7 +47,7 @@ export type Evaluation = { timeout?: Duration; /** Stamped into the experiment, e.g. `{ model }`. */ about?: Record; - /** Jev for string and `ask` scorers. Defaults to `jev()`. */ + /** Jev for string and `ask` scorers. Those checks need `judge.invoke` from `judgeModel`. */ judge?: Judge; /** Skip the console summary. */ quiet?: boolean; @@ -271,11 +271,16 @@ export async function runEvaluation(spec: Evaluation): Promise let judge: Judge | undefined = spec.judge; const defined = spec.scorers ?? { matches: matches() as Scorer }; const { code, jev: jevChecks } = toCheckParts(name, defined); + if (Object.keys(jevChecks).length > 0 && judge?.invoke === undefined) { + throw new Error( + `evaluation "${name}": string and ask checks need judge.invoke from judgeModel`, + ); + } const checkNames = Object.keys(defined); const jevGroup = - Object.keys(jevChecks).length === 0 + Object.keys(jevChecks).length === 0 || judge === undefined ? undefined - : { judge: (judge ??= jev()), checks: jevChecks }; + : { judge, checks: jevChecks }; const allowances = toAllowances(name, spec.allow ?? {}, checkNames); const cases = named( name, diff --git a/src/jev.ts b/src/jev.ts index d07f4f4..107a081 100644 --- a/src/jev.ts +++ b/src/jev.ts @@ -1,29 +1,12 @@ -import { - evaluate, - type Decision, - type EndpointConfig, - type EvaluateConfig, - type EvaluateDeps, - type Question, -} from "@corbits/system-one"; - -import { toMs, type Duration } from "./timeout.js"; +import type { Decision, Question } from "@corbits/system-one"; + import type { Outcome, Run } from "./types.js"; export type JevOptions = { - /** "official" (default), "gateway", or the URL of a custom endpoint. */ - endpoint?: string; - model?: string; - /** Defaults to the endpoint's environment key, as `@corbits/system-one` reads it. */ - apiKey?: string; - /** Bounds one Jev call. `@corbits/system-one` defaults to 1500ms. */ - timeout?: Duration; /** Pass bars for every Jev check through this judge. */ thresholds?: Thresholds; /** Consulted when a decision lands unsure. `sure` doubles as the escalation gate. */ escalate?: Escalation; - /** Replaces `fetch` and the scheduler, e.g. to stub Jev in tests. */ - deps?: EvaluateDeps; }; /** Yes/no passes at p >= `yes` and fails at p <= `no`; `ask` is unsure under `sure`. */ @@ -34,11 +17,23 @@ export type Resolved = { yes: number; no: number; sure: number }; /** The one documented set of numbers, tuned per workload with `calibrate`. */ export const DEFAULTS: Resolved = { yes: 0.7, no: 0.3, sure: 0.6 }; +/** What one judge call sends. Questions are id-keyed; boolean checks are `noul`. */ +export type JudgePayload = { + readonly state: Record; + readonly questions: Readonly< + Record + >; +}; + export type Judge = { - readonly config: EvaluateConfig; readonly thresholds?: Resolved; readonly escalate?: Escalation; - readonly deps?: EvaluateDeps; + /** + * One call per case attempt. `judgeModel` sets this; `jev()` alone does not, + * so it is not a model. + * Resolves to `{ decisions }` or `{ fallback: true, reason, detail? }`. + */ + readonly invoke?: (payload: JudgePayload) => Promise; }; /** @@ -68,27 +63,12 @@ function resolve(label: string, raw: Thresholds): Resolved { return out; } -function toEndpoint(endpoint?: string, model?: string): EndpointConfig | undefined { - const named = model === undefined ? {} : { model }; - if (endpoint === undefined) - return model === undefined ? undefined : { kind: "official", ...named }; - if (endpoint === "official" || endpoint === "gateway") return { kind: endpoint, ...named }; - return { kind: "custom", url: endpoint, ...named }; -} - export function jev(options: JevOptions = {}): Judge { - const config: EvaluateConfig = {}; - const endpoint = toEndpoint(options.endpoint, options.model); - if (endpoint !== undefined) config.endpoint = endpoint; - if (options.apiKey !== undefined) config.apiKey = options.apiKey; - if (options.timeout !== undefined) config.timeoutMs = toMs(options.timeout, "jev timeout"); const thresholds = options.thresholds === undefined ? undefined : resolve("thresholds", options.thresholds); return { - config, ...(thresholds === undefined ? {} : { thresholds }), ...(options.escalate === undefined ? {} : { escalate: options.escalate }), - ...(options.deps === undefined ? {} : { deps: options.deps }), }; } @@ -333,6 +313,59 @@ function toState(run: Run): Record { return state; } +function wireQuestions(questions: readonly Question[]): JudgePayload["questions"] { + const wire: Record< + string, + { type: string; instructions: Question["instructions"]; criteria?: unknown } + > = {}; + for (const question of questions) { + const mapped: { type: string; instructions: Question["instructions"]; criteria?: unknown } = { + type: question.type === "boolean" ? "noul" : question.type, + instructions: question.instructions, + }; + if ("criteria" in question && question.criteria !== undefined) + mapped.criteria = question.criteria; + wire[question.id] = mapped; + } + return wire; +} + +type JudgeReply = { fallback: true; reason: string; detail?: string } | { decisions: Decision[] }; + +function readReply(out: unknown): JudgeReply | undefined { + if (typeof out !== "object" || out === null) return undefined; + const record = out as Record; + if (record["fallback"] === true) { + const reason = typeof record["reason"] === "string" ? record["reason"] : "parse-error"; + const detail = typeof record["detail"] === "string" ? record["detail"] : undefined; + return { fallback: true, reason, ...(detail === undefined ? {} : { detail }) }; + } + if (Array.isArray(record["decisions"])) return { decisions: record["decisions"] as Decision[] }; + return undefined; +} + +/** Fail closed: a thrown invoke is a transport fallback, same as a typed one. */ +async function callJudge(judge: Judge, questions: Question[], run: Run): Promise { + if (judge.invoke === undefined) throw new Error("jev: judge has no invoke"); + let out: unknown; + try { + out = await judge.invoke({ state: toState(run), questions: wireQuestions(questions) }); + } catch (cause) { + return { + fallback: true, + reason: "network", + detail: cause instanceof Error ? cause.message : String(cause), + }; + } + return ( + readReply(out) ?? { + fallback: true, + reason: "parse-error", + detail: "judge output was not decisions or a fallback", + } + ); +} + /** * Answer every check in one Jev call, keyed by check name. Fail-closed: * transport fallbacks, thrown errors and missing decisions all fail — a @@ -346,16 +379,13 @@ export async function askJevBatch( ): Promise> { const names = Object.keys(checks); const questions = names.map((id) => toQuestion(id, checks[id] as Ask | Sentence | string)); - let out; + let out: JudgeReply; try { - out = await evaluate( - { state: toState(run), questions, config: judge.config }, - judge.deps === undefined ? undefined : { deps: judge.deps }, - ); + out = await callJudge(judge, questions, run); } catch (cause) { return failedAll(names, `jev threw: ${cause instanceof Error ? cause.message : String(cause)}`); } - if (out.fallback) { + if ("fallback" in out) { return failedAll(names, `jev fallback (${out.reason}): ${out.detail ?? "no detail"}`); } const decisions = new Map(out.decisions.map((decision) => [decision.id, decision])); @@ -387,15 +417,13 @@ export async function decideBatch( run: Run, ): Promise> { const names = Object.keys(checks); - const out = await evaluate( - { - state: toState(run), - questions: names.map((id) => toQuestion(id, checks[id] as Ask | Sentence | string)), - config: judge.config, - }, - judge.deps === undefined ? undefined : { deps: judge.deps }, + const out = await callJudge( + judge, + names.map((id) => toQuestion(id, checks[id] as Ask | Sentence | string)), + run, ); - if (out.fallback) throw new Error(`jev fallback (${out.reason}): ${out.detail ?? "no detail"}`); + if ("fallback" in out) + throw new Error(`jev fallback (${out.reason}): ${out.detail ?? "no detail"}`); return Object.fromEntries(out.decisions.map((decision) => [decision.id, decision])); } diff --git a/src/workflow.ts b/src/workflow.ts index 55e5c20..5d9d915 100644 --- a/src/workflow.ts +++ b/src/workflow.ts @@ -1,13 +1,17 @@ import { + defineWorkflow, runLocal, type RunLocalOptions, type RunResult, type StepInvoker, + type StepPrimitive, type WorkflowAuthorizeFn, type WorkflowDefinition, } from "@intx/workflow"; import { TRACE, type Traced } from "./runs.js"; +import { jev, type Escalation, type Judge, type Thresholds } from "./jev.js"; +import { toMs, type Duration } from "./timeout.js"; import type { Step } from "./types.js"; const allowAll: WorkflowAuthorizeFn = async () => ({ @@ -82,3 +86,118 @@ export function workflow( }, }; } + +export type JudgeModelOptions = { + /** A step, or a workflow whose only primitive is that step. */ + model: StepPrimitive | WorkflowDefinition; + /** Runs the judge step. Required: a definition alone cannot call a model. */ + invokeStep: StepInvoker; + thresholds?: Thresholds; + escalate?: Escalation; + /** Bounds the judge step. */ + timeout?: Duration; +}; + +function isWorkflow(model: StepPrimitive | WorkflowDefinition): model is WorkflowDefinition { + return "steps" in model && "stepOrder" in model; +} + +function assertJudgeStep(step: StepPrimitive): void { + if (step.kind !== "step") throw new Error("judgeModel: the only primitive must be a step"); + if (step.input !== undefined) throw new Error("judgeModel: step must not set input"); + if (step.inference !== undefined) throw new Error("judgeModel: step must not set inference"); + if (step.triggers !== undefined && step.triggers !== 1) { + throw new Error("judgeModel: step triggers must be 1"); + } +} + +function takeJudgeStep(model: StepPrimitive | WorkflowDefinition): StepPrimitive { + if (!isWorkflow(model)) { + if (!("kind" in model) || model.kind !== "step") { + throw new Error("judgeModel: model must be a step or a one-step workflow"); + } + assertJudgeStep(model); + return model; + } + if (model.triggers.length !== 1) + throw new Error("judgeModel: workflow must have exactly one trigger"); + if (model.stepOrder.length === 0) throw new Error("judgeModel: workflow has no steps"); + if (model.stepOrder.length !== 1) + throw new Error("judgeModel: workflow must have exactly one step"); + const id = model.stepOrder[0] as string; + const primitive = model.steps[id]; + if (primitive === undefined || primitive.kind !== "step") { + throw new Error("judgeModel: the only primitive must be a step"); + } + assertJudgeStep(primitive); + return primitive; +} + +/** + * A Jev judge whose model is one workflow step. The step input is forced to + * the trigger payload `{ state, questions }`, so the agent's inference sources + * never enter the request. One `invokeStep` call per case attempt. The step + * output is the judge reply: `{ decisions }` or `{ fallback: true, reason }`. + */ +export function judgeModel(options: JudgeModelOptions): Judge { + if (options.invokeStep === undefined) throw new Error("judgeModel needs invokeStep"); + const primitive = takeJudgeStep(options.model); + const timeoutMs = + options.timeout === undefined ? undefined : toMs(options.timeout, "judgeModel timeout"); + const forced: StepPrimitive = { + ...primitive, + id: "", + input: { from: "trigger.payload" }, + ...(timeoutMs === undefined ? {} : { timeout: timeoutMs }), + }; + const definition = defineWorkflow({ + id: "jev-judge", + trigger: { type: "manual" }, + steps: { judge: forced }, + }); + const base = jev({ + ...(options.thresholds === undefined ? {} : { thresholds: options.thresholds }), + ...(options.escalate === undefined ? {} : { escalate: options.escalate }), + }); + return { + ...base, + invoke: async (payload) => { + let calls = 0; + const run = runLocal(definition, { + authorize: allowAll, + hasUpstreamSignalResolver: true, + triggerPayload: payload, + invokeStep: async (request) => { + calls += 1; + if (calls > 1) throw new Error("judgeModel: more than one invoke"); + try { + const result = await options.invokeStep(request); + if ("suspend" in result) { + return { + output: { fallback: true, reason: "network", detail: "judge step suspended" }, + }; + } + return result; + } catch (cause) { + return { + output: { + fallback: true, + reason: "network", + detail: cause instanceof Error ? cause.message : String(cause), + }, + }; + } + }, + }); + const finished = await run.complete; + if (finished.terminalStatus !== "completed") { + return { + fallback: true, + reason: "network", + detail: `judge step ended ${finished.terminalStatus}`, + }; + } + return finished.outputs["judge"]; + }, + }; +} From 37fad73e5204da3cb43b10275a5b25b214760f48 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 26 Oct 2026 20:48:34 -0700 Subject: [PATCH 04/16] fix: accept a one-step judge workflow's default input defineWorkflow stamps a lone step with { from: "trigger.payload" } even when the author omits input. judgeModel treated that stamp as a custom input and never ran the step. --- e2e/jev.e2e.test.ts | 61 +++++++++++++++++++++++++++++++++++++++++++-- src/workflow.ts | 12 ++++++++- 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/e2e/jev.e2e.test.ts b/e2e/jev.e2e.test.ts index 12c8dcd..65328a2 100644 --- a/e2e/jev.e2e.test.ts +++ b/e2e/jev.e2e.test.ts @@ -3,7 +3,14 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { defineAgent } from "@intx/agent"; -import { defineWorkflow, step, type StepInvoker, type WorkflowDefinition } from "@intx/workflow"; +import { + childWorkflow, + defineWorkflow, + map, + step, + type StepInvoker, + type WorkflowDefinition, +} from "@intx/workflow"; import { ask, @@ -606,7 +613,16 @@ describe("judgeModel", () => { ); expect(() => judgeModel({ - model: step({ agent, input: { from: "trigger.payload" } }), + model: step({ agent, input: { from: "steps.other.output" } }), + invokeStep, + }), + ).toThrow("must not set input"); + expect(() => + judgeModel({ + model: defineWorkflow({ + id: "custom-input", + steps: { only: step({ agent, input: { literal: "fixed" } }) }, + }), invokeStep, }), ).toThrow("must not set input"); @@ -625,6 +641,20 @@ describe("judgeModel", () => { }, }); expect(() => judgeModel({ model: two, invokeStep })).toThrow("exactly one step"); + const mapped = defineWorkflow({ + id: "mapped", + steps: { + fan: map({ over: { from: "trigger.payload" }, step: step({ agent }) }), + }, + }); + expect(() => judgeModel({ model: mapped, invokeStep })).toThrow("must be a step"); + const child = defineWorkflow({ + id: "parent", + steps: { + inner: childWorkflow({ definition: defineWorkflow({ id: "inner", agent }) }), + }, + }); + expect(() => judgeModel({ model: child, invokeStep })).toThrow("must be a step"); const empty = { id: "empty", triggers: [{ type: "manual" }], @@ -653,4 +683,31 @@ describe("judgeModel", () => { }), ).toThrow("needs invokeStep"); }); + + test("a one-step workflow is invoked once with the case payload", async () => { + const question = "Is this reply ready to send to the customer?"; + const payload = { + state: { input: "refund?", output: "reply to refund?", expectedOutput: "yes" }, + questions: { judged: { type: "noul", instructions: question } }, + }; + const forms = [ + step({ agent }), + defineWorkflow({ id: "plural", steps: { only: step({ agent }) } }), + defineWorkflow({ + id: "explicit", + steps: { only: step({ agent, input: { from: "trigger.payload" } }) }, + }), + defineWorkflow({ id: "singular", agent }), + ]; + for (const model of forms) { + const bodies: unknown[] = []; + const seen: StepInvoker = async (request) => { + bodies.push(request.input); + return { output: { decisions: [{ id: "judged", type: "noul", noul: 0.9 }] } }; + }; + const { outcome } = await judged(question, judgeModel({ model, invokeStep: seen })); + expect(outcome?.pass).toBe(true); + expect(bodies).toEqual([payload]); + } + }); }); diff --git a/src/workflow.ts b/src/workflow.ts index 5d9d915..05d5a22 100644 --- a/src/workflow.ts +++ b/src/workflow.ts @@ -102,9 +102,19 @@ function isWorkflow(model: StepPrimitive | WorkflowDefinition): model is Workflo return "steps" in model && "stepOrder" in model; } +/** + * `defineWorkflow` stamps a lone step that omits `input` with this selector. + * It is the same binding `judgeModel` forces, so it is not a custom input. + */ +function isStampedTriggerInput(input: NonNullable): boolean { + return "from" in input && input.from === "trigger.payload" && Object.keys(input).length === 1; +} + function assertJudgeStep(step: StepPrimitive): void { if (step.kind !== "step") throw new Error("judgeModel: the only primitive must be a step"); - if (step.input !== undefined) throw new Error("judgeModel: step must not set input"); + if (step.input !== undefined && !isStampedTriggerInput(step.input)) { + throw new Error("judgeModel: step must not set input"); + } if (step.inference !== undefined) throw new Error("judgeModel: step must not set inference"); if (step.triggers !== undefined && step.triggers !== 1) { throw new Error("judgeModel: step triggers must be 1"); From 5c768ace58d4f07a37ff62db05dae5a1794fc774 Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 02:05:06 -0700 Subject: [PATCH 05/16] feat: add string includes and prebuilt judge scorers --- README.md | 25 ++-- e2e/prebuilt.e2e.test.ts | 154 ++++++++++++++++++++++++ src/checks.ts | 26 +++++ src/index.ts | 1 + src/prebuilt.ts | 245 +++++++++++++++++++++++++++++++++++++++ src/workflow.ts | 11 ++ 6 files changed, 450 insertions(+), 12 deletions(-) create mode 100644 e2e/prebuilt.e2e.test.ts create mode 100644 src/prebuilt.ts diff --git a/README.md b/README.md index de6bfb8..50da473 100644 --- a/README.md +++ b/README.md @@ -47,18 +47,19 @@ returned. `unsure (p=…)` in between. `{ question, yes, no }` sets its own bars. - **Helpers**: -| helper | passes when | -| ----------------------------- | ------------------------------------------------------------------------------ | -| `matches(field?)` | output deep-equals expectedOutput, or the same field (key or accessor) of each | -| `fitsSchema(schema)` | output satisfies a Standard Schema (arktype, zod, valibot) | -| `mentions(words)` | output contains every word, case-insensitive; value is the fraction found | -| `avoids(words)` | output contains none of the words | -| `ranSteps(names)` | `tool` and `step` names match `names`, order ignored; value is their F1 | -| `stepsAtMost(n)` | the run took at most `n` steps | -| `under({ ms })` | the run finished within `ms` | -| `ask(q, { options, pass })` | Jev picks an option named in `pass` | -| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt` | -| `ask(q, { levels, pass })` | Jev grades at level `pass` or above (levels lowest first) | +| helper | passes when | +| ----------------------------- | ----------------------------------------------------------------------------------------------------------- | +| `matches(field?)` | output deep-equals expectedOutput, or the same field (key or accessor) of each | +| `includes()` | both strings, and output contains expectedOutput after NFKC, trim, and whitespace collapse; value is 1 or 0 | +| `fitsSchema(schema)` | output satisfies a Standard Schema (arktype, zod, valibot) | +| `mentions(words)` | output contains every word, case-insensitive; value is the fraction found | +| `avoids(words)` | output contains none of the words | +| `ranSteps(names)` | `tool` and `step` names match `names`, order ignored; value is their F1 | +| `stepsAtMost(n)` | the run took at most `n` steps | +| `under({ ms })` | the run finished within `ms` | +| `ask(q, { options, pass })` | Jev picks an option named in `pass` | +| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt` | +| `ask(q, { levels, pass })` | Jev grades at level `pass` or above (levels lowest first) | `mentions` and `avoids` also take `(r) => words` for per-case words. `mentions` and `ranSteps` take `{ atLeast }` to pass on partial credit. diff --git a/e2e/prebuilt.e2e.test.ts b/e2e/prebuilt.e2e.test.ts new file mode 100644 index 0000000..b1f8bcf --- /dev/null +++ b/e2e/prebuilt.e2e.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, test } from "bun:test"; + +import { includes, matches, runEvaluation } from "../src/index.js"; +import { + answerRelevancy, + correctness, + faithfulness, + parsePrebuilt, + trajectoryAccuracy, + type PrebuiltId, +} from "../src/workflow.js"; + +const source = { provider: "ollama", model: "pinned" }; + +function judgeOf( + id: PrebuiltId, + definition: { steps: Record; stepOrder: readonly string[] }, +) { + expect(definition.stepOrder).toEqual(["judge"]); + const judge = definition.steps["judge"]; + expect(judge).toMatchObject({ + kind: "step", + input: { from: "trigger.payload" }, + agent: { + id: `${id}-judge`, + capabilities: [], + inference: { sources: [source] }, + }, + }); + const agent = (judge as { agent: { toolFactories: unknown[]; systemPrompt: string } }).agent; + expect(agent.toolFactories).toEqual([]); + expect(agent.systemPrompt.length).toBeGreaterThan(0); +} + +describe("includes", () => { + test("matches and includes share one string expectedOutput", async () => { + const report = await runEvaluation({ + name: "includes-mix", + task: (input: string) => `Answer: ${input}`, + data: [{ input: "Refund", expectedOutput: " reFUND " }], + scorers: { exact: matches(), has: includes() }, + allow: { exact: 1 }, + }); + expect(report.ok).toBe(true); + expect(report.cases[0]?.checks["exact"]?.pass).toBe(false); + expect(report.cases[0]?.checks["has"]).toEqual({ pass: true, value: 1 }); + }); + + test("fails closed on a non-string, a missing expected, or an empty needle", () => { + const check = includes(); + const run = { input: "", steps: [], ms: 0 }; + expect(check({ ...run, output: { a: 1 }, expectedOutput: '{"a":1}' })).toMatchObject({ + pass: false, + value: 0, + }); + expect(check({ ...run, output: "hello" })).toMatchObject({ pass: false, value: 0 }); + expect(check({ ...run, output: "hello", expectedOutput: " \n\t " })).toMatchObject({ + pass: false, + value: 0, + }); + expect(check({ ...run, output: "see the file now", expectedOutput: "file now" })).toEqual({ + pass: true, + value: 1, + }); + }); +}); + +describe("prebuilt judges", () => { + test("each factory is one judge step pinned to the source", () => { + const run = { + input: "q", + output: "a", + expectedOutput: "e", + steps: [{ name: "lookup", kind: "tool" as const }], + ms: 1, + }; + const checks = [ + faithfulness(source), + answerRelevancy(source), + correctness(source), + trajectoryAccuracy(source, ["lookup"]), + ]; + for (const check of checks) judgeOf(check.id, check.definition); + expect(faithfulness(source).payload(run)).toEqual({ context: "e", answer: "a" }); + expect(faithfulness(source, { context: () => "ctx" }).payload(run)).toEqual({ + context: "ctx", + answer: "a", + }); + expect(answerRelevancy(source).payload({ ...run, expectedOutput: "ignored" })).toEqual({ + question: "q", + answer: "a", + }); + expect(correctness(source).payload(run)).toEqual({ input: "q", expected: "e", answer: "a" }); + expect(trajectoryAccuracy(source, (r) => r.steps.map((s) => s.name)).payload(run)).toEqual({ + expected: ["lookup"], + actual: run.steps, + }); + expect(faithfulness(source).payload({ ...run, output: 1 })).toMatchObject({ + pass: false, + value: 0, + }); + expect(() => faithfulness({ provider: "", model: "m" })).toThrow("non-empty"); + }); + + test("parsePrebuilt accepts one object and rejects fences and a score past 1", () => { + expect(parsePrebuilt("faithfulness", { category: "supported" })).toEqual({ + pass: true, + value: 1, + label: "supported", + }); + expect(parsePrebuilt("faithfulness", '{"category":"unverifiable"}')).toMatchObject({ + pass: false, + value: 0.5, + label: "unverifiable", + }); + expect( + parsePrebuilt("correctness", { category: "incorrect", explanation: "no" }), + ).toMatchObject({ + pass: false, + value: 0, + label: "incorrect", + }); + expect(parsePrebuilt("answerRelevancy", { score: 1, explanation: "on topic" })).toEqual({ + pass: true, + value: 1, + explanation: "on topic", + }); + expect( + parsePrebuilt("trajectoryAccuracy", { score: 0.5, explanation: "missed a step" }), + ).toEqual({ + pass: false, + value: 0.5, + explanation: "missed a step", + }); + const fenced = '```json\n{"score":1,"explanation":"ok"}\n```'; + expect(parsePrebuilt("answerRelevancy", fenced)).toEqual({ + pass: false, + value: 0, + explanation: "judge output failed closed", + }); + expect(parsePrebuilt("answerRelevancy", { score: 1.5, explanation: "too high" })).toEqual({ + pass: false, + value: 0, + explanation: "judge output failed closed", + }); + expect( + parsePrebuilt("correctness", '{"category":"correct"}{"category":"incorrect"}'), + ).toMatchObject({ pass: false, value: 0 }); + expect(parsePrebuilt("faithfulness", "the answer is supported")).toMatchObject({ + pass: false, + value: 0, + }); + }); +}); diff --git a/src/checks.ts b/src/checks.ts index c13bc41..613a289 100644 --- a/src/checks.ts +++ b/src/checks.ts @@ -27,6 +27,32 @@ export function toOutcome(name: string, raw: unknown): Outcome { }; } +/** Case-insensitive substring. Value is 1 or 0. Strings only; no coercion. */ +export function includes(): CheckFn { + return ({ output, expectedOutput }) => { + if (typeof output !== "string" || typeof expectedOutput !== "string") { + return { + pass: false, + value: 0, + explanation: "includes needs string output and expectedOutput", + }; + } + const needle = collapsed(expectedOutput); + if (needle.length === 0) { + return { pass: false, value: 0, explanation: "includes needle is empty" }; + } + const pass = collapsed(output).includes(needle); + return pass + ? { pass: true, value: 1 } + : { pass: false, value: 0, explanation: "output does not include the expected text" }; + }; +} + +/** NFKC, trim, collapse internal whitespace, then lowercase. */ +function collapsed(value: string): string { + return value.normalize("NFKC").trim().replace(/\s+/g, " ").toLowerCase(); +} + /** Output deep-equals expectedOutput, or one field of each (a key or an accessor). */ export function matches( field?: string | ((value: T) => unknown), diff --git a/src/index.ts b/src/index.ts index efe1428..5ab8218 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,7 @@ export type { export { avoids, fitsSchema, + includes, matches, mentions, ranSteps, diff --git a/src/prebuilt.ts b/src/prebuilt.ts new file mode 100644 index 0000000..14a0916 --- /dev/null +++ b/src/prebuilt.ts @@ -0,0 +1,245 @@ +import { defineAgent } from "@intx/agent"; +import { defineWorkflow, step, type WorkflowDefinition } from "@intx/workflow"; + +import type { Outcome, Run, Step } from "./types.js"; + +/** Pinned inference source for a prebuilt judge. Both fields are required. */ +export type ModelSource = { provider: string; model: string }; + +export type PrebuiltId = "faithfulness" | "answerRelevancy" | "correctness" | "trajectoryAccuracy"; + +/** + * A one-step judge definition plus the payload for its trigger and the + * outcome used when the run cannot be judged. This does not call a model. + */ +export type Prebuilt

= { + id: PrebuiltId; + definition: WorkflowDefinition; + payload: (run: Run) => P | Outcome; + failure: Outcome; +}; + +const CLOSED: Outcome = { + pass: false, + value: 0, + explanation: "judge output failed closed", +}; + +const FAITH = ["contradicted", "unverifiable", "supported"] as const; +const CORRECT = ["incorrect", "partial", "correct"] as const; + +const RUBRIC = { + faithfulness: [ + "You judge whether every claim in the answer is faithful to the context.", + "Reply with one JSON object and nothing else. No prose. No markdown fences.", + '{"category":"contradicted"|"unverifiable"|"supported","explanation":"..."}', + "Order, lowest first: contradicted, unverifiable, supported.", + "contradicted: the answer conflicts with the context.", + "unverifiable: the answer adds claims the context neither supports nor contradicts.", + "supported: every claim in the answer is supported by the context.", + "Pick the lowest category that applies.", + ].join("\n"), + answerRelevancy: [ + "You judge how completely the answer addresses the question.", + "Reply with one JSON object and nothing else. No prose. No markdown fences.", + '{"score":0,"explanation":"..."}', + "score is a finite number from 0 to 1. 1 means the answer is fully relevant.", + ].join("\n"), + correctness: [ + "You judge whether the answer is correct for the input, given the expected answer.", + "Reply with one JSON object and nothing else. No prose. No markdown fences.", + '{"category":"incorrect"|"partial"|"correct","explanation":"..."}', + "Order, lowest first: incorrect, partial, correct.", + "incorrect: the answer is wrong. partial: some of it is right. correct: it matches.", + "Pick the lowest category that applies.", + ].join("\n"), + trajectoryAccuracy: [ + "You judge how closely the actual steps follow the expected trajectory.", + "Reply with one JSON object and nothing else. No prose. No markdown fences.", + '{"score":0,"explanation":"..."}', + "score is a finite number from 0 to 1. 1 means the trajectory is accurate.", + "expected is the names that should have run. actual is the steps that did.", + ].join("\n"), +} as const; + +function requireSource(source: ModelSource): ModelSource { + if (source.provider.trim() === "" || source.model.trim() === "") { + throw new Error("prebuilt judge needs a non-empty provider and model"); + } + return { provider: source.provider, model: source.model }; +} + +function judgeWorkflow(id: PrebuiltId, source: ModelSource): WorkflowDefinition { + const pinned = requireSource(source); + return defineWorkflow({ + id, + trigger: { type: "manual" }, + steps: { + judge: step({ + agent: defineAgent({ + id: `${id}-judge`, + systemPrompt: RUBRIC[id], + tools: [], + capabilities: [], + inference: { sources: [pinned] }, + }), + input: { from: "trigger.payload" }, + }), + }, + }); +} + +function closed(explanation: string): Outcome { + return { pass: false, value: 0, explanation }; +} + +function isOutcome(value: object): value is Outcome { + return "pass" in value && "value" in value; +} + +export function faithfulness( + source: ModelSource, + options: { context?: (run: Run) => unknown } = {}, +): Prebuilt<{ context: string; answer: string }> { + const contextOf = options.context; + return { + id: "faithfulness", + definition: judgeWorkflow("faithfulness", source), + failure: CLOSED, + payload: (run) => { + if (typeof run.output !== "string") return closed("faithfulness needs a string answer"); + const context = + contextOf !== undefined + ? contextOf(run) + : typeof run.expectedOutput === "string" + ? run.expectedOutput + : run.input; + if (typeof context !== "string") return closed("faithfulness needs a string context"); + return { context, answer: run.output }; + }, + }; +} + +export function answerRelevancy( + source: ModelSource, +): Prebuilt<{ question: string; answer: string }> { + return { + id: "answerRelevancy", + definition: judgeWorkflow("answerRelevancy", source), + failure: CLOSED, + payload: (run) => { + if (typeof run.input !== "string" || typeof run.output !== "string") { + return closed("answerRelevancy needs a string question and answer"); + } + return { question: run.input, answer: run.output }; + }, + }; +} + +export function correctness( + source: ModelSource, +): Prebuilt<{ input: string; expected: string; answer: string }> { + return { + id: "correctness", + definition: judgeWorkflow("correctness", source), + failure: CLOSED, + payload: (run) => { + if ( + typeof run.input !== "string" || + typeof run.expectedOutput !== "string" || + typeof run.output !== "string" + ) { + return closed("correctness needs string input, expected, and answer"); + } + return { input: run.input, expected: run.expectedOutput, answer: run.output }; + }, + }; +} + +export function trajectoryAccuracy( + source: ModelSource, + expected: readonly string[] | ((run: Run) => readonly string[]), +): Prebuilt<{ expected: readonly string[]; actual: readonly Step[] }> { + return { + id: "trajectoryAccuracy", + definition: judgeWorkflow("trajectoryAccuracy", source), + failure: CLOSED, + payload: (run) => { + const names = typeof expected === "function" ? expected(run) : expected; + if (!Array.isArray(names) || names.some((name) => typeof name !== "string")) { + return closed("trajectoryAccuracy needs a string array of expected step names"); + } + return { expected: names, actual: run.steps }; + }, + }; +} + +function asObject(output: unknown): Record | undefined { + if (typeof output === "string") { + const text = output.trim(); + if (text.length === 0 || text.includes("```")) return undefined; + try { + const value: unknown = JSON.parse(text); + if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined; + return value as Record; + } catch { + return undefined; + } + } + if (typeof output !== "object" || output === null || Array.isArray(output)) return undefined; + if (isOutcome(output)) return undefined; + return output as Record; +} + +function explanationOf(raw: Record, keys: readonly string[]): string | undefined { + for (const key of Object.keys(raw)) { + if (!keys.includes(key)) return undefined; + } + if (!("explanation" in raw)) return ""; + return typeof raw["explanation"] === "string" ? raw["explanation"] : undefined; +} + +function categoryOutcome( + raw: Record, + scale: readonly string[], + passAt: string, +): Outcome | undefined { + const explanation = explanationOf(raw, ["category", "explanation"]); + const category = raw["category"]; + if (explanation === undefined || typeof category !== "string") return undefined; + const index = scale.indexOf(category); + if (index < 0) return undefined; + const outcome: Outcome = { + pass: category === passAt, + value: index / 2, + label: category, + }; + if (explanation.length > 0) outcome.explanation = explanation; + return outcome; +} + +function scoreOutcome(raw: Record): Outcome | undefined { + const explanation = explanationOf(raw, ["score", "explanation"]); + const score = raw["score"]; + if (explanation === undefined || explanation.length === 0) return undefined; + if (typeof score !== "number" || !Number.isFinite(score) || score < 0 || score > 1) { + return undefined; + } + return { pass: score === 1, value: score, explanation }; +} + +/** Parse one judge object. Fences, prose, a second object, or a bad score fail closed. */ +export function parsePrebuilt(id: PrebuiltId, output: unknown): Outcome { + const raw = asObject(output); + if (raw === undefined) return CLOSED; + if (id === "faithfulness") { + return categoryOutcome(raw, FAITH, "supported") ?? CLOSED; + } + if (id === "correctness") { + return categoryOutcome(raw, CORRECT, "correct") ?? CLOSED; + } + if (id === "answerRelevancy" || id === "trajectoryAccuracy") { + return scoreOutcome(raw) ?? CLOSED; + } + return CLOSED; +} diff --git a/src/workflow.ts b/src/workflow.ts index 05d5a22..1d17c10 100644 --- a/src/workflow.ts +++ b/src/workflow.ts @@ -211,3 +211,14 @@ export function judgeModel(options: JudgeModelOptions): Judge { }, }; } + +export { + answerRelevancy, + correctness, + faithfulness, + parsePrebuilt, + trajectoryAccuracy, + type ModelSource, + type Prebuilt, + type PrebuiltId, +} from "./prebuilt.js"; From 022ab029a84d7d41c3a1ffddeb27915f6d3fd834 Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 02:13:15 -0700 Subject: [PATCH 06/16] feat: record agent steps from stream() An empty step list is not an observed empty tool path, so ranSteps([]) fails unless a watched run actually used no tools. --- README.md | 31 ++-- e2e/agent-trajectory.e2e.test.ts | 297 +++++++++++++++++++++++++++++++ src/checks.ts | 7 + src/runs.ts | 246 ++++++++++++++++++++++++- 4 files changed, 559 insertions(+), 22 deletions(-) create mode 100644 e2e/agent-trajectory.e2e.test.ts diff --git a/README.md b/README.md index 50da473..95eb742 100644 --- a/README.md +++ b/README.md @@ -47,19 +47,19 @@ returned. `unsure (p=…)` in between. `{ question, yes, no }` sets its own bars. - **Helpers**: -| helper | passes when | -| ----------------------------- | ----------------------------------------------------------------------------------------------------------- | -| `matches(field?)` | output deep-equals expectedOutput, or the same field (key or accessor) of each | -| `includes()` | both strings, and output contains expectedOutput after NFKC, trim, and whitespace collapse; value is 1 or 0 | -| `fitsSchema(schema)` | output satisfies a Standard Schema (arktype, zod, valibot) | -| `mentions(words)` | output contains every word, case-insensitive; value is the fraction found | -| `avoids(words)` | output contains none of the words | -| `ranSteps(names)` | `tool` and `step` names match `names`, order ignored; value is their F1 | -| `stepsAtMost(n)` | the run took at most `n` steps | -| `under({ ms })` | the run finished within `ms` | -| `ask(q, { options, pass })` | Jev picks an option named in `pass` | -| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt` | -| `ask(q, { levels, pass })` | Jev grades at level `pass` or above (levels lowest first) | +| helper | passes when | +| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `matches(field?)` | output deep-equals expectedOutput, or the same field (key or accessor) of each | +| `includes()` | both strings, and output contains expectedOutput after NFKC, trim, and whitespace collapse; value is 1 or 0 | +| `fitsSchema(schema)` | output satisfies a Standard Schema (arktype, zod, valibot) | +| `mentions(words)` | output contains every word, case-insensitive; value is the fraction found | +| `avoids(words)` | output contains none of the words | +| `ranSteps(names)` | `tool` and `step` names match `names`, order ignored; value is their F1. Agent trajectory comes from `stream()`. A reply-only send without `stream` does not satisfy `ranSteps([])` | +| `stepsAtMost(n)` | the run took at most `n` steps | +| `under({ ms })` | the run finished within `ms` | +| `ask(q, { options, pass })` | Jev picks an option named in `pass` | +| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt` | +| `ask(q, { levels, pass })` | Jev grades at level `pass` or above (levels lowest first) | `mentions` and `avoids` also take `(r) => words` for per-case words. `mentions` and `ranSteps` take `{ atLeast }` to pass on partial credit. @@ -100,7 +100,10 @@ space cannot absorb — those fail closed by design. - **Function**: `(input, signal) => output`. `signal` aborts on `timeout`. - **Agent**: an `@intx/agent` Agent, detected by `send`. Input is sent - as a message; output is the reply. + as a message; output is the reply. When the agent has `stream()`, steps + are recorded from that stream (`inference.done` as `llm`, `tool.start` / + `tool.done` as `tool`). A reply-only send without `stream` leaves `steps` + empty, which does not satisfy `ranSteps([])`. - **Workflow**: `workflow(definition, { invokeStep })` from `@corbits/evals/workflow`. A definition cannot run locally without a step invoker, and the root never loads `@intx/workflow`, so a bare diff --git a/e2e/agent-trajectory.e2e.test.ts b/e2e/agent-trajectory.e2e.test.ts new file mode 100644 index 0000000..267a7ac --- /dev/null +++ b/e2e/agent-trajectory.e2e.test.ts @@ -0,0 +1,297 @@ +import { describe, expect, test } from "bun:test"; + +import { EvalFailure, ranSteps, runEvaluation } from "../src/index.js"; +import { toTraced, type AgentLike } from "../src/runs.js"; + +type Event = { type: string; data?: unknown }; +type SendResult = + | { type: "reply"; reply: string; turn?: { model?: string } } + | { type: "suspended" }; + +function createStub(options: { + stream: boolean; + onSend: (input: string, emit: (event: Event) => void) => Promise; +}): { agent: AgentLike; maxConsumers: () => number } { + let consumers = 0; + let maxConsumers = 0; + let buffer: Event[] = [{ type: "reactor.start" }]; + const waiters: Array<(result: IteratorResult) => void> = []; + + function emit(event: Event): void { + const waiter = waiters.shift(); + if (waiter !== undefined) waiter({ value: event, done: false }); + else buffer.push(event); + } + + const agent: AgentLike = { + send: (content) => options.onSend(content, emit), + }; + if (options.stream) { + agent.stream = () => ({ + [Symbol.asyncIterator](): AsyncIterator { + consumers += 1; + maxConsumers = Math.max(maxConsumers, consumers); + if (consumers > 1) throw new Error("two consumers"); + return { + next(): Promise> { + const queued = buffer.shift(); + if (queued !== undefined) return Promise.resolve({ value: queued, done: false }); + return new Promise((resolve) => { + waiters.push(resolve); + }); + }, + async return(): Promise> { + consumers -= 1; + const waiter = waiters.shift(); + if (waiter !== undefined) waiter({ value: undefined, done: true }); + return { value: undefined, done: true }; + }, + }; + }, + }); + } + return { agent, maxConsumers: () => maxConsumers }; +} + +async function failure(pending: Promise): Promise { + try { + await pending; + } catch (cause) { + if (cause instanceof EvalFailure) return cause; + throw cause; + } + throw new Error("expected runEvaluation to fail"); +} + +describe("agent trajectory", () => { + test("a watched tool cycle records llm and tool steps", async () => { + const { agent } = createStub({ + stream: true, + onSend: async (input, emit) => { + emit({ + type: "inference.done", + data: { + turn: { model: "stub", content: [{ type: "text", text: "looking" }] }, + }, + }); + emit({ + type: "inference.tool_call.end", + data: { callId: "c1", name: "search", arguments: { q: input } }, + }); + emit({ + type: "tool.start", + data: { call: { id: "c1", name: "search", arguments: { q: input } } }, + }); + emit({ type: "tool.update", data: { callId: "c1", partial: "x" } }); + emit({ type: "inference.text.delta", data: { token: "nope" } }); + emit({ + type: "tool.done", + data: { result: { callId: "orphan", content: "no start" } }, + }); + emit({ + type: "tool.done", + data: { result: { callId: "c1", content: "found" } }, + }); + emit({ + type: "inference.done", + data: { + turn: { + model: "stub", + content: [ + { type: "text", text: "done " }, + { type: "thinking", text: "hidden" }, + { type: "text", text: "now" }, + ], + }, + }, + }); + return { type: "reply", reply: "done now", turn: { model: "stub" } }; + }, + }); + const report = await runEvaluation({ + name: "tool-cycle", + task: agent, + data: [{ input: "order" }], + scorers: { ran: ranSteps(["search"]), none: ranSteps([]) }, + allow: { none: 1 }, + }); + expect(report.cases[0]?.steps).toEqual([ + { name: "stub", kind: "llm", output: "looking" }, + { name: "search", kind: "tool", input: { q: "order" }, output: "found" }, + { name: "stub", kind: "llm", output: "done now" }, + ]); + expect(report.cases[0]?.checks["ran"]?.pass).toBe(true); + expect(report.cases[0]?.checks["none"]?.pass).toBe(false); + }); + + test("a watched reply with no inference or tool is one llm step", async () => { + const { agent } = createStub({ + stream: true, + onSend: async () => ({ type: "reply", reply: "hello", turn: { model: "stub-model" } }), + }); + const report = await runEvaluation({ + name: "reply-only-watched", + task: agent, + data: [{ input: "hi" }], + scorers: { none: ranSteps([]), search: ranSteps(["search"]) }, + allow: { search: 1 }, + }); + expect(report.cases[0]?.steps).toEqual([{ name: "stub-model", kind: "llm", output: "hello" }]); + expect(report.cases[0]?.checks["none"]?.pass).toBe(true); + expect(report.cases[0]?.checks["search"]?.pass).toBe(false); + }); + + test("a watched reply without a model name is assistant", async () => { + const { agent } = createStub({ + stream: true, + onSend: async () => ({ type: "reply", reply: "hello" }), + }); + const report = await runEvaluation({ + name: "reply-only-assistant", + task: agent, + data: [{ input: "hi" }], + scorers: { none: ranSteps([]) }, + }); + expect(report.cases[0]?.steps).toEqual([{ name: "assistant", kind: "llm", output: "hello" }]); + }); + + test("send without stream leaves steps empty and ranSteps([]) fails", async () => { + const { agent } = createStub({ + stream: false, + onSend: async () => ({ type: "reply", reply: "hello", turn: { model: "stub" } }), + }); + const err = await failure( + runEvaluation({ + name: "send-only", + task: agent, + data: [{ input: "hi" }], + scorers: { none: ranSteps([]) }, + }), + ); + expect(err.report.cases[0]?.steps).toEqual([]); + expect(err.report.cases[0]?.checks["none"]).toMatchObject({ + pass: false, + value: 0, + explanation: "an empty step list is not an observed empty tool path", + }); + }); + + test("a suspended send still fails the case", async () => { + const { agent } = createStub({ + stream: false, + onSend: async () => ({ type: "suspended" }), + }); + const err = await failure( + runEvaluation({ + name: "suspended", + task: agent, + data: [{ input: "hi", expectedOutput: "hi" }], + }), + ); + expect(err.report.cases[0]?.error).toBe('agent did not reply (got "suspended")'); + }); + + test("overlapping cases on one agent do not mix steps", async () => { + const { agent, maxConsumers } = createStub({ + stream: true, + onSend: async (input, emit) => { + await new Promise((resolve) => setTimeout(resolve, 20)); + emit({ + type: "inference.done", + data: { turn: { model: "stub", content: [{ type: "text", text: input }] } }, + }); + emit({ + type: "tool.start", + data: { call: { id: input, name: input, arguments: { q: input } } }, + }); + emit({ + type: "tool.done", + data: { result: { callId: input, content: input } }, + }); + return { type: "reply", reply: input, turn: { model: "stub" } }; + }, + }); + const report = await runEvaluation({ + name: "overlap", + task: agent, + concurrency: 2, + data: [ + { name: "a", input: "search" }, + { name: "b", input: "lookup" }, + ], + scorers: { ran: ranSteps([]) }, + allow: { ran: 2 }, + }); + const byName = Object.fromEntries( + report.cases.map((c) => [c.name, c.steps.map((s) => s.name)]), + ); + expect(byName).toEqual({ + a: ["stub", "search"], + b: ["stub", "lookup"], + }); + expect(maxConsumers()).toBe(1); + }); + + test("a reply inference.done emitted after send still lands", async () => { + const { agent } = createStub({ + stream: true, + onSend: async (_input, emit) => { + emit({ + type: "tool.start", + data: { call: { id: "c1", name: "search", arguments: { q: "late" } } }, + }); + emit({ + type: "tool.done", + data: { result: { callId: "c1", content: "found" } }, + }); + setTimeout(() => { + emit({ + type: "inference.done", + data: { + turn: { model: "reply-model", content: [{ type: "text", text: "from-stream" }] }, + }, + }); + }, 0); + return { type: "reply", reply: "from-send", turn: { model: "send-model" } }; + }, + }); + const report = await runEvaluation({ + name: "late-inference", + task: agent, + data: [{ input: "late" }], + scorers: { ran: ranSteps(["search"]) }, + }); + expect(report.cases[0]?.steps).toEqual([ + { name: "search", kind: "tool", input: { q: "late" }, output: "found" }, + { name: "reply-model", kind: "llm", output: "from-stream" }, + ]); + }); + + test("an aborted waiter does not send", async () => { + let sends = 0; + let releaseA: () => void = () => {}; + const gate = new Promise((resolve) => { + releaseA = resolve; + }); + const { agent } = createStub({ + stream: true, + onSend: async () => { + sends += 1; + await gate; + return { type: "reply", reply: "a", turn: { model: "stub" } }; + }, + }); + const run = toTraced(agent); + const inFlight = new AbortController(); + const queued = new AbortController(); + const pendingA = run("a", inFlight.signal); + await new Promise((resolve) => setTimeout(resolve, 10)); + const pendingB = run("b", queued.signal); + await new Promise((resolve) => setTimeout(resolve, 10)); + queued.abort(); + releaseA(); + await pendingA; + await expect(pendingB).rejects.toThrow("run aborted before send"); + expect(sends).toBe(1); + }); +}); diff --git a/src/checks.ts b/src/checks.ts index 613a289..c784fe7 100644 --- a/src/checks.ts +++ b/src/checks.ts @@ -181,6 +181,13 @@ export function ranSteps(names: readonly string[], options: AtLeast = {}): Check const actual = stepNames(run).sort(); const explanation = `expected [${expected.join(", ")}], got [${actual.join(", ")}]`; if (expected.length === 0) { + if (run.steps.length === 0) { + return { + pass: false, + value: 0, + explanation: "an empty step list is not an observed empty tool path", + }; + } return { pass: actual.length === 0, value: actual.length === 0 ? 1 : 0, explanation }; } const counts = new Map(); diff --git a/src/runs.ts b/src/runs.ts index f446d98..c7c5a95 100644 --- a/src/runs.ts +++ b/src/runs.ts @@ -5,12 +5,16 @@ export type Trace = { output: O; steps: Step[] }; export type RunFn = (input: I, signal: AbortSignal) => O | Promise; +type StreamEvent = { type: string; data?: unknown }; + +type AgentSendResult = + | { type: "reply"; reply: string; turn?: { model?: string } } + | { type: "suspended" }; + /** Structural `@intx/agent` Agent, so the root never imports it. */ export type AgentLike = { - send( - content: string, - opts?: { signal?: AbortSignal }, - ): Promise<{ type: "reply"; reply: string } | { type: "suspended" }>; + send(content: string, opts?: { signal?: AbortSignal }): Promise; + stream?(): AsyncIterable; }; export const TRACE: unique symbol = Symbol("@corbits/evals trace"); @@ -28,8 +32,236 @@ function isWorkflowDefinition(value: object): boolean { return "stepOrder" in value && "steps" in value && "triggers" in value; } +const tails = new WeakMap>(); + +function withAgentLock(agent: object, fn: () => Promise): Promise { + const prev = tails.get(agent) ?? Promise.resolve(); + let release: (() => void) | undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + tails.set( + agent, + prev.then( + () => gate, + () => gate, + ), + ); + return prev + .then( + () => fn(), + () => fn(), + ) + .finally(() => { + release?.(); + }); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function joinedText(turn: unknown): { name: string; output: string } { + if (!isRecord(turn)) return { name: "assistant", output: "" }; + const name = typeof turn.model === "string" && turn.model.length > 0 ? turn.model : "assistant"; + if (!Array.isArray(turn.content)) return { name, output: "" }; + const parts: string[] = []; + for (const block of turn.content) { + if (isRecord(block) && block.type === "text" && typeof block.text === "string") { + parts.push(block.text); + } + } + return { name, output: parts.join("") }; +} + +function recordEvent(event: StreamEvent, steps: Step[], tools: Map): boolean { + if (event.type === "inference.done") { + const turn = isRecord(event.data) ? event.data.turn : undefined; + const text = joinedText(turn); + steps.push({ name: text.name, kind: "llm", output: text.output }); + return true; + } + if (event.type === "tool.start") { + const call = isRecord(event.data) ? event.data.call : undefined; + if (!isRecord(call) || typeof call.name !== "string") return false; + const step: Step = { name: call.name, kind: "tool", input: call.arguments }; + steps.push(step); + if (typeof call.id === "string") tools.set(call.id, step); + return true; + } + if (event.type === "tool.done") { + const result = isRecord(event.data) ? event.data.result : undefined; + if (!isRecord(result) || typeof result.callId !== "string") return false; + const step = tools.get(result.callId); + if (step === undefined) return false; + step.output = "content" in result ? result.content : result; + } + return false; +} + +type Raced = { ready: true; result: T } | { ready: false }; + +function raceReady(pending: Promise): Promise> { + return new Promise((resolve, reject) => { + let settled = false; + pending.then( + (result) => { + settled = true; + resolve({ ready: true, result }); + }, + (error: unknown) => { + settled = true; + reject(error); + }, + ); + queueMicrotask(() => { + if (!settled) resolve({ ready: false }); + }); + }); +} + +/** Drop events already queued at subscribe. The returned read is the first live one. */ +async function discardBuffered( + iterator: AsyncIterator, +): Promise<{ pending: Promise> }> { + let pending = iterator.next(); + for (;;) { + const raced = await raceReady(pending); + if (!raced.ready) return { pending }; + if (raced.result.done === true) return { pending: Promise.resolve(raced.result) }; + pending = iterator.next(); + } +} + +async function takeReady( + pending: Promise>, +): Promise | undefined> { + const raced = await raceReady(pending); + if (!raced.ready) return undefined; + return raced.result; +} + +/** Quiet once a macrotask passes with no event, so a post-send `setTimeout(0)` is kept. */ +async function nextEventOrIdle( + pending: Promise>, +): Promise<{ idle: true } | { idle: false; result: IteratorResult }> { + let timer: ReturnType | undefined; + const idle = new Promise<{ idle: true }>((resolve) => { + timer = setTimeout(() => { + resolve({ idle: true }); + }, 0); + }); + try { + return await Promise.race([pending.then((result) => ({ idle: false as const, result })), idle]); + } finally { + clearTimeout(timer); + } +} + +async function collect( + iterator: AsyncIterator, + first: Promise>, + steps: Step[], + send: Promise, +): Promise { + let saw = false; + const tools = new Map(); + let sendSettled = false; + const mark = send.then( + () => { + sendSettled = true; + }, + () => { + sendSettled = true; + }, + ); + const note = (event: StreamEvent): void => { + if (recordEvent(event, steps, tools)) saw = true; + }; + let next = first; + while (!sendSettled) { + const winner = await Promise.race([ + next.then((result) => ({ kind: "event" as const, result })), + mark.then(() => ({ kind: "send" as const })), + ]); + if (winner.kind === "send") break; + if (winner.result.done === true) break; + note(winner.result.value); + next = iterator.next(); + } + for (;;) { + const winner = await nextEventOrIdle(next); + if (winner.idle) { + const ready = await takeReady(next); + if (ready === undefined || ready.done === true) break; + note(ready.value); + next = iterator.next(); + continue; + } + if (winner.result.done === true) break; + note(winner.result.value); + next = iterator.next(); + } + void next.then( + () => {}, + () => {}, + ); + return saw; +} + +function replyModel(res: AgentSendResult): string { + if (res.type !== "reply") return "assistant"; + const model = res.turn?.model; + return typeof model === "string" && model.length > 0 ? model : "assistant"; +} + +async function sendAgent( + agent: AgentLike, + input: string, + signal: AbortSignal, +): Promise> { + if (agent.stream === undefined) { + const res = await agent.send(input, { signal }); + if (res.type !== "reply") throw new Error(`agent did not reply (got "${res.type}")`); + return { output: res.reply as O, steps: [] }; + } + const stream = agent.stream; + return withAgentLock(agent, async () => { + if (signal.aborted) throw new Error("run aborted before send"); + const iterator = stream()[Symbol.asyncIterator](); + const steps: Step[] = []; + try { + const first = await discardBuffered(iterator); + let settled: { ok: true; res: AgentSendResult } | { ok: false; error: unknown } | undefined; + const send = agent.send(input, { signal }).then( + (res) => { + settled = { ok: true, res }; + }, + (error: unknown) => { + settled = { ok: false, error }; + }, + ); + const saw = await collect(iterator, first.pending, steps, send); + await send; + if (settled === undefined || !settled.ok) { + throw settled === undefined ? new Error("agent send did not settle") : settled.error; + } + if (settled.res.type !== "reply") { + throw new Error(`agent did not reply (got "${settled.res.type}")`); + } + if (!saw) { + steps.push({ name: replyModel(settled.res), kind: "llm", output: settled.res.reply }); + } + return { output: settled.res.reply as O, steps }; + } finally { + await iterator.return?.(); + } + }); +} + /** - * Agents run with no recorded steps. A raw workflow definition is refused: + * Agents record steps from `stream()` when it exists. Without it, a reply + * is returned with no steps. A raw workflow definition is refused: * running it needs a step invoker, which `workflow()` takes. */ export function toTraced( @@ -47,9 +279,7 @@ export function toTraced( return async (input, signal) => { if (typeof input !== "string") throw new Error("agent input must be a string"); if (signal.aborted) throw new Error("run aborted before send"); - const res = await run.send(input, { signal }); - if (res.type !== "reply") throw new Error(`agent did not reply (got "${res.type}")`); - return { output: res.reply as O, steps: [] }; + return sendAgent(run, input, signal); }; } if (isWorkflowDefinition(run)) { From 20584a429157e4707d3b54a15c6a772eaaccdd94 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 26 Oct 2026 21:02:37 -0700 Subject: [PATCH 07/16] fix: isolate aborted turns and name denied tools --- README.md | 6 +- e2e/agent-trajectory.e2e.test.ts | 115 ++++++++++++++++++++++++++++++- src/runner.ts | 9 ++- src/runs.ts | 89 +++++++++++++++++++++--- 4 files changed, 206 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 95eb742..0210ce3 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,10 @@ space cannot absorb — those fail closed by design. - **Agent**: an `@intx/agent` Agent, detected by `send`. Input is sent as a message; output is the reply. When the agent has `stream()`, steps are recorded from that stream (`inference.done` as `llm`, `tool.start` / - `tool.done` as `tool`). A reply-only send without `stream` leaves `steps` - empty, which does not satisfy `ranSteps([])`. + `tool.done` as `tool`). A policy deny emits `tool.done` without + `tool.start`; the tool is still recorded when `inference.tool_call.end` + or the assistant `tool_call` block names it. A reply-only send without + `stream` leaves `steps` empty, which does not satisfy `ranSteps([])`. - **Workflow**: `workflow(definition, { invokeStep })` from `@corbits/evals/workflow`. A definition cannot run locally without a step invoker, and the root never loads `@intx/workflow`, so a bare diff --git a/e2e/agent-trajectory.e2e.test.ts b/e2e/agent-trajectory.e2e.test.ts index 267a7ac..37462ee 100644 --- a/e2e/agent-trajectory.e2e.test.ts +++ b/e2e/agent-trajectory.e2e.test.ts @@ -10,7 +10,7 @@ type SendResult = function createStub(options: { stream: boolean; - onSend: (input: string, emit: (event: Event) => void) => Promise; + onSend: (input: string, emit: (event: Event) => void, signal: AbortSignal) => Promise; }): { agent: AgentLike; maxConsumers: () => number } { let consumers = 0; let maxConsumers = 0; @@ -24,7 +24,8 @@ function createStub(options: { } const agent: AgentLike = { - send: (content) => options.onSend(content, emit), + send: (content, opts) => + options.onSend(content, emit, opts?.signal ?? new AbortController().signal), }; if (options.stream) { agent.stream = () => ({ @@ -294,4 +295,114 @@ describe("agent trajectory", () => { await expect(pendingB).rejects.toThrow("run aborted before send"); expect(sends).toBe(1); }); + + test("an aborted turn's late tool does not land on the next case", async () => { + const { agent } = createStub({ + stream: true, + onSend: (input, emit, signal) => { + if (input !== "slow") { + return Promise.resolve({ type: "reply", reply: "found", turn: { model: "stub" } }); + } + return new Promise((_resolve, reject) => { + const abort = (): void => { + reject(signal.reason instanceof Error ? signal.reason : new Error("aborted")); + setTimeout(() => { + emit({ + type: "tool.start", + data: { call: { id: "late", name: "search", arguments: { q: "slow" } } }, + }); + emit({ type: "connector.reply", data: { content: "late" } }); + }, 20); + }; + if (signal.aborted) abort(); + else signal.addEventListener("abort", abort, { once: true }); + }); + }, + }); + const err = await failure( + runEvaluation({ + name: "abort-then-lookup", + task: agent, + concurrency: 1, + timeout: 200, + data: [ + { name: "slow", input: "slow" }, + { name: "lookup", input: "lookup" }, + ], + scorers: { none: ranSteps([]), search: ranSteps(["search"]) }, + }), + ); + const lookup = err.report.cases.find((c) => c.name === "lookup"); + expect(err.report.cases[0]?.error).toContain("timed out"); + expect(lookup?.error).toBeUndefined(); + expect(lookup?.steps).toEqual([{ name: "stub", kind: "llm", output: "found" }]); + expect(lookup?.checks["none"]?.pass).toBe(true); + expect(lookup?.checks["search"]?.pass).toBe(false); + }); + + test("a policy deny records the tool name without tool.start", async () => { + const { agent } = createStub({ + stream: true, + onSend: async (input, emit) => { + if (input === "from-block") { + emit({ + type: "inference.done", + data: { + turn: { + model: "stub", + content: [ + { type: "text", text: "no" }, + { type: "tool_call", id: "c1", name: "search", arguments: { q: input } }, + ], + }, + }, + }); + } else { + emit({ + type: "inference.done", + data: { turn: { model: "stub", content: [{ type: "text", text: "no" }] } }, + }); + emit({ + type: "inference.tool_call.end", + data: { callId: "c1", name: "search", arguments: { q: input } }, + }); + emit({ + type: "inference.tool_call.end", + data: { callId: "empty", name: "", arguments: {} }, + }); + } + emit({ + type: "tool.done", + data: { result: { callId: "nameless", content: "no name" } }, + }); + emit({ + type: "tool.done", + data: { result: { callId: "empty", content: "skip" } }, + }); + emit({ + type: "tool.done", + data: { result: { callId: "c1", content: "blocked", isError: true } }, + }); + return { type: "reply", reply: "no", turn: { model: "stub" } }; + }, + }); + const report = await runEvaluation({ + name: "policy-deny", + task: agent, + data: [ + { name: "from-end", input: "from-end" }, + { name: "from-block", input: "from-block" }, + ], + scorers: { ran: ranSteps(["search"]), none: ranSteps([]) }, + allow: { none: 2 }, + }); + for (const evalCase of report.cases) { + expect(evalCase.steps).toEqual([ + { name: "stub", kind: "llm", output: "no" }, + { name: "search", kind: "tool", input: { q: evalCase.input }, output: "blocked" }, + ]); + expect(evalCase.checks["ran"]?.pass).toBe(true); + expect(evalCase.checks["none"]?.pass).toBe(false); + } + }); }); diff --git a/src/runner.ts b/src/runner.ts index 798eff8..7bd55d9 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -47,7 +47,14 @@ export async function runOnce( }, timeoutMs); }); try { - const trace = await Promise.race([spec.run(evalCase.input, controller.signal), expiry]); + const pending = spec.run(evalCase.input, controller.signal); + // Timeout settles this race while an aborted turn may still be draining. + // Observe that later rejection so it is not unhandled. + void pending.then( + () => {}, + () => {}, + ); + const trace = await Promise.race([pending, expiry]); const run: Run = { input: evalCase.input, output: trace.output, diff --git a/src/runs.ts b/src/runs.ts index c7c5a95..ce5cab2 100644 --- a/src/runs.ts +++ b/src/runs.ts @@ -74,27 +74,74 @@ function joinedText(turn: unknown): { name: string; output: string } { return { name, output: parts.join("") }; } -function recordEvent(event: StreamEvent, steps: Step[], tools: Map): boolean { +type KnownCall = { name: string; input: unknown; step?: Step }; + +function rememberCall( + tools: Map, + id: string, + name: string, + input: unknown, +): void { + if (name.length === 0 || tools.has(id)) return; + tools.set(id, { name, input }); +} + +/** Names a blocked call can carry without `tool.start`. */ +function rememberTurnCalls(turn: unknown, tools: Map): void { + if (!isRecord(turn) || !Array.isArray(turn.content)) return; + for (const block of turn.content) { + if (!isRecord(block) || block.type !== "tool_call") continue; + if (typeof block.id !== "string" || typeof block.name !== "string") continue; + rememberCall(tools, block.id, block.name, block.arguments); + } +} + +function toolName(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function recordEvent(event: StreamEvent, steps: Step[], tools: Map): boolean { + if (event.type === "inference.tool_call.end") { + const data = isRecord(event.data) ? event.data : undefined; + if (data === undefined || typeof data.callId !== "string") return false; + const name = toolName(data.name); + if (name === undefined) return false; + rememberCall(tools, data.callId, name, data.arguments); + return false; + } if (event.type === "inference.done") { const turn = isRecord(event.data) ? event.data.turn : undefined; + rememberTurnCalls(turn, tools); const text = joinedText(turn); steps.push({ name: text.name, kind: "llm", output: text.output }); return true; } if (event.type === "tool.start") { const call = isRecord(event.data) ? event.data.call : undefined; - if (!isRecord(call) || typeof call.name !== "string") return false; - const step: Step = { name: call.name, kind: "tool", input: call.arguments }; + if (!isRecord(call)) return false; + const id = typeof call.id === "string" ? call.id : undefined; + const known = id === undefined ? undefined : tools.get(id); + const name = toolName(call.name) ?? known?.name; + if (name === undefined) return false; + if (known?.step !== undefined) return true; + const step: Step = { name, kind: "tool", input: call.arguments ?? known?.input }; steps.push(step); - if (typeof call.id === "string") tools.set(call.id, step); + if (id !== undefined) tools.set(id, { name, input: step.input, step }); return true; } if (event.type === "tool.done") { const result = isRecord(event.data) ? event.data.result : undefined; if (!isRecord(result) || typeof result.callId !== "string") return false; - const step = tools.get(result.callId); - if (step === undefined) return false; + const known = tools.get(result.callId); + if (known === undefined) return false; + let step = known.step; + if (step === undefined) { + step = { name: known.name, kind: "tool", input: known.input }; + steps.push(step); + known.step = step; + } step.output = "content" in result ? result.content : result; + return true; } return false; } @@ -158,21 +205,35 @@ async function nextEventOrIdle( } } +/** The reactor settles the in-flight send on these, then stops this turn. */ +function isTurnEnd(event: StreamEvent): boolean { + if ( + event.type === "connector.reply" || + event.type === "reactor.gate.blocked" || + event.type === "reactor.done" + ) { + return true; + } + return event.type === "reactor.error" && isRecord(event.data) && event.data.fatal === true; +} async function collect( iterator: AsyncIterator, first: Promise>, steps: Step[], send: Promise, + signal: AbortSignal, ): Promise { let saw = false; - const tools = new Map(); + const tools = new Map(); let sendSettled = false; + let sendFailed = false; const mark = send.then( () => { sendSettled = true; }, () => { sendSettled = true; + sendFailed = true; }, ); const note = (event: StreamEvent): void => { @@ -189,7 +250,19 @@ async function collect( note(winner.result.value); next = iterator.next(); } + // Abort rejects `send` while the reactor keeps emitting. Hold this + // subscription until the turn ends so a late tool cannot be recorded + // on the next case, where it would also suppress that case's model step. + let holdForTurn = sendFailed && signal.aborted; for (;;) { + if (holdForTurn) { + const result = await next; + if (result.done === true) break; + note(result.value); + next = iterator.next(); + if (isTurnEnd(result.value)) holdForTurn = false; + continue; + } const winner = await nextEventOrIdle(next); if (winner.idle) { const ready = await takeReady(next); @@ -241,7 +314,7 @@ async function sendAgent( settled = { ok: false, error }; }, ); - const saw = await collect(iterator, first.pending, steps, send); + const saw = await collect(iterator, first.pending, steps, send, signal); await send; if (settled === undefined || !settled.ok) { throw settled === undefined ? new Error("agent send did not settle") : settled.error; From 533d676878cd2ae22caf6842ef5c060c071f8b2d Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 26 Oct 2026 21:12:10 -0700 Subject: [PATCH 08/16] fix: hold an aborted turn on the raw send rejection --- e2e/agent-trajectory.e2e.test.ts | 8 ++++++-- src/runs.ts | 7 ++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/e2e/agent-trajectory.e2e.test.ts b/e2e/agent-trajectory.e2e.test.ts index 37462ee..4d2449f 100644 --- a/e2e/agent-trajectory.e2e.test.ts +++ b/e2e/agent-trajectory.e2e.test.ts @@ -301,7 +301,11 @@ describe("agent trajectory", () => { stream: true, onSend: (input, emit, signal) => { if (input !== "slow") { - return Promise.resolve({ type: "reply", reply: "found", turn: { model: "stub" } }); + return new Promise((resolve) => { + setTimeout(() => { + resolve({ type: "reply", reply: "found", turn: { model: "stub" } }); + }, 80); + }); } return new Promise((_resolve, reject) => { const abort = (): void => { @@ -312,7 +316,7 @@ describe("agent trajectory", () => { data: { call: { id: "late", name: "search", arguments: { q: "slow" } } }, }); emit({ type: "connector.reply", data: { content: "late" } }); - }, 20); + }, 30); }; if (signal.aborted) abort(); else signal.addEventListener("abort", abort, { once: true }); diff --git a/src/runs.ts b/src/runs.ts index ce5cab2..435f34b 100644 --- a/src/runs.ts +++ b/src/runs.ts @@ -220,7 +220,7 @@ async function collect( iterator: AsyncIterator, first: Promise>, steps: Step[], - send: Promise, + send: Promise, signal: AbortSignal, ): Promise { let saw = false; @@ -306,7 +306,8 @@ async function sendAgent( try { const first = await discardBuffered(iterator); let settled: { ok: true; res: AgentSendResult } | { ok: false; error: unknown } | undefined; - const send = agent.send(input, { signal }).then( + const pending = agent.send(input, { signal }); + const send = pending.then( (res) => { settled = { ok: true, res }; }, @@ -314,7 +315,7 @@ async function sendAgent( settled = { ok: false, error }; }, ); - const saw = await collect(iterator, first.pending, steps, send, signal); + const saw = await collect(iterator, first.pending, steps, pending, signal); await send; if (settled === undefined || !settled.ok) { throw settled === undefined ? new Error("agent send did not settle") : settled.error; From 33f9d10f51ca30d348777669ebb871b399131c29 Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 02:24:41 -0700 Subject: [PATCH 09/16] feat: add the corbits-eval CLI --- README.md | 29 +++++++++++++++++ bun.lock | 17 ++++++++++ e2e/cli.e2e.test.ts | 34 ++++++++++++++++++++ e2e/fixtures/fail.ts | 5 +++ e2e/fixtures/missing-export.ts | 1 + e2e/fixtures/pass.ts | 5 +++ package.json | 9 ++++-- src/cli/index.ts | 12 +++++++ src/cli/run.ts | 30 ++++++++++++++++++ src/evalFile.ts | 57 ++++++++++++++++++++++++++++++++++ 10 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 e2e/cli.e2e.test.ts create mode 100644 e2e/fixtures/fail.ts create mode 100644 e2e/fixtures/missing-export.ts create mode 100644 e2e/fixtures/pass.ts create mode 100644 src/cli/index.ts create mode 100644 src/cli/run.ts create mode 100644 src/evalFile.ts diff --git a/README.md b/README.md index 0210ce3..309a59d 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,35 @@ evaluation, with no file: `runEvaluation({ ...spec, compare: before.experiment } Flips print as `fixed` / `broke`, new and gone cases as `added` / `removed`. `quiet: true` skips the summary. +## CLI + +The package bin is `corbits-eval` (`./dist/cli/index.js`). + +```sh +corbits-eval run ./evals/refund.ts +corbits-eval run ./evals/refund.ts --format json +``` + +The file is an ES module path resolved from the current working directory. +`corbits-eval` loads the `evaluation` export when it is a spec object, otherwise +the default export when that is a spec object, otherwise a zero-argument +function on either of those exports. The function must return a spec. No other +exports are read. + +A spec is the object `runEvaluation` accepts (`name`, `data`, `task`, +`scorers`, and the rest). Case rows use `expectedOutput`. The CLI calls +`runEvaluation`; it does not score on its own. + +Exit codes: + +| code | when | +| ---- | ------------------------------------------------------------------ | +| 0 | the run passed its gate | +| 1 | the evaluation failed its gate | +| 2 | the file is missing, the import failed, or there is no spec export | + +`--format json` (and `--json`) come from incur and print the result as JSON. + ## Calibrating Thresholds are per-workload starting points, not constants. `calibrate` diff --git a/bun.lock b/bun.lock index 78def42..9de4ace 100644 --- a/bun.lock +++ b/bun.lock @@ -6,6 +6,7 @@ "name": "@corbits/evals", "dependencies": { "@corbits/system-one": "^0.3.0", + "incur": "^0.7.1", }, "devDependencies": { "@intx/agent": "^0.5.0", @@ -30,6 +31,8 @@ "@ark/util": ["@ark/util@0.56.2", "", {}, "sha512-9kU2sUE38FZEGG7l3hamYMBieLYEJh2L1mrYD2eXpT+78EnQSV1bhjxJhnxGBMSTbtwpBSDNSK+K60WvaI/DTQ=="], + "@cfworker/json-schema": ["@cfworker/json-schema@4.1.1", "", {}, "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og=="], + "@corbits/system-one": ["@corbits/system-one@0.3.0", "", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": "^0.4.0", "@intx/types": "^0.4.0" } }, "sha512-rH1MSTyTph1O3wRjVuh2gbunfPLXK0HFDjOym9CkXtcDpRa6J+sWiDsnHdIla3KnVxPWQ2dTbYYck6KlTghtIw=="], "@intx/agent": ["@intx/agent@0.5.0", "", { "dependencies": { "@intx/inference": "0.5.0", "@intx/log": "0.5.0", "@intx/mime": "0.5.0", "@intx/types": "0.5.0", "arktype": "^2.1.29" } }, "sha512-EWkRfduYJii67goq0cdO6qgRtV0aLe2jGk2aZJhurOivluy3tx0SlhiO2YaQmA4du+qBNzlMV9MMfKkmQ/zs+Q=="], @@ -50,6 +53,8 @@ "@logtape/logtape": ["@logtape/logtape@2.3.11", "", {}, "sha512-T7T+IShQBxofHCs7J1kVMZPLLHYAXAOqPEc/xwzd6/2q8va/D/t/HP7uCeJzTQOywpXF1tbyz15O4VHT8NF2hw=="], + "@modelcontextprotocol/server": ["@modelcontextprotocol/server@2.0.0-alpha.4", "", { "dependencies": { "zod": "^4.2.0" } }, "sha512-/KEo3ZJ50HlagHp0lz2vPgfBZFtXHu6zTBXT9XqPc+4O9i4+IbBVWKq/a9yAfv/ifp0u3+mRo8SUk5kMKzhN8A=="], + "@oxfmt/binding-android-arm-eabi": ["@oxfmt/binding-android-arm-eabi@0.70.0", "", { "os": "android", "cpu": "arm" }, "sha512-Xd7YO4/T2axEj6FTLcj4Why3mTBqFMg+x24xtorT4Lb2+1g82090GH0a/4U1m0pABGYiix2bq1pqkYrmV3f0Sw=="], "@oxfmt/binding-android-arm64": ["@oxfmt/binding-android-arm64@0.70.0", "", { "os": "android", "cpu": "arm64" }, "sha512-x9rlMYyKXdgKdYyUJzGsK1ZV8P4di/J32ipzcS6Jet6p9r9UAh28neXIMtdlSaJJycdi61Z4YkcLKLpk8ueFjg=="], @@ -126,6 +131,10 @@ "@oxlint/binding-win32-x64-msvc": ["@oxlint/binding-win32-x64-msvc@1.85.0", "", { "os": "win32", "cpu": "x64" }, "sha512-pBebIPUpKKhWrhSMWhy8TdAZBewiXnfxmaAGxhzxM1068GagqFaTwgKlU6e+UyJ2sPR+VoHouhXuGJkQjsrDvA=="], + "@scalar/openapi-types": ["@scalar/openapi-types@0.8.0", "", {}, "sha512-WmaxVSfvY5K/TwcG2B2TU1WOe1As1uc2s7myswtP6dBlcjU3hM08SApxv/jmyGaCE8t4gO5BBhmHY4pDUfmr2g=="], + + "@toon-format/toon": ["@toon-format/toon@2.3.1", "", {}, "sha512-sWqEMeAJGMda9qRrfPKrX3C4c33CO9SJuvJzzrcBEn5sbDB+k6pSLkDsfN0rVnLQwR97MV3sDgVbcxQNsq8xVw=="], + "@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], "@types/node": ["@types/node@26.6.4", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg=="], @@ -138,6 +147,8 @@ "hono": ["hono@4.13.13", "", {}, "sha512-CQ46U0ZkAGmbT/4UxdzzGJpacP2IeKgY4a5/tOI9AABbpOMfK739wfDXmv1usCk+3RkKj1hQy4/fjhiwa2xlrA=="], + "incur": ["incur@0.7.1", "", { "dependencies": { "@cfworker/json-schema": "^4.1.1", "@modelcontextprotocol/server": "2.0.0-alpha.4", "@scalar/openapi-types": "^0.8.0", "@toon-format/toon": "^2.3.1", "tokenx": "^1.3.0", "yaml": "^2.8.2", "zod": "^4.3.6" }, "bin": { "incur": "dist/cli/index.js", "incur.src": "src/cli/index.ts" } }, "sha512-tXEStxASXtVWOjY7wNBBMy03+3HPpiLPkrEaqNb5iYerRNRWqTYqwdAEgLfU6Ga2OUrz9LpRuMWFvdj5kIaUlg=="], + "oxfmt": ["oxfmt@0.70.0", "", { "dependencies": { "tinypool": "2.1.2" }, "optionalDependencies": { "@oxfmt/binding-android-arm-eabi": "0.70.0", "@oxfmt/binding-android-arm64": "0.70.0", "@oxfmt/binding-darwin-arm64": "0.70.0", "@oxfmt/binding-darwin-x64": "0.70.0", "@oxfmt/binding-freebsd-x64": "0.70.0", "@oxfmt/binding-linux-arm-gnueabihf": "0.70.0", "@oxfmt/binding-linux-arm-musleabihf": "0.70.0", "@oxfmt/binding-linux-arm64-gnu": "0.70.0", "@oxfmt/binding-linux-arm64-musl": "0.70.0", "@oxfmt/binding-linux-ppc64-gnu": "0.70.0", "@oxfmt/binding-linux-riscv64-gnu": "0.70.0", "@oxfmt/binding-linux-riscv64-musl": "0.70.0", "@oxfmt/binding-linux-s390x-gnu": "0.70.0", "@oxfmt/binding-linux-x64-gnu": "0.70.0", "@oxfmt/binding-linux-x64-musl": "0.70.0", "@oxfmt/binding-openharmony-arm64": "0.70.0", "@oxfmt/binding-win32-arm64-msvc": "0.70.0", "@oxfmt/binding-win32-ia32-msvc": "0.70.0", "@oxfmt/binding-win32-x64-msvc": "0.70.0" }, "peerDependencies": { "svelte": "^5.0.0", "vite-plus": "*" }, "optionalPeers": ["svelte", "vite-plus"], "bin": { "oxfmt": "bin/oxfmt" } }, "sha512-IsHxZ4y0wQLLMhnrJblBJgZsLDzfULrJnAw5j/QqsTlMa/m3AqsbToi+W71uhBGaqlqq/PbbjvHc09TJwdv3Tw=="], "oxlint": ["oxlint@1.85.0", "", { "optionalDependencies": { "@oxlint/binding-android-arm-eabi": "1.85.0", "@oxlint/binding-android-arm64": "1.85.0", "@oxlint/binding-darwin-arm64": "1.85.0", "@oxlint/binding-darwin-x64": "1.85.0", "@oxlint/binding-freebsd-x64": "1.85.0", "@oxlint/binding-linux-arm-gnueabihf": "1.85.0", "@oxlint/binding-linux-arm-musleabihf": "1.85.0", "@oxlint/binding-linux-arm64-gnu": "1.85.0", "@oxlint/binding-linux-arm64-musl": "1.85.0", "@oxlint/binding-linux-ppc64-gnu": "1.85.0", "@oxlint/binding-linux-riscv64-gnu": "1.85.0", "@oxlint/binding-linux-riscv64-musl": "1.85.0", "@oxlint/binding-linux-s390x-gnu": "1.85.0", "@oxlint/binding-linux-x64-gnu": "1.85.0", "@oxlint/binding-linux-x64-musl": "1.85.0", "@oxlint/binding-openharmony-arm64": "1.85.0", "@oxlint/binding-win32-arm64-msvc": "1.85.0", "@oxlint/binding-win32-ia32-msvc": "1.85.0", "@oxlint/binding-win32-x64-msvc": "1.85.0" }, "peerDependencies": { "oxlint-tsgolint": ">=7.0.2001", "vite-plus": "*" }, "optionalPeers": ["oxlint-tsgolint", "vite-plus"], "bin": { "oxlint": "bin/oxlint" } }, "sha512-bc26s97nuvPj1ViyPsqmKecVkUWFMEdtayO8MaQ6oiLfs1pj94cQlZZhrh4BPNlr9HQosjhIlwgZKsfcwmcNgg=="], @@ -146,10 +157,16 @@ "tinypool": ["tinypool@2.1.2", "", {}, "sha512-9YodfrxS9g9IbFr/KOjE5bAeJ0p61n3bW6mqvy0jtoeKd1kTW1Cxm0oulm6KX2lyM9Gl6WIe8nEbY7LWv5ZJww=="], + "tokenx": ["tokenx@1.6.0", "", {}, "sha512-CKTjk345ajvBAUp5xUI9a5KKN0zU0lBueVHQbCskH1Hp6WkUKsPW2qGCYNs0pxNyfzxfo+IIjdt2W4sMbw/qBw=="], + "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], "undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="], + "yaml": ["yaml@2.9.1", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw=="], + + "zod": ["zod@4.6.7", "", {}, "sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg=="], + "@corbits/system-one/@intx/inference": ["@intx/inference@0.4.0", "", { "dependencies": { "@intx/log": "0.4.0", "@intx/types": "0.4.0", "arktype": "^2.1.29" } }, "sha512-cuaqCqotmjg0OzGUm8QmFJr1xX5wPVS0LeuhrxHsj0X3ImoCzVB4e/ttTc+DaP3K1tTMQ/8Q8HHCAPX36mOecQ=="], "@corbits/system-one/@intx/inference/@intx/log": ["@intx/log@0.4.0", "", { "dependencies": { "@logtape/hono": "^2.0.2", "@logtape/logtape": "^2.0.2" }, "peerDependencies": { "hono": "^4.0.0" }, "optionalPeers": ["hono"] }, "sha512-f1IPZ/9YpEqlA0I2UZPOlRJOTDoOFEZRwZoUVuw8DpgR1WatOMm4hA0E8o6Mtz+pA0ux7gVdls4y/JZcgypsfQ=="], diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts new file mode 100644 index 0000000..dd15919 --- /dev/null +++ b/e2e/cli.e2e.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; + +const root = join(import.meta.dir, ".."); +const cli = join(root, "src/cli/index.ts"); +const fixtures = join(import.meta.dir, "fixtures"); + +async function run(file: string): Promise { + const proc = Bun.spawn(["bun", cli, "run", file], { + cwd: root, + stdout: "pipe", + stderr: "pipe", + }); + await proc.exited; + return proc.exitCode ?? 1; +} + +describe("corbits-eval run", () => { + test("a passing evaluation exits 0", async () => { + expect(await run(join(fixtures, "pass.ts"))).toBe(0); + }); + + test("a failed gate exits 1", async () => { + expect(await run(join(fixtures, "fail.ts"))).toBe(1); + }); + + test("a module with no spec exits 2", async () => { + expect(await run(join(fixtures, "missing-export.ts"))).toBe(2); + }); + + test("a missing file exits 2", async () => { + expect(await run(join(fixtures, "does-not-exist.ts"))).toBe(2); + }); +}); diff --git a/e2e/fixtures/fail.ts b/e2e/fixtures/fail.ts new file mode 100644 index 0000000..aee0080 --- /dev/null +++ b/e2e/fixtures/fail.ts @@ -0,0 +1,5 @@ +export const evaluation = { + name: "fail", + task: (input: string) => input.toUpperCase(), + data: [{ input: "hi", expectedOutput: "nope" }], +}; diff --git a/e2e/fixtures/missing-export.ts b/e2e/fixtures/missing-export.ts new file mode 100644 index 0000000..6437c87 --- /dev/null +++ b/e2e/fixtures/missing-export.ts @@ -0,0 +1 @@ +export const note = "not a spec"; diff --git a/e2e/fixtures/pass.ts b/e2e/fixtures/pass.ts new file mode 100644 index 0000000..a0732e6 --- /dev/null +++ b/e2e/fixtures/pass.ts @@ -0,0 +1,5 @@ +export const evaluation = { + name: "pass", + task: (input: string) => input.toUpperCase(), + data: [{ input: "hi", expectedOutput: "HI" }], +}; diff --git a/package.json b/package.json index 4cc752a..6fc7452 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,9 @@ "type": "git", "url": "git+https://github.com/corbitsdev/corbits-evals.git" }, + "bin": { + "corbits-eval": "./dist/cli/index.js" + }, "files": [ "dist", "README.md", @@ -51,7 +54,8 @@ "check": "bun run typecheck && bun run lint && bun run format:check && bun run test && bun run test:e2e" }, "dependencies": { - "@corbits/system-one": "^0.3.0" + "@corbits/system-one": "^0.3.0", + "incur": "^0.7.1" }, "devDependencies": { "@intx/agent": "^0.5.0", @@ -72,6 +76,7 @@ } }, "engines": { - "bun": ">=1.2.0" + "bun": ">=1.2.0", + "node": ">=22" } } diff --git a/src/cli/index.ts b/src/cli/index.ts new file mode 100644 index 0000000..573f412 --- /dev/null +++ b/src/cli/index.ts @@ -0,0 +1,12 @@ +#!/usr/bin/env bun +import { Cli } from "incur"; + +const cli = Cli.create("corbits-eval", { + description: "Run a corbits evaluation file", +}).fs(new URL("./", import.meta.url)); + +if (import.meta.main) { + await cli.serve(); +} + +export { cli }; diff --git a/src/cli/run.ts b/src/cli/run.ts new file mode 100644 index 0000000..f1f08d5 --- /dev/null +++ b/src/cli/run.ts @@ -0,0 +1,30 @@ +import { Cli, z } from "incur"; + +import { loadEvalFile } from "../evalFile.js"; +import { EvalFailure, runEvaluation } from "../evaluation.js"; + +export default Cli.command({ + description: "Run an evaluation module", + args: z.object({ + file: z.string().describe("Evaluation module path, resolved from the current directory"), + }), + async run(c) { + let spec; + try { + spec = await loadEvalFile(c.args.file); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + return c.error({ code: "EVAL_FILE", message, exitCode: 2 }); + } + + try { + const report = await runEvaluation(spec); + return { name: report.name, ok: report.ok, passed: report.passed, total: report.total }; + } catch (cause) { + if (cause instanceof EvalFailure) { + return c.error({ code: "EVAL_FAILED", message: cause.message, exitCode: 1 }); + } + throw cause; + } + }, +}); diff --git a/src/evalFile.ts b/src/evalFile.ts new file mode 100644 index 0000000..cd5a337 --- /dev/null +++ b/src/evalFile.ts @@ -0,0 +1,57 @@ +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +import type { Evaluation } from "./evaluation.js"; + +/** A missing file, a failed import, or a module with no evaluation spec. */ +export class EvalFileError extends Error { + constructor(message: string) { + super(message); + this.name = "EvalFileError"; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function isSpec(value: unknown): value is Evaluation { + return isRecord(value) && typeof value["name"] === "string" && "data" in value && "task" in value; +} + +function isZeroArg(value: unknown): value is () => unknown { + return typeof value === "function" && value.length === 0; +} + +async function fromFactory( + value: unknown, +): Promise | undefined> { + if (!isZeroArg(value)) return undefined; + const produced: unknown = await value(); + if (isSpec(produced)) return produced; + throw new EvalFileError("evaluation function did not return a spec"); +} + +/** + * Load an evaluation module. Uses `evaluation` when it is a spec, else `default` + * when it is a spec, else a zero-arg function on either export. + */ +export async function loadEvalFile(file: string): Promise> { + const href = pathToFileURL(resolve(file)).href; + let mod: Record; + try { + mod = (await import(href)) as Record; + } catch (cause) { + const detail = cause instanceof Error ? cause.message : String(cause); + throw new EvalFileError(`could not import ${file}: ${detail}`); + } + + const evaluation = mod["evaluation"]; + const fallback = mod["default"]; + if (isSpec(evaluation)) return evaluation; + if (isSpec(fallback)) return fallback; + + const produced = (await fromFactory(evaluation)) ?? (await fromFactory(fallback)); + if (produced !== undefined) return produced; + throw new EvalFileError(`${file} has no evaluation spec export`); +} From e20e9d8da994a7d44bda66f1bc55af7afe266a4d Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 26 Oct 2026 20:44:03 -0700 Subject: [PATCH 10/16] fix: print only json and exit 2 when an eval does not run JSON output sets quiet so the human summary does not prefix stdout. A module that is not a runnable spec is rejected at load time. Config errors that never start a run exit 2 instead of 1. --- README.md | 12 +++---- e2e/cli.e2e.test.ts | 70 +++++++++++++++++++++++++++++++++---- e2e/fixtures/empty.ts | 5 +++ e2e/fixtures/needs-judge.ts | 6 ++++ e2e/fixtures/not-a-spec.ts | 1 + src/cli/run.ts | 5 +-- src/evalFile.ts | 20 ++++++++++- 7 files changed, 103 insertions(+), 16 deletions(-) create mode 100644 e2e/fixtures/empty.ts create mode 100644 e2e/fixtures/needs-judge.ts create mode 100644 e2e/fixtures/not-a-spec.ts diff --git a/README.md b/README.md index 309a59d..5825640 100644 --- a/README.md +++ b/README.md @@ -186,13 +186,13 @@ A spec is the object `runEvaluation` accepts (`name`, `data`, `task`, Exit codes: -| code | when | -| ---- | ------------------------------------------------------------------ | -| 0 | the run passed its gate | -| 1 | the evaluation failed its gate | -| 2 | the file is missing, the import failed, or there is no spec export | +| code | when | +| ---- | ---------------------------------------------------------------------------------------------- | +| 0 | the run passed its gate | +| 1 | the evaluation failed its gate | +| 2 | the file is missing, the import failed, there is no spec export, or the evaluation did not run | -`--format json` (and `--json`) come from incur and print the result as JSON. +`--format json` (and `--json`) come from incur and print only that result as JSON on stdout, with no human summary. Exit 1 is only a failed gate (`EVAL_FAILED`). Config that stops the run before it starts, such as empty `data` or a string or `ask` scorer with no `judge.invoke`, exits 2. ## Calibrating diff --git a/e2e/cli.e2e.test.ts b/e2e/cli.e2e.test.ts index dd15919..e49c7bd 100644 --- a/e2e/cli.e2e.test.ts +++ b/e2e/cli.e2e.test.ts @@ -5,30 +5,86 @@ const root = join(import.meta.dir, ".."); const cli = join(root, "src/cli/index.ts"); const fixtures = join(import.meta.dir, "fixtures"); -async function run(file: string): Promise { - const proc = Bun.spawn(["bun", cli, "run", file], { +async function run( + file: string, + args: readonly string[] = [], +): Promise<{ code: number; stdout: string; stderr: string }> { + const proc = Bun.spawn(["bun", cli, "run", file, ...args], { cwd: root, stdout: "pipe", stderr: "pipe", }); + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); await proc.exited; - return proc.exitCode ?? 1; + return { code: proc.exitCode ?? 1, stdout, stderr }; +} + +function jsonBody(stdout: string): unknown { + return JSON.parse(stdout); } describe("corbits-eval run", () => { test("a passing evaluation exits 0", async () => { - expect(await run(join(fixtures, "pass.ts"))).toBe(0); + expect((await run(join(fixtures, "pass.ts"))).code).toBe(0); }); test("a failed gate exits 1", async () => { - expect(await run(join(fixtures, "fail.ts"))).toBe(1); + expect((await run(join(fixtures, "fail.ts"))).code).toBe(1); }); test("a module with no spec exits 2", async () => { - expect(await run(join(fixtures, "missing-export.ts"))).toBe(2); + expect((await run(join(fixtures, "missing-export.ts"))).code).toBe(2); }); test("a missing file exits 2", async () => { - expect(await run(join(fixtures, "does-not-exist.ts"))).toBe(2); + expect((await run(join(fixtures, "does-not-exist.ts"))).code).toBe(2); + }); + + test("--format json stdout is only JSON for a pass and a gate failure", async () => { + const passed = await run(join(fixtures, "pass.ts"), ["--format", "json"]); + expect(passed.code).toBe(0); + expect(passed.stdout).not.toContain("cases passed"); + expect(jsonBody(passed.stdout)).toMatchObject({ name: "pass", ok: true }); + + const failed = await run(join(fixtures, "fail.ts"), ["--format", "json"]); + expect(failed.code).toBe(1); + expect(failed.stdout).not.toContain("cases passed"); + expect(jsonBody(failed.stdout)).toMatchObject({ code: "EVAL_FAILED" }); + }); + + test("--json stdout is only JSON for a pass and a gate failure", async () => { + const passed = await run(join(fixtures, "pass.ts"), ["--json"]); + expect(passed.code).toBe(0); + expect(passed.stdout).not.toContain("cases passed"); + expect(jsonBody(passed.stdout)).toMatchObject({ name: "pass", ok: true }); + + const failed = await run(join(fixtures, "fail.ts"), ["--json"]); + expect(failed.code).toBe(1); + expect(failed.stdout).not.toContain("cases passed"); + expect(jsonBody(failed.stdout)).toMatchObject({ code: "EVAL_FAILED" }); + }); + + test("a non-spec export exits 2", async () => { + const result = await run(join(fixtures, "not-a-spec.ts")); + expect(result.code).toBe(2); + expect(result.stdout).toContain("no evaluation spec"); + expect(result.stdout).not.toContain("cases.map is not a function"); + }); + + test("empty data exits 2", async () => { + const result = await run(join(fixtures, "empty.ts")); + expect(result.code).toBe(2); + expect(result.stdout).toContain("EVAL_CONFIG"); + expect(result.stdout).toContain("has no data"); + }); + + test("a string scorer with no judge exits 2", async () => { + const result = await run(join(fixtures, "needs-judge.ts")); + expect(result.code).toBe(2); + expect(result.stdout).toContain("EVAL_CONFIG"); + expect(result.stdout).toContain("string and ask checks need judge.invoke"); }); }); diff --git a/e2e/fixtures/empty.ts b/e2e/fixtures/empty.ts new file mode 100644 index 0000000..7002e16 --- /dev/null +++ b/e2e/fixtures/empty.ts @@ -0,0 +1,5 @@ +export const evaluation = { + name: "empty", + task: (input: string) => input, + data: [], +}; diff --git a/e2e/fixtures/needs-judge.ts b/e2e/fixtures/needs-judge.ts new file mode 100644 index 0000000..3ed3db4 --- /dev/null +++ b/e2e/fixtures/needs-judge.ts @@ -0,0 +1,6 @@ +export const evaluation = { + name: "needs-judge", + task: (input: string) => input, + data: [{ input: "hi" }], + scorers: { fine: "is it fine?" }, +}; diff --git a/e2e/fixtures/not-a-spec.ts b/e2e/fixtures/not-a-spec.ts new file mode 100644 index 0000000..a68f193 --- /dev/null +++ b/e2e/fixtures/not-a-spec.ts @@ -0,0 +1 @@ +export const evaluation = { name: "bad", data: 1, task: "nope" }; diff --git a/src/cli/run.ts b/src/cli/run.ts index f1f08d5..241e551 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -18,13 +18,14 @@ export default Cli.command({ } try { - const report = await runEvaluation(spec); + const report = await runEvaluation(c.format === "json" ? { ...spec, quiet: true } : spec); return { name: report.name, ok: report.ok, passed: report.passed, total: report.total }; } catch (cause) { if (cause instanceof EvalFailure) { return c.error({ code: "EVAL_FAILED", message: cause.message, exitCode: 1 }); } - throw cause; + const message = cause instanceof Error ? cause.message : String(cause); + return c.error({ code: "EVAL_CONFIG", message, exitCode: 2 }); } }, }); diff --git a/src/evalFile.ts b/src/evalFile.ts index cd5a337..430c0b9 100644 --- a/src/evalFile.ts +++ b/src/evalFile.ts @@ -2,6 +2,7 @@ import { resolve } from "node:path"; import { pathToFileURL } from "node:url"; import type { Evaluation } from "./evaluation.js"; +import { TRACE } from "./runs.js"; /** A missing file, a failed import, or a module with no evaluation spec. */ export class EvalFileError extends Error { @@ -15,8 +16,25 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } +/** A task `toTraced` can run: a function, `workflow()`, or an agent with `send`. */ +function isRunnable(value: unknown): boolean { + if (typeof value === "function") return true; + if (!isRecord(value)) return false; + if (TRACE in value) return true; + return typeof value["send"] === "function"; +} + +function isData(value: unknown): boolean { + return Array.isArray(value) || typeof value === "string"; +} + function isSpec(value: unknown): value is Evaluation { - return isRecord(value) && typeof value["name"] === "string" && "data" in value && "task" in value; + return ( + isRecord(value) && + typeof value["name"] === "string" && + isData(value["data"]) && + isRunnable(value["task"]) + ); } function isZeroArg(value: unknown): value is () => unknown { From 00bd196307669fc19cbe76d46d90fe6a960200e0 Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 02:27:21 -0700 Subject: [PATCH 11/16] feat: rename the jev constructor to systemOne --- README.md | 7 ++++--- e2e/jev.e2e.test.ts | 8 ++++---- src/index.ts | 1 + src/jev.ts | 8 ++++++-- src/workflow.ts | 4 ++-- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 5825640..b4bdbf1 100644 --- a/README.md +++ b/README.md @@ -70,12 +70,13 @@ A check that throws, hangs past `timeout`, or returns anything else fails itself closed; its siblings still run. Jev transport failures fail closed. Jev sees `{ input, output, expectedOutput, steps }`, and a case's string and -`ask` checks answer in one call. The model is not configured on `jev()`. +`ask` checks answer in one call. The model is not configured on `systemOne()`. Build it with `judgeModel({ model, invokeStep, thresholds, escalate, timeout })` from `@corbits/evals/workflow`. `model` is one workflow step (or a one-step workflow). `invokeStep` runs that step; its input is `{ state, questions }` and its output is `{ decisions }` or `{ fallback: true, reason }`. -`jev({ thresholds, escalate })` only sets bars and an escalation agent. +`systemOne({ thresholds, escalate })` only sets bars and an escalation agent. +`jev` is an alias of `systemOne`. Thresholds: a yes/no check passes at p(yes) >= `yes` and fails at p(yes) <= `no` (0.7 / 0.3); `ask` fails as unsure when its top @@ -85,7 +86,7 @@ Thresholds are tuned to a judge version, so pin the step agent's model when they must stay stable. Escalation: `sure` doubles as the escalation gate. Give the judge an -agent — `escalate` on `jev` or `judgeModel`, anything with +agent — `escalate` on `systemOne` or `judgeModel`, anything with `send(prompt)` returning `{ type: "reply", reply }` or `{ type: "suspended" }`, e.g. an `@intx/agent` agent from a `@corbits/*` adapter — and every unsure decision is re-asked with the exact answer diff --git a/e2e/jev.e2e.test.ts b/e2e/jev.e2e.test.ts index 65328a2..7c801ff 100644 --- a/e2e/jev.e2e.test.ts +++ b/e2e/jev.e2e.test.ts @@ -16,7 +16,7 @@ import { ask, EvalFailure, runEvaluation, - jev, + systemOne, type Scorer, type Escalation, type Judge, @@ -253,8 +253,8 @@ describe("thresholds", () => { }); test("bad thresholds throw at definition", async () => { - expect(() => jev({ thresholds: { yes: 0.3, no: 0.5 } })).toThrow("must not exceed"); - expect(() => jev({ thresholds: { sure: 1.5 } })).toThrow("[0, 1]"); + expect(() => systemOne({ thresholds: { yes: 0.3, no: 0.5 } })).toThrow("must not exceed"); + expect(() => systemOne({ thresholds: { sure: 1.5 } })).toThrow("[0, 1]"); expect(() => ask("Ready?", { options: { a: "a" }, pass: "a", sure: 0 })).toThrow("(0, 1]"); const { judge } = stubJev(noul(0.9)); await expect(judged({ question, yes: 0.2, no: 0.5 }, judge)).rejects.toThrow( @@ -596,7 +596,7 @@ describe("judgeModel", () => { task: (input: string) => input, data: "/no/such/cases.jsonl", scorers: { ready: ask("Ready?", { options: { a: "a" }, pass: "a" }) }, - judge: jev({ thresholds: { yes: 0.9 } }), + judge: systemOne({ thresholds: { yes: 0.9 } }), }), ).rejects.toThrow("judge.invoke"); const report = await runEvaluation({ diff --git a/src/index.ts b/src/index.ts index 5ab8218..7011680 100644 --- a/src/index.ts +++ b/src/index.ts @@ -42,6 +42,7 @@ export { export type { Experiment, ExperimentRow } from "./reports.js"; export { ask, + systemOne, jev, DEFAULTS, type Ask, diff --git a/src/jev.ts b/src/jev.ts index 107a081..060e072 100644 --- a/src/jev.ts +++ b/src/jev.ts @@ -29,7 +29,7 @@ export type Judge = { readonly thresholds?: Resolved; readonly escalate?: Escalation; /** - * One call per case attempt. `judgeModel` sets this; `jev()` alone does not, + * One call per case attempt. `judgeModel` sets this; `systemOne()` alone does not, * so it is not a model. * Resolves to `{ decisions }` or `{ fallback: true, reason, detail? }`. */ @@ -63,7 +63,8 @@ function resolve(label: string, raw: Thresholds): Resolved { return out; } -export function jev(options: JevOptions = {}): Judge { +/** Thresholds and an optional escalation agent. Not a model and not an HTTP client. */ +export function systemOne(options: JevOptions = {}): Judge { const thresholds = options.thresholds === undefined ? undefined : resolve("thresholds", options.thresholds); return { @@ -72,6 +73,9 @@ export function jev(options: JevOptions = {}): Judge { }; } +/** Same function as `systemOne`. */ +export { systemOne as jev }; + /** A Jev question with named answers. A plain string check is the yes/no case. */ export type Ask = { readonly question: string; diff --git a/src/workflow.ts b/src/workflow.ts index 1d17c10..810b48e 100644 --- a/src/workflow.ts +++ b/src/workflow.ts @@ -10,7 +10,7 @@ import { } from "@intx/workflow"; import { TRACE, type Traced } from "./runs.js"; -import { jev, type Escalation, type Judge, type Thresholds } from "./jev.js"; +import { systemOne, type Escalation, type Judge, type Thresholds } from "./jev.js"; import { toMs, type Duration } from "./timeout.js"; import type { Step } from "./types.js"; @@ -165,7 +165,7 @@ export function judgeModel(options: JudgeModelOptions): Judge { trigger: { type: "manual" }, steps: { judge: forced }, }); - const base = jev({ + const base = systemOne({ ...(options.thresholds === undefined ? {} : { thresholds: options.thresholds }), ...(options.escalate === undefined ? {} : { escalate: options.escalate }), }); From bbb6ba8fb644558b65666fc9670220245a342d49 Mon Sep 17 00:00:00 2001 From: "user.email" Date: Mon, 26 Oct 2026 02:29:14 -0700 Subject: [PATCH 12/16] docs: describe the current eval API State format 2 experiments, expectedOutput JSONL rows, and ranSteps([]) on an empty step list. --- AGENTS.md | 16 ++++++++++++++++ README.md | 50 ++++++++++++++++++++++++++++++++++---------------- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6bc8f7c..8ed0d6c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,6 +4,22 @@ `@corbits/evals` package. +Public call: `runEvaluation({ name, data, task, scorers })`. There is no +`defineEval`, no `evaluate(name, options)`, and no `runEval`. Ground truth +is `expectedOutput`. JSONL keys `expect` and `expected` throw. + +The judge model is `judgeModel()` from `@corbits/evals/workflow`: one step, +inference source pinned on the agent. Thresholds are `systemOne`. `jev` is +an alias of `systemOne`, not an endpoint. Do not document `jev({ endpoint })`. + +Saved experiments are format 2 (`{ format: 2, name, cases }`). Format 1 is +rejected. `ranSteps([])` fails when `steps` is empty; an observed model-only +run still passes. CLI is `corbits-eval run ` and exits 0, 1, or 2. + +`includes` and the four prebuilt workflow scorers +(`faithfulness`, `answerRelevancy`, `correctness`, `trajectoryAccuracy`) exist. +Do not document toxicity, bias, or pii. + ## Conventions - Read `README.md` before changing public behavior. diff --git a/README.md b/README.md index b4bdbf1..cf5069c 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ console.log(report.markdown()); ## Checks -`checks` maps a name to a check. Without `checks`, one `matches` check +`scorers` maps a name to a check. Without `scorers`, one `matches` check compares output with `expectedOutput`. Every check gets the run `r = { input, output, expectedOutput, steps, ms }`; `output` is whatever the run returned. @@ -47,19 +47,19 @@ returned. `unsure (p=…)` in between. `{ question, yes, no }` sets its own bars. - **Helpers**: -| helper | passes when | -| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `matches(field?)` | output deep-equals expectedOutput, or the same field (key or accessor) of each | -| `includes()` | both strings, and output contains expectedOutput after NFKC, trim, and whitespace collapse; value is 1 or 0 | -| `fitsSchema(schema)` | output satisfies a Standard Schema (arktype, zod, valibot) | -| `mentions(words)` | output contains every word, case-insensitive; value is the fraction found | -| `avoids(words)` | output contains none of the words | -| `ranSteps(names)` | `tool` and `step` names match `names`, order ignored; value is their F1. Agent trajectory comes from `stream()`. A reply-only send without `stream` does not satisfy `ranSteps([])` | -| `stepsAtMost(n)` | the run took at most `n` steps | -| `under({ ms })` | the run finished within `ms` | -| `ask(q, { options, pass })` | Jev picks an option named in `pass` | -| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt` | -| `ask(q, { levels, pass })` | Jev grades at level `pass` or above (levels lowest first) | +| helper | passes when | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `matches(field?)` | output deep-equals expectedOutput, or the same field (key or accessor) of each | +| `includes()` | both strings, and output contains expectedOutput after NFKC, trim, and whitespace collapse; value is 1 or 0 | +| `fitsSchema(schema)` | output satisfies a Standard Schema (arktype, zod, valibot) | +| `mentions(words)` | output contains every word, case-insensitive; value is the fraction found | +| `avoids(words)` | output contains none of the words | +| `ranSteps(names)` | `tool` and `step` names match `names`, order ignored; value is their F1. Agent trajectory comes from `stream()`. `ranSteps([])` fails when `steps` is empty. An observed model-only run (llm steps, no tool or step names) still passes | +| `stepsAtMost(n)` | the run took at most `n` steps | +| `under({ ms })` | the run finished within `ms` | +| `ask(q, { options, pass })` | Jev picks an option named in `pass` | +| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt` | +| `ask(q, { levels, pass })` | Jev grades at level `pass` or above (levels lowest first) | `mentions` and `avoids` also take `(r) => words` for per-case words. `mentions` and `ranSteps` take `{ atLeast }` to pass on partial credit. @@ -106,7 +106,8 @@ space cannot absorb — those fail closed by design. `tool.done` as `tool`). A policy deny emits `tool.done` without `tool.start`; the tool is still recorded when `inference.tool_call.end` or the assistant `tool_call` block names it. A reply-only send without - `stream` leaves `steps` empty, which does not satisfy `ranSteps([])`. + `stream` leaves `steps` empty, which fails `ranSteps([])`. An observed + model-only run still passes. - **Workflow**: `workflow(definition, { invokeStep })` from `@corbits/evals/workflow`. A definition cannot run locally without a step invoker, and the root never loads `@intx/workflow`, so a bare @@ -123,7 +124,8 @@ attempt passes. `concurrency` (default 4) bounds cases in flight; ## Cases An array of `{ name?, input, expectedOutput?, must?, skip? }`, or the path of a -JSONL file of the same objects. `name` keys the case in reports and +JSONL file of the same objects. The ground-truth key is `expectedOutput`. +`expect` and `expected` throw. `name` keys the case in reports and compared runs and defaults to the input. ## Passing @@ -159,6 +161,8 @@ cases; `.report` holds every result. An experiment is each case × scorer pass and value — plus a Jev scorer's probability distribution and confidence — and `about`, no times. +Saved files are format 2: `{ format: 2, name, cases }`. Any other `format`, +including format 1, is rejected. `compare` only reads; a missing file skips drops. `save` only writes, on every run. One path for both compares with the last experiment, then replaces it. `compare` also takes an experiment from an earlier `runEvaluation` of the same @@ -195,6 +199,20 @@ Exit codes: `--format json` (and `--json`) come from incur and print only that result as JSON on stdout, with no human summary. Exit 1 is only a failed gate (`EVAL_FAILED`). Config that stops the run before it starts, such as empty `data` or a string or `ask` scorer with no `judge.invoke`, exits 2. +## Workflow scorers + +`@corbits/evals/workflow` also exports four prebuilt one-step judges. Each +takes a pinned `{ provider, model }` and does not call a model by itself: +`faithfulness`, `answerRelevancy`, `correctness`, and `trajectoryAccuracy`. +Pair them with `judgeModel` when you want those rubrics run. `includes` stays +a code check on the root package. + +## Breaking changes + +- Saved experiments must be format 2 (`{ format: 2, name, cases }`). Format 1 is rejected. +- JSONL rows use `expectedOutput`. `expect` and `expected` throw. +- `ranSteps([])` fails when `steps` is empty. An observed model-only run still passes. + ## Calibrating Thresholds are per-workload starting points, not constants. `calibrate` From 480eb51cc1e924ad26d71e4fa64be2f6ef49d398 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 26 Oct 2026 20:42:26 -0700 Subject: [PATCH 13/16] docs: correct how prebuilt rubrics are run --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cf5069c..3c624bd 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,12 @@ Exit codes: `@corbits/evals/workflow` also exports four prebuilt one-step judges. Each takes a pinned `{ provider, model }` and does not call a model by itself: `faithfulness`, `answerRelevancy`, `correctness`, and `trajectoryAccuracy`. -Pair them with `judgeModel` when you want those rubrics run. `includes` stays +Each returns `{ definition, payload, failure }`. Run `definition` with +`workflow`, using `payload(run)` as the trigger. An outcome from `payload` +is the result; `failure` is the closed miss when the run cannot be judged. +`parsePrebuilt(id, output)` reads the step's category or score JSON. +`judgeModel` expects `{ decisions }` or `{ fallback }` and does not run +these rubrics. `includes` stays a code check on the root package. ## Breaking changes From b878c6d77361b0914aa422df13f13d058a3b7064 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 26 Oct 2026 21:30:29 -0700 Subject: [PATCH 14/16] chore(release): 0.2.0 Depend on @corbits/system-one ^0.4.0. Leave the lockfile unchanged because 0.4.0 is not published yet. --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 6fc7452..fd8e7f5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@corbits/evals", - "version": "0.1.2", + "version": "0.2.0", "description": "Evals for agents and workflows in one runEvaluation call: code, Jev and helper scorers over JSONL or inline data, gated per scorer on allowed misses and drops.", "keywords": [ "agents", @@ -54,7 +54,7 @@ "check": "bun run typecheck && bun run lint && bun run format:check && bun run test && bun run test:e2e" }, "dependencies": { - "@corbits/system-one": "^0.3.0", + "@corbits/system-one": "^0.4.0", "incur": "^0.7.1" }, "devDependencies": { From 7c0a9953f7b4729e5199f89fd476cfaddcca828d Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 26 Oct 2026 21:36:28 -0700 Subject: [PATCH 15/16] chore(deps): lock @corbits/system-one at 0.4.0 --- bun.lock | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/bun.lock b/bun.lock index 9de4ace..efa8121 100644 --- a/bun.lock +++ b/bun.lock @@ -5,7 +5,7 @@ "": { "name": "@corbits/evals", "dependencies": { - "@corbits/system-one": "^0.3.0", + "@corbits/system-one": "^0.4.0", "incur": "^0.7.1", }, "devDependencies": { @@ -33,7 +33,7 @@ "@cfworker/json-schema": ["@cfworker/json-schema@4.1.1", "", {}, "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og=="], - "@corbits/system-one": ["@corbits/system-one@0.3.0", "", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": "^0.4.0", "@intx/types": "^0.4.0" } }, "sha512-rH1MSTyTph1O3wRjVuh2gbunfPLXK0HFDjOym9CkXtcDpRa6J+sWiDsnHdIla3KnVxPWQ2dTbYYck6KlTghtIw=="], + "@corbits/system-one": ["@corbits/system-one@0.4.0", "", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": "^0.5.0", "@intx/types": "^0.5.0" } }, "sha512-v61GdkbcSS72/sY3G6RdfsjvUD2PDzzR246oUaGQVeR/NJaA0vb5XjklYKGeaj9lLyDENBuEW5wqNYlq/SEnDg=="], "@intx/agent": ["@intx/agent@0.5.0", "", { "dependencies": { "@intx/inference": "0.5.0", "@intx/log": "0.5.0", "@intx/mime": "0.5.0", "@intx/types": "0.5.0", "arktype": "^2.1.29" } }, "sha512-EWkRfduYJii67goq0cdO6qgRtV0aLe2jGk2aZJhurOivluy3tx0SlhiO2YaQmA4du+qBNzlMV9MMfKkmQ/zs+Q=="], @@ -166,11 +166,5 @@ "yaml": ["yaml@2.9.1", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw=="], "zod": ["zod@4.6.7", "", {}, "sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg=="], - - "@corbits/system-one/@intx/inference": ["@intx/inference@0.4.0", "", { "dependencies": { "@intx/log": "0.4.0", "@intx/types": "0.4.0", "arktype": "^2.1.29" } }, "sha512-cuaqCqotmjg0OzGUm8QmFJr1xX5wPVS0LeuhrxHsj0X3ImoCzVB4e/ttTc+DaP3K1tTMQ/8Q8HHCAPX36mOecQ=="], - - "@corbits/system-one/@intx/inference/@intx/log": ["@intx/log@0.4.0", "", { "dependencies": { "@logtape/hono": "^2.0.2", "@logtape/logtape": "^2.0.2" }, "peerDependencies": { "hono": "^4.0.0" }, "optionalPeers": ["hono"] }, "sha512-f1IPZ/9YpEqlA0I2UZPOlRJOTDoOFEZRwZoUVuw8DpgR1WatOMm4hA0E8o6Mtz+pA0ux7gVdls4y/JZcgypsfQ=="], - - "@corbits/system-one/@intx/inference/@intx/types": ["@intx/types@0.4.0", "", { "dependencies": { "arktype": "^2.1.29", "semver": "^7.7.2" } }, "sha512-nJ2fcmskMOZNMRfbHs5ueKpCANHNttoL+sZgIoFaVUF/QLpFWfVp0YV72CQAaa9bVv9j5kRSwtSOCCGSlwQ6Dw=="], } } From 02a9cb1782b57bdfe19dd9d85c3cb347c6eaff74 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 26 Oct 2026 21:39:34 -0700 Subject: [PATCH 16/16] fix: fail closed on a box decision --- e2e/jev.e2e.test.ts | 22 ++++++++++++++++++++++ src/jev.ts | 7 ++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/e2e/jev.e2e.test.ts b/e2e/jev.e2e.test.ts index 7c801ff..837157a 100644 --- a/e2e/jev.e2e.test.ts +++ b/e2e/jev.e2e.test.ts @@ -128,6 +128,28 @@ describe("sentence checks", () => { expect(outcome?.explanation).toContain("jev fallback"); expect(outcome?.explanation).toContain("boom"); }); + + test("a box decision fails closed and is not scored as a choice or a noul", async () => { + const box = { + type: "box", + box: { top: 0.1, bottom: 0.4, left: 0.2, right: 0.5 }, + confidence: null, + }; + const closed = { + pass: false, + value: 0, + explanation: "jev returned a box decision", + }; + expect((await answered(box, question)).outcome).toEqual(closed); + expect( + ( + await answered( + { ...box, confidence: 0.91 }, + ask("Ready?", { options: { ready: "send", revise: "work" }, pass: "ready" }), + ) + ).outcome, + ).toEqual(closed); + }); }); describe("batching", () => { diff --git a/src/jev.ts b/src/jev.ts index 060e072..9978a46 100644 --- a/src/jev.ts +++ b/src/jev.ts @@ -240,7 +240,10 @@ export function checkThresholds(check: Ask | Sentence | string): void { } /** The decision's full distribution; the answer never carries it alone. */ -function dist(decision: Decision): { probabilities: Record; confidence?: number } { +function dist(decision: Exclude): { + probabilities: Record; + confidence?: number; +} { if (decision.type === "noul") { const yes = r3(decision.noul); return { probabilities: { yes, no: r3(1 - decision.noul) } }; @@ -257,6 +260,8 @@ export function toOutcome( decision: Decision, judge: Judge, ): Outcome { + // A box locates a region. It has no distribution and is not a yes/no, choice, or score. + if (decision.type === "box") return failed("jev returned a box decision"); const t = thresholdsFor(check, judge); const d = dist(decision); if (decision.type === "noul") {