diff --git a/README.md b/README.md index 392c03d..eb1b167 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ console.log(report.markdown()); ## Checks `checks` maps a name to a check. Without `checks`, one `matches` check -compares output with `expect`. Every check gets the run -`r = { input, output, expect, steps, ms }`; `output` is whatever the run +compares output with `expectedOutput`. Every check gets the run +`r = { input, output, expectedOutput, steps, ms }`; `output` is whatever the run returned. - **Code**: `(r) => boolean | number`, sync or async. A number is 0..1 @@ -47,18 +47,18 @@ returned. `unsure (p=…)` in between. `{ question, yes, no }` sets its own bars. - **Helpers**: -| helper | passes when | -| ----------------------------- | -------------------------------------------------------------------------- | -| `matches(field?)` | output deep-equals expect, or the same field (key or accessor) of each | -| `fitsSchema(schema)` | output satisfies a Standard Schema (arktype, zod, valibot) | -| `mentions(words)` | output contains every word, case-insensitive; value is the fraction found | -| `avoids(words)` | output contains none of the words | -| `ranSteps(names)` | `tool` and `step` names match `names`, order ignored; value is their F1 | -| `stepsAtMost(n)` | the run took at most `n` steps | -| `under({ ms })` | the run finished within `ms` | -| `ask(q, { options, pass })` | Jev picks an option named in `pass` | -| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt` | -| `ask(q, { levels, pass })` | Jev grades at level `pass` or above (levels lowest first) | +| helper | passes when | +| ----------------------------- | ------------------------------------------------------------------------------ | +| `matches(field?)` | output deep-equals expectedOutput, or the same field (key or accessor) of each | +| `fitsSchema(schema)` | output satisfies a Standard Schema (arktype, zod, valibot) | +| `mentions(words)` | output contains every word, case-insensitive; value is the fraction found | +| `avoids(words)` | output contains none of the words | +| `ranSteps(names)` | `tool` and `step` names match `names`, order ignored; value is their F1 | +| `stepsAtMost(n)` | the run took at most `n` steps | +| `under({ ms })` | the run finished within `ms` | +| `ask(q, { options, pass })` | Jev picks an option named in `pass` | +| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt` | +| `ask(q, { levels, pass })` | Jev grades at level `pass` or above (levels lowest first) | `mentions` and `avoids` also take `(r) => words` for per-case words. `mentions` and `ranSteps` take `{ atLeast }` to pass on partial credit. @@ -68,7 +68,7 @@ returned. A check that throws, hangs past `timeout`, or returns anything else fails itself closed; its siblings still run. Jev transport failures fail closed. -Jev sees `{ input, output, expect, steps }`, and a case's string and +Jev sees `{ input, output, expectedOutput, steps }`, and a case's string and `ask` checks answer in one call. Configure it with `judge: jev({ endpoint, model, apiKey, timeout })`; `endpoint` is `"official"` (default), `"gateway"` or a URL. Without an `apiKey` the @@ -112,7 +112,7 @@ attempt passes. `concurrency` (default 4) bounds cases in flight; ## Cases -An array of `{ name?, input, expect?, must?, skip? }`, or the path of a +An array of `{ name?, input, expectedOutput?, must?, skip? }`, or the path of a JSONL file of the same objects. `name` keys the case in reports and compared runs and defaults to the input. @@ -165,7 +165,7 @@ verdicts agree, land unsure, or confidently disagree at each threshold: ```ts const table = await calibrate("refund-agent", { run: refundAgent, - cases: labeledCases, // { input, expect?, labels: { readyToSend: true } } + cases: labeledCases, // { input, expectedOutput?, labels: { readyToSend: true } } checks: { readyToSend: "Is this reply ready to send to the customer?" }, judge, }); diff --git a/e2e/evaluate.e2e.test.ts b/e2e/evaluate.e2e.test.ts index 3a58acb..be27513 100644 --- a/e2e/evaluate.e2e.test.ts +++ b/e2e/evaluate.e2e.test.ts @@ -39,8 +39,8 @@ describe("runs", () => { name: "upper", task: upper, data: [ - { input: "hi", expect: "HI" }, - { name: "yo", input: "yo", expect: "YO" }, + { input: "hi", expectedOutput: "HI" }, + { name: "yo", input: "yo", expectedOutput: "YO" }, ], }); expect(report.ok).toBe(true); @@ -60,7 +60,7 @@ describe("runs", () => { const report = await runEvaluation({ name: "agent", task: agent, - data: [{ input: "refund", expect: "re: refund" }], + data: [{ input: "refund", expectedOutput: "re: refund" }], }); expect(report.ok).toBe(true); expect(seen[0]).toBeInstanceOf(AbortSignal); @@ -86,7 +86,7 @@ describe("runs", () => { const report = await runEvaluation({ name: "objects", task: (input: { amount: number }) => ({ decision: "refund", amount: input.amount }), - data: [{ input: { amount: 5 }, expect: { decision: "refund", amount: 5 } }], + data: [{ input: { amount: 5 }, expectedOutput: { decision: "refund", amount: 5 } }], scorers: { whole: matches(), decision: matches("decision"), @@ -102,7 +102,7 @@ describe("runs", () => { const path = join(dir, "cases.jsonl"); await Bun.write( path, - '{"name":"a","input":"hi","expect":"HI"}\n\n{"input":"yo","expect":"YO"}\n', + '{"name":"a","input":"hi","expectedOutput":"HI"}\n\n{"input":"yo","expectedOutput":"YO"}\n', ); const report = await runEvaluation({ name: "jsonl", task: upper, data: path }); expect(report.cases.map((c) => c.name)).toEqual(["a", "yo"]); @@ -190,7 +190,7 @@ describe("checks", () => { const err = await failure( runEvaluation({ name: "no-expect", task: upper, data: [{ input: "hi" }] }), ); - expect(err.report.cases[0]?.checks["matches"]?.explanation).toBe("case has no expect"); + expect(err.report.cases[0]?.checks["matches"]?.explanation).toBe("case has no expectedOutput"); }); test("a throwing, hanging or malformed check fails itself closed, not its siblings", async () => { @@ -198,7 +198,7 @@ describe("checks", () => { runEvaluation({ name: "bad-checks", task: upper, - data: [{ input: "hi", expect: "HI" }], + data: [{ input: "hi", expectedOutput: "HI" }], scorers: { exact: matches(), boom: () => { @@ -273,7 +273,7 @@ describe("gates", () => { }); test("bad options are rejected up front", async () => { - const base = { task: upper, data: [{ input: "a", expect: "A" }] }; + const base = { task: upper, data: [{ input: "a", expectedOutput: "A" }] }; await expect(runEvaluation({ name: "o", ...base, concurrency: Number.NaN })).rejects.toThrow( "concurrency", ); @@ -293,7 +293,7 @@ describe("gates", () => { runEvaluation({ name: "md", task: upper, - data: [{ name: "a|b\nc", input: "", expect: "no" }], + data: [{ name: "a|b\nc", input: "", expectedOutput: "no" }], }), ); const md = err.report.markdown(); @@ -306,9 +306,9 @@ describe("gates", () => { describe("passing", () => { const cases = [ - { name: "a", input: "a", expect: "A" }, - { name: "b", input: "b", expect: "B" }, - { name: "c", input: "c", expect: "nope" }, + { name: "a", input: "a", expectedOutput: "A" }, + { name: "b", input: "b", expectedOutput: "B" }, + { name: "c", input: "c", expectedOutput: "nope" }, ]; const checks = { exact: matches(), short: (r: { output: string }) => r.output.length === 1 }; @@ -368,7 +368,7 @@ describe("passing", () => { scorers: { exact: (r) => { seen.push(r.input); - return r.output === r.expect; + return r.output === r.expectedOutput; }, }, }); @@ -382,7 +382,7 @@ describe("passing", () => { const path = join(await tempDir(), "cases.jsonl"); await Bun.write( path, - '{"name":"a","input":"a","expect":"A","must":["exact"]}\n{"name":"c","input":"c","expect":"nope","skip":["exact"]}\n', + '{"name":"a","input":"a","expectedOutput":"A","must":["exact"]}\n{"name":"c","input":"c","expectedOutput":"nope","skip":["exact"]}\n', ); const report = await runEvaluation({ name: "jsonl-must", @@ -429,9 +429,9 @@ describe("passing", () => { describe("compare and save", () => { const cases = [ - { name: "a", input: "a", expect: "A" }, - { name: "b", input: "b", expect: "B" }, - { name: "c", input: "c", expect: "nope" }, + { name: "a", input: "a", expectedOutput: "A" }, + { name: "b", input: "b", expectedOutput: "B" }, + { name: "c", input: "c", expectedOutput: "nope" }, ]; const allow = { matches: { miss: 2 } }; const swap = (input: string) => (input === "b" ? "?" : input === "c" ? "nope" : "A"); @@ -566,7 +566,7 @@ describe("compare and save", () => { test("added and removed cases are reported, not counted as drops", async () => { const before = await runEvaluation({ name: "upper", task: upper, data: cases, allow }); - const changed = [...cases.slice(1), { name: "d", input: "d", expect: "nope" }]; + const changed = [...cases.slice(1), { name: "d", input: "d", expectedOutput: "nope" }]; const report = await runEvaluation({ name: "upper", task: upper, diff --git a/e2e/jev.e2e.test.ts b/e2e/jev.e2e.test.ts index 4926f72..a37ab9b 100644 --- a/e2e/jev.e2e.test.ts +++ b/e2e/jev.e2e.test.ts @@ -50,7 +50,7 @@ async function judged(check: Scorer, judge: Judge) { report = await runEvaluation({ name: "jev", task: (input: string) => `reply to ${input}`, - data: [{ input: "refund?", expect: "yes" }], + data: [{ input: "refund?", expectedOutput: "yes" }], scorers: { judged: check }, judge, }); @@ -81,7 +81,7 @@ describe("sentence checks", () => { expect(bodies[0]?.["state"]).toEqual({ input: "refund?", output: "reply to refund?", - expect: "yes", + expectedOutput: "yes", }); expect(bodies[0]?.["questions"]).toEqual({ judged: { type: "noul", instructions: question } }); }); diff --git a/e2e/workflow.e2e.test.ts b/e2e/workflow.e2e.test.ts index 00724b0..060de90 100644 --- a/e2e/workflow.e2e.test.ts +++ b/e2e/workflow.e2e.test.ts @@ -39,7 +39,7 @@ describe("workflow runs", () => { const report = await runEvaluation({ name: "workflow", task: workflow(pipeline, { invokeStep }), - data: [{ input: "refund me", expect: "DRAFT: REFUND ME" }], + data: [{ input: "refund me", expectedOutput: "DRAFT: REFUND ME" }], scorers: { exact: matches(), order: (r) => r.steps.map((s) => s.name).join(">") === "drafter>reviewer", @@ -68,7 +68,10 @@ describe("workflow runs", () => { output: (run) => ({ draft: run.outputs["draft"], review: run.outputs["review"] }), }), data: [ - { input: { order: 42 }, expect: { draft: "draft: order 42", review: "DRAFT: ORDER 42" } }, + { + input: { order: 42 }, + expectedOutput: { draft: "draft: order 42", review: "DRAFT: ORDER 42" }, + }, ], }); expect(report.ok).toBe(true); @@ -84,7 +87,7 @@ describe("workflow runs", () => { return { output: "draft" }; }, }), - data: [{ input: "x", expect: "X" }], + data: [{ input: "x", expectedOutput: "X" }], }); throw new Error("expected a failure"); } catch (cause) { diff --git a/examples/ports/factuality.ts b/examples/ports/factuality.ts index c448467..011b3a6 100644 --- a/examples/ports/factuality.ts +++ b/examples/ports/factuality.ts @@ -77,11 +77,11 @@ const CASES: { id: string; question: string; expert: string; submission: string; const report = await runEvaluation({ name: "port-factuality", task: (question: string) => CASES.find((c) => c.question === question)?.submission ?? "", - data: CASES.map((c) => ({ name: c.id, input: c.question, expect: c.expert })), + data: CASES.map((c) => ({ name: c.id, input: c.question, expectedOutput: c.expert })), scorers: { baseline_exact: matches(), jev_factuality: ask( - "Compare the factual content of the output with the expert answer in expect. Ignore style, grammar, punctuation. Pick one option.", + "Compare the factual content of the output with the expert answer in expectedOutput. Ignore style, grammar, punctuation. Pick one option.", { options: CRITERIA, pass: ["A", "B", "C", "E"] }, ), }, diff --git a/examples/ports/keyword-coverage.ts b/examples/ports/keyword-coverage.ts index f87f75a..c0a0426 100644 --- a/examples/ports/keyword-coverage.ts +++ b/examples/ports/keyword-coverage.ts @@ -39,10 +39,10 @@ const CASES: { id: string; reference: string; output: string; truth: boolean }[] const report = await runEvaluation({ name: "port-keyword-coverage", task: (id: string) => CASES.find((c) => c.id === id)?.output ?? "", - data: CASES.map((c) => ({ input: c.id, expect: c.reference })), + data: CASES.map((c) => ({ input: c.id, expectedOutput: c.reference })), scorers: { baseline_exact: matches(), - port_coverage: (r) => coverage(r.expect ?? "", r.output) >= 0.5, + port_coverage: (r) => coverage(r.expectedOutput ?? "", r.output) >= 0.5, }, allow: { baseline_exact: CASES.length, port_coverage: CASES.length }, }); diff --git a/src/calibrate.ts b/src/calibrate.ts index 3afed93..6453f9d 100644 --- a/src/calibrate.ts +++ b/src/calibrate.ts @@ -11,7 +11,7 @@ import type { Outcome } from "./types.js"; export type Labeled = { name?: string; input: I; - expect?: E; + expectedOutput?: E; /** The expected verdict per check name. */ labels: Readonly>; }; @@ -115,7 +115,7 @@ export async function calibrate( const evalCase = { name: key, input: labeled.input, - ...(labeled.expect === undefined ? {} : { expect: labeled.expect }), + ...(labeled.expectedOutput === undefined ? {} : { expectedOutput: labeled.expectedOutput }), }; const runResult = await runOnce(spec, evalCase); const decisions = await decideBatch(options.judge, options.checks, runResult); diff --git a/src/checks.ts b/src/checks.ts index c1b0b3e..c13bc41 100644 --- a/src/checks.ts +++ b/src/checks.ts @@ -27,7 +27,7 @@ export function toOutcome(name: string, raw: unknown): Outcome { }; } -/** Output deep-equals expect, or one field of each (a key or an accessor). */ +/** Output deep-equals expectedOutput, or one field of each (a key or an accessor). */ export function matches( field?: string | ((value: T) => unknown), ): CheckFn { @@ -38,10 +38,11 @@ export function matches( ? (value as Record)[field] : undefined; }; - return ({ output, expect }) => { - if (expect === undefined) return { pass: false, value: 0, explanation: "case has no expect" }; + return ({ output, expectedOutput }) => { + if (expectedOutput === undefined) + return { pass: false, value: 0, explanation: "case has no expectedOutput" }; const got = get(output); - const want = get(expect); + const want = get(expectedOutput); if (Bun.deepEquals(got, want)) return true; return { pass: false, @@ -73,7 +74,7 @@ export function fitsSchema(schema: StandardSchema): CheckFn { }; } -/** Fixed words, or words picked per case (e.g. from `expect`). */ +/** Fixed words, or words picked per case (e.g. from `expectedOutput`). */ export type Words = | readonly string[] | ((run: Run) => readonly string[]); diff --git a/src/datasets.test.ts b/src/datasets.test.ts new file mode 100644 index 0000000..1ef9596 --- /dev/null +++ b/src/datasets.test.ts @@ -0,0 +1,11 @@ +import { expect, test } from "bun:test"; + +import { parseJsonl } from "./datasets.js"; + +test("parseJsonl loads expectedOutput and rejects expect or expected", () => { + expect(parseJsonl('{"input":"hi","expectedOutput":"HI"}\n')).toEqual([ + { input: "hi", expectedOutput: "HI" }, + ]); + expect(() => parseJsonl('{"input":"hi","expect":"HI"}\n')).toThrow("expectedOutput"); + expect(() => parseJsonl('{"input":"hi","expected":"HI"}\n')).toThrow("expectedOutput"); +}); diff --git a/src/datasets.ts b/src/datasets.ts index 9fc5aa7..057c08b 100644 --- a/src/datasets.ts +++ b/src/datasets.ts @@ -21,7 +21,10 @@ function toCase(row: Record, where: string): Case { if (!("input" in row)) throw new Error(`${where}: missing "input"`); const evalCase: Case = { input: row["input"] }; if (name !== undefined) evalCase.name = name; - if (row["expect"] !== undefined) evalCase.expect = row["expect"]; + if (row["expect"] !== undefined || row["expected"] !== undefined) { + throw new Error(`${where}: use "expectedOutput", not "expect" or "expected"`); + } + if (row["expectedOutput"] !== undefined) evalCase.expectedOutput = row["expectedOutput"]; const must = checkNames(row, "must", where); if (must !== undefined) evalCase.must = must; const skip = checkNames(row, "skip", where); diff --git a/src/jev.ts b/src/jev.ts index 7147092..d07f4f4 100644 --- a/src/jev.ts +++ b/src/jev.ts @@ -328,7 +328,7 @@ export function toOutcome( function toState(run: Run): Record { const state: Record = { input: run.input, output: run.output }; - if (run.expect !== undefined) state["expect"] = run.expect; + if (run.expectedOutput !== undefined) state["expectedOutput"] = run.expectedOutput; if (run.steps.length > 0) state["steps"] = run.steps; return state; } diff --git a/src/runner.ts b/src/runner.ts index 27d690e..798eff8 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -4,7 +4,12 @@ import type { Trace } from "./runs.js"; import { withTimeout } from "./timeout.js"; import type { CaseResult, Outcome, Run } from "./types.js"; -export type NamedCase = { name: string; input: I; expect?: E; skip?: readonly string[] }; +export type NamedCase = { + name: string; + input: I; + expectedOutput?: E; + skip?: readonly string[]; +}; /** String and `ask` checks Jev answers in one call per attempt. */ export type JevGroup = { judge: Judge; checks: Record }; @@ -49,7 +54,7 @@ export async function runOnce( steps: trace.steps, ms: Date.now() - started, }; - if (evalCase.expect !== undefined) run.expect = evalCase.expect; + if (evalCase.expectedOutput !== undefined) run.expectedOutput = evalCase.expectedOutput; return run; } finally { clearTimeout(timer); @@ -168,7 +173,7 @@ async function runCase( pass: error === undefined && Object.values(checks).every((o) => o.pass), checks, }; - if (evalCase.expect !== undefined) result.expect = evalCase.expect; + if (evalCase.expectedOutput !== undefined) result.expectedOutput = evalCase.expectedOutput; if (last !== undefined) result.output = last.output; if (error !== undefined) result.error = error; return result; diff --git a/src/types.ts b/src/types.ts index 756c72b..10079c3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -12,7 +12,7 @@ export type Case = { /** Keys the case in reports and compared runs. Defaults to the input. */ name?: string; input: I; - expect?: E; + expectedOutput?: E; /** Checks that must pass on this case, whatever `allow` says. */ must?: readonly string[]; /** Checks this case is not run or counted for. */ @@ -23,7 +23,7 @@ export type Case = { export type Run = { input: I; output: O; - expect?: E; + expectedOutput?: E; steps: Step[]; ms: number; }; @@ -51,7 +51,7 @@ export type CheckFn = ( export type CaseResult = { name: string; input: I; - expect?: E; + expectedOutput?: E; output?: O; steps: Step[]; /** Whole case: every attempt's run plus checks. */