feat!: name the case ground truth expectedOutput

README.md

@@ -35,8 +35,8 @@ console.log(report.markdown());
 ## Checks
 
 `checks` maps a name to a check. Without `checks`, one `matches` check
-compares output with `expect`. Every check gets the run
-`r = { input, output, expect, steps, ms }`; `output` is whatever the run
+compares output with `expectedOutput`. Every check gets the run
+`r = { input, output, expectedOutput, steps, ms }`; `output` is whatever the run
 returned.
 
 - **Code**: `(r) => boolean | number`, sync or async. A number is 0..1
@@ -47,18 +47,18 @@ returned.
   `unsure (p=…)` in between. `{ question, yes, no }` sets its own bars.
 - **Helpers**:
 
-| helper                        | passes when                                                                |
-| ----------------------------- | -------------------------------------------------------------------------- |
-| `matches(field?)`             | output deep-equals expect, or the same field (key or accessor) of each     |
-| `fitsSchema(schema)`          | output satisfies a Standard Schema (arktype, zod, valibot)                 |
-| `mentions(words)`             | output contains every word, case-insensitive; value is the fraction found  |
-| `avoids(words)`               | output contains none of the words                                          |
-| `ranSteps(names)`             | `tool` and `step` names match `names`, order ignored; value is their F1    |
-| `stepsAtMost(n)`              | the run took at most `n` steps                                             |
-| `under({ ms })`               | the run finished within `ms`                                               |
-| `ask(q, { options, pass })`   | Jev picks an option named in `pass`                                        |
-| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt` |
-| `ask(q, { levels, pass })`    | Jev grades at level `pass` or above (levels lowest first)                  |
+| helper                        | passes when                                                                    |
+| ----------------------------- | ------------------------------------------------------------------------------ |
+| `matches(field?)`             | output deep-equals expectedOutput, or the same field (key or accessor) of each |
+| `fitsSchema(schema)`          | output satisfies a Standard Schema (arktype, zod, valibot)                     |
+| `mentions(words)`             | output contains every word, case-insensitive; value is the fraction found      |
+| `avoids(words)`               | output contains none of the words                                              |
+| `ranSteps(names)`             | `tool` and `step` names match `names`, order ignored; value is their F1        |
+| `stepsAtMost(n)`              | the run took at most `n` steps                                                 |
+| `under({ ms })`               | the run finished within `ms`                                                   |
+| `ask(q, { options, pass })`   | Jev picks an option named in `pass`                                            |
+| `ask(q, { options, passAt })` | Jev picks an option; its credit (0..1) is the value and passes at `passAt`     |
+| `ask(q, { levels, pass })`    | Jev grades at level `pass` or above (levels lowest first)                      |
 
 `mentions` and `avoids` also take `(r) => words` for per-case words.
 `mentions` and `ranSteps` take `{ atLeast }` to pass on partial credit.
@@ -68,7 +68,7 @@ returned.
 A check that throws, hangs past `timeout`, or returns anything else fails
 itself closed; its siblings still run. Jev transport failures fail closed.
 
-Jev sees `{ input, output, expect, steps }`, and a case's string and
+Jev sees `{ input, output, expectedOutput, steps }`, and a case's string and
 `ask` checks answer in one call. Configure it with
 `judge: jev({ endpoint, model, apiKey, timeout })`; `endpoint` is
 `"official"` (default), `"gateway"` or a URL. Without an `apiKey` the
@@ -112,7 +112,7 @@ attempt passes. `concurrency` (default 4) bounds cases in flight;
 
 ## Cases
 
-An array of `{ name?, input, expect?, must?, skip? }`, or the path of a
+An array of `{ name?, input, expectedOutput?, must?, skip? }`, or the path of a
 JSONL file of the same objects. `name` keys the case in reports and
 compared runs and defaults to the input.
 
@@ -165,7 +165,7 @@ verdicts agree, land unsure, or confidently disagree at each threshold:
 ```ts
 const table = await calibrate("refund-agent", {
   run: refundAgent,
-  cases: labeledCases, // { input, expect?, labels: { readyToSend: true } }
+  cases: labeledCases, // { input, expectedOutput?, labels: { readyToSend: true } }
   checks: { readyToSend: "Is this reply ready to send to the customer?" },
   judge,
 });

e2e/evaluate.e2e.test.ts

@@ -39,8 +39,8 @@ describe("runs", () => {
       name: "upper",
       task: upper,
       data: [
-        { input: "hi", expect: "HI" },
-        { name: "yo", input: "yo", expect: "YO" },
+        { input: "hi", expectedOutput: "HI" },
+        { name: "yo", input: "yo", expectedOutput: "YO" },
       ],
     });
     expect(report.ok).toBe(true);
@@ -60,7 +60,7 @@ describe("runs", () => {
     const report = await runEvaluation({
       name: "agent",
       task: agent,
-      data: [{ input: "refund", expect: "re: refund" }],
+      data: [{ input: "refund", expectedOutput: "re: refund" }],
     });
     expect(report.ok).toBe(true);
     expect(seen[0]).toBeInstanceOf(AbortSignal);
@@ -86,7 +86,7 @@ describe("runs", () => {
     const report = await runEvaluation({
       name: "objects",
       task: (input: { amount: number }) => ({ decision: "refund", amount: input.amount }),
-      data: [{ input: { amount: 5 }, expect: { decision: "refund", amount: 5 } }],
+      data: [{ input: { amount: 5 }, expectedOutput: { decision: "refund", amount: 5 } }],
       scorers: {
         whole: matches(),
         decision: matches("decision"),
@@ -102,7 +102,7 @@ describe("runs", () => {
     const path = join(dir, "cases.jsonl");
     await Bun.write(
       path,
-      '{"name":"a","input":"hi","expect":"HI"}\n\n{"input":"yo","expect":"YO"}\n',
+      '{"name":"a","input":"hi","expectedOutput":"HI"}\n\n{"input":"yo","expectedOutput":"YO"}\n',
     );
     const report = await runEvaluation({ name: "jsonl", task: upper, data: path });
     expect(report.cases.map((c) => c.name)).toEqual(["a", "yo"]);
@@ -190,15 +190,15 @@ describe("checks", () => {
     const err = await failure(
       runEvaluation({ name: "no-expect", task: upper, data: [{ input: "hi" }] }),
     );
-    expect(err.report.cases[0]?.checks["matches"]?.explanation).toBe("case has no expect");
+    expect(err.report.cases[0]?.checks["matches"]?.explanation).toBe("case has no expectedOutput");
   });
 
   test("a throwing, hanging or malformed check fails itself closed, not its siblings", async () => {
     const err = await failure(
       runEvaluation({
         name: "bad-checks",
         task: upper,
-        data: [{ input: "hi", expect: "HI" }],
+        data: [{ input: "hi", expectedOutput: "HI" }],
         scorers: {
           exact: matches(),
           boom: () => {
@@ -273,7 +273,7 @@ describe("gates", () => {
   });
 
   test("bad options are rejected up front", async () => {
-    const base = { task: upper, data: [{ input: "a", expect: "A" }] };
+    const base = { task: upper, data: [{ input: "a", expectedOutput: "A" }] };
     await expect(runEvaluation({ name: "o", ...base, concurrency: Number.NaN })).rejects.toThrow(
       "concurrency",
     );
@@ -293,7 +293,7 @@ describe("gates", () => {
       runEvaluation({
         name: "md",
         task: upper,
-        data: [{ name: "a|b\nc", input: "<x>", expect: "no" }],
+        data: [{ name: "a|b\nc", input: "<x>", expectedOutput: "no" }],
       }),
     );
     const md = err.report.markdown();
@@ -306,9 +306,9 @@ describe("gates", () => {
 
 describe("passing", () => {
   const cases = [
-    { name: "a", input: "a", expect: "A" },
-    { name: "b", input: "b", expect: "B" },
-    { name: "c", input: "c", expect: "nope" },
+    { name: "a", input: "a", expectedOutput: "A" },
+    { name: "b", input: "b", expectedOutput: "B" },
+    { name: "c", input: "c", expectedOutput: "nope" },
   ];
   const checks = { exact: matches(), short: (r: { output: string }) => r.output.length === 1 };
 
@@ -368,7 +368,7 @@ describe("passing", () => {
       scorers: {
         exact: (r) => {
           seen.push(r.input);
-          return r.output === r.expect;
+          return r.output === r.expectedOutput;
         },
       },
     });
@@ -382,7 +382,7 @@ describe("passing", () => {
     const path = join(await tempDir(), "cases.jsonl");
     await Bun.write(
       path,
-      '{"name":"a","input":"a","expect":"A","must":["exact"]}\n{"name":"c","input":"c","expect":"nope","skip":["exact"]}\n',
+      '{"name":"a","input":"a","expectedOutput":"A","must":["exact"]}\n{"name":"c","input":"c","expectedOutput":"nope","skip":["exact"]}\n',
     );
     const report = await runEvaluation({
       name: "jsonl-must",
@@ -429,9 +429,9 @@ describe("passing", () => {
 
 describe("compare and save", () => {
   const cases = [
-    { name: "a", input: "a", expect: "A" },
-    { name: "b", input: "b", expect: "B" },
-    { name: "c", input: "c", expect: "nope" },
+    { name: "a", input: "a", expectedOutput: "A" },
+    { name: "b", input: "b", expectedOutput: "B" },
+    { name: "c", input: "c", expectedOutput: "nope" },
   ];
   const allow = { matches: { miss: 2 } };
   const swap = (input: string) => (input === "b" ? "?" : input === "c" ? "nope" : "A");
@@ -566,7 +566,7 @@ describe("compare and save", () => {
 
   test("added and removed cases are reported, not counted as drops", async () => {
     const before = await runEvaluation({ name: "upper", task: upper, data: cases, allow });
-    const changed = [...cases.slice(1), { name: "d", input: "d", expect: "nope" }];
+    const changed = [...cases.slice(1), { name: "d", input: "d", expectedOutput: "nope" }];
     const report = await runEvaluation({
       name: "upper",
       task: upper,

e2e/jev.e2e.test.ts

@@ -50,7 +50,7 @@ async function judged(check: Scorer<string, string, string>, judge: Judge) {
     report = await runEvaluation({
       name: "jev",
       task: (input: string) => `reply to ${input}`,
-      data: [{ input: "refund?", expect: "yes" }],
+      data: [{ input: "refund?", expectedOutput: "yes" }],
       scorers: { judged: check },
       judge,
     });
@@ -81,7 +81,7 @@ describe("sentence checks", () => {
     expect(bodies[0]?.["state"]).toEqual({
       input: "refund?",
       output: "reply to refund?",
-      expect: "yes",
+      expectedOutput: "yes",
     });
     expect(bodies[0]?.["questions"]).toEqual({ judged: { type: "noul", instructions: question } });
   });

e2e/workflow.e2e.test.ts

@@ -39,7 +39,7 @@ describe("workflow runs", () => {
     const report = await runEvaluation({
       name: "workflow",
       task: workflow<string, string>(pipeline, { invokeStep }),
-      data: [{ input: "refund me", expect: "DRAFT: REFUND ME" }],
+      data: [{ input: "refund me", expectedOutput: "DRAFT: REFUND ME" }],
       scorers: {
         exact: matches(),
         order: (r) => r.steps.map((s) => s.name).join(">") === "drafter>reviewer",
@@ -68,7 +68,10 @@ describe("workflow runs", () => {
         output: (run) => ({ draft: run.outputs["draft"], review: run.outputs["review"] }),
       }),
       data: [
-        { input: { order: 42 }, expect: { draft: "draft: order 42", review: "DRAFT: ORDER 42" } },
+        {
+          input: { order: 42 },
+          expectedOutput: { draft: "draft: order 42", review: "DRAFT: ORDER 42" },
+        },
       ],
     });
     expect(report.ok).toBe(true);
@@ -84,7 +87,7 @@ describe("workflow runs", () => {
             return { output: "draft" };
           },
         }),
-        data: [{ input: "x", expect: "X" }],
+        data: [{ input: "x", expectedOutput: "X" }],
       });
       throw new Error("expected a failure");
     } catch (cause) {

examples/ports/factuality.ts

@@ -77,11 +77,11 @@ const CASES: { id: string; question: string; expert: string; submission: string;
 const report = await runEvaluation({
   name: "port-factuality",
   task: (question: string) => CASES.find((c) => c.question === question)?.submission ?? "",
-  data: CASES.map((c) => ({ name: c.id, input: c.question, expect: c.expert })),
+  data: CASES.map((c) => ({ name: c.id, input: c.question, expectedOutput: c.expert })),
   scorers: {
     baseline_exact: matches(),
     jev_factuality: ask(
-      "Compare the factual content of the output with the expert answer in expect. Ignore style, grammar, punctuation. Pick one option.",
+      "Compare the factual content of the output with the expert answer in expectedOutput. Ignore style, grammar, punctuation. Pick one option.",
       { options: CRITERIA, pass: ["A", "B", "C", "E"] },
     ),
   },

examples/ports/keyword-coverage.ts

@@ -39,10 +39,10 @@ const CASES: { id: string; reference: string; output: string; truth: boolean }[]
 const report = await runEvaluation({
   name: "port-keyword-coverage",
   task: (id: string) => CASES.find((c) => c.id === id)?.output ?? "",
-  data: CASES.map((c) => ({ input: c.id, expect: c.reference })),
+  data: CASES.map((c) => ({ input: c.id, expectedOutput: c.reference })),
   scorers: {
     baseline_exact: matches(),
-    port_coverage: (r) => coverage(r.expect ?? "", r.output) >= 0.5,
+    port_coverage: (r) => coverage(r.expectedOutput ?? "", r.output) >= 0.5,
   },
   allow: { baseline_exact: CASES.length, port_coverage: CASES.length },
 });

src/calibrate.ts

@@ -11,7 +11,7 @@ import type { Outcome } from "./types.js";
 export type Labeled<I = unknown, E = unknown> = {
   name?: string;
   input: I;
-  expect?: E;
+  expectedOutput?: E;
   /** The expected verdict per check name. */
   labels: Readonly<Record<string, boolean>>;
 };
@@ -115,7 +115,7 @@ export async function calibrate<I, O, E = O>(
     const evalCase = {
       name: key,
       input: labeled.input,
-      ...(labeled.expect === undefined ? {} : { expect: labeled.expect }),
+      ...(labeled.expectedOutput === undefined ? {} : { expectedOutput: labeled.expectedOutput }),
     };
     const runResult = await runOnce(spec, evalCase);
     const decisions = await decideBatch(options.judge, options.checks, runResult);

src/checks.ts

@@ -27,7 +27,7 @@ export function toOutcome(name: string, raw: unknown): Outcome {
   };
 }
 
-/** Output deep-equals expect, or one field of each (a key or an accessor). */
+/** Output deep-equals expectedOutput, or one field of each (a key or an accessor). */
 export function matches<T = unknown>(
   field?: string | ((value: T) => unknown),
 ): CheckFn<unknown, T, T> {
@@ -38,10 +38,11 @@ export function matches<T = unknown>(
       ? (value as Record<string, unknown>)[field]
       : undefined;
   };
-  return ({ output, expect }) => {
-    if (expect === undefined) return { pass: false, value: 0, explanation: "case has no expect" };
+  return ({ output, expectedOutput }) => {
+    if (expectedOutput === undefined)
+      return { pass: false, value: 0, explanation: "case has no expectedOutput" };
     const got = get(output);
-    const want = get(expect);
+    const want = get(expectedOutput);
     if (Bun.deepEquals(got, want)) return true;
     return {
       pass: false,
@@ -73,7 +74,7 @@ export function fitsSchema(schema: StandardSchema): CheckFn {
   };
 }
 
-/** Fixed words, or words picked per case (e.g. from `expect`). */
+/** Fixed words, or words picked per case (e.g. from `expectedOutput`). */
 export type Words<I = unknown, O = unknown, E = unknown> =
   | readonly string[]
   | ((run: Run<I, O, E>) => readonly string[]);

src/datasets.test.ts

@@ -0,0 +1,11 @@
+import { expect, test } from "bun:test";
+
+import { parseJsonl } from "./datasets.js";
+
+test("parseJsonl loads expectedOutput and rejects expect or expected", () => {
+  expect(parseJsonl('{"input":"hi","expectedOutput":"HI"}\n')).toEqual([
+    { input: "hi", expectedOutput: "HI" },
+  ]);
+  expect(() => parseJsonl('{"input":"hi","expect":"HI"}\n')).toThrow("expectedOutput");
+  expect(() => parseJsonl('{"input":"hi","expected":"HI"}\n')).toThrow("expectedOutput");
+});

src/datasets.ts

@@ -21,7 +21,10 @@ function toCase(row: Record<string, unknown>, where: string): Case {
   if (!("input" in row)) throw new Error(`${where}: missing "input"`);
   const evalCase: Case = { input: row["input"] };
   if (name !== undefined) evalCase.name = name;
-  if (row["expect"] !== undefined) evalCase.expect = row["expect"];
+  if (row["expect"] !== undefined || row["expected"] !== undefined) {
+    throw new Error(`${where}: use "expectedOutput", not "expect" or "expected"`);
+  }
+  if (row["expectedOutput"] !== undefined) evalCase.expectedOutput = row["expectedOutput"];
   const must = checkNames(row, "must", where);
   if (must !== undefined) evalCase.must = must;
   const skip = checkNames(row, "skip", where);

src/jev.ts

@@ -328,7 +328,7 @@ export function toOutcome(
 
 function toState(run: Run): Record<string, unknown> {
   const state: Record<string, unknown> = { input: run.input, output: run.output };
-  if (run.expect !== undefined) state["expect"] = run.expect;
+  if (run.expectedOutput !== undefined) state["expectedOutput"] = run.expectedOutput;
   if (run.steps.length > 0) state["steps"] = run.steps;
   return state;
 }

src/runner.ts

@@ -4,7 +4,12 @@ import type { Trace } from "./runs.js";
 import { withTimeout } from "./timeout.js";
 import type { CaseResult, Outcome, Run } from "./types.js";
 
-export type NamedCase<I, E> = { name: string; input: I; expect?: E; skip?: readonly string[] };
+export type NamedCase<I, E> = {
+  name: string;
+  input: I;
+  expectedOutput?: E;
+  skip?: readonly string[];
+};
 
 /** String and `ask` checks Jev answers in one call per attempt. */
 export type JevGroup = { judge: Judge; checks: Record<string, Ask | Sentence | string> };
@@ -49,7 +54,7 @@ export async function runOnce<I, O, E>(
       steps: trace.steps,
       ms: Date.now() - started,
     };
-    if (evalCase.expect !== undefined) run.expect = evalCase.expect;
+    if (evalCase.expectedOutput !== undefined) run.expectedOutput = evalCase.expectedOutput;
     return run;
   } finally {
     clearTimeout(timer);
@@ -168,7 +173,7 @@ async function runCase<I, O, E>(
     pass: error === undefined && Object.values(checks).every((o) => o.pass),
     checks,
   };
-  if (evalCase.expect !== undefined) result.expect = evalCase.expect;
+  if (evalCase.expectedOutput !== undefined) result.expectedOutput = evalCase.expectedOutput;
   if (last !== undefined) result.output = last.output;
   if (error !== undefined) result.error = error;
   return result;