diff --git a/README.md b/README.md index a69acf0..8fc45a3 100644 --- a/README.md +++ b/README.md @@ -70,12 +70,13 @@ A check that throws, hangs past `timeout`, or returns anything else fails itself closed; its siblings still run. Jev transport failures fail closed. Jev sees `{ input, output, expectedOutput, steps }`, and a case's string and -`ask` checks answer in one call. The model is not configured on `jev()`. +`ask` checks answer in one call. The model is not configured on `systemOne()`. Build it with `judgeModel({ model, invokeStep, thresholds, escalate, timeout })` from `@corbits/evals/workflow`. `model` is one workflow step (or a one-step workflow). `invokeStep` runs that step; its input is `{ state, questions }` and its output is `{ decisions }` or `{ fallback: true, reason }`. -`jev({ thresholds, escalate })` only sets bars and an escalation agent. +`systemOne({ thresholds, escalate })` only sets bars and an escalation agent. +`jev` is an alias of `systemOne`. Thresholds: a yes/no check passes at p(yes) >= `yes` and fails at p(yes) <= `no` (0.7 / 0.3); `ask` fails as unsure when its top @@ -85,7 +86,7 @@ Thresholds are tuned to a judge version, so pin the step agent's model when they must stay stable. Escalation: `sure` doubles as the escalation gate. Give the judge an -agent — `escalate` on `jev` or `judgeModel`, anything with +agent — `escalate` on `systemOne` or `judgeModel`, anything with `send(prompt)` returning `{ type: "reply", reply }` or `{ type: "suspended" }`, e.g. an `@intx/agent` agent from a `@corbits/*` adapter — and every unsure decision is re-asked with the exact answer diff --git a/e2e/jev.e2e.test.ts b/e2e/jev.e2e.test.ts index 12c8dcd..53a4e3d 100644 --- a/e2e/jev.e2e.test.ts +++ b/e2e/jev.e2e.test.ts @@ -9,7 +9,7 @@ import { ask, EvalFailure, runEvaluation, - jev, + systemOne, type Scorer, type Escalation, type Judge, @@ -246,8 +246,8 @@ describe("thresholds", () => { }); test("bad thresholds throw at definition", async () => { - expect(() => jev({ thresholds: { yes: 0.3, no: 0.5 } })).toThrow("must not exceed"); - expect(() => jev({ thresholds: { sure: 1.5 } })).toThrow("[0, 1]"); + expect(() => systemOne({ thresholds: { yes: 0.3, no: 0.5 } })).toThrow("must not exceed"); + expect(() => systemOne({ thresholds: { sure: 1.5 } })).toThrow("[0, 1]"); expect(() => ask("Ready?", { options: { a: "a" }, pass: "a", sure: 0 })).toThrow("(0, 1]"); const { judge } = stubJev(noul(0.9)); await expect(judged({ question, yes: 0.2, no: 0.5 }, judge)).rejects.toThrow( @@ -589,7 +589,7 @@ describe("judgeModel", () => { task: (input: string) => input, data: "/no/such/cases.jsonl", scorers: { ready: ask("Ready?", { options: { a: "a" }, pass: "a" }) }, - judge: jev({ thresholds: { yes: 0.9 } }), + judge: systemOne({ thresholds: { yes: 0.9 } }), }), ).rejects.toThrow("judge.invoke"); const report = await runEvaluation({ diff --git a/src/index.ts b/src/index.ts index 5ab8218..7011680 100644 --- a/src/index.ts +++ b/src/index.ts @@ -42,6 +42,7 @@ export { export type { Experiment, ExperimentRow } from "./reports.js"; export { ask, + systemOne, jev, DEFAULTS, type Ask, diff --git a/src/jev.ts b/src/jev.ts index 107a081..060e072 100644 --- a/src/jev.ts +++ b/src/jev.ts @@ -29,7 +29,7 @@ export type Judge = { readonly thresholds?: Resolved; readonly escalate?: Escalation; /** - * One call per case attempt. `judgeModel` sets this; `jev()` alone does not, + * One call per case attempt. `judgeModel` sets this; `systemOne()` alone does not, * so it is not a model. * Resolves to `{ decisions }` or `{ fallback: true, reason, detail? }`. */ @@ -63,7 +63,8 @@ function resolve(label: string, raw: Thresholds): Resolved { return out; } -export function jev(options: JevOptions = {}): Judge { +/** Thresholds and an optional escalation agent. Not a model and not an HTTP client. */ +export function systemOne(options: JevOptions = {}): Judge { const thresholds = options.thresholds === undefined ? undefined : resolve("thresholds", options.thresholds); return { @@ -72,6 +73,9 @@ export function jev(options: JevOptions = {}): Judge { }; } +/** Same function as `systemOne`. */ +export { systemOne as jev }; + /** A Jev question with named answers. A plain string check is the yes/no case. */ export type Ask = { readonly question: string; diff --git a/src/workflow.ts b/src/workflow.ts index 4a186c9..52da52c 100644 --- a/src/workflow.ts +++ b/src/workflow.ts @@ -10,7 +10,7 @@ import { } from "@intx/workflow"; import { TRACE, type Traced } from "./runs.js"; -import { jev, type Escalation, type Judge, type Thresholds } from "./jev.js"; +import { systemOne, type Escalation, type Judge, type Thresholds } from "./jev.js"; import { toMs, type Duration } from "./timeout.js"; import type { Step } from "./types.js"; @@ -155,7 +155,7 @@ export function judgeModel(options: JudgeModelOptions): Judge { trigger: { type: "manual" }, steps: { judge: forced }, }); - const base = jev({ + const base = systemOne({ ...(options.thresholds === undefined ? {} : { thresholds: options.thresholds }), ...(options.escalate === undefined ? {} : { escalate: options.escalate }), });