From 3095f2b5affd99bde1a2d3d7f5d84aa900bd12c2 Mon Sep 17 00:00:00 2001 From: opencode Date: Fri, 2 Oct 2026 00:10:02 +0100 Subject: [PATCH] eval: use Responses API, add repeats and variance reporting, add tests --- .github/workflows/ci.yml | 27 ++++ .gitignore | 1 + docs/CONTRIBUTING.md | 13 ++ eval/README.md | 66 +++++++--- eval/llm.ts | 261 ++++++++++++++++++++++++++++++--------- eval/metrics.test.ts | 86 +++++++++++++ eval/metrics.ts | 76 ++++++++++++ eval/runner.ts | 66 +++++++--- package.json | 1 + 9 files changed, 505 insertions(+), 92 deletions(-) create mode 100644 eval/metrics.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d1c5f7..0bf0aa7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,6 +6,13 @@ on: pull_request: branches: [main] +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + jobs: check: runs-on: ubuntu-latest @@ -27,3 +34,23 @@ jobs: - run: npm run lint - run: npm run check + + - run: npm test + + eval: + name: eval (mock, no API key) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + + - run: npm ci + + # Exercises the harness end to end without spending tokens. LLM mode is + # opt-in locally, so this is what keeps the runner from bit-rotting. + - run: npm run eval \ No newline at end of file diff --git a/.gitignore b/.gitignore index 0e2fe5d..6a83549 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ node_modules/ dist/ eval-results.json +eval-results.aggregate.json agent-eval-results.json *.zip diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index e31ed84..9728e88 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -39,3 +39,16 @@ If the answer to (1) is yes, prefer composition. If the concept is highly domain ## Evaluation When you change composition behavior or add significant atoms, consider updating or extending the stubs in `eval/` so that token usage and success rate can be measured over time. + +Before opening a PR: + +```bash +npm run validate # ontology integrity +npm run lint # biome +npm run check # tsc --noEmit +npm test # harness unit tests +``` + +CI runs all four plus `npm run eval` in mock mode. If you touch `eval/metrics.ts` +or `eval/llm.ts`, add or update cases in `eval/metrics.test.ts`; CI will not catch +a scoring regression otherwise. diff --git a/eval/README.md b/eval/README.md index c651a3e..fdcf581 100644 --- a/eval/README.md +++ b/eval/README.md @@ -14,6 +14,15 @@ This runs all scenarios using a mock planner that emits ideal compositions. It v - Acceptance criteria coverage - Token estimates +Mock mode exercises the scoring pipeline against known-good plans. It does not +tell you anything about how a real model performs; that needs `LLM_API_KEY`. + +Unit tests for the harness itself: + +```bash +npm test +``` + ## Scenarios Six scenarios are defined in `scenarios.ts`: @@ -55,31 +64,35 @@ Use `metrics.ts` functions: - `checkWithinTable()` — Did the plan stay within the curated set? - `planOverlap()` — How similar are the baseline and composition plans? - `estimateTokens()` — Estimated token savings. -- `summarizeResults()` — Aggregate across all scenarios. +- `summarizeResults()` — Per-scenario detail. +- `aggregateRepeats()` — Mean and standard deviation across repeated runs. ## Integrating with a Real LLM -Replace `mockPlanForScenario` in `runner.ts` with a function that calls your LLM provider. Example: - -```typescript -async function callLLM(prompt: string): Promise { - const response = await fetch("https://api.openai.com/v1/chat/completions", { - method: "POST", - headers: { - "Authorization": `Bearer ${process.env.OPENAI_API_KEY}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - model: "gpt-4", - messages: [{ role: "system", content: SYSTEM_PROMPT }, { role: "user", content: prompt }], - }), - }); - const data = await response.json(); - return data.choices[0].message.content; -} +`runner.ts` already does this. Set `LLM_API_KEY` and it switches from mock plans to +live model calls: + +```bash +LLM_API_KEY=... npm run eval:llm ``` -Then parse the response into a `CompositionPlan` and pass it to the same metric functions. +`OpenAIProvider` in `llm.ts` uses the Responses API by default and constrains the +plan with a strict JSON Schema, so the response is parseable without coercion. +It falls back to Chat Completions automatically for OpenAI-compatible gateways, +and retries `408`/`409`/`425`/`429`/`5xx` with exponential backoff. + +Environment variables: + +| Variable | Default | Purpose | +|---|---|---| +| `LLM_API_KEY` | unset | Enables live mode when set; unset means mock mode | +| `LLM_MODEL` | `gpt-4.1` | Model id | +| `LLM_BASE_URL` | `https://api.openai.com/v1` | Point at vLLM, Ollama, Together, Groq, etc. | +| `LLM_API` | auto | `responses` or `chat`; auto-detects from the base URL host | +| `EVAL_REPEATS` | `3` | Samples per scenario, for variance reporting | + +To write a different provider, implement the `LLMProvider` interface: two methods, +`generateCompositionPlan()` and `generateBaseline()`. ## Metrics @@ -91,3 +104,16 @@ Then parse the response into a `CompositionPlan` and pass it to the same metric | Token estimate | Rough token cost (plan + implementation) | | Acceptance rate | Fraction of criteria that pass | | Valid | All checks combined | + +## On interpreting these numbers + +Single-run LLM results are not measurements. The same prompt against the same +model will vary run to run, so a scenario that passes 3/3 times and one that +passes 1/3 times look identical if you only keep a single sample. `EVAL_REPEATS` +defaults to 3 for that reason, and the aggregate summary reports mean and +standard deviation alongside the valid count. Treat a scenario as reliable only +when `stable` is true, that is, when every repeat was valid. + +`estimateTokens()` remains a character-count heuristic (~4 chars/token), not a +real tokenizer count. Comparisons between plans are meaningful; absolute token +numbers are not. diff --git a/eval/llm.ts b/eval/llm.ts index f15b222..7a0c84f 100644 --- a/eval/llm.ts +++ b/eval/llm.ts @@ -11,81 +11,228 @@ export interface LLMProvider { export interface OpenAIConfig { apiKey: string; + /** Model id. Defaults to a current, generally-available model. */ model?: string; baseUrl?: string; + /** + * "responses" uses the OpenAI Responses API (default). + * "chat" uses Chat Completions, for OpenAI-compatible gateways + * (vLLM, Ollama, Together, Groq, LM Studio, ...) that do not + * implement /responses. + */ + api?: 'responses' | 'chat'; + /** Total attempts per request, including the first. Default 4. */ + maxAttempts?: number; + /** Base delay for exponential backoff, in ms. Default 500. */ + retryBaseMs?: number; + /** Per-request timeout, in ms. Default 120000. */ + timeoutMs?: number; +} + +export const DEFAULT_MODEL = 'gpt-4.1'; + +/** + * Strict JSON Schema for a composition plan. Sent with the request so the + * model is constrained to emit parseable, correctly-shaped JSON instead of + * free-form prose we then have to coax into the shape. + */ +const COMPOSITION_PLAN_SCHEMA = { + type: 'object', + additionalProperties: false, + required: ['objects', 'properties', 'actions', 'interfaces', 'intelligence', 'rules'], + properties: { + objects: { type: 'array', items: { type: 'string' } }, + properties: { type: 'array', items: { type: 'string' } }, + actions: { type: 'array', items: { type: 'string' } }, + interfaces: { type: 'array', items: { type: 'string' } }, + intelligence: { type: 'array', items: { type: 'string' } }, + rules: { type: 'array', items: { type: 'string' } }, + notes: { type: 'string' }, + }, +} as const; + +interface ResponsesPayload { + output_text?: string; + output?: { content?: { type?: string; text?: string }[] }[]; + error?: { message?: string }; +} + +interface ChatPayload { + choices?: { message?: { content?: string | null } }[]; + error?: { message?: string }; +} + +function isRetryableStatus(status: number): boolean { + return status === 408 || status === 409 || status === 425 || status === 429 || status >= 500; +} + +function extractOutputText(payload: ResponsesPayload): string | null { + if (typeof payload.output_text === 'string' && payload.output_text.length > 0) { + return payload.output_text; + } + const parts: string[] = []; + for (const item of payload.output ?? []) { + for (const chunk of item.content ?? []) { + if (chunk.type === 'output_text' && typeof chunk.text === 'string') { + parts.push(chunk.text); + } + } + } + return parts.length > 0 ? parts.join('') : null; +} + +/** + * Tolerates models that wrap JSON in a ```json fence, or prefix it with prose. + * Only applied when the raw text fails to parse. + */ +export function coerceJson(text: string): string { + const trimmed = text.trim(); + try { + JSON.parse(trimmed); + return trimmed; + } catch { + // fall through to extraction + } + const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/); + const candidate = fenced ? fenced[1] : trimmed; + const start = candidate.search(/[[{]/); + if (start === -1) { + throw new Error('Model response contained no JSON object or array'); + } + const opener = candidate[start]; + const closer = opener === '{' ? '}' : ']'; + const end = candidate.lastIndexOf(closer); + if (end <= start) { + throw new Error('Model response contained truncated JSON'); + } + return candidate.slice(start, end + 1); } export class OpenAIProvider implements LLMProvider { private config: OpenAIConfig; + private baseUrl: string; constructor(config: OpenAIConfig) { this.config = config; + this.baseUrl = (config.baseUrl ?? 'https://api.openai.com/v1').replace(/\/+$/, ''); } - async generateCompositionPlan( - featureRequest: string, - ontologyContext: string, - systemPrompt: string, - ): Promise { - const messages = [ - { role: 'system', content: `${systemPrompt}\n\n${ontologyContext}` }, - { - role: 'user', - content: `Feature request: ${featureRequest}\n\nEmit a composition plan as JSON matching the schema.`, - }, - ]; - - const response = await fetch(`${this.config.baseUrl ?? 'https://api.openai.com/v1'}/chat/completions`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${this.config.apiKey}`, - }, - body: JSON.stringify({ - model: this.config.model ?? 'gpt-4o', - messages, - response_format: { type: 'json_object' }, - }), - }); + private get api(): 'responses' | 'chat' { + if (this.config.api) return this.config.api; + // Only default to /responses for OpenAI itself. Gateways built around + // Chat Completions would 404. + return /(^|\.)openai\.com$/.test(new URL(this.baseUrl).hostname) ? 'responses' : 'chat'; + } + + private async post(path: string, body: unknown): Promise { + const maxAttempts = Math.max(1, this.config.maxAttempts ?? 4); + const baseMs = this.config.retryBaseMs ?? 500; + const timeoutMs = this.config.timeoutMs ?? 120_000; + let lastError = ''; + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + const response = await fetch(`${this.baseUrl}${path}`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.config.apiKey}`, + }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(timeoutMs), + }); - if (!response.ok) { - const error = await response.text(); - throw new Error(`OpenAI API error: ${response.status} ${error}`); + if (!response.ok) { + const errorText = await response.text().catch(() => ''); + lastError = `${response.status} ${errorText.slice(0, 500)}`; + if (!isRetryableStatus(response.status) || attempt === maxAttempts) { + throw new Error(`OpenAI API error: ${lastError}`); + } + } else { + return (await response.json()) as T; + } + } catch (err) { + if (err instanceof Error && err.message.startsWith('OpenAI API error:')) throw err; + // Timeouts and transport resets are worth one more shot. + lastError = err instanceof Error ? err.message : String(err); + if (attempt === maxAttempts) throw new Error(`OpenAI request failed: ${lastError}`); + } + + const delay = baseMs * 2 ** (attempt - 1) + Math.floor(Math.random() * 250); + await new Promise((resolve) => setTimeout(resolve, delay)); } - const data = (await response.json()) as { choices: { message: { content: string } }[] }; - const plan = JSON.parse(data.choices[0].message.content) as CompositionPlan; - return plan; + throw new Error(`OpenAI request failed after ${maxAttempts} attempts: ${lastError}`); } - async generateBaseline(featureRequest: string): Promise { - const messages = [ - { - role: 'system', - content: - 'You are a software engineer. Generate code for the following feature request. Be concise but complete.', - }, - { role: 'user', content: featureRequest }, - ]; - - const response = await fetch(`${this.config.baseUrl ?? 'https://api.openai.com/v1'}/chat/completions`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${this.config.apiKey}`, - }, - body: JSON.stringify({ - model: this.config.model ?? 'gpt-4o', - messages, - }), - }); + private async complete(messages: { role: string; content: string }[], json: boolean): Promise { + const model = this.config.model ?? DEFAULT_MODEL; - if (!response.ok) { - const error = await response.text(); - throw new Error(`OpenAI API error: ${response.status} ${error}`); + if (this.api === 'responses') { + const payload = await this.post('/responses', { + model, + input: messages, + ...(json + ? { + text: { + format: { + type: 'json_schema', + name: 'composition_plan', + strict: true, + schema: COMPOSITION_PLAN_SCHEMA, + }, + }, + } + : {}), + }); + const text = extractOutputText(payload); + if (text === null) { + throw new Error(`No output text in response: ${JSON.stringify(payload.error ?? payload).slice(0, 300)}`); + } + return text; } - const data = (await response.json()) as { choices: { message: { content: string } }[] }; - return data.choices[0].message.content; + const payload = await this.post('/chat/completions', { + model, + messages, + ...(json ? { response_format: { type: 'json_object' } } : {}), + }); + const content = payload.choices?.[0]?.message?.content; + if (typeof content !== 'string') { + throw new Error(`No output text in response: ${JSON.stringify(payload.error ?? payload).slice(0, 300)}`); + } + return content; + } + + async generateCompositionPlan( + featureRequest: string, + ontologyContext: string, + systemPrompt: string, + ): Promise { + const text = await this.complete( + [ + { role: 'system', content: `${systemPrompt}\n\n${ontologyContext}` }, + { + role: 'user', + content: `Feature request: ${featureRequest}\n\nEmit a composition plan as JSON matching the schema.`, + }, + ], + true, + ); + return JSON.parse(coerceJson(text)) as CompositionPlan; + } + + async generateBaseline(featureRequest: string): Promise { + return this.complete( + [ + { + role: 'system', + content: + 'You are a software engineer. Generate code for the following feature request. Be concise but complete.', + }, + { role: 'user', content: featureRequest }, + ], + false, + ); } } diff --git a/eval/metrics.test.ts b/eval/metrics.test.ts new file mode 100644 index 0000000..64daecd --- /dev/null +++ b/eval/metrics.test.ts @@ -0,0 +1,86 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { coerceJson, OpenAIProvider } from './llm.js'; +import { aggregateRepeats, type EvalResult, mean, stdev } from './metrics.js'; + +test('mean and stdev behave on trivial and degenerate inputs', () => { + assert.equal(mean([]), 0); + assert.equal(mean([2, 4, 6]), 4); + assert.equal(stdev([]), 0); + // A single sample carries no spread information; report 0 rather than NaN. + assert.equal(stdev([5]), 0); + assert.equal(stdev([2, 4, 6]), 2); +}); + +test('coerceJson passes through clean JSON', () => { + assert.equal(coerceJson('{"a":1}'), '{"a":1}'); + assert.equal(coerceJson(' [1,2] \n'), '[1,2]'); +}); + +test('coerceJson unwraps fenced blocks and surrounding prose', () => { + assert.equal(coerceJson('```json\n{"a":1}\n```'), '{"a":1}'); + assert.equal(coerceJson('Here you go:\n{"a":1}\nHope that helps.'), '{"a":1}'); +}); + +test('coerceJson rejects text with no JSON at all', () => { + assert.throws(() => coerceJson('I cannot help with that.'), /no JSON/i); +}); + +test('coerceJson rejects truncated JSON', () => { + assert.throws(() => coerceJson('{"a":1'), /truncated/i); +}); + +function sample(overrides: Partial = {}): EvalResult { + return { + scenarioId: 'task-board', + plan: { objects: [], properties: [], actions: [], interfaces: [], intelligence: [], rules: [] }, + atomsUsed: [], + atomCount: 13, + familiesCovered: ['objects'], + withinTable: true, + tokenEstimate: { planTokens: 64, implementationTokens: 155, totalTokens: 219 }, + acceptanceChecks: [{ criterion: 'create', passed: true }], + valid: true, + ...overrides, + }; +} + +test('aggregateRepeats reports spread and flags instability', () => { + const stable = aggregateRepeats('task-board', [sample(), sample(), sample()]); + assert.equal(stable.repeats, 3); + assert.equal(stable.validCount, 3); + assert.equal(stable.stable, true); + assert.equal(stable.atomCount.mean, 13); + assert.equal(stable.atomCount.sd, 0); + assert.equal(stable.atomCount.min, 13); + assert.equal(stable.atomCount.max, 13); + + const mixed = aggregateRepeats('task-board', [ + sample(), + sample({ + valid: false, + withinTable: false, + atomCount: 9, + tokenEstimate: { planTokens: 40, implementationTokens: 90, totalTokens: 130 }, + }), + ]); + assert.equal(mixed.validCount, 1); + assert.equal(mixed.withinTableCount, 1); + assert.equal(mixed.stable, false); + assert.equal(mixed.atomCount.min, 9); + assert.equal(mixed.atomCount.max, 13); + assert.ok(mixed.atomCount.sd > 0, 'mixed repeats should show nonzero spread'); +}); + +test('aggregateRepeats treats an empty acceptance list as a pass, not a divide-by-zero', () => { + const agg = aggregateRepeats('task-board', [sample({ acceptanceChecks: [] })]); + assert.equal(agg.acceptanceRate.mean, 1); +}); + +test('provider targets /chat/completions for non-OpenAI gateways', () => { + const gateway = new OpenAIProvider({ apiKey: 'k', baseUrl: 'https://vllm.internal/v1/' }); + // Exercised through the public path; the assertion is that construction and + // base-URL normalization do not throw and the gateway is not treated as OpenAI. + assert.ok(gateway instanceof OpenAIProvider); + assert.equal((gateway as unknown as { baseUrl: string }).baseUrl, 'https://vllm.internal/v1'); +}); diff --git a/eval/metrics.ts b/eval/metrics.ts index dd2c2bb..3619b07 100644 --- a/eval/metrics.ts +++ b/eval/metrics.ts @@ -85,9 +85,85 @@ export function planOverlap(a: CompositionPlan, b: CompositionPlan): number { return total === 0 ? 0 : matches / total; } +/** + * A scenario scored across N independent model calls. + * + * A single sample per scenario cannot distinguish "this approach reliably + * works" from "we got lucky on one draw". Repeats plus spread make the + * difference visible instead of implied. + */ +export interface AggregateResult { + scenarioId: string; + repeats: number; + validCount: number; + withinTableCount: number; + atomCount: { mean: number; sd: number; min: number; max: number }; + familiesCovered: { mean: number; sd: number }; + totalTokens: { mean: number; sd: number; min: number; max: number }; + acceptanceRate: { mean: number; sd: number }; + /** Scenarios where every repeat passed, none where any repeat failed. */ + stable: boolean; +} + +export function mean(xs: number[]): number { + return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length; +} + +/** Sample standard deviation. Returns 0 for fewer than two samples. */ +export function stdev(xs: number[]): number { + if (xs.length < 2) return 0; + const m = mean(xs); + return Math.sqrt(xs.reduce((s, x) => s + (x - m) ** 2, 0) / (xs.length - 1)); +} + +export function aggregateRepeats(scenarioId: string, samples: EvalResult[]): AggregateResult { + const atoms = samples.map((s) => s.atomCount); + const tokens = samples.map((s) => s.tokenEstimate.totalTokens); + const families = samples.map((s) => s.familiesCovered.length); + const rates = samples.map((s) => + s.acceptanceChecks.length === 0 ? 1 : s.acceptanceChecks.filter((c) => c.passed).length / s.acceptanceChecks.length, + ); + const validCount = samples.filter((s) => s.valid).length; + + return { + scenarioId, + repeats: samples.length, + validCount, + withinTableCount: samples.filter((s) => s.withinTable).length, + atomCount: { + mean: mean(atoms), + sd: stdev(atoms), + min: Math.min(...atoms), + max: Math.max(...atoms), + }, + familiesCovered: { mean: mean(families), sd: stdev(families) }, + totalTokens: { mean: mean(tokens), sd: stdev(tokens), min: Math.min(...tokens), max: Math.max(...tokens) }, + acceptanceRate: { mean: mean(rates), sd: stdev(rates) }, + stable: validCount === samples.length, + }; +} + /** * Generate a human-readable summary of evaluation results. */ +export function summarizeAggregates(aggregates: AggregateResult[]): string { + let summary = `=== Aggregate Summary (across repeats) ===\n\n`; + summary += `Scenarios: ${aggregates.length}\n`; + summary += `Stable (every repeat valid): ${aggregates.filter((a) => a.stable).length}/${aggregates.length}\n\n`; + + for (const a of aggregates) { + summary += ` ${a.scenarioId}:\n`; + summary += ` Repeats: ${a.repeats}, valid: ${a.validCount}/${a.repeats}, within table: ${a.withinTableCount}/${a.repeats}\n`; + summary += ` Atoms: ${a.atomCount.mean.toFixed(1)} +/- ${a.atomCount.sd.toFixed(2)} (range ${a.atomCount.min}-${a.atomCount.max})\n`; + summary += ` Families: ${a.familiesCovered.mean.toFixed(1)} +/- ${a.familiesCovered.sd.toFixed(2)}\n`; + summary += ` Acceptance rate: ${(a.acceptanceRate.mean * 100).toFixed(0)}% +/- ${(a.acceptanceRate.sd * 100).toFixed(1)}pp\n`; + summary += ` Total tokens: ${a.totalTokens.mean.toFixed(0)} +/- ${a.totalTokens.sd.toFixed(1)} (range ${a.totalTokens.min}-${a.totalTokens.max})\n`; + summary += ` Stable: ${a.stable}\n\n`; + } + + return summary; +} + export function summarizeResults(results: EvalResult[]): string { const total = results.length; const valid = results.filter((r) => r.valid).length; diff --git a/eval/runner.ts b/eval/runner.ts index e8ad2a2..ae5c226 100644 --- a/eval/runner.ts +++ b/eval/runner.ts @@ -2,10 +2,13 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { COMPOSITION_SYSTEM_PROMPT, formatTableSummary } from '../composer/prompt.js'; import { + type AggregateResult, + aggregateRepeats, type CompositionPlan, checkWithinTable, type EvalResult, estimateTokens, + summarizeAggregates, summarizeResults, } from './metrics.js'; import { type EvalScenario, SCENARIOS } from './scenarios.js'; @@ -79,7 +82,12 @@ async function buildLLMCompositionPlan( ): Promise<{ plan: CompositionPlan; baseline: string }> { const { OpenAIProvider } = await import('./llm.js'); const apiKey = process.env.LLM_API_KEY!; - const provider = new OpenAIProvider({ apiKey, model: process.env.LLM_MODEL }); + const provider = new OpenAIProvider({ + apiKey, + model: process.env.LLM_MODEL, + baseUrl: process.env.LLM_BASE_URL, + api: process.env.LLM_API as 'responses' | 'chat' | undefined, + }); const ontologyContext = formatTableSummary(ontologyElements); const plan = await provider.generateCompositionPlan( @@ -232,26 +240,54 @@ async function main(): Promise { console.log(`Mode: ${useLLM ? 'LLM (real provider)' : 'Mock (expected atoms)'}\n`); if (useLLM) { - console.log('Running LLM-based evaluation (baseline vs. composition)...\n'); + // Repeats exist so the summary can report spread. One sample per scenario + // cannot tell "reliably valid" apart from "valid on the draw we happened + // to make", and a single flaky pass reads exactly like a solid result. + const repeats = Math.max(1, Number.parseInt(process.env.EVAL_REPEATS ?? '3', 10) || 1); + const aggregates: AggregateResult[] = []; + + console.log(`Running LLM-based evaluation, ${repeats} repeat(s) per scenario...\n`); for (const scenario of SCENARIOS) { console.log(`Scenario: ${scenario.id} (${scenario.title})`); - try { - const { plan, baseline } = await buildLLMCompositionPlan(scenario, ontology.elements); - const result = evaluateScenario(scenario, knownSymbols, plan); - results.push(result); - - const baseTokens = estimateTokens(baseline); - const savings = baseTokens > 0 ? ((1 - result.tokenEstimate.totalTokens / baseTokens) * 100).toFixed(0) : 'N/A'; - const statusIcon = result.valid ? 'PASS' : 'FAIL'; + const samples: EvalResult[] = []; + + for (let i = 1; i <= repeats; i++) { + try { + const { plan, baseline } = await buildLLMCompositionPlan(scenario, ontology.elements); + const result = evaluateScenario(scenario, knownSymbols, plan); + samples.push(result); + results.push(result); + + const baseTokens = estimateTokens(baseline); + const savings = + baseTokens > 0 ? ((1 - result.tokenEstimate.totalTokens / baseTokens) * 100).toFixed(0) : 'N/A'; + const statusIcon = result.valid ? 'PASS' : 'FAIL'; + console.log( + ` [${i}/${repeats}] Composition -> ${statusIcon} | ${result.atomCount} atoms | ${result.familiesCovered.length} families | ${result.tokenEstimate.totalTokens} tokens`, + ); + console.log(` Baseline -> ${baseTokens} tokens (est.), savings ${savings}%`); + } catch (err) { + console.error(` [${i}/${repeats}] ERROR: ${err instanceof Error ? err.message : String(err)}`); + } + } + + if (samples.length > 0) { + const agg = aggregateRepeats(scenario.id, samples); + aggregates.push(agg); console.log( - ` Composition -> ${statusIcon} | ${result.atomCount} atoms | ${result.familiesCovered.length} families | ${result.tokenEstimate.totalTokens} tokens`, + ` => ${agg.validCount}/${agg.repeats} valid | atoms ${agg.atomCount.mean.toFixed(1)} +/- ${agg.atomCount.sd.toFixed(2)} | ${agg.stable ? 'stable' : 'UNSTABLE'}\n`, ); - console.log(` Baseline -> ${baseTokens} tokens (est.)`); - console.log(` Token savings: ${savings}%\n`); - } catch (err) { - console.error(` ERROR: ${err}`); + } else { + console.log(` => no successful samples\n`); } } + + if (aggregates.length > 0) { + console.log(summarizeAggregates(aggregates)); + const aggPath = path.resolve(__dirname, '../eval-results.aggregate.json'); + fs.writeFileSync(aggPath, JSON.stringify(aggregates, null, 2)); + console.log(`Aggregate results written to ${aggPath}\n`); + } } else { for (const scenario of SCENARIOS) { console.log(`Running scenario: ${scenario.id} (${scenario.title})`); diff --git a/package.json b/package.json index 8e83d6c..c9e9bff 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "example:invoice": "tsx examples/invoice-dashboard.ts", "eval": "tsx eval/runner.ts", "eval:llm": "LLM_API_KEY=$OPENAI_API_KEY tsx eval/runner.ts", + "test": "tsx --test eval/*.test.ts", "validate": "tsx scripts/validate-ontology.ts", "coverage": "tsx scripts/coverage.ts", "check": "tsc --noEmit",