diff --git a/docs/jev.md b/docs/jev.md new file mode 100644 index 00000000..02fc5187 --- /dev/null +++ b/docs/jev.md @@ -0,0 +1,159 @@ +# Jev without another runtime + +Jev belongs in a bounded decision, judge, or analyst callback. Runtime already owns +execution, graph scheduling, continuation, journals, and budgets. Do not add a +`JevAgent`, a second graph runner, or another critique/revise loop. + +## Graphs: use the existing registry adapter + +After the Eval change that publishes `@tangle-network/agent-eval/jev`, construct an +ordinary `jevAnalyst` and register it in the existing Eval registry. Then use +`analystsFromRegistry` to attach it to an `analyzes` edge. The application owns the +scoped native transport, trace selection/redaction, findings, and paid-call account. + +```ts +import { AnalystRegistry } from '@tangle-network/agent-eval/analyst' +import { jevAnalyst } from '@tangle-network/agent-eval/jev' +import { + analystsFromRegistry, + promptHandle, + runGraph, +} from '@tangle-network/agent-runtime/kernel' + +const registry = new AnalystRegistry({ + hooks: { + // Retain failed/skipped summaries and usage, not only the returned findings. + onAfterAnalyze: persistAnalystSummary, + }, +}) + +registry.register(jevAnalyst({ + id: 'evidence-review', + description: 'Assess supplied execution evidence against the task requirements', + inputKind: 'trace-store', + model: pinnedJevModel, + version: policyVersion, + questions: reviewQuestions, + evaluate: nativeEvaluation, // Router /v1/systemone; single accounted attempt. + renderState: renderAuthorizedTraceEvidence, + findings: mapAnswersToEvidenceBackedFindings, + maximumCharge: enforcedMaximumCharge, + receipt: readNativeReceipt, +})) + +const graph = { + nodes: [ + { id: 'driver', profile: driverProfile }, + { id: 'worker', profile: workerProfile }, + ], + edges: [ + { + kind: 'delegates' as const, + from: 'driver', to: 'worker', + directive: promptHandle('delegates/worker-brief/v1'), + continuity: 'resume' as const, + maxTraversals: authorizedTraversalLimit, + }, + { + kind: 'analyzes' as const, + analyst: 'evidence-review', over: ['worker'], to: 'driver', + directive: promptHandle('analyzes/findings-report/v1'), + }, + ], + budget: authorizedBudget, + deliverable: independentCompletionCheck, +} + +const result = await runGraph(graph, { + ...executionOptions, + analysts: analystsFromRegistry(registry, [{ + id: 'evidence-review', description: 'Review execution evidence', area: 'verification', + }], { runOpts: { + signal, + costLedger: sharedCostLedger, + costPhase: 'graph-review', + } }), +}) +``` + +The named values above are caller-owned domain/execution configuration, not hidden +defaults. Use real span/event/artifact references in findings. A probability-like +answer is a model judgment, not a causal diagnosis or a demonstrated calibration. +Do not reveal final-test rubrics to workers through review feedback. + +`runGraph` keeps node pinning, versioned directives, traversal caps, resume semantics, +and edge-delivery evidence. A function-shaped analyst does not become free: its +native call must use the shared paid-call ledger with a genuine enforced bound. +Do not assume a separate analyst ledger automatically debits the Supervisor's token +pool. Allocate explicit analyst authority and reconcile its receipts in the total +experiment account; where one Supervisor pool must own every call, use the existing +metered agent-analyst path rather than hiding inference inside a supposedly pure lens. + +The registry isolates analyst failures; its findings list alone is not proof of a +successful analysis. Persist and inspect the existing summaries. A failed optional +review must not be interpreted as a clean bill of health or an authorization to act. + +## Before and after local inference + +Use the actual call boundaries, not a new hook bus: + +- `ToolLoopHooks.beforeTurn` is awaited and can prepare the conversation. It runs + **before** the existing compaction step. +- `ToolLoopCompaction.distill` owns compaction at a clean tool-call boundary. Do not + add a second compactor or orphan tool requests from their results. +- A caller-owned `ToolLoopChat` can prepare the final outgoing request and inspect + the completed result. Preserve the provided signal, call/correlation identity, + tool grants, served-model evidence, and all physical-attempt receipts. +- `RuntimeHooks` is observation-only. Its notifier does not await a required model + decision and deliberately isolates observer errors. Use a durable sink/queue for + analysis that must survive coordinator termination. + +A normal awaited composition is enough: + +```ts +const brain: ToolLoopChat = async (messages, tools, context) => { + context?.signal.throwIfAborted() + const prepared = await prepareAllowedContext(messages, context) + context?.signal.throwIfAborted() + const response = await modelCall(prepared, tools, context) + await inspectCompletedResponse(response, context) + return response +} +``` + +This is an ordering example, not a complete metering wrapper. When preparation or +inspection calls Jev, admit those calls through the shared paid-call account and +preserve their receipts even if a later operation fails. Do not return only the +main model's usage and call the entire program measured. Do not hide paid work in +an observer. Optional optimization failures should preserve the ordinary agent path; +authorization and irreversible actions remain deterministic controls. + +A blocking output policy must run before delivery. It cannot retract streamed text +or undo completed tool effects. Prefer explicit generation → evaluation → revision +composition over silently rewriting a user's answer after it has been delivered. + +## Sandbox coverage + +A sandbox prompt may contain many native model requests. Session preparation is not +a portable barrier before each request inside Claude Code, Codex, or another external +harness. Use the substrate's supported native hooks and document their actual scope. +Do not add a misleading universal `beforeInference` flag to `agent-provider-tangle`. + +## Optimization + +The new Eval judge returns the existing `JudgeConfig`, so current campaigns and +`improve()` remain the path for comparing prompt variants, questions, rubric levels, +context policies, and output-selection policies. Keep train, selection, and final-test +partitions separate, and retain exact candidate/model versions plus all inner costs. +Jev selects/evaluates bounded options; generative models still produce arbitrary +arguments, code, summaries, and prompt rewrites. + +## Verification and dependencies + +`tests/kernel/jev-composition.test.ts` exercises the existing registry/graph and local +call ordering with deterministic injected responses. It is not a live Jev benchmark. +No new Runtime dependency or public orchestration primitive is needed. Production +consumers need the released Eval adapter and the native Router endpoint; do not +claim support merely because an unpublished import appears in a snippet. + +Related changes: agent-eval #761, tangle-router #531, agent-dev-container #7620. diff --git a/src/runtime/supervise-surface.ts b/src/runtime/supervise-surface.ts index 23e02831..f2ef7d65 100644 --- a/src/runtime/supervise-surface.ts +++ b/src/runtime/supervise-surface.ts @@ -162,6 +162,13 @@ export function analystsFromRegistry( { traceStore: trace }, { ...opts?.runOpts, only: [kindId] }, ) + // Registry failures are isolated, not successful reviews with zero findings. + const summary = result.per_analyst.find((entry) => entry.analyst_id === kindId) + if (summary?.status !== 'ok') { + throw new ValidationError( + `analyst ${JSON.stringify(kindId)} did not complete: ${summary?.status ?? 'missing result'}`, + ) + } return result.findings }, ...(authoring === undefined @@ -218,7 +225,7 @@ export interface AnalystAuthoring { * * The version is DERIVED from the definition's own canonical digest, never supplied by the manager. * That is what makes an invented lens reproducible: the same authored words always compile to the - * same analyst version, two different wordings can never share one, and a finding's `analyst_id` + * same version, two different wordings can never share one, and a finding's `analyst_id` * plus version names the exact text that produced it. */ function traceAnalystFromAuthored(definition: AuthoredAnalystDefinition): TraceAnalystDefinition { diff --git a/src/runtime/tool-loop.ts b/src/runtime/tool-loop.ts index b6f2fd7d..447bd1e0 100644 --- a/src/runtime/tool-loop.ts +++ b/src/runtime/tool-loop.ts @@ -64,7 +64,7 @@ export type ToolLoopChat = ( export interface ToolLoopHooks { /** Run before each inference turn (e.g. flush queued steers into `messages`). */ beforeTurn?(turn: number, messages: ToolLoopMessageRecord[]): void | Promise - /** Return true to stop before the next turn (e.g. pool starved / deadline passed / aborted). */ + /** Stop predicate, rechecked after awaited preparation before spending on inference. */ stopBefore?(turn: number): boolean /** Each turn's usage, for metering into a conserved budget pool. */ onUsage?(usage: { input: number; output: number }): void @@ -206,10 +206,15 @@ export async function runBrainLoop(opts: { for (let turn = 1; maxTurns === 0 || turn <= maxTurns; turn += 1) { if (opts.hooks?.stopBefore?.(turn)) break await opts.hooks?.beforeTurn?.(turn, messages) + // Preparation may run a classifier or await a steer while authority changes. + if (opts.hooks?.stopBefore?.(turn)) break // Close the chapter BEFORE the inference turn that would otherwise re-bill the whole transcript: // distill the accumulated middle to a compact note so this and every later turn pay O(working-set), // not O(total-history). A clean boundary — the prior turn's tool replies are already folded in. - if (opts.compaction) await maybeCompact(messages, opts.compaction, turn) + if (opts.compaction) { + await maybeCompact(messages, opts.compaction, turn) + if (opts.hooks?.stopBefore?.(turn)) break + } const r = await opts.chat(messages, opts.tools) completedTurns = turn if (r.usage) { diff --git a/tests/kernel/jev-composition.test.ts b/tests/kernel/jev-composition.test.ts new file mode 100644 index 00000000..040b287a --- /dev/null +++ b/tests/kernel/jev-composition.test.ts @@ -0,0 +1,121 @@ +import { type AnalystContext, CostLedger } from '@tangle-network/agent-eval' +import { AnalystRegistry } from '@tangle-network/agent-eval/analyst' +import { describe, expect, it } from 'vitest' +import { shotLoop } from '../../examples/graphs/shot-loop' +import { analystsFromRegistry } from '../../src/runtime/supervise-surface' +import { runBrainLoop } from '../../src/runtime/tool-loop' +import { runGraphWithTestBrain } from '../../src/testing' + +// These are composition proofs over deterministic fixtures, not live Jev quality tests. +describe('native decision composition without another runtime', () => { + it('delivers an ordinary Eval analyst through the existing graph and ledger', async () => { + const contexts: AnalystContext[] = [] + const registry = new AnalystRegistry() + const sharedCostLedger = new CostLedger() + const controller = new AbortController() + registry.register({ + id: 'verify', + description: 'Fixture for a native evidence-review analyst', + inputKind: 'trace-store', + version: 'native-fixture-v1', + // No network or real model runs in this fixture. + cost: { kind: 'deterministic' }, + async analyze(store, context) { + expect(store).toBeDefined() + contexts.push(context) + const answer = { type: 'noul', noul: 0.75 } as const + return [ + { + schema_version: '1.0.0', + finding_id: `fixture-${context.runId}`, + analyst_id: 'verify', + produced_at: new Date(0).toISOString(), + severity: 'info', + area: 'verification', + claim: 'Injected native-answer fixture reached the graph reviewer', + confidence: answer.noul, + evidence_refs: [], + derived_from_judge: true, + metadata: { fixture: true, nativeAnswer: answer }, + }, + ] + }, + }) + const { graph, opts } = shotLoop() + const analysts = analystsFromRegistry( + registry, + [ + { + id: 'verify', + description: 'Native-answer fixture', + area: 'verification', + }, + ], + { + runOpts: { + signal: controller.signal, + costLedger: sharedCostLedger, + costPhase: 'graph-review', + tags: { policy: 'native-fixture-v1' }, + }, + }, + ) + const result = await runGraphWithTestBrain(graph, { + ...opts, + runId: 'jev-composition-fixture', + analysts, + }) + expect(contexts.length).toBeGreaterThan(0) + expect(contexts.every((context) => context.costLedger === sharedCostLedger)).toBe(true) + expect(contexts.every((context) => context.costPhase === 'graph-review')).toBe(true) + expect(contexts.every((context) => context.tags?.policy === 'native-fixture-v1')).toBe(true) + expect(result.ledger.some((edge) => edge.kind === 'analyzes')).toBe(true) + expect(result.ledger.some((edge) => edge.edge === 'analyzes:verify:coder->reviewer')).toBe(true) + expect(graph.nodes.map((node) => node.id)).toEqual(['reviewer', 'coder']) + }) + + it('prepares before compaction and observes the exact post-compaction request', async () => { + const order: string[] = [] + const result = await runBrainLoop({ + initialMessages: [ + { role: 'system', content: 'Keep mandatory instructions.' }, + { role: 'user', content: 'Original task.' }, + { role: 'assistant', content: 'Prior work.' }, + ], + tools: [], + maxTurns: 1, + hooks: { + async beforeTurn(_turn, messages) { + order.push('prepare') + messages.push({ role: 'user', content: 'Selected optional context.' }) + }, + }, + compaction: { + thresholdTokens: 1, + preserveHead: 2, + distill: async () => { + order.push('compact') + return 'Retained progress.' + }, + }, + chat: async (messages) => { + order.push('inference') + expect(messages[0]?.content).toBe('Keep mandatory instructions.') + expect(messages[1]?.content).toBe('Original task.') + expect(messages.at(-1)?.content).toContain('Retained progress.') + const response = { content: 'answer', toolCalls: [], usage: { input: 10, output: 2 } } + // Awaited caller composition, not RuntimeHooks or a second loop. + await Promise.resolve().then(() => { + order.push('inspect') + }) + return response + }, + execute: async () => { + throw new Error('No tool should execute') + }, + }) + expect(order).toEqual(['prepare', 'compact', 'inference', 'inspect']) + expect(result.final).toBe('answer') + expect(result.usage).toEqual({ input: 10, output: 2 }) + }) +}) diff --git a/tests/kernel/supervise-surface.test.ts b/tests/kernel/supervise-surface.test.ts index 21832d16..dda5bef7 100644 --- a/tests/kernel/supervise-surface.test.ts +++ b/tests/kernel/supervise-surface.test.ts @@ -171,7 +171,10 @@ describe('analystsFromRegistry — the eval registry as a supervise lens', () => list: () => DEFAULT_TRACE_ANALYST_KINDS.map((kind) => ({ id: kind.id })), run: async (runId, inputs, opts) => { calls.push({ runId, only: opts?.only, hasStore: inputs.traceStore !== undefined }) - return { findings: [finding] } as never + return { + findings: [finding], + per_analyst: [{ analyst_id: opts?.only?.[0], status: 'ok' }], + } as never }, } } @@ -198,6 +201,41 @@ describe('analystsFromRegistry — the eval registry as a supervise lens', () => expect(registry.calls[0]?.hasStore).toBe(true) }) + it.each(['failed', 'skipped'] as const)( + 'does not report a %s analyst as a clean review', + async (status) => { + const registry = fakeRegistry() + registry.run = async () => + ({ + findings: [], + per_analyst: [{ analyst_id: 'failure-mode', status }], + }) as never + await expect( + analystsFromRegistry(registry).run('failure-mode', toolSpansToTraceAnalysisStore([span])), + ).rejects.toThrow(`did not complete: ${status}`) + }, + ) + + it('refuses a missing execution receipt even when there are no findings', async () => { + const registry = fakeRegistry() + registry.run = async () => ({ findings: [], per_analyst: [] }) as never + await expect( + analystsFromRegistry(registry).run('failure-mode', toolSpansToTraceAnalysisStore([span])), + ).rejects.toThrow('missing result') + }) + + it('preserves a successful review with no findings', async () => { + const registry = fakeRegistry() + registry.run = async () => + ({ + findings: [], + per_analyst: [{ analyst_id: 'failure-mode', status: 'ok' }], + }) as never + await expect( + analystsFromRegistry(registry).run('failure-mode', toolSpansToTraceAnalysisStore([span])), + ).resolves.toEqual([]) + }) + it('refuses a kind the registry does not have, at adapt time', () => { expect(() => analystsFromRegistry(fakeRegistry(), [ diff --git a/tests/kernel/tool-loop-preparation-stop.test.ts b/tests/kernel/tool-loop-preparation-stop.test.ts new file mode 100644 index 00000000..21461bee --- /dev/null +++ b/tests/kernel/tool-loop-preparation-stop.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it, vi } from 'vitest' +import { runBrainLoop } from '../../src/runtime/tool-loop' + +const initialMessages = [ + { role: 'system', content: 'Keep the task constraints.' }, + { role: 'user', content: 'Complete the task.' }, + { role: 'assistant', content: 'Earlier work.' }, + { role: 'user', content: 'More evidence.' }, +] + +const response = { + content: 'done', + toolCalls: [], + usage: { input: 10, output: 2 }, +} + +describe('tool-loop preparation authority', () => { + it('does not compact or infer after cancellation during a classifier hook', async () => { + const controller = new AbortController() + const chat = vi.fn(async () => response) + const distill = vi.fn(async () => 'digest') + const execute = vi.fn(async () => 'tool output') + const result = await runBrainLoop({ + initialMessages, + chat, + tools: [], + execute, + hooks: { + stopBefore: () => controller.signal.aborted, + beforeTurn: async (_turn, messages) => { + await Promise.resolve() + messages.push({ role: 'user', content: 'Classifier observation retained.' }) + controller.abort() + }, + }, + compaction: { thresholdTokens: 1, distill }, + }) + expect(chat).not.toHaveBeenCalled() + expect(distill).not.toHaveBeenCalled() + expect(execute).not.toHaveBeenCalled() + expect(result.turns).toBe(0) + expect(result.usage).toEqual({ input: 0, output: 0 }) + expect(result.messages.at(-1)?.content).toBe('Classifier observation retained.') + }) + + it('does not infer when compaction consumes the remaining execution budget', async () => { + let budgetAvailable = true + const chat = vi.fn(async () => response) + const onCompact = vi.fn() + const result = await runBrainLoop({ + initialMessages, + chat, + tools: [], + execute: async () => 'unused', + hooks: { stopBefore: () => !budgetAvailable }, + compaction: { + thresholdTokens: 1, + distill: async () => { + await Promise.resolve() + budgetAvailable = false + return 'Completed compaction retained for resume.' + }, + onCompact, + }, + }) + expect(chat).not.toHaveBeenCalled() + expect(onCompact).toHaveBeenCalledOnce() + expect(result.turns).toBe(0) + expect(result.messages.at(-1)?.content).toContain('Completed compaction retained for resume.') + expect(result.messages.slice(0, 2)).toEqual(initialMessages.slice(0, 2)) + }) + + it('retains completed turns, tool results, and metering when a later hook stops the run', async () => { + let stopped = false + const onUsage = vi.fn() + const chat = vi.fn(async () => ({ + content: 'Check the evidence.', + toolCalls: [{ id: 'tool-1', name: 'read', arguments: '{}' }], + usage: { input: 10, output: 2 }, + })) + const execute = vi.fn(async () => 'Observed evidence.') + const result = await runBrainLoop({ + initialMessages: initialMessages.slice(0, 2), + tools: [], + maxTurns: 0, + chat, + execute, + hooks: { + stopBefore: () => stopped, + beforeTurn: async (turn) => { + await Promise.resolve() + if (turn === 2) stopped = true + }, + onUsage, + }, + }) + expect(chat).toHaveBeenCalledOnce() + expect(execute).toHaveBeenCalledOnce() + expect(onUsage).toHaveBeenCalledExactlyOnceWith({ input: 10, output: 2 }) + expect(result.turns).toBe(1) + expect(result.toolCalls).toBe(1) + expect(result.usage).toEqual({ input: 10, output: 2 }) + expect(result.messages.at(-1)).toEqual({ + role: 'tool', + tool_call_id: 'tool-1', + content: 'Observed evidence.', + }) + }) +})