From 481a91a2599cbfc7161d4a98b3ecb4f87e8cedae Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 14 Sep 2026 02:08:09 -0700 Subject: [PATCH] fix(eval): enforce deciding evidence in knowledge promotion --- .github/workflows/ci.yml | 5 -- CHANGELOG.md | 9 +++ README.md | 24 +++++- api-surface.json | 2 +- package.json | 4 +- pnpm-lock.yaml | 10 +-- scripts/verify-official-optimizers.mjs | 2 +- scripts/verify-package.mjs | 8 +- src/memory/improvement/promotion.ts | 53 +++++++++---- src/memory/improvement/run.ts | 15 ++++ src/rag-improvement-loop.ts | 2 +- src/rag-improvement-phases.ts | 8 +- src/release.ts | 4 +- tests/eval-integrity.test.ts | 96 +++++++++++++++++++++++ tests/kb-improvement/optimization.test.ts | 24 +++--- tests/memory/improvement.test.ts | 83 +++++++++++++++++++- tests/rag-improvement-loop.test.ts | 48 +++++++++++- 17 files changed, 344 insertions(+), 53 deletions(-) create mode 100644 tests/eval-integrity.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbed27f..235b553 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,11 +43,6 @@ jobs: - name: Verify packed package lifecycle run: pnpm verify:package - - name: Verify packed package with Eval 0.174 - run: node scripts/verify-package.mjs - env: - AGENT_KNOWLEDGE_EVAL_VERSION: 0.174.0 - official-optimizers: runs-on: ubuntu-latest timeout-minutes: 20 diff --git a/CHANGELOG.md b/CHANGELOG.md index b5342a5..37e29a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog +## 17.0.0 — 2026-09-14 + +Requires Eval `>=0.181.0 <0.182.0` and uses its explicit held-out release decision type. +Memory activation checks the deciding safety interval and refuses insufficient or indeterminate evidence. +RAG promotion also requires Eval's complete final decision; a positive diagnostic bound cannot override an inconclusive result. +Source-unit mappings are captured before execution and bound to resume identity; repetitions retain their measured coverage. +Retrieval, RAG, and KB-policy optimization inherit Eval's optional source-unit claims and durable final-evidence controls. +Long finite run lineages remain readable; callers can set `maxAncestors` when they need a bound. + ## 16.0.1 — 2026-09-12 Run-scoped promotion verifies and freezes source pages, including cited support, before writing the shared store. diff --git a/README.md b/README.md index bd2b350..8f59adc 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Supply application callbacks for those decisions, or use `@tangle-network/agent- ## Install ```bash -pnpm add @tangle-network/agent-knowledge@15.0.3 @tangle-network/agent-eval@0.180.0 @tangle-network/agent-interface@2.6.0 +pnpm add @tangle-network/agent-knowledge@17.0.0 @tangle-network/agent-eval@0.181.0 @tangle-network/agent-interface@2.6.0 ``` Requires Node.js 20.19 or later. @@ -411,6 +411,18 @@ Official external methods must report observed package identity. Custom in-process methods have no external package identity, so their behavior must be covered by `executionRef`. Treat `accountingComplete: false` as incomplete evidence for activation. +Retrieval, RAG, serialized-candidate, and KB-policy optimization accept Eval's optional `claim` and `finalEvidence` options. +Use `claim.independentUnit` to group questions from the same source. +A `new-units` claim requires separate source units for development and final evaluation. +Results retain scenario scores, source-unit scores, and both observation counts. +Claim metadata declares the intended scope; it does not authenticate labels or establish certification. + +For fresh final evidence, pass a shared durable ledger from `openFinalEvidenceLedger()` in `@tangle-network/agent-eval/experiment`. +The policy also requires a request ID and the actual evaluator's content digest. +Eval reserves final units before search and records exposure before measurement. +Interrupted measurements consume that evidence, and another request cannot restore its freshness. +See [Eval's integrity guide](https://github.com/tangle-network/agent-eval/blob/main/docs/evaluation-integrity.md) for the full controls and limits. + Retrieval and answer generation remain callbacks. This lets the same evaluation code work with local search, vector databases, hybrid search, rerankers, and hosted RAG services. Adaptive diagnosis, acquisition, and update callbacks finish before retrieval or RAG final scoring starts. @@ -418,6 +430,10 @@ Only answer evaluation, the terminal promotion decision, and the returned result Answer-quality evidence must name at least two final scenario IDs, immutable dataset and evaluator references, non-empty finite metrics, and observed cost accounting. Promotion also requires `answerQualityCostCeiling`. +`calibrateRagAnswerJudge()` checks supplied strong and weak fixtures; it does not measure an evaluator's error rates. +For evaluator admission, use `auditEvaluator()` from `@tangle-network/agent-eval/meta-eval` with actual judgments of independently verified controls. +The application must enforce evaluator and auditor separation and retain evidence for the labels. + ## Integrate memory systems `@tangle-network/agent-knowledge/memory` defines `AgentMemoryAdapter` and adapters for Mem0, Graphiti, and Neo4j Agent Memory. @@ -429,6 +445,12 @@ Use them to compare a provider against no memory or another provider on the same `runAgentMemoryImprovement` accepts a complete `OptimizationMethod`, evaluates each serialized configuration in an isolated provider branch, and activates only a winner that passes a separate final comparison. Set `implementationRef` to `git:<40 lowercase hex>` or `sha256:<64 lowercase hex>` covering the installed implementation, method configuration, candidate construction, execution behavior, and external configuration so incompatible state cannot resume. The run records one immutable candidate reference for each memory configuration and refuses cached results if that reference changes. +Critical dimensions use Eval's deciding interval and observation minimum. +Missing, insufficient, or indeterminate safety evidence holds activation. +An interval that crosses the safety margin remains uncertain; it is not reported as an observed regression. +Set `significance.independentUnitByScenarioId` when final sequences share a source unit. +The run captures this mapping before execution and binds it to resume identity. +Paired repetitions retain their coverage without increasing the independent-unit count. Each improvement candidate declares a maximum for one sequence and one recovery attempt. The adapter must enforce that maximum with its provider before starting external work. The adapter callback must call `recordExternalCost()` with each observed charge. diff --git a/api-surface.json b/api-surface.json index 19c0622..2c2d33c 100644 --- a/api-surface.json +++ b/api-surface.json @@ -296,7 +296,7 @@ "KnowledgeRelationWalkOptions": "value fe4fbf78ea49", "KnowledgeRelationWalkStep": "value dce4639198b3", "KnowledgeRelease": "value 28df725e17fa", - "KnowledgeReleaseInput": "value 113e4cc7bdc7", + "KnowledgeReleaseInput": "value 844c1ad8d42f", "KnowledgeReleaseReport": "value 5f72778544bd", "KnowledgeResearchLoopContext": "value 1a4d66267646", "KnowledgeResearchLoopDecision": "value 648813512b86", diff --git a/package.json b/package.json index 52abb6e..a036634 100644 --- a/package.json +++ b/package.json @@ -84,14 +84,14 @@ "zod": "4.5.4" }, "peerDependencies": { - "@tangle-network/agent-eval": ">=0.174.0 <0.181.0", + "@tangle-network/agent-eval": ">=0.181.0 <0.182.0", "@tangle-network/agent-interface": "^2.0.0" }, "devDependencies": { "@arethetypeswrong/cli": "^0.18.5", "@biomejs/biome": "^2.5.11", "@neo4j-labs/agent-memory": "0.4.1", - "@tangle-network/agent-eval": "0.180.0", + "@tangle-network/agent-eval": "0.181.0", "@tangle-network/agent-interface": "2.0.0", "@types/node": "^26.4.0", "mem0ai": "3.1.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3014a5e..bb73411 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -35,8 +35,8 @@ importers: specifier: 0.4.1 version: 0.4.1 '@tangle-network/agent-eval': - specifier: 0.180.0 - version: 0.180.0 + specifier: 0.181.0 + version: 0.181.0 '@tangle-network/agent-interface': specifier: 2.0.0 version: 2.0.0 @@ -755,8 +755,8 @@ packages: '@modelcontextprotocol/sdk': optional: true - '@tangle-network/agent-eval@0.180.0': - resolution: {integrity: sha512-sQFExEf/3eaVM0/PbNWsFJ4cSWRgQayx6VB0nR6yy1Av9XNzQ7RuVMllOmJYz3anoJ4xHjiMpbrFlRnu4MM5VQ==} + '@tangle-network/agent-eval@0.181.0': + resolution: {integrity: sha512-TP2Z37oTo/8fDGDwSrAMrjgZ86FA9Vk6wwdpr1+l8cw3wS5T7cPsO2UxjG/9nY2Bq5wrWBROVDKpYo5VlEVCAg==} engines: {node: '>=20.19.0'} hasBin: true @@ -2959,7 +2959,7 @@ snapshots: '@tangle-network/agent-interface': 2.0.0 zod: 4.5.4 - '@tangle-network/agent-eval@0.180.0': + '@tangle-network/agent-eval@0.181.0': dependencies: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.5.4) '@hono/node-server': 2.0.12(hono@4.12.32) diff --git a/scripts/verify-official-optimizers.mjs b/scripts/verify-official-optimizers.mjs index a8bb87e..4a0071c 100644 --- a/scripts/verify-official-optimizers.mjs +++ b/scripts/verify-official-optimizers.mjs @@ -18,7 +18,7 @@ const agentEvalVersion = sourcePackage.devDependencies?.['@tangle-network/agent- if (!/^\d+\.\d+\.\d+$/.test(agentEvalVersion)) { throw new Error('@tangle-network/agent-eval must have one exact development pin') } -const expectedEvalPeerRange = '>=0.174.0 <0.181.0' +const expectedEvalPeerRange = expectedPeerRange(agentEvalVersion) if (sourcePackage.peerDependencies?.['@tangle-network/agent-eval'] !== expectedEvalPeerRange) { throw new Error( `@tangle-network/agent-eval peer range must be ${expectedEvalPeerRange} to match the development pin`, diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index b02c546..b5a1311 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -67,12 +67,12 @@ const agentCorePackage = '@tangle-network/agent-core' const agentInterfacePackage = '@tangle-network/agent-interface' const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '..') const sourcePackage = JSON.parse(readFileSync(join(repoRoot, 'package.json'), 'utf8')) -const agentEvalVersion = - process.env.AGENT_KNOWLEDGE_EVAL_VERSION ?? exactDevelopmentPin(sourcePackage, agentEvalPackage) -if (!['0.174.0', '0.175.0', '0.176.0', '0.177.0', '0.178.0', '0.179.0', '0.180.0'].includes(agentEvalVersion)) { +const agentEvalDevelopmentVersion = exactDevelopmentPin(sourcePackage, agentEvalPackage) +const agentEvalVersion = process.env.AGENT_KNOWLEDGE_EVAL_VERSION ?? agentEvalDevelopmentVersion +if (!caretAdmits(`^${agentEvalDevelopmentVersion}`, agentEvalVersion)) { throw new Error(`unsupported Eval compatibility test version: ${agentEvalVersion}`) } -const agentEvalPeerRange = '>=0.174.0 <0.181.0' +const agentEvalPeerRange = expectedPeerRange(agentEvalDevelopmentVersion) const agentInterfaceVersion = exactDevelopmentPin(sourcePackage, agentInterfacePackage) const agentInterfacePeerRange = expectedPeerRange(agentInterfaceVersion) const zodVersion = exactVersion(sourcePackage.dependencies?.zod, 'zod runtime dependency') diff --git a/src/memory/improvement/promotion.ts b/src/memory/improvement/promotion.ts index 18ad04f..c1a844d 100644 --- a/src/memory/improvement/promotion.ts +++ b/src/memory/improvement/promotion.ts @@ -41,26 +41,30 @@ export function decidePromotion(input: { options.significance, ) const tolerance = options.criticalDimensionTolerance ?? 0.05 - const criticalDimensions = (options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS).map( + const criticalChecks = (options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS).map( (dimension) => { const expectedN = applicableSequenceCount(options.finalSequences, dimension) * (options.reps ?? 1) const pairs = pairedDimension(finalEvaluation, dimension) const comparison = heldoutSignificance(pairs, { ...options.significance, - deltaThreshold: 0, + deltaThreshold: -tolerance, }) + const measured = expectedN > 0 && pairs.before.length === expectedN return { - dimension, - n: comparison.n, - expectedN, - measured: expectedN > 0 && comparison.n === expectedN, - meanDelta: comparison.bootstrap.mean, - low: comparison.bootstrap.low, - high: comparison.bootstrap.high, - tolerance, - regressed: - expectedN > 0 && comparison.n === expectedN && comparison.bootstrap.low < -tolerance, + decision: comparison.decision, + report: { + dimension, + n: comparison.n, + expectedN, + measured, + meanDelta: comparison.bootstrap.mean, + low: comparison.decision.low, + high: comparison.decision.high, + tolerance, + regressed: + measured && !comparison.decision.indeterminate && comparison.decision.high < -tolerance, + }, } }, ) @@ -84,22 +88,32 @@ export function decidePromotion(input: { if (!significance.significant) { reasons.push( significance.fewRuns - ? `only ${significance.n} paired final cells; more are required` + ? `only ${significance.n} paired final observations; at least ${significance.minimumRequired} are required` : 'final lift is not confidently above the promotion threshold', ) } if (winnerScore < (options.minFinalScore ?? 0)) { reasons.push(`winner final score ${winnerScore} is below the required minimum`) } - for (const dimension of criticalDimensions) { + for (const { report: dimension, decision } of criticalChecks) { if (!dimension.measured) { reasons.push( dimension.expectedN === 0 ? `critical dimension ${dimension.dimension} has no applicable final histories` : `critical dimension ${dimension.dimension} was measured on ${dimension.n}/${dimension.expectedN} applicable paired final cells`, ) + } else if (!decision.sufficient) { + reasons.push( + `critical dimension ${dimension.dimension} needs at least ${decision.minimumPairs} paired final observations; measured ${decision.n}`, + ) + } else if (decision.indeterminate) { + reasons.push(`critical dimension ${dimension.dimension} has an indeterminate final interval`) } else if (dimension.regressed) { - reasons.push(`${dimension.dimension} may regress beyond ${tolerance}`) + reasons.push(`critical dimension ${dimension.dimension} regresses beyond ${tolerance}`) + } else if (!decision.promote) { + reasons.push( + `critical dimension ${dimension.dimension} does not exclude a regression beyond ${tolerance}`, + ) } } return { @@ -109,7 +123,7 @@ export function decidePromotion(input: { winnerScore, lift: winnerScore - baselineScore, significance, - criticalDimensions, + criticalDimensions: criticalChecks.map((check) => check.report), } } @@ -124,6 +138,13 @@ export function normalizedPromotionPolicy( resamples: options.significance?.resamples ?? 2000, seed: options.significance?.seed ?? 1337, statistic: options.significance?.statistic ?? 'mean', + ...(options.significance?.independentUnitByScenarioId === undefined + ? {} + : { + independentUnitByScenarioId: [...options.significance.independentUnitByScenarioId].sort( + ([left], [right]) => (left < right ? -1 : left > right ? 1 : 0), + ), + }), }, criticalDimensions: [...(options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS)], criticalDimensionTolerance: options.criticalDimensionTolerance ?? 0.05, diff --git a/src/memory/improvement/run.ts b/src/memory/improvement/run.ts index ff25e24..29f24b9 100644 --- a/src/memory/improvement/run.ts +++ b/src/memory/improvement/run.ts @@ -47,6 +47,21 @@ import { assertMemoryImprovementOptions } from './validation' export async function runAgentMemoryImprovement( options: RunAgentMemoryImprovementOptions, ): Promise> { + if (options.significance !== undefined) { + options = { + ...options, + significance: { + ...options.significance, + ...(options.significance.independentUnitByScenarioId === undefined + ? {} + : { + independentUnitByScenarioId: new Map( + options.significance.independentUnitByScenarioId, + ), + }), + }, + } + } assertMemoryImprovementOptions(options) const storage = options.storage ?? fsCampaignStorage() const runDir = resolveRunDir(options.runDir, options.repo) diff --git a/src/rag-improvement-loop.ts b/src/rag-improvement-loop.ts index 748ce52..01de28a 100644 --- a/src/rag-improvement-loop.ts +++ b/src/rag-improvement-loop.ts @@ -152,7 +152,7 @@ export interface RunRagKnowledgeImprovementLoopOptions { answerQualityCostCeiling?: number /** * Makes a side-effect-free promotion decision after the library has rejected - * missing, regressing, unaccounted, or over-budget final evidence. + * missing, inconclusive, regressing, unaccounted, or over-budget final evidence. */ decidePromotion?: (input: RagPromotionInput) => MaybePromise enabledPhases?: readonly RagKnowledgeImprovementPhase[] diff --git a/src/rag-improvement-phases.ts b/src/rag-improvement-phases.ts index 91ac3c9..75f305d 100644 --- a/src/rag-improvement-phases.ts +++ b/src/rag-improvement-phases.ts @@ -267,8 +267,12 @@ function rejectUnsafePromotionEvidence(evidence: { if (optimizerSource && optimizerSource.evidence !== 'observed') { reasons.push(`${label} optimizer package identity was not observed`) } - if (comparison.best.liftCi.low < 0) { - reasons.push(`${label} final comparison does not rule out a regression`) + if (!comparison.best.decision.promote) { + reasons.push( + comparison.best.decision.low < 0 + ? `${label} final comparison does not rule out a regression` + : `${label} final comparison does not establish the required improvement`, + ) } if ( costCeiling !== undefined && diff --git a/src/release.ts b/src/release.ts index cef84cc..7213a9b 100644 --- a/src/release.ts +++ b/src/release.ts @@ -1,7 +1,7 @@ import { type DatasetScenario, evaluateReleaseConfidence, - type GateDecision, + type HeldOutGateDecision, type ReleaseConfidenceScorecard, type ReleaseTraceEvidence, type RunRecord, @@ -27,7 +27,7 @@ export interface KnowledgeReleaseInput { candidateRuns: RunRecord[] baselineRuns?: RunRecord[] traces?: ReleaseTraceEvidence[] - gateDecision?: GateDecision | null + gateDecision?: HeldOutGateDecision | null /** Scenario corpus used to prove train and holdout split coverage. */ scenarios?: readonly DatasetScenario[] /** diff --git a/tests/eval-integrity.test.ts b/tests/eval-integrity.test.ts new file mode 100644 index 0000000..3623270 --- /dev/null +++ b/tests/eval-integrity.test.ts @@ -0,0 +1,96 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { + defineEvaluationClaim, + openFinalEvidenceLedger, +} from '@tangle-network/agent-eval/experiment' +import { hashCanonical } from '@tangle-network/agent-eval/ledger-core' +import { expect, it } from 'vitest' +import { + type RetrievalEvalArtifact, + type RetrievalEvalScenario, + runRetrievalImprovementLoop, +} from '../src/index' +import { fixedOptimizationMethod, testExecutionRef } from './support/optimization' + +it('forwards source-unit claims and consumes fresh final evidence through retrieval optimization', async () => { + const root = await mkdtemp(join(tmpdir(), 'knowledge-final-evidence-')) + try { + const ledger = openFinalEvidenceLedger({ path: join(root, 'evidence.jsonl') }) + const evaluatorDigest = hashCanonical({ evaluator: 'retrieval-integrity-fixture' }) + const claim = defineEvaluationClaim({ + use: 'comparison', + population: { id: 'fixture-sources', description: 'Offline adapter fixtures' }, + samplingFrame: 'Authored fixtures for execution checks only', + independentUnit: 'source.id', + generalization: 'new-units', + minimumEffect: 0.05, + }) + const fixed = fixedOptimizationMethod('{}') + let searches = 0 + const run = () => + runRetrievalImprovementLoop({ + executionRef: testExecutionRef('retrieval-integrity-fixture'), + baseline: { k: 1 }, + method: { + name: fixed.name, + async optimize(input) { + searches += 1 + const reserved = await ledger.read() + expect(reserved.succeeded).toBe(true) + if (!reserved.succeeded) throw new Error(reserved.error.message) + expect(reserved.value).toHaveLength(1) + expect(reserved.value[0]?.exposure).toBeNull() + expect(reserved.value[0]?.reservation.unitIds).toEqual(['final-1', 'final-2']) + return fixed.optimize(input) + }, + }, + trainScenarios: [scenario('train', 'train')], + selectionScenarios: [scenario('selection', 'selection')], + finalScenarios: [ + scenario('final-1-a', 'final-1'), + scenario('final-1-b', 'final-1'), + scenario('final-2-a', 'final-2'), + scenario('final-2-b', 'final-2'), + ], + retrieve: async () => ({ + hits: [{ pageId: 'answer', path: 'knowledge/answer.md', rank: 1 }], + }), + runDir: '/runs/retrieval-integrity-fixture', + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + claim, + finalEvidence: { ledger, requestId: 'retrieval-fixture', evaluatorDigest }, + resamples: 200, + }) + + const result = await run() + expect(result.comparison.claim).toEqual(claim) + expect(result.comparison).toMatchObject({ + observationUnit: 'registered', + pairedCellN: 4, + units: { observations: 4, independentUnits: 2 }, + }) + expect(result.comparison.best.scenarioScores).toHaveLength(4) + expect(result.comparison.best.unitScores).toHaveLength(2) + expect(result.comparison.finalEvidence?.record.exposure?.measurement.evaluatorDigest).toBe( + evaluatorDigest, + ) + await expect(run().then(() => undefined)).rejects.toMatchObject({ kind: 'conflict' }) + expect(searches).toBe(1) + } finally { + await rm(root, { recursive: true, force: true }) + } +}) + +function scenario(id: string, sourceId: string): RetrievalEvalScenario { + return { + id, + kind: 'retrieval-eval', + query: `question ${id}`, + expected: { kind: 'page', pageId: 'answer' }, + source: { id: sourceId }, + } +} diff --git a/tests/kb-improvement/optimization.test.ts b/tests/kb-improvement/optimization.test.ts index f41a275..f16f882 100644 --- a/tests/kb-improvement/optimization.test.ts +++ b/tests/kb-improvement/optimization.test.ts @@ -155,6 +155,7 @@ describe('optimizeKnowledgeBasePolicy', () => { let finalEvaluatorCalls = 0 const updatedIterations: number[] = [] const finalDispatches: string[] = [] + const finalIds = ['a', 'b', 'c', 'd', 'e', 'f'].map((suffix) => `single-final-${suffix}`) const scenario = (id: string): RagAnswerEvalScenario => ({ id, kind: 'rag-answer-eval', @@ -199,20 +200,24 @@ describe('optimizeKnowledgeBasePolicy', () => { method, trainScenarios: [scenario('single-final-train')], selectionScenarios: [scenario('single-final-selection')], - finalScenarios: [scenario('single-final-a'), scenario('single-final-b')], - async run({ scenario: item }) { - if (item.id.startsWith('single-final-') && !item.id.endsWith('train')) { - if (item.id === 'single-final-a' || item.id === 'single-final-b') { - finalDispatches.push(item.id) - } + finalScenarios: finalIds.map(scenario), + async run({ config, scenario: item }) { + if (finalIds.includes(item.id)) finalDispatches.push(item.id) + return { + query: item.query, + answer: 'answer', + contexts: [], + metadata: { score: config.mode === 'candidate' ? 1 : 0 }, } - return { query: item.query, answer: 'answer', contexts: [] } }, judges: [ { name: 'single-final-quality', dimensions: [{ key: 'quality', description: 'answer quality' }], - score: () => ({ composite: 1, dimensions: { quality: 1 } }), + score: ({ artifact }) => { + const score = Number(artifact.metadata?.score ?? 0) + return { composite: score, dimensions: { quality: score } } + }, }, ], storage: inMemoryCampaignStorage(), @@ -247,7 +252,8 @@ describe('optimizeKnowledgeBasePolicy', () => { expect(promotionCalls).toBe(1) expect(developmentEvaluatorCalls).toBe(2) expect(finalEvaluatorCalls).toBe(1) - expect(new Set(finalDispatches)).toEqual(new Set(['single-final-a', 'single-final-b'])) + expect(new Set(finalDispatches)).toEqual(new Set(finalIds)) + expect(finalDispatches).toHaveLength(finalIds.length * 2) expect(result.state.status).toBe('rejected') expect(result.state.candidates).toHaveLength(2) }) diff --git a/tests/memory/improvement.test.ts b/tests/memory/improvement.test.ts index 2134228..5d0ae4f 100644 --- a/tests/memory/improvement.test.ts +++ b/tests/memory/improvement.test.ts @@ -22,13 +22,54 @@ import { createScopedTestAdapter, runAgentMemoryImprovement } from '../support/m type Config = { visibility: 'private' | 'team' } -const FINAL_SEQUENCE_IDS = ['final-a', 'final-b', 'final-c', 'final-d', 'final-e', 'final-f'] +// Tied binary safety outcomes need enough evidence to clear the 0.05 regression margin. +const FINAL_SEQUENCE_IDS = Array.from({ length: 14 }, (_, batch) => + ['a', 'b', 'c', 'd', 'e', 'f'].map((suffix) => + batch === 0 ? `final-${suffix}` : `final-${suffix}-${batch}`, + ), +).flat() function immutableRef(value: string): string { return `sha256:${createHash('sha256').update(value).digest('hex')}` } describe('agent memory improvement', () => { + it('holds activation when tied safety outcomes cannot exclude the allowed regression', async () => { + let activationCalls = 0 + let activeConfig: Config = { visibility: 'private' } + const result = await runAgentMemoryImprovement( + baseOptions({ + finalSequences: Array.from({ length: 20 }, (_, index) => + improvementSequence(`safety-final-${index}`, 'test', index % 2), + ), + criticalDimensions: ['memory_stale_safe'], + criticalDimensionTolerance: 0.05, + activation: { + ref: immutableRef('safety-policy'), + async readCurrent() { + return activeConfig + }, + async compareAndSet({ config }) { + activationCalls += 1 + activeConfig = config + }, + }, + }), + ) + + expect(result.decision.significance?.significant).toBe(true) + expect(result.decision.status).toBe('hold') + expect(result.decision.criticalDimensions[0]?.low).toBeLessThan(-0.05) + expect(result.decision.criticalDimensions[0]?.high).toBeGreaterThan(0.05) + expect(result.decision.criticalDimensions[0]?.regressed).toBe(false) + expect(result.decision.reasons).toContain( + 'critical dimension memory_stale_safe does not exclude a regression beyond 0.05', + ) + expect(result.activation.status).toBe('not-eligible') + expect(activationCalls).toBe(0) + expect(activeConfig).toEqual({ visibility: 'private' }) + }) + it('runs a complete method, keeps final data private, resumes, and activates once', async () => { const storage = inMemoryCampaignStorage() const methodInputs: string[][] = [] @@ -82,7 +123,7 @@ describe('agent memory improvement', () => { expect(methodInputs).toEqual([['train-a', 'selection-a']]) expect(result.winnerConfig).toEqual({ visibility: 'team' }) expect(result.winnerSurface).toBe('{"visibility":"team"}') - expect(result.finalEvaluation.pairs).toHaveLength(6) + expect(result.finalEvaluation.pairs).toHaveLength(FINAL_SEQUENCE_IDS.length) expect(result.finalEvaluation.pairs.map((pair) => pair.sequenceId)).toEqual(FINAL_SEQUENCE_IDS) expect(result.decision).toMatchObject({ status: 'promote', winnerScore: 1 }) expect(result.decision.baselineScore).toBeCloseTo(61 / 96) @@ -413,6 +454,42 @@ describe('agent memory improvement', () => { expect(result.activation.status).toBe('not-eligible') expect(activationCalls).toBe(0) }) + + it('binds independent source units before resuming a memory improvement', async () => { + const finalSequences = finalImprovementSequences().slice(0, 6) + const independentUnitByScenarioId = new Map( + finalSequences.map(({ id }, index) => [id, `source-${index}`]), + ) + const method = selectingMethod([{ visibility: 'private' }, { visibility: 'team' }]) + const options = baseOptions({ + finalSequences, + reps: 2, + significance: { resamples: 200, seed: 7, independentUnitByScenarioId }, + method: { + name: method.name, + async optimize(input) { + independentUnitByScenarioId.set(finalSequences[0]!.id, 'source-1') + return method.optimize(input) + }, + }, + }) + const result = await runAgentMemoryImprovement(options) + expect(result.decision.significance).toMatchObject({ + n: 6, + pairedCellN: 12, + observationUnit: 'registered', + }) + expect(result.decision.criticalDimensions[0]).toMatchObject({ + n: 6, + expectedN: 12, + measured: true, + }) + + independentUnitByScenarioId.set(FINAL_SEQUENCE_IDS[0]!, 'changed-source') + await expect(runAgentMemoryImprovement(options).then(() => undefined)).rejects.toThrow( + 'does not match its persisted inputs or implementation', + ) + }) }) function baseOptions( @@ -542,6 +619,6 @@ function improvementSequence( function finalImprovementSequences(): AgentMemorySequence[] { const sameAgentProbeCounts = [0, 1, 2, 3, 1, 2] return FINAL_SEQUENCE_IDS.map((id, index) => - improvementSequence(id, 'test', sameAgentProbeCounts[index]), + improvementSequence(id, 'test', sameAgentProbeCounts[index % sameAgentProbeCounts.length]), ) } diff --git a/tests/rag-improvement-loop.test.ts b/tests/rag-improvement-loop.test.ts index d592202..71e7e0a 100644 --- a/tests/rag-improvement-loop.test.ts +++ b/tests/rag-improvement-loop.test.ts @@ -138,7 +138,7 @@ describe('RAG knowledge improvement loop', () => { makeScenario('q-selection-b'), makeScenario('q-selection-c'), ], - finalScenarios: [makeScenario('q-final-a'), makeScenario('q-final-b')], + finalScenarios: Array.from({ length: 6 }, (_, index) => makeScenario(`q-final-${index}`)), method: fixedOptimizationMethod('{"k":2}'), retrieve: async ({ k, scenario }) => { if (scenario.id.startsWith('q-final-')) finalRetrievalCalls += 1 @@ -339,6 +339,52 @@ describe('RAG knowledge improvement loop', () => { expect(decisionCalls).toBe(0) }) + it('refuses a positive final lift when the deciding evidence is indeterminate', async () => { + let decisionCalls = 0 + const scenario = (id: string): RetrievalEvalScenario => ({ + id, + kind: 'retrieval-eval', + query: id, + expected: { kind: 'page', pageId: 'gold' }, + }) + const result = await runRagKnowledgeImprovementLoop({ + goal: 'Reject an indeterminate final comparison', + enabledPhases: ['retrieval-tuning', 'promotion'], + retrieval: { + executionRef: testExecutionRef('rag-indeterminate-lift'), + baseline: { k: 1 }, + trainScenarios: [scenario('train')], + selectionScenarios: [scenario('selection')], + finalScenarios: Array.from({ length: 6 }, (_, index) => scenario(`final-${index}`)), + method: fixedOptimizationMethod('{"k":2}'), + retrieve: async () => ({ hits: [] }), + judges: [ + { + name: 'constant-continuous-gain', + dimensions: [{ key: 'quality', description: 'Fixed continuous fixture score' }], + async score({ artifact }) { + const quality = artifact.config.k === 1 ? 0.25 : 0.75 + return { composite: quality, dimensions: { quality } } + }, + }, + ], + runDir: 'memory://rag-indeterminate-lift', + storage: inMemoryCampaignStorage(), + expectUsage: 'off', + resamples: 200, + }, + decidePromotion() { + decisionCalls += 1 + return { promoted: true, reason: 'caller requested promotion' } + }, + }) + + expect(result.retrieval?.comparison.best.liftCi.low).toBeGreaterThan(0) + expect(result.retrieval?.comparison.best.decision.indeterminate).toBe(true) + expect(result.promotion).toMatchObject({ promoted: false }) + expect(decisionCalls).toBe(0) + }) + it('does not call the promotion decision without final evidence', async () => { let decisionCalls = 0 const result = await runRagKnowledgeImprovementLoop({