diff --git a/libs/e2e-harness/src/drift-lib.spec.ts b/libs/e2e-harness/src/drift-lib.spec.ts index 94a80332b..58a9f8fa9 100644 --- a/libs/e2e-harness/src/drift-lib.spec.ts +++ b/libs/e2e-harness/src/drift-lib.spec.ts @@ -55,3 +55,59 @@ test('diffFixtures: unpairable entries are listed, not errored', () => { assert.deepEqual(d.unmatchedCommitted, ['only-committed||']); assert.deepEqual(d.unmatchedRecorded, ['only-recorded||']); }); + +const withMeta = (e: FixtureEntry, metadata: Record): FixtureEntry => ({ ...e, metadata }); + +test('diffFixtures: matching metadata hashes report no prompt change', () => { + const meta = { systemHash: 'aaaa1111', toolsHash: 'bbbb2222' }; + const d = diffFixtures([withMeta(text('hi', 'hello'), meta)], [withMeta(text('hi', 'hello world'), meta)]); + assert.equal(d.promptChanged.length, 0); + assert.equal(d.changed.length, 0); +}); + +test('diffFixtures: changed systemHash is bucketed as promptChanged, not changed', () => { + const d = diffFixtures( + [withMeta(text('hi', 'hello'), { systemHash: 'aaaa1111', toolsHash: 'bbbb2222' })], + [withMeta(text('hi', 'hello there'), { systemHash: 'cccc3333', toolsHash: 'bbbb2222' })] + ); + assert.equal(d.promptChanged.length, 1); + assert.match(d.promptChanged[0].reason, /systemHash: aaaa1111 -> cccc3333/); + assert.doesNotMatch(d.promptChanged[0].reason, /toolsHash/); + assert.equal(d.changed.length, 0); +}); + +test('diffFixtures: changed toolsHash only is reported as promptChanged', () => { + const d = diffFixtures( + [withMeta(tool('plan', ['research']), { systemHash: 'aaaa1111', toolsHash: 'bbbb2222' })], + [withMeta(tool('plan', ['research']), { systemHash: 'aaaa1111', toolsHash: 'dddd4444' })] + ); + assert.equal(d.promptChanged.length, 1); + assert.match(d.promptChanged[0].reason, /toolsHash: bbbb2222 -> dddd4444/); + assert.doesNotMatch(d.promptChanged[0].reason, /systemHash/); +}); + +test('diffFixtures: absent metadata on either side is never a prompt change', () => { + // committed has metadata, recorded does not — and vice versa — and neither has any + const d = diffFixtures( + [ + withMeta(text('a', 'x'), { systemHash: 'aaaa1111' }), + text('b', 'x'), + text('c', 'x'), + ], + [ + text('a', 'x'), + withMeta(text('b', 'x'), { systemHash: 'eeee5555' }), + text('c', 'x'), + ] + ); + assert.equal(d.promptChanged.length, 0); +}); + +test('diffFixtures: structural drift and prompt change are reported independently', () => { + const d = diffFixtures( + [withMeta(tool('plan', ['research']), { systemHash: 'aaaa1111' })], + [withMeta(tool('plan', ['book']), { systemHash: 'cccc3333' })] + ); + assert.equal(d.changed.length, 1); + assert.equal(d.promptChanged.length, 1); +}); diff --git a/libs/e2e-harness/src/drift-lib.ts b/libs/e2e-harness/src/drift-lib.ts index 99a55ba0e..67a825d5c 100644 --- a/libs/e2e-harness/src/drift-lib.ts +++ b/libs/e2e-harness/src/drift-lib.ts @@ -6,6 +6,9 @@ export interface FixtureEntry { match: Record; response: Record; + /** Stamped by the aimock recorder: fingerprints of the system prompt and + * tool definitions that produced the recording. Older fixtures lack it. */ + metadata?: { systemHash?: string; toolsHash?: string }; } export interface EntrySummary { @@ -18,10 +21,16 @@ export interface EntrySummary { * recorder emits this ("fixture may be incomplete") when it cannot parse * tool-call deltas out of a stream. A recorder artifact, not drift. */ incomplete: boolean; + systemHash?: string; + toolsHash?: string; } export interface DriftReport { changed: Array<{ key: string; reason: string; committed: EntrySummary; recorded: EntrySummary }>; + /** Pairs whose recorder-stamped systemHash/toolsHash differ: the prompt or + * tool definitions moved underneath the fixture — not model drift. Only + * reported when BOTH sides carry the hash. Independent of `changed`. */ + promptChanged: Array<{ key: string; reason: string; committed: EntrySummary; recorded: EntrySummary }>; unmatchedCommitted: string[]; unmatchedRecorded: string[]; /** Recorded entries the recorder itself marked as possibly incomplete @@ -44,20 +53,23 @@ export function summarizeEntry(e: FixtureEntry): EntrySummary { .sort() : []; const content = e.response?.['content']; - return { + const summary: EntrySummary = { key: entryKey(e), kind: names.length > 0 ? 'toolCalls' : 'text', toolNames: names, lengthBucket: Math.floor(Math.log10(Math.max(1, JSON.stringify(e.response ?? {}).length))), incomplete: names.length === 0 && (content === '' || content === undefined), }; + if (typeof e.metadata?.systemHash === 'string') summary.systemHash = e.metadata.systemHash; + if (typeof e.metadata?.toolsHash === 'string') summary.toolsHash = e.metadata.toolsHash; + return summary; } export function diffFixtures(committed: FixtureEntry[], recorded: FixtureEntry[]): DriftReport { const byKey = (list: FixtureEntry[]) => new Map(list.map((e) => [entryKey(e), summarizeEntry(e)])); const c = byKey(committed); const r = byKey(recorded); - const report: DriftReport = { changed: [], unmatchedCommitted: [], unmatchedRecorded: [], incompleteRecordings: [] }; + const report: DriftReport = { changed: [], promptChanged: [], unmatchedCommitted: [], unmatchedRecorded: [], incompleteRecordings: [] }; for (const [key, cs] of c) { const rs = r.get(key); if (!rs) { report.unmatchedCommitted.push(key); continue; } @@ -67,6 +79,14 @@ export function diffFixtures(committed: FixtureEntry[], recorded: FixtureEntry[] if (cs.toolNames.join(',') !== rs.toolNames.join(',')) reasons.push(`toolNames: [${cs.toolNames}] -> [${rs.toolNames}]`); if (cs.lengthBucket !== rs.lengthBucket) reasons.push(`lengthBucket: ${cs.lengthBucket} -> ${rs.lengthBucket}`); if (reasons.length) report.changed.push({ key, reason: reasons.join('; '), committed: cs, recorded: rs }); + // Hash mismatch means our prompt/tools moved, not the model. Absent hashes + // (pre-metadata fixtures) prove nothing, so only compare when both exist. + const promptReasons: string[] = []; + if (cs.systemHash && rs.systemHash && cs.systemHash !== rs.systemHash) + promptReasons.push(`systemHash: ${cs.systemHash} -> ${rs.systemHash}`); + if (cs.toolsHash && rs.toolsHash && cs.toolsHash !== rs.toolsHash) + promptReasons.push(`toolsHash: ${cs.toolsHash} -> ${rs.toolsHash}`); + if (promptReasons.length) report.promptChanged.push({ key, reason: promptReasons.join('; '), committed: cs, recorded: rs }); } for (const key of r.keys()) if (!c.has(key)) report.unmatchedRecorded.push(key); return report; diff --git a/libs/e2e-harness/src/drift.ts b/libs/e2e-harness/src/drift.ts index 85f86118b..ccc83caa3 100644 --- a/libs/e2e-harness/src/drift.ts +++ b/libs/e2e-harness/src/drift.ts @@ -27,5 +27,5 @@ const FIXTURES_DIR = resolve(fixturesDir); const report = diffFixtures(loadDir(FIXTURES_DIR), loadDir(resolve(recordedDir))); console.log(JSON.stringify(report, null, 2)); console.error( - `[drift] changed=${report.changed.length} incompleteRecordings=${report.incompleteRecordings.length} unmatchedCommitted=${report.unmatchedCommitted.length} unmatchedRecorded=${report.unmatchedRecorded.length}` + `[drift] changed=${report.changed.length} promptChanged=${report.promptChanged.length} incompleteRecordings=${report.incompleteRecordings.length} unmatchedCommitted=${report.unmatchedCommitted.length} unmatchedRecorded=${report.unmatchedRecorded.length}` );