From ee65a5ae4928d656b09e00fa891201f17ac77812 Mon Sep 17 00:00:00 2001 From: Brian Love Date: Tue, 1 Sep 2026 19:59:06 -0700 Subject: [PATCH] feat(e2e-harness): bucket metadata-hash mismatches as promptChanged in the drift differ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The aimock recorder stamps fixtures with systemHash/toolsHash fingerprints of the system prompt and tool definitions that produced each recording, but the structural differ ignored them — so a red weekly drift run could not distinguish "our prompt moved underneath the fixture" from model drift. diffFixtures now compares committed vs recorded hashes when BOTH sides carry one and reports mismatches in a new DriftReport.promptChanged category, independent of `changed`. Entries without metadata (pre-stamp fixtures) are never reported. The differ stays advisory: drift.ts exit codes are unchanged; its stderr summary gains a promptChanged count. Co-Authored-By: Claude Fable 5 --- libs/e2e-harness/src/drift-lib.spec.ts | 56 ++++++++++++++++++++++++++ libs/e2e-harness/src/drift-lib.ts | 24 ++++++++++- libs/e2e-harness/src/drift.ts | 2 +- 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/libs/e2e-harness/src/drift-lib.spec.ts b/libs/e2e-harness/src/drift-lib.spec.ts index 94a80332b..58a9f8fa9 100644 --- a/libs/e2e-harness/src/drift-lib.spec.ts +++ b/libs/e2e-harness/src/drift-lib.spec.ts @@ -55,3 +55,59 @@ test('diffFixtures: unpairable entries are listed, not errored', () => { assert.deepEqual(d.unmatchedCommitted, ['only-committed||']); assert.deepEqual(d.unmatchedRecorded, ['only-recorded||']); }); + +const withMeta = (e: FixtureEntry, metadata: Record): FixtureEntry => ({ ...e, metadata }); + +test('diffFixtures: matching metadata hashes report no prompt change', () => { + const meta = { systemHash: 'aaaa1111', toolsHash: 'bbbb2222' }; + const d = diffFixtures([withMeta(text('hi', 'hello'), meta)], [withMeta(text('hi', 'hello world'), meta)]); + assert.equal(d.promptChanged.length, 0); + assert.equal(d.changed.length, 0); +}); + +test('diffFixtures: changed systemHash is bucketed as promptChanged, not changed', () => { + const d = diffFixtures( + [withMeta(text('hi', 'hello'), { systemHash: 'aaaa1111', toolsHash: 'bbbb2222' })], + [withMeta(text('hi', 'hello there'), { systemHash: 'cccc3333', toolsHash: 'bbbb2222' })] + ); + assert.equal(d.promptChanged.length, 1); + assert.match(d.promptChanged[0].reason, /systemHash: aaaa1111 -> cccc3333/); + assert.doesNotMatch(d.promptChanged[0].reason, /toolsHash/); + assert.equal(d.changed.length, 0); +}); + +test('diffFixtures: changed toolsHash only is reported as promptChanged', () => { + const d = diffFixtures( + [withMeta(tool('plan', ['research']), { systemHash: 'aaaa1111', toolsHash: 'bbbb2222' })], + [withMeta(tool('plan', ['research']), { systemHash: 'aaaa1111', toolsHash: 'dddd4444' })] + ); + assert.equal(d.promptChanged.length, 1); + assert.match(d.promptChanged[0].reason, /toolsHash: bbbb2222 -> dddd4444/); + assert.doesNotMatch(d.promptChanged[0].reason, /systemHash/); +}); + +test('diffFixtures: absent metadata on either side is never a prompt change', () => { + // committed has metadata, recorded does not — and vice versa — and neither has any + const d = diffFixtures( + [ + withMeta(text('a', 'x'), { systemHash: 'aaaa1111' }), + text('b', 'x'), + text('c', 'x'), + ], + [ + text('a', 'x'), + withMeta(text('b', 'x'), { systemHash: 'eeee5555' }), + text('c', 'x'), + ] + ); + assert.equal(d.promptChanged.length, 0); +}); + +test('diffFixtures: structural drift and prompt change are reported independently', () => { + const d = diffFixtures( + [withMeta(tool('plan', ['research']), { systemHash: 'aaaa1111' })], + [withMeta(tool('plan', ['book']), { systemHash: 'cccc3333' })] + ); + assert.equal(d.changed.length, 1); + assert.equal(d.promptChanged.length, 1); +}); diff --git a/libs/e2e-harness/src/drift-lib.ts b/libs/e2e-harness/src/drift-lib.ts index 99a55ba0e..67a825d5c 100644 --- a/libs/e2e-harness/src/drift-lib.ts +++ b/libs/e2e-harness/src/drift-lib.ts @@ -6,6 +6,9 @@ export interface FixtureEntry { match: Record; response: Record; + /** Stamped by the aimock recorder: fingerprints of the system prompt and + * tool definitions that produced the recording. Older fixtures lack it. */ + metadata?: { systemHash?: string; toolsHash?: string }; } export interface EntrySummary { @@ -18,10 +21,16 @@ export interface EntrySummary { * recorder emits this ("fixture may be incomplete") when it cannot parse * tool-call deltas out of a stream. A recorder artifact, not drift. */ incomplete: boolean; + systemHash?: string; + toolsHash?: string; } export interface DriftReport { changed: Array<{ key: string; reason: string; committed: EntrySummary; recorded: EntrySummary }>; + /** Pairs whose recorder-stamped systemHash/toolsHash differ: the prompt or + * tool definitions moved underneath the fixture — not model drift. Only + * reported when BOTH sides carry the hash. Independent of `changed`. */ + promptChanged: Array<{ key: string; reason: string; committed: EntrySummary; recorded: EntrySummary }>; unmatchedCommitted: string[]; unmatchedRecorded: string[]; /** Recorded entries the recorder itself marked as possibly incomplete @@ -44,20 +53,23 @@ export function summarizeEntry(e: FixtureEntry): EntrySummary { .sort() : []; const content = e.response?.['content']; - return { + const summary: EntrySummary = { key: entryKey(e), kind: names.length > 0 ? 'toolCalls' : 'text', toolNames: names, lengthBucket: Math.floor(Math.log10(Math.max(1, JSON.stringify(e.response ?? {}).length))), incomplete: names.length === 0 && (content === '' || content === undefined), }; + if (typeof e.metadata?.systemHash === 'string') summary.systemHash = e.metadata.systemHash; + if (typeof e.metadata?.toolsHash === 'string') summary.toolsHash = e.metadata.toolsHash; + return summary; } export function diffFixtures(committed: FixtureEntry[], recorded: FixtureEntry[]): DriftReport { const byKey = (list: FixtureEntry[]) => new Map(list.map((e) => [entryKey(e), summarizeEntry(e)])); const c = byKey(committed); const r = byKey(recorded); - const report: DriftReport = { changed: [], unmatchedCommitted: [], unmatchedRecorded: [], incompleteRecordings: [] }; + const report: DriftReport = { changed: [], promptChanged: [], unmatchedCommitted: [], unmatchedRecorded: [], incompleteRecordings: [] }; for (const [key, cs] of c) { const rs = r.get(key); if (!rs) { report.unmatchedCommitted.push(key); continue; } @@ -67,6 +79,14 @@ export function diffFixtures(committed: FixtureEntry[], recorded: FixtureEntry[] if (cs.toolNames.join(',') !== rs.toolNames.join(',')) reasons.push(`toolNames: [${cs.toolNames}] -> [${rs.toolNames}]`); if (cs.lengthBucket !== rs.lengthBucket) reasons.push(`lengthBucket: ${cs.lengthBucket} -> ${rs.lengthBucket}`); if (reasons.length) report.changed.push({ key, reason: reasons.join('; '), committed: cs, recorded: rs }); + // Hash mismatch means our prompt/tools moved, not the model. Absent hashes + // (pre-metadata fixtures) prove nothing, so only compare when both exist. + const promptReasons: string[] = []; + if (cs.systemHash && rs.systemHash && cs.systemHash !== rs.systemHash) + promptReasons.push(`systemHash: ${cs.systemHash} -> ${rs.systemHash}`); + if (cs.toolsHash && rs.toolsHash && cs.toolsHash !== rs.toolsHash) + promptReasons.push(`toolsHash: ${cs.toolsHash} -> ${rs.toolsHash}`); + if (promptReasons.length) report.promptChanged.push({ key, reason: promptReasons.join('; '), committed: cs, recorded: rs }); } for (const key of r.keys()) if (!c.has(key)) report.unmatchedRecorded.push(key); return report; diff --git a/libs/e2e-harness/src/drift.ts b/libs/e2e-harness/src/drift.ts index 85f86118b..ccc83caa3 100644 --- a/libs/e2e-harness/src/drift.ts +++ b/libs/e2e-harness/src/drift.ts @@ -27,5 +27,5 @@ const FIXTURES_DIR = resolve(fixturesDir); const report = diffFixtures(loadDir(FIXTURES_DIR), loadDir(resolve(recordedDir))); console.log(JSON.stringify(report, null, 2)); console.error( - `[drift] changed=${report.changed.length} incompleteRecordings=${report.incompleteRecordings.length} unmatchedCommitted=${report.unmatchedCommitted.length} unmatchedRecorded=${report.unmatchedRecorded.length}` + `[drift] changed=${report.changed.length} promptChanged=${report.promptChanged.length} incompleteRecordings=${report.incompleteRecordings.length} unmatchedCommitted=${report.unmatchedCommitted.length} unmatchedRecorded=${report.unmatchedRecorded.length}` );