From 228ef13e504bcb2d57f3996239efb78ddf930d05 Mon Sep 17 00:00:00 2001
From: Guillaume Flambard <g.flambard@gmail.com>
Date: Fri, 16 Oct 2026 14:43:47 +0200
Subject: [PATCH] fix(p0): reject modified adjudication source snapshots

---
 packages/agents/holdout-adjudication.mjs | 20 ++++++++++++++----
 tests/holdout-adjudication.test.mjs      | 26 ++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 4 deletions(-)

diff --git a/packages/agents/holdout-adjudication.mjs b/packages/agents/holdout-adjudication.mjs
index 9282fee..93f5e65 100644
--- a/packages/agents/holdout-adjudication.mjs
+++ b/packages/agents/holdout-adjudication.mjs
@@ -1,5 +1,6 @@
 import {createHash} from 'node:crypto';
 import {readFile,writeFile} from 'node:fs/promises';
+import {isDeepStrictEqual} from 'node:util';
 import {parseReviewerAnnotations,evaluateIndependentReviews} from './holdout-review.mjs';
 const hash=x=>createHash('sha256').update(JSON.stringify(x)).digest('hex');
 const LABELS=new Set(['same_problem','different_problem','uncertain']);
@@ -7,7 +8,8 @@ export function prepareAdjudication(blind,reviewA,reviewB){
  const a=parseReviewerAnnotations(blind,reviewA),b=parseReviewerAnnotations(blind,reviewB);
  if(a.reviewer===b.reviewer)throw Error('Distinct reviewer pseudonyms are required');
  const disputed=blind.pairs.filter(p=>a.labels.get(p.pairId)!==b.labels.get(p.pairId));
- const entries=disputed.map(p=>({pairId:p.pairId,left:p.left,right:p.right,label:null,reason:null}));
+ // Return detached copies so editing a review queue cannot mutate the original blind evidence.
+ const entries=disputed.map(p=>({pairId:p.pairId,left:structuredClone(p.left),right:structuredClone(p.right),label:null,reason:null}));
  return {kind:'human_adjudication_queue',blindDigest:hash(blind),reviewDigests:[hash(reviewA),hash(reviewB)],
   reviewerPseudonyms:[a.reviewer,b.reviewer],items:entries,
   note:'Disagreements or unanswered labels require an external human adjudicator; no labels are filled automatically.'};
@@ -17,10 +19,20 @@ export function finalizeAdjudication(blind,reviewA,reviewB,queue,{predictions=[]
  if(queue?.kind!=='human_adjudication_queue'||queue.blindDigest!==expected.blindDigest||
   JSON.stringify(queue.reviewDigests)!==JSON.stringify(expected.reviewDigests)||
   !Array.isArray(queue.items)||queue.items.length!==expected.items.length)throw Error('Adjudication does not match frozen source and reviewer submissions');
- const expectedIds=new Set(expected.items.map(x=>x.pairId)),resolved=new Map();
+ // The pair ID and outer file digests are not enough: a filled adjudication queue
+ // could keep the same ID while presenting a different excerpt to the adjudicator.
+ // Compare the whole source snapshots against the authoritative frozen blind queue.
+ const frozenById=new Map(expected.items.map(item=>[item.pairId,item]));
+ const expectedFields=['label','left','pairId','reason','right'];
+ const resolved=new Map();
  for(const item of queue.items){
-  if(!expectedIds.has(item?.pairId)||resolved.has(item.pairId)||!LABELS.has(item.label)||
-   typeof item.reason!=='string'||!item.reason.trim())throw Error('Human adjudication incomplete or invalid');
+  const frozen=frozenById.get(item?.pairId);
+  if(!frozen||resolved.has(item.pairId)||
+   !isDeepStrictEqual(Object.keys(item).sort(),expectedFields)||
+   !isDeepStrictEqual(item.left,frozen.left)||!isDeepStrictEqual(item.right,frozen.right))
+   throw Error('Adjudication source snapshot differs from frozen blind queue');
+  if(!LABELS.has(item.label)||typeof item.reason!=='string'||!item.reason.trim())
+   throw Error('Human adjudication incomplete or invalid');
   resolved.set(item.pairId,item.label);
  }
  const a=parseReviewerAnnotations(blind,reviewA),b=parseReviewerAnnotations(blind,reviewB);
diff --git a/tests/holdout-adjudication.test.mjs b/tests/holdout-adjudication.test.mjs
index d1b8b98..4159a8b 100644
--- a/tests/holdout-adjudication.test.mjs
+++ b/tests/holdout-adjudication.test.mjs
@@ -40,3 +40,29 @@ test('rejects tampered frozen inputs and prevents omitted or fabricated adjudica
 test('reviewer pseudonym alone cannot qualify as independent human evidence',()=>{
  assert.throws(()=>prepareAdjudication(blind,A,{...B,reviewer:'reviewer-a'}),/Distinct reviewer/);
 });
+
+test('adjudication queue snapshots are detached from frozen blind evidence',()=>{
+ const before=structuredClone(blind.pairs[0].left);
+ const queue=prepareAdjudication(blind,A,B);
+ queue.items[0].left.body='A misleading replacement excerpt';
+ assert.deepEqual(blind.pairs[0].left,before);
+ assert.notDeepEqual(queue.items[0].left,before);
+});
+
+test('finalization rejects changes to any frozen source snapshot or queue structure',()=>{
+ const mutations=[
+  q=>{q.items[0].left.body='Different observed failure';},
+  q=>{q.items[0].right.url='https://example.com/other';},
+  q=>{delete q.items[0].right.title;},
+  q=>{q.items[0].left.extra='fabricated';},
+  q=>{q.items[0].unexpected='unfrozen';},
+  q=>{q.items[0].left=structuredClone(q.items[1].left);}
+ ];
+ for(const mutate of mutations){
+  const queue=prepareAdjudication(blind,A,B);
+  queue.items[0].label='same_problem';queue.items[0].reason='Same failure';
+  queue.items[1].label='uncertain';queue.items[1].reason='Insufficient detail';
+  mutate(queue);
+  assert.throws(()=>finalizeAdjudication(blind,A,B,queue),/source snapshot differs/);
+ }
+});