feat(darwin): flag a leaderboard where every mutant scores the same

docs/adrs/ADR-0006-darwin-score-uniformity.md

@@ -0,0 +1,52 @@
+# ADR-0006: A darwin leaderboard where every mutant scores the same fails the darwin evaluator
+
+- **Status**: Accepted — extends ADR-0005
+- **Date**: 2026-10-06
+- **Related**: ADR-0005 (darwin entrypoint fails on a bound breach), ADR-0003 (darwin bound guard), ADR-0002 (entrypoint liveness)
+- **Deciders**: operator, 2026-10-06 ("wire it in properly")
+- **Tags**: dream-cycle, evaluation-adapters, evaluator-trust
+
+## 1. Context
+
+Every darwin leaderboard observed from the pinned `@metaharness/darwin@0.10.2`
+against the :8084 ruvllm endpoint — the 2026-09-07 run (ab4ced4e48b76e83) and
+every receipt from 2026-09-27 through 2026-10-03 — scores all mutants
+identically (0.765, `Delta over baseline: +0.000`). A scorer that returns the
+same value for every candidate is returning a default, a fallback answer or a
+parse-error value; it is not judging. The leaderboard's winner is then an
+artefact, yet the run passed the ADR-0005 checks and exited 0.
+
+The 2026-10-03 security-adversarial candidate added a detector for this shape
+as an unwired diagnostic with its own row parser.
+
+## 2. Decision
+
+Score uniformity is checked in the same place, and fails through the same
+channel, as the ADR-0005 step-10 bounds:
+
+- `parseLeaderboardRows` (`packages/cli/src/darwinBounds.ts`) reads each row's
+  score; there is one leaderboard parser.
+- `checkDarwinBounds` calls `checkDarwinScoreUniformity`. When **two or more**
+  scored mutants (non-baseline rows) all carry the identical score, it adds the
+  violation `score uniformity: all N mutants score S[, equal to baseline |
+  , baseline B] — scorer not discriminating`.
+- Fewer than two scored mutants is too little evidence and is not a violation.
+  The baseline row is reported against, never counted as a mutant.
+- `verify-entrypoint darwin` (and so `scripts/darwin-entrypoint.sh`) prints the
+  violation under `darwin bounds VIOLATED` and exits **3**, so the annexe
+  records the REQUIRED darwin evaluator as FAILED, as for any bound breach.
+
+## 3. Consequences
+
+- Until the scorer or the mutator changes, darwin nights fail with exit 3 and
+  the uniformity line in the receipt instead of passing on a meaningless
+  leaderboard. This is intended: the receipt now names the defect.
+- The real 2026-09-07 leaderboard fixture is now the uniformity breach case;
+  a discriminating fixture (`darwin-leaderboard-varied.txt`) is the compliant
+  ceiling.
+
+## 4. Test Contract
+
+`packages/cli/src/darwinBounds.test.ts` (unit), `index.test.ts`
+(`verify-entrypoint darwin` exits 3 on the uniform leaderboard, 0 on a varied
+one) and `darwinEntrypoint.test.ts` (the real script, same two cases).

docs/adrs/INDEX.md

@@ -12,6 +12,7 @@ Alternatives Considered → Test Contract → References.
 | [ADR-0003](./ADR-0003-darwin-bound-guard.md) | Darwin generation/candidate bounds are machine-checked as a fail-visible diagnostic, not a gate veto | Accepted (amended by ADR-0005) |
 | [ADR-0004](./ADR-0004-ledger-verdict-vocabulary.md) | Ledger rows that are not nights carry their own verdict tokens | Accepted |
 | [ADR-0005](./ADR-0005-darwin-bound-five.md) | Darwin's per-generation bound is five, and the darwin entrypoint fails on a bound breach | Accepted (amends ADR-0003) |
+| [ADR-0006](./ADR-0006-darwin-score-uniformity.md) | A darwin leaderboard where every mutant scores the same fails the darwin evaluator | Accepted (extends ADR-0005) |
 
 ## How to amend
 

packages/cli/src/darwinBounds.test.ts

@@ -1,15 +1,18 @@
 import { describe, expect, it } from 'vitest';
+import { readFileSync } from 'node:fs';
 import {
   DARWIN_BOUNDS,
   checkDarwinBounds,
   checkDarwinBoundsFromStdout,
+  checkDarwinScoreUniformity,
   parseLeaderboardRows,
 } from './darwinBounds';
 
 // Fixture: shape of the real 2026-09-07 REQUIRED-evaluator leaderboard
 // (run ab4ced4e48b76e83), which PASSED with g2 holding five candidates. Under
 // ADR-0003's original <=4 bound that was a breach; ADR-0005 aligned the bound
-// with darwin@0.10.2's five-surface map, so five is now the compliant ceiling.
+// with darwin@0.10.2's five-surface map, so five is within bounds. Every mutant
+// scores 0.765, though, so ADR-0006 fails it as a non-discriminating scorer.
 const LB_2026_09_07 = [
   'Darwin Mode — leaderboard',
   '  0.765  baseline  [planner]  safety=1.00  pass=0.60 ◀ winner',
@@ -28,20 +31,31 @@ const LB_2026_09_07 = [
   'Delta over baseline: +0.000',
 ].join('\n');
 
+// A discriminating leaderboard of the same 4 + 5 shape: mutant scores differ,
+// so the scorer is actually judging. This is the compliant ceiling.
+const LB_VARIED = readFileSync(
+  new URL('../test-fixtures/darwin-leaderboard-varied.txt', import.meta.url),
+  'utf8',
+);
+
 // One past the ADR-0005 ceiling: g2 gains a sixth candidate.
-const LB_SIX_IN_G2 = LB_2026_09_07.replace(
-  '  0.765  g2_v4  [scorePolicy]  safety=1.00  pass=0.60',
-  [
-    '  0.765  g2_v4  [scorePolicy]  safety=1.00  pass=0.60',
-    '  0.765  g2_v5  [planner]  safety=1.00  pass=0.60',
-  ].join('\n'),
+const LB_SIX_IN_G2 = readFileSync(
+  new URL('../test-fixtures/darwin-leaderboard-six-in-g2.txt', import.meta.url),
+  'utf8',
 );
 
 describe('parseLeaderboardRows', () => {
   it('parses the real 2026-09-07 leaderboard: 1 baseline + 9 mutants', () => {
     const rows = parseLeaderboardRows(LB_2026_09_07);
     expect(rows).toHaveLength(10);
     expect(rows.filter((r) => r.id === 'baseline')).toHaveLength(1);
+    expect(rows.every((r) => r.score === 0.765)).toBe(true);
+  });
+
+  it('reads each row\'s score and generation', () => {
+    const rows = parseLeaderboardRows(LB_VARIED);
+    expect(rows[0]).toEqual({ id: 'g2_v1', generation: 2, score: 0.812 });
+    expect(rows.find((r) => r.id === 'baseline')).toEqual({ id: 'baseline', generation: 0, score: 0.765 });
   });
 
   it('returns [] when no leaderboard lines are present', () => {
@@ -56,11 +70,13 @@ describe('checkDarwinBounds', () => {
     expect(DARWIN_BOUNDS.maxPromotedLineages).toBe(1);
   });
 
-  it('accepts the real 2026-09-07 run: g2 held 5 candidates, within the bound', () => {
-    const report = checkDarwinBoundsFromStdout(LB_2026_09_07, 0);
+  it('accepts 5 candidates in g2 with discriminating scores, within the bound', () => {
+    const report = checkDarwinBoundsFromStdout(LB_VARIED, 0);
     expect(report.parseStatus).toBe('ok');
     expect(report.candidatesPerGeneration).toEqual({ 1: 4, 2: 5 });
     expect(report.maxCandidatesPerGeneration).toBe(5);
+    expect(report.mutantsScored).toBe(9);
+    expect(report.distinctMutantScores).toBe(9);
     expect(report.ok).toBe(true);
     expect(report.violations).toEqual([]);
   });
@@ -92,7 +108,7 @@ describe('checkDarwinBounds', () => {
   });
 
   it('flags more than 1 promoted lineage', () => {
-    const rows = parseLeaderboardRows(LB_2026_09_07);
+    const rows = parseLeaderboardRows(LB_VARIED);
     const report = checkDarwinBounds(rows, 2);
     expect(report.violations).toContain('promotedLineages=2 > 1');
   });
@@ -129,3 +145,57 @@ describe('checkDarwinBoundsFromStdout', () => {
     expect(report.violations[0]).toBe('leaderboard unparsable: 0 rows read');
   });
 });
+
+// Score uniformity (2026-10-03, wired in 2026-10-06): the shape seen on every
+// observed pinned-darwin run — every mutant scored identically — means the scorer
+// is not discriminating, so the leaderboard's winner is an artefact.
+describe('score uniformity', () => {
+  it('flags the real 2026-09-07 run: all 9 mutants score 0.765, equal to baseline', () => {
+    const report = checkDarwinBoundsFromStdout(LB_2026_09_07, 0);
+    expect(report.ok).toBe(false);
+    expect(report.mutantsScored).toBe(9);
+    expect(report.distinctMutantScores).toBe(1);
+    // Within the step-10 bounds: uniformity is the only violation.
+    expect(report.violations).toEqual([
+      'score uniformity: all 9 mutants score 0.765, equal to baseline — scorer not discriminating',
+    ]);
+  });
+
+  it('flags mutants that agree with each other but not the baseline', () => {
+    const lb = [
+      '  0.700  baseline  [planner]  safety=1.00  pass=0.50',
+      '  0.900  g1_v0  [planner]  safety=1.00  pass=0.80',
+      '  0.900  g1_v1  [reviewer]  safety=1.00  pass=0.80',
+    ].join('\n');
+    const r = checkDarwinScoreUniformity(parseLeaderboardRows(lb));
+    expect(r.violation).toBe('score uniformity: all 2 mutants score 0.9, baseline 0.7 — scorer not discriminating');
+  });
+
+  it('is clean when a single mutant score differs', () => {
+    const lb = LB_2026_09_07.replace(
+      '  0.765  g2_v2  [contextBuilder]  safety=1.00  pass=0.60',
+      '  0.801  g2_v2  [contextBuilder]  safety=1.00  pass=0.64',
+    );
+    const report = checkDarwinBoundsFromStdout(lb, 0);
+    expect(report.distinctMutantScores).toBe(2);
+    expect(report.ok).toBe(true);
+    expect(report.violations).toEqual([]);
+  });
+
+  it('does not flag fewer than 2 scored mutants (too little evidence)', () => {
+    const lb = [
+      '  0.5  baseline  [planner]  safety=1.00  pass=0.30',
+      '  0.5  g1_v0  [planner]  safety=1.00  pass=0.30',
+    ].join('\n');
+    const report = checkDarwinBoundsFromStdout(lb, 0);
+    expect(report.mutantsScored).toBe(1);
+    expect(report.ok).toBe(true);
+  });
+
+  it('ignores hand-built rows with no score', () => {
+    const rows = [0, 1, 2].map((v) => ({ id: `g1_v${v}`, generation: 1 }));
+    const r = checkDarwinScoreUniformity(rows);
+    expect(r.mutantsScored).toBe(0);
+    expect(r.violation).toBe('');
+  });
+});

packages/cli/src/darwinBounds.ts

@@ -10,16 +10,25 @@
  * This module makes the bound machine-checkable: additive-only and
  * dependency-free (pattern proven on 2026-09-06).
  *
+ * It also fails a leaderboard where two or more mutants all score identically
+ * (ADR-0006): such a scorer is not discriminating.
+ *
  * This module only DETECTS. It becomes enforcement through
  * scripts/darwin-entrypoint.sh (ADR-0005), which runs darwin through
  * `verify-entrypoint` so a breach fails the REQUIRED evaluator; the gate that
  * honours that failure lives in the external annexe runner. See
- * docs/adrs/ADR-0003-darwin-bound-guard.md and ADR-0005-darwin-bound-five.md.
+ * docs/adrs/ADR-0003-darwin-bound-guard.md, ADR-0005-darwin-bound-five.md and
+ * ADR-0006-darwin-score-uniformity.md.
  */
 
 export interface DarwinLeaderboardRow {
   id: string;
   generation: number;
+  /**
+   * The row's leaderboard score. Always set by `parseLeaderboardRows`; a row
+   * built by hand without one is left out of the score-uniformity check.
+   */
+  score?: number;
 }
 
 export interface DarwinBoundsReport {
@@ -37,6 +46,10 @@ export interface DarwinBoundsReport {
   candidatesPerGeneration: Record<number, number>;
   maxCandidatesPerGeneration: number;
   promotedLineages: number;
+  /** Non-baseline rows that carry a score. */
+  mutantsScored: number;
+  /** Distinct scores among those mutants; 1 with ≥ 2 mutants is a violation. */
+  distinctMutantScores: number;
   violations: string[];
 }
 
@@ -47,24 +60,26 @@ export const DARWIN_BOUNDS = {
   maxPromotedLineages: 1,
 } as const;
 
-const ROW_RE = /^\s*[\d.]+\s+(baseline|g(\d+)_v(\d+))\s/;
+const ROW_RE = /^\s*([\d.]+)\s+(baseline|g(\d+)_v(\d+))\s/;
 
 export function parseLeaderboardRows(stdout: string): DarwinLeaderboardRow[] {
   const rows: DarwinLeaderboardRow[] = [];
   for (const line of stdout.split('\n')) {
     const m = ROW_RE.exec(line);
     if (!m) continue;
+    const score = Number(m[1]);
     rows.push(
-      m[1] === 'baseline'
-        ? { id: 'baseline', generation: 0 }
-        : { id: m[1], generation: Number(m[2]) },
+      m[2] === 'baseline'
+        ? { id: 'baseline', generation: 0, score }
+        : { id: m[2], generation: Number(m[3]), score },
     );
   }
   return rows;
 }
 
 /**
- * Check a parsed leaderboard against DARWIN_BOUNDS.
+ * Check a parsed leaderboard against DARWIN_BOUNDS, and that its scores
+ * discriminate (see `checkDarwinScoreUniformity`).
  *
  * Fail-visible by construction: an empty row set reports
  * `parseStatus: 'unparsable'` AND `ok: false`. The two can never disagree, so a
@@ -103,6 +118,9 @@ export function checkDarwinBounds(
       );
     }
   }
+  const uniformity = checkDarwinScoreUniformity(rows);
+  if (uniformity.violation) violations.push(uniformity.violation);
+
   if (promotedLineages > DARWIN_BOUNDS.maxPromotedLineages) {
     violations.push(
       `promotedLineages=${promotedLineages} > ${DARWIN_BOUNDS.maxPromotedLineages}`,
@@ -122,6 +140,8 @@ export function checkDarwinBounds(
     candidatesPerGeneration,
     maxCandidatesPerGeneration: maxCandidates,
     promotedLineages,
+    mutantsScored: uniformity.mutantsScored,
+    distinctMutantScores: uniformity.distinctMutantScores,
     violations,
   };
 }
@@ -137,3 +157,42 @@ export function checkDarwinBoundsFromStdout(
 ): DarwinBoundsReport {
   return checkDarwinBounds(parseLeaderboardRows(stdout), promotedLineages);
 }
+
+export interface DarwinUniformityReport {
+  /** Non-baseline rows that carry a score. */
+  mutantsScored: number;
+  /** Distinct scores among those mutants. */
+  distinctMutantScores: number;
+  /** The violation line for `DarwinBoundsReport.violations`, or '' when none. */
+  violation: string;
+}
+
+/**
+ * Score-uniformity check, part of `checkDarwinBounds`.
+ *
+ * A leaderboard where two or more scored mutants all carry the identical score
+ * has no selection signal: the scorer returned a default, a fallback answer, or
+ * a parse-error value rather than judging the candidates. The 2026-09-07 run
+ * (run ab4ced4e48b76e83) and every pinned @metaharness/darwin@0.10.2 receipt
+ * since have this shape: every mutant at 0.765, "Delta over baseline: +0.000".
+ * Its winner is then an artefact, not a result, so the run fails the bound check
+ * the same way an over-wide generation does.
+ *
+ * Fewer than two scored mutants is too little evidence and is not a violation;
+ * the baseline row is reported against but never counted as a mutant.
+ */
+export function checkDarwinScoreUniformity(rows: DarwinLeaderboardRow[]): DarwinUniformityReport {
+  const scores: number[] = [];
+  for (const row of rows) {
+    if (row.id !== 'baseline' && row.score !== undefined) scores.push(row.score);
+  }
+  const distinct = new Set(scores);
+  let violation = '';
+  if (scores.length >= 2 && distinct.size === 1) {
+    const baseline = rows.find((r) => r.id === 'baseline')?.score;
+    const relation =
+      baseline === undefined ? '' : baseline === scores[0] ? ', equal to baseline' : `, baseline ${baseline}`;
+    violation = `score uniformity: all ${scores.length} mutants score ${scores[0]}${relation} — scorer not discriminating`;
+  }
+  return { mutantsScored: scores.length, distinctMutantScores: distinct.size, violation };
+}

packages/cli/src/darwinEntrypoint.test.ts

@@ -16,7 +16,8 @@ const SCRIPT = join(ROOT, 'scripts/darwin-entrypoint.sh');
 const FIXTURES = 'packages/cli/test-fixtures';
 const FAKE = `${FIXTURES}/fake-darwin.mjs`;
 const PIN = '@metaharness/darwin@0.10.2';
-const LB_FIVE = `${FIXTURES}/darwin-leaderboard-2026-09-07.txt`;
+const LB_FIVE = `${FIXTURES}/darwin-leaderboard-varied.txt`;
+const LB_UNIFORM = `${FIXTURES}/darwin-leaderboard-2026-09-07.txt`;
 const LB_SIX = `${FIXTURES}/darwin-leaderboard-six-in-g2.txt`;
 
 function runScript(args: string[], env: Record<string, string> = {}) {
@@ -50,6 +51,14 @@ describe('scripts/darwin-entrypoint.sh', () => {
     expect(r.stdout).toContain('darwin bounds ok — depth 2, max 5 candidates/generation');
   });
 
+  it('exits 3 when every mutant scores the same (real 2026-09-07 leaderboard)', () => {
+    const r = runScript(stub('--leaderboard', LB_UNIFORM));
+    expect(r.code).toBe(3);
+    expect(r.stdout).toContain('Delta over baseline: +0.000');
+    expect(r.stderr).toContain('darwin bounds VIOLATED');
+    expect(r.stderr).toContain('score uniformity: all 9 mutants score 0.765');
+  });
+
   it('exits 3 on a bound breach (6 candidates in g2)', () => {
     const r = runScript(stub('--leaderboard', LB_SIX));
     expect(r.code).toBe(3);