From 3d3a550f418a65edcd2dcf1d7952c95366e579f7 Mon Sep 17 00:00:00 2001 From: Dream Machine Date: Sat, 3 Oct 2026 01:34:40 +0000 Subject: [PATCH 1/2] dream(security-adversarial): candidate for 2026-10-03-dream-machine --- packages/cli/src/darwinBounds.test.ts | 46 +++++++++++++++++++ packages/cli/src/darwinBounds.ts | 64 +++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) diff --git a/packages/cli/src/darwinBounds.test.ts b/packages/cli/src/darwinBounds.test.ts index 9296c60..52c26bd 100644 --- a/packages/cli/src/darwinBounds.test.ts +++ b/packages/cli/src/darwinBounds.test.ts @@ -3,6 +3,7 @@ import { DARWIN_BOUNDS, checkDarwinBounds, checkDarwinBoundsFromStdout, + checkDarwinScoreUniformity, parseLeaderboardRows, } from './darwinBounds'; @@ -129,3 +130,48 @@ describe('checkDarwinBoundsFromStdout', () => { expect(report.violations[0]).toBe('leaderboard unparsable: 0 rows read'); }); }); + +// Mutator-diversity diagnostic (2026-10-03): the shape seen on every observed +// pinned-darwin run — every mutant scored exactly at the baseline. +const LB_ONE_MUTANT_DIFFERS = LB_2026_09_07.replace( + ' 0.765 g2_v2 [contextBuilder] safety=1.00 pass=0.60', + ' 0.801 g2_v2 [contextBuilder] safety=1.00 pass=0.64', +); + +describe('checkDarwinScoreUniformity (diagnostic, never a bound)', () => { + it('flags the observed run shape: every mutant scores exactly the baseline', () => { + const r = checkDarwinScoreUniformity(LB_2026_09_07); + expect(r.uniformWithBaseline).toBe(true); + expect(r.scoredRows).toBe(10); + expect(r.mutantsScored).toBe(9); + expect(r.distinctMutantScores).toBe(1); + expect(r.ok).toBe(false); + expect(r.signal).toContain('all 9 mutants score exactly the baseline 0.765'); + }); + + it('is clean when one mutant score differs from the baseline', () => { + const r = checkDarwinScoreUniformity(LB_ONE_MUTANT_DIFFERS); + expect(r.ok).toBe(true); + expect(r.signal).toBe(''); + expect(r.uniformWithBaseline).toBe(false); + expect(r.distinctMutantScores).toBe(2); + }); + + it('does not flag a lone mutant that ties the baseline (too little evidence)', () => { + const lb = [ + ' 0.5 baseline [planner] safety=1.00 pass=0.30', + ' 0.5 g1_v0 [planner] safety=1.00 pass=0.30', + ].join('\n'); + const r = checkDarwinScoreUniformity(lb); + expect(r.mutantsScored).toBe(1); + expect(r.uniformWithBaseline).toBe(false); + expect(r.ok).toBe(true); + }); + + it('says unassessed when no scored rows can be read', () => { + const r = checkDarwinScoreUniformity('no leaderboard here'); + expect(r.ok).toBe(false); + expect(r.scoredRows).toBe(0); + expect(r.signal).toContain('unassessed'); + }); +}); diff --git a/packages/cli/src/darwinBounds.ts b/packages/cli/src/darwinBounds.ts index 0a7d20d..3d367be 100644 --- a/packages/cli/src/darwinBounds.ts +++ b/packages/cli/src/darwinBounds.ts @@ -137,3 +137,67 @@ export function checkDarwinBoundsFromStdout( ): DarwinBoundsReport { return checkDarwinBounds(parseLeaderboardRows(stdout), promotedLineages); } + +const SCORED_ROW_RE = /^\s*([\d.]+)\s+(baseline|g\d+_v\d+)\s/; + +export interface DarwinUniformityReport { + /** true when no uniformity signal was emitted — diagnostic, never a bound. */ + ok: boolean; + /** Human-readable signal; '' when ok. */ + signal: string; + /** Leaderboard rows (baseline + mutants) from which a score was read. */ + scoredRows: number; + /** Non-baseline rows from which a score was read. */ + mutantsScored: number; + /** Distinct scores among mutants. Witness datum, not a threshold. */ + distinctMutantScores: number; + /** Every scored mutant matched the baseline exactly (delta +0.000 shape). */ + uniformWithBaseline: boolean; +} + +/** + * Diagnostic for the mutator-diversity question carried since 2026-09-08 (and + * again 2026-09-29): every darwin leaderboard observed from the pinned + * @metaharness/darwin@0.10.2 against the :8084 ruvllm endpoint — the 2026-09-07 + * run and every receipt from 2026-09-27 through 2026-10-03 — scores ALL mutants + * exactly at the baseline (uniform 0.765, Delta over baseline: +0.000). Such a + * leaderboard carries no selection signal: either the mutator proposed nothing + * distinguishable, or the endpoint is not really scoring (the 09-08 operator + * note names the Loom-façade possibility). + * + * Like this module's bounds before ADR-0005, this only DETECTS: it adds no + * violation and changes no exit code, and must not gate anything until a human + * decides the policy — tonight's live run is itself uniform, so wiring this in + * as a veto would fail every REQUIRED darwin evaluation until the endpoint or + * the mutator changes. + */ +export function checkDarwinScoreUniformity(stdout: string): DarwinUniformityReport { + const scored: { id: string; score: number }[] = []; + for (const line of stdout.split('\n')) { + const m = SCORED_ROW_RE.exec(line); + if (m) scored.push({ id: m[2], score: Number(m[1]) }); + } + const mutants = scored.filter((r) => r.id !== 'baseline'); + const baselineScore = scored.find((r) => r.id === 'baseline')?.score; + const distinctMutantScores = new Set(mutants.map((r) => r.score)).size; + const uniformWithBaseline = + baselineScore !== undefined && + mutants.length >= 2 && + mutants.every((r) => r.score === baselineScore); + + let signal = ''; + if (scored.length === 0) { + signal = 'score uniformity unassessed: 0 scored rows read'; + } else if (uniformWithBaseline) { + signal = `score uniformity suspect: all ${mutants.length} mutants score exactly the baseline ${baselineScore}`; + } + + return { + ok: signal === '', + signal, + scoredRows: scored.length, + mutantsScored: mutants.length, + distinctMutantScores, + uniformWithBaseline, + }; +} From 49d9efd180fbbfdf918a8f932f29b76e3cffbd51 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Tue, 6 Oct 2026 13:49:26 +0000 Subject: [PATCH 2/2] feat(darwin): flag a leaderboard where every mutant scores the same Wire the score-uniformity check into the darwin bound check (ADR-0006). parseLeaderboardRows now reads each row's score, replacing the second parser (SCORED_ROW_RE). checkDarwinBounds reports two or more scored mutants with one identical score as a violation, so verify-entrypoint darwin and scripts/darwin-entrypoint.sh exit 3 under "darwin bounds VIOLATED", the channel ADR-0005 bound breaches already use. Fewer than two scored mutants is not a breach. The real 2026-09-07 fixture (all 0.765) is now the uniformity breach case; a discriminating fixture is the compliant ceiling. Co-Authored-By: jjohare --- docs/adrs/ADR-0006-darwin-score-uniformity.md | 52 ++++++++ docs/adrs/INDEX.md | 1 + packages/cli/src/darwinBounds.test.ts | 106 ++++++++++------- packages/cli/src/darwinBounds.ts | 111 +++++++++--------- packages/cli/src/darwinEntrypoint.test.ts | 11 +- packages/cli/src/index.test.ts | 52 ++++++-- packages/cli/src/index.ts | 4 +- .../darwin-leaderboard-six-in-g2.txt | 26 ++-- .../darwin-leaderboard-varied.txt | 15 +++ scripts/darwin-entrypoint.sh | 3 +- 10 files changed, 258 insertions(+), 123 deletions(-) create mode 100644 docs/adrs/ADR-0006-darwin-score-uniformity.md create mode 100644 packages/cli/test-fixtures/darwin-leaderboard-varied.txt diff --git a/docs/adrs/ADR-0006-darwin-score-uniformity.md b/docs/adrs/ADR-0006-darwin-score-uniformity.md new file mode 100644 index 0000000..e5b1875 --- /dev/null +++ b/docs/adrs/ADR-0006-darwin-score-uniformity.md @@ -0,0 +1,52 @@ +# ADR-0006: A darwin leaderboard where every mutant scores the same fails the darwin evaluator + +- **Status**: Accepted — extends ADR-0005 +- **Date**: 2026-10-06 +- **Related**: ADR-0005 (darwin entrypoint fails on a bound breach), ADR-0003 (darwin bound guard), ADR-0002 (entrypoint liveness) +- **Deciders**: operator, 2026-10-06 ("wire it in properly") +- **Tags**: dream-cycle, evaluation-adapters, evaluator-trust + +## 1. Context + +Every darwin leaderboard observed from the pinned `@metaharness/darwin@0.10.2` +against the :8084 ruvllm endpoint — the 2026-09-07 run (ab4ced4e48b76e83) and +every receipt from 2026-09-27 through 2026-10-03 — scores all mutants +identically (0.765, `Delta over baseline: +0.000`). A scorer that returns the +same value for every candidate is returning a default, a fallback answer or a +parse-error value; it is not judging. The leaderboard's winner is then an +artefact, yet the run passed the ADR-0005 checks and exited 0. + +The 2026-10-03 security-adversarial candidate added a detector for this shape +as an unwired diagnostic with its own row parser. + +## 2. Decision + +Score uniformity is checked in the same place, and fails through the same +channel, as the ADR-0005 step-10 bounds: + +- `parseLeaderboardRows` (`packages/cli/src/darwinBounds.ts`) reads each row's + score; there is one leaderboard parser. +- `checkDarwinBounds` calls `checkDarwinScoreUniformity`. When **two or more** + scored mutants (non-baseline rows) all carry the identical score, it adds the + violation `score uniformity: all N mutants score S[, equal to baseline | + , baseline B] — scorer not discriminating`. +- Fewer than two scored mutants is too little evidence and is not a violation. + The baseline row is reported against, never counted as a mutant. +- `verify-entrypoint darwin` (and so `scripts/darwin-entrypoint.sh`) prints the + violation under `darwin bounds VIOLATED` and exits **3**, so the annexe + records the REQUIRED darwin evaluator as FAILED, as for any bound breach. + +## 3. Consequences + +- Until the scorer or the mutator changes, darwin nights fail with exit 3 and + the uniformity line in the receipt instead of passing on a meaningless + leaderboard. This is intended: the receipt now names the defect. +- The real 2026-09-07 leaderboard fixture is now the uniformity breach case; + a discriminating fixture (`darwin-leaderboard-varied.txt`) is the compliant + ceiling. + +## 4. Test Contract + +`packages/cli/src/darwinBounds.test.ts` (unit), `index.test.ts` +(`verify-entrypoint darwin` exits 3 on the uniform leaderboard, 0 on a varied +one) and `darwinEntrypoint.test.ts` (the real script, same two cases). diff --git a/docs/adrs/INDEX.md b/docs/adrs/INDEX.md index 9530c79..b218151 100644 --- a/docs/adrs/INDEX.md +++ b/docs/adrs/INDEX.md @@ -12,6 +12,7 @@ Alternatives Considered → Test Contract → References. | [ADR-0003](./ADR-0003-darwin-bound-guard.md) | Darwin generation/candidate bounds are machine-checked as a fail-visible diagnostic, not a gate veto | Accepted (amended by ADR-0005) | | [ADR-0004](./ADR-0004-ledger-verdict-vocabulary.md) | Ledger rows that are not nights carry their own verdict tokens | Accepted | | [ADR-0005](./ADR-0005-darwin-bound-five.md) | Darwin's per-generation bound is five, and the darwin entrypoint fails on a bound breach | Accepted (amends ADR-0003) | +| [ADR-0006](./ADR-0006-darwin-score-uniformity.md) | A darwin leaderboard where every mutant scores the same fails the darwin evaluator | Accepted (extends ADR-0005) | ## How to amend diff --git a/packages/cli/src/darwinBounds.test.ts b/packages/cli/src/darwinBounds.test.ts index 52c26bd..c85e396 100644 --- a/packages/cli/src/darwinBounds.test.ts +++ b/packages/cli/src/darwinBounds.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest'; +import { readFileSync } from 'node:fs'; import { DARWIN_BOUNDS, checkDarwinBounds, @@ -10,7 +11,8 @@ import { // Fixture: shape of the real 2026-09-07 REQUIRED-evaluator leaderboard // (run ab4ced4e48b76e83), which PASSED with g2 holding five candidates. Under // ADR-0003's original <=4 bound that was a breach; ADR-0005 aligned the bound -// with darwin@0.10.2's five-surface map, so five is now the compliant ceiling. +// with darwin@0.10.2's five-surface map, so five is within bounds. Every mutant +// scores 0.765, though, so ADR-0006 fails it as a non-discriminating scorer. const LB_2026_09_07 = [ 'Darwin Mode — leaderboard', ' 0.765 baseline [planner] safety=1.00 pass=0.60 ◀ winner', @@ -29,13 +31,17 @@ const LB_2026_09_07 = [ 'Delta over baseline: +0.000', ].join('\n'); +// A discriminating leaderboard of the same 4 + 5 shape: mutant scores differ, +// so the scorer is actually judging. This is the compliant ceiling. +const LB_VARIED = readFileSync( + new URL('../test-fixtures/darwin-leaderboard-varied.txt', import.meta.url), + 'utf8', +); + // One past the ADR-0005 ceiling: g2 gains a sixth candidate. -const LB_SIX_IN_G2 = LB_2026_09_07.replace( - ' 0.765 g2_v4 [scorePolicy] safety=1.00 pass=0.60', - [ - ' 0.765 g2_v4 [scorePolicy] safety=1.00 pass=0.60', - ' 0.765 g2_v5 [planner] safety=1.00 pass=0.60', - ].join('\n'), +const LB_SIX_IN_G2 = readFileSync( + new URL('../test-fixtures/darwin-leaderboard-six-in-g2.txt', import.meta.url), + 'utf8', ); describe('parseLeaderboardRows', () => { @@ -43,6 +49,13 @@ describe('parseLeaderboardRows', () => { const rows = parseLeaderboardRows(LB_2026_09_07); expect(rows).toHaveLength(10); expect(rows.filter((r) => r.id === 'baseline')).toHaveLength(1); + expect(rows.every((r) => r.score === 0.765)).toBe(true); + }); + + it('reads each row\'s score and generation', () => { + const rows = parseLeaderboardRows(LB_VARIED); + expect(rows[0]).toEqual({ id: 'g2_v1', generation: 2, score: 0.812 }); + expect(rows.find((r) => r.id === 'baseline')).toEqual({ id: 'baseline', generation: 0, score: 0.765 }); }); it('returns [] when no leaderboard lines are present', () => { @@ -57,11 +70,13 @@ describe('checkDarwinBounds', () => { expect(DARWIN_BOUNDS.maxPromotedLineages).toBe(1); }); - it('accepts the real 2026-09-07 run: g2 held 5 candidates, within the bound', () => { - const report = checkDarwinBoundsFromStdout(LB_2026_09_07, 0); + it('accepts 5 candidates in g2 with discriminating scores, within the bound', () => { + const report = checkDarwinBoundsFromStdout(LB_VARIED, 0); expect(report.parseStatus).toBe('ok'); expect(report.candidatesPerGeneration).toEqual({ 1: 4, 2: 5 }); expect(report.maxCandidatesPerGeneration).toBe(5); + expect(report.mutantsScored).toBe(9); + expect(report.distinctMutantScores).toBe(9); expect(report.ok).toBe(true); expect(report.violations).toEqual([]); }); @@ -93,7 +108,7 @@ describe('checkDarwinBounds', () => { }); it('flags more than 1 promoted lineage', () => { - const rows = parseLeaderboardRows(LB_2026_09_07); + const rows = parseLeaderboardRows(LB_VARIED); const report = checkDarwinBounds(rows, 2); expect(report.violations).toContain('promotedLineages=2 > 1'); }); @@ -131,47 +146,56 @@ describe('checkDarwinBoundsFromStdout', () => { }); }); -// Mutator-diversity diagnostic (2026-10-03): the shape seen on every observed -// pinned-darwin run — every mutant scored exactly at the baseline. -const LB_ONE_MUTANT_DIFFERS = LB_2026_09_07.replace( - ' 0.765 g2_v2 [contextBuilder] safety=1.00 pass=0.60', - ' 0.801 g2_v2 [contextBuilder] safety=1.00 pass=0.64', -); +// Score uniformity (2026-10-03, wired in 2026-10-06): the shape seen on every +// observed pinned-darwin run — every mutant scored identically — means the scorer +// is not discriminating, so the leaderboard's winner is an artefact. +describe('score uniformity', () => { + it('flags the real 2026-09-07 run: all 9 mutants score 0.765, equal to baseline', () => { + const report = checkDarwinBoundsFromStdout(LB_2026_09_07, 0); + expect(report.ok).toBe(false); + expect(report.mutantsScored).toBe(9); + expect(report.distinctMutantScores).toBe(1); + // Within the step-10 bounds: uniformity is the only violation. + expect(report.violations).toEqual([ + 'score uniformity: all 9 mutants score 0.765, equal to baseline — scorer not discriminating', + ]); + }); -describe('checkDarwinScoreUniformity (diagnostic, never a bound)', () => { - it('flags the observed run shape: every mutant scores exactly the baseline', () => { - const r = checkDarwinScoreUniformity(LB_2026_09_07); - expect(r.uniformWithBaseline).toBe(true); - expect(r.scoredRows).toBe(10); - expect(r.mutantsScored).toBe(9); - expect(r.distinctMutantScores).toBe(1); - expect(r.ok).toBe(false); - expect(r.signal).toContain('all 9 mutants score exactly the baseline 0.765'); + it('flags mutants that agree with each other but not the baseline', () => { + const lb = [ + ' 0.700 baseline [planner] safety=1.00 pass=0.50', + ' 0.900 g1_v0 [planner] safety=1.00 pass=0.80', + ' 0.900 g1_v1 [reviewer] safety=1.00 pass=0.80', + ].join('\n'); + const r = checkDarwinScoreUniformity(parseLeaderboardRows(lb)); + expect(r.violation).toBe('score uniformity: all 2 mutants score 0.9, baseline 0.7 — scorer not discriminating'); }); - it('is clean when one mutant score differs from the baseline', () => { - const r = checkDarwinScoreUniformity(LB_ONE_MUTANT_DIFFERS); - expect(r.ok).toBe(true); - expect(r.signal).toBe(''); - expect(r.uniformWithBaseline).toBe(false); - expect(r.distinctMutantScores).toBe(2); + it('is clean when a single mutant score differs', () => { + const lb = LB_2026_09_07.replace( + ' 0.765 g2_v2 [contextBuilder] safety=1.00 pass=0.60', + ' 0.801 g2_v2 [contextBuilder] safety=1.00 pass=0.64', + ); + const report = checkDarwinBoundsFromStdout(lb, 0); + expect(report.distinctMutantScores).toBe(2); + expect(report.ok).toBe(true); + expect(report.violations).toEqual([]); }); - it('does not flag a lone mutant that ties the baseline (too little evidence)', () => { + it('does not flag fewer than 2 scored mutants (too little evidence)', () => { const lb = [ ' 0.5 baseline [planner] safety=1.00 pass=0.30', ' 0.5 g1_v0 [planner] safety=1.00 pass=0.30', ].join('\n'); - const r = checkDarwinScoreUniformity(lb); - expect(r.mutantsScored).toBe(1); - expect(r.uniformWithBaseline).toBe(false); - expect(r.ok).toBe(true); + const report = checkDarwinBoundsFromStdout(lb, 0); + expect(report.mutantsScored).toBe(1); + expect(report.ok).toBe(true); }); - it('says unassessed when no scored rows can be read', () => { - const r = checkDarwinScoreUniformity('no leaderboard here'); - expect(r.ok).toBe(false); - expect(r.scoredRows).toBe(0); - expect(r.signal).toContain('unassessed'); + it('ignores hand-built rows with no score', () => { + const rows = [0, 1, 2].map((v) => ({ id: `g1_v${v}`, generation: 1 })); + const r = checkDarwinScoreUniformity(rows); + expect(r.mutantsScored).toBe(0); + expect(r.violation).toBe(''); }); }); diff --git a/packages/cli/src/darwinBounds.ts b/packages/cli/src/darwinBounds.ts index 3d367be..2072920 100644 --- a/packages/cli/src/darwinBounds.ts +++ b/packages/cli/src/darwinBounds.ts @@ -10,16 +10,25 @@ * This module makes the bound machine-checkable: additive-only and * dependency-free (pattern proven on 2026-09-06). * + * It also fails a leaderboard where two or more mutants all score identically + * (ADR-0006): such a scorer is not discriminating. + * * This module only DETECTS. It becomes enforcement through * scripts/darwin-entrypoint.sh (ADR-0005), which runs darwin through * `verify-entrypoint` so a breach fails the REQUIRED evaluator; the gate that * honours that failure lives in the external annexe runner. See - * docs/adrs/ADR-0003-darwin-bound-guard.md and ADR-0005-darwin-bound-five.md. + * docs/adrs/ADR-0003-darwin-bound-guard.md, ADR-0005-darwin-bound-five.md and + * ADR-0006-darwin-score-uniformity.md. */ export interface DarwinLeaderboardRow { id: string; generation: number; + /** + * The row's leaderboard score. Always set by `parseLeaderboardRows`; a row + * built by hand without one is left out of the score-uniformity check. + */ + score?: number; } export interface DarwinBoundsReport { @@ -37,6 +46,10 @@ export interface DarwinBoundsReport { candidatesPerGeneration: Record; maxCandidatesPerGeneration: number; promotedLineages: number; + /** Non-baseline rows that carry a score. */ + mutantsScored: number; + /** Distinct scores among those mutants; 1 with ≥ 2 mutants is a violation. */ + distinctMutantScores: number; violations: string[]; } @@ -47,24 +60,26 @@ export const DARWIN_BOUNDS = { maxPromotedLineages: 1, } as const; -const ROW_RE = /^\s*[\d.]+\s+(baseline|g(\d+)_v(\d+))\s/; +const ROW_RE = /^\s*([\d.]+)\s+(baseline|g(\d+)_v(\d+))\s/; export function parseLeaderboardRows(stdout: string): DarwinLeaderboardRow[] { const rows: DarwinLeaderboardRow[] = []; for (const line of stdout.split('\n')) { const m = ROW_RE.exec(line); if (!m) continue; + const score = Number(m[1]); rows.push( - m[1] === 'baseline' - ? { id: 'baseline', generation: 0 } - : { id: m[1], generation: Number(m[2]) }, + m[2] === 'baseline' + ? { id: 'baseline', generation: 0, score } + : { id: m[2], generation: Number(m[3]), score }, ); } return rows; } /** - * Check a parsed leaderboard against DARWIN_BOUNDS. + * Check a parsed leaderboard against DARWIN_BOUNDS, and that its scores + * discriminate (see `checkDarwinScoreUniformity`). * * Fail-visible by construction: an empty row set reports * `parseStatus: 'unparsable'` AND `ok: false`. The two can never disagree, so a @@ -103,6 +118,9 @@ export function checkDarwinBounds( ); } } + const uniformity = checkDarwinScoreUniformity(rows); + if (uniformity.violation) violations.push(uniformity.violation); + if (promotedLineages > DARWIN_BOUNDS.maxPromotedLineages) { violations.push( `promotedLineages=${promotedLineages} > ${DARWIN_BOUNDS.maxPromotedLineages}`, @@ -122,6 +140,8 @@ export function checkDarwinBounds( candidatesPerGeneration, maxCandidatesPerGeneration: maxCandidates, promotedLineages, + mutantsScored: uniformity.mutantsScored, + distinctMutantScores: uniformity.distinctMutantScores, violations, }; } @@ -138,66 +158,41 @@ export function checkDarwinBoundsFromStdout( return checkDarwinBounds(parseLeaderboardRows(stdout), promotedLineages); } -const SCORED_ROW_RE = /^\s*([\d.]+)\s+(baseline|g\d+_v\d+)\s/; - export interface DarwinUniformityReport { - /** true when no uniformity signal was emitted — diagnostic, never a bound. */ - ok: boolean; - /** Human-readable signal; '' when ok. */ - signal: string; - /** Leaderboard rows (baseline + mutants) from which a score was read. */ - scoredRows: number; - /** Non-baseline rows from which a score was read. */ + /** Non-baseline rows that carry a score. */ mutantsScored: number; - /** Distinct scores among mutants. Witness datum, not a threshold. */ + /** Distinct scores among those mutants. */ distinctMutantScores: number; - /** Every scored mutant matched the baseline exactly (delta +0.000 shape). */ - uniformWithBaseline: boolean; + /** The violation line for `DarwinBoundsReport.violations`, or '' when none. */ + violation: string; } /** - * Diagnostic for the mutator-diversity question carried since 2026-09-08 (and - * again 2026-09-29): every darwin leaderboard observed from the pinned - * @metaharness/darwin@0.10.2 against the :8084 ruvllm endpoint — the 2026-09-07 - * run and every receipt from 2026-09-27 through 2026-10-03 — scores ALL mutants - * exactly at the baseline (uniform 0.765, Delta over baseline: +0.000). Such a - * leaderboard carries no selection signal: either the mutator proposed nothing - * distinguishable, or the endpoint is not really scoring (the 09-08 operator - * note names the Loom-façade possibility). + * Score-uniformity check, part of `checkDarwinBounds`. * - * Like this module's bounds before ADR-0005, this only DETECTS: it adds no - * violation and changes no exit code, and must not gate anything until a human - * decides the policy — tonight's live run is itself uniform, so wiring this in - * as a veto would fail every REQUIRED darwin evaluation until the endpoint or - * the mutator changes. + * A leaderboard where two or more scored mutants all carry the identical score + * has no selection signal: the scorer returned a default, a fallback answer, or + * a parse-error value rather than judging the candidates. The 2026-09-07 run + * (run ab4ced4e48b76e83) and every pinned @metaharness/darwin@0.10.2 receipt + * since have this shape: every mutant at 0.765, "Delta over baseline: +0.000". + * Its winner is then an artefact, not a result, so the run fails the bound check + * the same way an over-wide generation does. + * + * Fewer than two scored mutants is too little evidence and is not a violation; + * the baseline row is reported against but never counted as a mutant. */ -export function checkDarwinScoreUniformity(stdout: string): DarwinUniformityReport { - const scored: { id: string; score: number }[] = []; - for (const line of stdout.split('\n')) { - const m = SCORED_ROW_RE.exec(line); - if (m) scored.push({ id: m[2], score: Number(m[1]) }); +export function checkDarwinScoreUniformity(rows: DarwinLeaderboardRow[]): DarwinUniformityReport { + const scores: number[] = []; + for (const row of rows) { + if (row.id !== 'baseline' && row.score !== undefined) scores.push(row.score); } - const mutants = scored.filter((r) => r.id !== 'baseline'); - const baselineScore = scored.find((r) => r.id === 'baseline')?.score; - const distinctMutantScores = new Set(mutants.map((r) => r.score)).size; - const uniformWithBaseline = - baselineScore !== undefined && - mutants.length >= 2 && - mutants.every((r) => r.score === baselineScore); - - let signal = ''; - if (scored.length === 0) { - signal = 'score uniformity unassessed: 0 scored rows read'; - } else if (uniformWithBaseline) { - signal = `score uniformity suspect: all ${mutants.length} mutants score exactly the baseline ${baselineScore}`; + const distinct = new Set(scores); + let violation = ''; + if (scores.length >= 2 && distinct.size === 1) { + const baseline = rows.find((r) => r.id === 'baseline')?.score; + const relation = + baseline === undefined ? '' : baseline === scores[0] ? ', equal to baseline' : `, baseline ${baseline}`; + violation = `score uniformity: all ${scores.length} mutants score ${scores[0]}${relation} — scorer not discriminating`; } - - return { - ok: signal === '', - signal, - scoredRows: scored.length, - mutantsScored: mutants.length, - distinctMutantScores, - uniformWithBaseline, - }; + return { mutantsScored: scores.length, distinctMutantScores: distinct.size, violation }; } diff --git a/packages/cli/src/darwinEntrypoint.test.ts b/packages/cli/src/darwinEntrypoint.test.ts index a15a937..5edda5c 100644 --- a/packages/cli/src/darwinEntrypoint.test.ts +++ b/packages/cli/src/darwinEntrypoint.test.ts @@ -16,7 +16,8 @@ const SCRIPT = join(ROOT, 'scripts/darwin-entrypoint.sh'); const FIXTURES = 'packages/cli/test-fixtures'; const FAKE = `${FIXTURES}/fake-darwin.mjs`; const PIN = '@metaharness/darwin@0.10.2'; -const LB_FIVE = `${FIXTURES}/darwin-leaderboard-2026-09-07.txt`; +const LB_FIVE = `${FIXTURES}/darwin-leaderboard-varied.txt`; +const LB_UNIFORM = `${FIXTURES}/darwin-leaderboard-2026-09-07.txt`; const LB_SIX = `${FIXTURES}/darwin-leaderboard-six-in-g2.txt`; function runScript(args: string[], env: Record = {}) { @@ -50,6 +51,14 @@ describe('scripts/darwin-entrypoint.sh', () => { expect(r.stdout).toContain('darwin bounds ok — depth 2, max 5 candidates/generation'); }); + it('exits 3 when every mutant scores the same (real 2026-09-07 leaderboard)', () => { + const r = runScript(stub('--leaderboard', LB_UNIFORM)); + expect(r.code).toBe(3); + expect(r.stdout).toContain('Delta over baseline: +0.000'); + expect(r.stderr).toContain('darwin bounds VIOLATED'); + expect(r.stderr).toContain('score uniformity: all 9 mutants score 0.765'); + }); + it('exits 3 on a bound breach (6 candidates in g2)', () => { const r = runScript(stub('--leaderboard', LB_SIX)); expect(r.code).toBe(3); diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 5968e21..f4894c0 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -293,7 +293,7 @@ describe('verify-entrypoint', () => { // ADR-0003. The real 2026-09-07 REQUIRED-evaluator leaderboard (run ab4ced4e48b76e83): // outcome=PASSED, exit 0, and five candidates in generation 2 against the then <=4 // bound. ADR-0005 raised the bound to 5 to match darwin@0.10.2's five-surface map, so - // this fixture is now the compliant ceiling and a sixth g2 row is the breach. + // its shape is within bounds; its uniform 0.765 scores fail it under ADR-0006. const DARWIN_LEADERBOARD_2026_09_07 = [ 'Darwin Mode — leaderboard', ' 0.765 baseline [planner] safety=1.00 pass=0.60 ◀ winner', @@ -312,11 +312,30 @@ describe('verify-entrypoint', () => { 'Delta over baseline: +0.000', ].join('\n'); - const DARWIN_LEADERBOARD_SIX_IN_G2 = DARWIN_LEADERBOARD_2026_09_07.replace( - ' 0.765 g2_v4 [scorePolicy] safety=1.00 pass=0.60', + // The same 4 + 5 shape with discriminating scores: the compliant ceiling. + const DARWIN_LEADERBOARD_VARIED = [ + 'Darwin Mode — leaderboard', + ' 0.812 g2_v1 [planner] safety=1.00 pass=0.66 ◀ winner', + ' 0.798 g2_v3 [toolPolicy] safety=1.00 pass=0.64', + ' 0.781 g1_v0 [planner] safety=1.00 pass=0.62', + ' 0.774 g2_v0 [reviewer] safety=1.00 pass=0.61', + ' 0.765 baseline [planner] safety=1.00 pass=0.60', + ' 0.765 g1_v2 [reviewer] safety=1.00 pass=0.60', + ' 0.752 g1_v1 [toolPolicy] safety=1.00 pass=0.58', + ' 0.749 g2_v2 [contextBuilder] safety=1.00 pass=0.58', + ' 0.741 g1_v3 [toolPolicy] safety=1.00 pass=0.57', + ' 0.733 g2_v4 [scorePolicy] safety=1.00 pass=0.56', + '', + 'Winner: g2_v1', + 'Lineage: baseline → g1_v0 → g2_v1', + 'Delta over baseline: +0.047', + ].join('\n'); + + const DARWIN_LEADERBOARD_SIX_IN_G2 = DARWIN_LEADERBOARD_VARIED.replace( + ' 0.733 g2_v4 [scorePolicy] safety=1.00 pass=0.56', [ - ' 0.765 g2_v4 [scorePolicy] safety=1.00 pass=0.60', - ' 0.765 g2_v5 [planner] safety=1.00 pass=0.60', + ' 0.733 g2_v4 [scorePolicy] safety=1.00 pass=0.56', + ' 0.728 g2_v5 [planner] safety=1.00 pass=0.55', ].join('\n'), ); @@ -336,14 +355,31 @@ describe('verify-entrypoint', () => { expect(r.err).toContain('promotedLineages unchecked'); }); - it('still exits 0 for a live darwin run that respects the bounds', async () => { - // Five in g2 — the real 2026-09-07 shape — is within the ADR-0005 bound. + // The real 2026-09-07 run sits inside the step-10 bounds, but every mutant scored + // 0.765: the scorer judged nothing, so the result is untrustworthy and fails the + // same way a bound breach does. + it('exits 3 when every scored mutant has the identical score', async () => { const io = mockIOWithExec(async () => ({ code: 0, stdout: DARWIN_LEADERBOARD_2026_09_07, stderr: '', })); const r = await run(['verify-entrypoint', 'darwin', '--cmd', 'npx @metaharness/darwin'], io); + expect(r.out).toContain('darwin: live'); + expect(r.code).toBe(3); + expect(r.err).toContain('darwin bounds VIOLATED (leaderboard ok)'); + expect(r.err).toContain(' - score uniformity: all 9 mutants score 0.765, equal to baseline'); + expect(r.out).not.toContain('bounds ok'); + }); + + it('still exits 0 for a live darwin run that respects the bounds', async () => { + // Five in g2 is within the ADR-0005 bound, and the scores discriminate. + const io = mockIOWithExec(async () => ({ + code: 0, + stdout: DARWIN_LEADERBOARD_VARIED, + stderr: '', + })); + const r = await run(['verify-entrypoint', 'darwin', '--cmd', 'npx @metaharness/darwin'], io); expect(r.code).toBe(0); expect(r.out).toContain('darwin bounds ok — depth 2, max 5 candidates/generation'); expect(r.err).not.toContain('VIOLATED'); @@ -355,7 +391,7 @@ describe('verify-entrypoint', () => { it('--passthrough echoes the command output ahead of the verdict', async () => { const io = mockIOWithExec(async () => ({ code: 0, - stdout: DARWIN_LEADERBOARD_2026_09_07, + stdout: DARWIN_LEADERBOARD_VARIED, stderr: 'darwin: ruvllm mutator warming up', })); const r = await run( diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 204966c..3848e87 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -357,7 +357,9 @@ export async function run(argv: string[], io: IO): Promise { // ADR-0003: a live darwin evaluator can still have run out of policy. On // 2026-09-07 it returned outcome=PASSED with five candidates in generation 2, // against the then-documented <=4/generation bound (raised to 5 by ADR-0005), - // and nothing noticed. Check the leaderboard it just printed. + // and nothing noticed. Check the leaderboard it just printed. The same check + // fails a leaderboard whose mutants all score identically (ADR-0006): the + // scorer judged nothing, so the run is untrustworthy, not compliant. // // Exit 3 is detection here. It becomes enforcement only because // scripts/darwin-entrypoint.sh (ADR-0005) makes this command the darwin diff --git a/packages/cli/test-fixtures/darwin-leaderboard-six-in-g2.txt b/packages/cli/test-fixtures/darwin-leaderboard-six-in-g2.txt index 8470e49..69e1b50 100644 --- a/packages/cli/test-fixtures/darwin-leaderboard-six-in-g2.txt +++ b/packages/cli/test-fixtures/darwin-leaderboard-six-in-g2.txt @@ -1,16 +1,16 @@ Darwin Mode — leaderboard - 0.765 baseline [planner] safety=1.00 pass=0.60 ◀ winner - 0.765 g1_v0 [planner] safety=1.00 pass=0.60 - 0.765 g1_v1 [toolPolicy] safety=1.00 pass=0.60 + 0.812 g2_v1 [planner] safety=1.00 pass=0.66 ◀ winner + 0.798 g2_v3 [toolPolicy] safety=1.00 pass=0.64 + 0.781 g1_v0 [planner] safety=1.00 pass=0.62 + 0.774 g2_v0 [reviewer] safety=1.00 pass=0.61 + 0.765 baseline [planner] safety=1.00 pass=0.60 0.765 g1_v2 [reviewer] safety=1.00 pass=0.60 - 0.765 g1_v3 [toolPolicy] safety=1.00 pass=0.60 - 0.765 g2_v0 [reviewer] safety=1.00 pass=0.60 - 0.765 g2_v1 [planner] safety=1.00 pass=0.60 - 0.765 g2_v2 [contextBuilder] safety=1.00 pass=0.60 - 0.765 g2_v3 [toolPolicy] safety=1.00 pass=0.60 - 0.765 g2_v4 [scorePolicy] safety=1.00 pass=0.60 - 0.765 g2_v5 [planner] safety=1.00 pass=0.60 + 0.752 g1_v1 [toolPolicy] safety=1.00 pass=0.58 + 0.749 g2_v2 [contextBuilder] safety=1.00 pass=0.58 + 0.741 g1_v3 [toolPolicy] safety=1.00 pass=0.57 + 0.733 g2_v4 [scorePolicy] safety=1.00 pass=0.56 + 0.728 g2_v5 [planner] safety=1.00 pass=0.55 -Winner: baseline -Lineage: baseline -Delta over baseline: +0.000 +Winner: g2_v1 +Lineage: baseline → g1_v0 → g2_v1 +Delta over baseline: +0.047 diff --git a/packages/cli/test-fixtures/darwin-leaderboard-varied.txt b/packages/cli/test-fixtures/darwin-leaderboard-varied.txt new file mode 100644 index 0000000..c7e399b --- /dev/null +++ b/packages/cli/test-fixtures/darwin-leaderboard-varied.txt @@ -0,0 +1,15 @@ +Darwin Mode — leaderboard + 0.812 g2_v1 [planner] safety=1.00 pass=0.66 ◀ winner + 0.798 g2_v3 [toolPolicy] safety=1.00 pass=0.64 + 0.781 g1_v0 [planner] safety=1.00 pass=0.62 + 0.774 g2_v0 [reviewer] safety=1.00 pass=0.61 + 0.765 baseline [planner] safety=1.00 pass=0.60 + 0.765 g1_v2 [reviewer] safety=1.00 pass=0.60 + 0.752 g1_v1 [toolPolicy] safety=1.00 pass=0.58 + 0.749 g2_v2 [contextBuilder] safety=1.00 pass=0.58 + 0.741 g1_v3 [toolPolicy] safety=1.00 pass=0.57 + 0.733 g2_v4 [scorePolicy] safety=1.00 pass=0.56 + +Winner: g2_v1 +Lineage: baseline → g1_v0 → g2_v1 +Delta over baseline: +0.047 diff --git a/scripts/darwin-entrypoint.sh b/scripts/darwin-entrypoint.sh index effd172..29bc08b 100755 --- a/scripts/darwin-entrypoint.sh +++ b/scripts/darwin-entrypoint.sh @@ -19,7 +19,8 @@ # 0 darwin live, bounds respected # 1 darwin failed (blocked) # 2 darwin exited 0 with no output (suspicious-silent) -# 3 darwin live but its leaderboard breached a bound, or could not be parsed +# 3 darwin live but its leaderboard breached a bound, could not be parsed, +# or scored every mutant identically (ADR-0006) # 64 refused: the command is not an exact-pinned darwin run in a mock/agent sandbox # 69 the CLI is not built (run `npm run build`) set -euo pipefail