From 10bb9ffc54a6da27efbdd53c259362a88327626b Mon Sep 17 00:00:00 2001 From: Byron Thanopoulos Date: Wed, 16 Sep 2026 19:19:44 +0100 Subject: [PATCH 1/3] test(teach): verify reading-track docs against rendered HTML The reading track (pm-01, pm-02) teaches PMs and reviewers to read a dashboard. Its output blocks quote figures rendered into index.html rather than CLI stdout, so the drift guard skipped both files and they were verified only by hand. Adds an HTML assertion mode: replay the doc's commands, read back the report they render, reduce it to visible text and match the documented figures against that. - open ... is rewritten to a no-op so CI never launches a browser - script/style bodies are stripped before matching, so a figure present only in the report's embedded JSON cannot satisfy an assertion the visible page would fail - whitespace is collapsed, since the renderer splits Passed 8/13 across elements into Passed 8 /13 Coverage 73 -> 77 asserted lines, 10 -> 12 exercises. --- scripts/verify-teaching-docs.ts | 96 +++++++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 10 deletions(-) diff --git a/scripts/verify-teaching-docs.ts b/scripts/verify-teaching-docs.ts index 1cf9acf..78fef01 100644 --- a/scripts/verify-teaching-docs.ts +++ b/scripts/verify-teaching-docs.ts @@ -17,7 +17,7 @@ * pnpm teach:verify --print # also print captured stdout per exercise */ import { execFileSync } from 'node:child_process'; -import { mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs'; +import { cpSync, mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; @@ -71,12 +71,14 @@ const STANDALONE_EXERCISES: { file: string; needsExamples: boolean }[] = [ ]; /** - * Not covered, deliberately: pm-01-reading-a-report.md and - * pm-02-reading-drift.md. Their ```text blocks quote figures rendered into the - * HTML report, not CLI stdout, and their steps call `open` to launch a browser. - * Both were verified by hand on 2026-09-15 by extracting the text of the - * generated index.html. Guarding them needs an HTML-aware assertion mode. + * Reading-track exercises for non-engineers. Unlike every other doc, their + * ```text blocks quote figures rendered into the HTML report rather than CLI + * stdout, so they are verified against the text of the generated index.html. + * + * Their steps also call `open` to launch a browser, which is stripped before + * replay (see runShellBlock) so CI never spawns a GUI. */ +const HTML_EXERCISES = ['pm-01-reading-a-report.md', 'pm-02-reading-drift.md']; interface Block { lang: string; @@ -122,7 +124,9 @@ function runShellBlock(script: string, cwd: string): { stdout: string; code: num // Exercises are written for the published CLI; point them at this checkout. const rewritten = script .replace(/npx eval-dashboards/g, `"${tsxBin}" "${cliEntry}"`) - .replace(/pnpm cli:dev/g, `"${tsxBin}" "${cliEntry}"`); + .replace(/pnpm cli:dev/g, `"${tsxBin}" "${cliEntry}"`) + // Reading-track docs tell a human to open the report; never spawn a GUI in CI. + .replace(/^\s*open\s+.*$/gm, ':'); try { const stdout = execFileSync('bash', ['-c', rewritten], { cwd, @@ -136,6 +140,35 @@ function runShellBlock(script: string, cwd: string): { stdout: string; code: num } } +/** + * Reduce an HTML document to its visible text, so documented figures can be + * matched against what a reader actually sees on the page. + * + * Script and style bodies are dropped first: they contain the report's own data + * as JSON, which would otherwise satisfy assertions the rendered page does not. + */ +function htmlToText(html: string): string { + return html + .replace(//gi, ' ') + .replace(//gi, ' ') + .replace(/<[^>]+>/g, ' ') + .replace(/ /g, ' ') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&/g, '&'); +} + +/** + * Collapse runs of whitespace so a documented figure matches regardless of how + * the renderer split it across elements: `Passed 8/13` becomes `Passed 8 /13` + * once tags are stripped, and header strips wrap arbitrarily. + */ +function normalizeForMatch(text: string): string { + return text.replace(/\s+/g, ''); +} + interface Drift { file: string; line: number; @@ -143,11 +176,17 @@ interface Drift { context: string; } -/** Replay one doc's shell blocks and diff its ```text blocks against reality. */ +/** + * Replay one doc's shell blocks and diff its ```text blocks against reality. + * + * `source` selects what "reality" means: 'stdout' asserts against what the CLI + * printed, 'html' against the visible text of every report the block rendered. + */ function verifyDoc( docRelPath: string, cwd: string, drifts: Drift[], + source: 'stdout' | 'html' = 'stdout', ): number { const markdown = readFileSync(path.join(repoRoot, docRelPath), 'utf8'); const blocks = extractBlocks(markdown); @@ -160,15 +199,37 @@ function verifyDoc( captured += stdout; } + if (source === 'html') { + // The CLI prints each report's path; read back what it actually rendered. + const reports = captured + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.endsWith('index.html')); + if (reports.length === 0) { + drifts.push({ + file: docRelPath, + line: 1, + documented: '(expected this doc to render at least one HTML report)', + context: captured.trim().split('\n').slice(-6).join('\n'), + }); + return 0; + } + captured = reports + .map((rel) => htmlToText(readFileSync(path.join(cwd, rel), 'utf8'))) + .join('\n'); + } + if (printMode) { process.stdout.write(`\n=== ${docRelPath} ===\n${captured}`); } + const haystack = normalizeForMatch(captured); + for (const block of blocks) { if (block.lang !== 'text') continue; for (const expected of assertableLines(block.body)) { asserted += 1; - if (!captured.includes(expected)) { + if (!haystack.includes(normalizeForMatch(expected))) { drifts.push({ file: docRelPath, line: block.startLine, @@ -211,15 +272,30 @@ function main(): number { rmSync(soloDir, { recursive: true, force: true }); } } + // Reading-track exercises assert against rendered HTML, not stdout. They + // copy examples/ in so the docs' relative fixture paths resolve unmodified. + for (const file of HTML_EXERCISES) { + const htmlDir = mkdtempSync(path.join(tmpdir(), 'teach-html-')); + try { + cpSync(path.join(repoRoot, 'examples'), path.join(htmlDir, 'examples'), { + recursive: true, + }); + assertedCount += verifyDoc(`docs/teach-exercises/${file}`, htmlDir, drifts, 'html'); + } finally { + rmSync(htmlDir, { recursive: true, force: true }); + } + } } finally { rmSync(workdir, { recursive: true, force: true }); } if (drifts.length === 0) { const labCount = exercisesOnly ? 0 : LABS.length; + const exerciseCount = + CHAIN.length + STANDALONE_EXERCISES.length + HTML_EXERCISES.length; console.log( `Teaching docs verified: ${assertedCount} documented output line(s) across ` + - `${CHAIN.length + STANDALONE_EXERCISES.length} exercises and ${labCount} labs ` + + `${exerciseCount} exercises and ${labCount} labs ` + `match real CLI output.`, ); return 0; From ba485e8ddbfcf80c95fb4c7814d496a5ce27c815 Mon Sep 17 00:00:00 2001 From: Byron Thanopoulos Date: Wed, 16 Sep 2026 19:33:19 +0100 Subject: [PATCH 2/3] test(teach): assert HTML figures against their own report Review caught a real hole. HTML mode joined every rendered report into one haystack, so a figure documented for one report could be satisfied by another. pm-02 contrasts a good run with a red one, and its whole lesson is telling them apart -- swapping the two header strips passed verification. Blocks now inherit the report named just above them in the prose (`eval-dashboard-red/index.html`:) and are asserted against that report alone. Failures name the report checked, so the message stays actionable. Verified: swapping pm-02's two header strips now fails with both lines flagged; restoring returns to green. --- scripts/verify-teaching-docs.ts | 50 ++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/scripts/verify-teaching-docs.ts b/scripts/verify-teaching-docs.ts index 78fef01..ad24fa6 100644 --- a/scripts/verify-teaching-docs.ts +++ b/scripts/verify-teaching-docs.ts @@ -84,6 +84,8 @@ interface Block { lang: string; body: string; startLine: number; + /** Nearest preceding inline-code path, e.g. `eval-dashboard-red/index.html`. */ + attributedTo?: string; } /** Extract fenced code blocks with their 1-indexed opening line number. */ @@ -91,16 +93,29 @@ function extractBlocks(markdown: string): Block[] { const lines = markdown.split('\n'); const blocks: Block[] = []; let open: { lang: string; startLine: number; buf: string[] } | null = null; + let lastAttribution: string | undefined; for (let i = 0; i < lines.length; i += 1) { const line = lines[i]; const fence = line.match(/^```(\w*)\s*$/); if (!fence) { - if (open) open.buf.push(line); + if (open) { + open.buf.push(line); + } else { + // Docs introduce a block by naming the file it came from, e.g. + // `eval-dashboard-good/index.html`: — remember the most recent one. + const named = line.match(/`([^`]*index\.html)`/); + if (named) lastAttribution = named[1]; + } continue; } if (open) { - blocks.push({ lang: open.lang, body: open.buf.join('\n'), startLine: open.startLine }); + blocks.push({ + lang: open.lang, + body: open.buf.join('\n'), + startLine: open.startLine, + attributedTo: lastAttribution, + }); open = null; } else { open = { lang: fence[1] || '', startLine: i + 1, buf: [] }; @@ -199,6 +214,10 @@ function verifyDoc( captured += stdout; } + // Per-report text, so a figure documented for one report cannot be satisfied + // by another. Keyed by the path the CLI printed, e.g. eval-dashboard/index.html. + const byReport = new Map(); + if (source === 'html') { // The CLI prints each report's path; read back what it actually rendered. const reports = captured @@ -214,19 +233,34 @@ function verifyDoc( }); return 0; } - captured = reports - .map((rel) => htmlToText(readFileSync(path.join(cwd, rel), 'utf8'))) - .join('\n'); + for (const rel of reports) { + byReport.set(rel, htmlToText(readFileSync(path.join(cwd, rel), 'utf8'))); + } + captured = [...byReport.values()].join('\n'); } if (printMode) { process.stdout.write(`\n=== ${docRelPath} ===\n${captured}`); } - const haystack = normalizeForMatch(captured); + const allText = normalizeForMatch(captured); for (const block of blocks) { if (block.lang !== 'text') continue; + + // When a doc names the report a block came from, assert against that report + // alone. pm-02 contrasts a good run with a red one; matching against the + // union would let the two sets of figures satisfy each other's assertions. + let haystack = allText; + let scope = ''; + if (source === 'html' && block.attributedTo) { + const match = [...byReport.entries()].find(([rel]) => rel.endsWith(block.attributedTo!)); + if (match) { + haystack = normalizeForMatch(match[1]); + scope = match[0]; + } + } + for (const expected of assertableLines(block.body)) { asserted += 1; if (!haystack.includes(normalizeForMatch(expected))) { @@ -234,7 +268,9 @@ function verifyDoc( file: docRelPath, line: block.startLine, documented: expected, - context: captured.trim().split('\n').slice(-6).join('\n'), + context: scope + ? `(asserted against ${scope})` + : captured.trim().split('\n').slice(-6).join('\n'), }); } } From a8800654f217c1d912f398606e813c8105fe9f97 Mon Sep 17 00:00:00 2001 From: Byron Thanopoulos Date: Wed, 16 Sep 2026 20:01:58 +0100 Subject: [PATCH 3/3] test(teach): close attribution gaps in HTML figure matching Proofreading found the previous fix only held for blocks that happened to be attributed, and that attribution itself was unreliable. Two holes, both allowing a false pass: - Attribution was read from any line, including the `open / index.html` commands inside sh fences. Those are instructions to the reader, not attributions, so every later block inherited one and a block could be matched against a report it did not come from. Only prose that names a report on its own line counts now. - When a doc rendered several reports and a block named none of them, matching silently fell back to the union of all of them - the exact hole this work set out to close. That is now a failure. Docs that render a single report still need no attribution, since there is nothing to confuse. A block naming a report that was never rendered now fails too, instead of silently widening the haystack. --- scripts/verify-teaching-docs.ts | 36 ++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/scripts/verify-teaching-docs.ts b/scripts/verify-teaching-docs.ts index ad24fa6..f47c2b4 100644 --- a/scripts/verify-teaching-docs.ts +++ b/scripts/verify-teaching-docs.ts @@ -104,8 +104,11 @@ function extractBlocks(markdown: string): Block[] { } else { // Docs introduce a block by naming the file it came from, e.g. // `eval-dashboard-good/index.html`: — remember the most recent one. - const named = line.match(/`([^`]*index\.html)`/); - if (named) lastAttribution = named[1]; + // Only prose counts: the `open /index.html` commands inside sh + // fences are instructions to the reader, not attributions, and treating + // them as such would silently attribute every later block. + const named = line.match(/^\s*`([^`]*index\.html)`\s*:?\s*$/); + lastAttribution = named ? named[1] : lastAttribution; } continue; } @@ -253,9 +256,32 @@ function verifyDoc( // union would let the two sets of figures satisfy each other's assertions. let haystack = allText; let scope = ''; - if (source === 'html' && block.attributedTo) { - const match = [...byReport.entries()].find(([rel]) => rel.endsWith(block.attributedTo!)); - if (match) { + if (source === 'html') { + // With one rendered report there is nothing to confuse, so prose need not + // name it. With several, attribution is mandatory: falling back to the + // union lets one report's figures satisfy another's assertions, which is + // precisely the drift pm-02 exists to teach. Fail loudly instead. + if (!block.attributedTo && byReport.size > 1) { + drifts.push({ + file: docRelPath, + line: block.startLine, + documented: + '(this doc renders several reports, so name the one this block came from above it, e.g. `eval-dashboard-red/index.html`:)', + context: `(rendered reports: ${[...byReport.keys()].join(', ')})`, + }); + continue; + } + if (block.attributedTo) { + const match = [...byReport.entries()].find(([rel]) => rel.endsWith(block.attributedTo!)); + if (!match) { + drifts.push({ + file: docRelPath, + line: block.startLine, + documented: `(block attributed to ${block.attributedTo}, which this doc never rendered)`, + context: `(rendered reports: ${[...byReport.keys()].join(', ')})`, + }); + continue; + } haystack = normalizeForMatch(match[1]); scope = match[0]; }