diff --git a/scripts/verify-teaching-docs.ts b/scripts/verify-teaching-docs.ts index 1cf9acf..f47c2b4 100644 --- a/scripts/verify-teaching-docs.ts +++ b/scripts/verify-teaching-docs.ts @@ -17,7 +17,7 @@ * pnpm teach:verify --print # also print captured stdout per exercise */ import { execFileSync } from 'node:child_process'; -import { mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs'; +import { cpSync, mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; @@ -71,17 +71,21 @@ const STANDALONE_EXERCISES: { file: string; needsExamples: boolean }[] = [ ]; /** - * Not covered, deliberately: pm-01-reading-a-report.md and - * pm-02-reading-drift.md. Their ```text blocks quote figures rendered into the - * HTML report, not CLI stdout, and their steps call `open` to launch a browser. - * Both were verified by hand on 2026-09-15 by extracting the text of the - * generated index.html. Guarding them needs an HTML-aware assertion mode. + * Reading-track exercises for non-engineers. Unlike every other doc, their + * ```text blocks quote figures rendered into the HTML report rather than CLI + * stdout, so they are verified against the text of the generated index.html. + * + * Their steps also call `open` to launch a browser, which is stripped before + * replay (see runShellBlock) so CI never spawns a GUI. */ +const HTML_EXERCISES = ['pm-01-reading-a-report.md', 'pm-02-reading-drift.md']; interface Block { lang: string; body: string; startLine: number; + /** Nearest preceding inline-code path, e.g. `eval-dashboard-red/index.html`. */ + attributedTo?: string; } /** Extract fenced code blocks with their 1-indexed opening line number. */ @@ -89,16 +93,32 @@ function extractBlocks(markdown: string): Block[] { const lines = markdown.split('\n'); const blocks: Block[] = []; let open: { lang: string; startLine: number; buf: string[] } | null = null; + let lastAttribution: string | undefined; for (let i = 0; i < lines.length; i += 1) { const line = lines[i]; const fence = line.match(/^```(\w*)\s*$/); if (!fence) { - if (open) open.buf.push(line); + if (open) { + open.buf.push(line); + } else { + // Docs introduce a block by naming the file it came from, e.g. + // `eval-dashboard-good/index.html`: — remember the most recent one. + // Only prose counts: the `open /index.html` commands inside sh + // fences are instructions to the reader, not attributions, and treating + // them as such would silently attribute every later block. + const named = line.match(/^\s*`([^`]*index\.html)`\s*:?\s*$/); + lastAttribution = named ? named[1] : lastAttribution; + } continue; } if (open) { - blocks.push({ lang: open.lang, body: open.buf.join('\n'), startLine: open.startLine }); + blocks.push({ + lang: open.lang, + body: open.buf.join('\n'), + startLine: open.startLine, + attributedTo: lastAttribution, + }); open = null; } else { open = { lang: fence[1] || '', startLine: i + 1, buf: [] }; @@ -122,7 +142,9 @@ function runShellBlock(script: string, cwd: string): { stdout: string; code: num // Exercises are written for the published CLI; point them at this checkout. const rewritten = script .replace(/npx eval-dashboards/g, `"${tsxBin}" "${cliEntry}"`) - .replace(/pnpm cli:dev/g, `"${tsxBin}" "${cliEntry}"`); + .replace(/pnpm cli:dev/g, `"${tsxBin}" "${cliEntry}"`) + // Reading-track docs tell a human to open the report; never spawn a GUI in CI. + .replace(/^\s*open\s+.*$/gm, ':'); try { const stdout = execFileSync('bash', ['-c', rewritten], { cwd, @@ -136,6 +158,35 @@ function runShellBlock(script: string, cwd: string): { stdout: string; code: num } } +/** + * Reduce an HTML document to its visible text, so documented figures can be + * matched against what a reader actually sees on the page. + * + * Script and style bodies are dropped first: they contain the report's own data + * as JSON, which would otherwise satisfy assertions the rendered page does not. + */ +function htmlToText(html: string): string { + return html + .replace(//gi, ' ') + .replace(//gi, ' ') + .replace(/<[^>]+>/g, ' ') + .replace(/ /g, ' ') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/&/g, '&'); +} + +/** + * Collapse runs of whitespace so a documented figure matches regardless of how + * the renderer split it across elements: `Passed 8/13` becomes `Passed 8 /13` + * once tags are stripped, and header strips wrap arbitrarily. + */ +function normalizeForMatch(text: string): string { + return text.replace(/\s+/g, ''); +} + interface Drift { file: string; line: number; @@ -143,11 +194,17 @@ interface Drift { context: string; } -/** Replay one doc's shell blocks and diff its ```text blocks against reality. */ +/** + * Replay one doc's shell blocks and diff its ```text blocks against reality. + * + * `source` selects what "reality" means: 'stdout' asserts against what the CLI + * printed, 'html' against the visible text of every report the block rendered. + */ function verifyDoc( docRelPath: string, cwd: string, drifts: Drift[], + source: 'stdout' | 'html' = 'stdout', ): number { const markdown = readFileSync(path.join(repoRoot, docRelPath), 'utf8'); const blocks = extractBlocks(markdown); @@ -160,20 +217,86 @@ function verifyDoc( captured += stdout; } + // Per-report text, so a figure documented for one report cannot be satisfied + // by another. Keyed by the path the CLI printed, e.g. eval-dashboard/index.html. + const byReport = new Map(); + + if (source === 'html') { + // The CLI prints each report's path; read back what it actually rendered. + const reports = captured + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.endsWith('index.html')); + if (reports.length === 0) { + drifts.push({ + file: docRelPath, + line: 1, + documented: '(expected this doc to render at least one HTML report)', + context: captured.trim().split('\n').slice(-6).join('\n'), + }); + return 0; + } + for (const rel of reports) { + byReport.set(rel, htmlToText(readFileSync(path.join(cwd, rel), 'utf8'))); + } + captured = [...byReport.values()].join('\n'); + } + if (printMode) { process.stdout.write(`\n=== ${docRelPath} ===\n${captured}`); } + const allText = normalizeForMatch(captured); + for (const block of blocks) { if (block.lang !== 'text') continue; + + // When a doc names the report a block came from, assert against that report + // alone. pm-02 contrasts a good run with a red one; matching against the + // union would let the two sets of figures satisfy each other's assertions. + let haystack = allText; + let scope = ''; + if (source === 'html') { + // With one rendered report there is nothing to confuse, so prose need not + // name it. With several, attribution is mandatory: falling back to the + // union lets one report's figures satisfy another's assertions, which is + // precisely the drift pm-02 exists to teach. Fail loudly instead. + if (!block.attributedTo && byReport.size > 1) { + drifts.push({ + file: docRelPath, + line: block.startLine, + documented: + '(this doc renders several reports, so name the one this block came from above it, e.g. `eval-dashboard-red/index.html`:)', + context: `(rendered reports: ${[...byReport.keys()].join(', ')})`, + }); + continue; + } + if (block.attributedTo) { + const match = [...byReport.entries()].find(([rel]) => rel.endsWith(block.attributedTo!)); + if (!match) { + drifts.push({ + file: docRelPath, + line: block.startLine, + documented: `(block attributed to ${block.attributedTo}, which this doc never rendered)`, + context: `(rendered reports: ${[...byReport.keys()].join(', ')})`, + }); + continue; + } + haystack = normalizeForMatch(match[1]); + scope = match[0]; + } + } + for (const expected of assertableLines(block.body)) { asserted += 1; - if (!captured.includes(expected)) { + if (!haystack.includes(normalizeForMatch(expected))) { drifts.push({ file: docRelPath, line: block.startLine, documented: expected, - context: captured.trim().split('\n').slice(-6).join('\n'), + context: scope + ? `(asserted against ${scope})` + : captured.trim().split('\n').slice(-6).join('\n'), }); } } @@ -211,15 +334,30 @@ function main(): number { rmSync(soloDir, { recursive: true, force: true }); } } + // Reading-track exercises assert against rendered HTML, not stdout. They + // copy examples/ in so the docs' relative fixture paths resolve unmodified. + for (const file of HTML_EXERCISES) { + const htmlDir = mkdtempSync(path.join(tmpdir(), 'teach-html-')); + try { + cpSync(path.join(repoRoot, 'examples'), path.join(htmlDir, 'examples'), { + recursive: true, + }); + assertedCount += verifyDoc(`docs/teach-exercises/${file}`, htmlDir, drifts, 'html'); + } finally { + rmSync(htmlDir, { recursive: true, force: true }); + } + } } finally { rmSync(workdir, { recursive: true, force: true }); } if (drifts.length === 0) { const labCount = exercisesOnly ? 0 : LABS.length; + const exerciseCount = + CHAIN.length + STANDALONE_EXERCISES.length + HTML_EXERCISES.length; console.log( `Teaching docs verified: ${assertedCount} documented output line(s) across ` + - `${CHAIN.length + STANDALONE_EXERCISES.length} exercises and ${labCount} labs ` + + `${exerciseCount} exercises and ${labCount} labs ` + `match real CLI output.`, ); return 0;