test(teach): verify reading-track docs against rendered HTML

scripts/verify-teaching-docs.ts

@@ -17,7 +17,7 @@
  *   pnpm teach:verify --print  # also print captured stdout per exercise
  */
 import { execFileSync } from 'node:child_process';
-import { mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs';
+import { cpSync, mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 
@@ -71,34 +71,54 @@ const STANDALONE_EXERCISES: { file: string; needsExamples: boolean }[] = [
 ];
 
 /**
- * Not covered, deliberately: pm-01-reading-a-report.md and
- * pm-02-reading-drift.md. Their ```text blocks quote figures rendered into the
- * HTML report, not CLI stdout, and their steps call `open` to launch a browser.
- * Both were verified by hand on 2026-09-15 by extracting the text of the
- * generated index.html. Guarding them needs an HTML-aware assertion mode.
+ * Reading-track exercises for non-engineers. Unlike every other doc, their
+ * ```text blocks quote figures rendered into the HTML report rather than CLI
+ * stdout, so they are verified against the text of the generated index.html.
+ *
+ * Their steps also call `open` to launch a browser, which is stripped before
+ * replay (see runShellBlock) so CI never spawns a GUI.
  */
+const HTML_EXERCISES = ['pm-01-reading-a-report.md', 'pm-02-reading-drift.md'];
 
 interface Block {
   lang: string;
   body: string;
   startLine: number;
+  /** Nearest preceding inline-code path, e.g. `eval-dashboard-red/index.html`. */
+  attributedTo?: string;
 }
 
 /** Extract fenced code blocks with their 1-indexed opening line number. */
 function extractBlocks(markdown: string): Block[] {
   const lines = markdown.split('\n');
   const blocks: Block[] = [];
   let open: { lang: string; startLine: number; buf: string[] } | null = null;
+  let lastAttribution: string | undefined;
 
   for (let i = 0; i < lines.length; i += 1) {
     const line = lines[i];
     const fence = line.match(/^```(\w*)\s*$/);
     if (!fence) {
-      if (open) open.buf.push(line);
+      if (open) {
+        open.buf.push(line);
+      } else {
+        // Docs introduce a block by naming the file it came from, e.g.
+        // `eval-dashboard-good/index.html`: — remember the most recent one.
+        // Only prose counts: the `open <report>/index.html` commands inside sh
+        // fences are instructions to the reader, not attributions, and treating
+        // them as such would silently attribute every later block.
+        const named = line.match(/^\s*`([^`]*index\.html)`\s*:?\s*$/);
+        lastAttribution = named ? named[1] : lastAttribution;
+      }
       continue;
     }
     if (open) {
-      blocks.push({ lang: open.lang, body: open.buf.join('\n'), startLine: open.startLine });
+      blocks.push({
+        lang: open.lang,
+        body: open.buf.join('\n'),
+        startLine: open.startLine,
+        attributedTo: lastAttribution,
+      });
       open = null;
     } else {
       open = { lang: fence[1] || '', startLine: i + 1, buf: [] };
@@ -122,7 +142,9 @@ function runShellBlock(script: string, cwd: string): { stdout: string; code: num
   // Exercises are written for the published CLI; point them at this checkout.
   const rewritten = script
     .replace(/npx eval-dashboards/g, `"${tsxBin}" "${cliEntry}"`)
-    .replace(/pnpm cli:dev/g, `"${tsxBin}" "${cliEntry}"`);
+    .replace(/pnpm cli:dev/g, `"${tsxBin}" "${cliEntry}"`)
+    // Reading-track docs tell a human to open the report; never spawn a GUI in CI.
+    .replace(/^\s*open\s+.*$/gm, ':');
   try {
     const stdout = execFileSync('bash', ['-c', rewritten], {
       cwd,
@@ -136,18 +158,53 @@ function runShellBlock(script: string, cwd: string): { stdout: string; code: num
   }
 }
 
+/**
+ * Reduce an HTML document to its visible text, so documented figures can be
+ * matched against what a reader actually sees on the page.
+ *
+ * Script and style bodies are dropped first: they contain the report's own data
+ * as JSON, which would otherwise satisfy assertions the rendered page does not.
+ */
+function htmlToText(html: string): string {
+  return html
+    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&amp;/g, '&');
+}
+
+/**
+ * Collapse runs of whitespace so a documented figure matches regardless of how
+ * the renderer split it across elements: `Passed 8/13` becomes `Passed 8 /13`
+ * once tags are stripped, and header strips wrap arbitrarily.
+ */
+function normalizeForMatch(text: string): string {
+  return text.replace(/\s+/g, '');
+}
+
 interface Drift {
   file: string;
   line: number;
   documented: string;
   context: string;
 }
 
-/** Replay one doc's shell blocks and diff its ```text blocks against reality. */
+/**
+ * Replay one doc's shell blocks and diff its ```text blocks against reality.
+ *
+ * `source` selects what "reality" means: 'stdout' asserts against what the CLI
+ * printed, 'html' against the visible text of every report the block rendered.
+ */
 function verifyDoc(
   docRelPath: string,
   cwd: string,
   drifts: Drift[],
+  source: 'stdout' | 'html' = 'stdout',
 ): number {
   const markdown = readFileSync(path.join(repoRoot, docRelPath), 'utf8');
   const blocks = extractBlocks(markdown);
@@ -160,20 +217,86 @@ function verifyDoc(
     captured += stdout;
   }
 
+  // Per-report text, so a figure documented for one report cannot be satisfied
+  // by another. Keyed by the path the CLI printed, e.g. eval-dashboard/index.html.
+  const byReport = new Map<string, string>();
+
+  if (source === 'html') {
+    // The CLI prints each report's path; read back what it actually rendered.
+    const reports = captured
+      .split('\n')
+      .map((l) => l.trim())
+      .filter((l) => l.endsWith('index.html'));
+    if (reports.length === 0) {
+      drifts.push({
+        file: docRelPath,
+        line: 1,
+        documented: '(expected this doc to render at least one HTML report)',
+        context: captured.trim().split('\n').slice(-6).join('\n'),
+      });
+      return 0;
+    }
+    for (const rel of reports) {
+      byReport.set(rel, htmlToText(readFileSync(path.join(cwd, rel), 'utf8')));
+    }
+    captured = [...byReport.values()].join('\n');
+  }
+
   if (printMode) {
     process.stdout.write(`\n=== ${docRelPath} ===\n${captured}`);
   }
 
+  const allText = normalizeForMatch(captured);
+
   for (const block of blocks) {
     if (block.lang !== 'text') continue;
+
+    // When a doc names the report a block came from, assert against that report
+    // alone. pm-02 contrasts a good run with a red one; matching against the
+    // union would let the two sets of figures satisfy each other's assertions.
+    let haystack = allText;
+    let scope = '';
+    if (source === 'html') {
+      // With one rendered report there is nothing to confuse, so prose need not
+      // name it. With several, attribution is mandatory: falling back to the
+      // union lets one report's figures satisfy another's assertions, which is
+      // precisely the drift pm-02 exists to teach. Fail loudly instead.
+      if (!block.attributedTo && byReport.size > 1) {
+        drifts.push({
+          file: docRelPath,
+          line: block.startLine,
+          documented:
+            '(this doc renders several reports, so name the one this block came from above it, e.g. `eval-dashboard-red/index.html`:)',
+          context: `(rendered reports: ${[...byReport.keys()].join(', ')})`,
+        });
+        continue;
+      }
+      if (block.attributedTo) {
+        const match = [...byReport.entries()].find(([rel]) => rel.endsWith(block.attributedTo!));
+        if (!match) {
+          drifts.push({
+            file: docRelPath,
+            line: block.startLine,
+            documented: `(block attributed to ${block.attributedTo}, which this doc never rendered)`,
+            context: `(rendered reports: ${[...byReport.keys()].join(', ')})`,
+          });
+          continue;
+        }
+        haystack = normalizeForMatch(match[1]);
+        scope = match[0];
+      }
+    }
+
     for (const expected of assertableLines(block.body)) {
       asserted += 1;
-      if (!captured.includes(expected)) {
+      if (!haystack.includes(normalizeForMatch(expected))) {
         drifts.push({
           file: docRelPath,
           line: block.startLine,
           documented: expected,
-          context: captured.trim().split('\n').slice(-6).join('\n'),
+          context: scope
+            ? `(asserted against ${scope})`
+            : captured.trim().split('\n').slice(-6).join('\n'),
         });
       }
     }
@@ -211,15 +334,30 @@ function main(): number {
         rmSync(soloDir, { recursive: true, force: true });
       }
     }
+    // Reading-track exercises assert against rendered HTML, not stdout. They
+    // copy examples/ in so the docs' relative fixture paths resolve unmodified.
+    for (const file of HTML_EXERCISES) {
+      const htmlDir = mkdtempSync(path.join(tmpdir(), 'teach-html-'));
+      try {
+        cpSync(path.join(repoRoot, 'examples'), path.join(htmlDir, 'examples'), {
+          recursive: true,
+        });
+        assertedCount += verifyDoc(`docs/teach-exercises/${file}`, htmlDir, drifts, 'html');
+      } finally {
+        rmSync(htmlDir, { recursive: true, force: true });
+      }
+    }
   } finally {
     rmSync(workdir, { recursive: true, force: true });
   }
 
   if (drifts.length === 0) {
     const labCount = exercisesOnly ? 0 : LABS.length;
+    const exerciseCount =
+      CHAIN.length + STANDALONE_EXERCISES.length + HTML_EXERCISES.length;
     console.log(
       `Teaching docs verified: ${assertedCount} documented output line(s) across ` +
-        `${CHAIN.length + STANDALONE_EXERCISES.length} exercises and ${labCount} labs ` +
+        `${exerciseCount} exercises and ${labCount} labs ` +
         `match real CLI output.`,
     );
     return 0;