Raise or parse correctly where the docs Markdown parser silently mangled input

docs/specs/website-docs.md

@@ -165,6 +165,9 @@ therefore supports a deliberate *subset* of CommonMark and raises
 general parser would. The public-doc lint turns that error into a build
 failure, which is what makes a hand-rolled parser safe as the guide grows.
 
+**Headings are ATX only.** **A fence closes only on a run of its own marker at
+least as long as the opener's**, so a longer outer fence nests an inner one.
+
 Raw HTML is disabled except for a narrow `<img>` allowlist carrying only `src`,
 `alt`, `width`, `height`, and `title`, with an `https:` source. Every other tag,
 and every other attribute on `<img>`, is rejected outright. The exception exists

website/scripts/docs-parser.js

@@ -222,6 +222,33 @@ export function parseInline(text, line) {
   return nodes;
 }
 
+/**
+ * The index of the `)` that closes the destination opened at `open`, or -1.
+ *
+ * Parentheses nest rather than closing at the first `)`, because a URL is
+ * allowed to contain a balanced pair and several the docs link to do
+ * (`.../wiki/Foo_(bar)`). Stopping at the first one truncates the href and
+ * still emits a link, so the page ships a live anchor pointing somewhere else
+ * — exactly the silent mangling this parser exists to avoid. A `"` toggles a
+ * title span, where a paren is literal.
+ */
+function matchDestination(text, open) {
+  let depth = 1;
+  let inTitle = false;
+  for (let i = open + 1; i < text.length; i++) {
+    const ch = text[i];
+    if (ch === '\\') { i++; continue; }
+    if (ch === '"') { inTitle = !inTitle; continue; }
+    if (inTitle) continue;
+    if (ch === '(') depth++;
+    else if (ch === ')') {
+      depth--;
+      if (depth === 0) return i;
+    }
+  }
+  return -1;
+}
+
 /** Match `[label](href "title")` starting at `start`. Returns null if not one. */
 function matchLink(text, start) {
   if (text[start] !== '[') return null;
@@ -237,7 +264,7 @@ function matchLink(text, start) {
   }
   if (depth !== 0 || text[i + 1] !== '(') return null;
   const label = text.slice(start + 1, i);
-  const close = text.indexOf(')', i + 2);
+  const close = matchDestination(text, i + 1);
   if (close === -1) return null;
   const target = text.slice(i + 2, close).trim();
   const titleMatch = /^(\S+)\s+"([^"]*)"$/.exec(target);
@@ -252,8 +279,57 @@ function matchLink(text, start) {
 
 const LIST_ITEM = /^(\s*)([-*+]|\d+[.)])\s+(.*)$/;
 
-/** Closing-fence matchers, one per marker character, compiled once. */
-const FENCE_CLOSE = { '`': /^\s*`{3,}\s*$/, '~': /^\s*~{3,}\s*$/ };
+/**
+ * Whether `line` closes a fence opened with `length` copies of `marker`.
+ *
+ * The closing run must be at least as long as the opening one. Without that
+ * length test a ````-fenced block closes at the first ``` inside it — the very
+ * form AGENTS.md prescribes for nesting one code block inside another — so the
+ * block's body spills out and renders as prose, with no error to notice.
+ */
+function closesFence(line, marker, length) {
+  const trimmed = line.trim();
+  return trimmed.length >= length && [...trimmed].every((ch) => ch === marker);
+}
+
+/**
+ * Whether `line` begins a new block, so a paragraph — or a blockquote's lazy
+ * continuation — ends before it instead of swallowing it. `next` supplies the
+ * single line of lookahead a table needs, a `|` row being a table only when a
+ * delimiter row follows it.
+ *
+ * Six of the seven block starts are here: ATX heading, list item, blockquote,
+ * fence, table, and block-level raw HTML. A standalone `<img>` is deliberately
+ * absent — it is inline-level, so GitHub keeps it in the paragraph and so do
+ * we. The seventh, the thematic break, is tested separately at each call site
+ * because `---` is also a setext underline, which has to raise first.
+ */
+function interruptsParagraph(line, next) {
+  return /^(#{1,6})\s+/.test(line)
+    || LIST_ITEM.test(line)
+    || /^\s*>\s?/.test(line)
+    || /^(\s*)(`{3,}|~{3,})/.test(line)
+    || (line.includes('|') && isDelimiterRow(next ?? ''))
+    || (/^\s*<\/?[a-zA-Z]/.test(line) && !/^\s*<img\b/i.test(line));
+}
+
+/**
+ * A setext underline (`===` / `---` under a line of text). GitHub renders one
+ * as a heading; this parser has no block for it, so left alone the underline
+ * joins the paragraph and ships as the literal text `Title ===`. Rejected
+ * rather than supported: `#` is the only heading form the sources use, and an
+ * error names the line instead of leaving a reader to spot the difference.
+ */
+const SETEXT_UNDERLINE = /^\s*(=+|-+)\s*$/;
+
+/**
+ * A thematic break (`***`, `___`, `- - -`). CommonMark lets one interrupt a
+ * paragraph, so without this the line joins the paragraph and `parseInline`
+ * eats the run as an empty emphasis span — `***` ships as a dropped rule and
+ * two lost characters. Checked after `SETEXT_UNDERLINE`, never before: `---`
+ * matches both, and GitHub reads it as a setext heading.
+ */
+const THEMATIC_BREAK = /^\s*([-*_])(\s*\1){2,}\s*$/;
 
 /**
  * Split a table row into cells, honouring backslash-escaped pipes so a cell
@@ -274,8 +350,14 @@ function splitRow(row, line) {
   return cells.map((c) => parseInline(c.trim(), line));
 }
 
+/**
+ * A GFM delimiter row (`| --- | :-- |`). The pipe is required: every `|` in the
+ * pattern is optional, so without this test a bare `---` reads as a one-column
+ * delimiter row and a preceding line that merely contains a pipe becomes a
+ * table — where the `---` is a setext underline, which has to raise instead.
+ */
 function isDelimiterRow(row) {
-  return /^\s*\|?\s*:?-{1,}:?\s*(\|\s*:?-{1,}:?\s*)*\|?\s*$/.test(row);
+  return row.includes('|') && /^\s*\|?\s*:?-{1,}:?\s*(\|\s*:?-{1,}:?\s*)*\|?\s*$/.test(row);
 }
 
 function alignmentsFrom(row) {
@@ -310,13 +392,14 @@ export function parseMarkdown(markdown, options = {}) {
     // Fenced code
     const fence = /^(\s*)(`{3,}|~{3,})\s*([\w+-]*)\s*$/.exec(raw);
     if (fence) {
-      const closeRe = FENCE_CLOSE[fence[2][0]];
+      const marker = fence[2][0];
+      const openLength = fence[2].length;
       const lang = fence[3] || null;
       const body = [];
       i++;
       let closed = false;
       while (i < lines.length) {
-        if (closeRe.test(lines[i])) { closed = true; i++; break; }
+        if (closesFence(lines[i], marker, openLength)) { closed = true; i++; break; }
         body.push(lines[i]);
         i++;
       }
@@ -339,7 +422,7 @@ export function parseMarkdown(markdown, options = {}) {
     }
 
     // Thematic break
-    if (/^\s*([-*_])(\s*\1){2,}\s*$/.test(raw)) {
+    if (THEMATIC_BREAK.test(raw)) {
       blocks.push({ type: 'thematicBreak' });
       i++;
       continue;
@@ -362,7 +445,13 @@ export function parseMarkdown(markdown, options = {}) {
     // Blockquote
     if (/^\s*>\s?/.test(raw)) {
       const body = [];
-      while (i < lines.length && (/^\s*>\s?/.test(lines[i]) || (lines[i].trim() !== '' && body.length > 0))) {
+      while (
+        i < lines.length
+        && (/^\s*>\s?/.test(lines[i])
+          || (lines[i].trim() !== '' && body.length > 0
+            && !interruptsParagraph(lines[i], lines[i + 1])
+            && !THEMATIC_BREAK.test(lines[i])))
+      ) {
         body.push(lines[i].replace(/^\s*>\s?/, ''));
         i++;
       }
@@ -395,7 +484,11 @@ export function parseMarkdown(markdown, options = {}) {
     const para = [];
     while (i < lines.length && lines[i].trim() !== '') {
       const l = lines[i];
-      if (para.length > 0 && (/^(#{1,6})\s+/.test(l) || LIST_ITEM.test(l) || /^\s*>\s?/.test(l) || /^(\s*)(`{3,}|~{3,})/.test(l))) break;
+      if (para.length > 0 && interruptsParagraph(l, lines[i + 1])) break;
+      if (para.length > 0 && SETEXT_UNDERLINE.test(l)) {
+        throw new UnsupportedMarkdownError('setext heading underline — use an ATX `#` heading', i + 1);
+      }
+      if (para.length > 0 && THEMATIC_BREAK.test(l)) break;
       para.push(l.trim());
       i++;
     }
@@ -446,6 +539,12 @@ function parseList(lines, start, slug) {
     i++;
     // Continuation lines: indented further, not themselves list items.
     while (i < lines.length && lines[i].trim() !== '' && !LIST_ITEM.test(lines[i]) && /^\s+/.test(lines[i])) {
+      // The same two guards a paragraph applies, so `Headings are ATX only`
+      // holds inside a list item too rather than shipping as literal text.
+      if (SETEXT_UNDERLINE.test(lines[i])) {
+        throw new UnsupportedMarkdownError('setext heading underline — use an ATX `#` heading', i + 1);
+      }
+      if (THEMATIC_BREAK.test(lines[i])) break;
       contentLines.push(lines[i].trim());
       i++;
     }

website/scripts/docs-parser.test.js

@@ -30,6 +30,17 @@ describe('inline', () => {
     expect(nodes.map((n) => n.type)).toEqual(['text', 'code', 'text', 'link', 'text', 'strong']);
   });
 
+  it('keeps a balanced paren inside a link destination', () => {
+    const nodes = parseInline('see [Foo](https://x.test/wiki/Foo_(bar)) now');
+    expect(nodes[1]).toMatchObject({ type: 'link', href: 'https://x.test/wiki/Foo_(bar)' });
+    expect(nodes[2]).toMatchObject({ type: 'text', value: ' now' });
+  });
+
+  it('keeps a paren inside a link title out of the destination scan', () => {
+    const nodes = parseInline('[a](/x "the (title)") end');
+    expect(nodes[0]).toMatchObject({ type: 'link', href: '/x', title: 'the (title)' });
+  });
+
   it('honours backslash escapes', () => {
     expect(inlineToText(parseInline('a \\| b'))).toBe('a | b');
   });
@@ -96,6 +107,76 @@ describe('blocks', () => {
     expect(() => parseMarkdown('```\nnope\n')).toThrow(/unterminated fenced code/);
   });
 
+  it('keeps a shorter fence inside a longer one as code, not prose', () => {
+    const md = '````markdown\n```bash\necho hi\n```\n````\n';
+    const { blocks } = parseMarkdown(md);
+    expect(blocks).toHaveLength(1);
+    expect(blocks[0]).toEqual({ type: 'code', lang: 'markdown', value: '```bash\necho hi\n```' });
+  });
+
+  it('rejects a setext underline instead of shipping it as text', () => {
+    expect(() => parseMarkdown('Title\n=====\n')).toThrow(UnsupportedMarkdownError);
+    expect(() => parseMarkdown('Title\n-----\n')).toThrow(/setext heading underline/);
+  });
+
+  it('ends a blockquote at a line that starts a new block', () => {
+    const { blocks } = parseMarkdown('> quoted\n# Heading\n');
+    expect(blocks.map((b) => b.type)).toEqual(['blockquote', 'heading']);
+    expect(blocks[1].id).toBe('heading');
+  });
+
+  it('still folds a lazy continuation line into the blockquote', () => {
+    const { blocks } = parseMarkdown('> quoted\ncontinues\n');
+    expect(blocks).toHaveLength(1);
+    expect(inlineToText(blocks[0].children[0].children)).toBe('quoted continues');
+  });
+
+  it('breaks a paragraph at a thematic break instead of eating it as emphasis', () => {
+    const { blocks } = parseMarkdown('Some text\n***\nNext\n');
+    expect(blocks.map((b) => b.type)).toEqual(['paragraph', 'thematicBreak', 'paragraph']);
+    expect(inlineToText(blocks[0].children)).toBe('Some text');
+    expect(inlineToText(blocks[2].children)).toBe('Next');
+  });
+
+  it('still reads a dashed underline as a setext heading, not a thematic break', () => {
+    expect(() => parseMarkdown('Title\n---\n')).toThrow(/setext heading underline/);
+  });
+
+  it('ends a blockquote at a thematic break rather than raising a setext error', () => {
+    const { blocks } = parseMarkdown('> quoted\n---\n');
+    expect(blocks.map((b) => b.type)).toEqual(['blockquote', 'thematicBreak']);
+  });
+
+  it('ends a blockquote at a table rather than swallowing its rows', () => {
+    const { blocks } = parseMarkdown('> quoted\n| a | b |\n|---|---|\n| 1 | 2 |\n');
+    expect(blocks.map((b) => b.type)).toEqual(['blockquote', 'table']);
+    expect(blocks[1].header.map(inlineToText)).toEqual(['a', 'b']);
+  });
+
+  it('ends a blockquote at block-level raw HTML, which then raises', () => {
+    expect(() => parseMarkdown('> quoted\n<div>\n')).toThrow(/raw HTML <div>/);
+  });
+
+  it('keeps a standalone <img> line inside the paragraph it follows', () => {
+    const { blocks } = parseMarkdown('text\n<img src="a.png" alt="a">\n');
+    expect(blocks).toHaveLength(1);
+    expect(blocks[0].children.map((n) => n.type)).toEqual(['text', 'image']);
+  });
+
+  it('rejects a setext underline inside a list item too', () => {
+    expect(() => parseMarkdown('- item\n  Title\n  =====\n')).toThrow(/setext heading underline/);
+  });
+
+  it('ends a list item at a thematic break in its continuation', () => {
+    const { blocks } = parseMarkdown('- item\n  ***\n');
+    expect(blocks.map((b) => b.type)).toEqual(['list', 'thematicBreak']);
+  });
+
+  it('reads a pipeless dashed underline as a setext heading, not a table delimiter row', () => {
+    expect(() => parseMarkdown('Intro\na | b\n---\n')).toThrow(/setext heading underline/);
+    expect(() => parseMarkdown('a | b\n---\n')).toThrow(/setext heading underline/);
+  });
+
   it('parses a table with an escaped pipe inside inline code', () => {
     const md = '| Key | Action |\n|-----|--------|\n| `\\|` or tmux `%` | Split |\n';
     const { blocks } = parseMarkdown(md);