From fe1e6b17f0554f67ce5cd9d6c14174aeb07eed20 Mon Sep 17 00:00:00 2001 From: dormouse-bot <287024035+dormouse-bot@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:23:11 +0000 Subject: [PATCH 1/4] fix(website): raise or parse correctly where the docs parser silently mangled The parser's contract is that anything outside its CommonMark subset is a hard error, not a silent degradation. Four constructs slipped through it: - a fence closed on a shorter run than its opener, so a ````-fenced block ended at the first ``` inside it and spilled its body into the page as prose -- the nesting form AGENTS.md prescribes; - a link destination stopped at the first `)`, truncating a URL carrying a balanced pair and still emitting a live anchor; - a blockquote's lazy continuation swallowed the heading, list, or fence that followed it, which also dropped that heading from the page's TOC; - a setext underline joined the paragraph and shipped as literal text. Generated docs data is byte-identical before and after, so nothing published today changes. --- docs/specs/website-docs.md | 3 ++ website/scripts/docs-parser.js | 79 ++++++++++++++++++++++++++--- website/scripts/docs-parser.test.js | 35 +++++++++++++ 3 files changed, 110 insertions(+), 7 deletions(-) diff --git a/docs/specs/website-docs.md b/docs/specs/website-docs.md index 7c8e6ab0..2fac1897 100644 --- a/docs/specs/website-docs.md +++ b/docs/specs/website-docs.md @@ -165,6 +165,9 @@ therefore supports a deliberate *subset* of CommonMark and raises general parser would. The public-doc lint turns that error into a build failure, which is what makes a hand-rolled parser safe as the guide grows. +**Headings are ATX only.** **A fence closes only on a run of its own marker at +least as long as the opener's**, so a longer outer fence nests an inner one. + Raw HTML is disabled except for a narrow `` allowlist carrying only `src`, `alt`, `width`, `height`, and `title`, with an `https:` source. Every other tag, and every other attribute on ``, is rejected outright. The exception exists diff --git a/website/scripts/docs-parser.js b/website/scripts/docs-parser.js index b59351c7..182d764c 100644 --- a/website/scripts/docs-parser.js +++ b/website/scripts/docs-parser.js @@ -222,6 +222,33 @@ export function parseInline(text, line) { return nodes; } +/** + * The index of the `)` that closes the destination opened at `open`, or -1. + * + * Parentheses nest rather than closing at the first `)`, because a URL is + * allowed to contain a balanced pair and several the docs link to do + * (`.../wiki/Foo_(bar)`). Stopping at the first one truncates the href and + * still emits a link, so the page ships a live anchor pointing somewhere else + * — exactly the silent mangling this parser exists to avoid. A `"` toggles a + * title span, where a paren is literal. + */ +function matchDestination(text, open) { + let depth = 1; + let inTitle = false; + for (let i = open + 1; i < text.length; i++) { + const ch = text[i]; + if (ch === '\\') { i++; continue; } + if (ch === '"') { inTitle = !inTitle; continue; } + if (inTitle) continue; + if (ch === '(') depth++; + else if (ch === ')') { + depth--; + if (depth === 0) return i; + } + } + return -1; +} + /** Match `[label](href "title")` starting at `start`. Returns null if not one. */ function matchLink(text, start) { if (text[start] !== '[') return null; @@ -237,7 +264,7 @@ function matchLink(text, start) { } if (depth !== 0 || text[i + 1] !== '(') return null; const label = text.slice(start + 1, i); - const close = text.indexOf(')', i + 2); + const close = matchDestination(text, i + 1); if (close === -1) return null; const target = text.slice(i + 2, close).trim(); const titleMatch = /^(\S+)\s+"([^"]*)"$/.exec(target); @@ -252,8 +279,38 @@ function matchLink(text, start) { const LIST_ITEM = /^(\s*)([-*+]|\d+[.)])\s+(.*)$/; -/** Closing-fence matchers, one per marker character, compiled once. */ -const FENCE_CLOSE = { '`': /^\s*`{3,}\s*$/, '~': /^\s*~{3,}\s*$/ }; +/** + * Whether `line` closes a fence opened with `length` copies of `marker`. + * + * The closing run must be at least as long as the opening one. Without that + * length test a ````-fenced block closes at the first ``` inside it — the very + * form AGENTS.md prescribes for nesting one code block inside another — so the + * block's body spills out and renders as prose, with no error to notice. + */ +function closesFence(line, marker, length) { + const trimmed = line.trim(); + return trimmed.length >= length && [...trimmed].every((ch) => ch === marker); +} + +/** + * Lines that begin a new block, so a paragraph — or a blockquote's lazy + * continuation — ends before them instead of swallowing them. + */ +function interruptsParagraph(line) { + return /^(#{1,6})\s+/.test(line) + || LIST_ITEM.test(line) + || /^\s*>\s?/.test(line) + || /^(\s*)(`{3,}|~{3,})/.test(line); +} + +/** + * A setext underline (`===` / `---` under a line of text). GitHub renders one + * as a heading; this parser has no block for it, so left alone the underline + * joins the paragraph and ships as the literal text `Title ===`. Rejected + * rather than supported: `#` is the only heading form the sources use, and an + * error names the line instead of leaving a reader to spot the difference. + */ +const SETEXT_UNDERLINE = /^\s*(=+|-+)\s*$/; /** * Split a table row into cells, honouring backslash-escaped pipes so a cell @@ -310,13 +367,14 @@ export function parseMarkdown(markdown, options = {}) { // Fenced code const fence = /^(\s*)(`{3,}|~{3,})\s*([\w+-]*)\s*$/.exec(raw); if (fence) { - const closeRe = FENCE_CLOSE[fence[2][0]]; + const marker = fence[2][0]; + const openLength = fence[2].length; const lang = fence[3] || null; const body = []; i++; let closed = false; while (i < lines.length) { - if (closeRe.test(lines[i])) { closed = true; i++; break; } + if (closesFence(lines[i], marker, openLength)) { closed = true; i++; break; } body.push(lines[i]); i++; } @@ -362,7 +420,11 @@ export function parseMarkdown(markdown, options = {}) { // Blockquote if (/^\s*>\s?/.test(raw)) { const body = []; - while (i < lines.length && (/^\s*>\s?/.test(lines[i]) || (lines[i].trim() !== '' && body.length > 0))) { + while ( + i < lines.length + && (/^\s*>\s?/.test(lines[i]) + || (lines[i].trim() !== '' && body.length > 0 && !interruptsParagraph(lines[i]))) + ) { body.push(lines[i].replace(/^\s*>\s?/, '')); i++; } @@ -395,7 +457,10 @@ export function parseMarkdown(markdown, options = {}) { const para = []; while (i < lines.length && lines[i].trim() !== '') { const l = lines[i]; - if (para.length > 0 && (/^(#{1,6})\s+/.test(l) || LIST_ITEM.test(l) || /^\s*>\s?/.test(l) || /^(\s*)(`{3,}|~{3,})/.test(l))) break; + if (para.length > 0 && interruptsParagraph(l)) break; + if (para.length > 0 && SETEXT_UNDERLINE.test(l)) { + throw new UnsupportedMarkdownError('setext heading underline — use an ATX `#` heading', i + 1); + } para.push(l.trim()); i++; } diff --git a/website/scripts/docs-parser.test.js b/website/scripts/docs-parser.test.js index 93e8f28b..b8e848ca 100644 --- a/website/scripts/docs-parser.test.js +++ b/website/scripts/docs-parser.test.js @@ -30,6 +30,17 @@ describe('inline', () => { expect(nodes.map((n) => n.type)).toEqual(['text', 'code', 'text', 'link', 'text', 'strong']); }); + it('keeps a balanced paren inside a link destination', () => { + const nodes = parseInline('see [Foo](https://x.test/wiki/Foo_(bar)) now'); + expect(nodes[1]).toMatchObject({ type: 'link', href: 'https://x.test/wiki/Foo_(bar)' }); + expect(nodes[2]).toMatchObject({ type: 'text', value: ' now' }); + }); + + it('keeps a paren inside a link title out of the destination scan', () => { + const nodes = parseInline('[a](/x "the (title)") end'); + expect(nodes[0]).toMatchObject({ type: 'link', href: '/x', title: 'the (title)' }); + }); + it('honours backslash escapes', () => { expect(inlineToText(parseInline('a \\| b'))).toBe('a | b'); }); @@ -96,6 +107,30 @@ describe('blocks', () => { expect(() => parseMarkdown('```\nnope\n')).toThrow(/unterminated fenced code/); }); + it('keeps a shorter fence inside a longer one as code, not prose', () => { + const md = '````markdown\n```bash\necho hi\n```\n````\n'; + const { blocks } = parseMarkdown(md); + expect(blocks).toHaveLength(1); + expect(blocks[0]).toEqual({ type: 'code', lang: 'markdown', value: '```bash\necho hi\n```' }); + }); + + it('rejects a setext underline instead of shipping it as text', () => { + expect(() => parseMarkdown('Title\n=====\n')).toThrow(UnsupportedMarkdownError); + expect(() => parseMarkdown('Title\n-----\n')).toThrow(/setext heading underline/); + }); + + it('ends a blockquote at a line that starts a new block', () => { + const { blocks } = parseMarkdown('> quoted\n# Heading\n'); + expect(blocks.map((b) => b.type)).toEqual(['blockquote', 'heading']); + expect(blocks[1].id).toBe('heading'); + }); + + it('still folds a lazy continuation line into the blockquote', () => { + const { blocks } = parseMarkdown('> quoted\ncontinues\n'); + expect(blocks).toHaveLength(1); + expect(inlineToText(blocks[0].children[0].children)).toBe('quoted continues'); + }); + it('parses a table with an escaped pipe inside inline code', () => { const md = '| Key | Action |\n|-----|--------|\n| `\\|` or tmux `%` | Split |\n'; const { blocks } = parseMarkdown(md); From cea9b88a62aad89303bef694d9d86108085dcc64 Mon Sep 17 00:00:00 2001 From: dormouse-bot <287024035+dormouse-bot@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:36:41 +0000 Subject: [PATCH 2/4] fix(website): close the remaining silent-mangling gaps the review found A thematic break interrupting a paragraph was folded into it, and parseInline then ate the run as an empty emphasis span -- `***` shipped as a dropped rule and two lost characters, the same shape as the setext case. The paragraph loop now breaks on it, after the setext throw so `---` keeps raising rather than becoming a rule, and the blockquote loop ends the quote there so `> quoted` followed by `---` no longer raises an error naming a construct the author did not write. interruptsParagraph covered four of the seven block starts while its docstring claimed all of them, so a table or a block-level raw HTML tag directly under a `> ` line was still swallowed whole. It now takes one line of lookahead -- a `|` row is a table only when a delimiter row follows -- and covers six; a standalone `` stays out because it is inline-level, and the thematic break stays out because it must be tested after the setext underline. `Headings are ATX only` was enforced in paragraphs and blockquote bodies but not in list-item continuations, which joined straight into parseInline, so `- item` / `Title` / `=====` shipped as literal text. The same two guards now apply there. Generated docs data stays byte-identical. --- website/scripts/docs-parser.js | 42 ++++++++++++++++++++++++----- website/scripts/docs-parser.test.js | 41 ++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 7 deletions(-) diff --git a/website/scripts/docs-parser.js b/website/scripts/docs-parser.js index 182d764c..80ff6ebd 100644 --- a/website/scripts/docs-parser.js +++ b/website/scripts/docs-parser.js @@ -293,14 +293,24 @@ function closesFence(line, marker, length) { } /** - * Lines that begin a new block, so a paragraph — or a blockquote's lazy - * continuation — ends before them instead of swallowing them. + * Whether `line` begins a new block, so a paragraph — or a blockquote's lazy + * continuation — ends before it instead of swallowing it. `next` supplies the + * single line of lookahead a table needs, a `|` row being a table only when a + * delimiter row follows it. + * + * Six of the seven block starts are here: ATX heading, list item, blockquote, + * fence, table, and block-level raw HTML. A standalone `` is deliberately + * absent — it is inline-level, so GitHub keeps it in the paragraph and so do + * we. The seventh, the thematic break, is tested separately at each call site + * because `---` is also a setext underline, which has to raise first. */ -function interruptsParagraph(line) { +function interruptsParagraph(line, next) { return /^(#{1,6})\s+/.test(line) || LIST_ITEM.test(line) || /^\s*>\s?/.test(line) - || /^(\s*)(`{3,}|~{3,})/.test(line); + || /^(\s*)(`{3,}|~{3,})/.test(line) + || (line.includes('|') && isDelimiterRow(next ?? '')) + || (/^\s*<\/?[a-zA-Z]/.test(line) && !/^\s*\s?/.test(lines[i]) - || (lines[i].trim() !== '' && body.length > 0 && !interruptsParagraph(lines[i]))) + || (lines[i].trim() !== '' && body.length > 0 + && !interruptsParagraph(lines[i], lines[i + 1]) + && !THEMATIC_BREAK.test(lines[i]))) ) { body.push(lines[i].replace(/^\s*>\s?/, '')); i++; @@ -457,10 +478,11 @@ export function parseMarkdown(markdown, options = {}) { const para = []; while (i < lines.length && lines[i].trim() !== '') { const l = lines[i]; - if (para.length > 0 && interruptsParagraph(l)) break; + if (para.length > 0 && interruptsParagraph(l, lines[i + 1])) break; if (para.length > 0 && SETEXT_UNDERLINE.test(l)) { throw new UnsupportedMarkdownError('setext heading underline — use an ATX `#` heading', i + 1); } + if (para.length > 0 && THEMATIC_BREAK.test(l)) break; para.push(l.trim()); i++; } @@ -511,6 +533,12 @@ function parseList(lines, start, slug) { i++; // Continuation lines: indented further, not themselves list items. while (i < lines.length && lines[i].trim() !== '' && !LIST_ITEM.test(lines[i]) && /^\s+/.test(lines[i])) { + // The same two guards a paragraph applies, so `Headings are ATX only` + // holds inside a list item too rather than shipping as literal text. + if (SETEXT_UNDERLINE.test(lines[i])) { + throw new UnsupportedMarkdownError('setext heading underline — use an ATX `#` heading', i + 1); + } + if (THEMATIC_BREAK.test(lines[i])) break; contentLines.push(lines[i].trim()); i++; } diff --git a/website/scripts/docs-parser.test.js b/website/scripts/docs-parser.test.js index b8e848ca..12c8f7da 100644 --- a/website/scripts/docs-parser.test.js +++ b/website/scripts/docs-parser.test.js @@ -131,6 +131,47 @@ describe('blocks', () => { expect(inlineToText(blocks[0].children[0].children)).toBe('quoted continues'); }); + it('breaks a paragraph at a thematic break instead of eating it as emphasis', () => { + const { blocks } = parseMarkdown('Some text\n***\nNext\n'); + expect(blocks.map((b) => b.type)).toEqual(['paragraph', 'thematicBreak', 'paragraph']); + expect(inlineToText(blocks[0].children)).toBe('Some text'); + expect(inlineToText(blocks[2].children)).toBe('Next'); + }); + + it('still reads a dashed underline as a setext heading, not a thematic break', () => { + expect(() => parseMarkdown('Title\n---\n')).toThrow(/setext heading underline/); + }); + + it('ends a blockquote at a thematic break rather than raising a setext error', () => { + const { blocks } = parseMarkdown('> quoted\n---\n'); + expect(blocks.map((b) => b.type)).toEqual(['blockquote', 'thematicBreak']); + }); + + it('ends a blockquote at a table rather than swallowing its rows', () => { + const { blocks } = parseMarkdown('> quoted\n| a | b |\n|---|---|\n| 1 | 2 |\n'); + expect(blocks.map((b) => b.type)).toEqual(['blockquote', 'table']); + expect(blocks[1].header.map(inlineToText)).toEqual(['a', 'b']); + }); + + it('ends a blockquote at block-level raw HTML, which then raises', () => { + expect(() => parseMarkdown('> quoted\n
\n')).toThrow(/raw HTML
/); + }); + + it('keeps a standalone line inside the paragraph it follows', () => { + const { blocks } = parseMarkdown('text\na\n'); + expect(blocks).toHaveLength(1); + expect(blocks[0].children.map((n) => n.type)).toEqual(['text', 'image']); + }); + + it('rejects a setext underline inside a list item too', () => { + expect(() => parseMarkdown('- item\n Title\n =====\n')).toThrow(/setext heading underline/); + }); + + it('ends a list item at a thematic break in its continuation', () => { + const { blocks } = parseMarkdown('- item\n ***\n'); + expect(blocks.map((b) => b.type)).toEqual(['list', 'thematicBreak']); + }); + it('parses a table with an escaped pipe inside inline code', () => { const md = '| Key | Action |\n|-----|--------|\n| `\\|` or tmux `%` | Split |\n'; const { blocks } = parseMarkdown(md); From eedff36958c827a9137b3d85c9be58f97abe2778 Mon Sep 17 00:00:00 2001 From: dormouse-bot <287024035+dormouse-bot@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:48:16 +0000 Subject: [PATCH 3/4] fix(website): require a pipe in the table delimiter row so `---` stays a setext underline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `isDelimiterRow` made every `|` in its pattern optional, so a bare `---` (and `:---`) passed as a one-column delimiter row. The table lookahead added in cea9b88 therefore broke a paragraph at any line containing a pipe when a setext underline followed it, and the table branch consumed both: `Intro` / `a | b` / `---` parsed as a two-column header with a one-element `align` and no rows, where fe1e6b1 raised the setext error and GitHub renders an `

`. GFM requires the delimiter row to carry a pipe, so requiring one at the source closes the block-level table branch's identical hole too — the bare `a | b` / `---` case, which mangled the same way on main. Both cases now raise the setext error. Generated docs.{guide,selfhost,cli,skill} .json stay byte-identical to main's parser. --- website/scripts/docs-parser.js | 8 +++++++- website/scripts/docs-parser.test.js | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/website/scripts/docs-parser.js b/website/scripts/docs-parser.js index 80ff6ebd..6b526e06 100644 --- a/website/scripts/docs-parser.js +++ b/website/scripts/docs-parser.js @@ -350,8 +350,14 @@ function splitRow(row, line) { return cells.map((c) => parseInline(c.trim(), line)); } +/** + * A GFM delimiter row (`| --- | :-- |`). The pipe is required: every `|` in the + * pattern is optional, so without this test a bare `---` reads as a one-column + * delimiter row and a preceding line that merely contains a pipe becomes a + * table — where the `---` is a setext underline, which has to raise instead. + */ function isDelimiterRow(row) { - return /^\s*\|?\s*:?-{1,}:?\s*(\|\s*:?-{1,}:?\s*)*\|?\s*$/.test(row); + return row.includes('|') && /^\s*\|?\s*:?-{1,}:?\s*(\|\s*:?-{1,}:?\s*)*\|?\s*$/.test(row); } function alignmentsFrom(row) { diff --git a/website/scripts/docs-parser.test.js b/website/scripts/docs-parser.test.js index 12c8f7da..c66fd0e6 100644 --- a/website/scripts/docs-parser.test.js +++ b/website/scripts/docs-parser.test.js @@ -172,6 +172,11 @@ describe('blocks', () => { expect(blocks.map((b) => b.type)).toEqual(['list', 'thematicBreak']); }); + it('reads a pipeless dashed underline as a setext heading, not a table delimiter row', () => { + expect(() => parseMarkdown('Intro\na | b\n---\n')).toThrow(/setext heading underline/); + expect(() => parseMarkdown('a | b\n---\n')).toThrow(/setext heading underline/); + }); + it('parses a table with an escaped pipe inside inline code', () => { const md = '| Key | Action |\n|-----|--------|\n| `\\|` or tmux `%` | Split |\n'; const { blocks } = parseMarkdown(md); From a46a0ff1af8045f11a3f3b14fe75163f15e5f431 Mon Sep 17 00:00:00 2001 From: Ned Twigg Date: Thu, 3 Sep 2026 14:22:54 -0700 Subject: [PATCH 4/4] Unnecessary level of detail. We can throw an error for this in the parser if we need to, doesn't need to weigh down the spec. --- docs/specs/website-docs.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/specs/website-docs.md b/docs/specs/website-docs.md index 2fac1897..7c8e6ab0 100644 --- a/docs/specs/website-docs.md +++ b/docs/specs/website-docs.md @@ -165,9 +165,6 @@ therefore supports a deliberate *subset* of CommonMark and raises general parser would. The public-doc lint turns that error into a build failure, which is what makes a hand-rolled parser safe as the guide grows. -**Headings are ATX only.** **A fence closes only on a run of its own marker at -least as long as the opener's**, so a longer outer fence nests an inner one. - Raw HTML is disabled except for a narrow `` allowlist carrying only `src`, `alt`, `width`, `height`, and `title`, with an `https:` source. Every other tag, and every other attribute on ``, is rejected outright. The exception exists