diff --git a/website/scripts/docs-parser.js b/website/scripts/docs-parser.js index b59351c7..6b526e06 100644 --- a/website/scripts/docs-parser.js +++ b/website/scripts/docs-parser.js @@ -222,6 +222,33 @@ export function parseInline(text, line) { return nodes; } +/** + * The index of the `)` that closes the destination opened at `open`, or -1. + * + * Parentheses nest rather than closing at the first `)`, because a URL is + * allowed to contain a balanced pair and several the docs link to do + * (`.../wiki/Foo_(bar)`). Stopping at the first one truncates the href and + * still emits a link, so the page ships a live anchor pointing somewhere else + * — exactly the silent mangling this parser exists to avoid. A `"` toggles a + * title span, where a paren is literal. + */ +function matchDestination(text, open) { + let depth = 1; + let inTitle = false; + for (let i = open + 1; i < text.length; i++) { + const ch = text[i]; + if (ch === '\\') { i++; continue; } + if (ch === '"') { inTitle = !inTitle; continue; } + if (inTitle) continue; + if (ch === '(') depth++; + else if (ch === ')') { + depth--; + if (depth === 0) return i; + } + } + return -1; +} + /** Match `[label](href "title")` starting at `start`. Returns null if not one. */ function matchLink(text, start) { if (text[start] !== '[') return null; @@ -237,7 +264,7 @@ function matchLink(text, start) { } if (depth !== 0 || text[i + 1] !== '(') return null; const label = text.slice(start + 1, i); - const close = text.indexOf(')', i + 2); + const close = matchDestination(text, i + 1); if (close === -1) return null; const target = text.slice(i + 2, close).trim(); const titleMatch = /^(\S+)\s+"([^"]*)"$/.exec(target); @@ -252,8 +279,57 @@ function matchLink(text, start) { const LIST_ITEM = /^(\s*)([-*+]|\d+[.)])\s+(.*)$/; -/** Closing-fence matchers, one per marker character, compiled once. */ -const FENCE_CLOSE = { '`': /^\s*`{3,}\s*$/, '~': /^\s*~{3,}\s*$/ }; +/** + * Whether `line` closes a fence opened with `length` copies of `marker`. + * + * The closing run must be at least as long as the opening one. Without that + * length test a ````-fenced block closes at the first ``` inside it — the very + * form AGENTS.md prescribes for nesting one code block inside another — so the + * block's body spills out and renders as prose, with no error to notice. + */ +function closesFence(line, marker, length) { + const trimmed = line.trim(); + return trimmed.length >= length && [...trimmed].every((ch) => ch === marker); +} + +/** + * Whether `line` begins a new block, so a paragraph — or a blockquote's lazy + * continuation — ends before it instead of swallowing it. `next` supplies the + * single line of lookahead a table needs, a `|` row being a table only when a + * delimiter row follows it. + * + * Six of the seven block starts are here: ATX heading, list item, blockquote, + * fence, table, and block-level raw HTML. A standalone `` is deliberately + * absent — it is inline-level, so GitHub keeps it in the paragraph and so do + * we. The seventh, the thematic break, is tested separately at each call site + * because `---` is also a setext underline, which has to raise first. + */ +function interruptsParagraph(line, next) { + return /^(#{1,6})\s+/.test(line) + || LIST_ITEM.test(line) + || /^\s*>\s?/.test(line) + || /^(\s*)(`{3,}|~{3,})/.test(line) + || (line.includes('|') && isDelimiterRow(next ?? '')) + || (/^\s*<\/?[a-zA-Z]/.test(line) && !/^\s* parseInline(c.trim(), line)); } +/** + * A GFM delimiter row (`| --- | :-- |`). The pipe is required: every `|` in the + * pattern is optional, so without this test a bare `---` reads as a one-column + * delimiter row and a preceding line that merely contains a pipe becomes a + * table — where the `---` is a setext underline, which has to raise instead. + */ function isDelimiterRow(row) { - return /^\s*\|?\s*:?-{1,}:?\s*(\|\s*:?-{1,}:?\s*)*\|?\s*$/.test(row); + return row.includes('|') && /^\s*\|?\s*:?-{1,}:?\s*(\|\s*:?-{1,}:?\s*)*\|?\s*$/.test(row); } function alignmentsFrom(row) { @@ -310,13 +392,14 @@ export function parseMarkdown(markdown, options = {}) { // Fenced code const fence = /^(\s*)(`{3,}|~{3,})\s*([\w+-]*)\s*$/.exec(raw); if (fence) { - const closeRe = FENCE_CLOSE[fence[2][0]]; + const marker = fence[2][0]; + const openLength = fence[2].length; const lang = fence[3] || null; const body = []; i++; let closed = false; while (i < lines.length) { - if (closeRe.test(lines[i])) { closed = true; i++; break; } + if (closesFence(lines[i], marker, openLength)) { closed = true; i++; break; } body.push(lines[i]); i++; } @@ -339,7 +422,7 @@ export function parseMarkdown(markdown, options = {}) { } // Thematic break - if (/^\s*([-*_])(\s*\1){2,}\s*$/.test(raw)) { + if (THEMATIC_BREAK.test(raw)) { blocks.push({ type: 'thematicBreak' }); i++; continue; @@ -362,7 +445,13 @@ export function parseMarkdown(markdown, options = {}) { // Blockquote if (/^\s*>\s?/.test(raw)) { const body = []; - while (i < lines.length && (/^\s*>\s?/.test(lines[i]) || (lines[i].trim() !== '' && body.length > 0))) { + while ( + i < lines.length + && (/^\s*>\s?/.test(lines[i]) + || (lines[i].trim() !== '' && body.length > 0 + && !interruptsParagraph(lines[i], lines[i + 1]) + && !THEMATIC_BREAK.test(lines[i]))) + ) { body.push(lines[i].replace(/^\s*>\s?/, '')); i++; } @@ -395,7 +484,11 @@ export function parseMarkdown(markdown, options = {}) { const para = []; while (i < lines.length && lines[i].trim() !== '') { const l = lines[i]; - if (para.length > 0 && (/^(#{1,6})\s+/.test(l) || LIST_ITEM.test(l) || /^\s*>\s?/.test(l) || /^(\s*)(`{3,}|~{3,})/.test(l))) break; + if (para.length > 0 && interruptsParagraph(l, lines[i + 1])) break; + if (para.length > 0 && SETEXT_UNDERLINE.test(l)) { + throw new UnsupportedMarkdownError('setext heading underline — use an ATX `#` heading', i + 1); + } + if (para.length > 0 && THEMATIC_BREAK.test(l)) break; para.push(l.trim()); i++; } @@ -446,6 +539,12 @@ function parseList(lines, start, slug) { i++; // Continuation lines: indented further, not themselves list items. while (i < lines.length && lines[i].trim() !== '' && !LIST_ITEM.test(lines[i]) && /^\s+/.test(lines[i])) { + // The same two guards a paragraph applies, so `Headings are ATX only` + // holds inside a list item too rather than shipping as literal text. + if (SETEXT_UNDERLINE.test(lines[i])) { + throw new UnsupportedMarkdownError('setext heading underline — use an ATX `#` heading', i + 1); + } + if (THEMATIC_BREAK.test(lines[i])) break; contentLines.push(lines[i].trim()); i++; } diff --git a/website/scripts/docs-parser.test.js b/website/scripts/docs-parser.test.js index 93e8f28b..c66fd0e6 100644 --- a/website/scripts/docs-parser.test.js +++ b/website/scripts/docs-parser.test.js @@ -30,6 +30,17 @@ describe('inline', () => { expect(nodes.map((n) => n.type)).toEqual(['text', 'code', 'text', 'link', 'text', 'strong']); }); + it('keeps a balanced paren inside a link destination', () => { + const nodes = parseInline('see [Foo](https://x.test/wiki/Foo_(bar)) now'); + expect(nodes[1]).toMatchObject({ type: 'link', href: 'https://x.test/wiki/Foo_(bar)' }); + expect(nodes[2]).toMatchObject({ type: 'text', value: ' now' }); + }); + + it('keeps a paren inside a link title out of the destination scan', () => { + const nodes = parseInline('[a](/x "the (title)") end'); + expect(nodes[0]).toMatchObject({ type: 'link', href: '/x', title: 'the (title)' }); + }); + it('honours backslash escapes', () => { expect(inlineToText(parseInline('a \\| b'))).toBe('a | b'); }); @@ -96,6 +107,76 @@ describe('blocks', () => { expect(() => parseMarkdown('```\nnope\n')).toThrow(/unterminated fenced code/); }); + it('keeps a shorter fence inside a longer one as code, not prose', () => { + const md = '````markdown\n```bash\necho hi\n```\n````\n'; + const { blocks } = parseMarkdown(md); + expect(blocks).toHaveLength(1); + expect(blocks[0]).toEqual({ type: 'code', lang: 'markdown', value: '```bash\necho hi\n```' }); + }); + + it('rejects a setext underline instead of shipping it as text', () => { + expect(() => parseMarkdown('Title\n=====\n')).toThrow(UnsupportedMarkdownError); + expect(() => parseMarkdown('Title\n-----\n')).toThrow(/setext heading underline/); + }); + + it('ends a blockquote at a line that starts a new block', () => { + const { blocks } = parseMarkdown('> quoted\n# Heading\n'); + expect(blocks.map((b) => b.type)).toEqual(['blockquote', 'heading']); + expect(blocks[1].id).toBe('heading'); + }); + + it('still folds a lazy continuation line into the blockquote', () => { + const { blocks } = parseMarkdown('> quoted\ncontinues\n'); + expect(blocks).toHaveLength(1); + expect(inlineToText(blocks[0].children[0].children)).toBe('quoted continues'); + }); + + it('breaks a paragraph at a thematic break instead of eating it as emphasis', () => { + const { blocks } = parseMarkdown('Some text\n***\nNext\n'); + expect(blocks.map((b) => b.type)).toEqual(['paragraph', 'thematicBreak', 'paragraph']); + expect(inlineToText(blocks[0].children)).toBe('Some text'); + expect(inlineToText(blocks[2].children)).toBe('Next'); + }); + + it('still reads a dashed underline as a setext heading, not a thematic break', () => { + expect(() => parseMarkdown('Title\n---\n')).toThrow(/setext heading underline/); + }); + + it('ends a blockquote at a thematic break rather than raising a setext error', () => { + const { blocks } = parseMarkdown('> quoted\n---\n'); + expect(blocks.map((b) => b.type)).toEqual(['blockquote', 'thematicBreak']); + }); + + it('ends a blockquote at a table rather than swallowing its rows', () => { + const { blocks } = parseMarkdown('> quoted\n| a | b |\n|---|---|\n| 1 | 2 |\n'); + expect(blocks.map((b) => b.type)).toEqual(['blockquote', 'table']); + expect(blocks[1].header.map(inlineToText)).toEqual(['a', 'b']); + }); + + it('ends a blockquote at block-level raw HTML, which then raises', () => { + expect(() => parseMarkdown('> quoted\n
\n')).toThrow(/raw HTML
/); + }); + + it('keeps a standalone line inside the paragraph it follows', () => { + const { blocks } = parseMarkdown('text\na\n'); + expect(blocks).toHaveLength(1); + expect(blocks[0].children.map((n) => n.type)).toEqual(['text', 'image']); + }); + + it('rejects a setext underline inside a list item too', () => { + expect(() => parseMarkdown('- item\n Title\n =====\n')).toThrow(/setext heading underline/); + }); + + it('ends a list item at a thematic break in its continuation', () => { + const { blocks } = parseMarkdown('- item\n ***\n'); + expect(blocks.map((b) => b.type)).toEqual(['list', 'thematicBreak']); + }); + + it('reads a pipeless dashed underline as a setext heading, not a table delimiter row', () => { + expect(() => parseMarkdown('Intro\na | b\n---\n')).toThrow(/setext heading underline/); + expect(() => parseMarkdown('a | b\n---\n')).toThrow(/setext heading underline/); + }); + it('parses a table with an escaped pipe inside inline code', () => { const md = '| Key | Action |\n|-----|--------|\n| `\\|` or tmux `%` | Split |\n'; const { blocks } = parseMarkdown(md);