diff --git a/src/gen-lexer.ts b/src/gen-lexer.ts index a4ed10c..a11eaaf 100644 --- a/src/gen-lexer.ts +++ b/src/gen-lexer.ts @@ -306,6 +306,10 @@ export function createLexer(grammar: CstGrammar, intern?: LexerIntern) { const kVoidNameTok = kOf(markup?.voidNameToken ?? null); const tTagOpen = markup ? (puLitOf.get(markup.tagOpen) ?? 0) : 0; const kNewlineModeTok = kOf(newline?.token ?? null); + // newline-only 'terminator' mode: a NEWLINE at EVERY block-context line break, placed at the break + // (see NewlineConfig.mode). The line-start boundary emission below is then skipped; the three + // places a break is consumed (a content line's break, a blank line, a tab-blank line) emit instead. + const nlTerminators = !indent && newline?.mode === 'terminator'; const kIndentTok = kOf(indent?.indentToken ?? null), kDedentTok = kOf(indent?.dedentToken ?? null), kIndentNewlineTok = kOf(indent?.newlineToken ?? null); const kBlockScalarTok = kOf(indent?.blockScalar?.token ?? null); const kRawBlockTok = kOf(indent?.rawBlock?.token ?? null); @@ -682,6 +686,7 @@ export function createLexer(grammar: CstGrammar, intern?: LexerIntern) { const ch = source[p]; if (p >= source.length) { pos = p; lineStart = false; continue; } // EOF — final DEDENTs emitted after the loop if (ch === '\n' || ch === '\r') { // blank line — ignored for structure + if (nlTerminators) push(mkNamed(newline!.token, '', p, kNewlineModeTok)); // …but a line break all the same pos = p + 1; if (ch === '\r' && source[pos] === '\n') pos++; continue; // still at a line start } @@ -696,6 +701,7 @@ export function createLexer(grammar: CstGrammar, intern?: LexerIntern) { let b = p; while (b < source.length && (source[b] === ' ' || source[b] === '\t')) b++; const bc = source[b]; if (b >= source.length || bc === '\n' || bc === '\r') { + if (nlTerminators && bc !== undefined) push(mkNamed(newline!.token, '', b, kNewlineModeTok)); pos = b; if (bc === '\r' && source[pos + 1] === '\n') pos += 2; else if (bc !== undefined) pos++; continue; } @@ -728,7 +734,7 @@ export function createLexer(grammar: CstGrammar, intern?: LexerIntern) { // ── newline-only mode: no indent stack — emit ONE NEWLINE at this real line boundary (a // leading boundary before any content is suppressed via emittedContent) and move on. ── if (!indent) { - if (emittedContent) push(mkNamed(newline!.token, '', pos, kNewlineModeTok)); + if (emittedContent && !nlTerminators) push(mkNamed(newline!.token, '', pos, kNewlineModeTok)); lineStart = false; atLineLead = true; continue; @@ -811,6 +817,7 @@ export function createLexer(grammar: CstGrammar, intern?: LexerIntern) { pos++; continue; } if (c === '\n' || c === '\r') { + if (nlTerminators && flowDepth === 0) push(mkNamed(newline!.token, '', pos, kNewlineModeTok)); pos++; if (c === '\r' && source[pos] === '\n') pos++; if (flowDepth === 0) lineStart = true; else if (indent) { diff --git a/src/types.ts b/src/types.ts index bf2817b..e5509a7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -438,7 +438,19 @@ export interface NewlineConfig { token: string; // token TYPE emitted at each significant line boundary (engine-emitted, like the indent tokens) flowOpen?: string[]; // punctuation that SUSPENDS newline significance while open (e.g. ['(', '[', '{']) flowClose?: string[]; // matching closers (e.g. [')', ']', '}']) - comment?: string; // line-comment introducer; a comment-only line emits no NEWLINE (e.g. '#') + comment?: string; // line-comment introducer; a comment-only line emits no NEWLINE in 'separator' mode (e.g. '#') + // WHAT a NEWLINE token means: + // 'separator' (default) — one NEWLINE between two content-bearing lines, placed at the start of + // the later line's content. Blank and comment-only lines collapse into it; nothing is emitted + // before the first content or after the last. The right shape for a statement SEPARATOR. + // 'terminator' — one NEWLINE at EVERY line break outside flow delimiters, placed AT the break + // (zero-width, before the `\n` / `\r\n`): blank lines, comment-only lines, leading and + // trailing breaks included; only a final line with no break has none. The right shape for a + // grammar whose AST keeps blank lines and comments as lines (dotenv / env-spec: `File = + // many(Line)`, `Line = [Stmt, opt(NEWLINE)] | [NEWLINE]`), and the shape the derived + // tree-sitter scanner already has (it is stateless: one NEWLINE per break where the grammar + // permits one). Positions are exact per line, so a blank-line node maps to its own line. + mode?: 'separator' | 'terminator'; } export interface PrecOperator { diff --git a/test/newline-mode.ts b/test/newline-mode.ts index feafe76..ca84063 100644 --- a/test/newline-mode.ts +++ b/test/newline-mode.ts @@ -14,9 +14,11 @@ import { mkdtempSync, writeFileSync, mkdirSync, existsSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import vm from 'node:vm'; -import { token, rule, defineGrammar, many, opt, sep, seq, plus, oneOf, range, star, noneOf, never } from '../src/api.ts'; +import { token, rule, defineGrammar, many, opt, alt, sep, seq, plus, oneOf, range, star, noneOf, never } from '../src/api.ts'; import { createLexer } from '../src/gen-lexer.ts'; import { createParser } from '../src/gen-parser.ts'; +import { emitParser, jsTarget } from '../src/emit.ts'; +import { objectify } from './emitted-obj.ts'; import { generateTmLanguage } from '../src/gen-tm.ts'; import { generateMonarch } from '../src/gen-monarch.ts'; import { generateTreeSitter } from '../src/gen-treesitter.ts'; @@ -167,5 +169,106 @@ if (hasCli()) { console.log('\ntree-sitter CLI not found — structural validation only (not a failure).'); } + +// ── 7. `mode: 'terminator'`: one NEWLINE at EVERY block-context line break, placed at the break ── +// (blank / comment-only / leading / trailing breaks included; none inside flow; none at EOF +// without a break). The shape a grammar that keeps blank lines and comments as AST lines needs. +{ + const LineT = rule(() => [[Stmt, opt(Newline)], [Newline]]); // Line = Stmt NEWLINE? | NEWLINE (a blank line) + const ProgramT = rule(() => [[many(LineT)]]); + const gT = defineGrammar({ + name: 'envspec-t', scopeName: 'source.envspec-t', + tokens: { Comment, Ident, Newline }, + rules: { Value, Stmt, Line: LineT, Program: ProgramT }, entry: ProgramT, + newline: { ...newline, mode: 'terminator' }, + }); + const lexT = createLexer(gT).tokenize; + const nls = (s: string) => lexT(s).filter(t => t.type === 'Newline').map(t => t.offset); + check("terminator: one NEWLINE between two statements, AT the break", JSON.stringify(nls('A=1\nB=2')) === '[3]'); + check("terminator: every blank line is its own NEWLINE", JSON.stringify(nls('A=1\n\n\nB=2')) === '[3,4,5]'); + check("terminator: leading breaks are emitted", JSON.stringify(nls('\n\nA=1')) === '[0,1]'); + check("terminator: a trailing break is emitted", JSON.stringify(nls('A=1\n')) === '[3]'); + check("terminator: no break, no NEWLINE", JSON.stringify(nls('A=1')) === '[]'); + check("terminator: a comment-only line still breaks", JSON.stringify(nls('A=1\n# note\nB=2')) === '[3,10]'); + check("terminator: a whitespace-only line (tabs) is a break", JSON.stringify(nls('A=1\n \t\nB=2')) === '[3,6]'); + check("terminator: CRLF counts once, at the CR", JSON.stringify(nls('A=1\r\n\r\nB=2')) === '[3,5]'); + check("terminator: breaks INSIDE flow ( … ) stay suspended", JSON.stringify(nls('A=fn(1,\n2)\nB=3')) === '[10]'); + check("terminator: still no INDENT/DEDENT", !lexT('A=1\n\nB=2').some(t => t.type === 'Indent' || t.type === 'Dedent')); + check("separator (default) is unchanged by the option's existence", countNL('A=1\n\n\nB=2') === 1 && tokenize('\n\nA=1')[0]?.type !== 'Newline'); + + const parseT = createParser(gT).parse; + const acceptsT = (s: string) => { try { return parseT(s).rule !== undefined; } catch { return false; } }; + check('terminator: parses statements with blank lines between and a trailing break', acceptsT('A=1\n\nB=2\n')); + check('terminator: parses leading blank lines', acceptsT('\n\nA=1')); + check('terminator: parses a lone statement without a break', acceptsT('A=1')); + check('terminator: parses a flow value spanning lines', acceptsT('A=fn(1,\n2)\nB=3')); + check('terminator: still rejects a malformed statement', !acceptsT('A B')); + // A blank line is its own Line node, at its own offset (what an editor maps to a line number). + const cst = parseT('A=1\n\nB=2') as any; + const nlLeaves = JSON.stringify(cst).match(/"tokenType":"Newline","offset":(\d+)/g) ?? []; + check('terminator: the blank line and the statement break are distinct leaves at 3 and 4', nlLeaves.length === 2 && nlLeaves[0].endsWith(':3') && nlLeaves[1].endsWith(':4')); + + // The generators do not care which mode is set (the tree-sitter scanner is already per-break). + check('terminator: TextMate generates', Object.keys(generateTmLanguage(gT).repository).length > 0); + check('terminator: Monarch generates', !!generateMonarch(gT).tokenizer.root); + const tsT = generateTreeSitter(gT, 'envspec_t'); + check('terminator: tree-sitter externals still include newline', tsT.externalTokens.includes('newline')); + check('terminator: tree-sitter scanner unchanged (stateless per-break scan_newline)', tsT.scannerC.includes('scan_newline')); + + // ── 8. The emitted engine inherits the mode (createLexer fallback bakes the newline config) ── + { + const dirE = mkdtempSync(join(tmpdir(), 'monogram-nl-emit-')); + const file = join(dirE, 'envspec-t.ts'); + writeFileSync(file, emitParser(gT, jsTarget)); + const p = (await import(file + '?v=' + Date.now())).createParser(); + for (const src of ['A=1\n\nB=2\n', '\n\nA=1', 'A=fn(1,\n2)\nB=3', 'A=1\n# note\nB=2']) { + const cst = p.parse(src); + const obj = objectify(p.tree, (fns: any) => p.visit(cst, fns)); + check(`terminator: emitted engine ≡ interpreter for ${JSON.stringify(src)}`, cst.errors.length === 0 && JSON.stringify(obj) === JSON.stringify(parseT(src))); + } + } + + // ── 9. The derived tree-sitter parser agrees: one newline node per line break, including blank + // lines and the trailing break (the claim that terminator mode is the scanner's native shape). ── + // On a grammar where the statement does not also claim the break: `Line = Stmt NEWLINE? | NEWLINE` + // is LR-ambiguous after a statement (shift the break into the statement, or reduce and start a + // blank Line), which the PEG engine resolves greedily but tree-sitter needs a declared + // conflict/precedence for; that resolution is a generator concern, not this mode's, so the + // scanner claim is checked on `Program = many(NEWLINE | Stmt)`. + const gT2 = defineGrammar({ + name: 'envspec-t2', scopeName: 'source.envspec-t2', + tokens: { Comment, Ident, Newline }, + rules: { Value, Stmt, Program: rule(() => [[many(alt(Newline, Stmt))]]) }, + newline: { ...newline, mode: 'terminator' }, + } as any); + const tsT2 = generateTreeSitter(gT2, 'envspec_t2'); + if (hasCli()) { + const dirT = mkdtempSync(join(tmpdir(), 'monogram-nl-term-')); + mkdirSync(join(dirT, 'src'), { recursive: true }); + mkdirSync(join(dirT, 'queries'), { recursive: true }); + writeFileSync(join(dirT, 'grammar.js'), tsT2.grammarJs); + writeFileSync(join(dirT, 'src', 'scanner.c'), tsT2.scannerC); + writeFileSync(join(dirT, 'queries', 'highlights.scm'), tsT2.highlightsScm); + writeFileSync(join(dirT, 'package.json'), JSON.stringify({ name: 'tree-sitter-envspec-t-monogram', version: '0.0.0' }, null, 2)); + let generatedT = false; + try { execFileSync(tsBin, ['generate'], { cwd: dirT, stdio: 'pipe' }); generatedT = true; } catch (e: any) { console.log(' generate (terminator) failed:', ((e.stderr || e.message || '') + '').split('\n').slice(0, 6).join('\n ')); } + check('terminator: tree-sitter generate succeeds (NEWLINE external, no indent scanner state)', generatedT); + if (generatedT) { + const parseTreeT = (input: string) => { + writeFileSync(join(dirT, 'in.env'), input); + try { return execFileSync(tsBin, ['parse', 'in.env'], { cwd: dirT, encoding: 'utf8' }); } + catch (e: any) { return ((e.stdout || '') + '\n' + (e.stderr || '')); } + }; + const newlines = (t: string) => (t.match(/\(newline /g) ?? []).length; + const t1 = parseTreeT('A=1\n\nB=2\n'); + check('terminator/tree-sitter: blank line and trailing break are newline nodes (3 total, no ERROR)', !t1.includes('ERROR') && newlines(t1) === 3); + const t2 = parseTreeT('\n\nA=1'); + check('terminator/tree-sitter: leading breaks are newline nodes (2, no ERROR)', !t2.includes('ERROR') && newlines(t2) === 2); + const t3 = parseTreeT('A=fn(1,\n2)\nB=3'); + check('terminator/tree-sitter: flow-internal break still suppressed (1 newline, no ERROR)', !t3.includes('ERROR') && newlines(t3) === 1); + } + } +} + console.log(fail === 0 ? `\n${ok}/${ok} newline-mode checks pass` : `\n${fail} of ${ok + fail} FAILED`); process.exit(fail === 0 ? 0 : 1);