feat(diagnostics): token and rule labels for expected-X messages

README.md

@@ -281,6 +281,8 @@ export default defineGrammar({
 });
 ```
 
+**Diagnostic labels.** The total parser's `expected …` messages name the missing token or rule by its grammar name (`expected 'Number'`, `expected Expr`). When a grammar name is an internal identifier rather than a word an end user should read, give it a display label: `token(pattern, { label: 'a number' })` / `rule(fn, { label: 'an expression' })` render `expected a number` / `expected an expression`. Labels change message text only: leaf `tokenType`s, `ruleNameOf`, the CST, and every derived artifact keep the grammar name, so adding labels parses byte-identically (`test/diagnostic-labels.ts`).
+
 Token patterns are **combinators, not regular expressions** — `seq` / `oneOf` / `range` / `noneOf` / `plus` / `star` / `altPattern` / `optPattern` / … assemble a structured pattern IR (regex is a *derived* backend, not the source of truth). A bare `RegExp` is not a valid token pattern: `token(/…/)` is a `TS2345` type error. Coming from regex:
 
 | RegExp | Combinator |

src/api.ts

@@ -14,6 +14,10 @@ export {
 
 interface TokenOptions {
   skip?: boolean;
+  // Display name for `expected …` diagnostics (the emitted engine's $missing rows): a token
+  // `NUM` reads as `expected 'NUM'` by default; `label: 'a number'` renders `expected a number`.
+  // Messages ONLY: leaf tokenTypes, scopes, and every derived artifact keep the grammar name.
+  label?: string;
   scope?: string;
   escape?: TokenPattern;
   // Highlight-only interpolation regions for ordinary string tokens (e.g. env-spec `${…}` / `$(…)`).
@@ -77,6 +81,10 @@ export function token(pattern: TokenPattern, opts?: TokenOptions): TokenRef {
 
 interface RuleOptions {
   type?: boolean;
+  // Display name for `expected …` diagnostics (a missing required rule): `expected Value` by
+  // default, `expected a value` with `label: 'a value'`. Messages only; `ruleNameOf`, the CST,
+  // and every derived artifact keep the rule name.
+  label?: string;
 }
 
 type Element = string | TokenRef | RuleRef | Marker | Combinator;
@@ -555,6 +563,7 @@ export function defineGrammar(config: GrammarConfig): CstGrammar & { name: strin
       blockOnly: tok.opts.blockOnly,
       flags,
       scope: tok.opts.scope,
+      label: tok.opts.label,
       escapePattern: tok.opts.escape,
       interpolation: tok.opts.interpolation
         ? (Array.isArray(tok.opts.interpolation) ? tok.opts.interpolation : [tok.opts.interpolation]).map((i) => ({ ...i }))
@@ -599,6 +608,7 @@ export function defineGrammar(config: GrammarConfig): CstGrammar & { name: strin
       name,
       body: convertAlternatives(alts, names),
       flags: r.opts.type ? ['type'] : [],
+      label: r.opts.label,
     };
   });
 

src/emit-parser.ts

@@ -1164,6 +1164,8 @@ export function emitJsParser(grammar: CstGrammar, lexSrc: string | null): string
   // Every token is BORN with tok.k (type kind) + tok.t (literal kind) and the stamp
   // flags — one monomorphic shape, one allocation, no post-pass.
   e.emit(`const TYPE_KIND = new Map<string, number>(${J([...st.typeKind])});`);
+  // Token diagnostic labels (TokenDecl.label): name → display string, for "expected …" only.
+  e.emit(`const TOKEN_LABELS = new Map<string, string>(${J(grammar.tokens.filter(t => t.label !== undefined).map(t => [t.name, t.label!]))});`);
   e.emit(`const LIT_KW = new Map<string, number>(${J([...st.kwLitKind])});`);
   e.emit(`const LIT_PU = new Map<string, number>(${J([...st.puLitKind])});`);
   e.emit(`const K_PUNCT = ${st.KIND_PUNCT};`);
@@ -1288,6 +1290,10 @@ export function emitJsParser(grammar: CstGrammar, lexSrc: string | null): string
   // node's rule name so trees stay byte-identical to the base grammar. Identical to
   // RULE_NAMES when no rule is forked (the common case).
   e.emit(`const RULE_DISPLAY = ${J([...grammar.rules.map(r => r.canon ?? r.name), '$template', '$error', '$missing'])};`);
+  // Diagnostic LABELS: what a `$missing` row's "expected …" names for a missing required RULE.
+  // `RuleDecl.label` substitutes a display string there and nowhere else (RULE_DISPLAY stays the
+  // node's reported rule name), so a grammar that adds labels parses byte-identically.
+  e.emit(`const RULE_LABELS = ${J([...grammar.rules.map(r => r.label ?? r.canon ?? r.name), '$template', '$error', '$missing'])};`);
   e.emit(`const RID_TEMPLATE = ${grammar.rules.length};`);
   e.emit(`const RID_ERROR = ${grammar.rules.length + 1};`);
   e.emit(`const RID_MISSING = ${grammar.rules.length + 2};`);
@@ -2530,6 +2536,10 @@ function tokTextAt(i: number) {
 // The k → type-name inverse, for reconstructing a token object (tokenAt).
 const K_NAMES: string[] = [];
 for (const [n, k] of TYPE_KIND) K_NAMES[k] = n;
+// The k → diagnostic-label inverse: a labelled token renders bare (expected a number), an
+// unlabelled one keeps the quoted grammar name (expected 'NUM').
+const K_LABELS: string[] = [];
+for (const [n, k] of TYPE_KIND) K_LABELS[k] = TOKEN_LABELS.get(n) ?? "'" + n + "'";
 // A per-token object view over the columns (gates / debugging — the parser never builds these).
 export function tokenAt(i: number) {
   return {
@@ -2930,9 +2940,9 @@ function missLit(v: number) {
 function missEntry(v: number, kb: number): Diag {
   let message;
   if (v >= 1 << 21) message = 'expected ' + VSETS[v >>> 21];
-  else if (v >= RULE_MISS_BASE) message = 'expected ' + RULE_DISPLAY[v - RULE_MISS_BASE];
+  else if (v >= RULE_MISS_BASE) message = 'expected ' + RULE_LABELS[v - RULE_MISS_BASE];
   else if (v > 0) message = "expected '" + LIT_NAMES[v] + "'";
-  else message = "expected '" + (K_NAMES[-v] ?? '?') + "'";
+  else message = 'expected ' + (K_LABELS[-v] ?? "'?'");
   return { offset: kb, end: kb, message };
 }
 function collectErrRows(id: number, charBase: number, tokBase: number) {

src/types.ts

@@ -21,6 +21,7 @@ export interface TokenDecl {
   pattern: TokenPattern;
   flags: string[];
   scope?: string;         // @scope(...) override
+  label?: string;         // display name for `expected …` diagnostics only (see api.ts TokenOptions.label)
   escapePattern?: TokenPattern; // @escape pattern — escape sequence pattern (highlight only)
   interpolation?: StringInterpolation[]; // highlight-only interpolation regions inside a string token (e.g. `${…}` / `$(…)`)
   // Highlight-only: this comment-scoped token matches only the INTRODUCER (e.g. a bare `#`)
@@ -553,6 +554,8 @@ export interface RuleDecl {
   // parser keeps the distinct `name` for its memo/adoption rule identity, but reports
   // `canon` as the node's rule name so trees stay byte-identical to the base grammar.
   canon?: string;
+  // Display name for `expected …` diagnostics only (see api.ts RuleOptions.label).
+  label?: string;
 }
 
 export interface CstGrammar {

test/check.ts

@@ -19,6 +19,7 @@ const GATES: Gate[] = [
   { group: 'core', name: 'agnostic', args: ['test/agnostic.ts'] },
   { group: 'core', name: 'left-recursion', args: ['test/left-recursion.ts'] },
   { group: 'core', name: 'newline-mode', args: ['test/newline-mode.ts'] },
+  { group: 'core', name: 'diagnostic-labels', args: ['test/diagnostic-labels.ts'] },
   { group: 'core', name: 'interpolation-metadata', args: ['test/interpolation-metadata.ts'] },
   { group: 'core', name: 'refactor-guard', args: ['test/refactor-guard.ts'] },
   { group: 'core', name: 'cst-text-invariant', args: ['test/cst-text-invariant.ts'] },

test/diagnostic-labels.ts

@@ -0,0 +1,117 @@
+// Gate: human-readable LABELS for `$missing` diagnostics.
+//
+// The emitted engine's "expected X" messages name what a required position was missing. By
+// default X is the raw grammar name (a token `NUM` reads as `expected 'NUM'`, a rule `Value`
+// as `expected Value`), which is fine for a language whose grammar names are already words but
+// leaks internal identifiers (`DEC_VALUE_TEXT`) to end users of any real editor. `token(p,
+// { label })` and `rule(fn, { label })` substitute a display string in exactly those messages
+// and NOWHERE else: leaf `tokenType`s, `ruleNameOf`, the CST, and every other artifact keep
+// the grammar name, so a grammar that adds labels parses byte-identically.
+//
+// Run with: node test/diagnostic-labels.ts
+import { writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { emitParser, jsTarget } from '../src/emit.ts';
+import { createParser } from '../src/gen-parser.ts';
+import { objectify } from './emitted-obj.ts';
+import { token, rule, defineGrammar, many, opt, plus, oneOf, range } from '../src/api.ts';
+import { generateTmLanguage } from '../src/gen-tm.ts';
+import { generateTreeSitter } from '../src/gen-treesitter.ts';
+import { generateLanguageConfig } from '../src/gen-vscode-config.ts';
+
+let ok = 0, fail = 0;
+const check = (label: string, cond: boolean) => { if (cond) ok++; else { fail++; console.log('  ✗', label); } };
+
+type Diag = { offset: number; end: number; message: string };
+
+function build(labelled: boolean) {
+  const WS = token(plus(oneOf(' ', '\t')), { skip: true });
+  const IDENT = token(plus(oneOf(range('a', 'z'))), { identifier: true });
+  const NUM = token(plus(oneOf(range('0', '9'))), labelled ? { label: 'a number' } : {});
+  const SEMI = token(';', {});
+  const Value = rule(() => [[NUM], [IDENT, '(', opt(NUM), ')']], labelled ? { label: 'a value' } : {});
+  // `opt('=', Value)`: once the optional group has consumed `=` it is committed, so a missing
+  // Value synthesizes a $missing row (the tsc-style rule the engine derives; see TOTAL-PARSING.md).
+  const Stmt = rule(() => [[IDENT, opt('=', Value), SEMI]]);
+  const Program = rule(() => [[many(Stmt)]]);
+  return defineGrammar({ name: 'labels', tokens: { WS, IDENT, NUM, SEMI }, rules: { Value, Stmt, Program }, entry: Program });
+}
+
+const plain = build(false);
+const labelled = build(true);
+
+// ── 1. defineGrammar carries the labels onto the declarations (and only when given) ──
+check('token label lands on TokenDecl', labelled.tokens.find(t => t.name === 'NUM')?.label === 'a number');
+check('rule label lands on RuleDecl', labelled.rules.find(r => r.name === 'Value')?.label === 'a value');
+check('an unlabelled token has no label', plain.tokens.find(t => t.name === 'NUM')?.label === undefined);
+check('an unlabelled rule has no label', plain.rules.find(r => r.name === 'Value')?.label === undefined);
+
+// ── 2. The emitted engine renders labels in `expected …` messages ──
+const dir = tmpdir();
+async function load(g: ReturnType<typeof build>, tag: string) {
+  const file = join(dir, `monogram-labels-${tag}-${process.pid}.ts`);
+  writeFileSync(file, emitParser(g, jsTarget));
+  const em = await import(file + '?v=' + Date.now());
+  return em.createParser() as { parse(s: string): { root: number; errors: Diag[] }; visit(c: unknown, fns: object): void; tree: any };
+}
+const pp = await load(plain, 'plain');
+const pl = await load(labelled, 'labelled');
+const msgs = (p: typeof pp, src: string) => p.parse(src).errors.map(e => e.message);
+
+// A required TOKEN missing: `a = 1` (no `;`) and `a = f(` (the `)` is a literal, unaffected).
+check("plain: missing named token → expected 'SEMI'", msgs(pp, 'a = 1').includes("expected 'SEMI'"));
+check("labelled: unlabelled token keeps the quoted grammar name", msgs(pl, 'a = 1').includes("expected 'SEMI'"));
+
+// A required RULE missing is exercised on the TypeScript grammar below (2b): whether a tiny
+// grammar synthesizes the rule or absorbs the statement is the recovery engine's call, not
+// this gate's subject.
+
+// A labelled TOKEN missing: `a = f(1` is a literal `)`; use `a = f(` + `;`? The optional NUM never
+// synthesizes, so exercise the token label through a grammar position where NUM is required:
+{
+  const WS = token(plus(oneOf(' ', '\t')), { skip: true });
+  const NUM = token(plus(oneOf(range('0', '9'))), { label: 'a number' });
+  const Pair = rule(() => [[NUM, ',', NUM]]);
+  const Top = rule(() => [[many(Pair)]]);
+  const g = defineGrammar({ name: 'labels2', tokens: { WS, NUM }, rules: { Pair, Top }, entry: Top });
+  const p = await load(g, 'pair');
+  check('labelled: missing token → expected a number (unquoted label)', msgs(p, '1,').includes('expected a number'));
+  check("labelled: the quoted raw token name is gone", !msgs(p, '1,').includes("expected 'NUM'"));
+}
+
+const tree = (p: typeof pp, src: string) => { const c = p.parse(src); return JSON.stringify(objectify(p.tree, (fns: any) => p.visit(c, fns))); };
+
+// ── 2b. A real grammar: label TypeScript's Expr rule and read `const a = ;` ──
+{
+  const ts = (await import('../typescript.ts')).default;
+  const labelledTs = { ...ts, rules: ts.rules.map((r: any) => r.name === 'Expr' ? { ...r, label: 'an expression' } : r) };
+  const p0 = await load(ts as any, 'ts-plain');
+  const p1 = await load(labelledTs as any, 'ts-labelled');
+  check('typescript: default message is expected Expr', msgs(p0, 'const a = ;').includes('expected Expr'));
+  check('typescript: labelled message is expected an expression', msgs(p1, 'const a = ;').includes('expected an expression'));
+  check('typescript: labelled tree is byte-identical', tree(p0, 'const a = ;\nfoo(1, [2, 3]);') === tree(p1, 'const a = ;\nfoo(1, [2, 3]);'));
+}
+
+// ── 3. Labels change messages ONLY: trees, leaf token types, and literal messages are identical ──
+for (const src of ['a = 1;', 'a = f(2);', 'a = ;', 'a = f(', 'a = 1']) {
+  check(`byte-identical tree for ${JSON.stringify(src)}`, tree(pp, src) === tree(pl, src));
+}
+check("literal messages unchanged: expected ')'", msgs(pl, 'a = f(').includes("expected ')'"));
+check('related info unchanged', JSON.stringify(pl.parse('a = f(').errors).includes("to match this '('"));
+check('valid input has no errors under labels', pl.parse('a = 1; b = f(2);').errors.length === 0);
+
+// ── 3b. Every derived artifact is unaffected: labels are not scopes, captures, or names ──
+check('TextMate grammar identical with and without labels', JSON.stringify(generateTmLanguage(plain)) === JSON.stringify(generateTmLanguage(labelled)));
+check('tree-sitter output identical with and without labels', JSON.stringify(generateTreeSitter(plain, 'labels')) === JSON.stringify(generateTreeSitter(labelled, 'labels')));
+check('language-configuration identical with and without labels', JSON.stringify(generateLanguageConfig(plain)) === JSON.stringify(generateLanguageConfig(labelled)));
+check('leaf tokenTypes keep the grammar name (no label leaks into the tree)', tree(pl, 'a = 1;').includes('"tokenType":"NUM"') && !tree(pl, 'a = 1;').includes('a number'));
+
+// ── 4. The interpreter is unaffected (it has no expected-X diagnostics to label) ──
+const interp = createParser(labelled);
+let threw = '';
+try { interp.parse('a = ;'); } catch (e) { threw = (e as Error).message; }
+check('interpreter still rejects with its own message', threw.startsWith('Parse error at offset'));
+
+console.log(`\n${ok}/${ok + fail} diagnostic-label checks pass${fail ? '' : ' ✓'}`);
+if (fail) process.exit(1);