grammar(ts): build TypeScript from marionebl's maintained fork

README.md

@@ -7,11 +7,17 @@
 > - **Impact is fast again.** `codegraph impact`, `codegraph_impact` and the viewer's blast
 >   radius had become 2–3x slower on busy symbols; the answer itself was unchanged
 >   (upstream [#3249](https://github.com/colbymchenry/codegraph/pull/3249), fixes #3248).
-> - **TypeScript variance annotations parse.** `interface Stream<out A, in E>` and other
->   `in` / `out` type parameters no longer turn the declaration, and in large files the whole
->   file, into a parse error that indexes as almost nothing (upstream
->   [#3226](https://github.com/colbymchenry/codegraph/pull/3226); the grammar's own fix PRs,
->   tree-sitter/tree-sitter-typescript#361 and #364, are unmerged). Re-index TypeScript projects after updating.
+> - **Much more TypeScript parses.** The TypeScript grammar is built from
+>   [marionebl/tree-sitter-typescript](https://github.com/marionebl/tree-sitter-typescript),
+>   a maintained fork of the stalled upstream grammar, with two fixes from
+>   [dokimasia's fork](https://github.com/dokimasia/tree-sitter-typescript) and our own patch.
+>   Variance annotations (`interface Stream<out A, in E>`), generic overloads after a member with
+>   no `;`, `export type *`, `import("m").X[]` and template-literal escapes no longer turn a
+>   declaration, and in large files the whole file, into a parse error that indexes as almost
+>   nothing. On Effect, Zod, Excalidraw and VS Code, the files that tsc accepts but the grammar
+>   rejects drop from 201 to 32 (upstream [#3226](https://github.com/colbymchenry/codegraph/pull/3226)
+>   covers variance only; see `docs/grammars/tree-sitter-typescript.md`). Re-index TypeScript
+>   projects after updating.
 >
 > Earlier fork fixes are now in upstream: `codegraph sync` on an index from an older engine
 > (#1842, which upstream made a warning rather than a refusal), imports onto an exported

__tests__/fixtures/ts-syntax/export-type-star.ts

@@ -0,0 +1,3 @@
+// Must keep parsing: `export type *` re-exports, TS 5.0 (fixed by the marionebl base).
+export type * from "./types";
+export type * as Types from "./types";

__tests__/fixtures/ts-syntax/export-type-star.tsx

@@ -0,0 +1,3 @@
+// Must keep parsing: `export type *` re-exports, TS 5.0 (fixed by the marionebl base).
+export type * from "./types";
+export type * as Types from "./types";

__tests__/fixtures/ts-syntax/import-type-primary.ts

@@ -0,0 +1,8 @@
+// Must keep parsing: an import type wherever a primary type can appear,
+// and `typeof import()` in a call's type arguments (dokimasia 55b0687, 96ce7e6).
+export type List = import("./types").Item[];
+export type Field = import("./types").Item["id"];
+export type Keys = keyof import("./types").Item;
+export type Boxed = import("./types").Box<string>;
+declare function load<T>(): T;
+export const m = load<typeof import("./types")>();

__tests__/fixtures/ts-syntax/import-type-primary.tsx

@@ -0,0 +1,8 @@
+// Must keep parsing: an import type wherever a primary type can appear,
+// and `typeof import()` in a call's type arguments (dokimasia 55b0687, 96ce7e6).
+export type List = import("./types").Item[];
+export type Field = import("./types").Item["id"];
+export type Keys = keyof import("./types").Item;
+export type Boxed = import("./types").Box<string>;
+declare function load<T>(): T;
+export const m = load<typeof import("./types")>();

__tests__/fixtures/ts-syntax/member-type-params-newline.ts

@@ -0,0 +1,16 @@
+// Must keep parsing: a member name with its type parameters on the next line
+// is one method (tsc), not a property plus a call signature. The class forms
+// fail to parse if split; the interface form only shows in node kinds
+// (__tests__/typescript-member-asi-parse.test.ts).
+export interface Repo {
+  find
+    <T>(id: string): T
+}
+export class Svc {
+  load
+    <T>(id: string): T { return null! }
+}
+export abstract class Base {
+  abstract make
+    <T>(): T
+}

__tests__/fixtures/ts-syntax/member-type-params-newline.tsx

@@ -0,0 +1,16 @@
+// Must keep parsing: a member name with its type parameters on the next line
+// is one method (tsc), not a property plus a call signature. The class forms
+// fail to parse if split; the interface form only shows in node kinds
+// (__tests__/typescript-member-asi-parse.test.ts).
+export interface Repo {
+  find
+    <T>(id: string): T
+}
+export class Svc {
+  load
+    <T>(id: string): T { return null! }
+}
+export abstract class Base {
+  abstract make
+    <T>(): T
+}

__tests__/fixtures/ts-syntax/overload-asi.ts

@@ -0,0 +1,11 @@
+// Must keep parsing: a generic call signature on its own line after a member
+// with no `;` (tree-sitter-typescript #335; fixed by the marionebl base).
+export interface Dual {
+  readonly name: string
+  <A>(self: A): A
+  <A, B>(self: A, that: B): A
+}
+export interface Keyed {
+  in: number
+  out: string
+}

__tests__/fixtures/ts-syntax/overload-asi.tsx

@@ -0,0 +1,11 @@
+// Must keep parsing: a generic call signature on its own line after a member
+// with no `;` (tree-sitter-typescript #335; fixed by the marionebl base).
+export interface Dual {
+  readonly name: string
+  <A>(self: A): A
+  <A, B>(self: A, that: B): A
+}
+export interface Keyed {
+  in: number
+  out: string
+}

__tests__/fixtures/ts-syntax/template-literal-escapes.ts

@@ -0,0 +1,5 @@
+// Must keep parsing: escape sequences inside template literal types
+// (fixed by the marionebl base).
+export type Line = `${string}\n`;
+export type Quoted = `\`${string}\``;
+export type Tab = `a\tb${number}`;

__tests__/fixtures/ts-syntax/template-literal-escapes.tsx

@@ -0,0 +1,5 @@
+// Must keep parsing: escape sequences inside template literal types
+// (fixed by the marionebl base).
+export type Line = `${string}\n`;
+export type Quoted = `\`${string}\``;
+export type Tab = `a\tb${number}`;

__tests__/typescript-member-asi-parse.test.ts

@@ -0,0 +1,102 @@
+/**
+ * The vendored grammar's scanner inserts an implicit `;` before a `<` that
+ * starts a line inside a type or class body, so a generic call-signature
+ * overload after a member with no `;` parses (tree-sitter-typescript #335):
+ *
+ *   interface I { a: string
+ *     <A>(self: A): A }        → property `a`, call signature
+ *
+ * Right after a bare member name, though, tsc reads the `<` as that member's
+ * type parameters, so `find\n<T>(id): T` is ONE method. Splitting it there
+ * turns a method into a property and drops its call edges, and the interface
+ * form parses without an error, so only the node kinds show it. The same holds
+ * for `(`: `find\n(id): T` is one method too.
+ *
+ * The scanner learns "right after a member name" from a never-produced
+ * external token that the grammar allows only there. Checking whether `(`
+ * could come next instead also fired after `typeof x` / `import("m").X`
+ * types and after `return`, which these cases pin.
+ * See docs/grammars/tree-sitter-typescript.md.
+ */
+import { describe, it, expect, beforeAll } from 'vitest';
+import type { Node } from 'web-tree-sitter';
+import { initGrammars, loadGrammarsForLanguages, getParser } from '../src/extraction/grammars';
+
+beforeAll(async () => {
+  await initGrammars();
+  await loadGrammarsForLanguages(['typescript', 'tsx']);
+});
+
+const MEMBER_KINDS = [
+  'method_signature',
+  'method_definition',
+  'abstract_method_signature',
+  'property_signature',
+  'public_field_definition',
+  'call_signature',
+  'construct_signature',
+];
+
+function members(language: 'typescript' | 'tsx', src: string): { hasError: boolean; kinds: string[] } {
+  const root: Node = getParser(language)!.parse(src)!.rootNode;
+  return { hasError: root.hasError, kinds: root.descendantsOfType(MEMBER_KINDS).map((n) => n.type) };
+}
+
+describe('TypeScript member ASI before `<`', () => {
+  it.each(['typescript', 'tsx'] as const)('%s: a member name then `<T>` on the next line is one method', (language) => {
+    const cases: Array<[string, string[]]> = [
+      ['interface Repo {\n  find\n    <T>(id: string): T\n}', ['method_signature']],
+      ['interface I {\n  a?\n  <T>(): T\n}', ['method_signature']],
+      ['class Svc {\n  load\n    <T>(id: string): T { return helper(id) }\n}', ['method_definition']],
+      ['abstract class A {\n  abstract m\n  <T>(): T\n}', ['abstract_method_signature']],
+      ['declare class D {\n  m\n  <T>(): T\n}', ['method_signature']],
+      ['interface I {\n  find\n  (id: string): T\n}', ['method_signature']],
+      ['interface I {\n  a?\n  (id: string): T\n}', ['method_signature']],
+      ['class C {\n  load\n  (id: string) { return helper(id) }\n}', ['method_definition']],
+    ];
+    for (const [src, want] of cases) {
+      const got = members(language, src);
+      expect(got.hasError, src).toBe(false);
+      expect(got.kinds, src).toEqual(want);
+    }
+  });
+
+  it.each(['typescript', 'tsx'] as const)('%s: after a type, `<` on the next line starts a call signature', (language) => {
+    const cases: Array<[string, string[]]> = [
+      ['interface I {\n  a: string\n  <A>(self: A): A\n}', ['property_signature', 'call_signature']],
+      ['interface I {\n  a: Foo\n  <A>(self: A): A\n}', ['property_signature', 'call_signature']],
+      ['interface I {\n  f(): void\n  <A>(self: A): A\n}', ['method_signature', 'call_signature']],
+      ['interface I {\n  (a: string)\n  <A>(self: A): A\n}', ['call_signature', 'call_signature']],
+      ['interface I {\n  a: string\n  (id: string): T\n}', ['property_signature', 'call_signature']],
+      ['interface I {\n  a: typeof b\n  <A>(self: A): A\n}', ['property_signature', 'call_signature']],
+      ['interface I {\n  a: typeof b.c\n  <A>(self: A): A\n}', ['property_signature', 'call_signature']],
+      ['interface I {\n  a: typeof this.x\n  <A>(self: A): A\n}', ['property_signature', 'call_signature']],
+      ['type O = {\n  a: keyof typeof b\n  <A>(self: A): A\n}', ['property_signature', 'call_signature']],
+      ['interface I {\n  a: import("m").X\n  <A>(self: A): A\n}', ['property_signature', 'call_signature']],
+      [
+        'type T = {\n  readonly a: number\n  <A>(self: A): A\n  new <B>(b: B): B\n}',
+        ['property_signature', 'call_signature', 'construct_signature'],
+      ],
+    ];
+    for (const [src, want] of cases) {
+      const got = members(language, src);
+      expect(got.hasError, src).toBe(false);
+      expect(got.kinds, src).toEqual(want);
+    }
+  });
+
+  it.each([
+    ['typescript', ['function f() {\n  return\n  <any>x\n}', 'function f() {\n  return\n  <T,>(x: T) => x\n}']],
+    ['tsx', ['function f() {\n  return\n  <div/>\n}', 'function f() {\n  return\n  <T,>(x: T) => x\n}']],
+  ] as const)('%s: `return` then `<` on the next line still ends the return', (language, sources) => {
+    // tsc: `return;` then a type-assertion / generic-arrow / JSX statement.
+    for (const src of sources) {
+      const root = getParser(language)!.parse(src)!.rootNode;
+      expect(root.hasError, src).toBe(false);
+      const ret = root.descendantsOfType('return_statement');
+      expect(ret, src).toHaveLength(1);
+      expect(ret[0]!.namedChildCount, src).toBe(0);
+      expect(root.descendantsOfType('expression_statement'), src).toHaveLength(1);
+    }
+  });
+});

__tests__/typescript-variance-parse.test.ts

@@ -2,9 +2,10 @@
  * TS 4.7 variance annotations (`interface Stream<out A, in E>`, `<in out T>`)
  * were parse errors in tree-sitter-typescript v0.23.2 (upstream #370), and in
  * a big file such errors can pile up into a whole-file ERROR. The vendored
- * grammar (wasm and kernel) is patched; see docs/grammars/tree-sitter-typescript.md.
+ * grammar (wasm and kernel) is built from a fork that parses them, plus our
+ * patch; see docs/grammars/tree-sitter-typescript.md.
  *
- * The patch makes `in`/`out` keywords where a type parameter starts, so the
+ * The grammar makes `in`/`out` keywords where a type parameter starts, so the
  * cases below also pin that `out` still works as a name where real code uses
  * it: the upstream fix PRs as written break `type F<out> = out[]`, and
  * accepting `out` as a type name everywhere broke `[out: T]` tuple labels
@@ -29,13 +30,19 @@ function parse(language: 'typescript' | 'tsx', src: string) {
   return getParser(language)!.parse(src)!.rootNode;
 }
 
-/** The modifier tokens of every type_parameter, and its name. */
+/**
+ * The modifier tokens of every type_parameter, and its name. `in` / `out`
+ * sit inside a `variance` node (the base grammar's shape); `const` sits
+ * directly under the type_parameter.
+ */
 function typeParams(root: Node): string[] {
   return root.descendantsOfType('type_parameter').map((p) => {
     const mods: string[] = [];
     for (let i = 0; i < p.childCount; i++) {
       const c = p.child(i)!;
-      if (!c.isNamed && ['in', 'out', 'const'].includes(c.type)) mods.push(c.type);
+      if (c.type === 'variance') {
+        for (let j = 0; j < c.childCount; j++) mods.push(c.child(j)!.type);
+      } else if (!c.isNamed && c.type === 'const') mods.push(c.type);
     }
     return [...mods, p.childForFieldName('name')!.text].join(' ');
   });

codegraph-kernel/Cargo.toml

@@ -21,7 +21,7 @@ regex = "1"
 # kernel-grammar-parity test asserts node-kind-table equality at test time;
 # bump these together with the wasm side or that gate fails.
 # typescript + tsx: vendored C (grammars/typescript, compiled in build.rs) —
-# v0.23.2 carries docs/grammars/tree-sitter-typescript.patch, regenerated.
+# marionebl 53071c9 carries docs/grammars/tree-sitter-typescript.patch, regenerated.
 tree-sitter-javascript = "0.25"
 tree-sitter-java = "0.23"
 tree-sitter-python = "0.23"

codegraph-kernel/build.rs

@@ -114,16 +114,18 @@ fn main() {
     println!("cargo:rerun-if-changed=grammars/swift");
 
     // TypeScript + TSX grammars — vendored C (sixth vendored-grammar-C
-    // language): tree-sitter/tree-sitter-typescript v0.23.2 (f975a62) with
+    // language): marionebl/tree-sitter-typescript 53071c9 (a maintained fork
+    // of tree-sitter/tree-sitter-typescript v0.23.2) with
     // docs/grammars/tree-sitter-typescript.patch applied to the grammar source,
     // then REGENERATED with tree-sitter-cli 0.25.10 (ABI 15) — the patch
     // touches grammar rules, so the checked-in parser.c can't be reused. The
     // vendored wasm is built from these same generated sources. Layout mirrors
     // the upstream repo so scanner.c's `../../common/scanner.h` resolves
-    // unedited. The patch: TS 4.7 variance annotations (`<in T>`, `<out T>`).
-    //   common/scanner.h           da66ef2bd14a3f7ea743e25ba068c6c9aae2c3509db200ff80c4a0e6116e564c (unpatched)
-    //   typescript/src/parser.c    eafc3e8cd89fbbfdcec9182b22446f19c31756f3c8c1212cfd7bffdb4b761424
-    //   tsx/src/parser.c           232fe5b8178180307a40b393713d82074920977b147638bae898874085bcee86
+    // unedited. See docs/grammars/tree-sitter-typescript.md for what the base
+    // and the patch each fix.
+    //   common/scanner.h           834a0b60fba90355410cf7456069d9e1dd1aac309fdc8485f80d64724fc1e98b (marionebl + patch)
+    //   typescript/src/parser.c    2617ac3e51b9938277d904d58b02e454183b0c86e04bb5fdd9792adb60506a26
+    //   tsx/src/parser.c           f6aadca29573f480ea13f5b3f9db803fceba34c4da50f1008f3594e2ffe60961
     // Flags crib the crate's bindings/rust/build.rs (c11). Each parser.c is
     // ~9 MB.
     for dialect in ["typescript", "tsx"] {

codegraph-kernel/grammars/typescript/common/scanner.h

@@ -13,6 +13,7 @@ enum TokenType {
     JSX_TEXT,
     FUNCTION_SIGNATURE_AUTOMATIC_SEMICOLON,
     ERROR_RECOVERY,
+    MEMBER_SIGNATURE_MARKER,
 };
 
 static void advance(TSLexer *lexer) { lexer->advance(lexer, false); }
@@ -123,7 +124,6 @@ static bool scan_automatic_semicolon(TSLexer *lexer, const bool *valid_symbols,
         case '*':
         case '%':
         case '>':
-        case '<':
         case '=':
         case '?':
         case '^':
@@ -139,14 +139,30 @@ static bool scan_automatic_semicolon(TSLexer *lexer, const bool *valid_symbols,
             }
             break;
 
-            // Don't insert a semicolon before a '[' or '(', unless we're parsing
-            // a type. Detect whether we're parsing a type or an expression using
-            // the validity of a binary operator token.
+            // Don't insert a semicolon before a '[', '(', or '<', unless we're
+            // parsing a type. Detect whether we're parsing a type or an expression
+            // using the validity of a binary operator token.
+            //
+            // This covers the case where a generic call-signature overload starts
+            // on a new line inside an object type, e.g.:
+            //
+            //   const f: {
+            //     (): string
+            //     <T>(): T    ← the `<` here needs an implicit semicolon before it
+            //   }
         case '(':
         case '[':
+        case '<':
             if (valid_symbols[LOGICAL_OR]) {
                 return false;
             }
+            // codegraph delta: right after a member name (`find`, `find?`) a
+            // `<` or `(` on the next line opens that member's signature, as tsc
+            // reads it, so no semicolon. MEMBER_SIGNATURE_MARKER is never
+            // produced; the grammar makes it valid only in that position.
+            if ((lexer->lookahead == '<' || lexer->lookahead == '(') && valid_symbols[MEMBER_SIGNATURE_MARKER]) {
+                return false;
+            }
             break;
 
             // Insert a semicolon before `--` and `++`, but not before binary `+` or `-`.
@@ -163,7 +179,10 @@ static bool scan_automatic_semicolon(TSLexer *lexer, const bool *valid_symbols,
             return lexer->lookahead != '=';
 
             // Don't insert a semicolon before `in` or `instanceof`, but do insert one
-            // before an identifier.
+            // before an identifier.  Also insert one before `in` when it is used as
+            // a property/method name (i.e. followed by `:` or `(`), because
+            //   interface Foo { name: string\n  in: string }
+            // must parse `in:` as a property name, not as a keyword operator.
         case 'i':
             skip(lexer);
 
@@ -172,7 +191,13 @@ static bool scan_automatic_semicolon(TSLexer *lexer, const bool *valid_symbols,
             }
             skip(lexer);
 
-            if (!iswalpha(lexer->lookahead)) {
+            if (!iswalpha(lexer->lookahead) && lexer->lookahead != '_') {
+                // `in` followed by non-word: could be `in `, `in:`, `in(`, etc.
+                // Insert ASI if the next non-space char is `:` or `(` (property name),
+                // but not if it is a space followed by an expression (keyword operator).
+                if (lexer->lookahead == ':' || lexer->lookahead == '(') {
+                    return true;
+                }
                 return false;
             }