grammar(ts): vendor our tree-sitter-typescript fork (84ffdb8)

README.md

@@ -10,12 +10,14 @@
 > - **Much more TypeScript parses.** The TypeScript grammar is built from
 >   [marionebl/tree-sitter-typescript](https://github.com/marionebl/tree-sitter-typescript),
 >   a maintained fork of the stalled upstream grammar, with two fixes from
->   [dokimasia's fork](https://github.com/dokimasia/tree-sitter-typescript) and our own patch.
->   Variance annotations (`interface Stream<out A, in E>`), generic overloads after a member with
->   no `;`, `export type *`, `import("m").X[]` and template-literal escapes no longer turn a
->   declaration, and in large files the whole file, into a parse error that indexes as almost
->   nothing. On Effect, Zod, Excalidraw and VS Code, the files that tsc accepts but the grammar
->   rejects drop from 201 to 32 (upstream [#3226](https://github.com/colbymchenry/codegraph/pull/3226)
+>   [dokimasia's fork](https://github.com/dokimasia/tree-sitter-typescript) and our own, kept in
+>   our grammar fork (checked against tsc in its CI). Variance annotations
+>   (`interface Stream<out A, in E>`), generic overloads after a member with no `;`, tagged
+>   templates with type arguments (``sql<Row>`...` ``), `export type *`, `import("m").X[]`,
+>   keyword-named tuple labels and template-literal escapes no longer turn a declaration, and in
+>   large files the whole file, into a parse error that indexes as almost nothing. On Effect, Zod,
+>   Excalidraw and VS Code, the files that tsc accepts but the grammar rejects drop from 201 to 5
+>   (upstream [#3226](https://github.com/colbymchenry/codegraph/pull/3226)
 >   covers variance only; see `docs/grammars/tree-sitter-typescript.md`). Re-index TypeScript
 >   projects after updating.
 >

__tests__/fixtures/ts-syntax/gap-unique-in-comparison.ts

@@ -0,0 +1,5 @@
+// Gap: `unique` lexes as the keyword of `unique symbol` wherever type
+// arguments can start, so `i < unique.length` fails. Fixing it by making
+// `unique` a reserved identifier broke `f<typeof unique>(x)` instead.
+declare const unique: string[];
+for (let i = 0; i < unique.length; i++) {}

__tests__/fixtures/ts-syntax/gap-unique-in-comparison.tsx

@@ -0,0 +1,5 @@
+// Gap: `unique` lexes as the keyword of `unique symbol` wherever type
+// arguments can start, so `i < unique.length` fails. Fixing it by making
+// `unique` a reserved identifier broke `f<typeof unique>(x)` instead.
+declare const unique: string[];
+for (let i = 0; i < unique.length; i++) {}

__tests__/fixtures/ts-syntax/tagged-template-type-args.ts

@@ -0,0 +1,9 @@
+// Must keep parsing: a tagged template with type arguments (Effect's sql<Row>`...`).
+declare function sql<T>(s: TemplateStringsArray, ...v: unknown[]): T;
+declare const table: string;
+export const rows = sql<{ id: number; name: string }>`SELECT * FROM ${table}`;
+export const one = sql<number>`SELECT 1`.valueOf();
+export const nested = sql<Array<{ readonly count: number }>>`
+  SELECT COUNT(*) as count FROM ${table}
+`;
+export const lt = 1 < 2;

__tests__/fixtures/ts-syntax/tagged-template-type-args.tsx

@@ -0,0 +1,9 @@
+// Must keep parsing: a tagged template with type arguments (Effect's sql<Row>`...`).
+declare function sql<T>(s: TemplateStringsArray, ...v: unknown[]): T;
+declare const table: string;
+export const rows = sql<{ id: number; name: string }>`SELECT * FROM ${table}`;
+export const one = sql<number>`SELECT 1`.valueOf();
+export const nested = sql<Array<{ readonly count: number }>>`
+  SELECT COUNT(*) as count FROM ${table}
+`;
+export const lt = 1 < 2;

__tests__/fixtures/ts-syntax/tuple-keyword-type-names.ts

@@ -0,0 +1,12 @@
+// Must keep parsing: tuple elements that are types named like contextual
+// keywords, and `typeof unique` in type arguments.
+type type = 1; type get = 2; type set = 3; type declare = 4;
+export type A = [type];
+export type B = [get, set];
+export type C = [declare?];
+export type D = [type[]];
+export function f(a: [get]): void {}
+declare const unique: number;
+declare function g<T>(x: T): T;
+export const r = g<typeof unique>(1);
+export type E = (typeof unique)[];

__tests__/fixtures/ts-syntax/tuple-keyword-type-names.tsx

@@ -0,0 +1,12 @@
+// Must keep parsing: tuple elements that are types named like contextual
+// keywords, and `typeof unique` in type arguments.
+type type = 1; type get = 2; type set = 3; type declare = 4;
+export type A = [type];
+export type B = [get, set];
+export type C = [declare?];
+export type D = [type[]];
+export function f(a: [get]): void {}
+declare const unique: number;
+declare function g<T>(x: T): T;
+export const r = g<typeof unique>(1);
+export type E = (typeof unique)[];

__tests__/fixtures/ts-syntax/tuple-label-keyword.ts

@@ -0,0 +1,7 @@
+// Must keep parsing: a tuple label named like a type keyword (VS Code's terminalVoice.ts).
+export type A = [spoken: string, symbol: string][];
+export type B = [string: number, number?: string];
+export type C = [any: 1, ...object: unknown[]];
+export type D = [any?];
+export type E = [readonly string[]];
+export type F = [readonly: string];

__tests__/fixtures/ts-syntax/tuple-label-keyword.tsx

@@ -0,0 +1,7 @@
+// Must keep parsing: a tuple label named like a type keyword (VS Code's terminalVoice.ts).
+export type A = [spoken: string, symbol: string][];
+export type B = [string: number, number?: string];
+export type C = [any: 1, ...object: unknown[]];
+export type D = [any?];
+export type E = [readonly string[]];
+export type F = [readonly: string];

__tests__/typescript-grammar-fork-shapes.test.ts

@@ -0,0 +1,89 @@
+/**
+ * Node shapes and extraction for the constructs our tree-sitter-typescript fork
+ * added (docs/grammars/tree-sitter-typescript.md). The tsc differential only
+ * checks that a file parses; these pin what it parses into, and the call edges
+ * codegraph derives from it, so a later grammar move cannot regress them silently.
+ */
+import { describe, it, expect, beforeAll } from 'vitest';
+import type { Node } from 'web-tree-sitter';
+import { extractFromSource } from '../src/extraction';
+import { initGrammars, loadGrammarsForLanguages, getParser } from '../src/extraction/grammars';
+
+beforeAll(async () => {
+  await initGrammars();
+  await loadGrammarsForLanguages(['typescript', 'tsx']);
+});
+
+const LANGS = ['typescript', 'tsx'] as const;
+
+function parse(language: 'typescript' | 'tsx', src: string): Node {
+  return getParser(language)!.parse(src)!.rootNode;
+}
+
+/** Names this file's code calls, as codegraph records them. */
+function calls(language: 'typescript' | 'tsx', src: string): string[] {
+  const file = language === 'tsx' ? 'src/a.tsx' : 'src/a.ts';
+  return extractFromSource(file, src, language)
+    .unresolvedReferences.filter((r) => r.referenceKind === 'calls')
+    .map((r) => r.referenceName);
+}
+
+describe('tagged templates with type arguments', () => {
+  it.each(LANGS)('%s: sql<Row>`...` is a call of sql with type arguments', (language) => {
+    const root = parse(language, 'sql<{ id: number }>`SELECT ${x}`;\nconst lt = a < b;');
+    expect(root.hasError).toBe(false);
+    const call = root.descendantsOfType('call_expression')[0]!;
+    expect(call.childForFieldName('function')!.text).toBe('sql');
+    expect(call.childForFieldName('type_arguments')!.type).toBe('type_arguments');
+    expect(call.childForFieldName('arguments')!.type).toBe('template_string');
+    expect(root.descendantsOfType('binary_expression')).toHaveLength(1);
+  });
+
+  it.each(LANGS)('%s: the tag is a call edge, also behind await and on a member', (language) => {
+    const src = [
+      'export async function load() {',
+      '  const a = sql<Row>`SELECT 1`;',
+      '  const b = await query<Row[]>`SELECT 2`;',
+      '  const c = db.tag<Row>`SELECT 3`;',
+      '}',
+    ].join('\n');
+    expect(calls(language, src)).toEqual(expect.arrayContaining(['sql', 'query', 'db.tag']));
+  });
+});
+
+describe('tuple labels named like type keywords', () => {
+  it.each(LANGS)('%s: keyword labels are names, keyword-named types stay types', (language) => {
+    const cases: Array<[string, Record<string, number>]> = [
+      ['type A = [symbol: string, number?: string];', { required_parameter: 1, optional_parameter: 1 }],
+      ['type B = [readonly: string, new: number];', { required_parameter: 2 }],
+      ['type C = [any?];', { optional_type: 1 }],
+      ['type D = [readonly string[]];', { readonly_type: 1 }],
+      ['type E = [type, get?];', { type_identifier: 3, optional_type: 1 }],
+      ['function f(a: [declare]) {}', { type_identifier: 1 }],
+    ];
+    for (const [src, want] of cases) {
+      const root = parse(language, src);
+      expect(root.hasError, src).toBe(false);
+      for (const [kind, n] of Object.entries(want)) {
+        expect(root.descendantsOfType(kind), `${src} ${kind}`).toHaveLength(n);
+      }
+    }
+  });
+});
+
+describe('typeof unique in type arguments', () => {
+  it.each(LANGS)('%s: f<typeof unique>(x) stays a generic call', (language) => {
+    const src = 'export function g() { return f<typeof unique>(1); }\ntype E = (typeof unique)[];';
+    expect(parse(language, src).hasError).toBe(false);
+    expect(calls(language, src)).toContain('f');
+    for (const more of [
+      'f<typeof unique.a>(x);',
+      'f<typeof unique[]>(x);',
+      'f<(typeof unique)[]>(x);',
+      'type A = (typeof unique);',
+      'type T = (typeof unique)["x"];',
+    ]) {
+      expect(parse(language, more).hasError, more).toBe(false);
+    }
+  });
+});

codegraph-kernel/Cargo.toml

@@ -21,7 +21,7 @@ regex = "1"
 # kernel-grammar-parity test asserts node-kind-table equality at test time;
 # bump these together with the wasm side or that gate fails.
 # typescript + tsx: vendored C (grammars/typescript, compiled in build.rs) —
-# marionebl 53071c9 carries docs/grammars/tree-sitter-typescript.patch, regenerated.
+# our fork inth3shadows/tree-sitter-typescript (marionebl 53071c9 + docs/grammars/tree-sitter-typescript.patch), regenerated.
 tree-sitter-javascript = "0.25"
 tree-sitter-java = "0.23"
 tree-sitter-python = "0.23"

codegraph-kernel/build.rs

@@ -114,18 +114,20 @@ fn main() {
     println!("cargo:rerun-if-changed=grammars/swift");
 
     // TypeScript + TSX grammars — vendored C (sixth vendored-grammar-C
-    // language): marionebl/tree-sitter-typescript 53071c9 (a maintained fork
-    // of tree-sitter/tree-sitter-typescript v0.23.2) with
-    // docs/grammars/tree-sitter-typescript.patch applied to the grammar source,
-    // then REGENERATED with tree-sitter-cli 0.25.10 (ABI 15) — the patch
-    // touches grammar rules, so the checked-in parser.c can't be reused. The
+    // language): our fork inth3shadows/tree-sitter-typescript (the commit is
+    // named in docs/grammars/tree-sitter-typescript.md), i.e.
+    // marionebl/tree-sitter-typescript 53071c9 (a maintained fork of
+    // tree-sitter/tree-sitter-typescript v0.23.2) plus
+    // docs/grammars/tree-sitter-typescript.patch, REGENERATED with
+    // tree-sitter-cli 0.25.10 (ABI 15) — marionebl's checked-in parser.c is
+    // ABI 14 and lacks the patch, so it can't be reused. The
     // vendored wasm is built from these same generated sources. Layout mirrors
     // the upstream repo so scanner.c's `../../common/scanner.h` resolves
     // unedited. See docs/grammars/tree-sitter-typescript.md for what the base
-    // and the patch each fix.
-    //   common/scanner.h           834a0b60fba90355410cf7456069d9e1dd1aac309fdc8485f80d64724fc1e98b (marionebl + patch)
-    //   typescript/src/parser.c    2617ac3e51b9938277d904d58b02e454183b0c86e04bb5fdd9792adb60506a26
-    //   tsx/src/parser.c           f6aadca29573f480ea13f5b3f9db803fceba34c4da50f1008f3594e2ffe60961
+    // and the fork each fix.
+    //   common/scanner.h           6bf2fd5994bc8a4b6777c5b8ee63fa2fbe5c206ec59b06cf398a9d78ae4c3142
+    //   typescript/src/parser.c    d4033c44e73006419f890b41780324c9bc26a2679e9045956210331a9d6d3c4a
+    //   tsx/src/parser.c           5372712651e0d900af85fb760f358a7b0b39a3ef624f575d4952c84e5e2ff2a7
     // Flags crib the crate's bindings/rust/build.rs (c11). Each parser.c is
     // ~9 MB.
     for dialect in ["typescript", "tsx"] {

codegraph-kernel/grammars/typescript/common/scanner.h

@@ -156,7 +156,7 @@ static bool scan_automatic_semicolon(TSLexer *lexer, const bool *valid_symbols,
             if (valid_symbols[LOGICAL_OR]) {
                 return false;
             }
-            // codegraph delta: right after a member name (`find`, `find?`) a
+            // right after a member name (`find`, `find?`) a
             // `<` or `(` on the next line opens that member's signature, as tsc
             // reads it, so no semicolon. MEMBER_SIGNATURE_MARKER is never
             // produced; the grammar makes it valid only in that position.