fix(extraction): vendor tree-sitter runtime 0.25.10 + #5997 (error-recovery livelock)

.gitattributes

@@ -7,3 +7,5 @@
 # reach the files inside it.
 codegraph-kernel/grammars/** linguist-vendored
 codegraph-kernel/grammars/**/parser.c linguist-generated
+# The kernel's patched copy of the tree-sitter runtime crate (docs/grammars/web-tree-sitter-runtime.md).
+codegraph-kernel/vendor/** linguist-vendored

.github/workflows/release.yml

@@ -188,7 +188,7 @@ jobs:
           if [ -f release/kernel/linux-x64/codegraph-kernel.node ]; then
             mkdir -p codegraph-kernel/prebuilds/linux-x64
             cp release/kernel/linux-x64/codegraph-kernel.node codegraph-kernel/prebuilds/linux-x64/
-            CODEGRAPH_KERNEL_EXPECT=1 npx vitest run __tests__/kernel-*.test.ts
+            CODEGRAPH_KERNEL_EXPECT=1 npx vitest run __tests__/kernel-*.test.ts __tests__/tsx-error-recovery-hang.test.ts
           else
             echo "::error::linux-x64 kernel prebuild missing despite required matrix"
             exit 1

.github/workflows/ts-differential.yml

@@ -13,6 +13,8 @@ on:
       - 'codegraph-kernel/grammars/typescript/**'
       - 'docs/grammars/tree-sitter-typescript.*'
       - 'scripts/ts-differential.mjs'
+      - 'scripts/tree-sitter-runtime.mjs'
+      - 'src/extraction/runtime/**'
       - 'scripts/ts-differential.corpora.json'
       - 'scripts/ts-differential.baseline.json'
       - '__tests__/fixtures/ts-syntax/**'
@@ -27,6 +29,8 @@ on:
       - 'codegraph-kernel/grammars/typescript/**'
       - 'docs/grammars/tree-sitter-typescript.*'
       - 'scripts/ts-differential.mjs'
+      - 'scripts/tree-sitter-runtime.mjs'
+      - 'src/extraction/runtime/**'
       - 'scripts/ts-differential.corpora.json'
       - 'scripts/ts-differential.baseline.json'
       - '__tests__/fixtures/ts-syntax/**'

CHANGELOG.md

@@ -53,6 +53,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Fixes
 
+- A TSX file with a broken JSX attribute such as `<a {{b:>c:d(e f)` no longer stalls indexing for about 90 seconds and then drops out of the index; it is indexed like any other file with a syntax error.
 - TypeScript variance annotations like `interface Stream<out A, in E>` or `<in out T>` now parse. Before, each one was a parse error, and in a large file enough of them could make the whole file fail to parse and show up with almost no symbols. Re-index TypeScript projects after upgrading.
 - In TypeScript, an awaited call with type arguments like `await fetchApi<User>("/me")` or `await api.get<T>(url)` now links to the function it calls. Before, the call was dropped, so typed fetch wrappers looked like they had no callers. (#2586)
 - In JavaScript and TypeScript, a call or `new` in a parameter's default value, like `function renderWithNote(note = buildNote())`, a destructured `{ size = defaultSize() }` or `constructor(private svc = new Service())`, now counts as the function's own call. Before, it was dropped, so `buildNote` was missing that caller and impact on it came back incomplete. Re-index JavaScript and TypeScript projects after upgrading. Thanks @inth3shadows for the report and the pull request. (#2588)

__tests__/grammar-wasm-bytes.test.ts

@@ -9,7 +9,7 @@
  */
 import { describe, it, expect } from 'vitest';
 import { Parser, Language as WasmLanguage } from 'web-tree-sitter';
-import { readGrammarWasmBytes } from '../src/extraction/grammars';
+import { initGrammars, readGrammarWasmBytes } from '../src/extraction/grammars';
 
 describe('readGrammarWasmBytes', () => {
   it('reads bytes for a tree-sitter-wasms grammar and a vendored grammar', async () => {
@@ -31,7 +31,7 @@ describe('readGrammarWasmBytes', () => {
   });
 
   it('produces bytes web-tree-sitter can load into a working parser', async () => {
-    await Parser.init();
+    await initGrammars();
     const bytes = await readGrammarWasmBytes(['javascript']);
     const language = await WasmLanguage.load(bytes.javascript);
     const parser = new Parser();

__tests__/tsx-error-recovery-hang.test.ts

@@ -0,0 +1,96 @@
+/**
+ * tree-sitter's error recovery before 0.27.1 never finishes on some broken
+ * input (tree-sitter/tree-sitter#5997; tree-sitter-typescript upstream #323):
+ * a TSX file holding `<a {{b:>c:d(e f)` timed out the parse worker three
+ * times (3 x 30s) and was then dropped from the index. The grammar is not at
+ * fault, so this pins the runtime: each input must give a tree, errors and
+ * all, well inside the budget. web-tree-sitter 0.25.10's own tree-sitter.wasm
+ * fails it; the vendored src/extraction/runtime/tree-sitter.wasm (0.25.10 +
+ * tree-sitter/tree-sitter#5997) passes. See docs/grammars/web-tree-sitter-runtime.md.
+ */
+import { describe, it, expect, beforeAll } from 'vitest';
+import * as fs from 'fs';
+import * as path from 'path';
+import { spawnSync } from 'child_process';
+import { createHash } from 'crypto';
+import { initGrammars, loadGrammarsForLanguages, getParser, RUNTIME_WASM_PATH } from '../src/extraction/grammars';
+import { parseWithinBudget } from '../src/extraction/parse-budget';
+
+// The vendored runtime wasm is paired with web-tree-sitter's JS glue: both must be 0.25.10.
+const VENDORED_RUNTIME_VERSION = '0.25.10';
+
+beforeAll(async () => {
+  await initGrammars();
+  await loadGrammarsForLanguages(['typescript', 'tsx']);
+});
+
+const HANG_INPUTS = [
+  '<a {{b:>c:d(e f)\n',
+  '<div style={{ textAlign:>Note: This is important. (And this)</div>\n',
+];
+
+describe('vendored tree-sitter runtime', () => {
+  it('is in place and matches the installed web-tree-sitter glue', () => {
+    expect(fs.existsSync(RUNTIME_WASM_PATH)).toBe(true);
+    const pkg = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'node_modules', 'web-tree-sitter', 'package.json'), 'utf8'));
+    expect(
+      pkg.version,
+      'web-tree-sitter changed: rebuild src/extraction/runtime/tree-sitter.wasm for it (docs/grammars/web-tree-sitter-runtime.md)',
+    ).toBe(VENDORED_RUNTIME_VERSION);
+  });
+});
+
+// The native kernel links the same runtime C (codegraph-kernel/vendor/tree-sitter, patched).
+// A native parse cannot be cancelled from JS, so it runs in a child with a hard timeout.
+// The kernel throws `defer: parse tree contains errors` for an erroring tree (the file then
+// goes to wasm): a finished parse that went through error recovery, which is what this pins.
+// Only tsx + the first input hangs an unpatched kernel; the rest pin that recovery still errors.
+const KERNEL_PATH = path.join(__dirname, '..', 'codegraph-kernel', 'prebuilds', `${process.platform}-${process.arch}`, 'codegraph-kernel.node');
+const kernelBuilt = fs.existsSync(KERNEL_PATH);
+const expectKernel = process.env.CODEGRAPH_KERNEL_EXPECT === '1';
+const KERNEL_CHILD = `
+const k = require(process.argv[1]);
+const [src, lang] = [process.argv[2], process.argv[3]];
+try { k.extractFile(lang === 'tsx' ? 'poc.tsx' : 'poc.ts', src, lang); console.log('clean'); }
+catch (e) { if (!String(e && e.message).startsWith('defer:')) throw e; console.log('defer'); }
+`;
+
+// The kernel's copy of the runtime C: crates.io tree-sitter 0.25.10 src/parser.c + the #5997
+// patch. Re-vendoring without the patch changes this hash; CI catches it without a kernel build.
+const VENDORED_PARSER_C = path.join(__dirname, '..', 'codegraph-kernel', 'vendor', 'tree-sitter', 'src', 'parser.c');
+const VENDORED_PARSER_C_SHA256 = '612383a469d79c61f8b5b749714d60a6a85ca2b49c708e10ad0901be5b3987d0';
+
+describe('kernel runtime', () => {
+  it('vendored parser.c carries the #5997 patch', () => {
+    const sha = createHash('sha256').update(fs.readFileSync(VENDORED_PARSER_C)).digest('hex');
+    expect(sha, 'codegraph-kernel/vendor/tree-sitter/src/parser.c changed: re-apply docs/grammars/web-tree-sitter-runtime.patch, then update this hash').toBe(VENDORED_PARSER_C_SHA256);
+  });
+
+  it.runIf(expectKernel)('kernel binary must exist when CODEGRAPH_KERNEL_EXPECT=1', () => {
+    expect(kernelBuilt, `expected kernel at ${KERNEL_PATH} — run scripts/build-kernel.sh`).toBe(true);
+  });
+});
+
+describe.skipIf(!kernelBuilt)('kernel: error recovery finishes on the same input', () => {
+  it.each(['tsx', 'typescript'] as const)('%s', (language) => {
+    for (const src of HANG_INPUTS) {
+      const child = spawnSync(process.execPath, ['-e', KERNEL_CHILD, KERNEL_PATH, src, language], {
+        encoding: 'utf8',
+        timeout: 10_000,
+      });
+      expect(child.error?.message, `${language}: ${JSON.stringify(src)} — kernel parse did not finish within 10s`).toBeUndefined();
+      expect(child.stdout.trim(), child.stderr).toBe('defer');
+    }
+  });
+});
+
+describe('error recovery finishes on input that once looped forever', () => {
+  it.each(['tsx', 'typescript'] as const)('%s', (language) => {
+    for (const src of HANG_INPUTS) {
+      const tree = parseWithinBudget(getParser(language)!, src, 2000);
+      expect(tree, `${language}: ${JSON.stringify(src)} did not finish within 2000ms`).not.toBeNull();
+      expect(tree!.rootNode.hasError).toBe(true);
+      tree!.delete();
+    }
+  });
+});

codegraph-kernel/Cargo.toml

@@ -12,7 +12,8 @@ crate-type = ["cdylib"]
 [dependencies]
 napi = { version = "3", default-features = false, features = ["napi8"] }
 napi-derive = "3"
-tree-sitter = "0.25"
+# Exact: the [patch.crates-io] entry below replaces exactly this version.
+tree-sitter = "=0.25.10"
 sha2 = "0.10"
 regex = "1"
 
@@ -76,3 +77,9 @@ cc = "1"
 lto = true
 codegen-units = 1
 strip = "symbols"
+
+# tree-sitter 0.25.10 + tree-sitter/tree-sitter#5997 (error-recovery livelock), the same
+# runtime C as the wasm path's src/extraction/runtime/tree-sitter.wasm. Patch:
+# docs/grammars/web-tree-sitter-runtime.patch. See docs/grammars/web-tree-sitter-runtime.md.
+[patch.crates-io]
+tree-sitter = { path = "vendor/tree-sitter" }

codegraph-kernel/vendor/tree-sitter/Cargo.toml

@@ -0,0 +1,170 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
+#
+# When uploading crates to the registry Cargo will automatically
+# "normalize" Cargo.toml files for maximal compatibility
+# with all versions of Cargo and also rewrite `path` dependencies
+# to registry (e.g., crates.io) dependencies.
+#
+# If you are reading this file be aware that the original Cargo.toml
+# will likely look very different (and much more reasonable).
+# See Cargo.toml.orig for the original contents.
+
+[package]
+edition = "2021"
+rust-version = "1.76"
+name = "tree-sitter"
+version = "0.25.10"
+authors = [
+    "Max Brunsfeld <maxbrunsfeld@gmail.com>",
+    "Amaan Qureshi <amaanq12@gmail.com>",
+]
+build = "binding_rust/build.rs"
+links = "tree-sitter"
+include = [
+    "/binding_rust/*",
+    "/Cargo.toml",
+    "/src/*.h",
+    "/src/*.c",
+    "/src/portable/*",
+    "/src/unicode/*",
+    "/src/wasm/*",
+    "/include/tree_sitter/api.h",
+]
+autolib = false
+autobins = false
+autoexamples = false
+autotests = false
+autobenches = false
+description = "Rust bindings to the Tree-sitter parsing library"
+homepage = "https://tree-sitter.github.io/tree-sitter"
+readme = "binding_rust/README.md"
+keywords = [
+    "incremental",
+    "parsing",
+]
+categories = [
+    "api-bindings",
+    "external-ffi-bindings",
+    "parsing",
+    "text-editors",
+]
+license = "MIT"
+repository = "https://github.com/tree-sitter/tree-sitter"
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = [
+    "--cfg",
+    "docsrs",
+]
+targets = [
+    "x86_64-unknown-linux-gnu",
+    "x86_64-pc-windows-gnu",
+]
+
+[features]
+default = ["std"]
+std = [
+    "regex/std",
+    "regex/perf",
+    "regex-syntax/unicode",
+]
+wasm = [
+    "std",
+    "wasmtime-c-api",
+]
+
+[lib]
+name = "tree_sitter"
+path = "binding_rust/lib.rs"
+
+[dependencies.regex]
+version = "1.11.1"
+features = ["unicode"]
+default-features = false
+
+[dependencies.regex-syntax]
+version = "0.8.5"
+default-features = false
+
+[dependencies.streaming-iterator]
+version = "0.1.9"
+
+[dependencies.tree-sitter-language]
+version = "0.1"
+
+[dependencies.wasmtime-c-api]
+version = "29.0.1"
+features = [
+    "cranelift",
+    "gc-drc",
+]
+optional = true
+default-features = false
+package = "wasmtime-c-api-impl"
+
+[build-dependencies.bindgen]
+version = "0.71.1"
+optional = true
+
+[build-dependencies.cc]
+version = "1.2.10"
+
+[build-dependencies.serde_json]
+version = "1.0.137"
+features = ["preserve_order"]
+
+[lints.clippy]
+branches_sharing_code = "allow"
+cast_lossless = "allow"
+cast_possible_truncation = "allow"
+cast_possible_wrap = "allow"
+cast_precision_loss = "allow"
+cast_sign_loss = "allow"
+checked_conversions = "allow"
+cognitive_complexity = "allow"
+collection_is_never_read = "allow"
+dbg_macro = "deny"
+fallible_impl_from = "allow"
+fn_params_excessive_bools = "allow"
+if_not_else = "allow"
+inline_always = "allow"
+items_after_statements = "allow"
+match_wildcard_for_single_variants = "allow"
+missing_errors_doc = "allow"
+missing_panics_doc = "allow"
+module_name_repetitions = "allow"
+multiple_crate_versions = "allow"
+needless_for_each = "allow"
+obfuscated_if_else = "allow"
+option_if_let_else = "allow"
+or_fun_call = "allow"
+range_plus_one = "allow"
+redundant_clone = "allow"
+redundant_closure_for_method_calls = "allow"
+ref_option = "allow"
+similar_names = "allow"
+string_lit_as_bytes = "allow"
+struct_excessive_bools = "allow"
+struct_field_names = "allow"
+todo = "deny"
+too_many_lines = "allow"
+transmute_undefined_repr = "allow"
+unnecessary_wraps = "allow"
+unused_self = "allow"
+used_underscore_items = "allow"
+
+[lints.clippy.cargo]
+level = "warn"
+priority = -1
+
+[lints.clippy.nursery]
+level = "warn"
+priority = -1
+
+[lints.clippy.pedantic]
+level = "warn"
+priority = -1
+
+[lints.rust]
+mismatched_lifetime_syntaxes = "allow"