tree-sitter-language-pack: update packages

.github/workflows/build-tree-sitter-language-pack.yml

@@ -74,15 +74,23 @@ jobs:
           python3 - <<'PY'
           import concurrent.futures as cf
           import json
+          import os
           import pathlib
           import re
+          import time
           import urllib.error
           import urllib.request
 
           NAMES = ["LICENSE", "LICENSE.md", "LICENSE.txt", "LICENCE", "LICENCE.md",
                    "LICENCE.txt", "COPYING", "COPYING.md", "COPYING.txt", "LICENSE-MIT",
                    "LICENSE-APACHE", "UNLICENSE", "license", "license.md"]
-          EXPECTED = 357
+          # Grammars whose repository publishes no licence file at the revision
+          # sources/language_definitions.json pins (the same 14 in every release
+          # from 1.15.8 to 1.22.1). Spelling them out means a grammar that loses
+          # its licence, or a fetch that silently fails, trips the assertion below
+          # instead of shrinking the wheel's licence set unnoticed.
+          UNLICENSED = {"brightscript", "cooklang", "corn", "ebnf", "eds", "eex", "elsa",
+                        "facility", "groovy", "move", "pgn", "slint", "t32", "vb"}
           LICENCE_RE = re.compile(r"(?i)^(licen[cs]e|copying|notice)")
           out = pathlib.Path("packages/python/grammar-licenses")
 
@@ -112,11 +120,21 @@ jobs:
               else:
                   return None
               for candidate in NAMES:
-                  try:
-                      with urllib.request.urlopen("%s/%s" % (base, candidate), timeout=60) as response:
-                          return write(language, candidate, response.read())
-                  except (urllib.error.URLError, TimeoutError):
-                      continue
+                  for attempt in range(6):
+                      try:
+                          with urllib.request.urlopen("%s/%s" % (base, candidate), timeout=60) as response:
+                              return write(language, candidate, response.read())
+                      except urllib.error.HTTPError as error:
+                          # Only a 404 means "no such file"; rate limiting (429) and
+                          # server errors are retried, never read as a missing licence.
+                          if error.code == 404:
+                              break
+                          if error.code not in (429, 500, 502, 503, 504) or attempt == 5:
+                              raise
+                      except (urllib.error.URLError, TimeoutError):
+                          if attempt == 5:
+                              raise
+                      time.sleep(5 * (attempt + 1))
               return None
 
 
@@ -125,7 +143,10 @@ jobs:
               staged = [name for name in pool.map(stage, sorted(definitions.items())) if name]
           missing = sorted(set(definitions) - set(staged))
           print("staged %d; no licence file at the pinned revision: %s" % (len(staged), ", ".join(missing)))
-          assert len(staged) == EXPECTED, len(staged)
+          assert set(missing) == UNLICENSED, sorted(set(missing) ^ UNLICENSED)
+          # the built wheel must carry exactly this many grammar licences
+          with open(os.environ["GITHUB_ENV"], "a") as env_file:
+              env_file.write("TSLP_LICENCES=%d\n" % len(staged))
           PY
 
       - name: Patch tree-sitter-language-pack source
@@ -163,14 +184,15 @@ jobs:
             TSLP_LANGUAGES=all
             TSLP_LINK_MODE=static
             TSLP_GRAMMAR_CFLAGS=-O2
+            TSLP_LICENCES=${{ env.TSLP_LICENCES }}
           CIBW_TEST_REQUIRES: pytest pytest-asyncio pytest-timeout
           CIBW_TEST_SOURCES: e2e/python
           # test_download.py needs upstream's prebuilt parser bundle for this
           # platform, which is exactly what riscv64 has none of.
           CIBW_TEST_COMMAND: >-
-            python -c "import importlib.metadata as md;
+            python -c "import importlib.metadata as md, os;
             n = len([p for p in md.files('tree-sitter-language-pack') if '.dist-info/licenses/grammar-licenses/' in str(p)]);
-            assert n == 357, n"
+            assert n == int(os.environ['TSLP_LICENCES']), n"
             && cd e2e/python
             && python -m pytest tests --ignore=tests/test_download.py
 

docs/packages/tree-sitter-language-pack.yaml

@@ -58,3 +58,8 @@ versions:
   - filename: tree_sitter_language_pack-1.20.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl
     sha256: 401ac0d672ee58292250b1667771d9c2d6c28bb94aecd266c59c26452278b18e
     requires-python: '>=3.10'
+- version: 1.21.0
+- version: 1.21.1
+- version: 1.21.3
+- version: 1.22.0
+- version: 1.22.1

patches/tree-sitter-language-pack/1.21.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch

@@ -0,0 +1,39 @@
+From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001
+From: Ludovic Henry <git@ludovic.dev>
+Date: Fri, 28 Aug 2026 09:00:00 +0200
+Subject: [PATCH] ship the vendored grammar licences in the wheel
+
+Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build]
+
+The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is
+371 third-party grammar repositories compiled into one shared library. Each
+carries its own MIT, Apache-2.0, BSD or ISC licence whose binary
+redistribution clause requires the copyright notice to travel with the
+binary, and the wheel ships only tree-sitter-language-pack's own LICENSE.
+
+project.license-files is an explicit PEP 639 list, so there is no default
+glob to extend and the grammar licences have to be named. The build stages
+them under packages/python/grammar-licenses/<language>/ from each grammar's
+pinned revision; 357 of the 371 grammars publish a licence file there, and
+the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa,
+facility, groovy, move, pgn, slint, t32, vb) publish none at the revision
+sources/language_definitions.json pins.
+
+Signed-off-by: Ludovic Henry <git@ludovic.dev>
+---
+ packages/python/pyproject.toml | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml
+index 0716ecf..297e674 100644
+--- a/packages/python/pyproject.toml
++++ b/packages/python/pyproject.toml
+@@ -16,7 +16,7 @@ version = "1.21.0"
+ description = "Pre-compiled tree-sitter grammars for 372 programming languages"
+ keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ]
+ license = "MIT"
+-license-files = [ "LICENSE" ]
++license-files = [ "LICENSE", "grammar-licenses/*/*" ]
+ authors = [ { name = "Na'aman Hirschfeld" } ]
+ requires-python = ">=3.10"
+ classifiers = [

patches/tree-sitter-language-pack/1.21.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch

@@ -0,0 +1,39 @@
+From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001
+From: Ludovic Henry <git@ludovic.dev>
+Date: Fri, 28 Aug 2026 09:00:00 +0200
+Subject: [PATCH] ship the vendored grammar licences in the wheel
+
+Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build]
+
+The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is
+371 third-party grammar repositories compiled into one shared library. Each
+carries its own MIT, Apache-2.0, BSD or ISC licence whose binary
+redistribution clause requires the copyright notice to travel with the
+binary, and the wheel ships only tree-sitter-language-pack's own LICENSE.
+
+project.license-files is an explicit PEP 639 list, so there is no default
+glob to extend and the grammar licences have to be named. The build stages
+them under packages/python/grammar-licenses/<language>/ from each grammar's
+pinned revision; 357 of the 371 grammars publish a licence file there, and
+the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa,
+facility, groovy, move, pgn, slint, t32, vb) publish none at the revision
+sources/language_definitions.json pins.
+
+Signed-off-by: Ludovic Henry <git@ludovic.dev>
+---
+ packages/python/pyproject.toml | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml
+index 729c9b1..c7b3c34 100644
+--- a/packages/python/pyproject.toml
++++ b/packages/python/pyproject.toml
+@@ -16,7 +16,7 @@ version = "1.21.1"
+ description = "Pre-compiled tree-sitter grammars for 372 programming languages"
+ keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ]
+ license = "MIT"
+-license-files = [ "LICENSE" ]
++license-files = [ "LICENSE", "grammar-licenses/*/*" ]
+ authors = [ { name = "Na'aman Hirschfeld" } ]
+ requires-python = ">=3.10"
+ classifiers = [

patches/tree-sitter-language-pack/1.21.3/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch

@@ -0,0 +1,39 @@
+From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001
+From: Ludovic Henry <git@ludovic.dev>
+Date: Fri, 28 Aug 2026 09:00:00 +0200
+Subject: [PATCH] ship the vendored grammar licences in the wheel
+
+Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build]
+
+The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is
+371 third-party grammar repositories compiled into one shared library. Each
+carries its own MIT, Apache-2.0, BSD or ISC licence whose binary
+redistribution clause requires the copyright notice to travel with the
+binary, and the wheel ships only tree-sitter-language-pack's own LICENSE.
+
+project.license-files is an explicit PEP 639 list, so there is no default
+glob to extend and the grammar licences have to be named. The build stages
+them under packages/python/grammar-licenses/<language>/ from each grammar's
+pinned revision; 357 of the 371 grammars publish a licence file there, and
+the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa,
+facility, groovy, move, pgn, slint, t32, vb) publish none at the revision
+sources/language_definitions.json pins.
+
+Signed-off-by: Ludovic Henry <git@ludovic.dev>
+---
+ packages/python/pyproject.toml | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml
+index 1f64ba1..522784e 100644
+--- a/packages/python/pyproject.toml
++++ b/packages/python/pyproject.toml
+@@ -16,7 +16,7 @@ version = "1.21.3"
+ description = "Pre-compiled tree-sitter grammars for 372 programming languages"
+ keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ]
+ license = "MIT"
+-license-files = [ "LICENSE" ]
++license-files = [ "LICENSE", "grammar-licenses/*/*" ]
+ authors = [
+   { name = "Na'aman Hirschfeld <naaman@xberg.io>" },
+   { name = "Tobias Perlstein <tobias@xberg.io>" },

patches/tree-sitter-language-pack/1.22.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch

@@ -0,0 +1,39 @@
+From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001
+From: Ludovic Henry <git@ludovic.dev>
+Date: Fri, 28 Aug 2026 09:00:00 +0200
+Subject: [PATCH] ship the vendored grammar licences in the wheel
+
+Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build]
+
+The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is
+371 third-party grammar repositories compiled into one shared library. Each
+carries its own MIT, Apache-2.0, BSD or ISC licence whose binary
+redistribution clause requires the copyright notice to travel with the
+binary, and the wheel ships only tree-sitter-language-pack's own LICENSE.
+
+project.license-files is an explicit PEP 639 list, so there is no default
+glob to extend and the grammar licences have to be named. The build stages
+them under packages/python/grammar-licenses/<language>/ from each grammar's
+pinned revision; 357 of the 371 grammars publish a licence file there, and
+the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa,
+facility, groovy, move, pgn, slint, t32, vb) publish none at the revision
+sources/language_definitions.json pins.
+
+Signed-off-by: Ludovic Henry <git@ludovic.dev>
+---
+ packages/python/pyproject.toml | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml
+index 2997284..9a0b375 100644
+--- a/packages/python/pyproject.toml
++++ b/packages/python/pyproject.toml
+@@ -16,7 +16,7 @@ version = "1.22.0"
+ description = "Pre-compiled tree-sitter grammars for 372 programming languages"
+ keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ]
+ license = "MIT"
+-license-files = [ "LICENSE" ]
++license-files = [ "LICENSE", "grammar-licenses/*/*" ]
+ authors = [
+   { name = "Na'aman Hirschfeld", email = "naaman@xberg.io" },
+   { name = "Tobias Perlstein", email = "tobias@xberg.io" },

patches/tree-sitter-language-pack/1.22.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch

@@ -0,0 +1,39 @@
+From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001
+From: Ludovic Henry <git@ludovic.dev>
+Date: Fri, 28 Aug 2026 09:00:00 +0200
+Subject: [PATCH] ship the vendored grammar licences in the wheel
+
+Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build]
+
+The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is
+371 third-party grammar repositories compiled into one shared library. Each
+carries its own MIT, Apache-2.0, BSD or ISC licence whose binary
+redistribution clause requires the copyright notice to travel with the
+binary, and the wheel ships only tree-sitter-language-pack's own LICENSE.
+
+project.license-files is an explicit PEP 639 list, so there is no default
+glob to extend and the grammar licences have to be named. The build stages
+them under packages/python/grammar-licenses/<language>/ from each grammar's
+pinned revision; 357 of the 371 grammars publish a licence file there, and
+the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa,
+facility, groovy, move, pgn, slint, t32, vb) publish none at the revision
+sources/language_definitions.json pins.
+
+Signed-off-by: Ludovic Henry <git@ludovic.dev>
+---
+ packages/python/pyproject.toml | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml
+index 19f417f..6dfbe34 100644
+--- a/packages/python/pyproject.toml
++++ b/packages/python/pyproject.toml
+@@ -16,7 +16,7 @@ version = "1.22.1"
+ description = "Pre-compiled tree-sitter grammars for 372 programming languages"
+ keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ]
+ license = "MIT"
+-license-files = [ "LICENSE" ]
++license-files = [ "LICENSE", "grammar-licenses/*/*" ]
+ authors = [
+   { name = "Na'aman Hirschfeld", email = "naaman@xberg.io" },
+   { name = "Tobias Perlstein", email = "tobias@xberg.io" },