From f1ec3f8899a84f18f8166fe3f6f3c850ff777510 Mon Sep 17 00:00:00 2001 From: "riseproject-dev[bot]" <330740410+riseproject-dev[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 12:14:43 +0000 Subject: [PATCH 1/3] tree-sitter-language-pack: Add versions 1.21.0, 1.21.1, 1.21.3, 1.22.0, 1.22.1 Signed-off-by: riseproject-dev[bot] <330740410+riseproject-dev[bot]@users.noreply.github.com> --- docs/packages/tree-sitter-language-pack.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/packages/tree-sitter-language-pack.yaml b/docs/packages/tree-sitter-language-pack.yaml index 32385b260185..be26c802d54d 100644 --- a/docs/packages/tree-sitter-language-pack.yaml +++ b/docs/packages/tree-sitter-language-pack.yaml @@ -58,3 +58,8 @@ versions: - filename: tree_sitter_language_pack-1.20.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl sha256: 401ac0d672ee58292250b1667771d9c2d6c28bb94aecd266c59c26452278b18e requires-python: '>=3.10' +- version: 1.21.0 +- version: 1.21.1 +- version: 1.21.3 +- version: 1.22.0 +- version: 1.22.1 From cbaf895dac5dd4a7224348d3c7578ed7c12287cc Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Fri, 23 Oct 2026 17:09:14 +0800 Subject: [PATCH 2/3] tree-sitter-language-pack: copy the grammar-licence patch for 1.21.0, 1.21.1, 1.21.3, 1.22.0 and 1.22.1 --- ...ndored-grammar-licences-in-the-wheel.patch | 39 +++++++++++++++++++ ...ndored-grammar-licences-in-the-wheel.patch | 39 +++++++++++++++++++ ...ndored-grammar-licences-in-the-wheel.patch | 39 +++++++++++++++++++ ...ndored-grammar-licences-in-the-wheel.patch | 39 +++++++++++++++++++ ...ndored-grammar-licences-in-the-wheel.patch | 39 +++++++++++++++++++ 5 files changed, 195 insertions(+) create mode 100644 patches/tree-sitter-language-pack/1.21.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch create mode 100644 patches/tree-sitter-language-pack/1.21.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch create mode 100644 patches/tree-sitter-language-pack/1.21.3/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch create mode 100644 patches/tree-sitter-language-pack/1.22.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch create mode 100644 patches/tree-sitter-language-pack/1.22.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch diff --git a/patches/tree-sitter-language-pack/1.21.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch b/patches/tree-sitter-language-pack/1.21.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch new file mode 100644 index 000000000000..3ab5f9ecf462 --- /dev/null +++ b/patches/tree-sitter-language-pack/1.21.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch @@ -0,0 +1,39 @@ +From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 28 Aug 2026 09:00:00 +0200 +Subject: [PATCH] ship the vendored grammar licences in the wheel + +Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build] + +The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is +371 third-party grammar repositories compiled into one shared library. Each +carries its own MIT, Apache-2.0, BSD or ISC licence whose binary +redistribution clause requires the copyright notice to travel with the +binary, and the wheel ships only tree-sitter-language-pack's own LICENSE. + +project.license-files is an explicit PEP 639 list, so there is no default +glob to extend and the grammar licences have to be named. The build stages +them under packages/python/grammar-licenses// from each grammar's +pinned revision; 357 of the 371 grammars publish a licence file there, and +the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa, +facility, groovy, move, pgn, slint, t32, vb) publish none at the revision +sources/language_definitions.json pins. + +Signed-off-by: Ludovic Henry +--- + packages/python/pyproject.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml +index 0716ecf..297e674 100644 +--- a/packages/python/pyproject.toml ++++ b/packages/python/pyproject.toml +@@ -16,7 +16,7 @@ version = "1.21.0" + description = "Pre-compiled tree-sitter grammars for 372 programming languages" + keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ] + license = "MIT" +-license-files = [ "LICENSE" ] ++license-files = [ "LICENSE", "grammar-licenses/*/*" ] + authors = [ { name = "Na'aman Hirschfeld" } ] + requires-python = ">=3.10" + classifiers = [ diff --git a/patches/tree-sitter-language-pack/1.21.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch b/patches/tree-sitter-language-pack/1.21.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch new file mode 100644 index 000000000000..88c711601015 --- /dev/null +++ b/patches/tree-sitter-language-pack/1.21.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch @@ -0,0 +1,39 @@ +From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 28 Aug 2026 09:00:00 +0200 +Subject: [PATCH] ship the vendored grammar licences in the wheel + +Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build] + +The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is +371 third-party grammar repositories compiled into one shared library. Each +carries its own MIT, Apache-2.0, BSD or ISC licence whose binary +redistribution clause requires the copyright notice to travel with the +binary, and the wheel ships only tree-sitter-language-pack's own LICENSE. + +project.license-files is an explicit PEP 639 list, so there is no default +glob to extend and the grammar licences have to be named. The build stages +them under packages/python/grammar-licenses// from each grammar's +pinned revision; 357 of the 371 grammars publish a licence file there, and +the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa, +facility, groovy, move, pgn, slint, t32, vb) publish none at the revision +sources/language_definitions.json pins. + +Signed-off-by: Ludovic Henry +--- + packages/python/pyproject.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml +index 729c9b1..c7b3c34 100644 +--- a/packages/python/pyproject.toml ++++ b/packages/python/pyproject.toml +@@ -16,7 +16,7 @@ version = "1.21.1" + description = "Pre-compiled tree-sitter grammars for 372 programming languages" + keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ] + license = "MIT" +-license-files = [ "LICENSE" ] ++license-files = [ "LICENSE", "grammar-licenses/*/*" ] + authors = [ { name = "Na'aman Hirschfeld" } ] + requires-python = ">=3.10" + classifiers = [ diff --git a/patches/tree-sitter-language-pack/1.21.3/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch b/patches/tree-sitter-language-pack/1.21.3/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch new file mode 100644 index 000000000000..9de269a713f4 --- /dev/null +++ b/patches/tree-sitter-language-pack/1.21.3/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch @@ -0,0 +1,39 @@ +From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 28 Aug 2026 09:00:00 +0200 +Subject: [PATCH] ship the vendored grammar licences in the wheel + +Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build] + +The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is +371 third-party grammar repositories compiled into one shared library. Each +carries its own MIT, Apache-2.0, BSD or ISC licence whose binary +redistribution clause requires the copyright notice to travel with the +binary, and the wheel ships only tree-sitter-language-pack's own LICENSE. + +project.license-files is an explicit PEP 639 list, so there is no default +glob to extend and the grammar licences have to be named. The build stages +them under packages/python/grammar-licenses// from each grammar's +pinned revision; 357 of the 371 grammars publish a licence file there, and +the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa, +facility, groovy, move, pgn, slint, t32, vb) publish none at the revision +sources/language_definitions.json pins. + +Signed-off-by: Ludovic Henry +--- + packages/python/pyproject.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml +index 1f64ba1..522784e 100644 +--- a/packages/python/pyproject.toml ++++ b/packages/python/pyproject.toml +@@ -16,7 +16,7 @@ version = "1.21.3" + description = "Pre-compiled tree-sitter grammars for 372 programming languages" + keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ] + license = "MIT" +-license-files = [ "LICENSE" ] ++license-files = [ "LICENSE", "grammar-licenses/*/*" ] + authors = [ + { name = "Na'aman Hirschfeld " }, + { name = "Tobias Perlstein " }, diff --git a/patches/tree-sitter-language-pack/1.22.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch b/patches/tree-sitter-language-pack/1.22.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch new file mode 100644 index 000000000000..1ac91111f755 --- /dev/null +++ b/patches/tree-sitter-language-pack/1.22.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch @@ -0,0 +1,39 @@ +From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 28 Aug 2026 09:00:00 +0200 +Subject: [PATCH] ship the vendored grammar licences in the wheel + +Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build] + +The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is +371 third-party grammar repositories compiled into one shared library. Each +carries its own MIT, Apache-2.0, BSD or ISC licence whose binary +redistribution clause requires the copyright notice to travel with the +binary, and the wheel ships only tree-sitter-language-pack's own LICENSE. + +project.license-files is an explicit PEP 639 list, so there is no default +glob to extend and the grammar licences have to be named. The build stages +them under packages/python/grammar-licenses// from each grammar's +pinned revision; 357 of the 371 grammars publish a licence file there, and +the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa, +facility, groovy, move, pgn, slint, t32, vb) publish none at the revision +sources/language_definitions.json pins. + +Signed-off-by: Ludovic Henry +--- + packages/python/pyproject.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml +index 2997284..9a0b375 100644 +--- a/packages/python/pyproject.toml ++++ b/packages/python/pyproject.toml +@@ -16,7 +16,7 @@ version = "1.22.0" + description = "Pre-compiled tree-sitter grammars for 372 programming languages" + keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ] + license = "MIT" +-license-files = [ "LICENSE" ] ++license-files = [ "LICENSE", "grammar-licenses/*/*" ] + authors = [ + { name = "Na'aman Hirschfeld", email = "naaman@xberg.io" }, + { name = "Tobias Perlstein", email = "tobias@xberg.io" }, diff --git a/patches/tree-sitter-language-pack/1.22.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch b/patches/tree-sitter-language-pack/1.22.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch new file mode 100644 index 000000000000..cf59bd01f3d6 --- /dev/null +++ b/patches/tree-sitter-language-pack/1.22.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch @@ -0,0 +1,39 @@ +From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 28 Aug 2026 09:00:00 +0200 +Subject: [PATCH] ship the vendored grammar licences in the wheel + +Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build] + +The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is +371 third-party grammar repositories compiled into one shared library. Each +carries its own MIT, Apache-2.0, BSD or ISC licence whose binary +redistribution clause requires the copyright notice to travel with the +binary, and the wheel ships only tree-sitter-language-pack's own LICENSE. + +project.license-files is an explicit PEP 639 list, so there is no default +glob to extend and the grammar licences have to be named. The build stages +them under packages/python/grammar-licenses// from each grammar's +pinned revision; 357 of the 371 grammars publish a licence file there, and +the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa, +facility, groovy, move, pgn, slint, t32, vb) publish none at the revision +sources/language_definitions.json pins. + +Signed-off-by: Ludovic Henry +--- + packages/python/pyproject.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml +index 19f417f..6dfbe34 100644 +--- a/packages/python/pyproject.toml ++++ b/packages/python/pyproject.toml +@@ -16,7 +16,7 @@ version = "1.22.1" + description = "Pre-compiled tree-sitter grammars for 372 programming languages" + keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ] + license = "MIT" +-license-files = [ "LICENSE" ] ++license-files = [ "LICENSE", "grammar-licenses/*/*" ] + authors = [ + { name = "Na'aman Hirschfeld", email = "naaman@xberg.io" }, + { name = "Tobias Perlstein", email = "tobias@xberg.io" }, From b0ae4b1f7c048fd2355753a408e3b9f4985a29fc Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Tue, 27 Oct 2026 17:30:18 +0100 Subject: [PATCH 3/3] tree-sitter-language-pack: derive the grammar-licence expectation instead of hard-coding it 1.21.0 and later vendor 372 grammars (baml is new) rather than 371, so the fixed count of 357 staged licences no longer holds: 358 are staged. The same 14 grammars publish no licence file at their pinned revision in every release, so name them explicitly and assert the missing set equals it, rather than comparing a count that has to be bumped per release. Only a 404 now counts as "no licence file here"; 429 and 5xx responses are retried and then fail the step, so rate limiting cannot silently drop a licence. The staged count is exported and the wheel test checks the wheel ships exactly that many grammar licences. --- .../build-tree-sitter-language-pack.yml | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-tree-sitter-language-pack.yml b/.github/workflows/build-tree-sitter-language-pack.yml index 5b104d9bfdff..6ad77df723cb 100644 --- a/.github/workflows/build-tree-sitter-language-pack.yml +++ b/.github/workflows/build-tree-sitter-language-pack.yml @@ -74,15 +74,23 @@ jobs: python3 - <<'PY' import concurrent.futures as cf import json + import os import pathlib import re + import time import urllib.error import urllib.request NAMES = ["LICENSE", "LICENSE.md", "LICENSE.txt", "LICENCE", "LICENCE.md", "LICENCE.txt", "COPYING", "COPYING.md", "COPYING.txt", "LICENSE-MIT", "LICENSE-APACHE", "UNLICENSE", "license", "license.md"] - EXPECTED = 357 + # Grammars whose repository publishes no licence file at the revision + # sources/language_definitions.json pins (the same 14 in every release + # from 1.15.8 to 1.22.1). Spelling them out means a grammar that loses + # its licence, or a fetch that silently fails, trips the assertion below + # instead of shrinking the wheel's licence set unnoticed. + UNLICENSED = {"brightscript", "cooklang", "corn", "ebnf", "eds", "eex", "elsa", + "facility", "groovy", "move", "pgn", "slint", "t32", "vb"} LICENCE_RE = re.compile(r"(?i)^(licen[cs]e|copying|notice)") out = pathlib.Path("packages/python/grammar-licenses") @@ -112,11 +120,21 @@ jobs: else: return None for candidate in NAMES: - try: - with urllib.request.urlopen("%s/%s" % (base, candidate), timeout=60) as response: - return write(language, candidate, response.read()) - except (urllib.error.URLError, TimeoutError): - continue + for attempt in range(6): + try: + with urllib.request.urlopen("%s/%s" % (base, candidate), timeout=60) as response: + return write(language, candidate, response.read()) + except urllib.error.HTTPError as error: + # Only a 404 means "no such file"; rate limiting (429) and + # server errors are retried, never read as a missing licence. + if error.code == 404: + break + if error.code not in (429, 500, 502, 503, 504) or attempt == 5: + raise + except (urllib.error.URLError, TimeoutError): + if attempt == 5: + raise + time.sleep(5 * (attempt + 1)) return None @@ -125,7 +143,10 @@ jobs: staged = [name for name in pool.map(stage, sorted(definitions.items())) if name] missing = sorted(set(definitions) - set(staged)) print("staged %d; no licence file at the pinned revision: %s" % (len(staged), ", ".join(missing))) - assert len(staged) == EXPECTED, len(staged) + assert set(missing) == UNLICENSED, sorted(set(missing) ^ UNLICENSED) + # the built wheel must carry exactly this many grammar licences + with open(os.environ["GITHUB_ENV"], "a") as env_file: + env_file.write("TSLP_LICENCES=%d\n" % len(staged)) PY - name: Patch tree-sitter-language-pack source @@ -163,14 +184,15 @@ jobs: TSLP_LANGUAGES=all TSLP_LINK_MODE=static TSLP_GRAMMAR_CFLAGS=-O2 + TSLP_LICENCES=${{ env.TSLP_LICENCES }} CIBW_TEST_REQUIRES: pytest pytest-asyncio pytest-timeout CIBW_TEST_SOURCES: e2e/python # test_download.py needs upstream's prebuilt parser bundle for this # platform, which is exactly what riscv64 has none of. CIBW_TEST_COMMAND: >- - python -c "import importlib.metadata as md; + python -c "import importlib.metadata as md, os; n = len([p for p in md.files('tree-sitter-language-pack') if '.dist-info/licenses/grammar-licenses/' in str(p)]); - assert n == 357, n" + assert n == int(os.environ['TSLP_LICENCES']), n" && cd e2e/python && python -m pytest tests --ignore=tests/test_download.py