diff --git a/.github/workflows/build-tree-sitter-language-pack.yml b/.github/workflows/build-tree-sitter-language-pack.yml index 5b104d9bfdff..6ad77df723cb 100644 --- a/.github/workflows/build-tree-sitter-language-pack.yml +++ b/.github/workflows/build-tree-sitter-language-pack.yml @@ -74,15 +74,23 @@ jobs: python3 - <<'PY' import concurrent.futures as cf import json + import os import pathlib import re + import time import urllib.error import urllib.request NAMES = ["LICENSE", "LICENSE.md", "LICENSE.txt", "LICENCE", "LICENCE.md", "LICENCE.txt", "COPYING", "COPYING.md", "COPYING.txt", "LICENSE-MIT", "LICENSE-APACHE", "UNLICENSE", "license", "license.md"] - EXPECTED = 357 + # Grammars whose repository publishes no licence file at the revision + # sources/language_definitions.json pins (the same 14 in every release + # from 1.15.8 to 1.22.1). Spelling them out means a grammar that loses + # its licence, or a fetch that silently fails, trips the assertion below + # instead of shrinking the wheel's licence set unnoticed. + UNLICENSED = {"brightscript", "cooklang", "corn", "ebnf", "eds", "eex", "elsa", + "facility", "groovy", "move", "pgn", "slint", "t32", "vb"} LICENCE_RE = re.compile(r"(?i)^(licen[cs]e|copying|notice)") out = pathlib.Path("packages/python/grammar-licenses") @@ -112,11 +120,21 @@ jobs: else: return None for candidate in NAMES: - try: - with urllib.request.urlopen("%s/%s" % (base, candidate), timeout=60) as response: - return write(language, candidate, response.read()) - except (urllib.error.URLError, TimeoutError): - continue + for attempt in range(6): + try: + with urllib.request.urlopen("%s/%s" % (base, candidate), timeout=60) as response: + return write(language, candidate, response.read()) + except urllib.error.HTTPError as error: + # Only a 404 means "no such file"; rate limiting (429) and + # server errors are retried, never read as a missing licence. + if error.code == 404: + break + if error.code not in (429, 500, 502, 503, 504) or attempt == 5: + raise + except (urllib.error.URLError, TimeoutError): + if attempt == 5: + raise + time.sleep(5 * (attempt + 1)) return None @@ -125,7 +143,10 @@ jobs: staged = [name for name in pool.map(stage, sorted(definitions.items())) if name] missing = sorted(set(definitions) - set(staged)) print("staged %d; no licence file at the pinned revision: %s" % (len(staged), ", ".join(missing))) - assert len(staged) == EXPECTED, len(staged) + assert set(missing) == UNLICENSED, sorted(set(missing) ^ UNLICENSED) + # the built wheel must carry exactly this many grammar licences + with open(os.environ["GITHUB_ENV"], "a") as env_file: + env_file.write("TSLP_LICENCES=%d\n" % len(staged)) PY - name: Patch tree-sitter-language-pack source @@ -163,14 +184,15 @@ jobs: TSLP_LANGUAGES=all TSLP_LINK_MODE=static TSLP_GRAMMAR_CFLAGS=-O2 + TSLP_LICENCES=${{ env.TSLP_LICENCES }} CIBW_TEST_REQUIRES: pytest pytest-asyncio pytest-timeout CIBW_TEST_SOURCES: e2e/python # test_download.py needs upstream's prebuilt parser bundle for this # platform, which is exactly what riscv64 has none of. CIBW_TEST_COMMAND: >- - python -c "import importlib.metadata as md; + python -c "import importlib.metadata as md, os; n = len([p for p in md.files('tree-sitter-language-pack') if '.dist-info/licenses/grammar-licenses/' in str(p)]); - assert n == 357, n" + assert n == int(os.environ['TSLP_LICENCES']), n" && cd e2e/python && python -m pytest tests --ignore=tests/test_download.py diff --git a/docs/packages/tree-sitter-language-pack.yaml b/docs/packages/tree-sitter-language-pack.yaml index 32385b260185..be26c802d54d 100644 --- a/docs/packages/tree-sitter-language-pack.yaml +++ b/docs/packages/tree-sitter-language-pack.yaml @@ -58,3 +58,8 @@ versions: - filename: tree_sitter_language_pack-1.20.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl sha256: 401ac0d672ee58292250b1667771d9c2d6c28bb94aecd266c59c26452278b18e requires-python: '>=3.10' +- version: 1.21.0 +- version: 1.21.1 +- version: 1.21.3 +- version: 1.22.0 +- version: 1.22.1 diff --git a/patches/tree-sitter-language-pack/1.21.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch b/patches/tree-sitter-language-pack/1.21.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch new file mode 100644 index 000000000000..3ab5f9ecf462 --- /dev/null +++ b/patches/tree-sitter-language-pack/1.21.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch @@ -0,0 +1,39 @@ +From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 28 Aug 2026 09:00:00 +0200 +Subject: [PATCH] ship the vendored grammar licences in the wheel + +Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build] + +The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is +371 third-party grammar repositories compiled into one shared library. Each +carries its own MIT, Apache-2.0, BSD or ISC licence whose binary +redistribution clause requires the copyright notice to travel with the +binary, and the wheel ships only tree-sitter-language-pack's own LICENSE. + +project.license-files is an explicit PEP 639 list, so there is no default +glob to extend and the grammar licences have to be named. The build stages +them under packages/python/grammar-licenses// from each grammar's +pinned revision; 357 of the 371 grammars publish a licence file there, and +the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa, +facility, groovy, move, pgn, slint, t32, vb) publish none at the revision +sources/language_definitions.json pins. + +Signed-off-by: Ludovic Henry +--- + packages/python/pyproject.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml +index 0716ecf..297e674 100644 +--- a/packages/python/pyproject.toml ++++ b/packages/python/pyproject.toml +@@ -16,7 +16,7 @@ version = "1.21.0" + description = "Pre-compiled tree-sitter grammars for 372 programming languages" + keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ] + license = "MIT" +-license-files = [ "LICENSE" ] ++license-files = [ "LICENSE", "grammar-licenses/*/*" ] + authors = [ { name = "Na'aman Hirschfeld" } ] + requires-python = ">=3.10" + classifiers = [ diff --git a/patches/tree-sitter-language-pack/1.21.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch b/patches/tree-sitter-language-pack/1.21.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch new file mode 100644 index 000000000000..88c711601015 --- /dev/null +++ b/patches/tree-sitter-language-pack/1.21.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch @@ -0,0 +1,39 @@ +From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 28 Aug 2026 09:00:00 +0200 +Subject: [PATCH] ship the vendored grammar licences in the wheel + +Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build] + +The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is +371 third-party grammar repositories compiled into one shared library. Each +carries its own MIT, Apache-2.0, BSD or ISC licence whose binary +redistribution clause requires the copyright notice to travel with the +binary, and the wheel ships only tree-sitter-language-pack's own LICENSE. + +project.license-files is an explicit PEP 639 list, so there is no default +glob to extend and the grammar licences have to be named. The build stages +them under packages/python/grammar-licenses// from each grammar's +pinned revision; 357 of the 371 grammars publish a licence file there, and +the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa, +facility, groovy, move, pgn, slint, t32, vb) publish none at the revision +sources/language_definitions.json pins. + +Signed-off-by: Ludovic Henry +--- + packages/python/pyproject.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml +index 729c9b1..c7b3c34 100644 +--- a/packages/python/pyproject.toml ++++ b/packages/python/pyproject.toml +@@ -16,7 +16,7 @@ version = "1.21.1" + description = "Pre-compiled tree-sitter grammars for 372 programming languages" + keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ] + license = "MIT" +-license-files = [ "LICENSE" ] ++license-files = [ "LICENSE", "grammar-licenses/*/*" ] + authors = [ { name = "Na'aman Hirschfeld" } ] + requires-python = ">=3.10" + classifiers = [ diff --git a/patches/tree-sitter-language-pack/1.21.3/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch b/patches/tree-sitter-language-pack/1.21.3/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch new file mode 100644 index 000000000000..9de269a713f4 --- /dev/null +++ b/patches/tree-sitter-language-pack/1.21.3/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch @@ -0,0 +1,39 @@ +From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 28 Aug 2026 09:00:00 +0200 +Subject: [PATCH] ship the vendored grammar licences in the wheel + +Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build] + +The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is +371 third-party grammar repositories compiled into one shared library. Each +carries its own MIT, Apache-2.0, BSD or ISC licence whose binary +redistribution clause requires the copyright notice to travel with the +binary, and the wheel ships only tree-sitter-language-pack's own LICENSE. + +project.license-files is an explicit PEP 639 list, so there is no default +glob to extend and the grammar licences have to be named. The build stages +them under packages/python/grammar-licenses// from each grammar's +pinned revision; 357 of the 371 grammars publish a licence file there, and +the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa, +facility, groovy, move, pgn, slint, t32, vb) publish none at the revision +sources/language_definitions.json pins. + +Signed-off-by: Ludovic Henry +--- + packages/python/pyproject.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml +index 1f64ba1..522784e 100644 +--- a/packages/python/pyproject.toml ++++ b/packages/python/pyproject.toml +@@ -16,7 +16,7 @@ version = "1.21.3" + description = "Pre-compiled tree-sitter grammars for 372 programming languages" + keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ] + license = "MIT" +-license-files = [ "LICENSE" ] ++license-files = [ "LICENSE", "grammar-licenses/*/*" ] + authors = [ + { name = "Na'aman Hirschfeld " }, + { name = "Tobias Perlstein " }, diff --git a/patches/tree-sitter-language-pack/1.22.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch b/patches/tree-sitter-language-pack/1.22.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch new file mode 100644 index 000000000000..1ac91111f755 --- /dev/null +++ b/patches/tree-sitter-language-pack/1.22.0/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch @@ -0,0 +1,39 @@ +From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 28 Aug 2026 09:00:00 +0200 +Subject: [PATCH] ship the vendored grammar licences in the wheel + +Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build] + +The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is +371 third-party grammar repositories compiled into one shared library. Each +carries its own MIT, Apache-2.0, BSD or ISC licence whose binary +redistribution clause requires the copyright notice to travel with the +binary, and the wheel ships only tree-sitter-language-pack's own LICENSE. + +project.license-files is an explicit PEP 639 list, so there is no default +glob to extend and the grammar licences have to be named. The build stages +them under packages/python/grammar-licenses// from each grammar's +pinned revision; 357 of the 371 grammars publish a licence file there, and +the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa, +facility, groovy, move, pgn, slint, t32, vb) publish none at the revision +sources/language_definitions.json pins. + +Signed-off-by: Ludovic Henry +--- + packages/python/pyproject.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml +index 2997284..9a0b375 100644 +--- a/packages/python/pyproject.toml ++++ b/packages/python/pyproject.toml +@@ -16,7 +16,7 @@ version = "1.22.0" + description = "Pre-compiled tree-sitter grammars for 372 programming languages" + keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ] + license = "MIT" +-license-files = [ "LICENSE" ] ++license-files = [ "LICENSE", "grammar-licenses/*/*" ] + authors = [ + { name = "Na'aman Hirschfeld", email = "naaman@xberg.io" }, + { name = "Tobias Perlstein", email = "tobias@xberg.io" }, diff --git a/patches/tree-sitter-language-pack/1.22.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch b/patches/tree-sitter-language-pack/1.22.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch new file mode 100644 index 000000000000..cf59bd01f3d6 --- /dev/null +++ b/patches/tree-sitter-language-pack/1.22.1/0001-ship-the-vendored-grammar-licences-in-the-wheel.patch @@ -0,0 +1,39 @@ +From f43cf6275f099fd9ef9a8b89dbfbfe0a2993f1fd Mon Sep 17 00:00:00 2001 +From: Ludovic Henry +Date: Fri, 28 Aug 2026 09:00:00 +0200 +Subject: [PATCH] ship the vendored grammar licences in the wheel + +Upstream-Status: Inappropriate [upstream's wheels download the grammars at runtime and link none of them, so the gap only exists in a TSLP_LANGUAGES build] + +The riscv64 wheel is built with TSLP_LANGUAGES=all, so _native.abi3.so is +371 third-party grammar repositories compiled into one shared library. Each +carries its own MIT, Apache-2.0, BSD or ISC licence whose binary +redistribution clause requires the copyright notice to travel with the +binary, and the wheel ships only tree-sitter-language-pack's own LICENSE. + +project.license-files is an explicit PEP 639 list, so there is no default +glob to extend and the grammar licences have to be named. The build stages +them under packages/python/grammar-licenses// from each grammar's +pinned revision; 357 of the 371 grammars publish a licence file there, and +the remaining 14 (brightscript, cooklang, corn, ebnf, eds, eex, elsa, +facility, groovy, move, pgn, slint, t32, vb) publish none at the revision +sources/language_definitions.json pins. + +Signed-off-by: Ludovic Henry +--- + packages/python/pyproject.toml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/packages/python/pyproject.toml b/packages/python/pyproject.toml +index 19f417f..6dfbe34 100644 +--- a/packages/python/pyproject.toml ++++ b/packages/python/pyproject.toml +@@ -16,7 +16,7 @@ version = "1.22.1" + description = "Pre-compiled tree-sitter grammars for 372 programming languages" + keywords = [ "language-pack", "parser", "syntax", "tree-sitter" ] + license = "MIT" +-license-files = [ "LICENSE" ] ++license-files = [ "LICENSE", "grammar-licenses/*/*" ] + authors = [ + { name = "Na'aman Hirschfeld", email = "naaman@xberg.io" }, + { name = "Tobias Perlstein", email = "tobias@xberg.io" },