From 160664a8aa3a21951aa017464cb71cd570751c9d Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Thu, 8 Oct 2026 21:41:33 -0700 Subject: [PATCH 1/4] Added type checking --- .claude/settings.json | 10 -- .github/workflows/release.yml | 8 +- .github/workflows/tests.yml | 14 +-- CHANGELOG.md | 22 ++++ CLAUDE.md | 2 +- MANIFEST.in | 1 + README.rst | 4 +- SECURITY.md | 11 -- docs/architecture/build-setup.md | 6 +- docs/changelog.md | 21 ---- docs/concepts/telltale-detection.md | 6 +- docs/guides/ci-cd.md | 4 +- docs/guides/local-development.md | 1 + docs/index.md | 4 +- docs/log.md | 13 ++- docs/release.md | 2 +- docs/typecheck-v2.1.0.md | 111 +++++++++++++++++++ pyproject.toml | 12 +++ src/portable_python/__init__.py | 126 ++++++++++------------ src/portable_python/cli.py | 18 ++-- src/portable_python/config.py | 58 ++++------ src/portable_python/cpython.py | 5 +- src/portable_python/external/xcpython.py | 9 +- src/portable_python/external/xtkinter.py | 4 +- src/portable_python/inspector.py | 99 ++++++++--------- src/portable_python/tracking.py | 88 +++++++-------- src/portable_python/versions.py | 131 ++++++++++++----------- tests/conftest.py | 6 -- tests/test_build.py | 6 +- tests/test_inspector.py | 2 +- tests/test_list.py | 12 ++- tests/test_setup.py | 26 ++++- tox.ini | 34 ++++++ 33 files changed, 509 insertions(+), 367 deletions(-) delete mode 100644 .claude/settings.json create mode 100644 CHANGELOG.md delete mode 100644 SECURITY.md delete mode 100644 docs/changelog.md create mode 100644 docs/typecheck-v2.1.0.md diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index da322a9..0000000 --- a/.claude/settings.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(.venv/bin/portable-python:*)", - "Bash(.venv/bin/pytest:*)", - "Bash(.venv/bin/python scripts/check_okf.py:*)", - "Bash(tox:*)" - ] - } -} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index acc3c4a..49baa32 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,13 +14,13 @@ jobs: id-token: write # mandatory for trusted publishing steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.14" - - uses: astral-sh/setup-uv@v7 - - run: uvx --with tox-uv tox -e py,style + - uses: astral-sh/setup-uv@v10.2.0 + - run: uvx --with tox-uv tox -e py,style,ty,pyrefly,pyright,mypy,zuban - run: uv build - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 045b62f..2a76478 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,12 +21,12 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} - - uses: astral-sh/setup-uv@v7 + - uses: astral-sh/setup-uv@v10.2.0 - run: uvx --with tox-uv tox -e py - uses: coverallsapp/github-action@v2 @@ -50,10 +50,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 with: python-version: "3.14" - - uses: astral-sh/setup-uv@v7 - - run: uvx --with tox-uv tox -e style + - uses: astral-sh/setup-uv@v10.2.0 + - run: uvx --with tox-uv tox -e style,ty,pyrefly,pyright,mypy,zuban diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e35ddb3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,22 @@ +# Changelog + +## v2.1.0 + +- portable-python's code is now checked by 5 type checkers (ty, pyrefly, pyright, mypy and zuban) on every change +- Bumped SQLite to 3.53.4 and gettext-tiny to 0.3.3; re-checked every other component against upstream (all already current, or deliberately held back — see the `version` property comments). +- Bug fixes: + - With `cpython-use-github: true`, a failed GitHub API call (e.g. rate limiting) crashed with a stack trace + - A build where `bin/python` could not be found crashed (instead of reporting it) when `cpython-validate-script` or `cpython-additional-packages` was configured + - Clearer error messages when the latest python version can't be determined, or a python family isn't supported +- Library use: `BuildSetup()` now takes a `PythonSpec`, get one via `required_python_spec("cpython:3.13.2")` (which validates it, `None` means latest) +- Tests cover more edge cases + +## v2.0.2 + +- Corrected `readline` static linking + +## v2.0.1 + +- Bumped the bundled components (OpenSSL, SQLite, Tcl/Tk, …) to current versions. +- Refreshed dependencies and modernized CI (now on uv / tox-uv, with updated GitHub Actions). +- Added an OKF documentation bundle under `docs/` (the old top-level docs folded into it). diff --git a/CLAUDE.md b/CLAUDE.md index 0e91795..e0e3385 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -22,4 +22,4 @@ The authoritative, self-contained documentation is the OKF knowledge bundle unde - Check **runez** before reimplementing file / system / CLI / logging / version helpers — see [`docs/guides/local-development.md`](docs/guides/local-development.md). - The docs use the OKF format; keep the bundle conformant (`scripts/check_okf.py docs`) when you edit it. -- As changes land, keep a top section in [`docs/changelog.md`](docs/changelog.md) for the upcoming release — `## Unreleased`, or the next version directly once it's known. Maintain it as PRs come in; it's double-checked at release. Don't fuss over marking released-vs-unreleased (small audience, `git tag` says what's out). +- As changes land, keep a top section in [`CHANGELOG.md`](CHANGELOG.md) for the upcoming release — `## Unreleased`, or the next version directly once it's known. Maintain it as PRs come in; it's double-checked at release. Don't fuss over marking released-vs-unreleased (small audience, `git tag` says what's out). diff --git a/MANIFEST.in b/MANIFEST.in index 9d5d250..3ce73c4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include LICENSE include README.rst +include CHANGELOG.md diff --git a/README.rst b/README.rst index a89c81b..fd9d3ff 100644 --- a/README.rst +++ b/README.rst @@ -193,9 +193,9 @@ Portable Python can be used as a python library to invoke builds, or inspect an Invoke a build from python code:: - from portable_python import BuildSetup + from portable_python import BuildSetup, required_python_spec - setup = BuildSetup("cpython:3.9.7") + setup = BuildSetup(required_python_spec("cpython:3.9.7")) setup.compile() diff --git a/SECURITY.md b/SECURITY.md deleted file mode 100644 index ed546cf..0000000 --- a/SECURITY.md +++ /dev/null @@ -1,11 +0,0 @@ -# Security Policy - -## Supported Versions - -| Version | Supported | -| ------- | ------------------ | -| 1.x | :white_check_mark: | - -## Reporting a Vulnerability - -To report any bug, please open an issue in our [issue tracker](https://github.com/codrsquad/portable-python/issues). diff --git a/docs/architecture/build-setup.md b/docs/architecture/build-setup.md index 904bb27..8030b2a 100644 --- a/docs/architecture/build-setup.md +++ b/docs/architecture/build-setup.md @@ -11,9 +11,9 @@ timestamp: 2026-06-23T00:00:00Z The build orchestrator (`__init__.py`). The [`build`](/cli/build.md) command — and library users — construct one from a python spec and call `compile()`: ```python -from portable_python import BuildSetup +from portable_python import BuildSetup, required_python_spec -BuildSetup("cpython:3.13.2").compile() +BuildSetup(required_python_spec("cpython:3.13.2")).compile() ``` ## Mental model @@ -26,4 +26,4 @@ It owns the [`Folders`](/architecture/config.md) (resolved paths) and the python - **Start reading at `compile()`** — it calls everything else in order. - A `--prefix` makes the build **non-portable**; without it the result is relocatable (see [ppp-marker](/concepts/ppp-marker.md) and [portability](/concepts/portability.md)). -- It requires a full `X.Y.Z` spec — a bare `3.13` is rejected; `"latest"` (or empty) resolves to the newest known version. +- It takes a `PythonSpec` with a full `X.Y.Z` version: `required_python_spec()` validates that (a bare `3.13` is rejected) and resolves `"latest"` (or empty) to the newest known version. diff --git a/docs/changelog.md b/docs/changelog.md deleted file mode 100644 index e54b541..0000000 --- a/docs/changelog.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -type: Changelog -title: Changelog -description: High-level release highlights for portable-python, newest first. -tags: [changelog, releases] -timestamp: 2026-06-23T00:00:00Z ---- - -## Unreleased - -- Bumped SQLite to 3.53.4 and gettext-tiny to 0.3.3; re-checked every other component against upstream (all already current, or deliberately held back — see the `version` property comments). - -## v2.0.2 - -- Corrected `readline` static linking - -## v2.0.1 - -- Bumped the bundled components (OpenSSL, SQLite, Tcl/Tk, …) to current versions. -- Refreshed dependencies and modernized CI (now on uv / tox-uv, with updated GitHub Actions). -- Added an OKF documentation bundle under `docs/` (the old top-level docs folded into it). diff --git a/docs/concepts/telltale-detection.md b/docs/concepts/telltale-detection.md index b02e5c7..f81ae34 100644 --- a/docs/concepts/telltale-detection.md +++ b/docs/concepts/telltale-detection.md @@ -12,15 +12,15 @@ Before compiling, the tool must decide, for each candidate [external module](/mo ## The `m_telltale` attribute -Each [`ModuleBuilder`](/architecture/module-builder.md) subclass declares `m_telltale`, a path (or list of paths) using an `{include}` placeholder that is expanded against the target's system include directories: +Each [`ModuleBuilder`](/architecture/module-builder.md) subclass declares `m_telltale`, a path (or several, space-separated) using an `{include}` placeholder that is expanded against the target's system include directories: ```python class Openssl(ModuleBuilder): m_telltale = "{include}/openssl/ssl.h" -class Sqlite(ModuleBuilder): - m_telltale = ["{include}/sqlite3.h"] +class LibFFI(ModuleBuilder): + m_telltale = "{include}/ffi.h {include}/ffi/ffi.h" ``` `PPG.find_telltale()` expands the placeholder against each `target.sys_include` directory and returns the first existing path, or `None`. diff --git a/docs/guides/ci-cd.md b/docs/guides/ci-cd.md index d1ad8b3..b239809 100644 --- a/docs/guides/ci-cd.md +++ b/docs/guides/ci-cd.md @@ -18,13 +18,13 @@ Runs on every push to `main` and every pull request targeting `main`, with `canc |-----|--------------| | `test` | Matrix over Python `3.10`–`3.14` on `ubuntu-latest` (`fail-fast: false`). Runs `uvx --with tox-uv tox -e py`, then uploads coverage to Coveralls as a **parallel** build, flagged per version. | | `coveralls-finish` | After all `test` jobs finish, signals Coveralls that the parallel build is complete. | -| `linters` | On Python `3.14`, runs `uvx --with tox-uv tox -e style` (ruff check + format diff). | +| `linters` | On Python `3.14`, runs `uvx --with tox-uv tox -e style,ty,pyrefly,pyright,mypy,zuban` (ruff check + format diff, then the type checkers, see [type checking](/typecheck-v2.1.0.md)). | ## Release (`release.yml`) Triggers on pushing a tag matching `v[0-9]*`. The single `publish` job (Python `3.14`, GitHub environment `release`): -1. `uvx --with tox-uv tox -e py,style` — full test + lint gate. +1. `uvx --with tox-uv tox -e py,style,ty,pyrefly,pyright,mypy,zuban` — full test + lint + type check gate. 2. `uv build` — build the sdist + wheel. 3. `pypa/gh-action-pypi-publish` — publish to PyPI via **trusted publishing** (OIDC, `id-token: write` — no API token is stored). diff --git a/docs/guides/local-development.md b/docs/guides/local-development.md index 8eddc90..017f337 100644 --- a/docs/guides/local-development.md +++ b/docs/guides/local-development.md @@ -25,6 +25,7 @@ tox # all envs (py310-314, coverage, style) tox -e py313 # one python version tox -e style # lint check (ruff) tox -e reformat # auto-fix formatting +tox -e ty # a type checker (also: pyrefly, pyright, mypy, basedpyright, zuban) .venv/bin/pytest tests/ # directly .venv/bin/pytest tests/test_build.py::test_build_rc -vv # a single test diff --git a/docs/index.md b/docs/index.md index 367ce58..dacce79 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,7 +19,7 @@ Start with the [Overview](/overview.md), then drill into the area you need. * [Configuration](/configuration/index.md) - The `portable-python.yml` file, its sections and overrides. * [Guides](/guides/index.md) - Playbooks for common tasks: building, extending, local development. * [Release process](/release.md) - How to cut a release: changelog highlights, then a version tag. -* [Changelog](/changelog.md) - High-level release highlights, newest first. +* [Type checking in v2.1.0](/typecheck-v2.1.0.md) - What introducing the type checkers found and fixed. ## Conventions @@ -29,4 +29,4 @@ Start with the [Overview](/overview.md), then drill into the area you need. ## Authority This bundle is the **authoritative** documentation for portable-python's internals, configuration, CLI, and development workflows. -The repository's `README.rst` remains the project's landing page (install, quick start, motivation). +The repository's `README.rst` remains the project's landing page (install, quick start, motivation), and its `CHANGELOG.md` has the high-level release highlights, newest first. diff --git a/docs/log.md b/docs/log.md index 07b27b1..88b7b5b 100644 --- a/docs/log.md +++ b/docs/log.md @@ -1,12 +1,23 @@ # Knowledge Bundle Update Log +## 2026-10-09 + +* **Creation**: Added the [type checking in v2.1.0](/typecheck-v2.1.0.md) report. +* **Update**: The changelog moved out of the bundle, to `CHANGELOG.md` at the repo root (shipped in the sdist, like `README.rst`); [release process](/release.md) and the index point there. +* **Update**: [CI/CD](/guides/ci-cd.md): the `linters` job and the release gate run the type checkers (ty, pyrefly, pyright, mypy, zuban). +* **Update**: `BuildSetup` takes a `PythonSpec` (via `required_python_spec()`) in [BuildSetup](/architecture/build-setup.md), and `m_telltale` is a space-separated `str` in [telltale detection](/concepts/telltale-detection.md). + +## 2026-10-08 + +* **Update**: Listed the type checker tox envs (`tox -e ty`, ...) in [local development](/guides/local-development.md). + ## 2026-08-20 * **Update**: Bumped SQLite to 3.53.4 and gettext-tiny to 0.3.3, and recorded the stay-put rationale for LibFFI, OpenSSL and Readline in their `version` property comments (per the [bump components playbook](/guides/bump-components.md)). ## 2026-06-23 -* **Creation**: Added the [release process](/release.md) playbook and the [changelog](/changelog.md). +* **Creation**: Added the [release process](/release.md) playbook and the changelog. * **Creation**: Added the [bump component versions](/guides/bump-components.md) guide. * **Update**: Pulled two verified facts in from the project wiki — the relativization tools (`patchelf` / `install_name_tool`) and `--prefix` / `--enable-shared` support ([PythonInspector](/architecture/python-inspector.md)), and the `#sha256=` download-checksum URL fragment ([configuration](/configuration/portable-python-yml.md)). Skipped the wiki's `sources:` download-url design — it's an unimplemented draft. * **Update**: Trimmed the CLI pages — dropped the per-command option tables (which mirror `--help`) for purpose + non-obvious gotchas + examples, pointing at `--help` for the full flag list. diff --git a/docs/release.md b/docs/release.md index b11d105..5ef77f9 100644 --- a/docs/release.md +++ b/docs/release.md @@ -22,7 +22,7 @@ Review what changed since the last release: git diff ..main # e.g. git diff v2.0.0..main ``` -Summarize the **highlights** in [`changelog.md`](/changelog.md) — short and high-level. It should +Summarize the **highlights** in `CHANGELOG.md` (at the repo root) — short and high-level. It should give a reader a rough idea of the main changes since the last release, not a blow-by-blow; skip the details. diff --git a/docs/typecheck-v2.1.0.md b/docs/typecheck-v2.1.0.md new file mode 100644 index 0000000..29dff18 --- /dev/null +++ b/docs/typecheck-v2.1.0.md @@ -0,0 +1,111 @@ +--- +type: Report +title: Type checking in v2.1.0 +description: What introducing 6 type checkers found in v2.0.2, and what v2.1.0 changed in response (bugs, design, minor). +tags: [report, type-checking, ci, quality] +timestamp: 2026-10-09T00:00:00Z +--- + +# Type checking: v2.0.2 → v2.1.0 + +What v2.1.0 changed in response to type checker findings on v2.0.2. + +| Checker | Version | v2.0.2 | v2.1.0 | Actionable | Bugs | Design | Minor | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| ty | 0.0.85 | 51 | 0 | 51 | 0 | 5 | 6 | +| pyrefly | 1.3.2 | 96 | 0 | 96 | 1 | 6 | 7 | +| pyright | 1.1.414 | 93 | 0 | 93 | 1 | 6 | 8 | +| mypy | 2.4.0 | 32 | 0 | 32 | 0 | 3 | 4 | +| basedpyright | 1.40.2 | 2,979 | 2,576 | 646 | 1 | 6 | 8 | +| zuban | 0.10.0 | 38 | 0 | 38 | 2 | 4 | 5 | + +- **v2.0.2**, **v2.1.0**: findings on that release's `src/` and `tests/`, both checked with v2.1.0's + settings (`tox.ini`, `pyproject.toml`). "v2.0.2" is commit `c298694`: v2.0.2 plus the component + bumps that preceded this work. +- **Actionable**: v2.0.2 findings resolved by a change in v2.1.0. +- **Bugs**, **Design**, **Minor**: how many of the changes listed under [Bug fixes](#bug-fixes), + [Design](#design) and [Minor](#minor) the checker spotted. +- basedpyright counts include its warnings: 110 errors + 2,869 warnings on v2.0.2, 5 + 2,571 on + v2.1.0. Its 5 remaining errors are import cycles (`__init__` ↔ `versions` ↔ `cpython` ↔ + `inspector`). basedpyright is available locally (`tox -e basedpyright`) but doesn't gate CI, the + other 5 checkers do (see [CI/CD](/guides/ci-cd.md)). +- v2.1.0's zeros need runez 5.10.1, which fixed an annotation portable-python tripped on (see + [Fixed in runez](#fixed-in-runez)). +- v2.0.2 had 3 `# type:` comments, v2.1.0 has none. + +## Bug fixes + +A checker pointed at code that misbehaved. + +- With `cpython-use-github: true`, a failed GitHub API call (rate limiting, network error) crashed + with a `TypeError`: `RestClient.get()` returns `None` on failure, and the response was iterated + as-is [zuban, once `get_available_versions()` had a return annotation: mypy-style checkers skip + the body of unannotated functions] +- When `bin/python` couldn't be found after `make install`, a configured `cpython-validate-script` + or `cpython-additional-packages` ran `None` as the python executable before the "Can't find + bin/python" check happened: `bin_python` is now always a `Path`, and aborts with that message on + first use [pyrefly, pyright, basedpyright, zuban] + +## Design + +portable-python's invariants are now stated in its types, and enforced where they used to be +assumed. + +- `BuildSetup` takes a `PythonSpec`, the new `required_python_spec()` turns CLI text (or + `None`/`"latest"`) into one, with a clear error when it isn't a full `X.Y.Z` + version [ty, pyrefly, pyright, basedpyright] +- `VersionFamily` is an ABC (`get_available_versions()` and `get_builder()` are abstract): `latest` + is always a `Version`, aborting with "Could not determine latest cpython version" instead of the + puzzling "Invalid python spec: None", and `available_versions` is always a dict [all six] +- `Folders`: `build`, `destdir`, `sources` and `ppp-marker` are required (clear error when not + configured), only `dist` and `logs` are optional, and folders are `Path`s [pyrefly, pyright, + basedpyright, zuban] +- `PPG.family()` always aborts for an unknown family (its `fatal=` parameter is gone) [all six] +- `m_telltale` is a plain `str` (several files are space-separated), instead of a mix of `str` and + `ClassVar[list]` [all six] +- Library tracking: `Trackable` is an ABC declaring `relative_path` and `tracked_category`, + `LibType` moved to `tracking.py` (the only enum `Tracker` was ever used with), and + `Tracker.report()` no longer overrides `represented()` with a different return type [ty, pyrefly, + pyright, basedpyright] + +## Fixed in runez + +portable-python tripped on this runez annotation, fixed in runez 5.10.1 rather than worked around. + +- `to_path(None)` returns `None`: it always did at runtime, but runez 5.10.0's annotation only + allowed `str | Path` [all six] + +## Removed + +A checker flagged code that turned out not to be needed. + +- The `__main__` block of `cli.py` re-imported `main` from its own module [ty, pyrefly, zuban] + +## Minor + +The checker was right, the fix changes little in practice. + +- Attributes say what they hold, including `None` (`ModuleBuilder.m_*`, `parent_module`, + `ModuleCollection`, `_bin_python`, `BuildSetup.build_context`, ...) [all six] +- Base class methods declare their return type, so overrides agree (`url`, `auto_select_reason()`, + `candidate_modules()`, `get_problem()`, `xenv_LDFLAGS()`) [all six] +- `config.py`: `ConfigSource.get_value()` walks the key in a loop, `SingleFileMatch` sets its + attributes in `__init__`, and `is_match()` returns a `bool` [all six] +- `# type:` comments became annotations [pyrefly, pyright, basedpyright, zuban] +- Inspector: report lists hold strings, `relative_path()` returns a `str`, no `x and x.y` + chains [pyrefly, mypy, zuban] +- Version listing checks `v.patch` truthiness instead of `v.patch > 0` [ty, pyrefly, pyright, + basedpyright] +- Tests: no poking at `Config._sources`, the mock's patcher is created in `__init__` [ty, pyrefly, + pyright, basedpyright] +- `runez.ActivateColors` instead of `runez.colors.ActivateColors` [pyright, basedpyright] +- `recompress` takes family and version from a spec only when the path names one [pyright, + basedpyright] +- `run_make(cpu_count=...)` is annotated [ty] + +## Not from a type checker + +- Folder paths containing a space are refused by portable-python itself, with the reason spelled + out ("escapes too complex with makefiles, shell scripts etc"), instead of relying on a niche + runez helper +- New tests: missing `bin/python`, python.org listing no versions, folder path with a space diff --git a/pyproject.toml b/pyproject.toml index 7959146..6765097 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,18 @@ directory = ".tox/test-reports/htmlcov" [tool.pytest.ini_options] cache_dir = ".tox/.cache" +[tool.pyright] # also read by basedpyright (it refuses a pyproject that has both sections) +pythonVersion = "3.10" +include = ["src", "tests"] + +[tool.mypy] # also read by zuban +python_version = "3.10" +files = ["src", "tests"] + +[tool.pyrefly] # without this section, pyrefly adopts [tool.pyright] (or [tool.mypy]) +python-version = "3.10" +project-includes = ["src", "tests"] + [tool.ruff] cache-dir = ".tox/.ruff_cache" line-length = 140 diff --git a/src/portable_python/__init__.py b/src/portable_python/__init__.py index 89eb30a..e9bea43 100644 --- a/src/portable_python/__init__.py +++ b/src/portable_python/__init__.py @@ -2,9 +2,9 @@ Designed to be used via portable-python CLI. Can be used programmatically too, example usage: - from portable_python import BuildSetup + from portable_python import BuildSetup, required_python_spec - setup = BuildSetup("cpython:3.9.6") + setup = BuildSetup(required_python_spec("cpython:3.9.6")) setup.compile() """ @@ -15,8 +15,8 @@ import os import pathlib import re +from collections.abc import Iterable from string import Template -from typing import ClassVar import runez from runez.http import RestClient @@ -147,9 +147,6 @@ def _resolved_isolation(self): return v - def __repr__(self): - return self.isolate_usr_local or "none" - def __enter__(self): runez.Anchored.add(self.setup.folders.base_folder) if self.isolate_usr_local == "mount-shadow": @@ -189,6 +186,18 @@ def __exit__(self, exc_type, exc_val, exc_tb): runez.Anchored.pop(self.setup.folders.base_folder) +def required_python_spec(text: str | None) -> PythonSpec: + """Python to build, with a full version (aborts with a clear message otherwise), default: latest cpython""" + spec = PythonSpec.from_object(PPG.cpython.latest if not text or text == "latest" else text) + if not spec: + runez.abort("Invalid python spec: %s" % runez.red(text)) + + if spec.version.given_components_count < 3: + runez.abort("Please provide full desired version: %s is not good enough" % runez.red(spec)) + + return spec + + class BuildSetup: """ Drives the compilation, external modules first, then the target python itself. @@ -198,33 +207,22 @@ class BuildSetup: # Internal, used to ensure files under {logs}/ folder sort alphabetically in the same order they were compiled log_counter = 0 - def __init__(self, python_spec=None, modules=None, prefix=None): + def __init__(self, python_spec: PythonSpec, modules: str | None = None, prefix: str | None = None): """ Parameters ---------- - python_spec : str | PythonSpec | None - Python to build (family and version) - modules : str | None + python_spec + Python to build (family and full version), see `required_python_spec()` + modules Modules to build (default: from config) - prefix : str | None + prefix --prefix to use """ - if not python_spec or python_spec == "latest": - python_spec = PPG.cpython.latest - - if not isinstance(python_spec, PythonSpec): - ps = PythonSpec.from_object(python_spec) - runez.abort_if(not ps, "Invalid python spec: %s" % runez.red(python_spec)) - python_spec = ps - - runez.abort_if(not python_spec.version or not python_spec.version.is_valid, "Invalid python spec: %s" % runez.red(python_spec)) - if len(python_spec.version.given_components) < 3: - runez.abort("Please provide full desired version: %s is not good enough" % runez.red(python_spec)) - self.python_spec = python_spec self.folders = PPG.get_folders( base=os.getcwd(), family=python_spec.family, version=python_spec.version, abi_suffix=python_spec.abi_suffix ) + self.build_context: BuildContext | None = None # Set during compile() self.desired_modules = modules prefix = self.folders.formatted(prefix) self.prefix = prefix @@ -244,7 +242,7 @@ def __init__(self, python_spec=None, modules=None, prefix=None): ) builder = PPG.family(python_spec.family).get_builder() - self.python_builder = builder(self) # type: PythonBuilder + self.python_builder = builder(self) def __repr__(self): return str(self.folders) @@ -301,41 +299,35 @@ def compile(self): class ModuleCollection: """Models a collection of sub-modules, with auto-detection and reporting as to what is active and why""" - candidates: list["ModuleBuilder"] = None - desired: str = None - selected: list["ModuleBuilder"] = None - - def __init__(self, parent_module: "ModuleBuilder", desired=None): - self.selected = [] - self.auto_selected = {} - self.candidates = [] + def __init__(self, parent_module: "ModuleBuilder", desired: str | None = None): + self.selected: list[ModuleBuilder] = [] + self.auto_selected: dict[str, str] = {} + self.candidates: list[ModuleBuilder] = [] self.desired = desired - self.module_by_name = {} # type: dict[str, ModuleBuilder] - candidates = parent_module.candidate_modules() - if candidates: - for module in candidates: - module = module(parent_module) - self.candidates.append(module) - self.module_by_name[module.m_name] = module + self.module_by_name: dict[str, ModuleBuilder] = {} + for module_class in parent_module.candidate_modules(): + module = module_class(parent_module) + self.candidates.append(module) + self.module_by_name[module.m_name] = module if desired == "all": self.selected = self.candidates return - desired = [] if desired == "none" else runez.flattened(desired, split=True) - desired = runez.flattened(desired, split=",") - unknown = [x for x in desired if x not in self.module_by_name] + names = [] if desired == "none" else runez.flattened(desired, split=True) + names = runez.flattened(names, split=",") + unknown = [x for x in names if x not in self.module_by_name] if unknown: runez.abort("Unknown modules: %s" % runez.joined(unknown, delimiter=", ", stringify=runez.red)) for candidate in self.candidates: - if candidate.m_name not in desired: + if candidate.m_name not in names: reason = candidate.auto_select_reason() if reason: self.auto_selected[candidate.m_name] = reason - desired.extend(self.auto_selected.keys()) - self.selected = [self.module_by_name[x] for x in desired] + names.extend(self.auto_selected.keys()) + self.selected = [self.module_by_name[x] for x in names] def __repr__(self): return "selected: %s (%s)" % (self.desired, runez.plural(self.selected, "module")) @@ -402,13 +394,13 @@ class LinkerOutcome(enum.Enum): class ModuleBuilder: """Common behavior for all external (typically C) modules to be compiled""" - m_build_cwd: str = None # Optional: relative (to unpacked source) folder where to run configure/make from - m_debian = None - m_include: str = None # Optional: subfolder to automatically list in CPATH when this module is active - m_telltale: ClassVar[list] # Optional: list of files that, if present, indicate this module is installed + m_build_cwd: str | None = None # Optional: relative (to unpacked source) folder where to run configure/make from + m_debian: str | None = None # Optional: debian package, prefixed with "!": required, "+": required if selected, "-": prevents static + m_include: str | None = None # Optional: subfolder to automatically list in CPATH when this module is active + m_telltale: str = "" # Optional: file(s) that, if present, indicate this module is installed (space separated) setup: BuildSetup - parent_module: "ModuleBuilder" = None + parent_module: "ModuleBuilder | None" = None _log_handler = None def __init__(self, parent_module): @@ -434,13 +426,14 @@ def __repr__(self): return "%s:%s" % (self.m_name, self.version) @classmethod - def candidate_modules(cls) -> list: + def candidate_modules(cls) -> Iterable[type["ModuleBuilder"]]: """All possible candidate external modules for this builder""" + return [] def selected_modules(self): return ModuleCollection(self, desired="all") - def auto_select_reason(self): + def auto_select_reason(self) -> str | None: """ If this module must be selected (build can't succeed without), descendant should return short explanation why """ @@ -480,11 +473,10 @@ def scan_note(self): return "no %s" % self.m_telltale def _find_telltale(self): - telltales = getattr(self, "m_telltale", runez.UNSET) - if telltales is runez.UNSET: - return telltales + if not self.m_telltale: + return runez.UNSET # Sub-modules don't have a telltale - return PPG.find_telltale(telltales) + return PPG.find_telltale(self.m_telltale) def active_module(self, name): return self.modules.active_module(name) @@ -523,7 +515,7 @@ def cfg_patches(self): return PPG.config.get_value("%s-patches" % self.m_name) @property - def url(self): + def url(self) -> str: """Url of source tarball, if any""" return "" @@ -607,7 +599,7 @@ def run_configure(self, program, *args, prefix=None): cmd = runez.flattened(*program, prefix, *args) return self._do_run(*cmd) - def run_make(self, *args, program="make", cpu_count=None): + def run_make(self, *args, program: str = "make", cpu_count: int | None = None): cmd = program.split() if cpu_count is None: available = multiprocessing.cpu_count() @@ -782,7 +774,7 @@ def _finalize(self): class PythonBuilder(ModuleBuilder): - _bin_python: pathlib.Path = None + _bin_python: pathlib.Path | None = None def __init__(self, parent_module): super().__init__(parent_module) @@ -799,15 +791,14 @@ def selected_modules(self): return ModuleCollection(self, desired=desired) @property - def bin_python(self): - """ - Returns - ------- - pathlib.Path | None - Path to freshly compiled bin/python, real file (not symlink) - """ + def bin_python(self) -> pathlib.Path: + """Path to freshly compiled bin/python, real file (not symlink)""" if self._bin_python is None: - self._bin_python = PPG.config.find_main_file(self.bin_folder / "python", self.version) + bin_python = PPG.config.find_main_file(self.bin_folder / "python", self.version) + if not bin_python: + runez.abort(f"Can't find bin/python in {self.bin_folder}") + + self._bin_python = bin_python return self._bin_python @@ -817,6 +808,7 @@ def version(self): def xenv_LDFLAGS(self): """Python builder does not reuse the common setting""" + yield from () def run_python(self, *args): """Run python command, using the freshly compiled python binary""" diff --git a/src/portable_python/cli.py b/src/portable_python/cli.py index 5e1fa81..35c23a6 100644 --- a/src/portable_python/cli.py +++ b/src/portable_python/cli.py @@ -5,7 +5,7 @@ from runez.pyenv import PythonSpec from runez.render import PrettyTable -from portable_python import BuildSetup, PPG +from portable_python import BuildSetup, PPG, required_python_spec from portable_python.inspector import LibAutoCorrect, PythonInspector LOG = logging.getLogger(__name__) @@ -41,7 +41,7 @@ def main(config, quiet, target): @click.argument("python_spec") def build(modules, prefix, python_spec): """Build a portable python binary""" - setup = BuildSetup(python_spec, modules=modules, prefix=prefix) + setup = BuildSetup(required_python_spec(python_spec), modules=modules, prefix=prefix) setup.compile() @@ -50,7 +50,7 @@ def build(modules, prefix, python_spec): @click.argument("python_spec", required=False) def build_report(modules, python_spec): """Show status of buildable modules, which will be auto-compiled""" - setup = BuildSetup(python_spec, modules=modules) + setup = BuildSetup(required_python_spec(python_spec), modules=modules) print(runez.bold(setup.python_spec)) report = setup.python_builder.modules.report() print(report) @@ -90,10 +90,7 @@ def inspect(modules, verbose, prefix, skip_so, path): @click.argument("family", default="cpython") def list_cmd(json, family): """List latest versions""" - fam = PPG.family(family, fatal=False) - if not fam: - runez.abort("Python family '%s' is not yet supported" % runez.red(family)) - + fam = PPG.family(family) if json: print(runez.represented_json(fam.available_versions)) return @@ -116,6 +113,7 @@ def _find_recompress_source(folders, path): candidates = [folders.base_folder, folders.build_folder, folders.dist, folders.destdir] for candidate in candidates: if candidate: + # TODO: `runez.to_path()` can be simplified away when all `folders.` attributes become of type Path candidate = runez.to_path(candidate) / path if candidate.exists(): return candidate.absolute() @@ -162,8 +160,8 @@ def recompress(path, ext): Mildly useful for comparing sizes from different compressions """ # noqa: D301 extension = runez.SYS_INFO.platform_id.canonical_compress_extension(ext) - pspec = PythonSpec.from_text(path) - folders = PPG.get_folders(base=".", family=pspec and pspec.family, version=pspec and pspec.version) + pspec = PythonSpec.from_text(path) # Best effort: a build folder like `3.9.7` yields a spec, archive names don't + folders = PPG.get_folders(base=".", family=pspec.family if pspec else "cpython", version=pspec.version if pspec else None) with runez.Anchored(folders.base_folder): actual_path = _find_recompress_source(folders, path) if not actual_path: @@ -205,6 +203,4 @@ def lib_auto_correct(commit, prefix, path): if __name__ == "__main__": - from portable_python.cli import main - main() diff --git a/src/portable_python/config.py b/src/portable_python/config.py index 3aee760..0ea57dd 100644 --- a/src/portable_python/config.py +++ b/src/portable_python/config.py @@ -90,8 +90,8 @@ def __init__(self, paths=None, target=None): self.target = target self.default = ConfigSource("default config", self.parsed_yaml(DEFAULT_CONFIG, "default config")) - self._sources = [] # type: list[ConfigSource] - self.by_path = {} + self._sources: list[ConfigSource] = [] + self.by_path: dict[str, ConfigSource] = {} for path in self.paths: self.load(path) @@ -228,9 +228,9 @@ def cleanup_globs(self, title, module, *globs): matcher = FileMatcher(spec) LOG.info("Applying clean-up spec: %s", matcher) cleaned = [] - for dirpath, dirnames, filenames in os.walk(module.install_folder): + for root, dirnames, filenames in os.walk(module.install_folder): removed = [] - dirpath = runez.to_path(dirpath) + dirpath = pathlib.Path(root) for name in dirnames: full_path = dirpath / name if matcher.is_match(full_path): @@ -270,7 +270,7 @@ def symlink_duplicates(self, folder): def real_path(path: pathlib.Path): if path and path.exists(): if path.is_symlink(): - path = runez.to_path(os.path.realpath(path)) + path = path.resolve() return path @@ -325,8 +325,7 @@ def load(self, path, base=None): front = True path = path[1:] - path = runez.resolved_path(path, base=base) - path = runez.to_path(path) + path = pathlib.Path(runez.resolved_path(path, base=base)) if path.exists(): with open(path) as fh: data = self.parsed_yaml(fh, path) @@ -368,22 +367,14 @@ def get_value(self, key): str | int | float | bool | dict | list | None Associated value, if any """ - return self._deep_get(self.data, key) + value = self.data + for k in key if isinstance(key, tuple) else (key,): + if not isinstance(value, dict): + return None - def _deep_get(self, data, key): - if not key or not isinstance(data, dict): - return None + value = value.get(k) - if isinstance(key, tuple): - if len(key) > 1: - value = self._deep_get(data, key[0]) - return self._deep_get(value, key[1:]) - - key = key[0] - - value = data.get(key) - if value is not None: - return value + return value class FileMatcher: @@ -395,42 +386,37 @@ def __init__(self, clean_spec): def __repr__(self): return runez.joined(self.matches) - def is_match(self, path: pathlib.Path): - for m in self.matches: - if m.is_match(path): - return path + def is_match(self, path: pathlib.Path) -> bool: + return any(m.is_match(path) for m in self.matches) class SingleFileMatch: - _on_folder = False - _rx_basename = None - _rx_path = None - def __init__(self, spec: str): self.spec = spec - if spec.endswith("/"): - spec = spec[:-1] - self._on_folder = True - + self._on_folder = spec.endswith("/") + spec = spec.removesuffix("/") + self._rx_path: re.Pattern[str] | None = None if "/" in spec: # lib/*/config-{python_mm}-\w+/ path = ".*/%s$" % os.path.dirname(spec).replace("*", ".*").strip("/") spec = os.path.basename(spec) self._rx_path = re.compile(path) - self._rx_basename = spec + self._basename_glob = spec def __repr__(self): return self.spec - def is_match(self, path: pathlib.Path): + def is_match(self, path: pathlib.Path) -> bool: if self._on_folder == path.is_dir(): if self._rx_path: m = self._rx_path.match(str(path.parent)) if not m: return False - return fnmatch.fnmatch(path.name, self._rx_basename) + return fnmatch.fnmatch(path.name, self._basename_glob) + + return False def _find_file_duplicates(seen, folder): diff --git a/src/portable_python/cpython.py b/src/portable_python/cpython.py index e1e2057..5f96c61 100644 --- a/src/portable_python/cpython.py +++ b/src/portable_python/cpython.py @@ -79,7 +79,7 @@ def build_information(self): "host-platform": runez.SYS_INFO.platform_info, "ldd-version": PythonInspector.tool_version("ldd"), "portable-python-version": runez.get_version(__package__), - "special-context": bc.isolate_usr_local and bc, + "special-context": bc.isolate_usr_local if bc else None, }, ) additional = PPG.config.get_value("manifest", "additional-info") @@ -280,7 +280,6 @@ def _finalize(self): self.run_python("-mpip", "install", "--no-cache-dir", "--upgrade", *runez.flattened(additional)) - runez.abort_if(not runez.DRYRUN and not self.bin_python, f"Can't find bin/python in {self.bin_folder}") PPG.config.ensure_main_file_symlinks(self) if not self.setup.prefix: self._relativize_sysconfig() @@ -305,7 +304,7 @@ def _finalize(self): info_path = PPG.config.get_value("manifest", "inspection-report") if info_path: - with runez.colors.ActivateColors(enable=False): + with runez.ActivateColors(enable=False): py_inspector = PythonInspector(self.install_folder, modules="all") contents = "%s\n" % py_inspector.represented(verbose=True) runez.write(self.install_folder / info_path, contents) diff --git a/src/portable_python/external/xcpython.py b/src/portable_python/external/xcpython.py index fe99784..572c74d 100644 --- a/src/portable_python/external/xcpython.py +++ b/src/portable_python/external/xcpython.py @@ -1,5 +1,4 @@ import os.path -from typing import ClassVar import runez @@ -81,7 +80,7 @@ class Gdbm(ModuleBuilder): """ m_debian = "libgdbm-dev" - m_telltale: ClassVar[list] = ["{include}/gdbm.h"] + m_telltale = "{include}/gdbm.h" @property def url(self): @@ -124,7 +123,7 @@ class LibFFI(ModuleBuilder): """ m_debian = "!libffi-dev" - m_telltale: ClassVar[list] = ["{include}/ffi.h", "{include}/ffi/ffi.h"] + m_telltale = "{include}/ffi.h {include}/ffi/ffi.h" xenv_CFLAGS = "-fPIC" @@ -306,7 +305,7 @@ class Sqlite(ModuleBuilder): """ m_debian = "+libsqlite3-dev" - m_telltale: ClassVar[list] = ["{include}/sqlite3.h"] + m_telltale = "{include}/sqlite3.h" xenv_CFLAGS = "-fPIC" @@ -349,7 +348,7 @@ class Uuid(ModuleBuilder): m_debian = "+uuid-dev" m_include = "uuid" - m_telltale: ClassVar[list] = ["{include}/uuid/uuid.h"] + m_telltale = "{include}/uuid/uuid.h" xenv_CFLAGS = "-fPIC" diff --git a/src/portable_python/external/xtkinter.py b/src/portable_python/external/xtkinter.py index 8d239d1..660ca40 100644 --- a/src/portable_python/external/xtkinter.py +++ b/src/portable_python/external/xtkinter.py @@ -5,8 +5,6 @@ Contributions are welcome! """ -from typing import ClassVar - import runez from portable_python import ModuleBuilder, PPG @@ -110,7 +108,7 @@ class TkInter(ModuleBuilder): """ m_debian = "-tk-dev" - m_telltale: ClassVar[list] = ["{include}/tk", "{include}/tk.h"] + m_telltale = "{include}/tk {include}/tk.h" @classmethod def candidate_modules(cls): diff --git a/src/portable_python/inspector.py b/src/portable_python/inspector.py index 55b52b7..fc3cb56 100644 --- a/src/portable_python/inspector.py +++ b/src/portable_python/inspector.py @@ -1,31 +1,21 @@ import collections -import enum import json import logging import os import re +from pathlib import Path +from typing import Any import runez from runez.render import PrettyTable -from portable_python.tracking import Trackable, Tracker +from portable_python.tracking import LibType, Trackable, Tracker from portable_python.versions import PPG LOG = logging.getLogger(__name__) RX_DYNLIB = re.compile(r"^.*\.(so(\.[0-9.]+)?|dylib)$") -class LibType(enum.Enum): - """Categorization for dynamic libs, value being the color we want to show them as""" - - base = "" - lib_static = "static" - libpython_so = "dynamic" - missing = "red" - other = "brown" - system = "blue" - - class TempChmod: """ Temporarily chmod a given file. @@ -140,23 +130,21 @@ def _auto_correct_macos(self, path): @staticmethod def _shared_ref_top_level(relative_path): - parts = runez.to_path(relative_path).parts - if len(parts) > 1: - parts = parts[:-1] # Remove basename - for part in parts: - yield part - if part != "..": - return + for part in runez.to_path(relative_path).parts[:-1]: # Excluding basename + yield part + if part != "..": + return class ModuleInfo: _regex = re.compile(r"^(.*?)\s*(\S+/(lib(64)?/.*))$") - def __init__(self, inspector: "PythonInspector", name: str, payload: dict): + def __init__(self, inspector: "PythonInspector", name: str, payload: dict[str, Any]): self.inspector = inspector self.name = name self.payload = payload - self.filepath = runez.to_path(payload.get("path")) + path = payload.get("path") + self.filepath = runez.to_path(path) if path else None self.note = payload.get("note") self.version = payload.get("version") self.version_field = payload.get("version_field") @@ -202,7 +190,7 @@ def report_rows(self): class CLibInfo(Trackable): - def __init__(self, inspector: "PythonInspector", path: str, version: str, basename: str): + def __init__(self, inspector: "PythonInspector", path: str, version: str | None, basename: str | None): self.inspector = inspector if not basename: basename = os.path.basename(path) @@ -242,10 +230,9 @@ def short_name(self): def represented(self, verbose=False): tc = self.tracked_category if verbose: - yield runez.joined("[%s]" % runez.colored(tc.name, color=tc.value), self.relative_path, self.version, delimiter=" ") + return runez.joined("[%s]" % runez.colored(tc.name, color=tc.value), self.relative_path, self.version, delimiter=" ") - else: - yield runez.colored(runez.joined(self.short_name, self.version, delimiter=":"), color=tc.value) + return runez.colored(runez.joined(self.short_name, self.version, delimiter=":"), color=tc.value) class SoInfo(Trackable): @@ -254,7 +241,7 @@ def __init__(self, inspector: "PythonInspector", path): self.path = runez.to_path(path) self.relative_path = inspector.relative_path(self.path) self.extension = self.path.name.rpartition(".")[2] - self.lib_tracker = Tracker(LibType, ".so") + self.lib_tracker = Tracker(".so") program, output = self._dot_so_listing(self.path) self.is_failed = "_failed" in self.path.name self.short_name = "%s*" % self.path.name.partition(".")[0] @@ -336,7 +323,7 @@ def add_ref(self, path, version=None, basename=None): info = CLibInfo(self.inspector, path, version, basename) self.lib_tracker.add(info) - def represented(self, verbose=None): + def represented(self, verbose=False): report = [] if verbose: path = self.inspector.relative_path(self.path) @@ -354,14 +341,14 @@ def represented(self, verbose=None): for tp in types: c = self.lib_tracker.category[tp] if c.items: - r = c.represented(verbose) + text = runez.joined(c.represented(verbose), delimiter="\n " if verbose else delimiter) if verbose: - r = " %s" % runez.joined(r, delimiter="\n ") + text = f" {text}" elif tp is LibType.missing: - r = "%s: %s" % (runez.red("missing"), runez.joined(r, delimiter=delimiter)) + text = f"{runez.red('missing')}: {text}" - report.append(r) + report.append(text) return runez.joined(report, delimiter=delimiter) @@ -379,7 +366,7 @@ def find_libs(folder): yield from find_libs(path) -def get_lib_type(install_folder, path, basename): +def get_lib_type(install_folder, path, basename) -> LibType: if basename.startswith("libpython") and path.startswith(install_folder): return LibType.libpython_so @@ -408,14 +395,15 @@ def __init__(self, spec, modules=None): script = os.path.join(os.path.dirname(__file__), "external/_inspect.py") r = runez.run(self.python.executable, script, arg, fatal=False, logger=print if runez.DRYRUN else logging.debug) self.output = r.output if r.succeeded else "exit code: %s\n%s" % (r.exit_code, r.full_output) - self.payload = None + self.payload: dict[str, Any] | None = None if self.output and self.output.startswith("{"): self.payload = json.loads(self.output) - self.reported_prefix = self.payload and self.payload.get("prefix") - self.srcdir = runez.to_path(self.payload and self.payload.get("srcdir")) + payload = self.payload or {} + self.reported_prefix = payload.get("prefix") + self.srcdir = runez.to_path(payload.get("srcdir")) self.lib_folder = _find_parent_subfolder(self.srcdir, "lib") - self.install_folder = self.lib_folder and str(self.lib_folder.parent) + self.install_folder = str(self.lib_folder.parent) if self.lib_folder else None def __repr__(self): return str(self.python) @@ -441,7 +429,7 @@ def module_info(self): if self.payload: return {k: ModuleInfo(self, k, v) for k, v in self.payload.get("report", {}).items()} - def relative_path(self, path): + def relative_path(self, path) -> str: with runez.Anchored(self.install_folder): p = runez.short(path, size=4096) m = re.match(r"^.*\.\.\./(.*)$", p) @@ -451,7 +439,7 @@ def relative_path(self, path): return p def libpython_report(self, items): - if not items: + if not items or not self.install_folder: # install_folder is known when there are items (they're found in it) return runez.green("-not used-") rel_paths = [getattr(x, "relative_path", x) for x in items] @@ -476,12 +464,11 @@ def represented(self, verbose=False): table.add_row("lib", runez.short(self.lib_folder)) table.add_row("srcdir", runez.short(self.srcdir)) - report.append(table) + report.append(str(table)) if verbose: - report.append(self.full_so_report) + report.append(str(self.full_so_report)) if self.full_so_report.problematic: # pragma: no cover (don't have a handy problematic python test case) - pb = self.full_so_report.problematic.represented(verbose=verbose) - report.append(runez.joined(pb, delimiter="\n")) + report.append(self.full_so_report.problematic.report(verbose=verbose)) report.append("\n-- Library users:") for what, users in self.full_so_report.lib_tracker.users.items(): color = what.tracked_category.value or "green" @@ -489,8 +476,7 @@ def represented(self, verbose=False): report.append(runez.short(overview)) if verbose: - pb = self.full_so_report.ok.represented(verbose=True) - report.append(runez.joined(pb, delimiter="\n")) + report.append(self.full_so_report.ok.report(verbose=True)) report = runez.joined(report or self.output, delimiter="\n") return runez.joined(report, delimiter="\n") @@ -530,21 +516,23 @@ def tool_version(tool_name): return PythonInspector.parsed_version(output.output or output.error) -def _find_parent_subfolder(folder, basename): +def _find_parent_subfolder(folder: Path | None, basename: str) -> Path | None: while folder and len(folder.parts) > 1: if folder.name == basename: return folder folder = folder.parent + return None + class FullSoReport: def __init__(self, inspector: PythonInspector): self.inspector = inspector self.size = 0 - self.lib_tracker = Tracker(LibType) - self.ok = Tracker(LibType, "OK") - self.problematic = Tracker(LibType, "problematic") + self.lib_tracker = Tracker() + self.ok = Tracker("OK") + self.problematic = Tracker("problematic") self.libpython_so = [] self.lib_static = [] for path in find_libs(self.inspector.lib_folder): @@ -564,21 +552,22 @@ def __init__(self, inspector: PythonInspector): def __repr__(self): return runez.joined(".so files: %s" % runez.represented_bytesize(self.size), self.problematic, self.ok, delimiter=", ") - def get_problem(self, portable) -> str: + def get_problem(self, portable) -> str | None: if portable: - allowed = PPG.config.get_value("allowed-system-libs") + allowed: str | None = PPG.config.get_value("allowed-system-libs") if allowed: # allowed-system-libs needs to be defined for the check to occur (opt-in) - allowed = re.compile(allowed) + rx_allowed = re.compile(allowed) uses_system = [x.relative_path for x in self.lib_tracker.category[LibType.system].items] - if uses_system: - uses_system = [x for x in uses_system if not allowed.match(x)] + uses_system = [x for x in uses_system if not rx_allowed.match(x)] if uses_system: return "Uses system libs: %s" % runez.red(runez.joined(uses_system)) if self.problematic: # pragma: no cover (don't have a handy problematic python test case) - return runez.joined(self.problematic.represented()) + return self.problematic.report() if not runez.DRYRUN and not self.ok: return "Internal error: %s" % runez.red("no OK libs found") + + return None diff --git a/src/portable_python/tracking.py b/src/portable_python/tracking.py index 5cd8009..c55e03f 100644 --- a/src/portable_python/tracking.py +++ b/src/portable_python/tracking.py @@ -1,23 +1,45 @@ +import abc +import enum +from collections.abc import Iterator + import runez -class Trackable: - tracked_category = None +class LibType(enum.Enum): + """Categorization for dynamic libs, value being the color we want to show them as""" + + base = "" + lib_static = "static" + libpython_so = "dynamic" + missing = "red" + other = "brown" + system = "blue" + + +class Trackable(abc.ABC): + """A file (dynamic lib, .so, ...) that can be tracked by a `Tracker`""" + + relative_path: str + tracked_category: LibType # Set by leaf trackables only, the ones yielded by `__iter__()` - def __eq__(self, other): + def __eq__(self, other: object) -> bool: return str(self) == str(other) - def __hash__(self): + def __hash__(self) -> int: return hash(str(self)) - def __iter__(self): + def __iter__(self) -> Iterator["Trackable"]: yield self + @abc.abstractmethod + def represented(self, verbose: bool = False) -> str: + """Textual representation, for reports""" + class TrackedCollection: - def __init__(self, name): + def __init__(self, name: str): self.name = name - self.items = [] + self.items: list[Trackable] = [] def __repr__(self): return "%s %s" % (len(self.items), self.name) @@ -25,50 +47,30 @@ def __repr__(self): def __bool__(self): return bool(self.items) - def add(self, item): + def add(self, item: Trackable): if item not in self.items: self.items.append(item) - def represented(self, verbose=False): - if self.items: - for item in self.items: - yield item.represented(verbose=verbose) + def represented(self, verbose: bool = False) -> Iterator[str]: + for item in self.items: + yield item.represented(verbose=verbose) class Tracker(TrackedCollection): - def __init__(self, enum, name=None): - self.kind = enum.__name__.replace("Type", "").lower() - super().__init__(name or self.kind) - self.enum = enum - self.category = {} - self.users = {} - for x in enum: - c = TrackedCollection("%s %s" % (x.name, self.kind)) - self.category[x] = c - - def add(self, item): - if item not in self.items: - self.items.append(item) + def __init__(self, name: str = "lib"): + super().__init__(name) + self.category = {x: TrackedCollection(f"{x.name} lib") for x in LibType} + self.users: dict[Trackable, set[Trackable]] = {} + def add(self, item: Trackable): + super().add(item) for trackable in item: - if trackable.tracked_category: - c = self.category[trackable.tracked_category] - c.add(trackable) - if trackable is not item: - u = self.users.get(trackable) - if u is None: - u = set() - self.users[trackable] = u - - u.add(item) - - def represented(self, verbose=False): - report = [] - if self.items: - for item in self.items: - report.append(item.represented(verbose)) - - report = runez.joined(report, delimiter="\n") + self.category[trackable.tracked_category].add(trackable) + if trackable is not item: + self.users.setdefault(trackable, set()).add(item) + + def report(self, verbose: bool = False) -> str: + report = runez.joined(self.represented(verbose), delimiter="\n") if verbose or report: report = "\n-- %s\n%s" % (runez.colored(self, "bold"), report) diff --git a/src/portable_python/versions.py b/src/portable_python/versions.py index a078789..a6f6b47 100644 --- a/src/portable_python/versions.py +++ b/src/portable_python/versions.py @@ -4,10 +4,13 @@ Not trying to do historical stuff here, older (or EOL-ed) versions will be removed from the list without notice. """ +import abc import logging import os import re -from typing import ClassVar +from collections.abc import Iterable +from pathlib import Path +from typing import ClassVar, TYPE_CHECKING import runez from runez.http import RestClient @@ -15,12 +18,15 @@ from portable_python.config import Config +if TYPE_CHECKING: + from portable_python import PythonBuilder -class VersionFamily: + +class VersionFamily(abc.ABC): """Common ancestor for python family implementations""" - _latest = None - _versions = None + _latest: Version | None = None + _versions: dict[str, Version] | None = None def __init__(self): self.family_name = self.__class__.__name__[:7].lower() @@ -28,47 +34,38 @@ def __init__(self): def __repr__(self): return self.family_name - def _fetch_versions(self): + def _fetch_versions(self) -> dict[str, Version]: if self._versions is None: self._versions = {} - versions = self.get_available_versions() - versions = versions and sorted(versions, reverse=True) - if versions: - self._latest = versions[0] - for v in versions: - if v.mm not in self._versions: - self._versions[v.mm] = v + versions = sorted(self.get_available_versions(), reverse=True) + self._latest = versions[0] if versions else None + for v in versions: + if v.mm not in self._versions: + self._versions[v.mm] = v + + return self._versions @property def latest(self) -> Version: """Latest version for this family""" self._fetch_versions() + if self._latest is None: + runez.abort(f"Could not determine latest {self} version") + return self._latest @property - def available_versions(self): - """Supplied by descendant: list of available versions""" - self._fetch_versions() - return self._versions - - def get_available_versions(self): - """ - Available versions. + def available_versions(self) -> dict[str, Version]: + """Latest version per major.minor (as supplied by descendant)""" + return self._fetch_versions() - Returns - ------- - list[Version] - """ + @abc.abstractmethod + def get_available_versions(self) -> Iterable[Version]: + """Available versions""" - def get_builder(self): - """ - Builder implementation for this family. - - Returns - ------- - portable_python.PythonBuilder - Builder implementation for this family - """ + @abc.abstractmethod + def get_builder(self) -> type["PythonBuilder"]: + """Builder implementation for this family""" class CPythonFamily(VersionFamily): @@ -80,11 +77,11 @@ class CPythonFamily(VersionFamily): def client(self): return RestClient() - def get_available_versions(self): + def get_available_versions(self) -> Iterable[Version]: """Available versions as per python.org/ftp""" if PPG.config.get_value("cpython-use-github"): r = self.client.get("https://api.github.com/repos/python/cpython/git/matching-refs/tags/v3.", logger=logging.debug) - for item in r: + for item in r or (): ref = item.get("ref") if ref and ref.startswith("refs/tags/v"): ref = ref[11:] @@ -107,10 +104,10 @@ def get_available_versions(self): v = Version(m.group(1)) if v.is_valid and v.is_final and self.min_version < v: # For .0 releases, double-check that it's not a release candidate - if v < upcoming or v.patch > 0 or self.client.url_exists(f"{base_url}/{v}/Python-{v}.tar.xz"): + if v < upcoming or v.patch or self.client.url_exists(f"{base_url}/{v}/Python-{v}.tar.xz"): yield v - def get_builder(self): + def get_builder(self) -> type["PythonBuilder"]: from portable_python.cpython import Cpython return Cpython @@ -119,20 +116,20 @@ def get_builder(self): class Folders: def __init__(self, config: Config, base=None, family=None, version=None, abi_suffix=None): self.config = config - self.base_folder = runez.resolved_path(base) + self.base_folder = runez.resolved_path(base) # TODO: revise this so that attributes that are paths, are of type Path self.family = family self.version = Version.from_object(version) self.mm = self.version and self.version.mm self.completions = config.completions(family=family, version=version, mm=self.mm, abi_suffix=abi_suffix or "") - self.build_folder = self._get_path("build") - self.completions["build"] = self.build_folder + self.build_folder = self._required_path("build") + self.completions["build"] = str(self.build_folder) self.components = self.build_folder / "components" self.deps = self.build_folder / "deps" - self.destdir = self._get_path("destdir") - self.dist = self._get_path("dist", required=False) - self.logs = self._get_path("logs", required=False) - self.ppp_marker = self._get_value("ppp-marker") - self.sources = self._get_path("sources") + self.destdir = self._required_path("destdir") + self.dist = self._get_path("dist") + self.logs = self._get_path("logs") + self.ppp_marker = self._required_value("ppp-marker") + self.sources = self._required_path("sources") def __repr__(self): return runez.short(self.build_folder) @@ -150,24 +147,36 @@ def resolved_destdir(self, relative_path=None): return folder - def _get_value(self, key, required=True): + def _get_value(self, key) -> str | None: value = self.config.get_value("folders", key, by_platform=False) - if required and not value: - runez.abort("Folder '%s' must be configured" % key) - if value: value = os.path.expandvars(value) value = self.formatted(value) return value - def _get_path(self, key, required=True): - path = self._get_value(key, required=required) - if path and self.base_folder: - path = runez.resolved_path(path, base=self.base_folder) + def _required_value(self, key) -> str: + value = self._get_value(key) + if not value: + runez.abort("Folder '%s' must be configured" % key) + + return value + + def _get_path(self, key) -> Path | None: + value = self._get_value(key) + return self._as_path(value) if value else None + + def _required_path(self, key) -> Path: + return self._as_path(self._required_value(key)) + + def _as_path(self, value: str) -> Path: + if self.base_folder: + value = runez.resolved_path(value, base=self.base_folder) + + if " " in value: + runez.abort("Refusing path with space (escapes too complex with makefiles, shell scripts etc): '%s'" % runez.short(value)) - if path: - return runez.to_path(path, no_spaces=True) + return Path(value) class PPG: @@ -185,7 +194,7 @@ class PPG: """ cpython = CPythonFamily() - families: ClassVar[dict] = {"cpython": cpython} + families: ClassVar[dict[str, VersionFamily]] = {"cpython": cpython} config = Config() target = config.target @@ -202,10 +211,10 @@ def get_folders(cls, base=None, family="cpython", version=None, abi_suffix=None) return Folders(config, base=base, family=family, version=version, abi_suffix=abi_suffix) @classmethod - def family(cls, family_name, fatal=True) -> VersionFamily: + def family(cls, family_name) -> VersionFamily: fam = cls.families.get(family_name) - if fatal and not fam: - runez.abort(f"Python family '{family_name}' is not yet supported") + if not fam: + runez.abort(f"Python family '{runez.red(family_name)}' is not yet supported") return fam @@ -217,8 +226,8 @@ def find_python(cls, spec): return cls._depot.find_python(spec) @classmethod - def find_telltale(cls, *telltales): - for tt in runez.flattened(telltales): + def find_telltale(cls, telltale: str): + for tt in runez.flattened(telltale, split=" "): for sys_include in runez.flattened(cls.target.sys_include): path = tt.format(include=sys_include) if os.path.exists(path): diff --git a/tests/conftest.py b/tests/conftest.py index c366aee..c35f72a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,9 +10,3 @@ runez.log.setup(debug=True, console_format="%(levelname)s %(message)s", locations=None) ClickRunner.default_main = main - - -def dummy_tarball(folders, basename, content=None): - runez.write("sample/README", content, logger=None) - runez.compress("sample", folders.sources / basename, logger=None) - runez.delete("sample", logger=None) diff --git a/tests/test_build.py b/tests/test_build.py index bf9777c..d880c43 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -4,7 +4,11 @@ from portable_python.versions import PPG -from .conftest import dummy_tarball + +def dummy_tarball(folders, basename, content=None): + runez.write("sample/README", content, logger=None) + runez.compress("sample", folders.sources / basename, logger=None) + runez.delete("sample", logger=None) def test_build_rc(cli): diff --git a/tests/test_inspector.py b/tests/test_inspector.py index 4b8b6be..ca9ecbc 100644 --- a/tests/test_inspector.py +++ b/tests/test_inspector.py @@ -130,6 +130,7 @@ def __init__(self, platform, prefix): self.program_handler = getattr(self, "_%s_run" % platform) self.prefix = prefix self.called = [] + self.mock = patch("runez.run", side_effect=self) def _linux_run(self, *args): if args[1] == "--print-rpath": @@ -149,7 +150,6 @@ def __call__(self, *args, **_): return runez.program.RunResult(code=0) def __enter__(self): - self.mock = patch("runez.run", side_effect=self) self.mock.start() return self diff --git a/tests/test_list.py b/tests/test_list.py index cd7cb6a..59c8793 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -1,6 +1,6 @@ from runez.http import RestClient -from portable_python import BuildSetup +from portable_python import BuildSetup, required_python_spec from portable_python.versions import CPythonFamily, PPG REST_CLIENT = RestClient() @@ -30,7 +30,7 @@ def test_list(cli, monkeypatch): # Edge cases monkeypatch.setattr(PPG, "config", None) PPG.grab_config(target="macos-arm64") - setup = BuildSetup() + setup = BuildSetup(required_python_spec(None)) assert setup.python_spec.version == PPG.cpython.latest cp = CPythonFamily() @@ -53,3 +53,11 @@ def test_list(cli, monkeypatch): cli.run("-c", cli.tests_path("sample-config1.yml"), "list") assert cli.succeeded assert cli.logged.stdout.contents().strip() == "cpython:\n 3.13: 3.13.3\n 3.9: 3.9.7" + + +@REST_CLIENT.mock({"https://www.python.org/ftp/python/": ""}) +def test_no_versions(cli, monkeypatch): + monkeypatch.setattr(PPG.cpython, "_versions", None) # Simulate python.org not reachable (or listing nothing) + cli.run("-n build latest") + assert cli.failed + assert "Could not determine latest cpython version" in cli.logged diff --git a/tests/test_setup.py b/tests/test_setup.py index 42fcb5c..52c5c89 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -1,7 +1,7 @@ import pytest import runez -from portable_python import BuildSetup, ModuleBuilder +from portable_python import BuildSetup, ModuleBuilder, required_python_spec from portable_python.versions import PPG @@ -39,6 +39,13 @@ def test_diagnostics(cli): assert "pp-dev.yml:\na: b\n\nportable-python.yml:\ninclude: +pp-dev.yml" in cli.logged.stdout +def test_folder_with_space(cli): + runez.write("pp.yml", "folders:\n build: my build", logger=None) + cli.run("-n", "-c", "pp.yml", "build", "3.9.7") + assert cli.failed + assert "Refusing path with space (escapes too complex with makefiles, shell scripts etc): 'my build'" in cli.logged + + def test_edge_cases(temp_folder, monkeypatch, logged): monkeypatch.setattr(PPG, "config", None) PPG.grab_config(runez.DEV.tests_path("sample-incomplete.yml")) @@ -51,7 +58,7 @@ def test_edge_cases(temp_folder, monkeypatch, logged): assert str(PPG.cpython) == "cpython" assert str(PPG.config) == "1 config source [linux-x86_64]" - setup = BuildSetup("3.9.6") + setup = BuildSetup(required_python_spec("3.9.6")) assert str(setup) == "build/cpython-3.9.6" mb = ModuleBuilder(setup) @@ -63,14 +70,23 @@ def test_edge_cases(temp_folder, monkeypatch, logged): assert outcome.name == "failed" assert reason == "broken, can't compile statically with foo present" - sources = getattr(PPG.config, "_sources", None) - monkeypatch.setattr(sources[0], "data", {"ext": "foo"}) + runez.write("pp.yml", "ext: foo", logger=None) + PPG.grab_config("pp.yml", target="linux-x86_64") assert not logged with pytest.raises(runez.system.AbortException): - _ = BuildSetup("3.9.6") + _ = BuildSetup(required_python_spec("3.9.6")) assert "Invalid extension 'foo'" in logged.pop() +def test_missing_bin_python(temp_folder, logged): + PPG.grab_config(target="linux-x86_64") + setup = BuildSetup(required_python_spec("3.9.6")) + with pytest.raises(runez.system.AbortException): + _ = setup.python_builder.bin_python + assert "Can't find bin/python in /" in logged + assert "/build/ppp-marker/3.9.6/bin\n" in logged.pop() + + def test_inspect(cli): cli.run("-n", "inspect", "foo", "-m+sys") assert cli.failed diff --git a/tox.ini b/tox.ini index 651da7e..c0cd1ac 100644 --- a/tox.ini +++ b/tox.ini @@ -29,3 +29,37 @@ deps = ruff commands = ruff check --fix ruff format + +# Type checkers +# types-PyYAML: yaml has no inline type hints (mypy and zuban insist on stubs, the others use them when present) +[testenv:basedpyright] +deps = basedpyright + types-PyYAML +# Without --pythonpath, basedpyright resolves imports against ./.venv rather than this tox env +commands = basedpyright --pythonpath {envpython} {posargs} + +[testenv:mypy] +deps = mypy + types-PyYAML +commands = mypy {posargs} + +[testenv:pyrefly] +deps = pyrefly + types-PyYAML +commands = pyrefly check {posargs} + +[testenv:pyright] +deps = pyright + types-PyYAML +commands = pyright {posargs} + +[testenv:ty] +deps = ty + types-PyYAML +commands = ty check {posargs} + +[testenv:zuban] +deps = zuban + types-PyYAML +# Without --python-executable, zuban doesn't resolve imports against this tox env +commands = zuban check --python-executable {envpython} {posargs} From 636510887b71c7873abfa5ce4c678a28bd98f2a2 Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Fri, 9 Oct 2026 16:35:54 -0700 Subject: [PATCH 2/4] Added tests --- CHANGELOG.md | 2 + docs/configuration/portable-python-yml.md | 2 +- docs/log.md | 1 + src/portable_python/__init__.py | 58 +++++++++++++++-------- src/portable_python/cpython.py | 7 ++- src/portable_python/external/xcpython.py | 22 ++++----- src/portable_python/external/xtkinter.py | 4 +- src/portable_python/inspector.py | 10 ++-- tests/test_build.py | 26 ++++++++++ tests/test_cleanup.py | 26 ++++++++-- tests/test_inspector.py | 29 +++++++++++- tests/test_report.py | 10 ++++ tests/test_setup.py | 36 +++++++++++++- 13 files changed, 182 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e35ddb3..5d52133 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ - With `cpython-use-github: true`, a failed GitHub API call (e.g. rate limiting) crashed with a stack trace - A build where `bin/python` could not be found crashed (instead of reporting it) when `cpython-validate-script` or `cpython-additional-packages` was configured - Clearer error messages when the latest python version can't be determined, or a python family isn't supported + - A `{module}-configure` setting with several options was passed to `./configure` as one single argument + - Dry-run (`-n`) could modify files of an existing build (source patches, shebangs, permissions), a configured patch on a missing file is now reported - Library use: `BuildSetup()` now takes a `PythonSpec`, get one via `required_python_spec("cpython:3.13.2")` (which validates it, `None` means latest) - Tests cover more edge cases diff --git a/docs/configuration/portable-python-yml.md b/docs/configuration/portable-python-yml.md index a07dd99..d0e479e 100644 --- a/docs/configuration/portable-python-yml.md +++ b/docs/configuration/portable-python-yml.md @@ -51,7 +51,7 @@ Replace `{module}` with the lowercased module name (e.g. `openssl`): | `{module}-version` | Version to build. | | `{module}-url` | Source URL (`$version` is substituted; may carry a `#sha256=…` fragment, verified on download). | | `{module}-src-suffix` | Archive extension when the URL lacks one. | -| `{module}-configure` | **Replace** the default configure args (`$deps_lib` substituted). | +| `{module}-configure` | **Replace** the default configure args (space-separated, shell-style quoting; `$lib_dir` and `$lib64_dir` substituted). | | `{module}-http-headers` | Headers for the download (env vars expanded). | | `{module}-patches` | Source patches to apply before compiling. | | `{module}-debian` | Debian package name for dependency detection. | diff --git a/docs/log.md b/docs/log.md index 88b7b5b..9d588ea 100644 --- a/docs/log.md +++ b/docs/log.md @@ -2,6 +2,7 @@ ## 2026-10-09 +* **Update**: [Configuration](/configuration/portable-python-yml.md): `{module}-configure` substitutes `$lib_dir` and `$lib64_dir` (not `$deps_lib`), and its value is split into separate args. * **Creation**: Added the [type checking in v2.1.0](/typecheck-v2.1.0.md) report. * **Update**: The changelog moved out of the bundle, to `CHANGELOG.md` at the repo root (shipped in the sdist, like `README.rst`); [release process](/release.md) and the index point there. * **Update**: [CI/CD](/guides/ci-cd.md): the `linters` job and the release gate run the type checkers (ty, pyrefly, pyright, mypy, zuban). diff --git a/src/portable_python/__init__.py b/src/portable_python/__init__.py index e9bea43..7d328ec 100644 --- a/src/portable_python/__init__.py +++ b/src/portable_python/__init__.py @@ -15,6 +15,7 @@ import os import pathlib import re +import shlex from collections.abc import Iterable from string import Template @@ -33,7 +34,7 @@ def is_binary_file(path): return RX_BINARY.match(path.name) -def patch_folder(folder, regex, replacement, ignore=None): +def patch_folder(folder, regex, replacement, ignore=None, dryrun: runez.DryrunSpec = runez.UNSET): """Replace all occurrences of 'old_text' by 'new_text' in all files in 'folder' Parameters @@ -46,33 +47,53 @@ def patch_folder(folder, regex, replacement, ignore=None): Replacement text ignore : re.Pattern | None Regex stating what to ignore + dryrun + Optionally override current dryrun setting """ for path in runez.ls_dir(folder): if not path.is_symlink() and (not ignore or not ignore.match(path.name)): if path.is_dir(): - patch_folder(path, regex, replacement, ignore=ignore) + patch_folder(path, regex, replacement, ignore=ignore, dryrun=dryrun) elif not is_binary_file(path): - patch_file(path, regex, replacement) + patch_file(path, regex, replacement, dryrun=dryrun) -def patch_file(path, regex, replacement): +def patch_file(path, regex, replacement, dryrun: runez.DryrunSpec = runez.UNSET): + """Replace `regex` with `replacement` in file `path`, if it occurs there""" + if not os.path.exists(path): + if not runez.log.resolved_dryrun(dryrun): # Sources to patch are not downloaded in dryrun mode + LOG.warning("Can't patch %s: file not found", runez.short(path)) + + return + + overview = f"{regex} -> {runez.short(replacement)} in {runez.short(path)}" try: with open(path) as fh: text = fh.read() new_text = re.sub(regex, replacement, text, flags=re.MULTILINE) - if text != new_text: + if text != new_text and not runez.log.hdry(f"apply patch: {overview}", dryrun=dryrun): with open(path, "w") as fh: fh.write(new_text) - LOG.info("Patched '%s' in %s", regex, runez.short(path)) + LOG.info("Patched %s", overview) except Exception as e: with open(path, errors="ignore") as fh: text = fh.read() if re.search(regex, text): - LOG.warning("Can't patch '%s': %s", runez.short(path), e) + LOG.warning("Can't patch %s: %s", overview, e) + + +def correct_permissions(path: pathlib.Path, expected: int, dryrun: runez.DryrunSpec = runez.UNSET): + """Ensure that `path` has `expected` permissions""" + current = path.stat().st_mode & 0o777 + if current != expected: + overview = f"permissions for {runez.short(path)} to {oct(expected)} (was {oct(current)})" + if not runez.log.hdry(f"correct {overview}", dryrun=dryrun): + LOG.info("Corrected %s", overview) + path.chmod(expected) class FolderMask: @@ -163,7 +184,7 @@ def __enter__(self): self.masked_folders.append(mask) mask.mount() - except BaseException: # pragma: no cover, ensure cleanup if any folder couldn't be masked + except BaseException: # Ensure cleanup if any folder couldn't be masked self.cleanup() raise @@ -506,10 +527,13 @@ def cfg_url(self, version): def cfg_src_suffix(self): return PPG.config.get_value("%s-src-suffix" % self.m_name) - def cfg_configure(self, deps_lib_dir, deps_lib64_dir): + def cfg_configure(self, deps_lib_dir, deps_lib64_dir) -> list[str] | None: + """Configure args from config (`$lib_dir` and `$lib64_dir` substituted), shell-style split into separate args""" if configure := PPG.config.get_value("%s-configure" % self.m_name): - configure_template = Template(configure) - return configure_template.substitute(lib_dir=deps_lib_dir, lib64_dir=deps_lib64_dir) + configure = Template(configure).substitute(lib_dir=deps_lib_dir, lib64_dir=deps_lib64_dir) + return shlex.split(configure) + + return None def cfg_patches(self): return PPG.config.get_value("%s-patches" % self.m_name) @@ -721,11 +745,7 @@ def compile(self): def _apply_patches(self): if patches := self.cfg_patches(): for patch in patches: - if runez.DRYRUN: - print(f"Would apply patch: {patch}") - else: - print(f"Applying patch: {patch}") - patch_file(patch["file"], patch["regex"], patch["replacement"]) + patch_file(patch["file"], patch["regex"], patch["replacement"]) def _get_env_vars(self): """Yield all found env vars, first found wins""" @@ -826,8 +846,4 @@ def _prepare(self): for lib_dir in self.deps_lib_dirs: for path in runez.ls_dir(lib_dir): if not path.name.endswith(".la"): - expected = 0o755 if path.is_dir() else 0o644 - current = path.stat().st_mode & 0o777 - if current != expected: - LOG.info("Corrected permissions for %s (was %s)", runez.short(path), oct(current)) - path.chmod(expected) + correct_permissions(path, 0o755 if path.is_dir() else 0o644) diff --git a/src/portable_python/cpython.py b/src/portable_python/cpython.py index 5f96c61..fcc38aa 100644 --- a/src/portable_python/cpython.py +++ b/src/portable_python/cpython.py @@ -404,11 +404,10 @@ def _relativize_shebang_file(self, path): except UnicodeError: return - if lines: - LOG.info("Auto-corrected shebang for %s" % runez.short(path)) + if lines and not runez.log.hdry(f"auto-correct shebang for {runez.short(path)}"): + LOG.info("Auto-corrected shebang for %s", runez.short(path)) with open(path, "w") as fh: - for line in lines: - fh.write(line) + fh.writelines(lines) class RelSysConf: diff --git a/src/portable_python/external/xcpython.py b/src/portable_python/external/xcpython.py index 572c74d..758144b 100644 --- a/src/portable_python/external/xcpython.py +++ b/src/portable_python/external/xcpython.py @@ -31,7 +31,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib_dir, self.deps_lib64_dir): - yield config_args + yield from config_args else: yield "--enable-shared=no" @@ -93,7 +93,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib_dir, self.deps_lib64_dir): - yield config_args + yield from config_args else: yield "--enable-shared=no" @@ -142,7 +142,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib_dir, self.deps_lib64_dir): - yield config_args + yield from config_args else: yield "--enable-shared=no" @@ -191,7 +191,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib_dir, self.deps_lib64_dir): - yield config_args + yield from config_args else: yield "-v" @@ -221,7 +221,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib_dir, self.deps_lib64_dir): - yield config_args + yield from config_args else: yield "--disable-shared" @@ -281,7 +281,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib_dir, self.deps_lib64_dir): - yield config_args + yield from config_args else: yield "--enable-shared=no" @@ -327,7 +327,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib_dir, self.deps_lib64_dir): - yield config_args + yield from config_args else: yield "--disable-shared" @@ -364,7 +364,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib_dir, self.deps_lib64_dir): - yield config_args + yield from config_args else: yield "--enable-shared=no" @@ -395,7 +395,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib_dir, self.deps_lib64_dir): - yield config_args + yield from config_args else: yield "--enable-shared=no" @@ -439,7 +439,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib_dir, self.deps_lib64_dir): - yield config_args + yield from config_args else: yield "--static" @@ -515,7 +515,7 @@ def version(self): def c_configure_args(self): if config_args := self.cfg_configure(self.deps_lib_dir, self.deps_lib64_dir): - yield config_args + yield from config_args else: pass # yield "--static" diff --git a/src/portable_python/external/xtkinter.py b/src/portable_python/external/xtkinter.py index 660ca40..abe0dd4 100644 --- a/src/portable_python/external/xtkinter.py +++ b/src/portable_python/external/xtkinter.py @@ -25,7 +25,7 @@ def url(self): def _prepare(self): for path in runez.ls_dir(self.m_src_build / "pkgs"): - if path.name.startswith(("sqlite", "tdbc")): # pragma: no cover, tcl/tk is "best effort" + if path.name.startswith(("sqlite", "tdbc")): # Remove packages we don't care about and can pull in unwanted symbols runez.delete(path) @@ -53,7 +53,7 @@ def c_configure_args(self): yield "--enable-threads" yield f"--with-tcl={self.deps_lib_dir}" yield "--without-x" - if PPG.target.is_macos: # pragma: no cover, tcl/tk is "best effort" + if PPG.target.is_macos: yield "--enable-aqua=yes" def _do_linux_compile(self): diff --git a/src/portable_python/inspector.py b/src/portable_python/inspector.py index fc3cb56..9b941a1 100644 --- a/src/portable_python/inspector.py +++ b/src/portable_python/inspector.py @@ -246,7 +246,7 @@ def __init__(self, inspector: "PythonInspector", path): self.is_failed = "_failed" in self.path.name self.short_name = "%s*" % self.path.name.partition(".")[0] if self.is_failed: - self.short_name += "_failed" # pragma: no cover + self.short_name += "_failed" if program and output: func = getattr(self, "parse_%s" % program) @@ -467,7 +467,7 @@ def represented(self, verbose=False): report.append(str(table)) if verbose: report.append(str(self.full_so_report)) - if self.full_so_report.problematic: # pragma: no cover (don't have a handy problematic python test case) + if self.full_so_report.problematic: report.append(self.full_so_report.problematic.report(verbose=verbose)) report.append("\n-- Library users:") for what, users in self.full_so_report.lib_tracker.users.items(): @@ -536,12 +536,12 @@ def __init__(self, inspector: PythonInspector): self.libpython_so = [] self.lib_static = [] for path in find_libs(self.inspector.lib_folder): - if path.name.endswith(".a"): # pragma: no cover + if path.name.endswith(".a"): self.lib_static.append(self.inspector.relative_path(path)) continue info = SoInfo(inspector, path) - if path.name.startswith("libpython"): # pragma: no cover (would need to do a full build with libpython.so...) + if path.name.startswith("libpython"): self.libpython_so.append(info) self.lib_tracker.add(info) @@ -564,7 +564,7 @@ def get_problem(self, portable) -> str | None: if uses_system: return "Uses system libs: %s" % runez.red(runez.joined(uses_system)) - if self.problematic: # pragma: no cover (don't have a handy problematic python test case) + if self.problematic: return self.problematic.report() if not runez.DRYRUN and not self.ok: diff --git a/tests/test_build.py b/tests/test_build.py index d880c43..f0aa0b4 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -19,6 +19,19 @@ def test_build_rc(cli): assert f"Would tar {install_dir} -> dist/cpython-{f.version}-macos-arm64.tar.gz" in cli.logged +def test_build_free_threaded(cli): + # Recent python on macos: zstd is required, libs that ./configure can't find via pkg-config there are spelled out + cli.run("-n", "-tmacos-arm64", "build", "3.14.4t", "-mreadline,tkinter") + assert cli.succeeded + assert " --enable-aqua=yes" in cli.logged # tk on macos + assert "zstd 1.5.7 static* [auto-selected] Required for 3.14 and up" in cli.logged + assert " --disable-gil" in cli.logged + assert "env LIBMPDEC_LIBS=build/deps/lib/libmpdec.a" in cli.logged + assert "env LIBREADLINE_LIBS=-Lbuild/deps/lib -lreadline -lncursesw" in cli.logged + assert "env LIBZSTD_LIBS=build/deps/lib/libzstd.a" in cli.logged + assert "Would tar build/ppp-marker/3.14.4t -> dist/cpython-3.14.4t-macos-arm64.tar.gz" in cli.logged + + SAMPLE_SYS_CONF = """ # sys config build_time_vars = {'a': '', @@ -73,10 +86,20 @@ def test_finalization(cli, monkeypatch): runez.touch(lib / "site-packages/pip", logger=None) _setup_exes(bin_folder) runez.touch(lib / "config-3.9/libpython3.9.a", logger=None) + pkgconfig = f.resolved_destdir("lib/pkgconfig") + runez.write(pkgconfig / "python3.pc", f"prefix=/ppp-marker/{f.version}\n", logger=None) monkeypatch.setenv("PP_X_DEBUG", "direct-finalize") monkeypatch.setenv("SOME_ENV", "some-env-value") with patch("runez.run", return_value=runez.program.RunResult(code=0)): + # Dryrun on an existing build: reports what finalization would change, without touching anything + cli.run("-n", "-tlinux-x86_64", "-c", cli.tests_path("sample-config1.yml"), "build", f.version, "-mbzip2") + assert cli.succeeded + assert f"Would auto-correct shebang for {bin_folder}/some-exe" in cli.logged + assert f"Would apply patch: prefix=/ppp-marker/{f.version} -> prefix=${{pcfiledir}}/../.. in {pkgconfig}/python3.pc" in cli.logged + assert list(runez.readlines(bin_folder / "some-exe")) == ["#!.../bin/python3", "hello"] + assert list(runez.readlines(pkgconfig / "python3.pc")) == [f"prefix=/ppp-marker/{f.version}"] + cli.run("-tlinux-x86_64", "-c", cli.tests_path("sample-config1.yml"), "build", f.version, "-mbzip2") assert cli.succeeded manifest = list(runez.readlines(f"build/ppp-marker/{f.version}/.manifest.yml")) @@ -88,6 +111,9 @@ def test_finalization(cli, monkeypatch): assert f"Symlink {bin_folder}/pip{f.mm} <- {bin_folder}/pip" in cli.logged assert f"Auto-corrected shebang for {bin_folder}/some-exe" in cli.logged assert f"Symlink {lib}/config-3.9/libpython3.9.a <- {lib.parent}/libpython3.9.a" in cli.logged + assert f"Patched prefix=/ppp-marker/{f.version} -> prefix=${{pcfiledir}}/../.. in {pkgconfig}/python3.pc" in cli.logged + + assert list(runez.readlines(pkgconfig / "python3.pc")) == ["prefix=${pcfiledir}/../.."] transformed = "\n".join(runez.readlines(sys_cfg)) assert transformed.strip() == SAMPLE_SYS_CONF_REL.strip() diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index 52e35d4..b4decf8 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -1,4 +1,5 @@ import os +from unittest.mock import patch import runez @@ -16,10 +17,14 @@ def test_cleanup(cli, monkeypatch): runez.write(f.components / "cpython/Lib/trace.py", "hmm\nmentions /usr/local", logger=None) # ignored because in Lib/ folder runez.write(f.components / "cpython/setup.py", "path.startswith('/usr/') and not path.startswith('/usr/local')", logger=None) runez.write(f.components / "cpython/foo", b"hello\xe4 /usr/local", logger=None) + runez.touch(f.components / "tcl/pkgs/sqlite3.40.0/README", logger=None) # Bundled tcl package we don't want + runez.touch(f.components / "tcl/pkgs/itcl4.2.3/README", logger=None) runez.touch(f.deps / "bin/bzcat", logger=None) runez.touch(f.deps / "include/readline/readline.h", logger=None) runez.touch(f.deps / "lib/libssl.a", logger=None) os.chmod(f.deps / "lib/libssl.a", 0o755) + runez.touch(f.deps / "lib64/libcrypto.a", logger=None) # Some libs install in lib64/ + os.chmod(f.deps / "lib64/libcrypto.a", 0o755) runez.touch(install_dir / "bin/python", logger=None) runez.touch(install_dir / "bin/easy_install", logger=None) runez.touch(install_dir / f"bin/pip{f.mm}", logger=None) @@ -32,14 +37,15 @@ def test_cleanup(cli, monkeypatch): cfg = cli.tests_path("sample-config1.yml") cli.run("-ntmacos-x86_64", f"-c{cfg}", "build", "-mopenssl,readline", f.version) assert cli.succeeded - assert "Patched '/usr/local\\b' in build/components/cpython/Mac/Makefile.in" in cli.logged + assert "Would apply patch: /usr/local\\b -> build/deps in build/components/cpython/Mac/Makefile.in" in cli.logged assert "MACOSX_DEPLOYMENT_TARGET=10.25" in cli.logged assert "Pass 1: Cleaned" in cli.logged assert "Pass 2: Cleaned" in cli.logged assert "PEP 668: Cleaned" in cli.logged - assert f"Corrected permissions for {f.deps}/lib/libssl.a" in cli.logged + assert f"Would correct permissions for {f.deps}/lib/libssl.a to 0o644" in cli.logged + assert f"Would correct permissions for {f.deps}/lib64/libcrypto.a to 0o644" in cli.logged + assert (f.deps / "lib/libssl.a").stat().st_mode & 0o777 == 0o755 # Untouched in dryrun mode assert f" install DESTDIR={f.build_folder}\n" in cli.logged - cli.match("Patched 'startswith(...)' in build/components/cpython/setup.py") assert "Exercising configured validation script" in cli.logged assert "Lib/trace.py" not in cli.logged @@ -47,6 +53,8 @@ def test_cleanup(cli, monkeypatch): assert cli.succeeded assert "MACOSX_DEPLOYMENT_TARGET" not in cli.logged assert "selected: all" in cli.logged + assert "Would delete build/components/tcl/pkgs/sqlite3.40.0" in cli.logged + assert "tcl/pkgs/itcl" not in cli.logged assert f"Would symlink {install_dir}/bin/pip{f.mm} <- {install_dir}/bin/pip" in cli.logged assert f"Would symlink {lib}/python{f.mm}/config-{f.mm}-darwin/libpython{f.mm}.a <- {lib}/libpython{f.mm}.a" in cli.logged assert f"Would tar {install_dir} -> dist/cpython-{f.version}-linux-x86_64.tar.gz" in cli.logged @@ -56,6 +64,14 @@ def test_cleanup(cli, monkeypatch): monkeypatch.setattr(BuildContext, "usr_local", os.getcwd()) cli.run("-ntmacos-arm64", f"-c{cfg}", "build", f.version) assert cli.succeeded - assert "Patched '/usr/local\\b' in build/components/cpython/setup.py" in cli.logged - assert "Can't patch 'build/components/cpython/foo'" in cli.logged + assert "Would apply patch: /usr/local\\b -> build/deps in build/components/cpython/setup.py" in cli.logged + assert "Can't patch /usr/local\\b -> build/deps in build/components/cpython/foo: " in cli.logged assert "isolate-usr-local: mount-shadow" in cli.logged + + # Folders that were already masked get cleaned up if masking fails mid-way + with patch("portable_python.FolderMask.mount", side_effect=[None, Exception("mount failed")]): + cli.run("-ntmacos-arm64", f"-c{cfg}", "build", f.version) + assert cli.failed + assert "mount failed" in cli.logged + assert f"Cleaning up isolation hack/mask for {os.getcwd()}/etc" in cli.logged + assert f"Cleaning up isolation hack/mask for {os.getcwd()}/include" in cli.logged diff --git a/tests/test_inspector.py b/tests/test_inspector.py index ca9ecbc..6c94094 100644 --- a/tests/test_inspector.py +++ b/tests/test_inspector.py @@ -1,6 +1,7 @@ import builtins import os import sys +from pathlib import Path from unittest.mock import patch import runez @@ -67,6 +68,31 @@ def test_inspect_python(temp_folder, monkeypatch): """ +def test_inspect_problematic(temp_folder): + # Simulate an installation where libs have missing dependencies (`ldd` reports some as "not found") + PPG.grab_config(target="linux-x86_64") + inspector = PythonInspector("invoker") + inspector.lib_folder = Path("lib") + for name in ("_dbm.so", "_tkinter.cpython-39-x86_64-linux-gnu_failed.so", "libpython3.9.so", "libpython3.9.a"): + runez.touch(inspector.lib_folder / name, logger=None) + + with patch("runez.which", side_effect=lambda program: program == "ldd"): + with patch("runez.run", return_value=runez.program.RunResult(code=0, output=LDD_SAMPLE)): + report = inspector.represented(verbose=True) + + assert "- lib/libpython3.9.a" in report + assert "- lib/libpython3.9.so" in report + assert "-- 3 problematic\n" in report + assert "lib/_tkinter.cpython-39-x86_64-linux-gnu_failed.so\n" in report + assert " [missing] libtinfo.so.5 5\n" in report + assert "\n-- Library users:\n" in report + assert "\ntinfo 3 users: " in report + assert "_tkinter*_failed.so" in report # Failed modules are marked as such (cpython renames them `*_failed.so`) + + problem = inspector.full_so_report.get_problem(portable=False) + assert problem.startswith("\n-- 3 problematic\n") + + def test_inspect_lib(logged): inspector = PythonInspector("invoker") with patch("runez.which", return_value="yup"): @@ -76,7 +102,8 @@ def test_inspect_lib(logged): assert not_there.is_problematic assert not_there.size == 0 assert "otool exited with code" in logged.pop() - assert inspector.libpython_report([not_there]) # Verify no crash, edge case testing + assert inspector.libpython_report([not_there]) == "- /dev/null/foo.platform.so" # Verify no crash, edge case testing + assert inspector.libpython_report([]) == "-not used-" with patch("runez.which", return_value=None): PPG.grab_config(target="macos-x86_64") diff --git a/tests/test_report.py b/tests/test_report.py index 8a293a0..f2720b3 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -1,5 +1,7 @@ from unittest.mock import patch +from portable_python.versions import PPG + def test_scan(cli): cli.run("-tmacos-x86_64", "build-report", "-mnone", "3.9.7") @@ -10,3 +12,11 @@ def test_scan(cli): assert cli.failed assert "needs tclsh" in cli.logged assert "Problematic modules:" in cli.logged + + with patch.object(PPG, "find_telltale", return_value=None): # Simulate a host without any system headers + cli.run("-tlinux-x86_64", "build-report", "3.16.0") + assert "zstd 1.5.7 static* broken, can't compile without libzstd-dev" in cli.logged # Auto-selected ('*') + assert "mpdec 4.0.1 static* broken, can't compile without libmpdec-dev" in cli.logged + + cli.run("-tmacos-arm64", "build-report", "3.16.0") + assert "mpdec 4.0.1 static* [auto-selected] Required for 3.16 and up" in cli.logged diff --git a/tests/test_setup.py b/tests/test_setup.py index 52c5c89..2005189 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -1,7 +1,9 @@ +from pathlib import Path + import pytest import runez -from portable_python import BuildSetup, ModuleBuilder, required_python_spec +from portable_python import BuildSetup, correct_permissions, ModuleBuilder, patch_file, required_python_spec from portable_python.versions import PPG @@ -31,6 +33,28 @@ def test_config(cli): assert " -> dist/cpython-3.9.7-linux-x86_64.tar.gz" in cli.logged # Default .tar.gz +def test_module_overrides(cli, monkeypatch): + monkeypatch.setenv("HTTP_PROXY", "http://my-proxy:8080") # Exercise proxy pass-through on downloads + monkeypatch.setenv("HTTPS_PROXY", "http://my-proxy:8443") + modules = "bdb gdbm libffi mpdec ncurses openssl readline sqlite uuid xz zlib".split() + config = [f"{m}-configure: --my-{m}-opt --my-{m}-libs=$lib_dir:$lib64_dir" for m in modules] + config.append("cpython-url: https://my-mirror/python-$version.tar.xz") + config.append("openssl-version: 1.1.1w") # Older versions have a different download url scheme + config.append("openssl-patches:\n - file: README\n regex: foo\n replacement: bar") + config.append(" - file: no-such-file\n regex: foo\n replacement: bar") + runez.write("pp.yml", "\n".join(config), logger=None) + runez.write("README", "Hello foo", logger=None) # Dryrun doesn't extract sources, so the build stays in this folder + cli.run("-n", "-tlinux-x86_64", "-c", "pp.yml", "build", "3.14.4", "-mall") + assert cli.succeeded + for m in modules: + assert f" --my-{m}-opt --my-{m}-libs=build/deps/lib:build/deps/lib64" in cli.logged # Separate args (not quoted together) + + assert "Would apply patch: foo -> bar in README" in cli.logged + assert "no-such-file" not in cli.logged # Not reported in dryrun mode (sources to patch are not downloaded) + assert "Would download https://my-mirror/python-3.14.4.tar.xz" in cli.logged + assert "Would download https://github.com/openssl/openssl/releases/download/OpenSSL_1_1_1w/openssl-1.1.1w.tar.gz" in cli.logged + + def test_diagnostics(cli): runez.write("pp-dev.yml", "a: b", logger=None) runez.write("portable-python.yml", "include: +pp-dev.yml", logger=None) @@ -70,6 +94,16 @@ def test_edge_cases(temp_folder, monkeypatch, logged): assert outcome.name == "failed" assert reason == "broken, can't compile statically with foo present" + patch_file("no-such-file", "foo", "bar") # Not in dryrun mode: a (configured) patch on a missing file is reported + assert "Can't patch no-such-file: file not found" in logged.pop() + + lib = Path("libfoo.a") + runez.touch(lib, logger=None) + lib.chmod(0o755) + correct_permissions(lib, 0o644) # Not in dryrun mode: permissions are effectively corrected + assert "Corrected permissions for libfoo.a to 0o644 (was 0o755)" in logged.pop() + assert lib.stat().st_mode & 0o777 == 0o644 + runez.write("pp.yml", "ext: foo", logger=None) PPG.grab_config("pp.yml", target="linux-x86_64") assert not logged From 3c088a7f670df90a6939fa169fcce197098a004f Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Sat, 10 Oct 2026 14:51:54 -0700 Subject: [PATCH 3/4] Renamed type checking report to docs/typecheck-findings.md, neutral wording on checkers Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/{typecheck-v2.1.0.md => typecheck-findings.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{typecheck-v2.1.0.md => typecheck-findings.md} (100%) diff --git a/docs/typecheck-v2.1.0.md b/docs/typecheck-findings.md similarity index 100% rename from docs/typecheck-v2.1.0.md rename to docs/typecheck-findings.md From 4a43898598871edd3db2b41c2ff2ab2d30e24fa4 Mon Sep 17 00:00:00 2001 From: Zoran Simic Date: Sat, 10 Oct 2026 14:52:14 -0700 Subject: [PATCH 4/4] Updated links to docs/typecheck-findings.md, neutral wording on checkers Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 +- docs/guides/ci-cd.md | 4 ++-- docs/guides/local-development.md | 10 +++++----- docs/index.md | 2 +- docs/log.md | 6 +++--- docs/typecheck-findings.md | 24 ++++++++++++------------ 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d52133..8ea5f04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## v2.1.0 -- portable-python's code is now checked by 5 type checkers (ty, pyrefly, pyright, mypy and zuban) on every change +- portable-python's code is now checked by multiple type checkers on every change - Bumped SQLite to 3.53.4 and gettext-tiny to 0.3.3; re-checked every other component against upstream (all already current, or deliberately held back — see the `version` property comments). - Bug fixes: - With `cpython-use-github: true`, a failed GitHub API call (e.g. rate limiting) crashed with a stack trace diff --git a/docs/guides/ci-cd.md b/docs/guides/ci-cd.md index b239809..0c269c1 100644 --- a/docs/guides/ci-cd.md +++ b/docs/guides/ci-cd.md @@ -18,13 +18,13 @@ Runs on every push to `main` and every pull request targeting `main`, with `canc |-----|--------------| | `test` | Matrix over Python `3.10`–`3.14` on `ubuntu-latest` (`fail-fast: false`). Runs `uvx --with tox-uv tox -e py`, then uploads coverage to Coveralls as a **parallel** build, flagged per version. | | `coveralls-finish` | After all `test` jobs finish, signals Coveralls that the parallel build is complete. | -| `linters` | On Python `3.14`, runs `uvx --with tox-uv tox -e style,ty,pyrefly,pyright,mypy,zuban` (ruff check + format diff, then the type checkers, see [type checking](/typecheck-v2.1.0.md)). | +| `linters` | On Python `3.14`, runs `uvx --with tox-uv tox -e style,` (ruff check + format diff, then the type checkers, see [type checker findings](/typecheck-findings.md)). | ## Release (`release.yml`) Triggers on pushing a tag matching `v[0-9]*`. The single `publish` job (Python `3.14`, GitHub environment `release`): -1. `uvx --with tox-uv tox -e py,style,ty,pyrefly,pyright,mypy,zuban` — full test + lint + type check gate. +1. `uvx --with tox-uv tox -e py,style,` — full test + lint + type check gate. 2. `uv build` — build the sdist + wheel. 3. `pypa/gh-action-pypi-publish` — publish to PyPI via **trusted publishing** (OIDC, `id-token: write` — no API token is stored). diff --git a/docs/guides/local-development.md b/docs/guides/local-development.md index 017f337..dd9fba7 100644 --- a/docs/guides/local-development.md +++ b/docs/guides/local-development.md @@ -21,11 +21,11 @@ uv sync The suite mocks `runez.run()` and builds in `--dryrun` mode, so it asserts the *planned* commands without actually compiling. Target is 100% coverage. ```shell -tox # all envs (py310-314, coverage, style) -tox -e py313 # one python version -tox -e style # lint check (ruff) -tox -e reformat # auto-fix formatting -tox -e ty # a type checker (also: pyrefly, pyright, mypy, basedpyright, zuban) +tox # all envs (py310-314, coverage, style) +tox -e py313 # one python version +tox -e style # lint check (ruff) +tox -e reformat # auto-fix formatting +tox -e # one tox env per type checker, see tox.ini .venv/bin/pytest tests/ # directly .venv/bin/pytest tests/test_build.py::test_build_rc -vv # a single test diff --git a/docs/index.md b/docs/index.md index dacce79..ecc5b3b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,7 +19,7 @@ Start with the [Overview](/overview.md), then drill into the area you need. * [Configuration](/configuration/index.md) - The `portable-python.yml` file, its sections and overrides. * [Guides](/guides/index.md) - Playbooks for common tasks: building, extending, local development. * [Release process](/release.md) - How to cut a release: changelog highlights, then a version tag. -* [Type checking in v2.1.0](/typecheck-v2.1.0.md) - What introducing the type checkers found and fixed. +* [Type checker findings](/typecheck-findings.md) - What introducing the type checkers found and fixed. ## Conventions diff --git a/docs/log.md b/docs/log.md index 9d588ea..c76aaa1 100644 --- a/docs/log.md +++ b/docs/log.md @@ -3,14 +3,14 @@ ## 2026-10-09 * **Update**: [Configuration](/configuration/portable-python-yml.md): `{module}-configure` substitutes `$lib_dir` and `$lib64_dir` (not `$deps_lib`), and its value is split into separate args. -* **Creation**: Added the [type checking in v2.1.0](/typecheck-v2.1.0.md) report. +* **Creation**: Added the [type checker findings](/typecheck-findings.md) report. * **Update**: The changelog moved out of the bundle, to `CHANGELOG.md` at the repo root (shipped in the sdist, like `README.rst`); [release process](/release.md) and the index point there. -* **Update**: [CI/CD](/guides/ci-cd.md): the `linters` job and the release gate run the type checkers (ty, pyrefly, pyright, mypy, zuban). +* **Update**: [CI/CD](/guides/ci-cd.md): the `linters` job and the release gate run the type checkers. * **Update**: `BuildSetup` takes a `PythonSpec` (via `required_python_spec()`) in [BuildSetup](/architecture/build-setup.md), and `m_telltale` is a space-separated `str` in [telltale detection](/concepts/telltale-detection.md). ## 2026-10-08 -* **Update**: Listed the type checker tox envs (`tox -e ty`, ...) in [local development](/guides/local-development.md). +* **Update**: Listed the type checker tox envs (`tox -e `) in [local development](/guides/local-development.md). ## 2026-08-20 diff --git a/docs/typecheck-findings.md b/docs/typecheck-findings.md index 29dff18..35ace7a 100644 --- a/docs/typecheck-findings.md +++ b/docs/typecheck-findings.md @@ -1,16 +1,16 @@ --- type: Report -title: Type checking in v2.1.0 -description: What introducing 6 type checkers found in v2.0.2, and what v2.1.0 changed in response (bugs, design, minor). +title: Type checker findings +description: What introducing multiple type checkers found in v2.0.2, and what v2.1 changed in response (bugs, design, minor). tags: [report, type-checking, ci, quality] timestamp: 2026-10-09T00:00:00Z --- -# Type checking: v2.0.2 → v2.1.0 +# Type checking: v2.0.2 → v2.1 -What v2.1.0 changed in response to type checker findings on v2.0.2. +What v2.1 changed in response to type checker findings on v2.0.2. -| Checker | Version | v2.0.2 | v2.1.0 | Actionable | Bugs | Design | Minor | +| Checker | Version | v2.0.2 | v2.1 | Actionable | Bugs | Design | Minor | | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | | ty | 0.0.85 | 51 | 0 | 51 | 0 | 5 | 6 | | pyrefly | 1.3.2 | 96 | 0 | 96 | 1 | 6 | 7 | @@ -19,19 +19,19 @@ What v2.1.0 changed in response to type checker findings on v2.0.2. | basedpyright | 1.40.2 | 2,979 | 2,576 | 646 | 1 | 6 | 8 | | zuban | 0.10.0 | 38 | 0 | 38 | 2 | 4 | 5 | -- **v2.0.2**, **v2.1.0**: findings on that release's `src/` and `tests/`, both checked with v2.1.0's +- **v2.0.2**, **v2.1**: findings on that release's `src/` and `tests/`, both checked with v2.1's settings (`tox.ini`, `pyproject.toml`). "v2.0.2" is commit `c298694`: v2.0.2 plus the component bumps that preceded this work. -- **Actionable**: v2.0.2 findings resolved by a change in v2.1.0. +- **Actionable**: v2.0.2 findings resolved by a change in v2.1. - **Bugs**, **Design**, **Minor**: how many of the changes listed under [Bug fixes](#bug-fixes), [Design](#design) and [Minor](#minor) the checker spotted. - basedpyright counts include its warnings: 110 errors + 2,869 warnings on v2.0.2, 5 + 2,571 on - v2.1.0. Its 5 remaining errors are import cycles (`__init__` ↔ `versions` ↔ `cpython` ↔ - `inspector`). basedpyright is available locally (`tox -e basedpyright`) but doesn't gate CI, the - other 5 checkers do (see [CI/CD](/guides/ci-cd.md)). -- v2.1.0's zeros need runez 5.10.1, which fixed an annotation portable-python tripped on (see + v2.1. Its 5 remaining errors are import cycles (`__init__` ↔ `versions` ↔ `cpython` ↔ + `inspector`). basedpyright is available locally (`tox -e basedpyright`), see [CI/CD](/guides/ci-cd.md) + for which checkers gate CI. +- v2.1's zeros need runez 5.10.1, which fixed an annotation portable-python tripped on (see [Fixed in runez](#fixed-in-runez)). -- v2.0.2 had 3 `# type:` comments, v2.1.0 has none. +- v2.0.2 had 3 `# type:` comments, v2.1 has none. ## Bug fixes