diff --git a/.github/AGENTS.md b/.github/AGENTS.md index f0d319bc..6777c6be 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -5,6 +5,7 @@ CI/CD workflows and repo automation. ## Workflows (source of truth) - `conda-package.yml` — Intel channel conda build/test pipeline - `conda-package-cf.yml` — conda-forge-oriented build/test pipeline +- `intel-ci.yml` — Intel Python CI: builds and tests `conda-recipe-cf` for the platforms and Python versions the recipe declares (`ci/matrix.py`); see `ci/README.md` - `build-with-clang.yml` — clang compatibility checks - `build_pip.yml` — editable pip build pipeline - `build-with-standard-clang.yml` — standard clang compatibility checks @@ -12,5 +13,5 @@ CI/CD workflows and repo automation. - `openssf-scorecard.yml` — security scanning ## Policy -- Treat workflow YAML as canonical for platform/Python matrices. +- Treat workflow YAML as canonical for platform/Python matrices, except `intel-ci.yml`, whose matrix is `conda-recipe-cf/conda_build_config.yaml`. - Avoid doc claims about CI coverage unless present in workflow config. diff --git a/.github/workflows/intel-ci.yml b/.github/workflows/intel-ci.yml new file mode 100644 index 00000000..37a12f35 --- /dev/null +++ b/.github/workflows/intel-ci.yml @@ -0,0 +1,135 @@ +# Intel Python CI (gh-ci) for mkl_fft: builds the conda-forge recipe, conda-recipe-cf, for every +# platform and Python version it declares, as ci/matrix.py reads them, and tests each package in +# a clean conda-forge environment with ci/test.py, exactly as Jenkins does. A pull request builds +# only the two newest regular Python versions. Intel's Jenkins waits for this whole run, then +# tests and scans each package on its own machines; see ci/README.md. Steps run their commands as +# Python, so they are the same on Linux and Windows. +name: Intel Python CI + +on: + push: + branches: [master] + tags: ['*'] + pull_request: + +permissions: + contents: read + +concurrency: + group: intel-ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + matrix: + name: Platforms and Python versions + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 10 + outputs: + matrix: ${{ steps.declared.outputs.matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Read the declared platforms and Python versions + id: declared + run: python ci/matrix.py --event "$GITHUB_EVENT_NAME" --github-output matrix + + build: + name: ${{ matrix.name }} Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The recipe takes its version from git describe. + fetch-depth: 0 + + # Miniforge 26.9.4-0 fails setup-miniconda's conda init on Windows runners; 26.7.2-0 works. + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + # ci/build.py runs with the conda base install's Python, from $CONDA, and conda-build runs + # as a module of it. + - name: Build and run the recipe tests + shell: python + env: + PYTHON_VERSION: ${{ matrix.python }} + run: | + import os, subprocess, sys + base = os.environ['CONDA'] + python = os.path.join(base, 'python.exe' if os.name == 'nt' else os.path.join('bin', 'python')) + subprocess.run([python, '-m', 'conda', 'install', '-y', '-q', '-n', 'base', + '--override-channels', '-c', 'conda-forge', 'conda-build'], check=True) + output = os.path.join(os.environ['RUNNER_TEMP'], 'output') + sys.exit(subprocess.run([python, 'ci/build.py', '--python', os.environ['PYTHON_VERSION'], + '--output', output]).returncode) + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: mkl_fft ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.conda + if-no-files-found: error + # Jenkins downloads them within minutes of the run, so pull requests and branches keep + # them 3 days; tags keep 30. + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }} + + test: + name: Test ${{ matrix.name }} Python ${{ matrix.python }} + needs: [matrix, build] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: mkl_fft ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + # ci/hooks.py runs itself again with the conda base install's Python, from $CONDA. + - name: Install the package and run ci/test.py + shell: python + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + import os, subprocess, sys + env = os.environ + temp = env['RUNNER_TEMP'] + sys.exit(subprocess.run([sys.executable, 'ci/hooks.py', '--python', env['PYTHON_VERSION'], + '--artifacts', os.path.join(temp, 'artifacts'), + '--work', os.path.join(temp, 'hooks'), + '--event', env['EVENT'], '--commit', env['COMMIT']]).returncode) diff --git a/AGENTS.md b/AGENTS.md index 2b9bc5e4..dda4883b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,7 @@ It provides accelerated FFT transforms while aiming to preserve upstream API beh - **Tests:** `mkl_fft/tests/` - **Vendored helpers:** `_vendored/` - **Packaging:** `conda-recipe/`, `conda-recipe-cf/` +- **Intel Python CI:** `ci/`, `.github/workflows/intel-ci.yml`; see `ci/README.md` - **Benchmarks:** `benchmarks/` ## Build/runtime basics diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 00000000..d362b0ea --- /dev/null +++ b/ci/README.md @@ -0,0 +1,28 @@ +# How Intel Python CI tests mkl_fft + +Intel Python CI (gh-ci) builds and tests mkl_fft's conda-forge packages in GitHub Actions. Intel's Jenkins then tests each package again on Intel machines and scans it, and on a release tag publishes the packages it tested. Everything it needs from this repository is in this directory, one workflow, and the recipe it builds. + +| File | Role | +|---|---| +| `../conda-recipe-cf/` | The recipe gh-ci builds. `conda_build_config.yaml` lists the Python versions. The recipe builds on Linux and Windows unless a platform-only skip in `meta.yaml`, such as `skip: true # [win]`, rules one out. | +| `../.github/workflows/intel-ci.yml` | One job reads the matrix, one job per platform and Python version builds and uploads `mkl_fft Python `, and one job per package tests it. | +| `matrix.py` | Reads the platforms and Python versions from the recipe. A pull request gets the two newest regular versions. | +| `build.py` | Builds one Python version with conda-build, which runs the recipe's own tests. | +| `hooks.py` | The workflow's test jobs: installs one package into a clean conda-forge environment with the requirements in `config.yml`, and runs `test.py` the way Jenkins does. | +| `config.yml` | Tells gh-ci which workflow builds the packages, what its artifacts are called, and what the tests need. | +| `test.py` | Runs the installed package's test suite with pytest, in Actions and again in Jenkins. | + +The scripts need only the Python standard library and are the same in every repository gh-ci tests, except that `matrix.py` here looks for `conda-recipe-cf/` first. Scanning is not the repository's to define: Jenkins runs its own scans. + +**A pull request into `master`.** Actions builds the two newest regular Python versions on Linux and Windows and tests each package. Jenkins notices the pull request within 3 minutes, waits for that whole run, and then tests and scans every package at once. Each cell posts its own status on the pull request, and `intel-ci/pr` sums them up. + +**A release tag.** Actions builds every declared version, and Jenkins tests and scans all of them before it publishes the release. + +To support another Python, add it to `conda-recipe-cf/conda_build_config.yaml`; nothing else lists versions. Try it locally from a conda base environment with conda-build: + +```bash +python ci/matrix.py # every platform and Python version +python ci/matrix.py --event pull_request # what a pull request builds +python ci/build.py --python VERSION --output /tmp/out # VERSION: one matrix.py lists +python ci/hooks.py --python VERSION --artifacts /tmp/out --work /tmp/hooks +``` diff --git a/ci/build.py b/ci/build.py new file mode 100644 index 00000000..d7977f98 --- /dev/null +++ b/ci/build.py @@ -0,0 +1,274 @@ +""" +Builds this repository's packages for one Python version. + +For the gh-ci GitHub Actions workflow, and for Coverity in Jenkins. + +Copy this file and ci/matrix.py into any repository gh-ci tests. They work out +the build system from the files present, and the Python versions from that +build system's own declaration, so most repositories need no changes to them: + + rattler-build .conda recipe.yaml or recipe/recipe.yaml, versions in + variants.yaml + conda-build .conda conda-recipe-cf/meta.yaml or meta.yaml, versions in + conda_build_config.yaml + python -m build .whl pyproject.toml, versions from its classifiers + +Usage: + python ci/build.py --python 3.14 --output OUT [--system rattler-build] + [--channel conda-forge] + +--python must be one of the declared versions; `python ci/matrix.py` lists them. + +Packages land directly in OUT, ready for actions/upload-artifact. A conda +recipe's own tests run during the build; Jenkins later runs ci/test.py against +the uploaded artifacts, and builds with this script once more under Coverity for +intel-innersource tags. Needs only the Python standard library. rattler-build is +used from PATH, or downloaded at a pinned version for Linux or Windows and +checked against its published sha256; conda-build, or the `build` package for +wheels, must already be installed for this interpreter, so run it with the conda +base install's Python to build with conda-build. +""" + +import argparse +import hashlib +import os +import platform +import shlex +import shutil +import subprocess +import sys +import sysconfig +import tempfile +import urllib.request +from pathlib import Path + +import matrix + +ROOT = matrix.ROOT +PACKAGE_SUFFIXES = (".conda", ".tar.bz2", ".whl") + +# The rattler-build release fetched when none is on PATH, per system, with the +# file name it goes under and the sha256 its release publishes. +RATTLER_BUILD_VERSION = "v0.77.0" +RATTLER_BUILD_RELEASES = { + "Linux": ( + "rattler-build-x86_64-unknown-linux-musl", + "rattler-build", + "5a5e8150c703b21e88a01f9130ea49a7a680198e9ac9414170c0a293fed7e5af", + ), + "Windows": ( + "rattler-build-x86_64-pc-windows-msvc.exe", + "rattler-build.exe", + "e144334f782cee9aa23e26eb8d095c143e1696fbbec7ba7534a156663d4abb2c", + ), +} + + +class BuildError(Exception): + """Raised when the build cannot start or produced nothing.""" + + +def ensure_rattler_build(tools: Path) -> None: + """ + Puts rattler-build on PATH, downloading the pinned release when none is + there. + """ + if shutil.which("rattler-build"): + return + release = RATTLER_BUILD_RELEASES.get(platform.system()) + if release is None or platform.machine().lower() not in ("x86_64", "amd64"): + raise BuildError( + "no rattler-build on PATH, and only the Linux and Windows x86_64 " + "releases are pinned here" + ) + asset, name, sha256 = release + print(f"build: fetching rattler-build {RATTLER_BUILD_VERSION}", flush=True) + url = ( + "https://github.com/prefix-dev/rattler-build/releases/download/" + f"{RATTLER_BUILD_VERSION}/{asset}" + ) + with urllib.request.urlopen(url, timeout=300) as reply: + binary = reply.read() + if hashlib.sha256(binary).hexdigest() != sha256: + raise BuildError( + f"rattler-build {RATTLER_BUILD_VERSION} does not match its pinned " + "sha256" + ) + tools.mkdir(parents=True, exist_ok=True) + target = tools / name + target.write_bytes(binary) + target.chmod(0o755) + os.environ["PATH"] = f'{tools}{os.pathsep}{os.environ.get("PATH", "")}' + + +def check_interpreter(python: str) -> None: + """Wheels build with this interpreter: it must be the requested one.""" + running = f"{sys.version_info.major}.{sys.version_info.minor}" + if sysconfig.get_config_var("Py_GIL_DISABLED"): + running += "t" + if running != python: + raise BuildError( + f"wheels for Python {python} must be built with that Python, " + f"not {running}; set up {python} first" + ) + + +def commands( + system: str, + selector: Path, + python: matrix.Python, + output: Path, + channels: list[str], + scratch: Path, +) -> list[list[str]]: + """ + The commands that build one Python version's packages into output. + + Each build system gets the version back exactly as its own file declares it, + and keeps every other variant setting from that file. + """ + channel_args = [arg for channel in channels for arg in ("-c", channel)] + if system == "rattler-build": + return [ + [ + "rattler-build", + "build", + "-r", + str(selector), + "--variant", + f"python={python.spec}", + "--output-dir", + str(scratch / "conda"), + *channel_args, + ] + ] + if system == "conda-build": + # conda runs as a module of this interpreter, so Windows needs no + # conda.bat on PATH. + return [ + [ + sys.executable, + "-m", + "conda", + "build", + str(selector.parent), + "--python", + python.spec, + "--no-anaconda-upload", + "--output-folder", + str(scratch / "conda"), + "--override-channels", + *channel_args, + ] + ] + check_interpreter(python.version) + return [ + [ + sys.executable, + "-m", + "build", + "--wheel", + "--outdir", + str(output), + str(ROOT), + ] + ] + + +def collect(scratch: Path, output: Path) -> list[Path]: + """Moves built packages into output, flat, and lists them.""" + output.mkdir(parents=True, exist_ok=True) + for package in sorted((scratch / "conda").rglob("*")): + if ( + package.is_file() + and package.name.endswith(PACKAGE_SUFFIXES) + and "broken" not in package.parts + ): + shutil.move(str(package), output / package.name) + built = sorted( + path + for path in output.iterdir() + if path.name.endswith(PACKAGE_SUFFIXES) + ) + if not built: + raise BuildError(f"the build left no package in {output}") + return built + + +def main(argv: list[str] | None = None) -> int: + """Builds one Python version.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--output", required=True, type=Path, help="where the packages go" + ) + parser.add_argument( + "--system", + default="", + choices=("", *matrix.SYSTEMS), + help="build system to use instead of the detected one", + ) + parser.add_argument( + "--channel", + action="append", + default=None, + help="conda channel to build against, repeatable; default conda-forge", + ) + default_tools = ( + Path(os.environ.get("RUNNER_TEMP", tempfile.gettempdir())) + / "gh-ci-tools" + ) + parser.add_argument( + "--tools", + type=Path, + default=default_tools, + help="where a downloaded rattler-build goes", + ) + args = parser.parse_args(argv) + + system, selector = matrix.detect(ROOT, args.system) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise BuildError( + f"Python {args.python} is not declared for {system}; " + f'declared: {", ".join(declared)}' + ) + python = declared[args.python] + print( + f"build: {system} from {selector.relative_to(ROOT)} " + f"for Python {python.version} ({python.spec})", + flush=True, + ) + if system == "rattler-build": + ensure_rattler_build(args.tools) + output = args.output.resolve() + with tempfile.TemporaryDirectory() as scratch: + for command in commands( + system, + selector, + python, + output, + args.channel or ["conda-forge"], + Path(scratch), + ): + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True) + built = collect(Path(scratch), output) + print("build: " + ", ".join(path.name for path in built)) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + BuildError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"build: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/config.yml b/ci/config.yml new file mode 100644 index 00000000..18673ae3 --- /dev/null +++ b/ci/config.yml @@ -0,0 +1,12 @@ +# How Intel Python CI (gh-ci) tests mkl_fft. .github/workflows/intel-ci.yml builds +# conda-recipe-cf for every platform and Python version the recipe declares, as ci/matrix.py +# reads them, and tests each package with ci/test.py. Intel's Jenkins then installs each package +# into a clean conda-forge environment on its own machines, runs ci/test.py there with the test +# requirements below, and scans it. {platform} and {python} match what the run built, so +# nothing here lists them. +version: 1 +workflow: intel-ci.yml +artifacts: + conda: "mkl_fft {platform} Python {python}" +test: + requires: [pytest, "scipy >=1.10", mkl-service] diff --git a/ci/hooks.py b/ci/hooks.py new file mode 100644 index 00000000..6607808b --- /dev/null +++ b/ci/hooks.py @@ -0,0 +1,389 @@ +""" +Runs ci/test.py against one built artifact, the way Intel Python CI does. + +For the GitHub Actions test job. + +Each Python version's artifact is installed into a fresh conda environment from +a local channel plus conda-forge, pinned to its exact build, with the test +requirements ci/config.yml lists, and the test runs with that environment's +Python from an empty working directory, on Linux or Windows. The interpreter +spec and the requirements are the ones Jenkins uses. It gets the same variables, +and nothing else from the runner, as it does in Jenkins, so a test that passes +here passes there. Jenkins runs ci/test.py again on its own machines, GPUs +included; the scans are Jenkins' own and never run here. + +Usage, from any Python 3.10 or newer; it runs itself again with the conda base +install's Python, from $CONDA, which setup-miniconda sets, or the conda on PATH: + python ci/hooks.py --python 3.14 --artifacts DIR --work DIR + +Needs conda, conda-package-handling and ruamel.yaml, which every conda base has; +installs conda-index into base when it is missing. +""" + +import argparse +import json +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import matrix + +# Every conda base has these, as dependencies of conda itself. ci/config.yml is +# read the way Jenkins reads it: with PyYAML when it is there, else ruamel.yaml. +try: + from conda_package_handling import api as CPH_API +except ModuleNotFoundError: + CPH_API = None +try: + import yaml as PYYAML +except ModuleNotFoundError: + PYYAML = None +try: + from ruamel.yaml import YAML as RUAMEL_YAML +except ModuleNotFoundError: + RUAMEL_YAML = None + +ROOT = matrix.ROOT +IS_WINDOWS = os.name == "nt" +PLATFORM = "windows" if IS_WINDOWS else "linux" +CONDA_SUFFIXES = (".conda", ".tar.bz2") +# What the test may see of the runner's environment; tokens and everything else +# are dropped. The Windows names are what any Windows process needs to start and +# find its temp directory. +KEEP = ( + "PATH", + "HOME", + "USER", + "LANG", + "TERM", + "TMPDIR", + "TZ", + "http_proxy", + "https_proxy", + "no_proxy", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CONDA_PKGS_DIRS", + "SYSTEMROOT", + "SYSTEMDRIVE", + "WINDIR", + "COMSPEC", + "PATHEXT", + "TEMP", + "TMP", + "USERPROFILE", + "APPDATA", + "LOCALAPPDATA", + "PROGRAMDATA", + "NUMBER_OF_PROCESSORS", + "PROCESSOR_ARCHITECTURE", +) + + +class HookError(Exception): + """Raised when the artifact cannot be installed or ci/test.py is missing.""" + + +def run(command: list[str], **options: object) -> None: + """Echoes and runs a command, never through a shell.""" + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True, **options) + + +def conda(*args: str) -> None: + """Runs conda from the base environment running this script.""" + run([sys.executable, "-m", "conda", *args]) + + +def stage_channel(packages: list[Path], channel: Path) -> None: + """ + Lays the packages out as an indexed local channel, by the subdir each one + declares. + """ + if CPH_API is None: + raise HookError( + "run this with a conda base Python, " + "which has conda-package-handling" + ) + for package in packages: + with tempfile.TemporaryDirectory() as scratch: + CPH_API.extract(str(package), scratch, components="info") + index = json.loads( + (Path(scratch) / "info" / "index.json").read_text( + encoding="utf-8" + ) + ) + subdir = channel / str(index.get("subdir") or "noarch") + subdir.mkdir(parents=True, exist_ok=True) + shutil.copy2(package, subdir / package.name) + (channel / "noarch").mkdir(parents=True, exist_ok=True) + if subprocess.run( + [sys.executable, "-c", "import conda_index"], check=False + ).returncode: + conda( + "install", + "-y", + "-q", + "-n", + "base", + "--override-channels", + "-c", + "conda-forge", + "conda-index", + ) + run([sys.executable, "-m", "conda_index", str(channel)]) + + +def test_python(version: str) -> str: + """The interpreter spec for the test environment, as Jenkins spells it.""" + minor, threaded = matrix.version_key(version) + # The .* matters: conda reads a bare 3.11 as exactly 3.11.0, not the newest + # 3.11. + if minor < matrix.FREE_THREADED_MINOR: + return f"3.{minor}.*" + # From 3.13 on conda-forge marks the ABI, so 3.14 cannot resolve to the + # free-threaded build. + return f'3.{minor}.* *_cp3{minor}{"t" if threaded else ""}' + + +def test_requirements() -> tuple[list[str], list[str]]: + """ + The conda-forge and pip packages ci/config.yml installs next to the + artifact. + """ + text = (ROOT / "ci" / "config.yml").read_text(encoding="utf-8") + if PYYAML is not None: + config = PYYAML.safe_load(text) + elif RUAMEL_YAML is not None: + config = RUAMEL_YAML(typ="safe", pure=True).load(text) + else: + raise HookError("reading ci/config.yml needs PyYAML or ruamel.yaml") + test = (config or {}).get("test") or {} + lists = [] + for key in ("requires", "pip_requires"): + values = test.get(key) or [] + if not isinstance(values, list) or not all( + isinstance(v, str) for v in values + ): + raise HookError( + f"ci/config.yml: test.{key} must be a list of package specs" + ) + lists.append(values) + return lists[0], lists[1] + + +def exact_spec(package: Path) -> str: + """name=version=build from a conda package file name.""" + stem = package.name + for suffix in CONDA_SUFFIXES: + stem = stem.removesuffix(suffix) + name, version, build = stem.rsplit("-", 2) + return f"{name}={version}={build}" + + +def env_python(env_dir: Path) -> Path: + """The interpreter of a conda environment.""" + return env_dir / "python.exe" if IS_WINDOWS else env_dir / "bin" / "python" + + +def env_path(env_dir: Path) -> list[Path]: + """ + The directories conda's activation would put first on PATH for an + environment. + """ + if not IS_WINDOWS: + return [env_dir / "bin"] + library = env_dir / "Library" + return [ + env_dir, + library / "mingw-w64" / "bin", + library / "usr" / "bin", + library / "bin", + env_dir / "Scripts", + env_dir / "bin", + ] + + +def test_env( + env_dir: Path, artifacts: Path, results: Path, args: argparse.Namespace +) -> dict: + """The variables Jenkins gives ci/test.py, and no runner tokens.""" + env = {key: value for key, value in os.environ.items() if key in KEEP} + env["PATH"] = os.pathsep.join( + [*map(str, env_path(env_dir)), env.get("PATH", "")] + ) + env.update( + { + "CONDA_PREFIX": str(env_dir), + "GH_CI_ARTIFACTS": str(artifacts), + "GH_CI_SOURCE": str(ROOT), + "GH_CI_RESULTS": str(results), + "GH_CI_KIND": "conda", + "GH_CI_PYTHON": args.python, + "GH_CI_PLATFORM": PLATFORM, + "GH_CI_EVENT": args.event, + "GH_CI_COMMIT": args.commit, + } + ) + return env + + +def summarize(lines: list[str]) -> None: + """Adds the results to the job summary page, in GitHub Actions.""" + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as page: + page.write("\n".join(lines) + "\n") + + +def install(args: argparse.Namespace) -> Path: + """ + Creates the test environment with the artifact, pinned to its exact build; + returns it. + """ + system, selector = matrix.detect(ROOT) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise HookError( + f"Python {args.python} is not declared; " + f'declared: {", ".join(declared)}' + ) + packages = sorted( + path + for path in args.artifacts.rglob("*") + if path.name.endswith(CONDA_SUFFIXES) + ) + if not packages: + raise HookError(f"{args.artifacts} holds no conda package") + requires, pip_requires = test_requirements() + channel, env_dir = args.work / "channel", args.work / "env" + stage_channel(packages, channel) + conda( + "create", + "-y", + "-q", + "-p", + str(env_dir), + "--override-channels", + "-c", + channel.as_uri(), + "-c", + "conda-forge", + f"python {test_python(args.python)}", + *requires, + *map(exact_spec, packages), + ) + if pip_requires: + run( + [ + str(env_python(env_dir)), + "-m", + "pip", + "install", + "--no-input", + *pip_requires, + ] + ) + return env_dir + + +def run_test(env_dir: Path, args: argparse.Namespace) -> int: + """Runs ci/test.py with the environment's Python; returns its status.""" + script = ROOT / "ci" / "test.py" + if not script.is_file(): + raise HookError("ci/test.py is required and missing") + results, scratch = ( + args.work / "results" / "test", + args.work / "scratch" / "test", + ) + for directory in (results, scratch): + directory.mkdir(parents=True, exist_ok=True) + command = [str(env_python(env_dir)), str(script)] + print(f"+ {shlex.join(command)}", flush=True) + status = subprocess.run( + command, + cwd=scratch, + check=False, + env=test_env(env_dir, args.artifacts, results, args), + ).returncode + result = "passed" if status == 0 else f"failed, exit {status}" + summarize( + [ + f"### {PLATFORM.capitalize()} Python {args.python}", + "", + "| Test | Result |", + "|---|---|", + f"| ci/test.py | {result} |", + ] + ) + return status + + +def conda_base_python() -> Path: + """The conda base install's Python: from $CONDA, else the conda on PATH.""" + root = os.environ.get("CONDA", "") + if not root: + found = shutil.which("conda") + if not found: + raise HookError( + "no conda base install: set CONDA or put conda on PATH" + ) + # conda lives in bin, Scripts or condabin, one level below the install. + root = str(Path(found).resolve().parent.parent) + python = env_python(Path(root)) + if not python.is_file(): + raise HookError( + f"{root} is not a conda base install; it has no {python.name}" + ) + return python + + +def main(argv: list[str] | None = None) -> int: + """Installs the artifact, then runs ci/test.py against it.""" + if CPH_API is None: + # conda and conda-package-handling come with the base install, so run + # there instead. + command = [ + str(conda_base_python()), + str(Path(__file__).resolve()), + *(sys.argv[1:] if argv is None else argv), + ] + return subprocess.run(command, check=False).returncode + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--artifacts", required=True, type=Path, help="the downloaded artifact" + ) + parser.add_argument( + "--work", required=True, type=Path, help="an empty scratch directory" + ) + parser.add_argument( + "--event", default="push", choices=("pr", "tag", "push") + ) + parser.add_argument("--commit", default="") + args = parser.parse_args(argv) + args.artifacts, args.work = args.artifacts.resolve(), args.work.resolve() + return run_test(install(args), args) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + HookError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"hooks: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/matrix.py b/ci/matrix.py new file mode 100644 index 00000000..bdb30639 --- /dev/null +++ b/ci/matrix.py @@ -0,0 +1,632 @@ +""" +The platforms and Python versions this repository builds. + +Both are read from the repository's own build files. + +Every build system already has a standard place for both, so each is written +once, where the build reads it, and the GitHub Actions matrix and Intel Python +CI (gh-ci) both follow it: + + rattler-build, selected by recipe.yaml + Python versions python: in variants.yaml next to the recipe + platforms build: skip: in the recipe + conda-build, selected by meta.yaml + Python versions python: in conda_build_config.yaml next to the recipe + platforms skip: true # [selector] lines in the recipe + wheel, selected by pyproject.toml + Python versions the Programming Language :: Python :: 3.N classifiers; + any Free Threading classifier adds 3.Nt for each from 3.13 + platforms the Operating System :: classifiers + +A recipe builds on every platform gh-ci builds, Linux and Windows, unless a skip +that depends on the platform alone rules one out, such as `skip: [win]` or +`skip: true # [not linux]`; a skip that also depends on something else, such +as `win and py < 311`, skips some builds, not the platform. A pyproject.toml +without Operating System classifiers, or with OS Independent, builds on both. +macOS is not a platform gh-ci builds. Of the declared platforms, the matrix +holds those GitHub Actions has runners for, BUILT_PLATFORMS: GitHub-hosted +runners cover Linux and Windows. + +meson-python, scikit-build-core and every other PEP 517 backend declare these in +pyproject.toml too, so the wheel row covers them. Another convention, such as +pixi.toml, is one more reader in READERS and PLATFORM_READERS and, if it is a +new build system, one more entry in SYSTEMS. + +Usage: + python ci/matrix.py [--system SYSTEM] [--event EVENT] [--github-output NAME] + +Prints the build matrix as JSON for strategy.matrix: {"include": [{"platform": +"linux", "name": "Linux", "python": "3.14"}, ...]}, every platform with every +Python version. For a pull_request event only the two newest regular versions +are listed, which is all gh-ci tests on a pull request. --github-output also +writes NAME= to $GITHUB_OUTPUT, for later jobs. Standard library only; +reading pyproject.toml needs Python 3.11 or newer for tomllib. +""" + +import argparse +import ast +import json +import os +import re +import sys +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +# Only the wheel reader needs it, and only it needs Python 3.11 or newer. +try: + import tomllib as TOMLLIB +except ModuleNotFoundError: + TOMLLIB = None + +ROOT = Path(__file__).resolve().parent.parent + +# Build system -> files that select it, in the order they are looked for. The +# conda-forge recipe comes first: gh-ci tests from conda-forge alone, and +# conda-recipe/ builds against Intel's channel. +SYSTEMS = { + "rattler-build": ("recipe.yaml", "recipe/recipe.yaml"), + "conda-build": ( + "conda-recipe-cf/meta.yaml", + "meta.yaml", + "conda-recipe/meta.yaml", + "recipe/meta.yaml", + ), + "wheel": ("pyproject.toml",), +} + +# The platforms gh-ci builds and tests, and how artifact names spell them. +PLATFORMS = {"linux": "Linux", "windows": "Windows"} +# The platforms the workflow can build on: GitHub-hosted runners have both. A +# declared platform missing here is reported and left out. +BUILT_PLATFORMS = ("linux", "windows") +# What a platform-only selector means on each platform, for both conda-build and +# rattler-build. +SELECTORS = { + "linux": { + "linux": True, + "linux64": True, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + "windows": { + "linux": False, + "linux64": False, + "unix": False, + "win": True, + "win64": True, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, +} +CONDA_SKIP_RE = re.compile(r"\s*skip\s*:\s*(?:true|True|yes)\s*#\s*\[(.+)\]\s*") +OS_CLASSIFIERS = { + "Operating System :: OS Independent": tuple(PLATFORMS), + "Operating System :: POSIX": ("linux",), + "Operating System :: Unix": ("linux",), + "Operating System :: POSIX :: Linux": ("linux",), + "Operating System :: Microsoft :: Windows": ("windows",), +} + +PULL_REQUEST_EVENTS = ("pull_request",) +PULL_REQUEST_COUNT = 2 +OLDEST_MINOR = 10 +FREE_THREADED_MINOR = 13 + +# A conda python variant: 3.12, 3.12.*, 3.12.* *_cpython, 3.13.* *_cp313 or +# 3.14.* *_cp314t. +CONDA_SPEC_RE = re.compile( + r"3\.(\d+)(?:\.\*)?(?:\s+\*_(?:cpython|cp3(\d+)(t?)))?" +) +CLASSIFIER_RE = re.compile(r"Programming Language :: Python :: 3\.(\d+)") +FREE_THREADING_CLASSIFIER = "Programming Language :: Python :: Free Threading" +LOWER_BOUND_RE = re.compile(r">=\s*3\.(\d+)") + + +class DeclarationError(Exception): + """ + Raised when the build files declare no Python versions, or ones this cannot + read. + """ + + +@dataclass(frozen=True) +class Python: + """One declared Python version.""" + + version: str # as artifact names and gh-ci spell it: 3.14 or 3.14t + spec: str # as the build system spells it, passed back to it unchanged + + +def version_key(version: str) -> tuple[int, bool]: + """(minor, free-threaded); also the sort order.""" + return int(version.rstrip("t").split(".")[1]), version.endswith("t") + + +def checked(minor: int, threaded: bool, where: str) -> str: + """The version string, once it is one gh-ci can test.""" + if minor < OLDEST_MINOR or (threaded and minor < FREE_THREADED_MINOR): + raise DeclarationError( + f'{where}: 3.{minor}{"t" if threaded else ""} is not supported; ' + f"the oldest is 3.{OLDEST_MINOR}, free-threaded from " + f"3.{FREE_THREADED_MINOR}" + ) + return f'3.{minor}{"t" if threaded else ""}' + + +def _strip_comment(line: str, where: str) -> str: + """ + The line without a trailing comment; conda-build selectors are refused, not + guessed at. + """ + in_quote = "" + for index, char in enumerate(line): + if char in "\"'" and in_quote in ("", char): + in_quote = "" if in_quote else char + elif ( + char == "#" + and not in_quote + and (index == 0 or line[index - 1].isspace()) + ): + if line[index:].lstrip("# ").startswith("["): + raise DeclarationError( + f"{where}: selectors in the python list are not supported" + ) + return line[:index].rstrip() + return line.rstrip() + + +def _unquote(value: str, where: str) -> str: + """A plain or quoted YAML scalar.""" + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + return value[1:-1] + if not value or value[0] in "{[&*!|>" or ":" in value: + raise DeclarationError( + f"{where}: write each python entry as a plain value, not {value!r}" + ) + return value + + +def yaml_python_list(text: str, where: str) -> list[str]: + """ + The top-level python: list of a variants file, without a YAML library. + + Accepts a block list or a one-line [a, b] list of plain or quoted values. + Anything richer, such as if/then selectors or templating, is refused so the + matrix is never guessed. + """ + lines = text.splitlines() + for number, line in enumerate(lines): + if not re.match(r"python\s*:", line): + continue + rest = _strip_comment(line.split(":", 1)[1], where).strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + raise DeclarationError(f"{where}: python must be a list") + values = [] + for item in lines[number + 1 :]: + item = _strip_comment(item, where) + entry = item.lstrip() + if not entry: + continue + if not entry.startswith("- "): + if not item[0].isspace(): + break # the next top-level key + raise DeclarationError( + f"{where}: python must be a plain list of versions" + ) + values.append(_unquote(entry[2:], where)) + return values + raise DeclarationError(f"{where} has no top-level python: list") + + +def from_conda_variants(path: Path) -> list[Python]: + """The python: list of a variants.yaml or a conda_build_config.yaml.""" + where = ( + path.relative_to(ROOT).as_posix() + if path.is_relative_to(ROOT) + else str(path) + ) + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError as ex: + raise DeclarationError( + f"{where} is missing; list the Python versions to build there" + ) from ex + pythons = [] + for spec in yaml_python_list(text, where): + match = CONDA_SPEC_RE.fullmatch(spec) + if not match or (match[2] and match[2] != match[1]): + raise DeclarationError( + f"{where}: {spec!r} is not a CPython variant such as " + '"3.13.* *_cp313" or "3.14.* *_cp314t"' + ) + pythons.append( + Python(checked(int(match[1]), bool(match[3]), where), spec) + ) + return pythons + + +def from_pyproject(path: Path) -> list[Python]: + """ + The 3.N classifiers in pyproject.toml, plus 3.Nt from 3.13 if free-threading + is declared. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + project = TOMLLIB.loads(path.read_text(encoding="utf-8")).get("project", {}) + classifiers = project.get("classifiers", []) + minors = sorted( + { + int(match[1]) + for classifier in classifiers + if (match := CLASSIFIER_RE.fullmatch(classifier)) + } + ) + if not minors: + raise DeclarationError( + "pyproject.toml lists no Programming Language :: Python :: 3.N " + "classifiers" + ) + bound = LOWER_BOUND_RE.search(project.get("requires-python", "")) + if bound and minors[0] < int(bound[1]): + raise DeclarationError( + f"pyproject.toml classifies 3.{minors[0]}, below requires-python " + f'{project["requires-python"]}' + ) + threaded = any( + classifier.startswith(FREE_THREADING_CLASSIFIER) + for classifier in classifiers + ) + versions = [checked(minor, False, "pyproject.toml") for minor in minors] + versions += [ + f"3.{minor}t" + for minor in minors + if threaded and minor >= FREE_THREADED_MINOR + ] + return [Python(version, version) for version in versions] + + +def skips_platform(expression: str, platform: str) -> bool | None: + """ + Whether a skip expression rules out a whole platform: True or False when it + depends on the platform alone, None when it also depends on anything else, + such as the Python version. + """ + try: + tree = ast.parse( + expression.strip().removeprefix("${{").removesuffix("}}").strip(), + mode="eval", + ).body + except SyntaxError: + return None + + def value(node: ast.AST) -> bool: + if isinstance(node, ast.BoolOp): + values = [value(item) for item in node.values] + return all(values) if isinstance(node.op, ast.And) else any(values) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not value(node.operand) + if isinstance(node, ast.Name) and node.id in SELECTORS[platform]: + return SELECTORS[platform][node.id] + if isinstance(node, ast.Constant) and isinstance(node.value, bool): + return node.value + raise ValueError(ast.dump(node)) + + try: + return value(tree) + except ValueError: + return None + + +def platforms_without(skips: list[str], where: str) -> list[str]: + """Every platform gh-ci builds, but those a platform-only skip rules out.""" + kept = [ + platform + for platform in PLATFORMS + if not any(skips_platform(skip, platform) for skip in skips) + ] + if not kept: + raise DeclarationError( + f"{where} skips every platform gh-ci builds: " + + ", ".join(PLATFORMS.values()) + ) + return kept + + +def yaml_build_skips(text: str, where: str) -> list[str]: + """ + The skip: entries directly under a rattler-build recipe's top-level build:, + as a list of plain or quoted values, a one-line [a, b] list, or one value; + none when there is no skip. + """ + lines = text.splitlines() + starts = [ + number + for number, line in enumerate(lines) + if re.match(r"build\s*:\s*(#.*)?$", line) + ] + if not starts: + return [] + child = "" + for number in range(starts[0] + 1, len(lines)): + line = ( + _strip_comment(lines[number], where) + if "#" in lines[number] + else lines[number] + ) + if not line.strip(): + continue + if not line[0].isspace(): + return [] # the next top-level key + indent = line[: len(line) - len(line.lstrip())] + child = child or indent + match = ( + re.match(r"skip\s*:(.*)", line.lstrip()) + if indent == child + else None + ) + if not match: + continue + rest = match[1].strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + return [_unquote(rest, where)] + values = [] + for item in lines[number + 1 :]: + entry = item.strip() + if not entry or entry.startswith("#"): + continue + if not entry.startswith("- "): + break + values.append(_unquote(entry[2:].split(" #")[0], where)) + return values + return [] + + +def from_rattler_recipe(recipe: Path) -> list[str]: + """The platforms a rattler-build recipe builds on, from its build: skip:.""" + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + return platforms_without( + yaml_build_skips(recipe.read_text(encoding="utf-8"), where), where + ) + + +def from_conda_recipe(recipe: Path) -> list[str]: + """ + The platforms a conda-build meta.yaml builds on, from its skip: true # + [selector] lines. + """ + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + skips = [ + match[1] + for line in recipe.read_text(encoding="utf-8").splitlines() + if (match := CONDA_SKIP_RE.fullmatch(line)) + ] + return platforms_without(skips, where) + + +def from_os_classifiers(path: Path) -> list[str]: + """ + The platforms pyproject.toml's Operating System classifiers name; both when + it names none. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + classifiers = ( + TOMLLIB.loads(path.read_text(encoding="utf-8")) + .get("project", {}) + .get("classifiers", []) + ) + named = [ + classifier + for classifier in classifiers + if classifier.startswith("Operating System ::") + ] + if not named: + return list(PLATFORMS) + found = { + platform + for classifier in named + for prefix, platforms in OS_CLASSIFIERS.items() + if classifier == prefix or classifier.startswith(prefix + " ::") + for platform in platforms + } + if not found: + raise DeclarationError( + "pyproject.toml names no Operating System gh-ci builds: " + + ", ".join(PLATFORMS.values()) + ) + return [platform for platform in PLATFORMS if platform in found] + + +# Build system -> where it declares Python versions, given the file that +# selected it. +READERS: dict[str, Callable[[Path], list[Python]]] = { + "rattler-build": lambda recipe: from_conda_variants( + recipe.parent / "variants.yaml" + ), + "conda-build": lambda recipe: from_conda_variants( + recipe.parent / "conda_build_config.yaml" + ), + "wheel": from_pyproject, +} + +# Build system -> where it declares the platforms it builds on, given the file +# that selected it. +PLATFORM_READERS: dict[str, Callable[[Path], list[str]]] = { + "rattler-build": from_rattler_recipe, + "conda-build": from_conda_recipe, + "wheel": from_os_classifiers, +} + + +def detect(root: Path, system: str = "") -> tuple[str, Path]: + """The build system and the file that selects it.""" + for name, candidates in SYSTEMS.items(): + if system and name != system: + continue + for candidate in candidates: + if (root / candidate).is_file(): + return name, root / candidate + wanted = system or " or ".join(SYSTEMS) + raise DeclarationError( + f"no {wanted} input found; looked for " + + ", ".join(c for name, cs in SYSTEMS.items() for c in cs) + ) + + +def declared(system: str, selector: Path) -> list[Python]: + """The build system's Python versions, sorted, each listed once.""" + pythons = READERS[system](selector) + versions = [python.version for python in pythons] + if not pythons: + raise DeclarationError(f"{system} declares no Python versions") + if len(set(versions)) != len(versions): + raise DeclarationError( + f"{system} declares a Python version twice: {versions}" + ) + return sorted(pythons, key=lambda python: version_key(python.version)) + + +def declared_platforms(system: str, selector: Path) -> list[str]: + """The platforms the build system builds on, in PLATFORMS order.""" + return PLATFORM_READERS[system](selector) + + +def for_event(pythons: list[Python], event: str) -> list[Python]: + """All of them, or for a pull request the two newest regular versions.""" + if event not in PULL_REQUEST_EVENTS: + return pythons + regular = [python for python in pythons if not python.version.endswith("t")] + return (regular or pythons)[-PULL_REQUEST_COUNT:] + + +def build_matrix(system: str, selector: Path, event: str) -> dict: + """ + Every declared platform Actions can build on, with every Python version the + event builds. + """ + pythons = for_event(declared(system, selector), event) + platforms = declared_platforms(system, selector) + waiting = [ + PLATFORMS[platform] + for platform in platforms + if platform not in BUILT_PLATFORMS + ] + if waiting: + print( + f'matrix: {", ".join(waiting)} declared but not built: ' + "no runners for it yet", + file=sys.stderr, + ) + built = [platform for platform in platforms if platform in BUILT_PLATFORMS] + if not built: + raise DeclarationError( + "the build files declare only platforms there are no runners " + "for yet: " + ", ".join(waiting) + ) + return { + "include": [ + { + "platform": platform, + "name": PLATFORMS[platform], + "python": python.version, + } + for platform in built + for python in pythons + ] + } + + +def main(argv: list[str] | None = None) -> int: + """Prints the matrix to build; writes it for GitHub Actions if asked.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--system", + default="", + choices=("", *SYSTEMS), + help="build system to read instead of the detected one", + ) + parser.add_argument( + "--event", default="", help="GitHub event name, such as pull_request" + ) + parser.add_argument( + "--github-output", + default="", + metavar="NAME", + help="also write NAME= to $GITHUB_OUTPUT", + ) + args = parser.parse_args(argv) + if args.github_output: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", args.github_output): + raise DeclarationError( + f"{args.github_output!r} is not an output name" + ) + if not os.environ.get("GITHUB_OUTPUT"): + raise DeclarationError( + "--github-output needs $GITHUB_OUTPUT, which Actions sets" + ) + system, selector = detect(ROOT, args.system) + matrix = build_matrix(system, selector, args.event) + cells = ", ".join( + f'{cell["name"]} {cell["python"]}' for cell in matrix["include"] + ) + print( + f"matrix: {system} from {selector.relative_to(ROOT)}: {cells}", + file=sys.stderr, + ) + text = json.dumps(matrix) + print(text) + if args.github_output: + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"{args.github_output}={text}\n") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except DeclarationError as error: + print(f"matrix: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/test.py b/ci/test.py new file mode 100644 index 00000000..2a24e953 --- /dev/null +++ b/ci/test.py @@ -0,0 +1,53 @@ +""" +Intel Python CI (gh-ci) test for mkl_fft. + +GitHub Actions, through ci/hooks.py, and Intel's Jenkins both run this with one +built conda package installed in a clean conda-forge environment, from an empty +working directory, so the tests import the installed mkl_fft rather than the +source tree. The test requirements come from ci/config.yml. Writes a JUnit +report for Jenkins. +""" + +import os +import pathlib +import sys +import sysconfig + +import pytest + +PACKAGE = "mkl_fft" + + +def running_python() -> str: + """This interpreter's version as gh-ci spells it, such as 3.14t.""" + version = f"{sys.version_info.major}.{sys.version_info.minor}" + threaded = sysconfig.get_config_var("Py_GIL_DISABLED") + return version + ("t" if threaded else "") + + +def main() -> int: + """Runs the installed package's test suite.""" + expected = os.environ["GH_CI_PYTHON"] + if running_python() != expected: + print(f"running Python {running_python()}, the cell is {expected}") + return 1 + # Every cell of a Jenkins build reports into the same build, so each names + # its own suite. + cell = f"{os.environ.get('GH_CI_PLATFORM', 'linux')}.py{expected}" + report = pathlib.Path(os.environ["GH_CI_RESULTS"]) / f"{PACKAGE}.xml" + return pytest.main( + [ + "-v", + "-p", + "no:cacheprovider", + f"--junitxml={report}", + "-o", + f"junit_suite_name={PACKAGE} {cell}", + "--pyargs", + PACKAGE, + ] + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conda-recipe-cf/AGENTS.md b/conda-recipe-cf/AGENTS.md index 9d8655e2..27e9e021 100644 --- a/conda-recipe-cf/AGENTS.md +++ b/conda-recipe-cf/AGENTS.md @@ -8,3 +8,4 @@ Conda-forge recipe context for `mkl_fft`. ## Guardrails - Keep conda-forge recipe semantics separate from Intel-channel recipe. - Align changes with `conda-package-cf.yml` workflow behavior. +- `conda_build_config.yaml`'s `python:` list is the Intel Python CI matrix, read by `ci/matrix.py`: plain versions, no selectors. diff --git a/conda-recipe-cf/conda_build_config.yaml b/conda-recipe-cf/conda_build_config.yaml index 822fd77b..17c37a0e 100644 --- a/conda-recipe-cf/conda_build_config.yaml +++ b/conda-recipe-cf/conda_build_config.yaml @@ -1,3 +1,16 @@ +# The Python versions to build, one per line and without selectors: ci/matrix.py +# reads them for .github/workflows/intel-ci.yml. NumPy 2, as in conda-forge's +# global pinning; conda-package-cf.yml passes its own with --python and --numpy. +python: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14.* *_cp314" + - "3.14.* *_cp314t" + - "3.15.* *_cp315" +numpy: + - '2' c_compiler: # [linux] - gcc # [linux] cxx_compiler: # [linux]