diff --git a/.github/scripts/envgen.py b/.github/scripts/envgen.py index 8c9bd64..e972ac3 100644 --- a/.github/scripts/envgen.py +++ b/.github/scripts/envgen.py @@ -27,6 +27,10 @@ off-index, while its base is an ordinary PyPI release. See ``req``. * GPU-only dropped - ``nvidia-*`` CUDA wheels, triton, flash-attn and deepspeed need a GPU a dev machine lacks (see DROP / DROP_PREFIX). + * Env-scoped drops - a pin dropped for specific environments only (DROP_BY_ENV) + when its version has no wheel for that env's Python and no + in-range version does either — e.g. pandas 1.5.3 on the + Python 3.12 runtimes DBR 16.4 / serverless-v3 (issue #18). * requires-python - taken from the runtime's Python version (major.minor). This module is imported by ``sync.py`` (the weekly discovery + reconciliation Action). @@ -54,6 +58,21 @@ "nvidia-", # nvidia-* CUDA wheels (and nvidia-ml-py): GPU tooling, no local use ) +# Environment-scoped drops: removed for the named environments only (DROP is global). +# pandas 1.5.3 has no cp312 wheel — pandas ships CPython 3.12 wheels only from 2.1.1 — +# so `uv sync` on a Python 3.12 runtime can't install it and falls back to a failing +# source build. DBR 16.4 and serverless-v3 are the only 3.12 runtimes still pinned to +# pandas 1.5.x: 13.3/14.3/15.4 predate 3.12, and 17.3+ / serverless-v4+ already ship +# pandas 2.x (which has cp312 wheels). No in-range (~=1.5) version has a cp312 wheel, +# so the pin can't be salvaged by widening — it is dropped for just these envs, and +# pandas resolves to an installable version locally. See issue #18. +DROP_BY_ENV = { + "16.4.x-scala2.12": {"pandas"}, + "16.4.x-cpu-ml-scala2.12": {"pandas"}, + "16.4.x-gpu-ml-scala2.12": {"pandas"}, + "serverless-v3": {"pandas"}, +} + def norm(name): # Strip the '*' footnote marker the release-notes tables append to some package @@ -102,12 +121,14 @@ def parse_requirements(text): return pkgs -def _filtered(pkgs): - # Inclusion only: drop the name-based DROP set and DROP_PREFIX. A PEP 440 local - # version segment is NOT a reason to drop — its base release is on PyPI — so - # those pins are kept here and req() strips the segment when rendering. +def _filtered(pkgs, env_name=None): + # Inclusion only: drop the global name-based DROP set and DROP_PREFIX, plus any + # environment-scoped drops (DROP_BY_ENV) for env_name. A PEP 440 local version + # segment is NOT a reason to drop — its base release is on PyPI — so those pins + # are kept here and req() strips the segment when rendering. + env_drop = DROP_BY_ENV.get(env_name, frozenset()) return {n: v for n, v in pkgs.items() - if n not in DROP and not n.startswith(DROP_PREFIX)} + if n not in DROP and n not in env_drop and not n.startswith(DROP_PREFIX)} def dbconnect_pin(pkgs): @@ -133,7 +154,7 @@ def build_pyproject(pkgs, env_name, python_version, dbconnect=None): not list it — the matching version is the runtime version, passed in explicitly. """ mm = ".".join(python_version.split(".")[:2]) # 3.12.3 -> 3.12 - body = {n: v for n, v in _filtered(pkgs).items() if n != "databricks-connect"} + body = {n: v for n, v in _filtered(pkgs, env_name).items() if n != "databricks-connect"} dev = f"databricks-connect~={dbconnect}.0" if dbconnect else dbconnect_pin(pkgs) project = "constraint-env-" + re.sub(r"[^a-z0-9]+", "-", env_name.lower()).strip("-") out = [ @@ -158,7 +179,7 @@ def build_pyproject(pkgs, env_name, python_version, dbconnect=None): def build_constraints(pkgs, env_name): - body = {n: v for n, v in _filtered(pkgs).items() if n != "databricks-connect"} + body = {n: v for n, v in _filtered(pkgs, env_name).items() if n != "databricks-connect"} out = [f"# constraints.txt file for Databricks {_label(env_name)}", ""] out += [req(n, body[n]) for n in sorted(body)] return "\n".join(out) + "\n" diff --git a/.github/scripts/test_envgen.py b/.github/scripts/test_envgen.py index 3c96377..fefb967 100644 --- a/.github/scripts/test_envgen.py +++ b/.github/scripts/test_envgen.py @@ -38,6 +38,27 @@ def test_keeps_installable_pins(self): self.assertEqual(_filtered(pkgs), pkgs) +class EnvScopedDropTest(unittest.TestCase): + def test_pandas_dropped_on_py312_1_5_runtimes(self): + # DBR 16.4 + serverless-v3 are the only Python-3.12 runtimes still on + # pandas 1.5.x, which has no cp312 wheel — dropped for just these envs. + pkgs = {"pandas": "1.5.3", "numpy": "2.1.3"} + for env in ( + "16.4.x-scala2.12", + "16.4.x-cpu-ml-scala2.12", + "16.4.x-gpu-ml-scala2.12", + "serverless-v3", + ): + self.assertEqual(_filtered(pkgs, env), {"numpy": "2.1.3"}, env) + + def test_pandas_kept_elsewhere(self): + # Other runtimes keep pandas: pre-3.12 envs have wheels for 1.5.x, and + # 17.3+/serverless-v4+ already ship pandas 2.x. No env_name = no scoped drop. + pkgs = {"pandas": "1.5.3", "numpy": "2.1.3"} + for env in ("15.4.x-scala2.12", "17.3.x-scala2.13", "serverless-v4", None): + self.assertEqual(_filtered(pkgs, env), pkgs, env) + + class ReqTest(unittest.TestCase): def test_strips_local_version_segment(self): # ~= is invalid with a local segment (PEP 440), and the segment names a build diff --git a/README.md b/README.md index dd24271..5fa0689 100644 --- a/README.md +++ b/README.md @@ -77,9 +77,17 @@ stay resolvable. Applied to **both** `pyproject.toml` and `constraints.txt`: → `flask~=1.1.2`), so `uv` resolves a platform-appropriate wheel. (Ubuntu system builds like `python-apt 2.7.7+ubuntu5.2` are dropped by name above instead, since their base version is not on PyPI.) - -The exact lists live in `DROP` / `DROP_PREFIX` and `_filtered()` / `req()` in -`.github/scripts/envgen.py`. +- **Environment-scoped drops** (dropped for named envs only) — a pin whose version + has no wheel for that environment's Python, where no in-range version has one + either. Today this is **`pandas` on DBR 16.4 and serverless-v3**: they are the only + Python-3.12 runtimes still pinned to `pandas 1.5.3`, which has no cp312 wheel (pandas + ships 3.12 wheels only from 2.1.1, so `~=1.5` can't reach one). The constraint is + dropped for just those envs, letting `pandas` resolve to an installable version + locally. Earlier runtimes (13.3/14.3/15.4) predate 3.12, and 17.3+ / serverless-v4+ + already ship pandas 2.x. + +The exact lists live in `DROP` / `DROP_PREFIX` / `DROP_BY_ENV` and `_filtered()` / +`req()` in `.github/scripts/envgen.py`. ## How it stays in sync diff --git a/python/dbr/16.4.x-cpu-ml-scala2.12/constraints.txt b/python/dbr/16.4.x-cpu-ml-scala2.12/constraints.txt index 3fb1850..df0df03 100644 --- a/python/dbr/16.4.x-cpu-ml-scala2.12/constraints.txt +++ b/python/dbr/16.4.x-cpu-ml-scala2.12/constraints.txt @@ -225,7 +225,6 @@ optuna-integration~=3.6.0 orjson~=3.10.16 overrides~=7.4.0 packaging~=24.1 -pandas~=1.5.3 pandocfilters~=1.5.0 paramiko~=3.4.0 parso~=0.8.3 diff --git a/python/dbr/16.4.x-cpu-ml-scala2.12/pyproject.toml b/python/dbr/16.4.x-cpu-ml-scala2.12/pyproject.toml index 9bfae3e..ed296d7 100644 --- a/python/dbr/16.4.x-cpu-ml-scala2.12/pyproject.toml +++ b/python/dbr/16.4.x-cpu-ml-scala2.12/pyproject.toml @@ -237,7 +237,6 @@ constraint-dependencies = [ "orjson~=3.10.16", "overrides~=7.4.0", "packaging~=24.1", - "pandas~=1.5.3", "pandocfilters~=1.5.0", "paramiko~=3.4.0", "parso~=0.8.3", diff --git a/python/dbr/16.4.x-gpu-ml-scala2.12/constraints.txt b/python/dbr/16.4.x-gpu-ml-scala2.12/constraints.txt index 531d11a..8f19c26 100644 --- a/python/dbr/16.4.x-gpu-ml-scala2.12/constraints.txt +++ b/python/dbr/16.4.x-gpu-ml-scala2.12/constraints.txt @@ -224,7 +224,6 @@ optuna-integration~=3.6.0 orjson~=3.10.16 overrides~=7.4.0 packaging~=24.1 -pandas~=1.5.3 pandocfilters~=1.5.0 paramiko~=3.4.0 parso~=0.8.3 diff --git a/python/dbr/16.4.x-gpu-ml-scala2.12/pyproject.toml b/python/dbr/16.4.x-gpu-ml-scala2.12/pyproject.toml index 903c296..eddfcda 100644 --- a/python/dbr/16.4.x-gpu-ml-scala2.12/pyproject.toml +++ b/python/dbr/16.4.x-gpu-ml-scala2.12/pyproject.toml @@ -236,7 +236,6 @@ constraint-dependencies = [ "orjson~=3.10.16", "overrides~=7.4.0", "packaging~=24.1", - "pandas~=1.5.3", "pandocfilters~=1.5.0", "paramiko~=3.4.0", "parso~=0.8.3", diff --git a/python/dbr/16.4.x-scala2.12/constraints.txt b/python/dbr/16.4.x-scala2.12/constraints.txt index 7bf6bdd..6819e8f 100644 --- a/python/dbr/16.4.x-scala2.12/constraints.txt +++ b/python/dbr/16.4.x-scala2.12/constraints.txt @@ -75,7 +75,6 @@ opentelemetry-api~=1.27.0 opentelemetry-sdk~=1.27.0 opentelemetry-semantic-conventions~=0.48b0 packaging~=24.1 -pandas~=1.5.3 parso~=0.8.3 pathspec~=0.10.3 patsy~=0.5.6 diff --git a/python/dbr/16.4.x-scala2.12/pyproject.toml b/python/dbr/16.4.x-scala2.12/pyproject.toml index 500b786..2c06589 100644 --- a/python/dbr/16.4.x-scala2.12/pyproject.toml +++ b/python/dbr/16.4.x-scala2.12/pyproject.toml @@ -87,7 +87,6 @@ constraint-dependencies = [ "opentelemetry-sdk~=1.27.0", "opentelemetry-semantic-conventions~=0.48b0", "packaging~=24.1", - "pandas~=1.5.3", "parso~=0.8.3", "pathspec~=0.10.3", "patsy~=0.5.6", diff --git a/python/serverless/serverless-v3/constraints.txt b/python/serverless/serverless-v3/constraints.txt index 781c9c3..4f2ae07 100644 --- a/python/serverless/serverless-v3/constraints.txt +++ b/python/serverless/serverless-v3/constraints.txt @@ -68,7 +68,6 @@ opentelemetry-api~=1.31.1 opentelemetry-sdk~=1.31.1 opentelemetry-semantic-conventions~=0.52b1 packaging~=24.1 -pandas~=1.5.3 parso~=0.8.3 pathspec~=0.10.3 patsy~=0.5.6 diff --git a/python/serverless/serverless-v3/pyproject.toml b/python/serverless/serverless-v3/pyproject.toml index 5826597..c398661 100644 --- a/python/serverless/serverless-v3/pyproject.toml +++ b/python/serverless/serverless-v3/pyproject.toml @@ -80,7 +80,6 @@ constraint-dependencies = [ "opentelemetry-sdk~=1.31.1", "opentelemetry-semantic-conventions~=0.52b1", "packaging~=24.1", - "pandas~=1.5.3", "parso~=0.8.3", "pathspec~=0.10.3", "patsy~=0.5.6",