diff --git a/.github/scripts/envgen.py b/.github/scripts/envgen.py index bf1ce7d..a246c8c 100644 --- a/.github/scripts/envgen.py +++ b/.github/scripts/envgen.py @@ -29,6 +29,10 @@ triton, flash-attn, deepspeed, horovod and pynvml — all need a GPU (and CUDA/MPI toolchain) a dev machine lacks (see DROP / DROP_PREFIX). + * Env-scoped drops - a pin dropped for specific environments only (DROP_BY_ENV) + when its version has no wheel for that env's Python and no + in-range version does either — e.g. pandas 1.5.3 on the + Python 3.12 runtimes DBR 16.4 / serverless-v3 (issue #18). * requires-python - taken from the runtime's Python version (major.minor). This module is imported by ``sync.py`` (the weekly discovery + reconciliation Action). @@ -58,6 +62,25 @@ "cuda-", # cuda-toolkit / cuda-bindings / cuda-pathfinder: CUDA tooling, GPU-only ) +# Environment-scoped drops: removed for the named environments only (DROP is global). +# pandas 1.5.3 has no cp312 wheel — pandas ships CPython 3.12 wheels only from 2.1.1 — +# so `uv sync` on a Python 3.12 runtime can't install it and falls back to a failing +# source build. DBR 16.4 and serverless-v3 are the only 3.12 runtimes still pinned to +# pandas 1.5.x: 13.3/14.3/15.4 predate 3.12, and 17.3+ / serverless-v4+ already ship +# pandas 2.x (which has cp312 wheels). No in-range (~=1.5) version has a cp312 wheel, +# so the pin can't be salvaged by widening — it is dropped for just these envs, and +# pandas resolves to an installable version locally. See issue #18. +# +# Value = the version prefix that triggers the drop, so the exception is version-gated +# and self-expiring: if one of these runtimes is ever re-pinned to a cp312-wheel 2.x, +# the version no longer matches and the pin is kept rather than silently discarded. +DROP_BY_ENV = { + "16.4.x-scala2.12": {"pandas": "1.5."}, + "16.4.x-cpu-ml-scala2.12": {"pandas": "1.5."}, + "16.4.x-gpu-ml-scala2.12": {"pandas": "1.5."}, + "serverless-v3": {"pandas": "1.5."}, +} + def norm(name): # Strip the '*' footnote marker the release-notes tables append to some package @@ -106,12 +129,15 @@ def parse_requirements(text): return pkgs -def _filtered(pkgs): - # Inclusion only: drop the name-based DROP set and DROP_PREFIX. A PEP 440 local - # version segment is NOT a reason to drop — its base release is on PyPI — so - # those pins are kept here and req() strips the segment when rendering. +def _filtered(pkgs, env_name=None): + # Inclusion only: drop the global name-based DROP set and DROP_PREFIX, plus any + # environment-scoped drops (DROP_BY_ENV) for env_name. A PEP 440 local version + # segment is NOT a reason to drop — its base release is on PyPI — so those pins + # are kept here and req() strips the segment when rendering. + env_drop = DROP_BY_ENV.get(env_name, {}) return {n: v for n, v in pkgs.items() - if n not in DROP and not n.startswith(DROP_PREFIX)} + if n not in DROP and not n.startswith(DROP_PREFIX) + and not (n in env_drop and v.startswith(env_drop[n]))} def dbconnect_pin(pkgs): @@ -137,7 +163,7 @@ def build_pyproject(pkgs, env_name, python_version, dbconnect=None): not list it — the matching version is the runtime version, passed in explicitly. """ mm = ".".join(python_version.split(".")[:2]) # 3.12.3 -> 3.12 - body = {n: v for n, v in _filtered(pkgs).items() if n != "databricks-connect"} + body = {n: v for n, v in _filtered(pkgs, env_name).items() if n != "databricks-connect"} dev = f"databricks-connect~={dbconnect}.0" if dbconnect else dbconnect_pin(pkgs) project = "constraint-env-" + re.sub(r"[^a-z0-9]+", "-", env_name.lower()).strip("-") out = [ @@ -162,7 +188,7 @@ def build_pyproject(pkgs, env_name, python_version, dbconnect=None): def build_constraints(pkgs, env_name): - body = {n: v for n, v in _filtered(pkgs).items() if n != "databricks-connect"} + body = {n: v for n, v in _filtered(pkgs, env_name).items() if n != "databricks-connect"} out = [f"# constraints.txt file for Databricks {_label(env_name)}", ""] out += [req(n, body[n]) for n in sorted(body)] return "\n".join(out) + "\n" diff --git a/.github/scripts/test_envgen.py b/.github/scripts/test_envgen.py index e061b2d..c5c1a51 100644 --- a/.github/scripts/test_envgen.py +++ b/.github/scripts/test_envgen.py @@ -41,6 +41,34 @@ def test_keeps_installable_pins(self): self.assertEqual(_filtered(pkgs), pkgs) +class EnvScopedDropTest(unittest.TestCase): + def test_pandas_dropped_on_py312_1_5_runtimes(self): + # DBR 16.4 + serverless-v3 are the only Python-3.12 runtimes still on + # pandas 1.5.x, which has no cp312 wheel — dropped for just these envs. + pkgs = {"pandas": "1.5.3", "numpy": "2.1.3"} + for env in ( + "16.4.x-scala2.12", + "16.4.x-cpu-ml-scala2.12", + "16.4.x-gpu-ml-scala2.12", + "serverless-v3", + ): + self.assertEqual(_filtered(pkgs, env), {"numpy": "2.1.3"}, env) + + def test_pandas_kept_elsewhere(self): + # Other runtimes keep pandas: pre-3.12 envs have wheels for 1.5.x, and + # 17.3+/serverless-v4+ already ship pandas 2.x. No env_name = no scoped drop. + pkgs = {"pandas": "1.5.3", "numpy": "2.1.3"} + for env in ("15.4.x-scala2.12", "17.3.x-scala2.13", "serverless-v4", None): + self.assertEqual(_filtered(pkgs, env), pkgs, env) + + def test_pandas_kept_if_repinned_to_cp312_version(self): + # The drop is version-gated so it self-expires: if a maintenance update ever + # re-pins pandas to a cp312-wheel release (2.x) on these envs, the pin is kept, + # not silently dropped. + pkgs = {"pandas": "2.2.3", "numpy": "2.1.3"} + self.assertEqual(_filtered(pkgs, "16.4.x-scala2.12"), pkgs) + + class ReqTest(unittest.TestCase): def test_strips_local_version_segment(self): # ~= is invalid with a local segment (PEP 440), and the segment names a build diff --git a/README.md b/README.md index ff8ce28..95e10d8 100644 --- a/README.md +++ b/README.md @@ -78,9 +78,18 @@ stay resolvable. Applied to **both** `pyproject.toml` and `constraints.txt`: → `flask~=1.1.2`), so `uv` resolves a platform-appropriate wheel. (Ubuntu system builds like `python-apt 2.7.7+ubuntu5.2` are dropped by name above instead, since their base version is not on PyPI.) - -The exact lists live in `DROP` / `DROP_PREFIX` and `_filtered()` / `req()` in -`.github/scripts/envgen.py`. +- **Environment-scoped drops** (dropped for named envs only) — a pin whose version + has no wheel for that environment's Python, where no in-range version has one + either. Today this is **`pandas` on DBR 16.4 and serverless-v3**: they are the only + Python-3.12 runtimes still pinned to `pandas 1.5.3`, which has no cp312 wheel (pandas + ships 3.12 wheels only from 2.1.1, so `~=1.5` can't reach one). The constraint is + dropped for just those envs, letting `pandas` resolve to an installable version + locally. Earlier runtimes (13.3/14.3/15.4) predate 3.12, and 17.3+ / serverless-v4+ + already ship pandas 2.x. The drop is version-gated to the `1.5.x` pin, so if one of + these runtimes is ever re-pinned to a cp312-wheel 2.x it keeps its pin. + +The exact lists live in `DROP` / `DROP_PREFIX` / `DROP_BY_ENV` and `_filtered()` / +`req()` in `.github/scripts/envgen.py`. ## How it stays in sync diff --git a/python/dbr/16.4.x-cpu-ml-scala2.12/constraints.txt b/python/dbr/16.4.x-cpu-ml-scala2.12/constraints.txt index 3fb1850..df0df03 100644 --- a/python/dbr/16.4.x-cpu-ml-scala2.12/constraints.txt +++ b/python/dbr/16.4.x-cpu-ml-scala2.12/constraints.txt @@ -225,7 +225,6 @@ optuna-integration~=3.6.0 orjson~=3.10.16 overrides~=7.4.0 packaging~=24.1 -pandas~=1.5.3 pandocfilters~=1.5.0 paramiko~=3.4.0 parso~=0.8.3 diff --git a/python/dbr/16.4.x-cpu-ml-scala2.12/pyproject.toml b/python/dbr/16.4.x-cpu-ml-scala2.12/pyproject.toml index 9bfae3e..ed296d7 100644 --- a/python/dbr/16.4.x-cpu-ml-scala2.12/pyproject.toml +++ b/python/dbr/16.4.x-cpu-ml-scala2.12/pyproject.toml @@ -237,7 +237,6 @@ constraint-dependencies = [ "orjson~=3.10.16", "overrides~=7.4.0", "packaging~=24.1", - "pandas~=1.5.3", "pandocfilters~=1.5.0", "paramiko~=3.4.0", "parso~=0.8.3", diff --git a/python/dbr/16.4.x-gpu-ml-scala2.12/constraints.txt b/python/dbr/16.4.x-gpu-ml-scala2.12/constraints.txt index 531d11a..8f19c26 100644 --- a/python/dbr/16.4.x-gpu-ml-scala2.12/constraints.txt +++ b/python/dbr/16.4.x-gpu-ml-scala2.12/constraints.txt @@ -224,7 +224,6 @@ optuna-integration~=3.6.0 orjson~=3.10.16 overrides~=7.4.0 packaging~=24.1 -pandas~=1.5.3 pandocfilters~=1.5.0 paramiko~=3.4.0 parso~=0.8.3 diff --git a/python/dbr/16.4.x-gpu-ml-scala2.12/pyproject.toml b/python/dbr/16.4.x-gpu-ml-scala2.12/pyproject.toml index 903c296..eddfcda 100644 --- a/python/dbr/16.4.x-gpu-ml-scala2.12/pyproject.toml +++ b/python/dbr/16.4.x-gpu-ml-scala2.12/pyproject.toml @@ -236,7 +236,6 @@ constraint-dependencies = [ "orjson~=3.10.16", "overrides~=7.4.0", "packaging~=24.1", - "pandas~=1.5.3", "pandocfilters~=1.5.0", "paramiko~=3.4.0", "parso~=0.8.3", diff --git a/python/dbr/16.4.x-scala2.12/constraints.txt b/python/dbr/16.4.x-scala2.12/constraints.txt index 7bf6bdd..6819e8f 100644 --- a/python/dbr/16.4.x-scala2.12/constraints.txt +++ b/python/dbr/16.4.x-scala2.12/constraints.txt @@ -75,7 +75,6 @@ opentelemetry-api~=1.27.0 opentelemetry-sdk~=1.27.0 opentelemetry-semantic-conventions~=0.48b0 packaging~=24.1 -pandas~=1.5.3 parso~=0.8.3 pathspec~=0.10.3 patsy~=0.5.6 diff --git a/python/dbr/16.4.x-scala2.12/pyproject.toml b/python/dbr/16.4.x-scala2.12/pyproject.toml index 500b786..2c06589 100644 --- a/python/dbr/16.4.x-scala2.12/pyproject.toml +++ b/python/dbr/16.4.x-scala2.12/pyproject.toml @@ -87,7 +87,6 @@ constraint-dependencies = [ "opentelemetry-sdk~=1.27.0", "opentelemetry-semantic-conventions~=0.48b0", "packaging~=24.1", - "pandas~=1.5.3", "parso~=0.8.3", "pathspec~=0.10.3", "patsy~=0.5.6", diff --git a/python/serverless/serverless-v3/constraints.txt b/python/serverless/serverless-v3/constraints.txt index 781c9c3..4f2ae07 100644 --- a/python/serverless/serverless-v3/constraints.txt +++ b/python/serverless/serverless-v3/constraints.txt @@ -68,7 +68,6 @@ opentelemetry-api~=1.31.1 opentelemetry-sdk~=1.31.1 opentelemetry-semantic-conventions~=0.52b1 packaging~=24.1 -pandas~=1.5.3 parso~=0.8.3 pathspec~=0.10.3 patsy~=0.5.6 diff --git a/python/serverless/serverless-v3/pyproject.toml b/python/serverless/serverless-v3/pyproject.toml index 5826597..c398661 100644 --- a/python/serverless/serverless-v3/pyproject.toml +++ b/python/serverless/serverless-v3/pyproject.toml @@ -80,7 +80,6 @@ constraint-dependencies = [ "opentelemetry-sdk~=1.31.1", "opentelemetry-semantic-conventions~=0.52b1", "packaging~=24.1", - "pandas~=1.5.3", "parso~=0.8.3", "pathspec~=0.10.3", "patsy~=0.5.6",