Drop pandas on DBR 16.4 + serverless-v3 (no cp312 wheel)

.github/scripts/envgen.py

@@ -29,6 +29,10 @@
                          triton, flash-attn, deepspeed, horovod and pynvml — all need
                          a GPU (and CUDA/MPI toolchain) a dev machine lacks (see
                          DROP / DROP_PREFIX).
+  * Env-scoped drops   - a pin dropped for specific environments only (DROP_BY_ENV)
+                         when its version has no wheel for that env's Python and no
+                         in-range version does either — e.g. pandas 1.5.3 on the
+                         Python 3.12 runtimes DBR 16.4 / serverless-v3 (issue #18).
   * requires-python    - taken from the runtime's Python version (major.minor).
 
 This module is imported by ``sync.py`` (the weekly discovery + reconciliation Action).
@@ -58,6 +62,25 @@
     "cuda-",          # cuda-toolkit / cuda-bindings / cuda-pathfinder: CUDA tooling, GPU-only
 )
 
+# Environment-scoped drops: removed for the named environments only (DROP is global).
+# pandas 1.5.3 has no cp312 wheel — pandas ships CPython 3.12 wheels only from 2.1.1 —
+# so `uv sync` on a Python 3.12 runtime can't install it and falls back to a failing
+# source build. DBR 16.4 and serverless-v3 are the only 3.12 runtimes still pinned to
+# pandas 1.5.x: 13.3/14.3/15.4 predate 3.12, and 17.3+ / serverless-v4+ already ship
+# pandas 2.x (which has cp312 wheels). No in-range (~=1.5) version has a cp312 wheel,
+# so the pin can't be salvaged by widening — it is dropped for just these envs, and
+# pandas resolves to an installable version locally. See issue #18.
+#
+# Value = the version prefix that triggers the drop, so the exception is version-gated
+# and self-expiring: if one of these runtimes is ever re-pinned to a cp312-wheel 2.x,
+# the version no longer matches and the pin is kept rather than silently discarded.
+DROP_BY_ENV = {
+    "16.4.x-scala2.12": {"pandas": "1.5."},
+    "16.4.x-cpu-ml-scala2.12": {"pandas": "1.5."},
+    "16.4.x-gpu-ml-scala2.12": {"pandas": "1.5."},
+    "serverless-v3": {"pandas": "1.5."},
+}
+
 
 def norm(name):
     # Strip the '*' footnote marker the release-notes tables append to some package
@@ -106,12 +129,15 @@ def parse_requirements(text):
     return pkgs
 
 
-def _filtered(pkgs):
-    # Inclusion only: drop the name-based DROP set and DROP_PREFIX. A PEP 440 local
-    # version segment is NOT a reason to drop — its base release is on PyPI — so
-    # those pins are kept here and req() strips the segment when rendering.
+def _filtered(pkgs, env_name=None):
+    # Inclusion only: drop the global name-based DROP set and DROP_PREFIX, plus any
+    # environment-scoped drops (DROP_BY_ENV) for env_name. A PEP 440 local version
+    # segment is NOT a reason to drop — its base release is on PyPI — so those pins
+    # are kept here and req() strips the segment when rendering.
+    env_drop = DROP_BY_ENV.get(env_name, {})
     return {n: v for n, v in pkgs.items()
-            if n not in DROP and not n.startswith(DROP_PREFIX)}
+            if n not in DROP and not n.startswith(DROP_PREFIX)
+            and not (n in env_drop and v.startswith(env_drop[n]))}
 
 
 def dbconnect_pin(pkgs):
@@ -137,7 +163,7 @@ def build_pyproject(pkgs, env_name, python_version, dbconnect=None):
     not list it — the matching version is the runtime version, passed in explicitly.
     """
     mm = ".".join(python_version.split(".")[:2])     # 3.12.3 -> 3.12
-    body = {n: v for n, v in _filtered(pkgs).items() if n != "databricks-connect"}
+    body = {n: v for n, v in _filtered(pkgs, env_name).items() if n != "databricks-connect"}
     dev = f"databricks-connect~={dbconnect}.0" if dbconnect else dbconnect_pin(pkgs)
     project = "constraint-env-" + re.sub(r"[^a-z0-9]+", "-", env_name.lower()).strip("-")
     out = [
@@ -162,7 +188,7 @@ def build_pyproject(pkgs, env_name, python_version, dbconnect=None):
 
 
 def build_constraints(pkgs, env_name):
-    body = {n: v for n, v in _filtered(pkgs).items() if n != "databricks-connect"}
+    body = {n: v for n, v in _filtered(pkgs, env_name).items() if n != "databricks-connect"}
     out = [f"# constraints.txt file for Databricks {_label(env_name)}", ""]
     out += [req(n, body[n]) for n in sorted(body)]
     return "\n".join(out) + "\n"

.github/scripts/test_envgen.py

@@ -41,6 +41,34 @@ def test_keeps_installable_pins(self):
         self.assertEqual(_filtered(pkgs), pkgs)
 
 
+class EnvScopedDropTest(unittest.TestCase):
+    def test_pandas_dropped_on_py312_1_5_runtimes(self):
+        # DBR 16.4 + serverless-v3 are the only Python-3.12 runtimes still on
+        # pandas 1.5.x, which has no cp312 wheel — dropped for just these envs.
+        pkgs = {"pandas": "1.5.3", "numpy": "2.1.3"}
+        for env in (
+            "16.4.x-scala2.12",
+            "16.4.x-cpu-ml-scala2.12",
+            "16.4.x-gpu-ml-scala2.12",
+            "serverless-v3",
+        ):
+            self.assertEqual(_filtered(pkgs, env), {"numpy": "2.1.3"}, env)
+
+    def test_pandas_kept_elsewhere(self):
+        # Other runtimes keep pandas: pre-3.12 envs have wheels for 1.5.x, and
+        # 17.3+/serverless-v4+ already ship pandas 2.x. No env_name = no scoped drop.
+        pkgs = {"pandas": "1.5.3", "numpy": "2.1.3"}
+        for env in ("15.4.x-scala2.12", "17.3.x-scala2.13", "serverless-v4", None):
+            self.assertEqual(_filtered(pkgs, env), pkgs, env)
+
+    def test_pandas_kept_if_repinned_to_cp312_version(self):
+        # The drop is version-gated so it self-expires: if a maintenance update ever
+        # re-pins pandas to a cp312-wheel release (2.x) on these envs, the pin is kept,
+        # not silently dropped.
+        pkgs = {"pandas": "2.2.3", "numpy": "2.1.3"}
+        self.assertEqual(_filtered(pkgs, "16.4.x-scala2.12"), pkgs)
+
+
 class ReqTest(unittest.TestCase):
     def test_strips_local_version_segment(self):
         # ~= is invalid with a local segment (PEP 440), and the segment names a build

README.md

@@ -78,9 +78,18 @@ stay resolvable. Applied to **both** `pyproject.toml` and `constraints.txt`:
   → `flask~=1.1.2`), so `uv` resolves a platform-appropriate wheel. (Ubuntu system
   builds like `python-apt 2.7.7+ubuntu5.2` are dropped by name above instead, since
   their base version is not on PyPI.)
-
-The exact lists live in `DROP` / `DROP_PREFIX` and `_filtered()` / `req()` in
-`.github/scripts/envgen.py`.
+- **Environment-scoped drops** (dropped for named envs only) — a pin whose version
+  has no wheel for that environment's Python, where no in-range version has one
+  either. Today this is **`pandas` on DBR 16.4 and serverless-v3**: they are the only
+  Python-3.12 runtimes still pinned to `pandas 1.5.3`, which has no cp312 wheel (pandas
+  ships 3.12 wheels only from 2.1.1, so `~=1.5` can't reach one). The constraint is
+  dropped for just those envs, letting `pandas` resolve to an installable version
+  locally. Earlier runtimes (13.3/14.3/15.4) predate 3.12, and 17.3+ / serverless-v4+
+  already ship pandas 2.x. The drop is version-gated to the `1.5.x` pin, so if one of
+  these runtimes is ever re-pinned to a cp312-wheel 2.x it keeps its pin.
+
+The exact lists live in `DROP` / `DROP_PREFIX` / `DROP_BY_ENV` and `_filtered()` /
+`req()` in `.github/scripts/envgen.py`.
 
 ## How it stays in sync
 

python/dbr/16.4.x-cpu-ml-scala2.12/constraints.txt

@@ -225,7 +225,6 @@ optuna-integration~=3.6.0
 orjson~=3.10.16
 overrides~=7.4.0
 packaging~=24.1
-pandas~=1.5.3
 pandocfilters~=1.5.0
 paramiko~=3.4.0
 parso~=0.8.3

python/dbr/16.4.x-cpu-ml-scala2.12/pyproject.toml

@@ -237,7 +237,6 @@ constraint-dependencies = [
     "orjson~=3.10.16",
     "overrides~=7.4.0",
     "packaging~=24.1",
-    "pandas~=1.5.3",
     "pandocfilters~=1.5.0",
     "paramiko~=3.4.0",
     "parso~=0.8.3",

python/dbr/16.4.x-gpu-ml-scala2.12/constraints.txt

@@ -224,7 +224,6 @@ optuna-integration~=3.6.0
 orjson~=3.10.16
 overrides~=7.4.0
 packaging~=24.1
-pandas~=1.5.3
 pandocfilters~=1.5.0
 paramiko~=3.4.0
 parso~=0.8.3

python/dbr/16.4.x-gpu-ml-scala2.12/pyproject.toml

@@ -236,7 +236,6 @@ constraint-dependencies = [
     "orjson~=3.10.16",
     "overrides~=7.4.0",
     "packaging~=24.1",
-    "pandas~=1.5.3",
     "pandocfilters~=1.5.0",
     "paramiko~=3.4.0",
     "parso~=0.8.3",

python/dbr/16.4.x-scala2.12/constraints.txt

@@ -75,7 +75,6 @@ opentelemetry-api~=1.27.0
 opentelemetry-sdk~=1.27.0
 opentelemetry-semantic-conventions~=0.48b0
 packaging~=24.1
-pandas~=1.5.3
 parso~=0.8.3
 pathspec~=0.10.3
 patsy~=0.5.6

python/dbr/16.4.x-scala2.12/pyproject.toml

@@ -87,7 +87,6 @@ constraint-dependencies = [
     "opentelemetry-sdk~=1.27.0",
     "opentelemetry-semantic-conventions~=0.48b0",
     "packaging~=24.1",
-    "pandas~=1.5.3",
     "parso~=0.8.3",
     "pathspec~=0.10.3",
     "patsy~=0.5.6",

python/serverless/serverless-v3/constraints.txt

@@ -68,7 +68,6 @@ opentelemetry-api~=1.31.1
 opentelemetry-sdk~=1.31.1
 opentelemetry-semantic-conventions~=0.52b1
 packaging~=24.1
-pandas~=1.5.3
 parso~=0.8.3
 pathspec~=0.10.3
 patsy~=0.5.6

python/serverless/serverless-v3/pyproject.toml

@@ -80,7 +80,6 @@ constraint-dependencies = [
     "opentelemetry-sdk~=1.31.1",
     "opentelemetry-semantic-conventions~=0.52b1",
     "packaging~=24.1",
-    "pandas~=1.5.3",
     "parso~=0.8.3",
     "pathspec~=0.10.3",
     "patsy~=0.5.6",