diff --git a/.github/scripts/envgen.py b/.github/scripts/envgen.py index 352802b..bf1ce7d 100644 --- a/.github/scripts/envgen.py +++ b/.github/scripts/envgen.py @@ -20,6 +20,15 @@ packages dropped be pip-installed locally or that ship vendored inside setuptools (see DROP / DROP_PREFIX). py4j is kept; pyspark is dropped so DB Connect supplies its own bundled build. + * Local segments - a PEP 440 local version segment (``+cu129`` / ``+cpu`` / + stripped ``+db1``) is stripped and the base release pinned + (``torch 2.9.0+cu129`` -> ``torch~=2.9.0``); ``~=`` is + invalid with a local segment and the build exists only + off-index, while its base is an ordinary PyPI release. See ``req``. + * GPU-only dropped - ``nvidia-*`` / ``cuda-*`` CUDA wheels and tooling, plus + triton, flash-attn, deepspeed, horovod and pynvml — all need + a GPU (and CUDA/MPI toolchain) a dev machine lacks (see + DROP / DROP_PREFIX). * requires-python - taken from the runtime's Python version (major.minor). This module is imported by ``sync.py`` (the weekly discovery + reconciliation Action). @@ -30,11 +39,24 @@ # libs, the spark client, pip itself, and deps vendored inside setuptools. DROP = { "pyspark", "dbus-python", "pygobject", "pip", "unattended-upgrades", + # Ubuntu system packages carried in the image but not pip-installable. Named + # explicitly so they stay dropped even though req() now strips local segments + # (their +ubuntu / +build local builds would otherwise resurrect as a base pin). + "python-apt", "distro-info", # setuptools-vendored "autocommand", "inflect", "typeguard", "backports-tarfile", "importlib-resources", "more-itertools", + # GPU-only: need an NVIDIA GPU + CUDA toolchain a dev machine does not have. + # (nvidia-* / cuda-* are dropped by prefix below.) horovod is the same class — a + # source-only distribution needing MPI/NCCL + a compiler to build. pynvml is the + # NVML binding (same as nvidia-ml-py); useless without a driver. + "triton", "flash-attn", "deepspeed", "horovod", "pynvml", } -DROP_PREFIX = ("jaraco-",) # jaraco.collections / jaraco.context / ... +DROP_PREFIX = ( + "jaraco-", # jaraco.collections / jaraco.context / ... (setuptools-vendored) + "nvidia-", # nvidia-* CUDA wheels (and nvidia-ml-py): GPU tooling, no local use + "cuda-", # cuda-toolkit / cuda-bindings / cuda-pathfinder: CUDA tooling, GPU-only +) def norm(name): @@ -44,10 +66,15 @@ def norm(name): def req(name, version): - """Render one requirement. Compatible-release ``~=`` allows patch bumps, but it - is invalid with a local version segment (PEP 440), and a local build like - ``+cpu`` / ``+cu118`` / ``+db1`` is exactly what distinguishes CPU vs GPU ML - images and Databricks-patched packages — so those are pinned exactly with ``==``. + """Render one requirement. Compatible-release ``~=`` allows patch bumps. + + A PEP 440 local version segment (``+cpu`` / ``+cu118`` / ``+db1``) is stripped: + ``~=`` is invalid with a local segment, and the segment names a build published + only off-index (``download.pytorch.org``) or rebuilt inside the image, while its + base release is an ordinary PyPI version. So the base is pinned instead + (``torch 2.9.0+cu129`` -> ``torch~=2.9.0``, ``flask 1.1.2+db1`` -> ``flask~=1.1.2``). + Ubuntu system builds like ``python-apt 2.7.7+ubuntu5.2`` would strip to a bogus + PyPI pin, so those are dropped by name in ``DROP`` before reaching here. ``databricks-sdk`` is a special case. It moves in lockstep with ``databricks-connect``, which is installed from PyPI in the dev group and declares @@ -60,8 +87,7 @@ def req(name, version): floor while letting databricks-connect's own metadata govern the exact version within that window. See issue #16. """ - if "+" in version: - return f"{name}=={version}" + version = version.split("+", 1)[0] if name == "databricks-sdk": version = ".".join(version.split(".")[:2]) return f"{name}~={version}" @@ -81,6 +107,9 @@ def parse_requirements(text): def _filtered(pkgs): + # Inclusion only: drop the name-based DROP set and DROP_PREFIX. A PEP 440 local + # version segment is NOT a reason to drop — its base release is on PyPI — so + # those pins are kept here and req() strips the segment when rendering. return {n: v for n, v in pkgs.items() if n not in DROP and not n.startswith(DROP_PREFIX)} diff --git a/.github/scripts/test_envgen.py b/.github/scripts/test_envgen.py new file mode 100644 index 0000000..e061b2d --- /dev/null +++ b/.github/scripts/test_envgen.py @@ -0,0 +1,107 @@ +"""Unit tests for envgen.py (run: python -m unittest test_envgen).""" +import unittest + +from envgen import _filtered, build_constraints, build_pyproject, dbconnect_pin, req + + +class FilterTest(unittest.TestCase): + def test_keeps_local_version_packages(self): + # A PEP 440 local version segment (+cu129 / +cpu / +db1) is NOT a reason to + # drop: the base release is on PyPI, so the pin is kept and the segment is + # stripped later by req(). Only GPU-only and system packages are dropped. + pkgs = {"torch": "2.9.0+cu129", "flask": "1.1.2+db1", "numpy": "2.1.3"} + self.assertEqual(_filtered(pkgs), pkgs) + + def test_drops_gpu_only_by_name(self): + # Every nvidia-* distribution is a CUDA runtime component; triton / + # flash-attn / deepspeed are GPU-only as well. _filtered receives keys already + # PEP 503-normalized by parse_requirements/norm, so it compares lowercased. + pkgs = { + "nvidia-cublas-cu12": "12.6.4.1", + "nvidia-cudnn-cu12": "9.5.1.17", + "triton": "3.3.0", + "flash-attn": "2.7.4.post1", + "deepspeed": "0.16.5", + "horovod": "0.28.1", + "pynvml": "11.5.0", + "cuda-toolkit": "13.0.2", + "cuda-bindings": "13.2.0", + "numpy": "2.1.3", + } + self.assertEqual(_filtered(pkgs), {"numpy": "2.1.3"}) + + def test_drops_system_packages(self): + # Ubuntu system packages carried in the image but not pip-installable. They + # must be dropped by name so stripping local segments does not resurrect them. + pkgs = {"python-apt": "2.7.7+ubuntu5.2", "distro-info": "1.7+build1", "numpy": "2.1.3"} + self.assertEqual(_filtered(pkgs), {"numpy": "2.1.3"}) + + def test_keeps_installable_pins(self): + pkgs = {"ray": "2.37.0", "databricks-sdk": "0.67.0", "numpy": "2.1.3", "pyarrow": "21.0.0"} + self.assertEqual(_filtered(pkgs), pkgs) + + +class ReqTest(unittest.TestCase): + def test_strips_local_version_segment(self): + # ~= is invalid with a local segment (PEP 440), and the segment names a build + # that only exists off-index; strip it so the base release is pinned instead. + self.assertEqual(req("torch", "2.9.0+cu129"), "torch~=2.9.0") + self.assertEqual(req("torch", "2.7.0+cpu"), "torch~=2.7.0") + self.assertEqual(req("flask", "1.1.2+db1"), "flask~=1.1.2") + + def test_compatible_release_default(self): + self.assertEqual(req("numpy", "2.1.3"), "numpy~=2.1.3") + + def test_databricks_sdk_widened_to_major_minor(self): + self.assertEqual(req("databricks-sdk", "0.67.0"), "databricks-sdk~=0.67") + + +class BuildArtifactsTest(unittest.TestCase): + # torchmetrics is a real ML package that must survive; it also guards against a + # bare-substring assertion mistaking "torch~=..." for "torchmetrics". + pkgs = { + "numpy": "2.1.3", + "torch": "2.9.0+cu129", + "torchmetrics": "1.6.0", + "nvidia-cublas-cu12": "12.6.4.1", + "triton": "3.3.0", + "python-apt": "2.7.7+ubuntu5.2", + "pyarrow": "21.0.0", + } + + def _check(self, out): + # +local stripped and pinned; ordinary pins kept. + self.assertIn("torch~=2.9.0", out) + self.assertIn("torchmetrics~=1.6.0", out) + self.assertIn("numpy~=2.1.3", out) + self.assertIn("pyarrow~=21.0.0", out) + self.assertNotIn("+cu129", out) + # Dropped by name — assert on the rendered pin, not a bare substring. + self.assertNotIn("nvidia-cublas-cu12~=", out) + self.assertNotIn("triton~=", out) + self.assertNotIn("python-apt~=", out) + + def test_pyproject(self): + self._check(build_pyproject(self.pkgs, "serverless-v4", "3.12.3")) + + def test_constraints(self): + self._check(build_constraints(self.pkgs, "serverless-v4")) + + +class DbconnectPinTest(unittest.TestCase): + def test_strips_local_version_segment(self): + # databricks-connect is installed from the dev group as a plain PyPI release; + # the pin is normalized to ~=MAJOR.0, so a local segment in the release-notes + # version is discarded and never lands in an artifact. dbconnect_pin reads raw + # pkgs (not _filtered), so this guards that the normalization does the stripping. + self.assertEqual( + dbconnect_pin({"databricks-connect": "17.3.1+db1"}), + "databricks-connect~=17.0", + ) + + def test_none_when_absent(self): + self.assertIsNone(dbconnect_pin({"numpy": "2.1.3"})) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..22bd2c4 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,34 @@ +name: test-envgen + +# Run the envgen unit tests so the artifact-transformation rules (drops, +local +# stripping, databricks-sdk / databricks-connect pinning) can't silently regress. +# envgen.py has no third-party dependencies, so this is a plain stdlib unittest run. + +on: + push: + paths: + - ".github/scripts/**" + - ".github/workflows/test.yml" + pull_request: + paths: + - ".github/scripts/**" + - ".github/workflows/test.yml" + +permissions: + contents: read + +jobs: + unittest: + # See sync.yml: GitHub-hosted ubuntu-latest is outside the databricks org IP + # allow list; linux-ubuntu-latest is the databricks-protected-runner-group. + runs-on: linux-ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" + + - name: Run envgen unit tests + run: python -m unittest discover -s .github/scripts -p "test_*.py" -v diff --git a/README.md b/README.md index 44d01d7..ff8ce28 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,36 @@ serverless), so resolving a target to its artifact is a deterministic lookup. so the pip path is constraints-only unless DB Connect is installed explicitly. Both are a mechanical transform of the official package list published in the -Databricks release notes — see `.github/scripts/envgen.py` for the rules. +Databricks release notes — see [what is intentionally not included](#what-is-intentionally-not-included) and +`.github/scripts/envgen.py` for the rules. + +## What is intentionally not included + +Not every package in the release-notes list is emitted verbatim. `envgen.py` drops +the ones that can't — or shouldn't — install on a developer machine, and strips +version markers that would make a pin unresolvable, so `uv sync` / `pip install -c` +stay resolvable. Applied to **both** `pyproject.toml` and `constraints.txt`: + +- **System / OS packages** (dropped) — `pip`, `pyspark` (DB Connect supplies its own + bundled build; `py4j` is kept), `dbus-python`, `pygobject`, `unattended-upgrades`, + `python-apt`, `distro-info`. +- **setuptools-vendored** (dropped) — `more-itertools`, `jaraco-*`, `inflect`, + `typeguard`, … — shipped inside setuptools, not installed standalone. +- **GPU-only distributions** (dropped) — the `nvidia-*` and `cuda-*` CUDA runtime + components and tooling, plus `triton`, `flash-attn`, `deepspeed`, `horovod`, + `pynvml`: they need an NVIDIA GPU (and a CUDA toolchain / MPI to build) a dev machine + lacks. (`nvidia-ml-py` and `pynvml` are pure-Python NVML bindings — installable + anywhere but useless without a driver, so dropped by prefix/name.) +- **Local version segments** (stripped, not dropped) — a `+cpu` / `+cuXXX` / `+db1` + segment names a build published only off-index (`download.pytorch.org`) or rebuilt + inside the image, and `~=` is invalid with a local segment. The segment is stripped + and the base release pinned (`torch 2.9.0+cu129` → `torch~=2.9.0`, `flask 1.1.2+db1` + → `flask~=1.1.2`), so `uv` resolves a platform-appropriate wheel. (Ubuntu system + builds like `python-apt 2.7.7+ubuntu5.2` are dropped by name above instead, since + their base version is not on PyPI.) + +The exact lists live in `DROP` / `DROP_PREFIX` and `_filtered()` / `req()` in +`.github/scripts/envgen.py`. ## How it stays in sync @@ -77,9 +106,10 @@ is best-effort. Nobody hand-edits the `python/` artifacts. - **DBR ML (CPU + GPU)** — for each `*-ml` runtime, a separate environment is produced per cluster type: `.x-cpu-ml-…` and `.x-gpu-ml-…`. Newer ML pages link downloadable `requirements-{cpu,gpu}-*.txt`; older ones render inline tables under - `python-libraries-on-{cpu,gpu}-clusters`. The GPU set carries the CUDA builds - (e.g. `torch==…+cu118`); the CPU set carries `…+cpu`. Local builds are pinned with - `==` (compatible-release `~=` is invalid with a `+local` segment). + `python-libraries-on-{cpu,gpu}-clusters`. The GPU set lists the CUDA builds + (e.g. `torch …+cu118`) and the CPU set lists `…+cpu`; the generated artifacts strip + the `+local` segment and pin the base release (see + [what is intentionally not included](#what-is-intentionally-not-included)). The Action runs it; you only need to run it locally to debug: @@ -104,5 +134,7 @@ doc for the full rationale. - [x] Serverless (v1–vN) — auto-discovered + synced; ML base environment (`-ml`) when published (v5+) - [x] DBR standard runtimes — auto-discovered from the index + HTML-table parsing - [x] DBR ML runtimes (CPU + GPU) — downloadable requirements or inline tables -- [ ] PyTorch index config in ML `pyproject.toml` (so `uv` fetches the matching - `+cpu` / `+cuXXX` torch build, not just pins it) +- [ ] PyTorch index config in ML `pyproject.toml`. Today the `+cpu` / `+cuXXX` + torch/torchvision builds are stripped to a base pin (see [what is intentionally not included](#what-is-intentionally-not-included)); + adding PyTorch's index would let `uv` fetch the exact `+cpu` / `+cuXXX` build the + runtime ships, rather than a base-version wheel. diff --git a/python/dbr/13.3.x-cpu-ml-scala2.12/constraints.txt b/python/dbr/13.3.x-cpu-ml-scala2.12/constraints.txt index 756d6b9..631135c 100644 --- a/python/dbr/13.3.x-cpu-ml-scala2.12/constraints.txt +++ b/python/dbr/13.3.x-cpu-ml-scala2.12/constraints.txt @@ -68,7 +68,7 @@ fastapi~=0.98.0 fastjsonschema~=2.18.0 fasttext~=0.9.2 filelock~=3.6.0 -flask==1.1.2+db1 +flask~=1.1.2 flatbuffers~=23.5.26 fonttools~=4.25.0 frozenlist~=1.4.0 @@ -94,7 +94,6 @@ gviz-api~=1.10.0 h11~=0.14.0 h5py~=3.7.0 holidays~=0.27.1 -horovod~=0.28.1 htmlmin~=0.1.12 httplib2~=0.20.2 httptools~=0.6.0 @@ -276,8 +275,8 @@ tiktoken~=0.4.0 tokenize-rt~=4.2.1 tokenizers~=0.13.3 tomli~=2.0.1 -torch==1.13.1+cpu -torchvision==0.14.1+cpu +torch~=1.13.1 +torchvision~=0.14.1 tornado~=6.1 tqdm~=4.64.1 traitlets~=5.1.1 diff --git a/python/dbr/13.3.x-cpu-ml-scala2.12/pyproject.toml b/python/dbr/13.3.x-cpu-ml-scala2.12/pyproject.toml index 1fb4a1a..bb380d7 100644 --- a/python/dbr/13.3.x-cpu-ml-scala2.12/pyproject.toml +++ b/python/dbr/13.3.x-cpu-ml-scala2.12/pyproject.toml @@ -80,7 +80,7 @@ constraint-dependencies = [ "fastjsonschema~=2.18.0", "fasttext~=0.9.2", "filelock~=3.6.0", - "flask==1.1.2+db1", + "flask~=1.1.2", "flatbuffers~=23.5.26", "fonttools~=4.25.0", "frozenlist~=1.4.0", @@ -106,7 +106,6 @@ constraint-dependencies = [ "h11~=0.14.0", "h5py~=3.7.0", "holidays~=0.27.1", - "horovod~=0.28.1", "htmlmin~=0.1.12", "httplib2~=0.20.2", "httptools~=0.6.0", @@ -288,8 +287,8 @@ constraint-dependencies = [ "tokenize-rt~=4.2.1", "tokenizers~=0.13.3", "tomli~=2.0.1", - "torch==1.13.1+cpu", - "torchvision==0.14.1+cpu", + "torch~=1.13.1", + "torchvision~=0.14.1", "tornado~=6.1", "tqdm~=4.64.1", "traitlets~=5.1.1", diff --git a/python/dbr/13.3.x-gpu-ml-scala2.12/constraints.txt b/python/dbr/13.3.x-gpu-ml-scala2.12/constraints.txt index 3b8ec36..c81a910 100644 --- a/python/dbr/13.3.x-gpu-ml-scala2.12/constraints.txt +++ b/python/dbr/13.3.x-gpu-ml-scala2.12/constraints.txt @@ -69,8 +69,7 @@ fastapi~=0.98.0 fastjsonschema~=2.18.0 fasttext~=0.9.2 filelock~=3.6.0 -flash-attn~=1.0.7 -flask==1.1.2+db1 +flask~=1.1.2 flatbuffers~=23.5.26 fonttools~=4.25.0 frozenlist~=1.4.0 @@ -96,7 +95,6 @@ gviz-api~=1.10.0 h11~=0.14.0 h5py~=3.7.0 holidays~=0.27.1 -horovod~=0.28.1 htmlmin~=0.1.12 httplib2~=0.20.2 httptools~=0.6.0 @@ -277,8 +275,8 @@ tiktoken~=0.4.0 tokenize-rt~=4.2.1 tokenizers~=0.13.3 tomli~=2.0.1 -torch==1.13.1+cu117 -torchvision==0.14.1+cu117 +torch~=1.13.1 +torchvision~=0.14.1 tornado~=6.1 tqdm~=4.64.1 traitlets~=5.1.1 diff --git a/python/dbr/13.3.x-gpu-ml-scala2.12/pyproject.toml b/python/dbr/13.3.x-gpu-ml-scala2.12/pyproject.toml index d359c2b..b9ce233 100644 --- a/python/dbr/13.3.x-gpu-ml-scala2.12/pyproject.toml +++ b/python/dbr/13.3.x-gpu-ml-scala2.12/pyproject.toml @@ -81,8 +81,7 @@ constraint-dependencies = [ "fastjsonschema~=2.18.0", "fasttext~=0.9.2", "filelock~=3.6.0", - "flash-attn~=1.0.7", - "flask==1.1.2+db1", + "flask~=1.1.2", "flatbuffers~=23.5.26", "fonttools~=4.25.0", "frozenlist~=1.4.0", @@ -108,7 +107,6 @@ constraint-dependencies = [ "h11~=0.14.0", "h5py~=3.7.0", "holidays~=0.27.1", - "horovod~=0.28.1", "htmlmin~=0.1.12", "httplib2~=0.20.2", "httptools~=0.6.0", @@ -289,8 +287,8 @@ constraint-dependencies = [ "tokenize-rt~=4.2.1", "tokenizers~=0.13.3", "tomli~=2.0.1", - "torch==1.13.1+cu117", - "torchvision==0.14.1+cu117", + "torch~=1.13.1", + "torchvision~=0.14.1", "tornado~=6.1", "tqdm~=4.64.1", "traitlets~=5.1.1", diff --git a/python/dbr/14.3.x-cpu-ml-scala2.12/constraints.txt b/python/dbr/14.3.x-cpu-ml-scala2.12/constraints.txt index 95e1ec7..0ea3a70 100644 --- a/python/dbr/14.3.x-cpu-ml-scala2.12/constraints.txt +++ b/python/dbr/14.3.x-cpu-ml-scala2.12/constraints.txt @@ -56,7 +56,6 @@ datasets~=2.15.0 dbl-tempo~=0.1.26 debugpy~=1.6.7 decorator~=5.1.1 -deepspeed~=0.12.4 defusedxml~=0.7.1 dill~=0.3.6 diskcache~=5.6.3 @@ -95,7 +94,6 @@ gviz-api~=1.10.0 h5py~=3.7.0 hjson~=3.1.0 holidays~=0.38 -horovod~=0.28.1 htmlmin~=0.1.12 httplib2~=0.20.2 huggingface-hub~=0.19.4 @@ -210,7 +208,6 @@ pyflakes~=3.1.0 pygments~=2.11.2 pyjwt~=2.3.0 pynacl~=1.5.0 -pynvml~=11.5.0 pyodbc~=4.0.32 pyparsing~=3.0.9 pyright~=1.1.294 @@ -281,8 +278,8 @@ tinycss2~=1.2.1 tokenize-rt~=4.2.1 tokenizers~=0.15.0 tomli~=2.0.1 -torch==2.0.1+cpu -torchvision==0.15.2+cpu +torch~=2.0.1 +torchvision~=0.15.2 tornado~=6.1 tqdm~=4.64.1 traitlets~=5.7.1 diff --git a/python/dbr/14.3.x-cpu-ml-scala2.12/pyproject.toml b/python/dbr/14.3.x-cpu-ml-scala2.12/pyproject.toml index 239aeb1..9e0cb1d 100644 --- a/python/dbr/14.3.x-cpu-ml-scala2.12/pyproject.toml +++ b/python/dbr/14.3.x-cpu-ml-scala2.12/pyproject.toml @@ -68,7 +68,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.6.7", "decorator~=5.1.1", - "deepspeed~=0.12.4", "defusedxml~=0.7.1", "dill~=0.3.6", "diskcache~=5.6.3", @@ -107,7 +106,6 @@ constraint-dependencies = [ "h5py~=3.7.0", "hjson~=3.1.0", "holidays~=0.38", - "horovod~=0.28.1", "htmlmin~=0.1.12", "httplib2~=0.20.2", "huggingface-hub~=0.19.4", @@ -222,7 +220,6 @@ constraint-dependencies = [ "pygments~=2.11.2", "pyjwt~=2.3.0", "pynacl~=1.5.0", - "pynvml~=11.5.0", "pyodbc~=4.0.32", "pyparsing~=3.0.9", "pyright~=1.1.294", @@ -293,8 +290,8 @@ constraint-dependencies = [ "tokenize-rt~=4.2.1", "tokenizers~=0.15.0", "tomli~=2.0.1", - "torch==2.0.1+cpu", - "torchvision==0.15.2+cpu", + "torch~=2.0.1", + "torchvision~=0.15.2", "tornado~=6.1", "tqdm~=4.64.1", "traitlets~=5.7.1", diff --git a/python/dbr/14.3.x-gpu-ml-scala2.12/constraints.txt b/python/dbr/14.3.x-gpu-ml-scala2.12/constraints.txt index 1010196..002b3fd 100644 --- a/python/dbr/14.3.x-gpu-ml-scala2.12/constraints.txt +++ b/python/dbr/14.3.x-gpu-ml-scala2.12/constraints.txt @@ -57,7 +57,6 @@ datasets~=2.15.0 dbl-tempo~=0.1.26 debugpy~=1.6.7 decorator~=5.1.1 -deepspeed~=0.12.4 defusedxml~=0.7.1 dill~=0.3.6 diskcache~=5.6.3 @@ -71,7 +70,6 @@ facets-overview~=1.1.1 fastjsonschema~=2.19.1 fasttext~=0.9.2 filelock~=3.9.0 -flash-attn~=2.3.6 flask~=2.2.5 flatbuffers~=23.5.26 fonttools~=4.25.0 @@ -98,7 +96,6 @@ gviz-api~=1.10.0 h5py~=3.7.0 hjson~=3.1.0 holidays~=0.38 -horovod~=0.28.1 htmlmin~=0.1.12 httplib2~=0.20.2 huggingface-hub~=0.19.4 @@ -213,7 +210,6 @@ pyflakes~=3.1.0 pygments~=2.11.2 pyjwt~=2.3.0 pynacl~=1.5.0 -pynvml~=11.5.0 pyodbc~=4.0.32 pyparsing~=3.0.9 pyright~=1.1.294 @@ -284,13 +280,12 @@ tinycss2~=1.2.1 tokenize-rt~=4.2.1 tokenizers~=0.15.0 tomli~=2.0.1 -torch==2.0.1+cu118 -torchvision==0.15.2+cu118 +torch~=2.0.1 +torchvision~=0.15.2 tornado~=6.1 tqdm~=4.64.1 traitlets~=5.7.1 transformers~=4.36.1 -triton~=2.0.0 typer~=0.9.0 typing-extensions~=4.4.0 typing-inspect~=0.9.0 diff --git a/python/dbr/14.3.x-gpu-ml-scala2.12/pyproject.toml b/python/dbr/14.3.x-gpu-ml-scala2.12/pyproject.toml index 33292db..f6e8c3a 100644 --- a/python/dbr/14.3.x-gpu-ml-scala2.12/pyproject.toml +++ b/python/dbr/14.3.x-gpu-ml-scala2.12/pyproject.toml @@ -69,7 +69,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.6.7", "decorator~=5.1.1", - "deepspeed~=0.12.4", "defusedxml~=0.7.1", "dill~=0.3.6", "diskcache~=5.6.3", @@ -83,7 +82,6 @@ constraint-dependencies = [ "fastjsonschema~=2.19.1", "fasttext~=0.9.2", "filelock~=3.9.0", - "flash-attn~=2.3.6", "flask~=2.2.5", "flatbuffers~=23.5.26", "fonttools~=4.25.0", @@ -110,7 +108,6 @@ constraint-dependencies = [ "h5py~=3.7.0", "hjson~=3.1.0", "holidays~=0.38", - "horovod~=0.28.1", "htmlmin~=0.1.12", "httplib2~=0.20.2", "huggingface-hub~=0.19.4", @@ -225,7 +222,6 @@ constraint-dependencies = [ "pygments~=2.11.2", "pyjwt~=2.3.0", "pynacl~=1.5.0", - "pynvml~=11.5.0", "pyodbc~=4.0.32", "pyparsing~=3.0.9", "pyright~=1.1.294", @@ -296,13 +292,12 @@ constraint-dependencies = [ "tokenize-rt~=4.2.1", "tokenizers~=0.15.0", "tomli~=2.0.1", - "torch==2.0.1+cu118", - "torchvision==0.15.2+cu118", + "torch~=2.0.1", + "torchvision~=0.15.2", "tornado~=6.1", "tqdm~=4.64.1", "traitlets~=5.7.1", "transformers~=4.36.1", - "triton~=2.0.0", "typer~=0.9.0", "typing-extensions~=4.4.0", "typing-inspect~=0.9.0", diff --git a/python/dbr/15.4.x-cpu-ml-scala2.12/constraints.txt b/python/dbr/15.4.x-cpu-ml-scala2.12/constraints.txt index 06328a5..08a7030 100644 --- a/python/dbr/15.4.x-cpu-ml-scala2.12/constraints.txt +++ b/python/dbr/15.4.x-cpu-ml-scala2.12/constraints.txt @@ -59,7 +59,6 @@ datasets~=2.19.1 dbl-tempo~=0.1.26 debugpy~=1.6.7 decorator~=5.1.1 -deepspeed~=0.14.4 defusedxml~=0.7.1 deprecated~=1.2.14 dill~=0.3.6 @@ -102,7 +101,6 @@ h11~=0.14.0 h5py~=3.10.0 hjson~=3.1.0 holidays~=0.45 -horovod==0.28.1+db1 htmlmin~=0.1.12 httpcore~=1.0.5 httplib2~=0.20.2 @@ -192,7 +190,6 @@ notebook~=6.5.4 notebook-shim~=0.2.2 numba~=0.57.1 numpy~=1.23.5 -nvidia-ml-py~=12.555.43 oauthlib~=3.2.0 oci~=2.126.4 openai~=1.35.3 @@ -316,9 +313,9 @@ tiktoken~=0.5.2 tinycss2~=1.2.1 tokenize-rt~=4.2.1 tokenizers~=0.19.0 -torch==2.3.1+cpu +torch~=2.3.1 torcheval~=0.0.7 -torchvision==0.18.1+cpu +torchvision~=0.18.1 tornado~=6.3.2 tqdm~=4.65.0 traitlets~=5.7.1 diff --git a/python/dbr/15.4.x-cpu-ml-scala2.12/pyproject.toml b/python/dbr/15.4.x-cpu-ml-scala2.12/pyproject.toml index 5ad236b..e85e484 100644 --- a/python/dbr/15.4.x-cpu-ml-scala2.12/pyproject.toml +++ b/python/dbr/15.4.x-cpu-ml-scala2.12/pyproject.toml @@ -71,7 +71,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.6.7", "decorator~=5.1.1", - "deepspeed~=0.14.4", "defusedxml~=0.7.1", "deprecated~=1.2.14", "dill~=0.3.6", @@ -114,7 +113,6 @@ constraint-dependencies = [ "h5py~=3.10.0", "hjson~=3.1.0", "holidays~=0.45", - "horovod==0.28.1+db1", "htmlmin~=0.1.12", "httpcore~=1.0.5", "httplib2~=0.20.2", @@ -204,7 +202,6 @@ constraint-dependencies = [ "notebook-shim~=0.2.2", "numba~=0.57.1", "numpy~=1.23.5", - "nvidia-ml-py~=12.555.43", "oauthlib~=3.2.0", "oci~=2.126.4", "openai~=1.35.3", @@ -328,9 +325,9 @@ constraint-dependencies = [ "tinycss2~=1.2.1", "tokenize-rt~=4.2.1", "tokenizers~=0.19.0", - "torch==2.3.1+cpu", + "torch~=2.3.1", "torcheval~=0.0.7", - "torchvision==0.18.1+cpu", + "torchvision~=0.18.1", "tornado~=6.3.2", "tqdm~=4.65.0", "traitlets~=5.7.1", diff --git a/python/dbr/15.4.x-gpu-ml-scala2.12/constraints.txt b/python/dbr/15.4.x-gpu-ml-scala2.12/constraints.txt index 633fc25..84c8388 100644 --- a/python/dbr/15.4.x-gpu-ml-scala2.12/constraints.txt +++ b/python/dbr/15.4.x-gpu-ml-scala2.12/constraints.txt @@ -59,7 +59,6 @@ datasets~=2.19.1 dbl-tempo~=0.1.26 debugpy~=1.6.7 decorator~=5.1.1 -deepspeed~=0.14.4 defusedxml~=0.7.1 deprecated~=1.2.14 dill~=0.3.6 @@ -75,7 +74,6 @@ farama-notifications~=0.0.4 fastjsonschema~=2.20.0 fasttext~=0.9.2 filelock~=3.13.4 -flash-attn~=2.5.9.post1 flask~=2.2.5 flatbuffers~=24.3.25 fonttools~=4.25.0 @@ -104,7 +102,6 @@ h11~=0.14.0 h5py~=3.10.0 hjson~=3.1.0 holidays~=0.45 -horovod==0.28.1+db1 htmlmin~=0.1.12 httpcore~=1.0.5 httplib2~=0.20.2 @@ -194,19 +191,6 @@ notebook~=6.5.4 notebook-shim~=0.2.2 numba~=0.57.1 numpy~=1.23.5 -nvidia-cublas-cu12~=12.1.3.1 -nvidia-cuda-cupti-cu12~=12.1.105 -nvidia-cuda-nvrtc-cu12~=12.1.105 -nvidia-cuda-runtime-cu12~=12.1.105 -nvidia-cudnn-cu12~=8.9.2.26 -nvidia-cufft-cu12~=11.0.2.54 -nvidia-curand-cu12~=10.3.2.106 -nvidia-cusolver-cu12~=11.4.5.107 -nvidia-cusparse-cu12~=12.1.0.106 -nvidia-ml-py~=12.555.43 -nvidia-nccl-cu12~=2.20.5 -nvidia-nvjitlink-cu12~=12.5.82 -nvidia-nvtx-cu12~=12.1.105 oauthlib~=3.2.0 oci~=2.126.4 openai~=1.35.3 @@ -330,14 +314,13 @@ tiktoken~=0.5.2 tinycss2~=1.2.1 tokenize-rt~=4.2.1 tokenizers~=0.19.0 -torch==2.3.1+cu121 +torch~=2.3.1 torcheval~=0.0.7 -torchvision==0.18.1+cu121 +torchvision~=0.18.1 tornado~=6.3.2 tqdm~=4.65.0 traitlets~=5.7.1 transformers~=4.41.2 -triton~=2.3.1 typer~=0.9.4 typing-extensions~=4.10.0 typing-inspect~=0.9.0 diff --git a/python/dbr/15.4.x-gpu-ml-scala2.12/pyproject.toml b/python/dbr/15.4.x-gpu-ml-scala2.12/pyproject.toml index 1673bdb..ad2679d 100644 --- a/python/dbr/15.4.x-gpu-ml-scala2.12/pyproject.toml +++ b/python/dbr/15.4.x-gpu-ml-scala2.12/pyproject.toml @@ -71,7 +71,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.6.7", "decorator~=5.1.1", - "deepspeed~=0.14.4", "defusedxml~=0.7.1", "deprecated~=1.2.14", "dill~=0.3.6", @@ -87,7 +86,6 @@ constraint-dependencies = [ "fastjsonschema~=2.20.0", "fasttext~=0.9.2", "filelock~=3.13.4", - "flash-attn~=2.5.9.post1", "flask~=2.2.5", "flatbuffers~=24.3.25", "fonttools~=4.25.0", @@ -116,7 +114,6 @@ constraint-dependencies = [ "h5py~=3.10.0", "hjson~=3.1.0", "holidays~=0.45", - "horovod==0.28.1+db1", "htmlmin~=0.1.12", "httpcore~=1.0.5", "httplib2~=0.20.2", @@ -206,19 +203,6 @@ constraint-dependencies = [ "notebook-shim~=0.2.2", "numba~=0.57.1", "numpy~=1.23.5", - "nvidia-cublas-cu12~=12.1.3.1", - "nvidia-cuda-cupti-cu12~=12.1.105", - "nvidia-cuda-nvrtc-cu12~=12.1.105", - "nvidia-cuda-runtime-cu12~=12.1.105", - "nvidia-cudnn-cu12~=8.9.2.26", - "nvidia-cufft-cu12~=11.0.2.54", - "nvidia-curand-cu12~=10.3.2.106", - "nvidia-cusolver-cu12~=11.4.5.107", - "nvidia-cusparse-cu12~=12.1.0.106", - "nvidia-ml-py~=12.555.43", - "nvidia-nccl-cu12~=2.20.5", - "nvidia-nvjitlink-cu12~=12.5.82", - "nvidia-nvtx-cu12~=12.1.105", "oauthlib~=3.2.0", "oci~=2.126.4", "openai~=1.35.3", @@ -342,14 +326,13 @@ constraint-dependencies = [ "tinycss2~=1.2.1", "tokenize-rt~=4.2.1", "tokenizers~=0.19.0", - "torch==2.3.1+cu121", + "torch~=2.3.1", "torcheval~=0.0.7", - "torchvision==0.18.1+cu121", + "torchvision~=0.18.1", "tornado~=6.3.2", "tqdm~=4.65.0", "traitlets~=5.7.1", "transformers~=4.41.2", - "triton~=2.3.1", "typer~=0.9.4", "typing-extensions~=4.10.0", "typing-inspect~=0.9.0", diff --git a/python/dbr/16.4.x-cpu-ml-scala2.12/constraints.txt b/python/dbr/16.4.x-cpu-ml-scala2.12/constraints.txt index 93158e1..3fb1850 100644 --- a/python/dbr/16.4.x-cpu-ml-scala2.12/constraints.txt +++ b/python/dbr/16.4.x-cpu-ml-scala2.12/constraints.txt @@ -66,7 +66,6 @@ datasets~=3.5.0 dbl-tempo~=0.1.26 debugpy~=1.6.7 decorator~=5.1.1 -deepspeed~=0.16.5 defusedxml~=0.7.1 deprecated~=1.2.18 dill~=0.3.8 @@ -350,11 +349,11 @@ tinycss2~=1.2.1 tokenize-rt~=4.2.1 tokenizers~=0.21.0 tomli~=2.0.1 -torch==2.6.0+cpu +torch~=2.6.0 torch-optimizer~=0.3.0 torcheval~=0.0.7 torchmetrics~=1.6.0 -torchvision==0.21.0+cpu +torchvision~=0.21.0 tornado~=6.4.1 tqdm~=4.66.4 traitlets~=5.14.3 diff --git a/python/dbr/16.4.x-cpu-ml-scala2.12/pyproject.toml b/python/dbr/16.4.x-cpu-ml-scala2.12/pyproject.toml index 71640e3..9bfae3e 100644 --- a/python/dbr/16.4.x-cpu-ml-scala2.12/pyproject.toml +++ b/python/dbr/16.4.x-cpu-ml-scala2.12/pyproject.toml @@ -78,7 +78,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.6.7", "decorator~=5.1.1", - "deepspeed~=0.16.5", "defusedxml~=0.7.1", "deprecated~=1.2.18", "dill~=0.3.8", @@ -362,11 +361,11 @@ constraint-dependencies = [ "tokenize-rt~=4.2.1", "tokenizers~=0.21.0", "tomli~=2.0.1", - "torch==2.6.0+cpu", + "torch~=2.6.0", "torch-optimizer~=0.3.0", "torcheval~=0.0.7", "torchmetrics~=1.6.0", - "torchvision==0.21.0+cpu", + "torchvision~=0.21.0", "tornado~=6.4.1", "tqdm~=4.66.4", "traitlets~=5.14.3", diff --git a/python/dbr/16.4.x-gpu-ml-scala2.12/constraints.txt b/python/dbr/16.4.x-gpu-ml-scala2.12/constraints.txt index b134562..531d11a 100644 --- a/python/dbr/16.4.x-gpu-ml-scala2.12/constraints.txt +++ b/python/dbr/16.4.x-gpu-ml-scala2.12/constraints.txt @@ -65,7 +65,6 @@ datasets~=3.5.0 dbl-tempo~=0.1.26 debugpy~=1.6.7 decorator~=5.1.1 -deepspeed~=0.16.5 defusedxml~=0.7.1 deprecated~=1.2.18 dill~=0.3.8 @@ -82,7 +81,6 @@ fastapi~=0.115.12 fastjsonschema~=2.21.1 fasttext-wheel~=0.9.2 filelock~=3.13.1 -flash-attn~=2.7.4.post1 flask~=2.2.5 flatbuffers~=25.2.10 fonttools~=4.51.0 @@ -211,19 +209,6 @@ notebook~=7.0.8 notebook-shim~=0.2.3 numba~=0.59.1 numpy~=1.26.4 -nvidia-cublas-cu12~=12.4.5.8 -nvidia-cuda-cupti-cu12~=12.4.127 -nvidia-cuda-nvrtc-cu12~=12.4.127 -nvidia-cuda-runtime-cu12~=12.4.127 -nvidia-cudnn-cu12~=9.1.0.70 -nvidia-cufft-cu12~=11.2.1.3 -nvidia-curand-cu12~=10.3.5.147 -nvidia-cusolver-cu12~=11.6.1.9 -nvidia-cusparse-cu12~=12.3.1.170 -nvidia-cusparselt-cu12~=0.6.2 -nvidia-nccl-cu12~=2.21.5 -nvidia-nvjitlink-cu12~=12.4.127 -nvidia-nvtx-cu12~=12.4.127 oauthlib~=3.2.0 oci~=2.150.0 openai~=1.69.0 @@ -363,16 +348,15 @@ tinycss2~=1.2.1 tokenize-rt~=4.2.1 tokenizers~=0.21.0 tomli~=2.0.1 -torch==2.6.0+cu124 +torch~=2.6.0 torch-optimizer~=0.3.0 torcheval~=0.0.7 torchmetrics~=1.6.0 -torchvision==0.21.0+cu124 +torchvision~=0.21.0 tornado~=6.4.1 tqdm~=4.66.4 traitlets~=5.14.3 transformers~=4.50.2 -triton~=3.2.0 typer~=0.15.2 types-protobuf~=3.20.3 types-psutil~=5.9.0 diff --git a/python/dbr/16.4.x-gpu-ml-scala2.12/pyproject.toml b/python/dbr/16.4.x-gpu-ml-scala2.12/pyproject.toml index a1e2b33..903c296 100644 --- a/python/dbr/16.4.x-gpu-ml-scala2.12/pyproject.toml +++ b/python/dbr/16.4.x-gpu-ml-scala2.12/pyproject.toml @@ -77,7 +77,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.6.7", "decorator~=5.1.1", - "deepspeed~=0.16.5", "defusedxml~=0.7.1", "deprecated~=1.2.18", "dill~=0.3.8", @@ -94,7 +93,6 @@ constraint-dependencies = [ "fastjsonschema~=2.21.1", "fasttext-wheel~=0.9.2", "filelock~=3.13.1", - "flash-attn~=2.7.4.post1", "flask~=2.2.5", "flatbuffers~=25.2.10", "fonttools~=4.51.0", @@ -223,19 +221,6 @@ constraint-dependencies = [ "notebook-shim~=0.2.3", "numba~=0.59.1", "numpy~=1.26.4", - "nvidia-cublas-cu12~=12.4.5.8", - "nvidia-cuda-cupti-cu12~=12.4.127", - "nvidia-cuda-nvrtc-cu12~=12.4.127", - "nvidia-cuda-runtime-cu12~=12.4.127", - "nvidia-cudnn-cu12~=9.1.0.70", - "nvidia-cufft-cu12~=11.2.1.3", - "nvidia-curand-cu12~=10.3.5.147", - "nvidia-cusolver-cu12~=11.6.1.9", - "nvidia-cusparse-cu12~=12.3.1.170", - "nvidia-cusparselt-cu12~=0.6.2", - "nvidia-nccl-cu12~=2.21.5", - "nvidia-nvjitlink-cu12~=12.4.127", - "nvidia-nvtx-cu12~=12.4.127", "oauthlib~=3.2.0", "oci~=2.150.0", "openai~=1.69.0", @@ -375,16 +360,15 @@ constraint-dependencies = [ "tokenize-rt~=4.2.1", "tokenizers~=0.21.0", "tomli~=2.0.1", - "torch==2.6.0+cu124", + "torch~=2.6.0", "torch-optimizer~=0.3.0", "torcheval~=0.0.7", "torchmetrics~=1.6.0", - "torchvision==0.21.0+cu124", + "torchvision~=0.21.0", "tornado~=6.4.1", "tqdm~=4.66.4", "traitlets~=5.14.3", "transformers~=4.50.2", - "triton~=3.2.0", "typer~=0.15.2", "types-protobuf~=3.20.3", "types-psutil~=5.9.0", diff --git a/python/dbr/17.3.x-cpu-ml-scala2.13/constraints.txt b/python/dbr/17.3.x-cpu-ml-scala2.13/constraints.txt index ef6d4de..c3cdae0 100644 --- a/python/dbr/17.3.x-cpu-ml-scala2.13/constraints.txt +++ b/python/dbr/17.3.x-cpu-ml-scala2.13/constraints.txt @@ -70,7 +70,6 @@ datasets~=3.5.0 dbl-tempo~=0.1.26 debugpy~=1.8.11 decorator~=5.1.1 -deepspeed~=0.16.5 defusedxml~=0.7.1 dill~=0.3.8 distlib~=0.3.9 @@ -214,7 +213,6 @@ notebook~=7.3.2 notebook-shim~=0.2.3 numba~=0.61.0 numpy~=2.1.3 -nvidia-nccl-cu12~=2.28.3 oauthlib~=3.2.0 oci~=2.160.2 openai~=1.69.0 @@ -359,11 +357,11 @@ tinycss2~=1.4.0 tokenize-rt~=6.1.0 tokenizers~=0.21.0 tomli~=2.0.1 -torch==2.7.0+cpu +torch~=2.7.0 torch-optimizer~=0.3.0 torcheval~=0.0.7 torchmetrics~=1.6.0 -torchvision==0.22.0+cpu +torchvision~=0.22.0 tornado~=6.4.2 tqdm~=4.67.1 traitlets~=5.14.3 diff --git a/python/dbr/17.3.x-cpu-ml-scala2.13/pyproject.toml b/python/dbr/17.3.x-cpu-ml-scala2.13/pyproject.toml index 9d75d2a..fefb81c 100644 --- a/python/dbr/17.3.x-cpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/17.3.x-cpu-ml-scala2.13/pyproject.toml @@ -82,7 +82,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.8.11", "decorator~=5.1.1", - "deepspeed~=0.16.5", "defusedxml~=0.7.1", "dill~=0.3.8", "distlib~=0.3.9", @@ -226,7 +225,6 @@ constraint-dependencies = [ "notebook-shim~=0.2.3", "numba~=0.61.0", "numpy~=2.1.3", - "nvidia-nccl-cu12~=2.28.3", "oauthlib~=3.2.0", "oci~=2.160.2", "openai~=1.69.0", @@ -371,11 +369,11 @@ constraint-dependencies = [ "tokenize-rt~=6.1.0", "tokenizers~=0.21.0", "tomli~=2.0.1", - "torch==2.7.0+cpu", + "torch~=2.7.0", "torch-optimizer~=0.3.0", "torcheval~=0.0.7", "torchmetrics~=1.6.0", - "torchvision==0.22.0+cpu", + "torchvision~=0.22.0", "tornado~=6.4.2", "tqdm~=4.67.1", "traitlets~=5.14.3", diff --git a/python/dbr/17.3.x-gpu-ml-scala2.13/constraints.txt b/python/dbr/17.3.x-gpu-ml-scala2.13/constraints.txt index e1c88a5..c051b59 100644 --- a/python/dbr/17.3.x-gpu-ml-scala2.13/constraints.txt +++ b/python/dbr/17.3.x-gpu-ml-scala2.13/constraints.txt @@ -69,7 +69,6 @@ datasets~=3.5.0 dbl-tempo~=0.1.26 debugpy~=1.8.11 decorator~=5.1.1 -deepspeed~=0.16.5 defusedxml~=0.7.1 dill~=0.3.8 distlib~=0.3.9 @@ -85,7 +84,6 @@ fastapi~=0.117.1 fastjsonschema~=2.21.2 fasttext-wheel~=0.9.2 filelock~=3.13.1 -flash-attn~=2.7.4.post1 flask~=2.2.5 flatbuffers~=25.2.10 fonttools~=4.55.3 @@ -214,20 +212,6 @@ notebook~=7.3.2 notebook-shim~=0.2.3 numba~=0.61.0 numpy~=2.1.3 -nvidia-cublas-cu12~=12.6.4.1 -nvidia-cuda-cupti-cu12~=12.6.80 -nvidia-cuda-nvrtc-cu12~=12.6.77 -nvidia-cuda-runtime-cu12~=12.6.77 -nvidia-cudnn-cu12~=9.5.1.17 -nvidia-cufft-cu12~=11.3.0.4 -nvidia-cufile-cu12~=1.11.1.6 -nvidia-curand-cu12~=10.3.7.77 -nvidia-cusolver-cu12~=11.7.1.2 -nvidia-cusparse-cu12~=12.5.4.2 -nvidia-cusparselt-cu12~=0.6.3 -nvidia-nccl-cu12~=2.26.2 -nvidia-nvjitlink-cu12~=12.6.85 -nvidia-nvtx-cu12~=12.6.77 oauthlib~=3.2.0 oci~=2.160.2 openai~=1.69.0 @@ -381,7 +365,6 @@ tornado~=6.4.2 tqdm~=4.67.1 traitlets~=5.14.3 transformers~=4.51.3 -triton~=3.3.0 typer~=0.19.2 types-python-dateutil~=2.9.0.20250822 typing-extensions~=4.12.2 diff --git a/python/dbr/17.3.x-gpu-ml-scala2.13/pyproject.toml b/python/dbr/17.3.x-gpu-ml-scala2.13/pyproject.toml index 761f8a0..87aa7cb 100644 --- a/python/dbr/17.3.x-gpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/17.3.x-gpu-ml-scala2.13/pyproject.toml @@ -81,7 +81,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.8.11", "decorator~=5.1.1", - "deepspeed~=0.16.5", "defusedxml~=0.7.1", "dill~=0.3.8", "distlib~=0.3.9", @@ -97,7 +96,6 @@ constraint-dependencies = [ "fastjsonschema~=2.21.2", "fasttext-wheel~=0.9.2", "filelock~=3.13.1", - "flash-attn~=2.7.4.post1", "flask~=2.2.5", "flatbuffers~=25.2.10", "fonttools~=4.55.3", @@ -226,20 +224,6 @@ constraint-dependencies = [ "notebook-shim~=0.2.3", "numba~=0.61.0", "numpy~=2.1.3", - "nvidia-cublas-cu12~=12.6.4.1", - "nvidia-cuda-cupti-cu12~=12.6.80", - "nvidia-cuda-nvrtc-cu12~=12.6.77", - "nvidia-cuda-runtime-cu12~=12.6.77", - "nvidia-cudnn-cu12~=9.5.1.17", - "nvidia-cufft-cu12~=11.3.0.4", - "nvidia-cufile-cu12~=1.11.1.6", - "nvidia-curand-cu12~=10.3.7.77", - "nvidia-cusolver-cu12~=11.7.1.2", - "nvidia-cusparse-cu12~=12.5.4.2", - "nvidia-cusparselt-cu12~=0.6.3", - "nvidia-nccl-cu12~=2.26.2", - "nvidia-nvjitlink-cu12~=12.6.85", - "nvidia-nvtx-cu12~=12.6.77", "oauthlib~=3.2.0", "oci~=2.160.2", "openai~=1.69.0", @@ -393,7 +377,6 @@ constraint-dependencies = [ "tqdm~=4.67.1", "traitlets~=5.14.3", "transformers~=4.51.3", - "triton~=3.3.0", "typer~=0.19.2", "types-python-dateutil~=2.9.0.20250822", "typing-extensions~=4.12.2", diff --git a/python/dbr/18.0.x-cpu-ml-scala2.13/constraints.txt b/python/dbr/18.0.x-cpu-ml-scala2.13/constraints.txt index 26126ef..90df94a 100644 --- a/python/dbr/18.0.x-cpu-ml-scala2.13/constraints.txt +++ b/python/dbr/18.0.x-cpu-ml-scala2.13/constraints.txt @@ -67,7 +67,6 @@ datasets~=4.3.0 dbl-tempo~=0.1.26 debugpy~=1.8.11 decorator~=5.1.1 -deepspeed~=0.18.1 defusedxml~=0.7.1 deltalake~=1.1.4 deprecated~=1.2.18 @@ -216,7 +215,6 @@ notebook~=7.3.2 notebook-shim~=0.2.4 numba~=0.61.0 numpy~=2.1.3 -nvidia-nccl-cu12~=2.28.7 oauthlib~=3.2.0 oci~=2.163.1 openai~=2.7.0 @@ -355,9 +353,9 @@ tinycss2~=1.4.0 tokenize-rt~=6.1.0 tokenizers~=0.22.1 tomli~=2.0.1 -torch==2.9.0+cpu +torch~=2.9.0 torcheval~=0.0.7 -torchvision==0.24.0+cpu +torchvision~=0.24.0 tornado~=6.5.1 tqdm~=4.67.1 traitlets~=5.14.3 diff --git a/python/dbr/18.0.x-cpu-ml-scala2.13/pyproject.toml b/python/dbr/18.0.x-cpu-ml-scala2.13/pyproject.toml index 866df78..32160da 100644 --- a/python/dbr/18.0.x-cpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/18.0.x-cpu-ml-scala2.13/pyproject.toml @@ -79,7 +79,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.8.11", "decorator~=5.1.1", - "deepspeed~=0.18.1", "defusedxml~=0.7.1", "deltalake~=1.1.4", "deprecated~=1.2.18", @@ -228,7 +227,6 @@ constraint-dependencies = [ "notebook-shim~=0.2.4", "numba~=0.61.0", "numpy~=2.1.3", - "nvidia-nccl-cu12~=2.28.7", "oauthlib~=3.2.0", "oci~=2.163.1", "openai~=2.7.0", @@ -367,9 +365,9 @@ constraint-dependencies = [ "tokenize-rt~=6.1.0", "tokenizers~=0.22.1", "tomli~=2.0.1", - "torch==2.9.0+cpu", + "torch~=2.9.0", "torcheval~=0.0.7", - "torchvision==0.24.0+cpu", + "torchvision~=0.24.0", "tornado~=6.5.1", "tqdm~=4.67.1", "traitlets~=5.14.3", diff --git a/python/dbr/18.0.x-gpu-ml-scala2.13/constraints.txt b/python/dbr/18.0.x-gpu-ml-scala2.13/constraints.txt index e867ce5..2c90b66 100644 --- a/python/dbr/18.0.x-gpu-ml-scala2.13/constraints.txt +++ b/python/dbr/18.0.x-gpu-ml-scala2.13/constraints.txt @@ -66,7 +66,6 @@ datasets~=4.3.0 dbl-tempo~=0.1.26 debugpy~=1.8.11 decorator~=5.1.1 -deepspeed~=0.18.1 defusedxml~=0.7.1 deltalake~=1.1.4 deprecated~=1.2.18 @@ -83,7 +82,6 @@ fastapi~=0.121.2 fastjsonschema~=2.21.1 filelock~=3.17.0 filetype~=1.2.0 -flash-attn~=2.8.3 flask~=2.2.5 flatbuffers~=25.9.23 fonttools~=4.55.3 @@ -216,22 +214,6 @@ notebook~=7.3.2 notebook-shim~=0.2.4 numba~=0.61.0 numpy~=2.1.3 -nvidia-cublas-cu12~=12.9.1.4 -nvidia-cuda-cupti-cu12~=12.9.79 -nvidia-cuda-nvrtc-cu12~=12.9.86 -nvidia-cuda-runtime-cu12~=12.9.79 -nvidia-cudnn-cu12~=9.10.2.21 -nvidia-cufft-cu12~=11.4.1.4 -nvidia-cufile-cu12~=1.14.1.1 -nvidia-curand-cu12~=10.3.10.19 -nvidia-cusolver-cu12~=11.7.5.82 -nvidia-cusparse-cu12~=12.5.10.65 -nvidia-cusparselt-cu12~=0.7.1 -nvidia-ml-py~=13.580.82 -nvidia-nccl-cu12~=2.27.5 -nvidia-nvjitlink-cu12~=12.9.86 -nvidia-nvshmem-cu12~=3.3.20 -nvidia-nvtx-cu12~=12.9.79 oauthlib~=3.2.0 oci~=2.163.1 openai~=2.7.0 @@ -370,14 +352,13 @@ tinycss2~=1.4.0 tokenize-rt~=6.1.0 tokenizers~=0.22.1 tomli~=2.0.1 -torch==2.9.0+cu129 +torch~=2.9.0 torcheval~=0.0.7 -torchvision==0.24.0+cu129 +torchvision~=0.24.0 tornado~=6.5.1 tqdm~=4.67.1 traitlets~=5.14.3 transformers~=4.57.1 -triton~=3.5.0 typer~=0.20.0 typer-slim~=0.20.0 types-python-dateutil~=2.9.0.20251108 diff --git a/python/dbr/18.0.x-gpu-ml-scala2.13/pyproject.toml b/python/dbr/18.0.x-gpu-ml-scala2.13/pyproject.toml index cf3cd37..c3dec6c 100644 --- a/python/dbr/18.0.x-gpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/18.0.x-gpu-ml-scala2.13/pyproject.toml @@ -78,7 +78,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.8.11", "decorator~=5.1.1", - "deepspeed~=0.18.1", "defusedxml~=0.7.1", "deltalake~=1.1.4", "deprecated~=1.2.18", @@ -95,7 +94,6 @@ constraint-dependencies = [ "fastjsonschema~=2.21.1", "filelock~=3.17.0", "filetype~=1.2.0", - "flash-attn~=2.8.3", "flask~=2.2.5", "flatbuffers~=25.9.23", "fonttools~=4.55.3", @@ -228,22 +226,6 @@ constraint-dependencies = [ "notebook-shim~=0.2.4", "numba~=0.61.0", "numpy~=2.1.3", - "nvidia-cublas-cu12~=12.9.1.4", - "nvidia-cuda-cupti-cu12~=12.9.79", - "nvidia-cuda-nvrtc-cu12~=12.9.86", - "nvidia-cuda-runtime-cu12~=12.9.79", - "nvidia-cudnn-cu12~=9.10.2.21", - "nvidia-cufft-cu12~=11.4.1.4", - "nvidia-cufile-cu12~=1.14.1.1", - "nvidia-curand-cu12~=10.3.10.19", - "nvidia-cusolver-cu12~=11.7.5.82", - "nvidia-cusparse-cu12~=12.5.10.65", - "nvidia-cusparselt-cu12~=0.7.1", - "nvidia-ml-py~=13.580.82", - "nvidia-nccl-cu12~=2.27.5", - "nvidia-nvjitlink-cu12~=12.9.86", - "nvidia-nvshmem-cu12~=3.3.20", - "nvidia-nvtx-cu12~=12.9.79", "oauthlib~=3.2.0", "oci~=2.163.1", "openai~=2.7.0", @@ -382,14 +364,13 @@ constraint-dependencies = [ "tokenize-rt~=6.1.0", "tokenizers~=0.22.1", "tomli~=2.0.1", - "torch==2.9.0+cu129", + "torch~=2.9.0", "torcheval~=0.0.7", - "torchvision==0.24.0+cu129", + "torchvision~=0.24.0", "tornado~=6.5.1", "tqdm~=4.67.1", "traitlets~=5.14.3", "transformers~=4.57.1", - "triton~=3.5.0", "typer~=0.20.0", "typer-slim~=0.20.0", "types-python-dateutil~=2.9.0.20251108", diff --git a/python/dbr/18.1.x-cpu-ml-scala2.13/constraints.txt b/python/dbr/18.1.x-cpu-ml-scala2.13/constraints.txt index fb7d7e9..4ab2cda 100644 --- a/python/dbr/18.1.x-cpu-ml-scala2.13/constraints.txt +++ b/python/dbr/18.1.x-cpu-ml-scala2.13/constraints.txt @@ -67,7 +67,6 @@ datasets~=4.3.0 dbl-tempo~=0.1.26 debugpy~=1.8.11 decorator~=5.1.1 -deepspeed~=0.18.1 defusedxml~=0.7.1 deltalake~=1.1.4 deprecated~=1.2.18 @@ -216,7 +215,6 @@ notebook~=7.3.2 notebook-shim~=0.2.4 numba~=0.61.0 numpy~=2.1.3 -nvidia-nccl-cu12~=2.28.7 oauthlib~=3.2.0 oci~=2.163.1 openai~=2.7.0 @@ -355,9 +353,9 @@ tinycss2~=1.4.0 tokenize-rt~=6.1.0 tokenizers~=0.22.1 tomli~=2.0.1 -torch==2.9.0+cpu +torch~=2.9.0 torcheval~=0.0.7 -torchvision==0.24.0+cpu +torchvision~=0.24.0 tornado~=6.5.1 tqdm~=4.67.1 traitlets~=5.14.3 diff --git a/python/dbr/18.1.x-cpu-ml-scala2.13/pyproject.toml b/python/dbr/18.1.x-cpu-ml-scala2.13/pyproject.toml index 7b32c62..430b595 100644 --- a/python/dbr/18.1.x-cpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/18.1.x-cpu-ml-scala2.13/pyproject.toml @@ -79,7 +79,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.8.11", "decorator~=5.1.1", - "deepspeed~=0.18.1", "defusedxml~=0.7.1", "deltalake~=1.1.4", "deprecated~=1.2.18", @@ -228,7 +227,6 @@ constraint-dependencies = [ "notebook-shim~=0.2.4", "numba~=0.61.0", "numpy~=2.1.3", - "nvidia-nccl-cu12~=2.28.7", "oauthlib~=3.2.0", "oci~=2.163.1", "openai~=2.7.0", @@ -367,9 +365,9 @@ constraint-dependencies = [ "tokenize-rt~=6.1.0", "tokenizers~=0.22.1", "tomli~=2.0.1", - "torch==2.9.0+cpu", + "torch~=2.9.0", "torcheval~=0.0.7", - "torchvision==0.24.0+cpu", + "torchvision~=0.24.0", "tornado~=6.5.1", "tqdm~=4.67.1", "traitlets~=5.14.3", diff --git a/python/dbr/18.1.x-gpu-ml-scala2.13/constraints.txt b/python/dbr/18.1.x-gpu-ml-scala2.13/constraints.txt index e0751ce..be9f2be 100644 --- a/python/dbr/18.1.x-gpu-ml-scala2.13/constraints.txt +++ b/python/dbr/18.1.x-gpu-ml-scala2.13/constraints.txt @@ -66,7 +66,6 @@ datasets~=4.3.0 dbl-tempo~=0.1.26 debugpy~=1.8.11 decorator~=5.1.1 -deepspeed~=0.18.1 defusedxml~=0.7.1 deltalake~=1.1.4 deprecated~=1.2.18 @@ -83,7 +82,6 @@ fastapi~=0.121.2 fastjsonschema~=2.21.1 filelock~=3.17.0 filetype~=1.2.0 -flash-attn~=2.8.3 flask~=2.2.5 flatbuffers~=25.9.23 fonttools~=4.55.3 @@ -216,22 +214,6 @@ notebook~=7.3.2 notebook-shim~=0.2.4 numba~=0.61.0 numpy~=2.1.3 -nvidia-cublas-cu12~=12.9.1.4 -nvidia-cuda-cupti-cu12~=12.9.79 -nvidia-cuda-nvrtc-cu12~=12.9.86 -nvidia-cuda-runtime-cu12~=12.9.79 -nvidia-cudnn-cu12~=9.10.2.21 -nvidia-cufft-cu12~=11.4.1.4 -nvidia-cufile-cu12~=1.14.1.1 -nvidia-curand-cu12~=10.3.10.19 -nvidia-cusolver-cu12~=11.7.5.82 -nvidia-cusparse-cu12~=12.5.10.65 -nvidia-cusparselt-cu12~=0.7.1 -nvidia-ml-py~=13.580.82 -nvidia-nccl-cu12~=2.27.5 -nvidia-nvjitlink-cu12~=12.9.86 -nvidia-nvshmem-cu12~=3.3.20 -nvidia-nvtx-cu12~=12.9.79 oauthlib~=3.2.0 oci~=2.163.1 openai~=2.7.0 @@ -370,14 +352,13 @@ tinycss2~=1.4.0 tokenize-rt~=6.1.0 tokenizers~=0.22.1 tomli~=2.0.1 -torch==2.9.0+cu129 +torch~=2.9.0 torcheval~=0.0.7 -torchvision==0.24.0+cu129 +torchvision~=0.24.0 tornado~=6.5.1 tqdm~=4.67.1 traitlets~=5.14.3 transformers~=4.57.1 -triton~=3.5.0 typer~=0.20.0 typer-slim~=0.20.0 types-python-dateutil~=2.9.0.20251108 diff --git a/python/dbr/18.1.x-gpu-ml-scala2.13/pyproject.toml b/python/dbr/18.1.x-gpu-ml-scala2.13/pyproject.toml index a693e10..52d57a4 100644 --- a/python/dbr/18.1.x-gpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/18.1.x-gpu-ml-scala2.13/pyproject.toml @@ -78,7 +78,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.8.11", "decorator~=5.1.1", - "deepspeed~=0.18.1", "defusedxml~=0.7.1", "deltalake~=1.1.4", "deprecated~=1.2.18", @@ -95,7 +94,6 @@ constraint-dependencies = [ "fastjsonschema~=2.21.1", "filelock~=3.17.0", "filetype~=1.2.0", - "flash-attn~=2.8.3", "flask~=2.2.5", "flatbuffers~=25.9.23", "fonttools~=4.55.3", @@ -228,22 +226,6 @@ constraint-dependencies = [ "notebook-shim~=0.2.4", "numba~=0.61.0", "numpy~=2.1.3", - "nvidia-cublas-cu12~=12.9.1.4", - "nvidia-cuda-cupti-cu12~=12.9.79", - "nvidia-cuda-nvrtc-cu12~=12.9.86", - "nvidia-cuda-runtime-cu12~=12.9.79", - "nvidia-cudnn-cu12~=9.10.2.21", - "nvidia-cufft-cu12~=11.4.1.4", - "nvidia-cufile-cu12~=1.14.1.1", - "nvidia-curand-cu12~=10.3.10.19", - "nvidia-cusolver-cu12~=11.7.5.82", - "nvidia-cusparse-cu12~=12.5.10.65", - "nvidia-cusparselt-cu12~=0.7.1", - "nvidia-ml-py~=13.580.82", - "nvidia-nccl-cu12~=2.27.5", - "nvidia-nvjitlink-cu12~=12.9.86", - "nvidia-nvshmem-cu12~=3.3.20", - "nvidia-nvtx-cu12~=12.9.79", "oauthlib~=3.2.0", "oci~=2.163.1", "openai~=2.7.0", @@ -382,14 +364,13 @@ constraint-dependencies = [ "tokenize-rt~=6.1.0", "tokenizers~=0.22.1", "tomli~=2.0.1", - "torch==2.9.0+cu129", + "torch~=2.9.0", "torcheval~=0.0.7", - "torchvision==0.24.0+cu129", + "torchvision~=0.24.0", "tornado~=6.5.1", "tqdm~=4.67.1", "traitlets~=5.14.3", "transformers~=4.57.1", - "triton~=3.5.0", "typer~=0.20.0", "typer-slim~=0.20.0", "types-python-dateutil~=2.9.0.20251108", diff --git a/python/dbr/18.2.x-cpu-ml-scala2.13/constraints.txt b/python/dbr/18.2.x-cpu-ml-scala2.13/constraints.txt index 2fd7b33..95e45aa 100644 --- a/python/dbr/18.2.x-cpu-ml-scala2.13/constraints.txt +++ b/python/dbr/18.2.x-cpu-ml-scala2.13/constraints.txt @@ -67,14 +67,12 @@ datasets~=4.3.0 dbl-tempo~=0.1.26 debugpy~=1.8.11 decorator~=5.1.1 -deepspeed~=0.18.1 defusedxml~=0.7.1 deltalake~=1.1.4 deprecated~=1.2.18 dill~=0.3.8 distlib~=0.3.9 distro~=1.9.0 -distro-info==1.7+build1 dm-tree~=0.1.9 docstring-to-markdown~=0.11 einops~=0.8.1 @@ -218,7 +216,6 @@ notebook~=7.3.2 notebook-shim~=0.2.4 numba~=0.61.0 numpy~=2.1.3 -nvidia-nccl-cu12~=2.28.7 oauthlib~=3.2.0 oci~=2.163.1 openai~=2.7.0 @@ -282,7 +279,6 @@ pyright~=1.1.394 pyroaring~=1.0.3 pytesseract~=0.3.10 pytest~=8.3.5 -python-apt==2.7.7+ubuntu5.2 python-dateutil~=2.9.0.post0 python-dotenv~=1.2.1 python-editor~=1.0.4 @@ -358,9 +354,9 @@ tinycss2~=1.4.0 tokenize-rt~=6.1.0 tokenizers~=0.22.1 tomli~=2.0.1 -torch==2.9.0+cpu +torch~=2.9.0 torcheval~=0.0.7 -torchvision==0.24.0+cpu +torchvision~=0.24.0 tornado~=6.5.1 tqdm~=4.67.1 traitlets~=5.14.3 diff --git a/python/dbr/18.2.x-cpu-ml-scala2.13/pyproject.toml b/python/dbr/18.2.x-cpu-ml-scala2.13/pyproject.toml index 71fc715..1a7cd30 100644 --- a/python/dbr/18.2.x-cpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/18.2.x-cpu-ml-scala2.13/pyproject.toml @@ -79,14 +79,12 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.8.11", "decorator~=5.1.1", - "deepspeed~=0.18.1", "defusedxml~=0.7.1", "deltalake~=1.1.4", "deprecated~=1.2.18", "dill~=0.3.8", "distlib~=0.3.9", "distro~=1.9.0", - "distro-info==1.7+build1", "dm-tree~=0.1.9", "docstring-to-markdown~=0.11", "einops~=0.8.1", @@ -230,7 +228,6 @@ constraint-dependencies = [ "notebook-shim~=0.2.4", "numba~=0.61.0", "numpy~=2.1.3", - "nvidia-nccl-cu12~=2.28.7", "oauthlib~=3.2.0", "oci~=2.163.1", "openai~=2.7.0", @@ -294,7 +291,6 @@ constraint-dependencies = [ "pyroaring~=1.0.3", "pytesseract~=0.3.10", "pytest~=8.3.5", - "python-apt==2.7.7+ubuntu5.2", "python-dateutil~=2.9.0.post0", "python-dotenv~=1.2.1", "python-editor~=1.0.4", @@ -370,9 +366,9 @@ constraint-dependencies = [ "tokenize-rt~=6.1.0", "tokenizers~=0.22.1", "tomli~=2.0.1", - "torch==2.9.0+cpu", + "torch~=2.9.0", "torcheval~=0.0.7", - "torchvision==0.24.0+cpu", + "torchvision~=0.24.0", "tornado~=6.5.1", "tqdm~=4.67.1", "traitlets~=5.14.3", diff --git a/python/dbr/18.2.x-gpu-ml-scala2.13/constraints.txt b/python/dbr/18.2.x-gpu-ml-scala2.13/constraints.txt index 1a4295e..b1c3bc7 100644 --- a/python/dbr/18.2.x-gpu-ml-scala2.13/constraints.txt +++ b/python/dbr/18.2.x-gpu-ml-scala2.13/constraints.txt @@ -66,14 +66,12 @@ datasets~=4.3.0 dbl-tempo~=0.1.26 debugpy~=1.8.11 decorator~=5.1.1 -deepspeed~=0.18.1 defusedxml~=0.7.1 deltalake~=1.1.4 deprecated~=1.2.18 dill~=0.3.8 distlib~=0.3.9 distro~=1.9.0 -distro-info==1.7+build1 dm-tree~=0.1.9 docstring-to-markdown~=0.11 einops~=0.8.1 @@ -85,7 +83,6 @@ fastapi~=0.121.2 fastjsonschema~=2.21.1 filelock~=3.17.0 filetype~=1.2.0 -flash-attn~=2.8.3 flask~=2.2.5 flatbuffers~=25.9.23 fonttools~=4.55.3 @@ -218,22 +215,6 @@ notebook~=7.3.2 notebook-shim~=0.2.4 numba~=0.61.0 numpy~=2.1.3 -nvidia-cublas-cu12~=12.9.1.4 -nvidia-cuda-cupti-cu12~=12.9.79 -nvidia-cuda-nvrtc-cu12~=12.9.86 -nvidia-cuda-runtime-cu12~=12.9.79 -nvidia-cudnn-cu12~=9.10.2.21 -nvidia-cufft-cu12~=11.4.1.4 -nvidia-cufile-cu12~=1.14.1.1 -nvidia-curand-cu12~=10.3.10.19 -nvidia-cusolver-cu12~=11.7.5.82 -nvidia-cusparse-cu12~=12.5.10.65 -nvidia-cusparselt-cu12~=0.7.1 -nvidia-ml-py~=13.580.82 -nvidia-nccl-cu12~=2.27.5 -nvidia-nvjitlink-cu12~=12.9.86 -nvidia-nvshmem-cu12~=3.3.20 -nvidia-nvtx-cu12~=12.9.79 oauthlib~=3.2.0 oci~=2.163.1 openai~=2.7.0 @@ -297,7 +278,6 @@ pyright~=1.1.394 pyroaring~=1.0.3 pytesseract~=0.3.10 pytest~=8.3.5 -python-apt==2.7.7+ubuntu5.2 python-dateutil~=2.9.0.post0 python-dotenv~=1.2.1 python-editor~=1.0.4 @@ -373,14 +353,13 @@ tinycss2~=1.4.0 tokenize-rt~=6.1.0 tokenizers~=0.22.1 tomli~=2.0.1 -torch==2.9.0+cu129 +torch~=2.9.0 torcheval~=0.0.7 -torchvision==0.24.0+cu129 +torchvision~=0.24.0 tornado~=6.5.1 tqdm~=4.67.1 traitlets~=5.14.3 transformers~=4.57.1 -triton~=3.5.0 typer~=0.20.0 typer-slim~=0.20.0 types-python-dateutil~=2.9.0.20251108 diff --git a/python/dbr/18.2.x-gpu-ml-scala2.13/pyproject.toml b/python/dbr/18.2.x-gpu-ml-scala2.13/pyproject.toml index dd67227..14f3919 100644 --- a/python/dbr/18.2.x-gpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/18.2.x-gpu-ml-scala2.13/pyproject.toml @@ -78,14 +78,12 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.8.11", "decorator~=5.1.1", - "deepspeed~=0.18.1", "defusedxml~=0.7.1", "deltalake~=1.1.4", "deprecated~=1.2.18", "dill~=0.3.8", "distlib~=0.3.9", "distro~=1.9.0", - "distro-info==1.7+build1", "dm-tree~=0.1.9", "docstring-to-markdown~=0.11", "einops~=0.8.1", @@ -97,7 +95,6 @@ constraint-dependencies = [ "fastjsonschema~=2.21.1", "filelock~=3.17.0", "filetype~=1.2.0", - "flash-attn~=2.8.3", "flask~=2.2.5", "flatbuffers~=25.9.23", "fonttools~=4.55.3", @@ -230,22 +227,6 @@ constraint-dependencies = [ "notebook-shim~=0.2.4", "numba~=0.61.0", "numpy~=2.1.3", - "nvidia-cublas-cu12~=12.9.1.4", - "nvidia-cuda-cupti-cu12~=12.9.79", - "nvidia-cuda-nvrtc-cu12~=12.9.86", - "nvidia-cuda-runtime-cu12~=12.9.79", - "nvidia-cudnn-cu12~=9.10.2.21", - "nvidia-cufft-cu12~=11.4.1.4", - "nvidia-cufile-cu12~=1.14.1.1", - "nvidia-curand-cu12~=10.3.10.19", - "nvidia-cusolver-cu12~=11.7.5.82", - "nvidia-cusparse-cu12~=12.5.10.65", - "nvidia-cusparselt-cu12~=0.7.1", - "nvidia-ml-py~=13.580.82", - "nvidia-nccl-cu12~=2.27.5", - "nvidia-nvjitlink-cu12~=12.9.86", - "nvidia-nvshmem-cu12~=3.3.20", - "nvidia-nvtx-cu12~=12.9.79", "oauthlib~=3.2.0", "oci~=2.163.1", "openai~=2.7.0", @@ -309,7 +290,6 @@ constraint-dependencies = [ "pyroaring~=1.0.3", "pytesseract~=0.3.10", "pytest~=8.3.5", - "python-apt==2.7.7+ubuntu5.2", "python-dateutil~=2.9.0.post0", "python-dotenv~=1.2.1", "python-editor~=1.0.4", @@ -385,14 +365,13 @@ constraint-dependencies = [ "tokenize-rt~=6.1.0", "tokenizers~=0.22.1", "tomli~=2.0.1", - "torch==2.9.0+cu129", + "torch~=2.9.0", "torcheval~=0.0.7", - "torchvision==0.24.0+cu129", + "torchvision~=0.24.0", "tornado~=6.5.1", "tqdm~=4.67.1", "traitlets~=5.14.3", "transformers~=4.57.1", - "triton~=3.5.0", "typer~=0.20.0", "typer-slim~=0.20.0", "types-python-dateutil~=2.9.0.20251108", diff --git a/python/dbr/19.x-cpu-ml-scala2.13/constraints.txt b/python/dbr/19.x-cpu-ml-scala2.13/constraints.txt index 87f8cb7..725fb83 100644 --- a/python/dbr/19.x-cpu-ml-scala2.13/constraints.txt +++ b/python/dbr/19.x-cpu-ml-scala2.13/constraints.txt @@ -52,7 +52,6 @@ datasets~=4.8.5 dbl-tempo~=0.1.26 debugpy~=1.8.16 decorator~=5.2.1 -deepspeed~=0.19.0 deltalake~=1.5.1 deprecated~=1.3.1 dill~=0.4.0 @@ -164,7 +163,6 @@ nltk~=3.9.2 nodeenv~=1.10.0 numba~=0.62.1 numpy~=2.3.4 -nvidia-nccl-cu12~=2.30.4 oauthlib~=3.2.0 openai~=2.37.0 opencensus~=0.11.4 @@ -272,9 +270,9 @@ tiktoken~=0.13.0 tokenize-rt~=6.2.0 tokenizers~=0.22.1 tomli~=2.0.1 -torch==2.12.0+cpu +torch~=2.12.0 torcheval~=0.0.7 -torchvision==0.27.0+cpu +torchvision~=0.27.0 tornado~=6.5.1 tqdm~=4.67.1 traitlets~=5.14.3 diff --git a/python/dbr/19.x-cpu-ml-scala2.13/pyproject.toml b/python/dbr/19.x-cpu-ml-scala2.13/pyproject.toml index 9e74580..fcc351e 100644 --- a/python/dbr/19.x-cpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/19.x-cpu-ml-scala2.13/pyproject.toml @@ -64,7 +64,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.8.16", "decorator~=5.2.1", - "deepspeed~=0.19.0", "deltalake~=1.5.1", "deprecated~=1.3.1", "dill~=0.4.0", @@ -176,7 +175,6 @@ constraint-dependencies = [ "nodeenv~=1.10.0", "numba~=0.62.1", "numpy~=2.3.4", - "nvidia-nccl-cu12~=2.30.4", "oauthlib~=3.2.0", "openai~=2.37.0", "opencensus~=0.11.4", @@ -284,9 +282,9 @@ constraint-dependencies = [ "tokenize-rt~=6.2.0", "tokenizers~=0.22.1", "tomli~=2.0.1", - "torch==2.12.0+cpu", + "torch~=2.12.0", "torcheval~=0.0.7", - "torchvision==0.27.0+cpu", + "torchvision~=0.27.0", "tornado~=6.5.1", "tqdm~=4.67.1", "traitlets~=5.14.3", diff --git a/python/dbr/19.x-gpu-ml-scala2.13/constraints.txt b/python/dbr/19.x-gpu-ml-scala2.13/constraints.txt index b613334..e721ec7 100644 --- a/python/dbr/19.x-gpu-ml-scala2.13/constraints.txt +++ b/python/dbr/19.x-gpu-ml-scala2.13/constraints.txt @@ -41,9 +41,6 @@ comm~=0.2.3 confection~=1.3.3 contourpy~=1.3.3 cryptography~=46.0.3 -cuda-bindings~=13.2.0 -cuda-pathfinder~=1.5.4 -cuda-toolkit~=13.0.2 cycler~=0.11.0 cymem~=2.0.13 databricks-agents~=1.10.2 @@ -54,7 +51,6 @@ datasets~=4.8.5 dbl-tempo~=0.1.26 debugpy~=1.8.16 decorator~=5.2.1 -deepspeed~=0.19.0 deltalake~=1.5.1 deprecated~=1.3.1 dill~=0.4.0 @@ -66,7 +62,6 @@ executing~=2.2.1 farama-notifications~=0.0.6 fastapi~=0.136.3 filelock~=3.20.0 -flash-attn~=2.8.3 flask~=2.2.5 fonttools~=4.60.1 frozenlist~=1.8.0 @@ -167,23 +162,6 @@ nltk~=3.9.2 nodeenv~=1.10.0 numba~=0.62.1 numpy~=2.3.4 -nvidia-cublas~=13.1.1.3 -nvidia-cuda-cupti~=13.0.85 -nvidia-cuda-nvrtc~=13.0.88 -nvidia-cuda-runtime~=13.0.96 -nvidia-cudnn-cu13~=9.20.0.48 -nvidia-cufft~=12.0.0.61 -nvidia-cufile~=1.15.1.6 -nvidia-curand~=10.4.0.35 -nvidia-cusolver~=12.0.4.66 -nvidia-cusparse~=12.6.3.3 -nvidia-cusparselt-cu13~=0.8.1 -nvidia-ml-py~=13.580.82 -nvidia-nccl-cu12~=2.30.4 -nvidia-nccl-cu13~=2.29.7 -nvidia-nvjitlink~=13.0.88 -nvidia-nvshmem-cu13~=3.4.5 -nvidia-nvtx~=13.0.85 oauthlib~=3.2.0 openai~=2.37.0 opencensus~=0.11.4 @@ -298,7 +276,6 @@ tornado~=6.5.1 tqdm~=4.67.1 traitlets~=5.14.3 transformers~=4.57.6 -triton~=3.7.0 typer~=0.25.1 typing-extensions~=4.15.0 typing-inspect~=0.9.0 diff --git a/python/dbr/19.x-gpu-ml-scala2.13/pyproject.toml b/python/dbr/19.x-gpu-ml-scala2.13/pyproject.toml index d396144..e001191 100644 --- a/python/dbr/19.x-gpu-ml-scala2.13/pyproject.toml +++ b/python/dbr/19.x-gpu-ml-scala2.13/pyproject.toml @@ -53,9 +53,6 @@ constraint-dependencies = [ "confection~=1.3.3", "contourpy~=1.3.3", "cryptography~=46.0.3", - "cuda-bindings~=13.2.0", - "cuda-pathfinder~=1.5.4", - "cuda-toolkit~=13.0.2", "cycler~=0.11.0", "cymem~=2.0.13", "databricks-agents~=1.10.2", @@ -66,7 +63,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.8.16", "decorator~=5.2.1", - "deepspeed~=0.19.0", "deltalake~=1.5.1", "deprecated~=1.3.1", "dill~=0.4.0", @@ -78,7 +74,6 @@ constraint-dependencies = [ "farama-notifications~=0.0.6", "fastapi~=0.136.3", "filelock~=3.20.0", - "flash-attn~=2.8.3", "flask~=2.2.5", "fonttools~=4.60.1", "frozenlist~=1.8.0", @@ -179,23 +174,6 @@ constraint-dependencies = [ "nodeenv~=1.10.0", "numba~=0.62.1", "numpy~=2.3.4", - "nvidia-cublas~=13.1.1.3", - "nvidia-cuda-cupti~=13.0.85", - "nvidia-cuda-nvrtc~=13.0.88", - "nvidia-cuda-runtime~=13.0.96", - "nvidia-cudnn-cu13~=9.20.0.48", - "nvidia-cufft~=12.0.0.61", - "nvidia-cufile~=1.15.1.6", - "nvidia-curand~=10.4.0.35", - "nvidia-cusolver~=12.0.4.66", - "nvidia-cusparse~=12.6.3.3", - "nvidia-cusparselt-cu13~=0.8.1", - "nvidia-ml-py~=13.580.82", - "nvidia-nccl-cu12~=2.30.4", - "nvidia-nccl-cu13~=2.29.7", - "nvidia-nvjitlink~=13.0.88", - "nvidia-nvshmem-cu13~=3.4.5", - "nvidia-nvtx~=13.0.85", "oauthlib~=3.2.0", "openai~=2.37.0", "opencensus~=0.11.4", @@ -310,7 +288,6 @@ constraint-dependencies = [ "tqdm~=4.67.1", "traitlets~=5.14.3", "transformers~=4.57.6", - "triton~=3.7.0", "typer~=0.25.1", "typing-extensions~=4.15.0", "typing-inspect~=0.9.0", diff --git a/python/serverless/serverless-v5-ml/constraints.txt b/python/serverless/serverless-v5-ml/constraints.txt index fbe29bd..ba246e1 100644 --- a/python/serverless/serverless-v5-ml/constraints.txt +++ b/python/serverless/serverless-v5-ml/constraints.txt @@ -66,7 +66,6 @@ datasets~=4.3.0 dbl-tempo~=0.1.26 debugpy~=1.8.11 decorator~=5.1.1 -deepspeed~=0.18.1 defusedxml~=0.7.1 deltalake~=1.1.4 deprecated~=1.2.18 @@ -210,7 +209,6 @@ notebook~=7.3.2 notebook-shim~=0.2.4 numba~=0.61.0 numpy~=2.1.3 -nvidia-nccl-cu12~=2.28.7 oauthlib~=3.2.0 oci~=2.163.1 openai~=2.7.0 @@ -347,9 +345,9 @@ tinycss2~=1.4.0 tokenize-rt~=6.1.0 tokenizers~=0.22.1 tomli~=2.0.1 -torch==2.9.0+cpu +torch~=2.9.0 torcheval~=0.0.7 -torchvision==0.24.0+cpu +torchvision~=0.24.0 tornado~=6.5.1 tqdm~=4.67.1 traitlets~=5.14.3 diff --git a/python/serverless/serverless-v5-ml/pyproject.toml b/python/serverless/serverless-v5-ml/pyproject.toml index 7d76472..a1f5f57 100644 --- a/python/serverless/serverless-v5-ml/pyproject.toml +++ b/python/serverless/serverless-v5-ml/pyproject.toml @@ -78,7 +78,6 @@ constraint-dependencies = [ "dbl-tempo~=0.1.26", "debugpy~=1.8.11", "decorator~=5.1.1", - "deepspeed~=0.18.1", "defusedxml~=0.7.1", "deltalake~=1.1.4", "deprecated~=1.2.18", @@ -222,7 +221,6 @@ constraint-dependencies = [ "notebook-shim~=0.2.4", "numba~=0.61.0", "numpy~=2.1.3", - "nvidia-nccl-cu12~=2.28.7", "oauthlib~=3.2.0", "oci~=2.163.1", "openai~=2.7.0", @@ -359,9 +357,9 @@ constraint-dependencies = [ "tokenize-rt~=6.1.0", "tokenizers~=0.22.1", "tomli~=2.0.1", - "torch==2.9.0+cpu", + "torch~=2.9.0", "torcheval~=0.0.7", - "torchvision==0.24.0+cpu", + "torchvision~=0.24.0", "tornado~=6.5.1", "tqdm~=4.67.1", "traitlets~=5.14.3",