From db97b2f729960b98dfd35198f61420ff2d8757cf Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Thu, 17 Sep 2026 19:21:14 -0400 Subject: [PATCH 1/7] Lock JupyterLite's browser packages at build time The package list moves to a single "jupyterlite" dependency group in pyproject.toml instead of being duplicated in the setup cell. --- doc/changes/dev/14314.other.rst | 1 + doc/conf.py | 2 + doc/jupyter_lite_config.py | 38 ++++++++++++++ doc/sphinxext/build_lite_wheel.py | 19 ++++--- doc/sphinxext/jupyterlite_data.py | 13 ++--- doc/sphinxext/jupyterlite_lock_specs.py | 66 +++++++++++++++++++++++++ doc/sphinxext/jupyterlite_setup_cell.py | 43 ++++++++++------ pyproject.toml | 13 +++++ 8 files changed, 168 insertions(+), 27 deletions(-) create mode 100644 doc/changes/dev/14314.other.rst create mode 100644 doc/jupyter_lite_config.py create mode 100644 doc/sphinxext/jupyterlite_lock_specs.py diff --git a/doc/changes/dev/14314.other.rst b/doc/changes/dev/14314.other.rst new file mode 100644 index 00000000000..43a577fb668 --- /dev/null +++ b/doc/changes/dev/14314.other.rst @@ -0,0 +1 @@ +Lock the JupyterLite browser kernel's extra packages at build time instead of installing them from the first notebook cell, by `Teon Brooks`_. diff --git a/doc/conf.py b/doc/conf.py index f28d499f539..e64423e5635 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -505,6 +505,8 @@ # sphinx-gallery writes link to auto_*/... at the root jupyterlite_contents = ["jupyterlite_contents/auto_*"] jupyterlite_bind_ipynb_suffix = False + # bakes the browser's extra packages into the Pyodide lock at build time + jupyterlite_config = "jupyter_lite_config.py" # served at the docs root (/mne_data/...) through html_extra_path below stage_lite_data(curpath / "lite_extra" / "mne_data") diff --git a/doc/jupyter_lite_config.py b/doc/jupyter_lite_config.py new file mode 100644 index 00000000000..e9acc669c90 --- /dev/null +++ b/doc/jupyter_lite_config.py @@ -0,0 +1,38 @@ +"""Build-time config for the JupyterLite site's Pyodide lock. + +Adds packages to the lock so Pyodide loads them at kernel initialization. +Packages Pyodide already curates keep their curated version. + +On a dev build, MNE is installed by jupyterlite_setup_cell instead. +""" + +# Authors: The MNE-Python contributors. +# License: BSD-3-Clause +# Copyright the MNE-Python contributors. + +import sys +from pathlib import Path + +sys.path.append(str(Path(__file__).parent / "sphinxext")) + +from jupyterlite_lock_specs import ( # noqa: E402 + jupyterlite_package_names, + jupyterlite_specs_to_lock, + mne_pypi_spec, +) + +c = get_config() # noqa: F821 + +specs = jupyterlite_specs_to_lock() +prefetch = jupyterlite_package_names() + +# a stable/maint build locks the real PyPI release instead of the dev wheel +# build_lite_wheel.py builds; see mne_pypi_spec for more info. +mne_spec = mne_pypi_spec() +if mne_spec is not None: + specs.append(mne_spec) + prefetch.append("mne") + +c.PyodideLockAddon.enabled = True +c.PyodideLockAddon.specs = specs +c.PyodideLockAddon.prefetch_extra = prefetch diff --git a/doc/sphinxext/build_lite_wheel.py b/doc/sphinxext/build_lite_wheel.py index 890e77c206c..3500b65f3a0 100644 --- a/doc/sphinxext/build_lite_wheel.py +++ b/doc/sphinxext/build_lite_wheel.py @@ -118,9 +118,16 @@ def build_wheel(): if __name__ == "__main__": - # Reuse a wheel that is already there, so repeat `make html` runs do not - # rebuild it. Remove doc/pypi (or `make clean`) to force a fresh one. - existing = find_wheels() - wheels = ", ".join(str(wheel) for wheel in (existing or build_wheel())) - verb = "Reusing" if existing else "Built" - print(f"[JupyterLite] {verb} MNE wheel(s) for the browser kernel: {wheels}") + from jupyterlite_lock_specs import mne_pypi_spec + + if mne_pypi_spec() is not None: + # a stable/maint build locks the real PyPI release instead + print("[JupyterLite] Stable build: no wheel needed, using the PyPI release") + else: + # Reuse a wheel that is already there, so repeat `make html` runs do + # not rebuild it. Remove doc/pypi (or `make clean`) to force a fresh + # one. + existing = find_wheels() + wheels = ", ".join(str(wheel) for wheel in (existing or build_wheel())) + verb = "Reusing" if existing else "Built" + print(f"[JupyterLite] {verb} MNE wheel(s) for the browser kernel: {wheels}") diff --git a/doc/sphinxext/jupyterlite_data.py b/doc/sphinxext/jupyterlite_data.py index 829380b4d3d..87ad21ee4f4 100644 --- a/doc/sphinxext/jupyterlite_data.py +++ b/doc/sphinxext/jupyterlite_data.py @@ -15,6 +15,7 @@ from pathlib import Path from build_lite_wheel import build_wheel, find_wheels +from jupyterlite_lock_specs import mne_pypi_spec from mne_doc_utils import sphinx_logger import mne @@ -175,9 +176,9 @@ def stage_lite_data(dst_base): sphinx_logger.info( f"[JupyterLite] Served data: {n_copied} files copied, {n_missing} missing" ) - # the development wheel, so the browser installs this MNE rather than the - # PyPI release: doc/sphinxext/build_lite_wheel.py puts it in doc/pypi, where - # the piplite addon indexes it; `make html` runs that first, so this is - # only a fallback for a bare sphinx-build - wheels = find_wheels() or build_wheel() - sphinx_logger.info(f"[JupyterLite] MNE wheel for the browser kernel: {wheels}") + # build the wheels for dev, default to pypi for release + if mne_pypi_spec() is None: + wheels = find_wheels() or build_wheel() + sphinx_logger.info(f"[JupyterLite] MNE wheel for the browser kernel: {wheels}") + else: + sphinx_logger.info("[JupyterLite] Stable build: locking the PyPI release") diff --git a/doc/sphinxext/jupyterlite_lock_specs.py b/doc/sphinxext/jupyterlite_lock_specs.py new file mode 100644 index 00000000000..fc66bb92852 --- /dev/null +++ b/doc/sphinxext/jupyterlite_lock_specs.py @@ -0,0 +1,66 @@ +"""Specs to add to Pyodide's lock for JupyterLite, versus what it already curates. + +Naming a package in ``PyodideLockAddon.specs`` drops its existing pin from +the lock (see ``pyodide_lock.uv_pip_compile.constraints_txt``) and resolves +it fresh from PyPI instead, a smaller and differently versioned set than +Pyodide's own curated wheels. :func:`jupyterlite_specs_to_lock` avoids that +by only including specs Pyodide's own lock does not already satisfy. + +:func:`mne_pypi_spec` is the same idea for MNE itself: a stable/maint build +can lock the real PyPI release, but a dev build's version has no PyPI release +to lock, so it needs the wheel ``build_lite_wheel.py`` builds instead. +""" + +# Authors: The MNE-Python contributors. +# License: BSD-3-Clause +# Copyright the MNE-Python contributors. + +import json +import tomllib +import urllib.request +from pathlib import Path + +from jupyterlite_pyodide_kernel.constants import PYODIDE_LOCK_DEFAULT_URL +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +_PYPROJECT_TOML = Path(__file__).parents[2] / "pyproject.toml" + + +def jupyterlite_specs(): + """PEP-508 specs in the "jupyterlite" dependency group of pyproject.toml.""" + with open(_PYPROJECT_TOML, "rb") as fid: + return tomllib.load(fid)["dependency-groups"]["jupyterlite"] + + +def jupyterlite_package_names(): + """Canonical package names in the "jupyterlite" dependency group.""" + return [canonicalize_name(Requirement(spec).name) for spec in jupyterlite_specs()] + + +def jupyterlite_specs_to_lock(): + """PEP-508 specs from the "jupyterlite" group Pyodide does not already satisfy.""" + with urllib.request.urlopen(PYODIDE_LOCK_DEFAULT_URL) as fid: + curated = json.load(fid)["packages"] + + to_lock = [] + for spec in jupyterlite_specs(): + req = Requirement(spec) + pkg = curated.get(canonicalize_name(req.name)) + if pkg is None or not req.specifier.contains(pkg["version"]): + to_lock.append(spec) + return to_lock + + +def mne_pypi_spec(): + """Return the ``mne==`` spec for a stable/maint build, else None. + + A ".dev" version (the docs building from ``main``) has no matching PyPI + release to lock against, so the browser kernel needs the wheel + ``build_lite_wheel.py`` builds from the working tree instead. + """ + import mne + + if ".dev" in mne.__version__: + return None + return f"mne=={mne.__version__}" diff --git a/doc/sphinxext/jupyterlite_setup_cell.py b/doc/sphinxext/jupyterlite_setup_cell.py index f0bd1b55c26..83faaa19a4c 100644 --- a/doc/sphinxext/jupyterlite_setup_cell.py +++ b/doc/sphinxext/jupyterlite_setup_cell.py @@ -1,35 +1,48 @@ """The setup cell prepended to every JupyterLite notebook. -It installs MNE into the browser kernel and hands over to -``mne.viz.backends._jupyterlite.setup_notebook``, which patches what Pyodide -does not provide. piplite (not micropip) prefers the development MNE wheel -bundled with the docs over PyPI, and ``keep_going`` reports a dependency with -no wheel instead of aborting. ``sys.platform`` is ``"emscripten"`` only inside -Pyodide, so the cell is a no-op in a local kernel and a notebook downloaded -from inside JupyterLite runs unchanged there. +It hands over to ``mne.viz.backends._jupyterlite.setup_notebook``, which +patches what Pyodide does not provide. ``sys.platform`` is ``"emscripten"`` +only inside Pyodide, so the cell is a no-op in a local kernel and a notebook +downloaded from inside JupyterLite runs unchanged there. The docs build prepends it only to the notebooks copied into the JupyterLite contents, not through ``first_notebook_cell``, which would also put it in the ``.ipynb`` offered for download. + +The "jupyterlite" dependency group in pyproject.toml, and MNE itself on a +stable/maint build, are already sitting in the browser kernel by the time +this cell runs, loaded at kernel start per jupyter_lite_config.py. A dev +build has no PyPI release of MNE to lock that way (see +jupyterlite_lock_specs.mne_pypi_spec), so this cell installs the development +wheel ``build_lite_wheel.py`` built instead. """ # Authors: The MNE-Python contributors. # License: BSD-3-Clause # Copyright the MNE-Python contributors. -# ruff: noqa: E501 # the install list stays on one line so the cell reads short - import ast -LITE_SETUP_CELL = """\ -# 💡 Added by the docs build: installs MNE into the browser kernel and adapts -# it to Pyodide. Does nothing outside JupyterLite. -import sys +from jupyterlite_lock_specs import mne_pypi_spec -if sys.platform == "emscripten": +_INSTALL_MNE = """\ import piplite - await piplite.install(["mne", "scikit-learn", "joblib", "pandas", "seaborn", "mne-connectivity", "nibabel", "pyvista-js", "pyxdf", "mffpy", "python-picard"], keep_going=True) # noqa: E501 + await piplite.install(["mne"], keep_going=True) +""" + +# a raw newline or backslash inside an f-string's {} needs Python >= 3.12 +# (PEP 701), below MNE's floor, so the piplite block above is substituted in +# by bare name; the \ right after it drops this template line's own newline, +# so a blank line between "if" and "from" only appears when that block +# (which ends with one) is substituted in +LITE_SETUP_CELL = f"""\ +# 💡 Added by the docs build: adapts MNE to Pyodide. Does nothing outside +# JupyterLite. +import sys + +if sys.platform == "emscripten": +{_INSTALL_MNE if mne_pypi_spec() is None else ""}\ from mne.viz.backends._jupyterlite import setup_notebook setup_notebook() diff --git a/pyproject.toml b/pyproject.toml index 91ca7ed19a3..a46a83df435 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,19 @@ doc = [ "sphinxcontrib-youtube", ] doc-full = ["rpy2", { include-group = "doc" }] +# Packages installed into the browser Pyodide kernel for JupyterLite notebooks +jupyterlite = [ + "joblib >= 0.8", + "mffpy >= 0.11.0", + "mne-connectivity", + "nibabel >= 5.2", + "pandas >= 2.2, != 3.0.4", + "python-picard >= 0.4", + "pyvista-js >= 0.16; python_version >= '3.12'", + "pyxdf", + "scikit-learn >= 1.5", + "seaborn >= 0.5, != 0.11.2", +] # Dependencies in a separate group for uv lockfile generation, currently only used for # the 'old' environment CI test run lockfile_extras = [ From 3ffe162107f0a1ead07e55a80ae64c7f92354b54 Mon Sep 17 00:00:00 2001 From: Teon L Brooks Date: Thu, 17 Sep 2026 19:43:48 -0400 Subject: [PATCH 2/7] cleanup --- doc/sphinxext/jupyterlite_lock_specs.py | 11 +++-------- doc/sphinxext/jupyterlite_setup_cell.py | 11 ----------- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/doc/sphinxext/jupyterlite_lock_specs.py b/doc/sphinxext/jupyterlite_lock_specs.py index fc66bb92852..45cc2a763bc 100644 --- a/doc/sphinxext/jupyterlite_lock_specs.py +++ b/doc/sphinxext/jupyterlite_lock_specs.py @@ -28,7 +28,7 @@ def jupyterlite_specs(): - """PEP-508 specs in the "jupyterlite" dependency group of pyproject.toml.""" + """Retrieve the "jupyterlite" dependency group from pyproject.toml.""" with open(_PYPROJECT_TOML, "rb") as fid: return tomllib.load(fid)["dependency-groups"]["jupyterlite"] @@ -39,7 +39,7 @@ def jupyterlite_package_names(): def jupyterlite_specs_to_lock(): - """PEP-508 specs from the "jupyterlite" group Pyodide does not already satisfy.""" + """Retrieve specs from "jupyterlite" group Pyodide does not already satisfy.""" with urllib.request.urlopen(PYODIDE_LOCK_DEFAULT_URL) as fid: curated = json.load(fid)["packages"] @@ -53,12 +53,7 @@ def jupyterlite_specs_to_lock(): def mne_pypi_spec(): - """Return the ``mne==`` spec for a stable/maint build, else None. - - A ".dev" version (the docs building from ``main``) has no matching PyPI - release to lock against, so the browser kernel needs the wheel - ``build_lite_wheel.py`` builds from the working tree instead. - """ + """Return the ``mne==`` spec for a stable/maint build, else None.""" import mne if ".dev" in mne.__version__: diff --git a/doc/sphinxext/jupyterlite_setup_cell.py b/doc/sphinxext/jupyterlite_setup_cell.py index 83faaa19a4c..ddc19465661 100644 --- a/doc/sphinxext/jupyterlite_setup_cell.py +++ b/doc/sphinxext/jupyterlite_setup_cell.py @@ -8,13 +8,6 @@ The docs build prepends it only to the notebooks copied into the JupyterLite contents, not through ``first_notebook_cell``, which would also put it in the ``.ipynb`` offered for download. - -The "jupyterlite" dependency group in pyproject.toml, and MNE itself on a -stable/maint build, are already sitting in the browser kernel by the time -this cell runs, loaded at kernel start per jupyter_lite_config.py. A dev -build has no PyPI release of MNE to lock that way (see -jupyterlite_lock_specs.mne_pypi_spec), so this cell installs the development -wheel ``build_lite_wheel.py`` built instead. """ # Authors: The MNE-Python contributors. @@ -31,10 +24,6 @@ await piplite.install(["mne"], keep_going=True) """ -# a raw newline or backslash inside an f-string's {} needs Python >= 3.12 -# (PEP 701), below MNE's floor, so the piplite block above is substituted in -# by bare name; the \ right after it drops this template line's own newline, -# so a blank line between "if" and "from" only appears when that block # (which ends with one) is substituted in LITE_SETUP_CELL = f"""\ # 💡 Added by the docs build: adapts MNE to Pyodide. Does nothing outside From b1d33ea1d37444ddc157a5d8c982a75aa5466fb5 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 18 Sep 2026 11:08:27 -0400 Subject: [PATCH 3/7] Simplify [circle full] --- doc/jupyter_lite_config.py | 26 +++++++++---------------- doc/sphinxext/build_lite_wheel.py | 19 ++++++------------ doc/sphinxext/jupyterlite_data.py | 11 ++++------- doc/sphinxext/jupyterlite_lock_specs.py | 13 ------------- doc/sphinxext/jupyterlite_setup_cell.py | 26 +++++++------------------ pyproject.toml | 23 +++++++++++++--------- 6 files changed, 40 insertions(+), 78 deletions(-) diff --git a/doc/jupyter_lite_config.py b/doc/jupyter_lite_config.py index e9acc669c90..07dd5b34af3 100644 --- a/doc/jupyter_lite_config.py +++ b/doc/jupyter_lite_config.py @@ -1,9 +1,8 @@ """Build-time config for the JupyterLite site's Pyodide lock. -Adds packages to the lock so Pyodide loads them at kernel initialization. -Packages Pyodide already curates keep their curated version. - -On a dev build, MNE is installed by jupyterlite_setup_cell instead. +Adds MNE and the packages the notebooks import to the lock, so Pyodide loads +them at kernel initialization. Packages Pyodide already curates keep their +curated version. """ # Authors: The MNE-Python contributors. @@ -18,21 +17,14 @@ from jupyterlite_lock_specs import ( # noqa: E402 jupyterlite_package_names, jupyterlite_specs_to_lock, - mne_pypi_spec, ) c = get_config() # noqa: F821 -specs = jupyterlite_specs_to_lock() -prefetch = jupyterlite_package_names() - -# a stable/maint build locks the real PyPI release instead of the dev wheel -# build_lite_wheel.py builds; see mne_pypi_spec for more info. -mne_spec = mne_pypi_spec() -if mne_spec is not None: - specs.append(mne_spec) - prefetch.append("mne") - c.PyodideLockAddon.enabled = True -c.PyodideLockAddon.specs = specs -c.PyodideLockAddon.prefetch_extra = prefetch +# the MNE wheel build_lite_wheel.py builds from this checkout (a tagged +# checkout gives the release): the solve takes it over PyPI, which would +# otherwise supply the latest release through mne-connectivity's mne>=1.6 +c.PyodideLockAddon.wheels = ["pypi"] +c.PyodideLockAddon.specs = ["mne", *jupyterlite_specs_to_lock()] +c.PyodideLockAddon.prefetch_extra = ["mne", *jupyterlite_package_names()] diff --git a/doc/sphinxext/build_lite_wheel.py b/doc/sphinxext/build_lite_wheel.py index 3500b65f3a0..890e77c206c 100644 --- a/doc/sphinxext/build_lite_wheel.py +++ b/doc/sphinxext/build_lite_wheel.py @@ -118,16 +118,9 @@ def build_wheel(): if __name__ == "__main__": - from jupyterlite_lock_specs import mne_pypi_spec - - if mne_pypi_spec() is not None: - # a stable/maint build locks the real PyPI release instead - print("[JupyterLite] Stable build: no wheel needed, using the PyPI release") - else: - # Reuse a wheel that is already there, so repeat `make html` runs do - # not rebuild it. Remove doc/pypi (or `make clean`) to force a fresh - # one. - existing = find_wheels() - wheels = ", ".join(str(wheel) for wheel in (existing or build_wheel())) - verb = "Reusing" if existing else "Built" - print(f"[JupyterLite] {verb} MNE wheel(s) for the browser kernel: {wheels}") + # Reuse a wheel that is already there, so repeat `make html` runs do not + # rebuild it. Remove doc/pypi (or `make clean`) to force a fresh one. + existing = find_wheels() + wheels = ", ".join(str(wheel) for wheel in (existing or build_wheel())) + verb = "Reusing" if existing else "Built" + print(f"[JupyterLite] {verb} MNE wheel(s) for the browser kernel: {wheels}") diff --git a/doc/sphinxext/jupyterlite_data.py b/doc/sphinxext/jupyterlite_data.py index 87ad21ee4f4..8fb06084220 100644 --- a/doc/sphinxext/jupyterlite_data.py +++ b/doc/sphinxext/jupyterlite_data.py @@ -15,7 +15,6 @@ from pathlib import Path from build_lite_wheel import build_wheel, find_wheels -from jupyterlite_lock_specs import mne_pypi_spec from mne_doc_utils import sphinx_logger import mne @@ -176,9 +175,7 @@ def stage_lite_data(dst_base): sphinx_logger.info( f"[JupyterLite] Served data: {n_copied} files copied, {n_missing} missing" ) - # build the wheels for dev, default to pypi for release - if mne_pypi_spec() is None: - wheels = find_wheels() or build_wheel() - sphinx_logger.info(f"[JupyterLite] MNE wheel for the browser kernel: {wheels}") - else: - sphinx_logger.info("[JupyterLite] Stable build: locking the PyPI release") + # the MNE wheel jupyter_lite_config.py locks for the browser: `make html` + # builds it first, so this is only a fallback for a bare sphinx-build + wheels = find_wheels() or build_wheel() + sphinx_logger.info(f"[JupyterLite] MNE wheel for the browser kernel: {wheels}") diff --git a/doc/sphinxext/jupyterlite_lock_specs.py b/doc/sphinxext/jupyterlite_lock_specs.py index 45cc2a763bc..ab795596d4b 100644 --- a/doc/sphinxext/jupyterlite_lock_specs.py +++ b/doc/sphinxext/jupyterlite_lock_specs.py @@ -5,10 +5,6 @@ it fresh from PyPI instead, a smaller and differently versioned set than Pyodide's own curated wheels. :func:`jupyterlite_specs_to_lock` avoids that by only including specs Pyodide's own lock does not already satisfy. - -:func:`mne_pypi_spec` is the same idea for MNE itself: a stable/maint build -can lock the real PyPI release, but a dev build's version has no PyPI release -to lock, so it needs the wheel ``build_lite_wheel.py`` builds instead. """ # Authors: The MNE-Python contributors. @@ -50,12 +46,3 @@ def jupyterlite_specs_to_lock(): if pkg is None or not req.specifier.contains(pkg["version"]): to_lock.append(spec) return to_lock - - -def mne_pypi_spec(): - """Return the ``mne==`` spec for a stable/maint build, else None.""" - import mne - - if ".dev" in mne.__version__: - return None - return f"mne=={mne.__version__}" diff --git a/doc/sphinxext/jupyterlite_setup_cell.py b/doc/sphinxext/jupyterlite_setup_cell.py index ddc19465661..bfa631523fe 100644 --- a/doc/sphinxext/jupyterlite_setup_cell.py +++ b/doc/sphinxext/jupyterlite_setup_cell.py @@ -1,9 +1,11 @@ """The setup cell prepended to every JupyterLite notebook. It hands over to ``mne.viz.backends._jupyterlite.setup_notebook``, which -patches what Pyodide does not provide. ``sys.platform`` is ``"emscripten"`` -only inside Pyodide, so the cell is a no-op in a local kernel and a notebook -downloaded from inside JupyterLite runs unchanged there. +patches what Pyodide does not provide; MNE itself and the packages the +notebooks import are in the Pyodide lock (see ``jupyter_lite_config.py``). +``sys.platform`` is ``"emscripten"`` only inside Pyodide, so the cell is a +no-op in a local kernel and a notebook downloaded from inside JupyterLite runs +unchanged there. The docs build prepends it only to the notebooks copied into the JupyterLite contents, not through ``first_notebook_cell``, which would also put it in the @@ -14,29 +16,15 @@ # License: BSD-3-Clause # Copyright the MNE-Python contributors. -import ast - -from jupyterlite_lock_specs import mne_pypi_spec - -_INSTALL_MNE = """\ - import piplite - - await piplite.install(["mne"], keep_going=True) -""" - -# (which ends with one) is substituted in -LITE_SETUP_CELL = f"""\ +LITE_SETUP_CELL = """\ # 💡 Added by the docs build: adapts MNE to Pyodide. Does nothing outside # JupyterLite. import sys if sys.platform == "emscripten": -{_INSTALL_MNE if mne_pypi_spec() is None else ""}\ from mne.viz.backends._jupyterlite import setup_notebook setup_notebook() """ # nothing else runs this before a reader does, so at least make sure it parses -compile( - LITE_SETUP_CELL, "lite_setup_cell", "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT -) +compile(LITE_SETUP_CELL, "lite_setup_cell", "exec") diff --git a/pyproject.toml b/pyproject.toml index a46a83df435..2fd89b7d710 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,9 @@ doc = [ # the two resolve together. Below either, the browser drops to Pyodide 0.29 # and its matplotlib 3.8.4, a minor under the 3.9 MNE declares. "jupyterlite-pyodide-kernel >= 0.8", + # [lock] brings pyodide-lock and uv for jupyter_lite_config.py; pyodide-lock + # needs 3.12, which every docs build has + "jupyterlite-pyodide-kernel[lock] >= 0.8; python_version >= '3.12'", "jupyterlite-sphinx >= 0.23", "memory_profiler >= 0.16", "mne-bids", @@ -44,18 +47,20 @@ doc = [ "sphinxcontrib-youtube", ] doc-full = ["rpy2", { include-group = "doc" }] -# Packages installed into the browser Pyodide kernel for JupyterLite notebooks +# Packages installed into the browser Pyodide kernel for JupyterLite notebooks; +# unpinned, since Pyodide's curated version wins where it has one and the rest +# resolve to the latest release (see doc/sphinxext/jupyterlite_lock_specs.py) jupyterlite = [ - "joblib >= 0.8", - "mffpy >= 0.11.0", + "joblib", + "mffpy", "mne-connectivity", - "nibabel >= 5.2", - "pandas >= 2.2, != 3.0.4", - "python-picard >= 0.4", - "pyvista-js >= 0.16; python_version >= '3.12'", + "nibabel", + "pandas", + "python-picard", + "pyvista-js", "pyxdf", - "scikit-learn >= 1.5", - "seaborn >= 0.5, != 0.11.2", + "scikit-learn", + "seaborn", ] # Dependencies in a separate group for uv lockfile generation, currently only used for # the 'old' environment CI test run From fef8fcf5b134ac321ad906907d6428aae43ee869 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 18 Sep 2026 12:26:29 -0400 Subject: [PATCH 4/7] FIX: Stabilize [circle full] --- azure-pipelines.yml | 87 +++++-------------------- mne/datasets/_fetch.py | 3 +- mne/datasets/_fsaverage/base.py | 4 +- mne/datasets/_phantom/base.py | 2 +- mne/datasets/config.py | 70 ++++++++++++-------- mne/datasets/utils.py | 11 ++-- mne/viz/backends/tests/test_renderer.py | 8 ++- 7 files changed, 76 insertions(+), 109 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d85b5999387..320e73c9ddd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -47,12 +47,6 @@ stages: PYTEST_XDIST_N: '2' # Microsoft-hosted agents have 2 cores MNE_TEST_ALLOW_SKIP: '^$' # nothing MNE_BROWSER_PRECOMPUTE: 'false' - # See tools/github_actions_env_vars.sh for why these are set (this job - # is on Python 3.14, so sys.monitoring can measure branches) - COVERAGE_CORE: 'sysmon' - UV_CACHE_DIR: '/home/vsts/.cache/uv' - UV_SYSTEM_PYTHON: 'true' # UsePythonVersion gives us a bare interpreter, not a venv - UV_NO_PROGRESS: 'true' steps: - bash: | set -xeo pipefail @@ -62,27 +56,21 @@ stages: displayName: Setup up Xvfb - task: Cache@2 inputs: - key: '"minimal-cmds" | "$(Agent.OS)" | $(MNE_TOOLS_DIR)/tools/get_minimal_commands.sh' + key: minimal-cmds-1 path: /home/vsts/minimal_cmds displayName: Cache minimal commands - - bash: $(MNE_TOOLS_DIR)/tools/get_minimal_commands.sh + - bash: ./tools/get_minimal_commands.sh displayName: Get minimal commands - task: UsePythonVersion@0 inputs: versionSpec: '3.14' architecture: 'x64' addToPath: true - - task: Cache@2 - inputs: - key: '"uv" | "$(Agent.OS)" | pyproject.toml' - restoreKeys: '"uv" | "$(Agent.OS)"' - path: $(UV_CACHE_DIR) - displayName: Cache uv - bash: | set -xeo pipefail - python -m pip install --progress-bar off --upgrade "uv>=0.9" - uv pip install --upgrade --only-binary=":all:" --no-binary=mne --no-binary=mne-qt-browser -e . --group=test "mne-qt-browser @ https://github.com/mne-tools/mne-qt-browser/archive/refs/heads/main.zip" pyvista scikit-learn python-picard qtpy nibabel sphinx-gallery "PySide6!=6.8.0,!=6.8.0.1,!=6.8.1.1,!=6.9.1" pandas neo pymatreader antio defusedxml curryreader pymef openmeeg - displayName: Install dependencies with uv + python -m pip install --progress-bar off --upgrade pip + python -m pip install --progress-bar off --upgrade --only-binary=":all:" -e . --group=test "mne-qt-browser @ https://github.com/mne-tools/mne-qt-browser/archive/refs/heads/main.zip" pyvista scikit-learn python-picard qtpy nibabel sphinx-gallery "PySide6!=6.8.0,!=6.8.0.1,!=6.8.1.1,!=6.9.1" pandas neo pymatreader antio defusedxml curryreader pymef openmeeg + displayName: Install dependencies with pip - bash: $(MNE_TOOLS_DIR)/tools/check_qt_import.sh PySide6 displayName: Check Qt import - bash: | @@ -94,7 +82,7 @@ stages: displayName: Get testing version - task: Cache@2 inputs: - key: mne_data-testing-$(testing_version)-misc-$(misc_version) + key: $(testing_version) path: /home/vsts/mne_data displayName: Cache testing data - bash: ./tools/github_actions_download.sh @@ -117,8 +105,6 @@ stages: PYTEST_XDIST_N: '2' # Microsoft-hosted agents have 2 cores TEST_OPTIONS: "--tb=short --cov=mne --cov-report=xml --cov-append -vv mne/gui mne/viz/_brain mne/viz/backends mne/viz/tests/test_evoked.py mne/report" MNE_TEST_ALLOW_SKIP: '^$' # nothing (can be overridden below) - # See tools/github_actions_env_vars.sh; this job is on Python 3.14 - COVERAGE_CORE: 'sysmon' steps: - bash: | set -xeo pipefail @@ -147,7 +133,7 @@ stages: displayName: Get testing version - task: Cache@2 inputs: - key: mne_data-testing-$(testing_version)-misc-$(misc_version) + key: $(testing_version) path: /home/vsts/mne_data displayName: Cache testing data - bash: ./tools/github_actions_download.sh @@ -180,22 +166,16 @@ stages: pool: vmImage: 'windows-latest' variables: - MNE_TOOLS_DIR: '$(Pipeline.Workspace)/mne-tools' MNE_LOGGING_LEVEL: 'warning' MNE_FORCE_SERIAL: 'true' PYTEST_XDIST_N: '2' # Microsoft-hosted agents have 2 cores OMP_DYNAMIC: 'false' PYTHONUNBUFFERED: 1 - # See tools/github_actions_env_vars.sh; both matrix legs are Python 3.14 - COVERAGE_CORE: 'sysmon' - # See tools/github_actions_env_vars.sh for why these three are set - NUMBA_CACHE_DIR: 'C:\Users\VssAdministrator\.cache\mne-numba' - NUMBA_CPU_NAME: 'generic' - NUMBA_CPU_FEATURES: '' - UV_CACHE_DIR: 'C:\Users\VssAdministrator\.cache\uv' - UV_SYSTEM_PYTHON: 'true' # UsePythonVersion gives us a bare interpreter, not a venv - UV_NO_PROGRESS: 'true' AZURE_CI_WINDOWS: 'true' + # Mesa's llvmpipe rasterizer threads can deadlock at process exit on + # Windows (pyvista/setup-headless-display-action#55); the action sets + # this itself on GHA, but here we only run its install script + LP_NUM_THREADS: '0' MNE_CI_KIND: $(TEST_MODE) MNE_TEST_ALLOW_SKIP: $(MNE_TEST_ALLOW_SKIP) timeoutInMinutes: 40 @@ -205,47 +185,25 @@ stages: 3.14 pip: TEST_MODE: 'pip' PYTHON_VERSION: '3.14' - MNE_TEST_ALLOW_SKIP: "^.*(Requires (spm|brainstorm) dataset|Requires FreeSurfer|CUDA not|on Windows|MNE_FORCE_SERIAL|PySide6 causes segfaults|SCIPY_ARRAY_API).*$" + MNE_TEST_ALLOW_SKIP: "^.*(Requires (spm|brainstorm) dataset|Requires MNE-C|Requires FreeSurfer|CUDA not|on Windows|MNE_FORCE_SERIAL|PySide6 causes segfaults|SCIPY_ARRAY_API).*$" 3.14 pip pre: TEST_MODE: 'pip-pre' PYTHON_VERSION: '3.14' - MNE_TEST_ALLOW_SKIP: "^.*(Requires (spm|brainstorm) dataset|Requires FreeSurfer|CUDA not|Numba not|on Windows|MNE_FORCE_SERIAL|PySide6 causes segfaults|SCIPY_ARRAY_API).*$" + MNE_TEST_ALLOW_SKIP: "^.*(Requires (spm|brainstorm) dataset|Requires MNE-C|Requires FreeSurfer|CUDA not|Numba not|on Windows|MNE_FORCE_SERIAL|PySide6 causes segfaults|SCIPY_ARRAY_API).*$" steps: - task: UsePythonVersion@0 inputs: versionSpec: $(PYTHON_VERSION) architecture: 'x64' addToPath: true - - bash: | - set -xeo pipefail - git clone --depth 1 https://github.com/mne-tools/mne-tools.git "$MNE_TOOLS_DIR" - displayName: Clone mne-tools - - task: Cache@2 - inputs: - key: '"minimal-cmds" | "$(Agent.OS)" | $(MNE_TOOLS_DIR)/tools/get_minimal_commands.sh' - path: C:\Users\VssAdministrator\minimal_cmds - displayName: Cache minimal commands - # MNE_TOOLS_DIR is a Windows path and these steps run under Git Bash, hence - # cygpath; backticks rather than $(...) so Azure does not read it as a macro - - bash: | - set -eo pipefail - MNE_TOOLS_UNIX=`cygpath -u "$MNE_TOOLS_DIR"` - bash "$MNE_TOOLS_UNIX/tools/get_minimal_commands.sh" - displayName: Get minimal commands - bash: | set -xeo pipefail git clone --depth 1 https://github.com/pyvista/setup-headless-display-action.git MESA3D_VERSION=24.3.0 bash setup-headless-display-action/windows/install_opengl.sh displayName: Install OpenGL - - task: Cache@2 - inputs: - key: '"uv" | "$(Agent.OS)" | "$(PYTHON_VERSION)" | "$(TEST_MODE)" | pyproject.toml' - restoreKeys: '"uv" | "$(Agent.OS)" | "$(PYTHON_VERSION)" | "$(TEST_MODE)"' - path: $(UV_CACHE_DIR) - displayName: Cache uv - bash: ./tools/azure_dependencies.sh displayName: Install dependencies - - script: uv pip install --only-binary=":all:" --no-binary=mne -e . + - script: pip install --only-binary=":all:" -e . displayName: Install mne - script: mne sys_info -pd displayName: Print config @@ -257,26 +215,11 @@ stages: displayName: Get testing version - task: Cache@2 inputs: - key: mne_data-testing-$(testing_version)-misc-$(misc_version) + key: $(testing_version) path: C:\Users\VssAdministrator\mne_data displayName: Cache testing data - bash: ./tools/github_actions_download.sh displayName: Download testing data - - bash: ./tools/get_numba_version.sh - displayName: Get numba version - - task: Cache@2 - inputs: - key: 'mne_numba | "$(Agent.OS)" | "$(PYTHON_VERSION)" | "$(numba_version)" | mne/**/*numba*.py' - path: C:\Users\VssAdministrator\.cache\mne-numba - displayName: Cache numba - # pip-pre has no numba when nightly wheels are missing; without it - # nothing creates the cache dir and the post-job save fails on tar - condition: ne(variables['numba_version'], 'none') - # Windows does not define HOME, and MNE-C dereferences it unconditionally: - # mne_forward_solution crashes with 0xC0000005 without it. Set after the - # steps above so that Git Bash keeps deriving its own MSYS-style HOME. - - script: 'echo ##vso[task.setvariable variable=HOME]%USERPROFILE%' - displayName: Set HOME for MNE-C - script: pytest -m "not (slowtest or pgtest)" -n $(PYTEST_XDIST_N) --dist loadscope --timeout=120 --timeout-method=thread -o faulthandler_timeout=110 --tb=short --cov=mne --cov-report=xml -vv mne displayName: Run tests - bash: bash <(curl -s https://codecov.io/bash) diff --git a/mne/datasets/_fetch.py b/mne/datasets/_fetch.py index 6ae66814dc4..0151e60dca8 100644 --- a/mne/datasets/_fetch.py +++ b/mne/datasets/_fetch.py @@ -126,7 +126,8 @@ def fetch_dataset( {'dataset_name': 'sample', 'archive_name': 'MNE-sample-data-processed.tar.gz', 'hash': 'md5:e8f30c4516abdc12a0c08e6bae57409c', - 'url': 'https://osf.io/download/86qa2?version=6', + 'url': 'https://github.com/mne-tools/mne-data/releases/download/' + 'sample-6/MNE-sample-data-processed.tar.gz', 'folder_name': 'MNE-sample-data', 'config_key': 'MNE_DATASETS_SAMPLE_PATH'} diff --git a/mne/datasets/_fsaverage/base.py b/mne/datasets/_fsaverage/base.py index e6a1aa51ea5..2e83b20e870 100644 --- a/mne/datasets/_fsaverage/base.py +++ b/mne/datasets/_fsaverage/base.py @@ -73,13 +73,13 @@ def fetch_fsaverage(subjects_dir=None, *, verbose=None): _manifest_check_download( manifest_path=FSAVERAGE_MANIFEST_PATH / "root.txt", destination=subjects_dir, - url="https://osf.io/download/3bxqt?version=2", + url="https://github.com/mne-tools/mne-data/releases/download/fsaverage-1.0/fsaverage-root.zip", hash_="5133fe92b7b8f03ae19219d5f46e4177", ) _manifest_check_download( manifest_path=FSAVERAGE_MANIFEST_PATH / "bem.txt", destination=subjects_dir / "fsaverage", - url="https://osf.io/download/7ve8g?version=4", + url="https://github.com/mne-tools/mne-data/releases/download/fsaverage-1.0/fsaverage-bem.zip", hash_="b31509cdcf7908af6a83dc5ee8f49fb1", ) return fs_dir diff --git a/mne/datasets/_phantom/base.py b/mne/datasets/_phantom/base.py index 0cd54957192..c2c17243246 100644 --- a/mne/datasets/_phantom/base.py +++ b/mne/datasets/_phantom/base.py @@ -48,7 +48,7 @@ def fetch_phantom(kind="otaniemi", subjects_dir=None, *, verbose=None): """ phantoms = dict( otaniemi=dict( - url="https://osf.io/download/j5czy?version=1", + url="https://github.com/mne-tools/mne-data/releases/download/phantom_otaniemi-1/phantom_otaniemi.zip", hash="42d17db5b1db3e30327ffb4cf2649de8", ), ) diff --git a/mne/datasets/config.py b/mne/datasets/config.py index 0e2c581f450..8b7ade4e9ff 100644 --- a/mne/datasets/config.py +++ b/mne/datasets/config.py @@ -140,7 +140,7 @@ MNE_DATASETS["fnirs_motor"] = dict( archive_name="MNE-fNIRS-motor-data.tgz", hash="md5:c4935d19ddab35422a69f3326a01fef8", - url="https://osf.io/download/dj3eh?version=1", + url="https://github.com/mne-tools/mne-data/releases/download/fnirs_motor-1/MNE-fNIRS-motor-data.tgz", folder_name="MNE-fNIRS-motor-data", config_key="MNE_DATASETS_FNIRS_MOTOR_PATH", ) @@ -148,7 +148,8 @@ MNE_DATASETS["ucl_opm_auditory"] = dict( archive_name="auditory_OPM_stationary.zip", hash="md5:b2d69aa2d656b960bd0c18968dc1a14d", - url="https://osf.io/download/tp324?version=1", # original is mwrt3 + # original is https://osf.io/mwrt3 + url="https://github.com/mne-tools/mne-data/releases/download/ucl_opm_auditory-1/auditory_OPM_stationary.zip", folder_name="auditory_OPM_stationary", config_key="MNE_DATASETS_UCL_OPM_AUDITORY_PATH", ) @@ -156,7 +157,7 @@ MNE_DATASETS["kiloword"] = dict( archive_name="MNE-kiloword-data.tar.gz", hash="md5:3a124170795abbd2e48aae8727e719a8", - url="https://osf.io/download/qkvf9?version=1", + url="https://github.com/mne-tools/mne-data/releases/download/kiloword-1/MNE-kiloword-data.tar.gz", folder_name="MNE-kiloword-data", config_key="MNE_DATASETS_KILOWORD_PATH", ) @@ -172,7 +173,7 @@ MNE_DATASETS["opm"] = dict( archive_name="MNE-OPM-data.tar.gz", hash="md5:370ad1dcfd5c47e029e692c85358a374", - url="https://osf.io/download/p6ae7?version=2", + url="https://github.com/mne-tools/mne-data/releases/download/opm-2/MNE-OPM-data.tar.gz", folder_name="MNE-OPM-data", config_key="MNE_DATASETS_OPM_PATH", ) @@ -180,7 +181,7 @@ MNE_DATASETS["phantom_kit"] = dict( archive_name="MNE-phantom-KIT-data.tar.gz", hash="md5:7bfdf40bbeaf17a66c99c695640e0740", - url="https://osf.io/download/fb6ya?version=1", + url="https://github.com/mne-tools/mne-data/releases/download/phantom_kit-1/MNE-phantom-KIT-data.tar.gz", folder_name="MNE-phantom-KIT-data", config_key="MNE_DATASETS_PHANTOM_KIT_PATH", ) @@ -188,7 +189,7 @@ MNE_DATASETS["phantom_4dbti"] = dict( archive_name="MNE-phantom-4DBTi.zip", hash="md5:938a601440f3ffa780d20a17bae039ff", - url="https://osf.io/download/v2brw?version=2", + url="https://github.com/mne-tools/mne-data/releases/download/phantom_4dbti-2/MNE-phantom-4DBTi.zip", folder_name="MNE-phantom-4DBTi", config_key="MNE_DATASETS_PHANTOM_4DBTI_PATH", ) @@ -196,7 +197,7 @@ MNE_DATASETS["phantom_kernel"] = dict( archive_name="MNE-phantom-kernel.tar.gz", hash="md5:4e2ad987dac1a20f95bae8ffeb2d41d6", - url="https://osf.io/download/dj7wz?version=1", + url="https://github.com/mne-tools/mne-data/releases/download/phantom_kernel-1/MNE-phantom-kernel.tar.gz", folder_name="MNE-phantom-kernel-data", config_key="MNE_DATASETS_PHANTOM_KERNEL_PATH", ) @@ -204,7 +205,7 @@ MNE_DATASETS["sample"] = dict( archive_name="MNE-sample-data-processed.tar.gz", hash="md5:e8f30c4516abdc12a0c08e6bae57409c", - url="https://osf.io/download/86qa2?version=6", + url="https://github.com/mne-tools/mne-data/releases/download/sample-6/MNE-sample-data-processed.tar.gz", folder_name="MNE-sample-data", config_key="MNE_DATASETS_SAMPLE_PATH", ) @@ -221,7 +222,7 @@ MNE_DATASETS["somato"] = dict( archive_name="MNE-somato-data.tar.gz", hash="md5:9a191907b326b9402341ee7a0d1240d8", - url="https://osf.io/download/tp4sg?version=8", + url="https://github.com/mne-tools/mne-data/releases/download/somato-8/MNE-somato-data.tar.gz", folder_name="MNE-somato-data", config_key="MNE_DATASETS_SOMATO_PATH", ) @@ -229,7 +230,7 @@ MNE_DATASETS["spm"] = dict( archive_name="MNE-spm-face.tar.gz", hash="md5:9f43f67150e3b694b523a21eb929ea75", - url="https://osf.io/download/je4s8?version=2", + url="https://github.com/mne-tools/mne-data/releases/download/spm-2/MNE-spm-face.tar.gz", folder_name="MNE-spm-face", config_key="MNE_DATASETS_SPM_FACE_PATH", ) @@ -241,17 +242,32 @@ folder_name="MNE-visual_92_categories-data", config_key="MNE_DATASETS_VISUAL_92_CATEGORIES_PATH", ) +# four parts, each under GitHub's 2 GiB release-asset limit MNE_DATASETS["visual_92_categories_1"] = dict( archive_name="MNE-visual_92_categories-data-part1.tar.gz", - hash="md5:74f50bbeb65740903eadc229c9fa759f", - url="https://osf.io/download/8ejrs?version=1", + hash="md5:b7ee73c69c1c086354ae370c66b41f0b", + url="https://github.com/mne-tools/mne-data/releases/download/visual_92_categories-2/MNE-visual_92_categories-data-part1.tar.gz", folder_name="MNE-visual_92_categories-data", config_key="MNE_DATASETS_VISUAL_92_CATEGORIES_PATH", ) MNE_DATASETS["visual_92_categories_2"] = dict( archive_name="MNE-visual_92_categories-data-part2.tar.gz", - hash="md5:203410a98afc9df9ae8ba9f933370e20", - url="https://osf.io/download/t4yjp?version=1", + hash="md5:f35b89c76a76f2341e68d60b46578621", + url="https://github.com/mne-tools/mne-data/releases/download/visual_92_categories-2/MNE-visual_92_categories-data-part2.tar.gz", + folder_name="MNE-visual_92_categories-data", + config_key="MNE_DATASETS_VISUAL_92_CATEGORIES_PATH", +) +MNE_DATASETS["visual_92_categories_3"] = dict( + archive_name="MNE-visual_92_categories-data-part3.tar.gz", + hash="md5:94b68c9c3ce267cc82106518129c6221", + url="https://github.com/mne-tools/mne-data/releases/download/visual_92_categories-2/MNE-visual_92_categories-data-part3.tar.gz", + folder_name="MNE-visual_92_categories-data", + config_key="MNE_DATASETS_VISUAL_92_CATEGORIES_PATH", +) +MNE_DATASETS["visual_92_categories_4"] = dict( + archive_name="MNE-visual_92_categories-data-part4.tar.gz", + hash="md5:59b0296b2ef1763481e6e5123b0ca207", + url="https://github.com/mne-tools/mne-data/releases/download/visual_92_categories-2/MNE-visual_92_categories-data-part4.tar.gz", folder_name="MNE-visual_92_categories-data", config_key="MNE_DATASETS_VISUAL_92_CATEGORIES_PATH", ) @@ -259,14 +275,14 @@ MNE_DATASETS["mtrf"] = dict( archive_name="mTRF_1.5.zip", hash="md5:273a390ebbc48da2c3184b01a82e4636", - url="https://osf.io/download/h85s2?version=1", + url="https://github.com/mne-tools/mne-data/releases/download/mtrf-1/mTRF_1.5.zip", folder_name="mTRF_1.5", config_key="MNE_DATASETS_MTRF_PATH", ) MNE_DATASETS["refmeg_noise"] = dict( archive_name="sample_reference_MEG_noise-raw.zip", hash="md5:779fecd890d98b73a4832e717d7c7c45", - url="https://osf.io/download/drt6v?version=1", + url="https://github.com/mne-tools/mne-data/releases/download/refmeg_noise-1/sample_reference_MEG_noise-raw.zip", folder_name="MNE-refmeg-noise-data", config_key="MNE_DATASETS_REFMEG_NOISE_PATH", ) @@ -274,7 +290,7 @@ MNE_DATASETS["ssvep"] = dict( archive_name="ssvep_example_data.zip", hash="md5:af866bbc0f921114ac9d683494fe87d6", - url="https://osf.io/download/z8h6k?version=5", + url="https://github.com/mne-tools/mne-data/releases/download/ssvep-5/ssvep_example_data.zip", folder_name="ssvep-example-data", config_key="MNE_DATASETS_SSVEP_PATH", ) @@ -282,7 +298,7 @@ MNE_DATASETS["erp_core"] = dict( archive_name="MNE-ERP-CORE-data.tar.gz", hash="md5:5866c0d6213bd7ac97f254c776f6c4b1", - url="https://osf.io/download/rzgba?version=1", + url="https://github.com/mne-tools/mne-data/releases/download/erp_core-1/MNE-ERP-CORE-data.tar.gz", folder_name="MNE-ERP-CORE-data", config_key="MNE_DATASETS_ERP_CORE_PATH", ) @@ -290,7 +306,7 @@ MNE_DATASETS["epilepsy_ecog"] = dict( archive_name="MNE-epilepsy-ecog-data.tar.gz", hash="md5:ffb139174afa0f71ec98adbbb1729dea", - url="https://osf.io/download/z4epq?version=1", + url="https://github.com/mne-tools/mne-data/releases/download/epilepsy_ecog-1/MNE-epilepsy-ecog-data.tar.gz", folder_name="MNE-epilepsy-ecog-data", config_key="MNE_DATASETS_EPILEPSY_ECOG_PATH", ) @@ -299,7 +315,7 @@ MNE_DATASETS["fieldtrip_cmc"] = dict( archive_name="SubjectCMC.zip", hash="md5:6f9fd6520f9a66e20994423808d2528c", - url="https://osf.io/download/j9b6s?version=1", + url="https://github.com/mne-tools/mne-data/releases/download/fieldtrip_cmc-1/SubjectCMC.zip", folder_name="MNE-fieldtrip_cmc-data", config_key="MNE_DATASETS_FIELDTRIP_CMC_PATH", ) @@ -308,28 +324,28 @@ MNE_DATASETS["bst_auditory"] = dict( archive_name="bst_auditory.tar.gz", hash="md5:fa371a889a5688258896bfa29dd1700b", - url="https://osf.io/download/5t9n8?version=1", + url="https://github.com/mne-tools/mne-data/releases/download/bst_auditory-1/bst_auditory.tar.gz", folder_name="MNE-brainstorm-data", config_key="MNE_DATASETS_BRAINSTORM_PATH", ) MNE_DATASETS["bst_phantom_ctf"] = dict( archive_name="bst_phantom_ctf.tar.gz", hash="md5:80819cb7f5b92d1a5289db3fb6acb33c", - url="https://osf.io/download/sxr8y?version=1", + url="https://github.com/mne-tools/mne-data/releases/download/bst_phantom_ctf-1/bst_phantom_ctf.tar.gz", folder_name="MNE-brainstorm-data", config_key="MNE_DATASETS_BRAINSTORM_PATH", ) MNE_DATASETS["bst_phantom_elekta"] = dict( archive_name="bst_phantom_elekta.tar.gz", hash="md5:1badccbe17998d18cc373526e86a7aaf", - url="https://osf.io/download/dpcku?version=1", + url="https://github.com/mne-tools/mne-data/releases/download/bst_phantom_elekta-1/bst_phantom_elekta.tar.gz", folder_name="MNE-brainstorm-data", config_key="MNE_DATASETS_BRAINSTORM_PATH", ) MNE_DATASETS["bst_raw"] = dict( archive_name="bst_raw.tar.gz", hash="md5:fa2efaaec3f3d462b319bc24898f440c", - url="https://osf.io/download/9675n?version=2", + url="https://github.com/mne-tools/mne-data/releases/download/bst_raw-2/bst_raw.tar.gz", folder_name="MNE-brainstorm-data", config_key="MNE_DATASETS_BRAINSTORM_PATH", ) @@ -355,7 +371,7 @@ # Zenodo can be slow, so we use the OSF mirror # url=('https://zenodo.org/record/3523071/files/' # 'hf_sef_evoked.tar.gz'), - url="https://osf.io/download/25f8d?version=2", + url="https://github.com/mne-tools/mne-data/releases/download/hf_sef_evoked-2/hf_sef_evoked.tar.gz", folder_name="hf_sef", config_key="MNE_DATASETS_HF_SEF_PATH", ) @@ -373,7 +389,9 @@ MNE_DATASETS["eyelink"] = dict( archive_name="MNE-eyelink-data.zip", hash="md5:68a6323ef17d655f1a659c3290ee1c3f", - url=("https://osf.io/download/xsu4g?version=1"), + url=( + "https://github.com/mne-tools/mne-data/releases/download/eyelink-1/MNE-eyelink-data.zip" + ), folder_name="MNE-eyelink-data", config_key="MNE_DATASETS_EYELINK_PATH", ) diff --git a/mne/datasets/utils.py b/mne/datasets/utils.py index bef7102ecad..a0dc490ca4d 100644 --- a/mne/datasets/utils.py +++ b/mne/datasets/utils.py @@ -250,9 +250,9 @@ def _download_mne_dataset( # handle case of multiple sub-datasets with different urls if name == "visual_92_categories": dataset_params = [] - for name in ["visual_92_categories_1", "visual_92_categories_2"]: - this_dataset = MNE_DATASETS[name] - this_dataset["dataset_name"] = name + for part in sorted(key for key in MNE_DATASETS if key.startswith(f"{name}_")): + this_dataset = MNE_DATASETS[part] + this_dataset["dataset_name"] = part dataset_params.append(this_dataset) return cast( @@ -435,7 +435,10 @@ def fetch_aparc_sub_parcellation(subjects_dir=None, verbose=None): subjects_dir = get_subjects_dir(subjects_dir, raise_error=True) destination = subjects_dir / "fsaverage" / "label" - urls = dict(lh="https://osf.io/download/p92yb", rh="https://osf.io/download/4kxny") + urls = dict( + lh="https://github.com/mne-tools/mne-data/releases/download/hcp_aparc_sub-1/lh.aparc_sub.annot", + rh="https://github.com/mne-tools/mne-data/releases/download/hcp_aparc_sub-1/rh.aparc_sub.annot", + ) hashes = dict( lh="9e4d8d6b90242b7e4b0145353436ef77", rh="dd6464db8e7762d969fc1d8087cd211b" ) diff --git a/mne/viz/backends/tests/test_renderer.py b/mne/viz/backends/tests/test_renderer.py index 75af51ee5e0..09072986fb7 100644 --- a/mne/viz/backends/tests/test_renderer.py +++ b/mne/viz/backends/tests/test_renderer.py @@ -4,6 +4,7 @@ import os import platform +import subprocess import sys from contextlib import nullcontext @@ -15,7 +16,6 @@ import mne from mne.datasets import testing from mne.transforms import quat_to_rot, rot_to_quat -from mne.utils import run_subprocess from mne.viz import Figure3D, get_3d_backend, set_3d_backend from mne.viz.backends._utils import ALLOWED_QUIVER_MODES from mne.viz.backends.renderer import _get_renderer @@ -299,7 +299,8 @@ def test_renderer(renderer, monkeypatch): f"assert backend != 'jupyterlite_notebook' or 'vtk' not in sys.modules", ] monkeypatch.setenv("MNE_3D_BACKEND", backend) - run_subprocess(cmd) + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=60) + assert proc.returncode == 0, proc.stderr def test_set_3d_backend_bad(monkeypatch, tmp_path): @@ -622,4 +623,5 @@ def test_lite_setup_notebook(renderer_lite, tmp_path): ) ) args = [sys.executable, str(script), str(tmp_path / "mne_data"), str(_data_path)] - run_subprocess(args, env=env) + proc = subprocess.run(args, capture_output=True, text=True, env=env, timeout=60) + assert proc.returncode == 0, proc.stderr From 3b00c3d207f99275d901661844ac6f6cd326492b Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 18 Sep 2026 13:34:23 -0400 Subject: [PATCH 5/7] FIX: Better [circle full] --- Makefile | 2 +- azure-pipelines.yml | 83 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 3ce4eda956d..12f4f302a8b 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ test-no-testing-data: in $(PYTESTS) mne test-doc: sample_data testing_data - $(PYTESTS) --tb=short --cov=mne --cov-report=xml --cov-branch --doctest-modules --doctest-ignore-import-errors --doctest-glob='*.rst' ./doc/ --ignore=./doc/auto_examples --ignore=./doc/auto_tutorials --ignore=./doc/_build --ignore=./doc/conf.py --ignore=doc/sphinxext --fulltrace + $(PYTESTS) --tb=short --cov=mne --cov-report=xml --cov-branch --doctest-modules --doctest-ignore-import-errors --doctest-glob='*.rst' ./doc/ --ignore=./doc/auto_examples --ignore=./doc/auto_tutorials --ignore=./doc/_build --ignore=./doc/conf.py --ignore=./doc/jupyter_lite_config.py --ignore=doc/sphinxext --fulltrace pre-commit: @pre-commit run -a --show-diff-on-failure diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 320e73c9ddd..02df54ffd81 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -47,6 +47,12 @@ stages: PYTEST_XDIST_N: '2' # Microsoft-hosted agents have 2 cores MNE_TEST_ALLOW_SKIP: '^$' # nothing MNE_BROWSER_PRECOMPUTE: 'false' + # See tools/github_actions_env_vars.sh for why these are set (this job + # is on Python 3.14, so sys.monitoring can measure branches) + COVERAGE_CORE: 'sysmon' + UV_CACHE_DIR: '/home/vsts/.cache/uv' + UV_SYSTEM_PYTHON: 'true' # UsePythonVersion gives us a bare interpreter, not a venv + UV_NO_PROGRESS: 'true' steps: - bash: | set -xeo pipefail @@ -56,21 +62,27 @@ stages: displayName: Setup up Xvfb - task: Cache@2 inputs: - key: minimal-cmds-1 + key: '"minimal-cmds" | "$(Agent.OS)" | $(MNE_TOOLS_DIR)/tools/get_minimal_commands.sh' path: /home/vsts/minimal_cmds displayName: Cache minimal commands - - bash: ./tools/get_minimal_commands.sh + - bash: $(MNE_TOOLS_DIR)/tools/get_minimal_commands.sh displayName: Get minimal commands - task: UsePythonVersion@0 inputs: versionSpec: '3.14' architecture: 'x64' addToPath: true + - task: Cache@2 + inputs: + key: '"uv" | "$(Agent.OS)" | pyproject.toml' + restoreKeys: '"uv" | "$(Agent.OS)"' + path: $(UV_CACHE_DIR) + displayName: Cache uv - bash: | set -xeo pipefail - python -m pip install --progress-bar off --upgrade pip - python -m pip install --progress-bar off --upgrade --only-binary=":all:" -e . --group=test "mne-qt-browser @ https://github.com/mne-tools/mne-qt-browser/archive/refs/heads/main.zip" pyvista scikit-learn python-picard qtpy nibabel sphinx-gallery "PySide6!=6.8.0,!=6.8.0.1,!=6.8.1.1,!=6.9.1" pandas neo pymatreader antio defusedxml curryreader pymef openmeeg - displayName: Install dependencies with pip + python -m pip install --progress-bar off --upgrade "uv>=0.9" + uv pip install --upgrade --only-binary=":all:" --no-binary=mne --no-binary=mne-qt-browser -e . --group=test "mne-qt-browser @ https://github.com/mne-tools/mne-qt-browser/archive/refs/heads/main.zip" pyvista scikit-learn python-picard qtpy nibabel sphinx-gallery "PySide6!=6.8.0,!=6.8.0.1,!=6.8.1.1,!=6.9.1" pandas neo pymatreader antio defusedxml curryreader pymef openmeeg + displayName: Install dependencies with uv - bash: $(MNE_TOOLS_DIR)/tools/check_qt_import.sh PySide6 displayName: Check Qt import - bash: | @@ -82,7 +94,7 @@ stages: displayName: Get testing version - task: Cache@2 inputs: - key: $(testing_version) + key: mne_data-testing-$(testing_version)-misc-$(misc_version) path: /home/vsts/mne_data displayName: Cache testing data - bash: ./tools/github_actions_download.sh @@ -105,6 +117,8 @@ stages: PYTEST_XDIST_N: '2' # Microsoft-hosted agents have 2 cores TEST_OPTIONS: "--tb=short --cov=mne --cov-report=xml --cov-append -vv mne/gui mne/viz/_brain mne/viz/backends mne/viz/tests/test_evoked.py mne/report" MNE_TEST_ALLOW_SKIP: '^$' # nothing (can be overridden below) + # See tools/github_actions_env_vars.sh; this job is on Python 3.14 + COVERAGE_CORE: 'sysmon' steps: - bash: | set -xeo pipefail @@ -133,7 +147,7 @@ stages: displayName: Get testing version - task: Cache@2 inputs: - key: $(testing_version) + key: mne_data-testing-$(testing_version)-misc-$(misc_version) path: /home/vsts/mne_data displayName: Cache testing data - bash: ./tools/github_actions_download.sh @@ -166,11 +180,21 @@ stages: pool: vmImage: 'windows-latest' variables: + MNE_TOOLS_DIR: '$(Pipeline.Workspace)/mne-tools' MNE_LOGGING_LEVEL: 'warning' MNE_FORCE_SERIAL: 'true' PYTEST_XDIST_N: '2' # Microsoft-hosted agents have 2 cores OMP_DYNAMIC: 'false' PYTHONUNBUFFERED: 1 + # See tools/github_actions_env_vars.sh; both matrix legs are Python 3.14 + COVERAGE_CORE: 'sysmon' + # See tools/github_actions_env_vars.sh for why these three are set + NUMBA_CACHE_DIR: 'C:\Users\VssAdministrator\.cache\mne-numba' + NUMBA_CPU_NAME: 'generic' + NUMBA_CPU_FEATURES: '' + UV_CACHE_DIR: 'C:\Users\VssAdministrator\.cache\uv' + UV_SYSTEM_PYTHON: 'true' # UsePythonVersion gives us a bare interpreter, not a venv + UV_NO_PROGRESS: 'true' AZURE_CI_WINDOWS: 'true' # Mesa's llvmpipe rasterizer threads can deadlock at process exit on # Windows (pyvista/setup-headless-display-action#55); the action sets @@ -185,25 +209,47 @@ stages: 3.14 pip: TEST_MODE: 'pip' PYTHON_VERSION: '3.14' - MNE_TEST_ALLOW_SKIP: "^.*(Requires (spm|brainstorm) dataset|Requires MNE-C|Requires FreeSurfer|CUDA not|on Windows|MNE_FORCE_SERIAL|PySide6 causes segfaults|SCIPY_ARRAY_API).*$" + MNE_TEST_ALLOW_SKIP: "^.*(Requires (spm|brainstorm) dataset|Requires FreeSurfer|CUDA not|on Windows|MNE_FORCE_SERIAL|PySide6 causes segfaults|SCIPY_ARRAY_API).*$" 3.14 pip pre: TEST_MODE: 'pip-pre' PYTHON_VERSION: '3.14' - MNE_TEST_ALLOW_SKIP: "^.*(Requires (spm|brainstorm) dataset|Requires MNE-C|Requires FreeSurfer|CUDA not|Numba not|on Windows|MNE_FORCE_SERIAL|PySide6 causes segfaults|SCIPY_ARRAY_API).*$" + MNE_TEST_ALLOW_SKIP: "^.*(Requires (spm|brainstorm) dataset|Requires FreeSurfer|CUDA not|Numba not|on Windows|MNE_FORCE_SERIAL|PySide6 causes segfaults|SCIPY_ARRAY_API).*$" steps: - task: UsePythonVersion@0 inputs: versionSpec: $(PYTHON_VERSION) architecture: 'x64' addToPath: true + - bash: | + set -xeo pipefail + git clone --depth 1 https://github.com/mne-tools/mne-tools.git "$MNE_TOOLS_DIR" + displayName: Clone mne-tools + - task: Cache@2 + inputs: + key: '"minimal-cmds" | "$(Agent.OS)" | $(MNE_TOOLS_DIR)/tools/get_minimal_commands.sh' + path: C:\Users\VssAdministrator\minimal_cmds + displayName: Cache minimal commands + # MNE_TOOLS_DIR is a Windows path and these steps run under Git Bash, hence + # cygpath; backticks rather than $(...) so Azure does not read it as a macro + - bash: | + set -eo pipefail + MNE_TOOLS_UNIX=`cygpath -u "$MNE_TOOLS_DIR"` + bash "$MNE_TOOLS_UNIX/tools/get_minimal_commands.sh" + displayName: Get minimal commands - bash: | set -xeo pipefail git clone --depth 1 https://github.com/pyvista/setup-headless-display-action.git MESA3D_VERSION=24.3.0 bash setup-headless-display-action/windows/install_opengl.sh displayName: Install OpenGL + - task: Cache@2 + inputs: + key: '"uv" | "$(Agent.OS)" | "$(PYTHON_VERSION)" | "$(TEST_MODE)" | pyproject.toml' + restoreKeys: '"uv" | "$(Agent.OS)" | "$(PYTHON_VERSION)" | "$(TEST_MODE)"' + path: $(UV_CACHE_DIR) + displayName: Cache uv - bash: ./tools/azure_dependencies.sh displayName: Install dependencies - - script: pip install --only-binary=":all:" -e . + - script: uv pip install --only-binary=":all:" --no-binary=mne -e . displayName: Install mne - script: mne sys_info -pd displayName: Print config @@ -215,11 +261,26 @@ stages: displayName: Get testing version - task: Cache@2 inputs: - key: $(testing_version) + key: mne_data-testing-$(testing_version)-misc-$(misc_version) path: C:\Users\VssAdministrator\mne_data displayName: Cache testing data - bash: ./tools/github_actions_download.sh displayName: Download testing data + - bash: ./tools/get_numba_version.sh + displayName: Get numba version + - task: Cache@2 + inputs: + key: 'mne_numba | "$(Agent.OS)" | "$(PYTHON_VERSION)" | "$(numba_version)" | mne/**/*numba*.py' + path: C:\Users\VssAdministrator\.cache\mne-numba + displayName: Cache numba + # pip-pre has no numba when nightly wheels are missing; without it + # nothing creates the cache dir and the post-job save fails on tar + condition: ne(variables['numba_version'], 'none') + # Windows does not define HOME, and MNE-C dereferences it unconditionally: + # mne_forward_solution crashes with 0xC0000005 without it. Set after the + # steps above so that Git Bash keeps deriving its own MSYS-style HOME. + - script: 'echo ##vso[task.setvariable variable=HOME]%USERPROFILE%' + displayName: Set HOME for MNE-C - script: pytest -m "not (slowtest or pgtest)" -n $(PYTEST_XDIST_N) --dist loadscope --timeout=120 --timeout-method=thread -o faulthandler_timeout=110 --tb=short --cov=mne --cov-report=xml -vv mne displayName: Run tests - bash: bash <(curl -s https://codecov.io/bash) From 38b552f88fa8aa58317667c888bb82369ce403ca Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 18 Sep 2026 13:35:01 -0400 Subject: [PATCH 6/7] TST [circle full] From 66ceb17e2b8b46257f8751f77e80c85a0def1c69 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 18 Sep 2026 15:07:51 -0400 Subject: [PATCH 7/7] FIX: Dont double --- mne/viz/backends/_jupyterlite.py | 2 ++ mne/viz/backends/tests/test_renderer.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/mne/viz/backends/_jupyterlite.py b/mne/viz/backends/_jupyterlite.py index 74aba331ff7..39651556525 100644 --- a/mne/viz/backends/_jupyterlite.py +++ b/mne/viz/backends/_jupyterlite.py @@ -40,6 +40,8 @@ def setup_notebook(data_path="/tmp/mne_data"): Where fetched data files are written; ``MNE_DATA`` is set to it. """ global _base, _data_root + if _data_root is not None: # the cell ran again: the wrappers below would + return # otherwise wrap their own wrapped selves and recurse import js # only exists inside Pyodide # The docs serve the data next to the pages (/mne_data/, via diff --git a/mne/viz/backends/tests/test_renderer.py b/mne/viz/backends/tests/test_renderer.py index 09072986fb7..61c846e62f9 100644 --- a/mne/viz/backends/tests/test_renderer.py +++ b/mne/viz/backends/tests/test_renderer.py @@ -589,6 +589,10 @@ def send(self): from mne.viz.backends._jupyterlite import setup_notebook setup_notebook(str(root)) +setup_notebook(str(root)) # running the cell twice must not wrap the wrappers +from mne.viz.backends import _jupyterlite + +assert _jupyterlite._orig["read_raw_kit"] is not mne.io.read_raw_kit assert mne.viz.get_3d_backend() == "jupyterlite_notebook" assert mne.datasets.sample.data_path() == root / "MNE-sample-data" data = root / "MNE-testing-data"