Lock JupyterLite's browser packages at build time

Makefile

@@ -41,7 +41,7 @@ test-no-testing-data: in
 	$(PYTESTS) mne
 
 test-doc: sample_data testing_data
-	$(PYTESTS) --tb=short --cov=mne --cov-report=xml --cov-branch --doctest-modules --doctest-ignore-import-errors --doctest-glob='*.rst' ./doc/ --ignore=./doc/auto_examples --ignore=./doc/auto_tutorials --ignore=./doc/_build --ignore=./doc/conf.py --ignore=doc/sphinxext --fulltrace
+	$(PYTESTS) --tb=short --cov=mne --cov-report=xml --cov-branch --doctest-modules --doctest-ignore-import-errors --doctest-glob='*.rst' ./doc/ --ignore=./doc/auto_examples --ignore=./doc/auto_tutorials --ignore=./doc/_build --ignore=./doc/conf.py --ignore=./doc/jupyter_lite_config.py --ignore=doc/sphinxext --fulltrace
 
 pre-commit:
 	@pre-commit run -a --show-diff-on-failure

azure-pipelines.yml

@@ -196,6 +196,10 @@ stages:
           UV_SYSTEM_PYTHON: 'true'  # UsePythonVersion gives us a bare interpreter, not a venv
           UV_NO_PROGRESS: 'true'
           AZURE_CI_WINDOWS: 'true'
+          # Mesa's llvmpipe rasterizer threads can deadlock at process exit on
+          # Windows (pyvista/setup-headless-display-action#55); the action sets
+          # this itself on GHA, but here we only run its install script
+          LP_NUM_THREADS: '0'
           MNE_CI_KIND: $(TEST_MODE)
           MNE_TEST_ALLOW_SKIP: $(MNE_TEST_ALLOW_SKIP)
         timeoutInMinutes: 40

doc/changes/dev/14314.other.rst

@@ -0,0 +1 @@
+Lock the JupyterLite browser kernel's extra packages at build time instead of installing them from the first notebook cell, by `Teon Brooks`_.

doc/conf.py

@@ -505,6 +505,8 @@
     # sphinx-gallery writes link to auto_*/... at the root
     jupyterlite_contents = ["jupyterlite_contents/auto_*"]
     jupyterlite_bind_ipynb_suffix = False
+    # bakes the browser's extra packages into the Pyodide lock at build time
+    jupyterlite_config = "jupyter_lite_config.py"
     # served at the docs root (/mne_data/...) through html_extra_path below
     stage_lite_data(curpath / "lite_extra" / "mne_data")
 

doc/jupyter_lite_config.py

@@ -0,0 +1,30 @@
+"""Build-time config for the JupyterLite site's Pyodide lock.
+
+Adds MNE and the packages the notebooks import to the lock, so Pyodide loads
+them at kernel initialization. Packages Pyodide already curates keep their
+curated version.
+"""
+
+# Authors: The MNE-Python contributors.
+# License: BSD-3-Clause
+# Copyright the MNE-Python contributors.
+
+import sys
+from pathlib import Path
+
+sys.path.append(str(Path(__file__).parent / "sphinxext"))
+
+from jupyterlite_lock_specs import (  # noqa: E402
+    jupyterlite_package_names,
+    jupyterlite_specs_to_lock,
+)
+
+c = get_config()  # noqa: F821
+
+c.PyodideLockAddon.enabled = True
+# the MNE wheel build_lite_wheel.py builds from this checkout (a tagged
+# checkout gives the release): the solve takes it over PyPI, which would
+# otherwise supply the latest release through mne-connectivity's mne>=1.6
+c.PyodideLockAddon.wheels = ["pypi"]
+c.PyodideLockAddon.specs = ["mne", *jupyterlite_specs_to_lock()]
+c.PyodideLockAddon.prefetch_extra = ["mne", *jupyterlite_package_names()]

doc/sphinxext/jupyterlite_data.py

@@ -175,9 +175,7 @@ def stage_lite_data(dst_base):
     sphinx_logger.info(
         f"[JupyterLite] Served data: {n_copied} files copied, {n_missing} missing"
     )
-    # the development wheel, so the browser installs this MNE rather than the
-    # PyPI release: doc/sphinxext/build_lite_wheel.py puts it in doc/pypi, where
-    # the piplite addon indexes it; `make html` runs that first, so this is
-    # only a fallback for a bare sphinx-build
+    # the MNE wheel jupyter_lite_config.py locks for the browser: `make html`
+    # builds it first, so this is only a fallback for a bare sphinx-build
     wheels = find_wheels() or build_wheel()
     sphinx_logger.info(f"[JupyterLite] MNE wheel for the browser kernel: {wheels}")

doc/sphinxext/jupyterlite_lock_specs.py

@@ -0,0 +1,48 @@
+"""Specs to add to Pyodide's lock for JupyterLite, versus what it already curates.
+
+Naming a package in ``PyodideLockAddon.specs`` drops its existing pin from
+the lock (see ``pyodide_lock.uv_pip_compile.constraints_txt``) and resolves
+it fresh from PyPI instead, a smaller and differently versioned set than
+Pyodide's own curated wheels. :func:`jupyterlite_specs_to_lock` avoids that
+by only including specs Pyodide's own lock does not already satisfy.
+"""
+
+# Authors: The MNE-Python contributors.
+# License: BSD-3-Clause
+# Copyright the MNE-Python contributors.
+
+import json
+import tomllib
+import urllib.request
+from pathlib import Path
+
+from jupyterlite_pyodide_kernel.constants import PYODIDE_LOCK_DEFAULT_URL
+from packaging.requirements import Requirement
+from packaging.utils import canonicalize_name
+
+_PYPROJECT_TOML = Path(__file__).parents[2] / "pyproject.toml"
+
+
+def jupyterlite_specs():
+    """Retrieve the "jupyterlite" dependency group from pyproject.toml."""
+    with open(_PYPROJECT_TOML, "rb") as fid:
+        return tomllib.load(fid)["dependency-groups"]["jupyterlite"]
+
+
+def jupyterlite_package_names():
+    """Canonical package names in the "jupyterlite" dependency group."""
+    return [canonicalize_name(Requirement(spec).name) for spec in jupyterlite_specs()]
+
+
+def jupyterlite_specs_to_lock():
+    """Retrieve specs from "jupyterlite" group Pyodide does not already satisfy."""
+    with urllib.request.urlopen(PYODIDE_LOCK_DEFAULT_URL) as fid:
+        curated = json.load(fid)["packages"]
+
+    to_lock = []
+    for spec in jupyterlite_specs():
+        req = Requirement(spec)
+        pkg = curated.get(canonicalize_name(req.name))
+        if pkg is None or not req.specifier.contains(pkg["version"]):
+            to_lock.append(spec)
+    return to_lock

doc/sphinxext/jupyterlite_setup_cell.py

@@ -1,12 +1,11 @@
 """The setup cell prepended to every JupyterLite notebook.
 
-It installs MNE into the browser kernel and hands over to
-``mne.viz.backends._jupyterlite.setup_notebook``, which patches what Pyodide
-does not provide. piplite (not micropip) prefers the development MNE wheel
-bundled with the docs over PyPI, and ``keep_going`` reports a dependency with
-no wheel instead of aborting. ``sys.platform`` is ``"emscripten"`` only inside
-Pyodide, so the cell is a no-op in a local kernel and a notebook downloaded
-from inside JupyterLite runs unchanged there.
+It hands over to ``mne.viz.backends._jupyterlite.setup_notebook``, which
+patches what Pyodide does not provide; MNE itself and the packages the
+notebooks import are in the Pyodide lock (see ``jupyter_lite_config.py``).
+``sys.platform`` is ``"emscripten"`` only inside Pyodide, so the cell is a
+no-op in a local kernel and a notebook downloaded from inside JupyterLite runs
+unchanged there.
 
 The docs build prepends it only to the notebooks copied into the JupyterLite
 contents, not through ``first_notebook_cell``, which would also put it in the
@@ -17,24 +16,15 @@
 # License: BSD-3-Clause
 # Copyright the MNE-Python contributors.
 
-# ruff: noqa: E501  # the install list stays on one line so the cell reads short
-
-import ast
-
 LITE_SETUP_CELL = """\
-# 💡 Added by the docs build: installs MNE into the browser kernel and adapts
-# it to Pyodide. Does nothing outside JupyterLite.
+# 💡 Added by the docs build: adapts MNE to Pyodide. Does nothing outside
+# JupyterLite.
 import sys
 
 if sys.platform == "emscripten":
-    import piplite
-
-    await piplite.install(["mne", "scikit-learn", "joblib", "pandas", "seaborn", "mne-connectivity", "nibabel", "pyvista-js", "pyxdf", "mffpy", "python-picard"], keep_going=True)  # noqa: E501
     from mne.viz.backends._jupyterlite import setup_notebook
 
     setup_notebook()
 """
 # nothing else runs this before a reader does, so at least make sure it parses
-compile(
-    LITE_SETUP_CELL, "lite_setup_cell", "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
-)
+compile(LITE_SETUP_CELL, "lite_setup_cell", "exec")

mne/datasets/_fetch.py

@@ -126,7 +126,8 @@ def fetch_dataset(
         {'dataset_name': 'sample',
          'archive_name': 'MNE-sample-data-processed.tar.gz',
          'hash': 'md5:e8f30c4516abdc12a0c08e6bae57409c',
-         'url': 'https://osf.io/download/86qa2?version=6',
+         'url': 'https://github.com/mne-tools/mne-data/releases/download/'
+                'sample-6/MNE-sample-data-processed.tar.gz',
          'folder_name': 'MNE-sample-data',
          'config_key': 'MNE_DATASETS_SAMPLE_PATH'}
 

mne/datasets/_fsaverage/base.py

@@ -77,13 +77,13 @@ def fetch_fsaverage(subjects_dir=None, *, verbose=None):
     _manifest_check_download(
         manifest_path=FSAVERAGE_MANIFEST_PATH / "root.txt",
         destination=subjects_dir,
-        url="https://osf.io/download/3bxqt?version=2",
+        url="https://github.com/mne-tools/mne-data/releases/download/fsaverage-1.0/fsaverage-root.zip",
         hash_="5133fe92b7b8f03ae19219d5f46e4177",
     )
     _manifest_check_download(
         manifest_path=FSAVERAGE_MANIFEST_PATH / "bem.txt",
         destination=subjects_dir / "fsaverage",
-        url="https://osf.io/download/7ve8g?version=4",
+        url="https://github.com/mne-tools/mne-data/releases/download/fsaverage-1.0/fsaverage-bem.zip",
         hash_="b31509cdcf7908af6a83dc5ee8f49fb1",
     )
     return fs_dir

mne/datasets/_phantom/base.py

@@ -60,7 +60,7 @@ def fetch_phantom(kind="otaniemi", subjects_dir=None, *, verbose=None):
     """
     phantoms = dict(
         otaniemi=dict(
-            url="https://osf.io/download/j5czy?version=1",
+            url="https://github.com/mne-tools/mne-data/releases/download/phantom_otaniemi-1/phantom_otaniemi.zip",
             hash="42d17db5b1db3e30327ffb4cf2649de8",
         ),
     )