Add downloadable cells dataset via scverse-misc

.github/workflows/test.yaml

@@ -59,7 +59,7 @@ jobs:
                   PLATFORM: ${{ matrix.os }}
                   DISPLAY: :42
               run: |
-                  uv run pytest --cov --color=yes --cov-report=xml -n auto --dist worksteal
+                  uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal
             - name: Upload coverage to Codecov
               uses: codecov/codecov-action@v6
               with:

docs/api/datasets.md

@@ -7,5 +7,6 @@ Convenience small datasets
 
 .. autofunction:: blobs
 .. autofunction:: blobs_annotating_element
+.. autofunction:: cells
 .. autofunction:: raccoon
 ```

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
     "spatial_image>=1.2.3",
     "scikit-image",
     "scipy!=1.17.0",
+    "scverse-misc[datasets]>=0.1.0",
     "typing_extensions>=4.8.0",
     "universal_pathlib>=0.2.6",
     "xarray>=2024.10.0",
@@ -108,6 +109,7 @@ addopts = [
 # These are all markers coming from xarray, dask or anndata. Added here to silence warnings.
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "network: marks tests that require network access; skipped by default, run with '--run-network'",
     "gpu: run test on GPU using CuPY.",
     "array_api: used by anndata.tests.helpers, not us",
     "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on",

src/spatialdata/datasets.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import warnings
+from pathlib import Path
 from typing import Any, Literal
 
 import dask.dataframe.core
@@ -31,7 +32,7 @@
 )
 from spatialdata.transformations import Identity
 
-__all__ = ["blobs", "raccoon"]
+__all__ = ["blobs", "cells", "raccoon"]
 
 
 def blobs(
@@ -79,6 +80,65 @@ def raccoon() -> SpatialData:
     return RaccoonDataset().raccoon()
 
 
+def _shipped_registry() -> tuple[str | None, dict[str, Any]]:
+    """Parse the ``datasets.yaml`` registry shipped inside the ``spatialdata`` package."""
+    import importlib.resources
+
+    from scverse_misc.datasets import parse_registry
+
+    registry = importlib.resources.files("spatialdata").joinpath("datasets.yaml")
+    with importlib.resources.as_file(registry) as registry_path:
+        base_url: str | None
+        datasets: dict[str, Any]
+        base_url, datasets = parse_registry(registry_path)
+    return base_url, datasets
+
+
+def _cache_dir(path: str | None) -> Path:
+    """Resolve the cache directory, defaulting to the OS cache location for ``"spatialdata"``."""
+    import pooch
+
+    return Path(path) if path is not None else Path(pooch.os_cache("spatialdata"))
+
+
+def cells(path: str | None = None) -> SpatialData:
+    """
+    Cells dataset.
+
+    Download the ``cells`` example dataset and load it as a :class:`~spatialdata.SpatialData`
+    object. The download is hash-verified and cached, so repeated calls reuse the local copy
+    instead of downloading again.
+
+    The dataset is a small region of a Xenium Prime Cervical Cancer sample and contains three
+    multiscale images (``he_aligned``, ``he_image``, ``morphology_focus``), three multiscale
+    label layers (``cell_labels``, ``nucleus_labels``, ``tissue_labels``), the ``transcripts``
+    points, the ``cell_boundaries`` and ``nucleus_boundaries`` shapes, and a cell-by-gene
+    ``table`` annotating the 94 cells.
+
+    Notes
+    -----
+    Derived from the 10x Genomics Xenium Prime Cervical Cancer FFPE dataset
+    (https://www.10xgenomics.com/datasets/xenium-prime-ffpe-human-cervical-cancer), subset to a
+    small tissue region. Licensed under `CC BY 4.0 <https://creativecommons.org/licenses/by/4.0/>`_;
+    see ``datasets.yaml`` for the attribution string shipped alongside the data.
+
+    Parameters
+    ----------
+    path
+        Directory in which to cache the downloaded data. If `None`, the default OS cache
+        location is used (:func:`pooch.os_cache` for ``"spatialdata"``).
+
+    Returns
+    -------
+    SpatialData object with the cells dataset.
+    """
+    from scverse_misc.datasets import fetch
+
+    base_url, datasets = _shipped_registry()
+    sdata: SpatialData = fetch(datasets["cells"], _cache_dir(path), base_url=base_url)
+    return sdata
+
+
 class RaccoonDataset:
     """Raccoon dataset."""
 

src/spatialdata/datasets.yaml

@@ -0,0 +1,27 @@
+# Registry of downloadable example datasets for ``spatialdata.datasets``.
+#
+# Parsed by ``scverse_misc.datasets.parse_registry`` and fetched (downloaded,
+# hash-verified, cached and loaded) via ``scverse_misc.datasets.fetch``.
+#
+#   type: spatialdata  ->  a .zip that extracts to a single .zarr store
+#
+# Every dataset must list its ``license``; datasets under a license that requires
+# attribution must also carry an ``attribution`` string crediting the original source, and
+# datasets under a license that requires linking the license (e.g. CC BY 4.0) must also carry
+# a ``license_url`` field.
+base_url: https://exampledata.scverse.org/spatialdata/
+datasets:
+    cells:
+        type: spatialdata
+        doc_header: Cells dataset as a SpatialData object.
+        license: CC BY 4.0
+        license_url: https://creativecommons.org/licenses/by/4.0/
+        attribution: >-
+            Derived from the 10x Genomics Xenium Prime Cervical Cancer FFPE dataset
+            (https://www.10xgenomics.com/datasets/xenium-prime-ffpe-human-cervical-cancer),
+            subset to a small tissue region. Licensed under CC BY 4.0
+            (https://creativecommons.org/licenses/by/4.0/).
+        files:
+            - name: cells.zip
+              s3_key: cells.zip
+              sha256: dc9613cb9e16fd2cd8d83f3a9586eeda4af5ba8ba366f1066efb51305820c5fb

tests/conftest.py

@@ -45,6 +45,21 @@
 )
 
 
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addoption(
+        "--run-network", action="store_true", default=False, help="run tests marked 'network' (e.g. dataset downloads)"
+    )
+
+
+def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
+    if config.getoption("--run-network"):
+        return
+    skip_network = pytest.mark.skip(reason="need --run-network option to run")
+    for item in items:
+        if "network" in item.keywords:
+            item.add_marker(skip_network)
+
+
 def _fast_deepcopy_sdata(sd: SpatialData) -> SpatialData:
     """
     Fast deepcopy for SpatialData objects in tests.

tests/datasets/test_datasets.py

@@ -1,6 +1,12 @@
 from __future__ import annotations
 
-from spatialdata.datasets import blobs, raccoon
+from pathlib import Path
+
+import pooch
+import pytest
+
+from spatialdata import SpatialData
+from spatialdata.datasets import _cache_dir, _shipped_registry, blobs, cells, raccoon
 
 
 def test_datasets() -> None:
@@ -26,3 +32,41 @@ def test_datasets() -> None:
     assert sdata_raccoon.images["raccoon"].shape == (3, 768, 1024)
     assert sdata_raccoon.labels["segmentation"].shape == (768, 1024)
     _ = str(sdata_raccoon)
+
+
+def test_cells_registry() -> None:
+    # Network-free: the shipped registry parses and exposes the cells dataset.
+    base_url, datasets = _shipped_registry()
+
+    assert base_url == "https://exampledata.scverse.org/spatialdata/"
+    entry = datasets["cells"]
+    assert entry.type == "spatialdata"
+    file = entry.file(name="cells.zip")
+    assert file.sha256 == "dc9613cb9e16fd2cd8d83f3a9586eeda4af5ba8ba366f1066efb51305820c5fb"
+    assert file.resolve_url(base_url) == "https://exampledata.scverse.org/spatialdata/cells.zip"
+
+
+def test_cache_dir() -> None:
+    # Network-free: both branches of the cache-directory resolution.
+    assert _cache_dir("/tmp/example") == Path("/tmp/example")
+    assert _cache_dir(None) == Path(pooch.os_cache("spatialdata"))
+
+
+@pytest.mark.network
+def test_cells_download(tmp_path) -> None:
+    # Downloads ~3 MB from the scverse example data bucket; skipped by default, opt in with `--run-network`.
+    sdata = cells(path=str(tmp_path))
+    assert isinstance(sdata, SpatialData)
+
+    assert set(sdata.images) == {"he_aligned", "he_image", "morphology_focus"}
+    assert sdata.images["he_aligned"]["scale0"]["image"].shape == (3, 430, 540)
+    assert sdata.images["he_image"]["scale0"]["image"].shape == (3, 423, 339)
+    assert sdata.images["morphology_focus"]["scale0"]["image"].shape == (4, 430, 540)
+
+    assert set(sdata.labels) == {"cell_labels", "nucleus_labels", "tissue_labels"}
+    assert sdata.labels["cell_labels"]["scale0"]["image"].shape == (430, 540)
+
+    assert len(sdata.shapes["cell_boundaries"]) == 94
+    assert len(sdata.shapes["nucleus_boundaries"]) == 94
+    assert len(sdata.points["transcripts"].compute()) == 19479
+    assert sdata.tables["table"].shape == (94, 5101)