diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index eff3010d3..bc65f074a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -59,7 +59,7 @@ jobs: PLATFORM: ${{ matrix.os }} DISPLAY: :42 run: | - uv run pytest --cov --color=yes --cov-report=xml -n auto --dist worksteal + uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal - name: Upload coverage to Codecov uses: codecov/codecov-action@v6 with: diff --git a/docs/api/datasets.md b/docs/api/datasets.md index 7bf6d5a61..d0c43b56c 100644 --- a/docs/api/datasets.md +++ b/docs/api/datasets.md @@ -7,5 +7,6 @@ Convenience small datasets .. autofunction:: blobs .. autofunction:: blobs_annotating_element +.. autofunction:: cells .. autofunction:: raccoon ``` diff --git a/pyproject.toml b/pyproject.toml index 76aded8a5..a8bce770a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ dependencies = [ "spatial_image>=1.2.3", "scikit-image", "scipy!=1.17.0", + "scverse-misc[datasets]>=0.1.0", "typing_extensions>=4.8.0", "universal_pathlib>=0.2.6", "xarray>=2024.10.0", @@ -108,6 +109,7 @@ addopts = [ # These are all markers coming from xarray, dask or anndata. Added here to silence warnings. markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "network: marks tests that require network access; skipped by default, run with '--run-network'", "gpu: run test on GPU using CuPY.", "array_api: used by anndata.tests.helpers, not us", "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on", diff --git a/src/spatialdata/datasets.py b/src/spatialdata/datasets.py index 37f529c72..51d360203 100644 --- a/src/spatialdata/datasets.py +++ b/src/spatialdata/datasets.py @@ -3,6 +3,7 @@ from __future__ import annotations import warnings +from pathlib import Path from typing import Any, Literal import dask.dataframe.core @@ -31,7 +32,7 @@ ) from spatialdata.transformations import Identity -__all__ = ["blobs", "raccoon"] +__all__ = ["blobs", "cells", "raccoon"] def blobs( @@ -79,6 +80,65 @@ def raccoon() -> SpatialData: return RaccoonDataset().raccoon() +def _shipped_registry() -> tuple[str | None, dict[str, Any]]: + """Parse the ``datasets.yaml`` registry shipped inside the ``spatialdata`` package.""" + import importlib.resources + + from scverse_misc.datasets import parse_registry + + registry = importlib.resources.files("spatialdata").joinpath("datasets.yaml") + with importlib.resources.as_file(registry) as registry_path: + base_url: str | None + datasets: dict[str, Any] + base_url, datasets = parse_registry(registry_path) + return base_url, datasets + + +def _cache_dir(path: str | None) -> Path: + """Resolve the cache directory, defaulting to the OS cache location for ``"spatialdata"``.""" + import pooch + + return Path(path) if path is not None else Path(pooch.os_cache("spatialdata")) + + +def cells(path: str | None = None) -> SpatialData: + """ + Cells dataset. + + Download the ``cells`` example dataset and load it as a :class:`~spatialdata.SpatialData` + object. The download is hash-verified and cached, so repeated calls reuse the local copy + instead of downloading again. + + The dataset is a small region of a Xenium Prime Cervical Cancer sample and contains three + multiscale images (``he_aligned``, ``he_image``, ``morphology_focus``), three multiscale + label layers (``cell_labels``, ``nucleus_labels``, ``tissue_labels``), the ``transcripts`` + points, the ``cell_boundaries`` and ``nucleus_boundaries`` shapes, and a cell-by-gene + ``table`` annotating the 94 cells. + + Notes + ----- + Derived from the 10x Genomics Xenium Prime Cervical Cancer FFPE dataset + (https://www.10xgenomics.com/datasets/xenium-prime-ffpe-human-cervical-cancer), subset to a + small tissue region. Licensed under `CC BY 4.0 `_; + see ``datasets.yaml`` for the attribution string shipped alongside the data. + + Parameters + ---------- + path + Directory in which to cache the downloaded data. If `None`, the default OS cache + location is used (:func:`pooch.os_cache` for ``"spatialdata"``). + + Returns + ------- + SpatialData object with the cells dataset. + """ + from scverse_misc.datasets import fetch + + base_url, datasets = _shipped_registry() + sdata: SpatialData = fetch(datasets["cells"], _cache_dir(path), base_url=base_url) + return sdata + + class RaccoonDataset: """Raccoon dataset.""" diff --git a/src/spatialdata/datasets.yaml b/src/spatialdata/datasets.yaml new file mode 100644 index 000000000..62e151d45 --- /dev/null +++ b/src/spatialdata/datasets.yaml @@ -0,0 +1,27 @@ +# Registry of downloadable example datasets for ``spatialdata.datasets``. +# +# Parsed by ``scverse_misc.datasets.parse_registry`` and fetched (downloaded, +# hash-verified, cached and loaded) via ``scverse_misc.datasets.fetch``. +# +# type: spatialdata -> a .zip that extracts to a single .zarr store +# +# Every dataset must list its ``license``; datasets under a license that requires +# attribution must also carry an ``attribution`` string crediting the original source, and +# datasets under a license that requires linking the license (e.g. CC BY 4.0) must also carry +# a ``license_url`` field. +base_url: https://exampledata.scverse.org/spatialdata/ +datasets: + cells: + type: spatialdata + doc_header: Cells dataset as a SpatialData object. + license: CC BY 4.0 + license_url: https://creativecommons.org/licenses/by/4.0/ + attribution: >- + Derived from the 10x Genomics Xenium Prime Cervical Cancer FFPE dataset + (https://www.10xgenomics.com/datasets/xenium-prime-ffpe-human-cervical-cancer), + subset to a small tissue region. Licensed under CC BY 4.0 + (https://creativecommons.org/licenses/by/4.0/). + files: + - name: cells.zip + s3_key: cells.zip + sha256: dc9613cb9e16fd2cd8d83f3a9586eeda4af5ba8ba366f1066efb51305820c5fb diff --git a/tests/conftest.py b/tests/conftest.py index 617acb90c..871b4280f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,6 +45,21 @@ ) +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--run-network", action="store_true", default=False, help="run tests marked 'network' (e.g. dataset downloads)" + ) + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + if config.getoption("--run-network"): + return + skip_network = pytest.mark.skip(reason="need --run-network option to run") + for item in items: + if "network" in item.keywords: + item.add_marker(skip_network) + + def _fast_deepcopy_sdata(sd: SpatialData) -> SpatialData: """ Fast deepcopy for SpatialData objects in tests. diff --git a/tests/datasets/test_datasets.py b/tests/datasets/test_datasets.py index 2237e253c..3fc66370c 100644 --- a/tests/datasets/test_datasets.py +++ b/tests/datasets/test_datasets.py @@ -1,6 +1,12 @@ from __future__ import annotations -from spatialdata.datasets import blobs, raccoon +from pathlib import Path + +import pooch +import pytest + +from spatialdata import SpatialData +from spatialdata.datasets import _cache_dir, _shipped_registry, blobs, cells, raccoon def test_datasets() -> None: @@ -26,3 +32,41 @@ def test_datasets() -> None: assert sdata_raccoon.images["raccoon"].shape == (3, 768, 1024) assert sdata_raccoon.labels["segmentation"].shape == (768, 1024) _ = str(sdata_raccoon) + + +def test_cells_registry() -> None: + # Network-free: the shipped registry parses and exposes the cells dataset. + base_url, datasets = _shipped_registry() + + assert base_url == "https://exampledata.scverse.org/spatialdata/" + entry = datasets["cells"] + assert entry.type == "spatialdata" + file = entry.file(name="cells.zip") + assert file.sha256 == "dc9613cb9e16fd2cd8d83f3a9586eeda4af5ba8ba366f1066efb51305820c5fb" + assert file.resolve_url(base_url) == "https://exampledata.scverse.org/spatialdata/cells.zip" + + +def test_cache_dir() -> None: + # Network-free: both branches of the cache-directory resolution. + assert _cache_dir("/tmp/example") == Path("/tmp/example") + assert _cache_dir(None) == Path(pooch.os_cache("spatialdata")) + + +@pytest.mark.network +def test_cells_download(tmp_path) -> None: + # Downloads ~3 MB from the scverse example data bucket; skipped by default, opt in with `--run-network`. + sdata = cells(path=str(tmp_path)) + assert isinstance(sdata, SpatialData) + + assert set(sdata.images) == {"he_aligned", "he_image", "morphology_focus"} + assert sdata.images["he_aligned"]["scale0"]["image"].shape == (3, 430, 540) + assert sdata.images["he_image"]["scale0"]["image"].shape == (3, 423, 339) + assert sdata.images["morphology_focus"]["scale0"]["image"].shape == (4, 430, 540) + + assert set(sdata.labels) == {"cell_labels", "nucleus_labels", "tissue_labels"} + assert sdata.labels["cell_labels"]["scale0"]["image"].shape == (430, 540) + + assert len(sdata.shapes["cell_boundaries"]) == 94 + assert len(sdata.shapes["nucleus_boundaries"]) == 94 + assert len(sdata.points["transcripts"].compute()) == 19479 + assert sdata.tables["table"].shape == (94, 5101)