diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index eff3010d3..bc65f074a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -59,7 +59,7 @@ jobs: PLATFORM: ${{ matrix.os }} DISPLAY: :42 run: | - uv run pytest --cov --color=yes --cov-report=xml -n auto --dist worksteal + uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal - name: Upload coverage to Codecov uses: codecov/codecov-action@v6 with: diff --git a/.gitmodules b/.gitmodules index cf0594cfa..57216b9bd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "docs/tutorials/notebooks"] path = docs/tutorials/notebooks - url = https://github.com/scverse/spatialdata-notebooks + url = https://github.com/scverse/spatialdata-tutorials diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 846497306..feb3aaa2d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,18 +9,18 @@ ci: skip: [] repos: - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.4 + rev: v3.9.6 hooks: - id: prettier exclude: ^.github/workflows/test.yaml - repo: https://github.com/pre-commit/mirrors-mypy - rev: v2.1.0 + rev: v2.3.1 hooks: - id: mypy additional_dependencies: [numpy, types-requests] exclude: tests/|docs/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.20 + rev: v0.16.3 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/docs/api/datasets.md b/docs/api/datasets.md index 7bf6d5a61..d0c43b56c 100644 --- a/docs/api/datasets.md +++ b/docs/api/datasets.md @@ -7,5 +7,6 @@ Convenience small datasets .. autofunction:: blobs .. autofunction:: blobs_annotating_element +.. autofunction:: cells .. autofunction:: raccoon ``` diff --git a/docs/contributing.md b/docs/contributing.md index 4faba0274..6026d4e1f 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -13,8 +13,7 @@ In addition to the packages needed to _use_ this package, you need additional py the documentation_. It's easy to install them using `pip`: ```bash -cd spatialdata-io -pip install -e ".[dev,test,doc]" +pip install -e . --group dev --group test --group docs ``` ## Code-style @@ -51,7 +50,7 @@ and [prettier][prettier-editors]. ## Writing tests ```{note} -Remember to first install the package with `pip install '-e[dev,test]'` +Remember to first install the package with `pip install -e . --group dev --group test` ``` This package uses [pytest][] for automated testing. Please [write tests][scanpy-test-docs] for every function added to the package. diff --git a/docs/index.md b/docs/index.md index 4b21c6ff7..9ad5ca49d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,8 +12,8 @@ This page provides documentation on how to install, use, and extend the core `sp - `spatialdata-io`: load data from common spatial omics technologies into `spatialdata` ([repository][spatialdata-io-repo], [documentation][spatialdata-io-docs]). - `spatialdata-plot`: Static plotting library for `spatialdata` ([repository][spatialdata-plot-repo], [documentation][spatialdata-plot-docs]). -- `napari-spatialdata-repo`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]). -- `SpatialData` (R): R implementation of the SpatialData framework ([repository][spatialdata-r-repo]). +- `napari-spatialdata`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]). +- `spatialdataR`: R implementation of the SpatialData framework ([repository][spatialdata-r-repo]). - `SpatialData.js`: JavaScript/TypeScript implementation of the SpatialData framework ([repository][spatialdata-js-repo]). Please see our publication {cite}`marconatoSpatialDataOpenUniversal2024` for citation and to learn more. @@ -110,6 +110,7 @@ tutorials/notebooks/notebooks.md tutorials/notebooks/datasets/README.md glossary.md design_doc.md +interoperability.md contributing.md changelog.md references.md @@ -123,5 +124,5 @@ references.md [napari-spatialdata-docs]: https://spatialdata.scverse.org/projects/napari/en/stable/notebooks/spatialdata.html [spatialdata-io-docs]: https://spatialdata.scverse.org/projects/io/en/stable/ [spatialdata-plot-docs]: https://spatialdata.scverse.org/projects/plot/en/stable/api.html -[spatialdata-r-repo]: https://github.com/HelenaLC/SpatialData +[spatialdata-r-repo]: https://github.com/HelenaLC/spatialdataR [spatialdata-js-repo]: https://github.com/Taylor-CCB-Group/SpatialData.js diff --git a/docs/interoperability.md b/docs/interoperability.md new file mode 100644 index 000000000..331450808 --- /dev/null +++ b/docs/interoperability.md @@ -0,0 +1,16 @@ +# Interoperability + +The on-disk representation of SpatialData can be read from other languages. Here we list interfaces for working with SpatialData from your language of choice: + +## R + +- [spatialdataR](https://helenalc.github.io/spatialdataR/) provides an R implementation of the `SpatialData` object, with out-of-memory images and labels, `duckdb`-backed points and shapes, and tables represented as `SingleCellExperiment` objects. + +## JavaScript and TypeScript + +- [SpatialData.js](https://github.com/Taylor-CCB-Group/SpatialData.js) provides a TypeScript and JavaScript library for interfacing with SpatialData stores. +- [Vitessce](https://vitessce.io/docs/data-file-types/#spatialdatazarr) reads `spatialdata.zarr` stores directly and uses them for interactive visualization. + +## File format + +The SpatialData on-disk format builds on [OME-NGFF](https://ngff.openmicroscopy.org/latest/). See the [design document](design_doc.md) for details of the current on-disk layout. diff --git a/pyproject.toml b/pyproject.toml index 03181eadb..a8bce770a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ dependencies = [ "spatial_image>=1.2.3", "scikit-image", "scipy!=1.17.0", + "scverse-misc[datasets]>=0.1.0", "typing_extensions>=4.8.0", "universal_pathlib>=0.2.6", "xarray>=2024.10.0", @@ -108,6 +109,7 @@ addopts = [ # These are all markers coming from xarray, dask or anndata. Added here to silence warnings. markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "network: marks tests that require network access; skipped by default, run with '--run-network'", "gpu: run test on GPU using CuPY.", "array_api: used by anndata.tests.helpers, not us", "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on", @@ -244,3 +246,38 @@ memray-flame = "memray flamegraph --temporal" [tool.pixi.environments] profiling = { features = ["profiling"], solve-group = "default" } + +[tool.hatch.envs.test] +dependency-groups = ["test"] + +[tool.hatch.envs.test-anndata-pandas] +template = "test" +extra-dependencies = ["zarr>=3"] +scripts.test = ["pip list|grep anndata && pip list|grep pandas && pytest {args}"] +scripts.test-readwrite = ["pip list|grep anndata && pip list|grep pandas && pytest tests/io/test_readwrite.py"] +scripts.test-all = ["pip list|grep anndata && pip list|grep pandas && pytest ."] + +[[tool.hatch.envs.test-anndata-pandas.matrix]] +anndata-pandas = [ + "0.13-2", + "0.13-3", + "0.12-2" + # no "0.12-3": support for pandas>=3 is available only in anndata>=0.13 +] + +[tool.hatch.envs.test-anndata-pandas.overrides] +matrix.anndata-pandas.extra-dependencies = [ + # every option where the if-condition is True gets included + + # anndata 0.13 + {value="anndata~=0.13", if = ["0.13-2", "0.13-3"]}, + + # anndata 0.12 + {value="anndata>=0.12,<0.13", if = ["0.12-2"]}, + + # pandas 2 + {value="pandas>=2.3,<3", if = ["0.13-2", "0.12-2"]}, + + # pandas 3 + {value="pandas~=3.0", if = ["0.13-3"]}, +] diff --git a/src/spatialdata/_core/spatialdata.py b/src/spatialdata/_core/spatialdata.py index fb55ab086..6ee2296c8 100644 --- a/src/spatialdata/_core/spatialdata.py +++ b/src/spatialdata/_core/spatialdata.py @@ -107,7 +107,7 @@ class SpatialData: - the table are stored as :class:`anndata.AnnData` objects, with the spatial coordinates stored in the obsm slot. - The table can annotate regions (shapesor labels) and can be used to store additional information. + The table can annotate regions (shapes or labels) and can be used to store additional information. Points are not regions but 0-dimensional locations. They can't be annotated by a table, but they can store annotation directly. """ @@ -1114,6 +1114,7 @@ def write( sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None, shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None, raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None, + convert_table_strings_to_categoricals: bool = False, ) -> None: """ Write the `SpatialData` object to a Zarr store. @@ -1166,6 +1167,9 @@ def write( compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically ordered for more efficient compression. + convert_table_strings_to_categoricals + If True, convert string columns of all tables to categoricals before writing. + Note that this will have a side effect of modifying string columns into categoricals in place. """ from spatialdata._io._utils import _resolve_zarr_store, _validate_compressor_args from spatialdata._io.format import _parse_formats @@ -1194,6 +1198,7 @@ def write( parsed_formats=parsed, shapes_geometry_encoding=shapes_geometry_encoding, raster_compressor=raster_compressor, + convert_table_strings_to_categoricals=convert_table_strings_to_categoricals, ) if self.path != file_path and update_sdata_path: @@ -1212,6 +1217,7 @@ def _write_element( parsed_formats: dict[str, SpatialDataFormatType] | None = None, shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None, raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None, + convert_table_strings_to_categoricals: bool = False, ) -> None: from spatialdata._io.io_zarr import _get_groups_for_element @@ -1279,6 +1285,7 @@ def _write_element( group=element_type_group, name=element_name, element_format=parsed_formats["tables"], + convert_strings_to_categoricals=convert_table_strings_to_categoricals, ) else: raise ValueError(f"Unknown element type: {element_type}") @@ -1290,6 +1297,7 @@ def write_element( sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None, shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None, raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None, + convert_table_strings_to_categoricals: bool = False, ) -> None: """ Write a single element, or a list of elements, to the Zarr store used for backing. @@ -1308,11 +1316,14 @@ def write_element( shapes_geometry_encoding Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet` for details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`. - raster_compressor + raster_compressor A lenght-1 dictionary with as key the type of compression to use for images and labels and as value the compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically ordered for more efficient compression. + convert_table_strings_to_categoricals + If True, and if element to be written is a table, convert string columns to categoricals before writing. + Note that this will have a side effect of modifying string columns into categoricals in place. Notes ----- @@ -1332,6 +1343,7 @@ def write_element( sdata_formats=sdata_formats, shapes_geometry_encoding=shapes_geometry_encoding, raster_compressor=raster_compressor, + convert_table_strings_to_categoricals=convert_table_strings_to_categoricals, ) return @@ -1368,6 +1380,7 @@ def write_element( parsed_formats=parsed_formats, shapes_geometry_encoding=shapes_geometry_encoding, raster_compressor=raster_compressor, + convert_table_strings_to_categoricals=convert_table_strings_to_categoricals, ) # After every write, metadata should be consolidated, otherwise this can lead to IO problems like when deleting. if self.has_consolidated_metadata(): diff --git a/src/spatialdata/_core/validation.py b/src/spatialdata/_core/validation.py index ad1e7a91e..b6c2eebfe 100644 --- a/src/spatialdata/_core/validation.py +++ b/src/spatialdata/_core/validation.py @@ -388,7 +388,7 @@ def __exit__( # Exceptions were collected that we want to raise as a combined validation error. if self._collector.errors: raise ValidationError( - title=self._message + "\nTo fix, run `spatialdata.utils.sanitize_table(adata)`.", + title=self._message + "\nTo fix, run `spatialdata.sanitize_table(adata)`.", errors=self._collector.errors, ) return True diff --git a/src/spatialdata/_io/exceptions.py b/src/spatialdata/_io/exceptions.py new file mode 100644 index 000000000..66f5802b7 --- /dev/null +++ b/src/spatialdata/_io/exceptions.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from ome_zarr.format import Format + + +class FormatVersionUnknownError(ValueError): + """Exception raised when an unknown element format is encountered.""" + + def __init__(self, element_type: str, version_encountered: Format): + self.element_type = element_type + self.version_encountered = version_encountered + self.message = ( + f"Encountered unknown element format version " + f"`{self.version_encountered}` for element of type `{self.element_type}`" + ) + super().__init__(self.message) + + +class WritingToZarrV2DeprecationWarning(DeprecationWarning): + """Warning raised when writing to zarr v2 format.""" + + message = ( + "Writing to zarr v2 format is currently deprecated in spatialdata " + "and will be removed in a future version. " + "Please consider writing to zarr v3." + ) diff --git a/src/spatialdata/_io/io_points.py b/src/spatialdata/_io/io_points.py index 03ef33389..bb203cad2 100644 --- a/src/spatialdata/_io/io_points.py +++ b/src/spatialdata/_io/io_points.py @@ -1,5 +1,6 @@ from __future__ import annotations +import warnings from pathlib import Path import zarr @@ -12,6 +13,7 @@ _write_metadata, overwrite_coordinate_transformations_non_raster, ) +from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning from spatialdata._io.format import CurrentPointsFormat, PointsFormats, _parse_version from spatialdata.models import get_axes_names from spatialdata.transformations._utils import ( @@ -65,6 +67,10 @@ def write_points( element_format The format of the points element used to store it. """ + if element_format.zarr_format == 2: + warnings.warn( + message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2 + ) axes = get_axes_names(points) transformations = _get_transformations(points) assert transformations is not None # mypy: validate_element() in _write_element guarantees this diff --git a/src/spatialdata/_io/io_raster.py b/src/spatialdata/_io/io_raster.py index 276f016bd..b9a2964f0 100644 --- a/src/spatialdata/_io/io_raster.py +++ b/src/spatialdata/_io/io_raster.py @@ -1,5 +1,6 @@ from __future__ import annotations +import warnings from collections.abc import Sequence from pathlib import Path from typing import Any, Literal, TypeGuard, cast @@ -23,6 +24,7 @@ overwrite_channel_names, overwrite_coordinate_transformations_raster, ) +from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning from spatialdata._io.format import ( CurrentRasterFormat, RasterFormatType, @@ -581,6 +583,11 @@ def write_image( raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None, **metadata: str | JSONDict | list[JSONDict], ) -> None: + if element_format.zarr_format == 2: + warnings.warn( + message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2 + ) + _write_raster( raster_type="image", raster_data=image, @@ -603,6 +610,11 @@ def write_labels( raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None, **metadata: JSONDict, ) -> None: + if element_format.zarr_format == 2: + warnings.warn( + message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2 + ) + _write_raster( raster_type="labels", raster_data=labels, diff --git a/src/spatialdata/_io/io_shapes.py b/src/spatialdata/_io/io_shapes.py index 3b6e18e39..f8528868d 100644 --- a/src/spatialdata/_io/io_shapes.py +++ b/src/spatialdata/_io/io_shapes.py @@ -1,5 +1,6 @@ from __future__ import annotations +import warnings from pathlib import Path from typing import Any, Literal @@ -15,6 +16,7 @@ _write_metadata, overwrite_coordinate_transformations_non_raster, ) +from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning from spatialdata._io.format import ( CurrentShapesFormat, ShapesFormats, @@ -93,6 +95,11 @@ def write_shapes( Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet` for details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`. """ + if element_format.zarr_format == 2: + warnings.warn( + message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2 + ) + from spatialdata.config import settings if geometry_encoding is None: diff --git a/src/spatialdata/_io/io_table.py b/src/spatialdata/_io/io_table.py index 3eb4b0927..931572389 100644 --- a/src/spatialdata/_io/io_table.py +++ b/src/spatialdata/_io/io_table.py @@ -1,5 +1,7 @@ from __future__ import annotations +import warnings +from importlib.metadata import version from pathlib import Path import numpy as np @@ -8,7 +10,10 @@ from anndata import read_zarr as read_anndata_zarr from anndata._io.specs import write_elem as write_adata from ome_zarr.format import Format +from packaging.version import Version +from spatialdata._io._utils import _resolve_zarr_store +from spatialdata._io.exceptions import FormatVersionUnknownError, WritingToZarrV2DeprecationWarning from spatialdata._io.format import ( CurrentTablesFormat, TablesFormats, @@ -55,17 +60,67 @@ def write_table( name: str, group_type: str = "ngff:regions_table", element_format: Format = CurrentTablesFormat(), + convert_strings_to_categoricals: bool = False, ) -> None: + """ + Write a table to a Zarr store. + + Parameters + ---------- + table + The table to write. + group + The table will be written into a subgroup of this group + name + The name of the subgroup of `group` to which table is to be written. + group_type + The type of the group. + element_format + The format to use for writing the table. + convert_strings_to_categoricals + If True, convert string columns to categoricals before writing. + Note that this will have a side effect of modifying dtypes of the input table in place. + """ + if element_format.zarr_format == 2: + warnings.warn( + message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2 + ) + if TableModel.ATTRS_KEY in table.uns: region, region_key, instance_key = get_table_keys(table) TableModel.validate(table) else: region, region_key, instance_key = (None, None, None) - write_adata(group, name, table) - tables_group = group[name] - tables_group.attrs["spatialdata-encoding-type"] = group_type - tables_group.attrs["region"] = region - tables_group.attrs["region_key"] = region_key - tables_group.attrs["instance_key"] = instance_key - tables_group.attrs["version"] = element_format.spatialdata_format_version + # Ensure the table group exists + table_group = group.require_group(name=name) + + if element_format not in TablesFormats.values(): + raise FormatVersionUnknownError(element_type="table", version_encountered=element_format) + + if element_format.zarr_format == 3 and Version(version("anndata")) >= Version("0.13"): + # `write_zarr` in anndata v0.13 and above can only write to zarr v3 + # solution of passing resolved store directly roughly based on: + # https://github.com/scverse/anndata/issues/1548#issuecomment-2199801855 + + # resolve the store from the group + resolved_store = _resolve_zarr_store(table_group) + + # Write the table to the path of the table group + table.write_zarr( + store=resolved_store, + consolidate_metadata=False, + convert_strings_to_categoricals=convert_strings_to_categoricals, + ) + + else: + if convert_strings_to_categoricals: + table.strings_to_categoricals() + + write_adata(group, name, table) + + table_group.attrs["spatialdata-encoding-type"] = group_type + table_group.attrs["region"] = region + table_group.attrs["region_key"] = region_key + table_group.attrs["instance_key"] = instance_key + table_group.attrs["version"] = element_format.spatialdata_format_version diff --git a/src/spatialdata/_io/io_zarr.py b/src/spatialdata/_io/io_zarr.py index 4c410fab0..9324f8b7f 100644 --- a/src/spatialdata/_io/io_zarr.py +++ b/src/spatialdata/_io/io_zarr.py @@ -5,7 +5,7 @@ from collections.abc import Callable from json import JSONDecodeError from pathlib import Path -from typing import Any, Literal, cast +from typing import Any, Literal import zarr.storage from anndata import AnnData @@ -71,7 +71,7 @@ def _read_zarr_group_spatialdata_element( reader_format = get_raster_format_for_read(elem_group, sdata_version) element = read_func( elem_group_path, - cast(Literal["image", "labels"], element_type), + element_type, reader_format, ) elif element_type in ["shapes", "points", "tables"]: @@ -136,7 +136,7 @@ def read_zarr( Path, URL, or zarr.Group to the zarr store (on-disk or remote). selection - List of elements to read from the zarr store (images, labels, points, shapes, table). If None, all elements are + List of elements to read from the zarr store (images, labels, points, shapes, tables). If None, all elements are read. on_bad_files diff --git a/src/spatialdata/datasets.py b/src/spatialdata/datasets.py index 37f529c72..51d360203 100644 --- a/src/spatialdata/datasets.py +++ b/src/spatialdata/datasets.py @@ -3,6 +3,7 @@ from __future__ import annotations import warnings +from pathlib import Path from typing import Any, Literal import dask.dataframe.core @@ -31,7 +32,7 @@ ) from spatialdata.transformations import Identity -__all__ = ["blobs", "raccoon"] +__all__ = ["blobs", "cells", "raccoon"] def blobs( @@ -79,6 +80,65 @@ def raccoon() -> SpatialData: return RaccoonDataset().raccoon() +def _shipped_registry() -> tuple[str | None, dict[str, Any]]: + """Parse the ``datasets.yaml`` registry shipped inside the ``spatialdata`` package.""" + import importlib.resources + + from scverse_misc.datasets import parse_registry + + registry = importlib.resources.files("spatialdata").joinpath("datasets.yaml") + with importlib.resources.as_file(registry) as registry_path: + base_url: str | None + datasets: dict[str, Any] + base_url, datasets = parse_registry(registry_path) + return base_url, datasets + + +def _cache_dir(path: str | None) -> Path: + """Resolve the cache directory, defaulting to the OS cache location for ``"spatialdata"``.""" + import pooch + + return Path(path) if path is not None else Path(pooch.os_cache("spatialdata")) + + +def cells(path: str | None = None) -> SpatialData: + """ + Cells dataset. + + Download the ``cells`` example dataset and load it as a :class:`~spatialdata.SpatialData` + object. The download is hash-verified and cached, so repeated calls reuse the local copy + instead of downloading again. + + The dataset is a small region of a Xenium Prime Cervical Cancer sample and contains three + multiscale images (``he_aligned``, ``he_image``, ``morphology_focus``), three multiscale + label layers (``cell_labels``, ``nucleus_labels``, ``tissue_labels``), the ``transcripts`` + points, the ``cell_boundaries`` and ``nucleus_boundaries`` shapes, and a cell-by-gene + ``table`` annotating the 94 cells. + + Notes + ----- + Derived from the 10x Genomics Xenium Prime Cervical Cancer FFPE dataset + (https://www.10xgenomics.com/datasets/xenium-prime-ffpe-human-cervical-cancer), subset to a + small tissue region. Licensed under `CC BY 4.0 `_; + see ``datasets.yaml`` for the attribution string shipped alongside the data. + + Parameters + ---------- + path + Directory in which to cache the downloaded data. If `None`, the default OS cache + location is used (:func:`pooch.os_cache` for ``"spatialdata"``). + + Returns + ------- + SpatialData object with the cells dataset. + """ + from scverse_misc.datasets import fetch + + base_url, datasets = _shipped_registry() + sdata: SpatialData = fetch(datasets["cells"], _cache_dir(path), base_url=base_url) + return sdata + + class RaccoonDataset: """Raccoon dataset.""" diff --git a/src/spatialdata/datasets.yaml b/src/spatialdata/datasets.yaml new file mode 100644 index 000000000..62e151d45 --- /dev/null +++ b/src/spatialdata/datasets.yaml @@ -0,0 +1,27 @@ +# Registry of downloadable example datasets for ``spatialdata.datasets``. +# +# Parsed by ``scverse_misc.datasets.parse_registry`` and fetched (downloaded, +# hash-verified, cached and loaded) via ``scverse_misc.datasets.fetch``. +# +# type: spatialdata -> a .zip that extracts to a single .zarr store +# +# Every dataset must list its ``license``; datasets under a license that requires +# attribution must also carry an ``attribution`` string crediting the original source, and +# datasets under a license that requires linking the license (e.g. CC BY 4.0) must also carry +# a ``license_url`` field. +base_url: https://exampledata.scverse.org/spatialdata/ +datasets: + cells: + type: spatialdata + doc_header: Cells dataset as a SpatialData object. + license: CC BY 4.0 + license_url: https://creativecommons.org/licenses/by/4.0/ + attribution: >- + Derived from the 10x Genomics Xenium Prime Cervical Cancer FFPE dataset + (https://www.10xgenomics.com/datasets/xenium-prime-ffpe-human-cervical-cancer), + subset to a small tissue region. Licensed under CC BY 4.0 + (https://creativecommons.org/licenses/by/4.0/). + files: + - name: cells.zip + s3_key: cells.zip + sha256: dc9613cb9e16fd2cd8d83f3a9586eeda4af5ba8ba366f1066efb51305820c5fb diff --git a/src/spatialdata/transformations/transformations.py b/src/spatialdata/transformations/transformations.py index deace0ae1..b06e4319e 100644 --- a/src/spatialdata/transformations/transformations.py +++ b/src/spatialdata/transformations/transformations.py @@ -110,6 +110,15 @@ def _get_default_coordinate_system( @abstractmethod def inverse(self) -> BaseTransformation: + """ + Return the inverse of the transformation. + + Returns + ------- + BaseTransformation + A new transformation that is the inverse of this one, such that applying + both in sequence yields the identity transformation. + """ pass # @abstractmethod @@ -118,6 +127,22 @@ def inverse(self) -> BaseTransformation: @abstractmethod def to_affine_matrix(self, input_axes: tuple[ValidAxis_t, ...], output_axes: tuple[ValidAxis_t, ...]) -> ArrayLike: + """ + Return the affine matrix representation of the transformation. + + Parameters + ---------- + input_axes + The axes of the input coordinate system, e.g. ``("x", "y")`` or ``("c", "z", "y", "x")``. + output_axes + The axes of the output coordinate system. + + Returns + ------- + ArrayLike + A homogeneous affine matrix of shape ``(len(output_axes) + 1, len(input_axes) + 1)``. + The last row is always ``[0, 0, ..., 1]`` (homogeneity). + """ pass def to_affine(self, input_axes: tuple[ValidAxis_t, ...], output_axes: tuple[ValidAxis_t, ...]) -> Affine: diff --git a/tests/conftest.py b/tests/conftest.py index 617acb90c..3aa3e0024 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ os.environ["NUMBA_DISABLE_JIT"] = "1" # If a pytest plugin already imported numba before this conftest ran, patch the cached config value too. if "numba.core.config" in sys.modules: - sys.modules["numba.core.config"].NUMBA_DISABLE_JIT = 1 + sys.modules["numba.core.config"].DISABLE_JIT = 1 import copy as _copy from collections.abc import Callable, Sequence @@ -45,6 +45,21 @@ ) +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--run-network", action="store_true", default=False, help="run tests marked 'network' (e.g. dataset downloads)" + ) + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + if config.getoption("--run-network"): + return + skip_network = pytest.mark.skip(reason="need --run-network option to run") + for item in items: + if "network" in item.keywords: + item.add_marker(skip_network) + + def _fast_deepcopy_sdata(sd: SpatialData) -> SpatialData: """ Fast deepcopy for SpatialData objects in tests. diff --git a/tests/datasets/test_datasets.py b/tests/datasets/test_datasets.py index 2237e253c..3fc66370c 100644 --- a/tests/datasets/test_datasets.py +++ b/tests/datasets/test_datasets.py @@ -1,6 +1,12 @@ from __future__ import annotations -from spatialdata.datasets import blobs, raccoon +from pathlib import Path + +import pooch +import pytest + +from spatialdata import SpatialData +from spatialdata.datasets import _cache_dir, _shipped_registry, blobs, cells, raccoon def test_datasets() -> None: @@ -26,3 +32,41 @@ def test_datasets() -> None: assert sdata_raccoon.images["raccoon"].shape == (3, 768, 1024) assert sdata_raccoon.labels["segmentation"].shape == (768, 1024) _ = str(sdata_raccoon) + + +def test_cells_registry() -> None: + # Network-free: the shipped registry parses and exposes the cells dataset. + base_url, datasets = _shipped_registry() + + assert base_url == "https://exampledata.scverse.org/spatialdata/" + entry = datasets["cells"] + assert entry.type == "spatialdata" + file = entry.file(name="cells.zip") + assert file.sha256 == "dc9613cb9e16fd2cd8d83f3a9586eeda4af5ba8ba366f1066efb51305820c5fb" + assert file.resolve_url(base_url) == "https://exampledata.scverse.org/spatialdata/cells.zip" + + +def test_cache_dir() -> None: + # Network-free: both branches of the cache-directory resolution. + assert _cache_dir("/tmp/example") == Path("/tmp/example") + assert _cache_dir(None) == Path(pooch.os_cache("spatialdata")) + + +@pytest.mark.network +def test_cells_download(tmp_path) -> None: + # Downloads ~3 MB from the scverse example data bucket; skipped by default, opt in with `--run-network`. + sdata = cells(path=str(tmp_path)) + assert isinstance(sdata, SpatialData) + + assert set(sdata.images) == {"he_aligned", "he_image", "morphology_focus"} + assert sdata.images["he_aligned"]["scale0"]["image"].shape == (3, 430, 540) + assert sdata.images["he_image"]["scale0"]["image"].shape == (3, 423, 339) + assert sdata.images["morphology_focus"]["scale0"]["image"].shape == (4, 430, 540) + + assert set(sdata.labels) == {"cell_labels", "nucleus_labels", "tissue_labels"} + assert sdata.labels["cell_labels"]["scale0"]["image"].shape == (430, 540) + + assert len(sdata.shapes["cell_boundaries"]) == 94 + assert len(sdata.shapes["nucleus_boundaries"]) == 94 + assert len(sdata.points["transcripts"].compute()) == 19479 + assert sdata.tables["table"].shape == (94, 5101) diff --git a/tests/io/test_readwrite.py b/tests/io/test_readwrite.py index 034c01d37..e9affd917 100644 --- a/tests/io/test_readwrite.py +++ b/tests/io/test_readwrite.py @@ -4,6 +4,7 @@ import os import tempfile from collections.abc import Callable +from importlib.metadata import version from pathlib import Path from typing import Any, Literal @@ -17,6 +18,7 @@ from anndata import AnnData from numpy.random import default_rng from packaging.version import Version +from pandas.testing import assert_series_equal from shapely import MultiPolygon, Polygon from upath import UPath from xarray import DataArray @@ -1289,13 +1291,13 @@ def test_read_sdata(tmp_path: Path, points: SpatialData) -> None: assert_spatial_data_objects_are_identical(sdata_from_path, sdata_from_zarr_group) -def test_sdata_with_nan_in_obs(tmp_path: Path) -> None: +@pytest.mark.parametrize("convert_strings_to_categoricals", (True, False)) +def test_sdata_with_nan_in_obs(tmp_path: Path, convert_strings_to_categoricals: bool) -> None: """Test writing SpatialData with mixed string/NaN values in obs works correctly. Regression test for https://github.com/scverse/spatialdata/issues/399 Previously this raised TypeError: expected unicode string, found nan. - Now the write succeeds, though NaN values in object-dtype columns are - converted to the string "nan" after round-trip. + Now the write succeeds, and NaN values are preserved round trip """ from spatialdata.models import TableModel @@ -1319,8 +1321,17 @@ def test_sdata_with_nan_in_obs(tmp_path: Path) -> None: assert sdata["table"].obs["column_only_region1"].iloc[1] is np.nan assert np.isnan(sdata["table"].obs["column_only_region2"].iloc[0]) + dtypes_before_writing = sdata["table"].obs.dtypes.copy() + path = tmp_path / "data.zarr" - sdata.write(path) + sdata.write(path, convert_table_strings_to_categoricals=convert_strings_to_categoricals) + + if convert_strings_to_categoricals: + expected_dtypes = dtypes_before_writing + expected_dtypes["column_only_region1"] = "category" + assert_series_equal(sdata["table"].obs.dtypes, expected_dtypes) + else: + assert_series_equal(sdata["table"].obs.dtypes, dtypes_before_writing) sdata2 = SpatialData.read(path) assert "column_only_region1" in sdata2["table"].obs.columns @@ -1329,8 +1340,12 @@ def test_sdata_with_nan_in_obs(tmp_path: Path) -> None: assert r1.iloc[0] == "string" assert r2.iloc[1] == 3 - if Version(pd.__version__) >= Version("3"): - assert pd.isna(r1.iloc[1]) - else: # After round-trip, NaN in object-dtype column becomes string "nan" on pandas 2 - assert r1.iloc[1] == "nan" assert np.isnan(r2.iloc[0]) + + if Version(version("pandas")) >= Version("3"): + assert pd.isna(r1.iloc[1]) + else: # After round-trip, NaN in object-dtype column becomes string + if convert_strings_to_categoricals: + assert pd.isna(r1.iloc[1]) + else: + assert r1.iloc[1] == "nan" diff --git a/tests/models/test_models.py b/tests/models/test_models.py index 3b173a15b..33a98a434 100644 --- a/tests/models/test_models.py +++ b/tests/models/test_models.py @@ -26,6 +26,7 @@ from spatial_image import to_spatial_image from xarray import DataArray, DataTree +import spatialdata from spatialdata._core.spatialdata import SpatialData from spatialdata._core.validation import ValidationError from spatialdata._types import ArrayLike @@ -624,6 +625,18 @@ def test_table_model_invalid_names(self, key: str, attr: str, parse: bool): else: TableModel.validate(adata) + def test_table_model_invalid_name_suggests_public_sanitizer(self): + adata = AnnData(np.array([[0]]), uns={"invalid name": {}}) + + with pytest.raises( + ValidationError, + match=r"`spatialdata\.sanitize_table\(adata\)`", + ): + TableModel.validate(adata) + + spatialdata.sanitize_table(adata) + TableModel.validate(adata) + @pytest.mark.parametrize( "keys", [