From 63a5780f29580ca98bdb26dbb44ef596a04e43e0 Mon Sep 17 00:00:00 2001 From: stanbot8 Date: Thu, 23 Jul 2026 13:59:53 -0700 Subject: [PATCH 01/12] fix: point table validation to public sanitizer --- src/spatialdata/_core/validation.py | 2 +- tests/models/test_models.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/spatialdata/_core/validation.py b/src/spatialdata/_core/validation.py index ad1e7a91e..b6c2eebfe 100644 --- a/src/spatialdata/_core/validation.py +++ b/src/spatialdata/_core/validation.py @@ -388,7 +388,7 @@ def __exit__( # Exceptions were collected that we want to raise as a combined validation error. if self._collector.errors: raise ValidationError( - title=self._message + "\nTo fix, run `spatialdata.utils.sanitize_table(adata)`.", + title=self._message + "\nTo fix, run `spatialdata.sanitize_table(adata)`.", errors=self._collector.errors, ) return True diff --git a/tests/models/test_models.py b/tests/models/test_models.py index 3b173a15b..33a98a434 100644 --- a/tests/models/test_models.py +++ b/tests/models/test_models.py @@ -26,6 +26,7 @@ from spatial_image import to_spatial_image from xarray import DataArray, DataTree +import spatialdata from spatialdata._core.spatialdata import SpatialData from spatialdata._core.validation import ValidationError from spatialdata._types import ArrayLike @@ -624,6 +625,18 @@ def test_table_model_invalid_names(self, key: str, attr: str, parse: bool): else: TableModel.validate(adata) + def test_table_model_invalid_name_suggests_public_sanitizer(self): + adata = AnnData(np.array([[0]]), uns={"invalid name": {}}) + + with pytest.raises( + ValidationError, + match=r"`spatialdata\.sanitize_table\(adata\)`", + ): + TableModel.validate(adata) + + spatialdata.sanitize_table(adata) + TableModel.validate(adata) + @pytest.mark.parametrize( "keys", [ From 6a805be18b47d2ec3226f4d6aa6d533ab2a14e7a Mon Sep 17 00:00:00 2001 From: Jan Gleixner Date: Wed, 5 Aug 2026 13:36:52 +0200 Subject: [PATCH 02/12] Fix typo in SpatialData documentation (#1173) Fix typo in documentation --- src/spatialdata/_core/spatialdata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spatialdata/_core/spatialdata.py b/src/spatialdata/_core/spatialdata.py index fb55ab086..088de3d77 100644 --- a/src/spatialdata/_core/spatialdata.py +++ b/src/spatialdata/_core/spatialdata.py @@ -107,7 +107,7 @@ class SpatialData: - the table are stored as :class:`anndata.AnnData` objects, with the spatial coordinates stored in the obsm slot. - The table can annotate regions (shapesor labels) and can be used to store additional information. + The table can annotate regions (shapes or labels) and can be used to store additional information. Points are not regions but 0-dimensional locations. They can't be annotated by a table, but they can store annotation directly. """ From 16d186d1e50cf20a6705540836d4233ae45b4847 Mon Sep 17 00:00:00 2001 From: Alberto Fabbri Date: Thu, 6 Aug 2026 19:21:53 +0200 Subject: [PATCH 03/12] Fix typo in parameter description of read_zarr function (#1171) --- src/spatialdata/_io/io_zarr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spatialdata/_io/io_zarr.py b/src/spatialdata/_io/io_zarr.py index 4c410fab0..7ba440533 100644 --- a/src/spatialdata/_io/io_zarr.py +++ b/src/spatialdata/_io/io_zarr.py @@ -136,7 +136,7 @@ def read_zarr( Path, URL, or zarr.Group to the zarr store (on-disk or remote). selection - List of elements to read from the zarr store (images, labels, points, shapes, table). If None, all elements are + List of elements to read from the zarr store (images, labels, points, shapes, tables). If None, all elements are read. on_bad_files From 29e0b0bddf0fb71901dbedaad3e49d02f2b21319 Mon Sep 17 00:00:00 2001 From: Lukas Heumos Date: Fri, 14 Aug 2026 13:16:35 +0200 Subject: [PATCH 04/12] Update submodule URL for spatialdata tutorials --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index cf0594cfa..57216b9bd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "docs/tutorials/notebooks"] path = docs/tutorials/notebooks - url = https://github.com/scverse/spatialdata-notebooks + url = https://github.com/scverse/spatialdata-tutorials From 4e1968129695ec4f3c5ee579ddf79f651ca18faf Mon Sep 17 00:00:00 2001 From: Jan Gleixner Date: Fri, 14 Aug 2026 13:21:20 +0200 Subject: [PATCH 05/12] Fix contributor dependency installation instructions (docs only) (#1174) Fix contributor dependency installation --- docs/contributing.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/contributing.md b/docs/contributing.md index 4faba0274..6026d4e1f 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -13,8 +13,7 @@ In addition to the packages needed to _use_ this package, you need additional py the documentation_. It's easy to install them using `pip`: ```bash -cd spatialdata-io -pip install -e ".[dev,test,doc]" +pip install -e . --group dev --group test --group docs ``` ## Code-style @@ -51,7 +50,7 @@ and [prettier][prettier-editors]. ## Writing tests ```{note} -Remember to first install the package with `pip install '-e[dev,test]'` +Remember to first install the package with `pip install -e . --group dev --group test` ``` This package uses [pytest][] for automated testing. Please [write tests][scanpy-test-docs] for every function added to the package. From 2debe5f2559ab61de02d0241a85961ef715a4ec0 Mon Sep 17 00:00:00 2001 From: Ajayrama Kumaraswamy Date: Tue, 18 Aug 2026 13:30:35 +0200 Subject: [PATCH 06/12] AnnData obs/var string to categoricals (#1170) * chore: added prek to dev deps * feat: writing table to zarr using AnnData.write_zarr for table version 2 * chore: added ruff to dev deps * fix: using internal resolve store function + categorical when writing tables to zarr v2 * fix: test no longer expectes 'nan' after table round trip instead of pd.NA/np.nan * feat: added hatch env configs for testing version combinations of anndata/pandas * feat: update hatch config for testing pandas/anndata versions * feat: added DeprecationWarnings when writing to zarr v2 * fix: if-raise instead of assert * refac + doc: pyproject.toml * fix: made version comparison of anndata more robust * fix: doc string * fix: made converting table strings to categorical optional + non-default * feat: documentation strings * feat: revised tests to work with new write parameter * fix: removed unnecessary redeclaration of table group * fix: doc string improvements * chore: docs touch-up; removed unnecessary .copy() --------- Co-authored-by: Luca Marconato Co-authored-by: LucaMarconato <2664412+LucaMarconato@users.noreply.github.com> --- pyproject.toml | 35 ++++++++++++++ src/spatialdata/_core/spatialdata.py | 15 +++++- src/spatialdata/_io/exceptions.py | 26 +++++++++++ src/spatialdata/_io/io_points.py | 6 +++ src/spatialdata/_io/io_raster.py | 12 +++++ src/spatialdata/_io/io_shapes.py | 7 +++ src/spatialdata/_io/io_table.py | 69 +++++++++++++++++++++++++--- tests/io/test_readwrite.py | 31 +++++++++---- 8 files changed, 185 insertions(+), 16 deletions(-) create mode 100644 src/spatialdata/_io/exceptions.py diff --git a/pyproject.toml b/pyproject.toml index 03181eadb..76aded8a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -244,3 +244,38 @@ memray-flame = "memray flamegraph --temporal" [tool.pixi.environments] profiling = { features = ["profiling"], solve-group = "default" } + +[tool.hatch.envs.test] +dependency-groups = ["test"] + +[tool.hatch.envs.test-anndata-pandas] +template = "test" +extra-dependencies = ["zarr>=3"] +scripts.test = ["pip list|grep anndata && pip list|grep pandas && pytest {args}"] +scripts.test-readwrite = ["pip list|grep anndata && pip list|grep pandas && pytest tests/io/test_readwrite.py"] +scripts.test-all = ["pip list|grep anndata && pip list|grep pandas && pytest ."] + +[[tool.hatch.envs.test-anndata-pandas.matrix]] +anndata-pandas = [ + "0.13-2", + "0.13-3", + "0.12-2" + # no "0.12-3": support for pandas>=3 is available only in anndata>=0.13 +] + +[tool.hatch.envs.test-anndata-pandas.overrides] +matrix.anndata-pandas.extra-dependencies = [ + # every option where the if-condition is True gets included + + # anndata 0.13 + {value="anndata~=0.13", if = ["0.13-2", "0.13-3"]}, + + # anndata 0.12 + {value="anndata>=0.12,<0.13", if = ["0.12-2"]}, + + # pandas 2 + {value="pandas>=2.3,<3", if = ["0.13-2", "0.12-2"]}, + + # pandas 3 + {value="pandas~=3.0", if = ["0.13-3"]}, +] diff --git a/src/spatialdata/_core/spatialdata.py b/src/spatialdata/_core/spatialdata.py index 088de3d77..6ee2296c8 100644 --- a/src/spatialdata/_core/spatialdata.py +++ b/src/spatialdata/_core/spatialdata.py @@ -1114,6 +1114,7 @@ def write( sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None, shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None, raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None, + convert_table_strings_to_categoricals: bool = False, ) -> None: """ Write the `SpatialData` object to a Zarr store. @@ -1166,6 +1167,9 @@ def write( compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically ordered for more efficient compression. + convert_table_strings_to_categoricals + If True, convert string columns of all tables to categoricals before writing. + Note that this will have a side effect of modifying string columns into categoricals in place. """ from spatialdata._io._utils import _resolve_zarr_store, _validate_compressor_args from spatialdata._io.format import _parse_formats @@ -1194,6 +1198,7 @@ def write( parsed_formats=parsed, shapes_geometry_encoding=shapes_geometry_encoding, raster_compressor=raster_compressor, + convert_table_strings_to_categoricals=convert_table_strings_to_categoricals, ) if self.path != file_path and update_sdata_path: @@ -1212,6 +1217,7 @@ def _write_element( parsed_formats: dict[str, SpatialDataFormatType] | None = None, shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None, raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None, + convert_table_strings_to_categoricals: bool = False, ) -> None: from spatialdata._io.io_zarr import _get_groups_for_element @@ -1279,6 +1285,7 @@ def _write_element( group=element_type_group, name=element_name, element_format=parsed_formats["tables"], + convert_strings_to_categoricals=convert_table_strings_to_categoricals, ) else: raise ValueError(f"Unknown element type: {element_type}") @@ -1290,6 +1297,7 @@ def write_element( sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None, shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None, raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None, + convert_table_strings_to_categoricals: bool = False, ) -> None: """ Write a single element, or a list of elements, to the Zarr store used for backing. @@ -1308,11 +1316,14 @@ def write_element( shapes_geometry_encoding Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet` for details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`. - raster_compressor + raster_compressor A lenght-1 dictionary with as key the type of compression to use for images and labels and as value the compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically ordered for more efficient compression. + convert_table_strings_to_categoricals + If True, and if element to be written is a table, convert string columns to categoricals before writing. + Note that this will have a side effect of modifying string columns into categoricals in place. Notes ----- @@ -1332,6 +1343,7 @@ def write_element( sdata_formats=sdata_formats, shapes_geometry_encoding=shapes_geometry_encoding, raster_compressor=raster_compressor, + convert_table_strings_to_categoricals=convert_table_strings_to_categoricals, ) return @@ -1368,6 +1380,7 @@ def write_element( parsed_formats=parsed_formats, shapes_geometry_encoding=shapes_geometry_encoding, raster_compressor=raster_compressor, + convert_table_strings_to_categoricals=convert_table_strings_to_categoricals, ) # After every write, metadata should be consolidated, otherwise this can lead to IO problems like when deleting. if self.has_consolidated_metadata(): diff --git a/src/spatialdata/_io/exceptions.py b/src/spatialdata/_io/exceptions.py new file mode 100644 index 000000000..66f5802b7 --- /dev/null +++ b/src/spatialdata/_io/exceptions.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from ome_zarr.format import Format + + +class FormatVersionUnknownError(ValueError): + """Exception raised when an unknown element format is encountered.""" + + def __init__(self, element_type: str, version_encountered: Format): + self.element_type = element_type + self.version_encountered = version_encountered + self.message = ( + f"Encountered unknown element format version " + f"`{self.version_encountered}` for element of type `{self.element_type}`" + ) + super().__init__(self.message) + + +class WritingToZarrV2DeprecationWarning(DeprecationWarning): + """Warning raised when writing to zarr v2 format.""" + + message = ( + "Writing to zarr v2 format is currently deprecated in spatialdata " + "and will be removed in a future version. " + "Please consider writing to zarr v3." + ) diff --git a/src/spatialdata/_io/io_points.py b/src/spatialdata/_io/io_points.py index 03ef33389..bb203cad2 100644 --- a/src/spatialdata/_io/io_points.py +++ b/src/spatialdata/_io/io_points.py @@ -1,5 +1,6 @@ from __future__ import annotations +import warnings from pathlib import Path import zarr @@ -12,6 +13,7 @@ _write_metadata, overwrite_coordinate_transformations_non_raster, ) +from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning from spatialdata._io.format import CurrentPointsFormat, PointsFormats, _parse_version from spatialdata.models import get_axes_names from spatialdata.transformations._utils import ( @@ -65,6 +67,10 @@ def write_points( element_format The format of the points element used to store it. """ + if element_format.zarr_format == 2: + warnings.warn( + message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2 + ) axes = get_axes_names(points) transformations = _get_transformations(points) assert transformations is not None # mypy: validate_element() in _write_element guarantees this diff --git a/src/spatialdata/_io/io_raster.py b/src/spatialdata/_io/io_raster.py index 276f016bd..b9a2964f0 100644 --- a/src/spatialdata/_io/io_raster.py +++ b/src/spatialdata/_io/io_raster.py @@ -1,5 +1,6 @@ from __future__ import annotations +import warnings from collections.abc import Sequence from pathlib import Path from typing import Any, Literal, TypeGuard, cast @@ -23,6 +24,7 @@ overwrite_channel_names, overwrite_coordinate_transformations_raster, ) +from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning from spatialdata._io.format import ( CurrentRasterFormat, RasterFormatType, @@ -581,6 +583,11 @@ def write_image( raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None, **metadata: str | JSONDict | list[JSONDict], ) -> None: + if element_format.zarr_format == 2: + warnings.warn( + message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2 + ) + _write_raster( raster_type="image", raster_data=image, @@ -603,6 +610,11 @@ def write_labels( raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None, **metadata: JSONDict, ) -> None: + if element_format.zarr_format == 2: + warnings.warn( + message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2 + ) + _write_raster( raster_type="labels", raster_data=labels, diff --git a/src/spatialdata/_io/io_shapes.py b/src/spatialdata/_io/io_shapes.py index 3b6e18e39..f8528868d 100644 --- a/src/spatialdata/_io/io_shapes.py +++ b/src/spatialdata/_io/io_shapes.py @@ -1,5 +1,6 @@ from __future__ import annotations +import warnings from pathlib import Path from typing import Any, Literal @@ -15,6 +16,7 @@ _write_metadata, overwrite_coordinate_transformations_non_raster, ) +from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning from spatialdata._io.format import ( CurrentShapesFormat, ShapesFormats, @@ -93,6 +95,11 @@ def write_shapes( Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet` for details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`. """ + if element_format.zarr_format == 2: + warnings.warn( + message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2 + ) + from spatialdata.config import settings if geometry_encoding is None: diff --git a/src/spatialdata/_io/io_table.py b/src/spatialdata/_io/io_table.py index 3eb4b0927..931572389 100644 --- a/src/spatialdata/_io/io_table.py +++ b/src/spatialdata/_io/io_table.py @@ -1,5 +1,7 @@ from __future__ import annotations +import warnings +from importlib.metadata import version from pathlib import Path import numpy as np @@ -8,7 +10,10 @@ from anndata import read_zarr as read_anndata_zarr from anndata._io.specs import write_elem as write_adata from ome_zarr.format import Format +from packaging.version import Version +from spatialdata._io._utils import _resolve_zarr_store +from spatialdata._io.exceptions import FormatVersionUnknownError, WritingToZarrV2DeprecationWarning from spatialdata._io.format import ( CurrentTablesFormat, TablesFormats, @@ -55,17 +60,67 @@ def write_table( name: str, group_type: str = "ngff:regions_table", element_format: Format = CurrentTablesFormat(), + convert_strings_to_categoricals: bool = False, ) -> None: + """ + Write a table to a Zarr store. + + Parameters + ---------- + table + The table to write. + group + The table will be written into a subgroup of this group + name + The name of the subgroup of `group` to which table is to be written. + group_type + The type of the group. + element_format + The format to use for writing the table. + convert_strings_to_categoricals + If True, convert string columns to categoricals before writing. + Note that this will have a side effect of modifying dtypes of the input table in place. + """ + if element_format.zarr_format == 2: + warnings.warn( + message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2 + ) + if TableModel.ATTRS_KEY in table.uns: region, region_key, instance_key = get_table_keys(table) TableModel.validate(table) else: region, region_key, instance_key = (None, None, None) - write_adata(group, name, table) - tables_group = group[name] - tables_group.attrs["spatialdata-encoding-type"] = group_type - tables_group.attrs["region"] = region - tables_group.attrs["region_key"] = region_key - tables_group.attrs["instance_key"] = instance_key - tables_group.attrs["version"] = element_format.spatialdata_format_version + # Ensure the table group exists + table_group = group.require_group(name=name) + + if element_format not in TablesFormats.values(): + raise FormatVersionUnknownError(element_type="table", version_encountered=element_format) + + if element_format.zarr_format == 3 and Version(version("anndata")) >= Version("0.13"): + # `write_zarr` in anndata v0.13 and above can only write to zarr v3 + # solution of passing resolved store directly roughly based on: + # https://github.com/scverse/anndata/issues/1548#issuecomment-2199801855 + + # resolve the store from the group + resolved_store = _resolve_zarr_store(table_group) + + # Write the table to the path of the table group + table.write_zarr( + store=resolved_store, + consolidate_metadata=False, + convert_strings_to_categoricals=convert_strings_to_categoricals, + ) + + else: + if convert_strings_to_categoricals: + table.strings_to_categoricals() + + write_adata(group, name, table) + + table_group.attrs["spatialdata-encoding-type"] = group_type + table_group.attrs["region"] = region + table_group.attrs["region_key"] = region_key + table_group.attrs["instance_key"] = instance_key + table_group.attrs["version"] = element_format.spatialdata_format_version diff --git a/tests/io/test_readwrite.py b/tests/io/test_readwrite.py index 034c01d37..e9affd917 100644 --- a/tests/io/test_readwrite.py +++ b/tests/io/test_readwrite.py @@ -4,6 +4,7 @@ import os import tempfile from collections.abc import Callable +from importlib.metadata import version from pathlib import Path from typing import Any, Literal @@ -17,6 +18,7 @@ from anndata import AnnData from numpy.random import default_rng from packaging.version import Version +from pandas.testing import assert_series_equal from shapely import MultiPolygon, Polygon from upath import UPath from xarray import DataArray @@ -1289,13 +1291,13 @@ def test_read_sdata(tmp_path: Path, points: SpatialData) -> None: assert_spatial_data_objects_are_identical(sdata_from_path, sdata_from_zarr_group) -def test_sdata_with_nan_in_obs(tmp_path: Path) -> None: +@pytest.mark.parametrize("convert_strings_to_categoricals", (True, False)) +def test_sdata_with_nan_in_obs(tmp_path: Path, convert_strings_to_categoricals: bool) -> None: """Test writing SpatialData with mixed string/NaN values in obs works correctly. Regression test for https://github.com/scverse/spatialdata/issues/399 Previously this raised TypeError: expected unicode string, found nan. - Now the write succeeds, though NaN values in object-dtype columns are - converted to the string "nan" after round-trip. + Now the write succeeds, and NaN values are preserved round trip """ from spatialdata.models import TableModel @@ -1319,8 +1321,17 @@ def test_sdata_with_nan_in_obs(tmp_path: Path) -> None: assert sdata["table"].obs["column_only_region1"].iloc[1] is np.nan assert np.isnan(sdata["table"].obs["column_only_region2"].iloc[0]) + dtypes_before_writing = sdata["table"].obs.dtypes.copy() + path = tmp_path / "data.zarr" - sdata.write(path) + sdata.write(path, convert_table_strings_to_categoricals=convert_strings_to_categoricals) + + if convert_strings_to_categoricals: + expected_dtypes = dtypes_before_writing + expected_dtypes["column_only_region1"] = "category" + assert_series_equal(sdata["table"].obs.dtypes, expected_dtypes) + else: + assert_series_equal(sdata["table"].obs.dtypes, dtypes_before_writing) sdata2 = SpatialData.read(path) assert "column_only_region1" in sdata2["table"].obs.columns @@ -1329,8 +1340,12 @@ def test_sdata_with_nan_in_obs(tmp_path: Path) -> None: assert r1.iloc[0] == "string" assert r2.iloc[1] == 3 - if Version(pd.__version__) >= Version("3"): - assert pd.isna(r1.iloc[1]) - else: # After round-trip, NaN in object-dtype column becomes string "nan" on pandas 2 - assert r1.iloc[1] == "nan" assert np.isnan(r2.iloc[0]) + + if Version(version("pandas")) >= Version("3"): + assert pd.isna(r1.iloc[1]) + else: # After round-trip, NaN in object-dtype column becomes string + if convert_strings_to_categoricals: + assert pd.isna(r1.iloc[1]) + else: + assert r1.iloc[1] == "nan" From f31ff45bed451c889bcf37ccfd775c354a06f5c6 Mon Sep 17 00:00:00 2001 From: Tim Treis Date: Tue, 18 Aug 2026 08:26:52 -0400 Subject: [PATCH 07/12] Add downloadable cells dataset via scverse-misc (#1149) * Add downloadable cells dataset via scverse-misc Expose spatialdata.datasets.cells() alongside blobs/raccoon. It downloads the cells example dataset and loads it as a SpatialData object, reusing the scverse-misc datasets infrastructure (parse_registry + fetch with the built-in spatialdata loader) rather than reimplementing a downloader. - ship src/spatialdata/datasets.yaml registry (base_url + cells.zip sha256) - add scverse-misc[datasets]>=0.0.10 dependency - bump requires-python and ruff target to 3.12 (scverse-misc requires >=3.12) - update CI matrix 3.11 -> 3.12 - docs + network-free registry test and a slow download test Co-Authored-By: Claude Opus 4.8 (1M context) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix mypy no-any-return in cells() Co-Authored-By: Claude Opus 4.8 (1M context) * Bump scverse-misc pin to >=0.1.0 (first released datasets version) Co-Authored-By: Claude Opus 4.8 (1M context) * Add license and attribution to cells dataset registry The cells dataset is derived from the 10x Genomics Xenium Prime Cervical Cancer FFPE sample, released under CC BY 4.0 (attribution required). Record the license and attribution in the registry and document the expectation that every dataset lists its license. * refactor(datasets): address review on cells() Extract _shipped_registry() and _cache_dir() helpers, shared between cells() and the tests, removing the duplicated registry-parsing block. _cache_dir() isolates the path-vs-default branch so both sides are covered without a network call. Describe the dataset contents in the docstring, and assert the concrete element counts and shapes in test_cells_download. Co-Authored-By: Claude Opus 4.8 * fix(datasets): satisfy mypy no-any-return in _shipped_registry parse_registry is untyped in the mypy env, so returning its result directly tripped no-any-return; assign through annotated locals. Co-Authored-By: Claude Opus 4.8 * chore: improve attribution and license info * test(datasets): gate network test behind opt-in `network` marker Rename the `slow` marker to `network` and flip it from opt-out (`-m "not slow"`) to opt-in: tests marked `network` are skipped unless `--run-network` is passed. Enable the flag in CI so `test_cells_download` runs there, and document the `license_url` convention for CC-BY-style datasets in datasets.yaml. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Luca Marconato Co-authored-by: LucaMarconato <2664412+LucaMarconato@users.noreply.github.com> --- .github/workflows/test.yaml | 2 +- docs/api/datasets.md | 1 + pyproject.toml | 2 ++ src/spatialdata/datasets.py | 62 ++++++++++++++++++++++++++++++++- src/spatialdata/datasets.yaml | 27 ++++++++++++++ tests/conftest.py | 15 ++++++++ tests/datasets/test_datasets.py | 46 +++++++++++++++++++++++- 7 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 src/spatialdata/datasets.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index eff3010d3..bc65f074a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -59,7 +59,7 @@ jobs: PLATFORM: ${{ matrix.os }} DISPLAY: :42 run: | - uv run pytest --cov --color=yes --cov-report=xml -n auto --dist worksteal + uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal - name: Upload coverage to Codecov uses: codecov/codecov-action@v6 with: diff --git a/docs/api/datasets.md b/docs/api/datasets.md index 7bf6d5a61..d0c43b56c 100644 --- a/docs/api/datasets.md +++ b/docs/api/datasets.md @@ -7,5 +7,6 @@ Convenience small datasets .. autofunction:: blobs .. autofunction:: blobs_annotating_element +.. autofunction:: cells .. autofunction:: raccoon ``` diff --git a/pyproject.toml b/pyproject.toml index 76aded8a5..a8bce770a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ dependencies = [ "spatial_image>=1.2.3", "scikit-image", "scipy!=1.17.0", + "scverse-misc[datasets]>=0.1.0", "typing_extensions>=4.8.0", "universal_pathlib>=0.2.6", "xarray>=2024.10.0", @@ -108,6 +109,7 @@ addopts = [ # These are all markers coming from xarray, dask or anndata. Added here to silence warnings. markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "network: marks tests that require network access; skipped by default, run with '--run-network'", "gpu: run test on GPU using CuPY.", "array_api: used by anndata.tests.helpers, not us", "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on", diff --git a/src/spatialdata/datasets.py b/src/spatialdata/datasets.py index 37f529c72..51d360203 100644 --- a/src/spatialdata/datasets.py +++ b/src/spatialdata/datasets.py @@ -3,6 +3,7 @@ from __future__ import annotations import warnings +from pathlib import Path from typing import Any, Literal import dask.dataframe.core @@ -31,7 +32,7 @@ ) from spatialdata.transformations import Identity -__all__ = ["blobs", "raccoon"] +__all__ = ["blobs", "cells", "raccoon"] def blobs( @@ -79,6 +80,65 @@ def raccoon() -> SpatialData: return RaccoonDataset().raccoon() +def _shipped_registry() -> tuple[str | None, dict[str, Any]]: + """Parse the ``datasets.yaml`` registry shipped inside the ``spatialdata`` package.""" + import importlib.resources + + from scverse_misc.datasets import parse_registry + + registry = importlib.resources.files("spatialdata").joinpath("datasets.yaml") + with importlib.resources.as_file(registry) as registry_path: + base_url: str | None + datasets: dict[str, Any] + base_url, datasets = parse_registry(registry_path) + return base_url, datasets + + +def _cache_dir(path: str | None) -> Path: + """Resolve the cache directory, defaulting to the OS cache location for ``"spatialdata"``.""" + import pooch + + return Path(path) if path is not None else Path(pooch.os_cache("spatialdata")) + + +def cells(path: str | None = None) -> SpatialData: + """ + Cells dataset. + + Download the ``cells`` example dataset and load it as a :class:`~spatialdata.SpatialData` + object. The download is hash-verified and cached, so repeated calls reuse the local copy + instead of downloading again. + + The dataset is a small region of a Xenium Prime Cervical Cancer sample and contains three + multiscale images (``he_aligned``, ``he_image``, ``morphology_focus``), three multiscale + label layers (``cell_labels``, ``nucleus_labels``, ``tissue_labels``), the ``transcripts`` + points, the ``cell_boundaries`` and ``nucleus_boundaries`` shapes, and a cell-by-gene + ``table`` annotating the 94 cells. + + Notes + ----- + Derived from the 10x Genomics Xenium Prime Cervical Cancer FFPE dataset + (https://www.10xgenomics.com/datasets/xenium-prime-ffpe-human-cervical-cancer), subset to a + small tissue region. Licensed under `CC BY 4.0 `_; + see ``datasets.yaml`` for the attribution string shipped alongside the data. + + Parameters + ---------- + path + Directory in which to cache the downloaded data. If `None`, the default OS cache + location is used (:func:`pooch.os_cache` for ``"spatialdata"``). + + Returns + ------- + SpatialData object with the cells dataset. + """ + from scverse_misc.datasets import fetch + + base_url, datasets = _shipped_registry() + sdata: SpatialData = fetch(datasets["cells"], _cache_dir(path), base_url=base_url) + return sdata + + class RaccoonDataset: """Raccoon dataset.""" diff --git a/src/spatialdata/datasets.yaml b/src/spatialdata/datasets.yaml new file mode 100644 index 000000000..62e151d45 --- /dev/null +++ b/src/spatialdata/datasets.yaml @@ -0,0 +1,27 @@ +# Registry of downloadable example datasets for ``spatialdata.datasets``. +# +# Parsed by ``scverse_misc.datasets.parse_registry`` and fetched (downloaded, +# hash-verified, cached and loaded) via ``scverse_misc.datasets.fetch``. +# +# type: spatialdata -> a .zip that extracts to a single .zarr store +# +# Every dataset must list its ``license``; datasets under a license that requires +# attribution must also carry an ``attribution`` string crediting the original source, and +# datasets under a license that requires linking the license (e.g. CC BY 4.0) must also carry +# a ``license_url`` field. +base_url: https://exampledata.scverse.org/spatialdata/ +datasets: + cells: + type: spatialdata + doc_header: Cells dataset as a SpatialData object. + license: CC BY 4.0 + license_url: https://creativecommons.org/licenses/by/4.0/ + attribution: >- + Derived from the 10x Genomics Xenium Prime Cervical Cancer FFPE dataset + (https://www.10xgenomics.com/datasets/xenium-prime-ffpe-human-cervical-cancer), + subset to a small tissue region. Licensed under CC BY 4.0 + (https://creativecommons.org/licenses/by/4.0/). + files: + - name: cells.zip + s3_key: cells.zip + sha256: dc9613cb9e16fd2cd8d83f3a9586eeda4af5ba8ba366f1066efb51305820c5fb diff --git a/tests/conftest.py b/tests/conftest.py index 617acb90c..871b4280f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,6 +45,21 @@ ) +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--run-network", action="store_true", default=False, help="run tests marked 'network' (e.g. dataset downloads)" + ) + + +def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None: + if config.getoption("--run-network"): + return + skip_network = pytest.mark.skip(reason="need --run-network option to run") + for item in items: + if "network" in item.keywords: + item.add_marker(skip_network) + + def _fast_deepcopy_sdata(sd: SpatialData) -> SpatialData: """ Fast deepcopy for SpatialData objects in tests. diff --git a/tests/datasets/test_datasets.py b/tests/datasets/test_datasets.py index 2237e253c..3fc66370c 100644 --- a/tests/datasets/test_datasets.py +++ b/tests/datasets/test_datasets.py @@ -1,6 +1,12 @@ from __future__ import annotations -from spatialdata.datasets import blobs, raccoon +from pathlib import Path + +import pooch +import pytest + +from spatialdata import SpatialData +from spatialdata.datasets import _cache_dir, _shipped_registry, blobs, cells, raccoon def test_datasets() -> None: @@ -26,3 +32,41 @@ def test_datasets() -> None: assert sdata_raccoon.images["raccoon"].shape == (3, 768, 1024) assert sdata_raccoon.labels["segmentation"].shape == (768, 1024) _ = str(sdata_raccoon) + + +def test_cells_registry() -> None: + # Network-free: the shipped registry parses and exposes the cells dataset. + base_url, datasets = _shipped_registry() + + assert base_url == "https://exampledata.scverse.org/spatialdata/" + entry = datasets["cells"] + assert entry.type == "spatialdata" + file = entry.file(name="cells.zip") + assert file.sha256 == "dc9613cb9e16fd2cd8d83f3a9586eeda4af5ba8ba366f1066efb51305820c5fb" + assert file.resolve_url(base_url) == "https://exampledata.scverse.org/spatialdata/cells.zip" + + +def test_cache_dir() -> None: + # Network-free: both branches of the cache-directory resolution. + assert _cache_dir("/tmp/example") == Path("/tmp/example") + assert _cache_dir(None) == Path(pooch.os_cache("spatialdata")) + + +@pytest.mark.network +def test_cells_download(tmp_path) -> None: + # Downloads ~3 MB from the scverse example data bucket; skipped by default, opt in with `--run-network`. + sdata = cells(path=str(tmp_path)) + assert isinstance(sdata, SpatialData) + + assert set(sdata.images) == {"he_aligned", "he_image", "morphology_focus"} + assert sdata.images["he_aligned"]["scale0"]["image"].shape == (3, 430, 540) + assert sdata.images["he_image"]["scale0"]["image"].shape == (3, 423, 339) + assert sdata.images["morphology_focus"]["scale0"]["image"].shape == (4, 430, 540) + + assert set(sdata.labels) == {"cell_labels", "nucleus_labels", "tissue_labels"} + assert sdata.labels["cell_labels"]["scale0"]["image"].shape == (430, 540) + + assert len(sdata.shapes["cell_boundaries"]) == 94 + assert len(sdata.shapes["nucleus_boundaries"]) == 94 + assert len(sdata.points["transcripts"].compute()) == 19479 + assert sdata.tables["table"].shape == (94, 5101) From 21abd1ef2291d9694819133ec5bea4aed23ff53e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:03:10 +0200 Subject: [PATCH 08/12] [pre-commit.ci] pre-commit autoupdate (#1163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/rbubley/mirrors-prettier: v3.9.4 → v3.9.6](https://github.com/rbubley/mirrors-prettier/compare/v3.9.4...v3.9.6) - [github.com/pre-commit/mirrors-mypy: v2.1.0 → v2.3.1](https://github.com/pre-commit/mirrors-mypy/compare/v2.1.0...v2.3.1) - [github.com/astral-sh/ruff-pre-commit: v0.15.20 → v0.16.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.15.20...v0.16.3) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 846497306..feb3aaa2d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,18 +9,18 @@ ci: skip: [] repos: - repo: https://github.com/rbubley/mirrors-prettier - rev: v3.9.4 + rev: v3.9.6 hooks: - id: prettier exclude: ^.github/workflows/test.yaml - repo: https://github.com/pre-commit/mirrors-mypy - rev: v2.1.0 + rev: v2.3.1 hooks: - id: mypy additional_dependencies: [numpy, types-requests] exclude: tests/|docs/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.20 + rev: v0.16.3 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] From 4d1020f6351515c5777a9f39590d17d4e92a92e1 Mon Sep 17 00:00:00 2001 From: Jan Gleixner Date: Wed, 19 Aug 2026 11:38:28 +0200 Subject: [PATCH 09/12] docs: add interoperability page (#1180) * docs: add interoperability page Closes #1176. Adds a short Interoperability page listing non-Python interfaces to the SpatialData on-disk format, mirroring the equivalent pages in anndata and mudata so the style is consistent across the three data structures. Also notes that the format builds on OME-NGFF and points at the design document for the current on-disk layout. * chore: fix precommit --------- Co-authored-by: Luca Marconato --- docs/index.md | 1 + docs/interoperability.md | 16 ++++++++++++++++ src/spatialdata/_io/io_zarr.py | 4 ++-- 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 docs/interoperability.md diff --git a/docs/index.md b/docs/index.md index 4b21c6ff7..afb23a4bf 100644 --- a/docs/index.md +++ b/docs/index.md @@ -110,6 +110,7 @@ tutorials/notebooks/notebooks.md tutorials/notebooks/datasets/README.md glossary.md design_doc.md +interoperability.md contributing.md changelog.md references.md diff --git a/docs/interoperability.md b/docs/interoperability.md new file mode 100644 index 000000000..331450808 --- /dev/null +++ b/docs/interoperability.md @@ -0,0 +1,16 @@ +# Interoperability + +The on-disk representation of SpatialData can be read from other languages. Here we list interfaces for working with SpatialData from your language of choice: + +## R + +- [spatialdataR](https://helenalc.github.io/spatialdataR/) provides an R implementation of the `SpatialData` object, with out-of-memory images and labels, `duckdb`-backed points and shapes, and tables represented as `SingleCellExperiment` objects. + +## JavaScript and TypeScript + +- [SpatialData.js](https://github.com/Taylor-CCB-Group/SpatialData.js) provides a TypeScript and JavaScript library for interfacing with SpatialData stores. +- [Vitessce](https://vitessce.io/docs/data-file-types/#spatialdatazarr) reads `spatialdata.zarr` stores directly and uses them for interactive visualization. + +## File format + +The SpatialData on-disk format builds on [OME-NGFF](https://ngff.openmicroscopy.org/latest/). See the [design document](design_doc.md) for details of the current on-disk layout. diff --git a/src/spatialdata/_io/io_zarr.py b/src/spatialdata/_io/io_zarr.py index 7ba440533..9324f8b7f 100644 --- a/src/spatialdata/_io/io_zarr.py +++ b/src/spatialdata/_io/io_zarr.py @@ -5,7 +5,7 @@ from collections.abc import Callable from json import JSONDecodeError from pathlib import Path -from typing import Any, Literal, cast +from typing import Any, Literal import zarr.storage from anndata import AnnData @@ -71,7 +71,7 @@ def _read_zarr_group_spatialdata_element( reader_format = get_raster_format_for_read(elem_group, sdata_version) element = read_func( elem_group_path, - cast(Literal["image", "labels"], element_type), + element_type, reader_format, ) elif element_type in ["shapes", "points", "tables"]: From 09b0bcba9b19caf24c5c7e8ac967f2d36a3abef6 Mon Sep 17 00:00:00 2001 From: Jan Gleixner Date: Wed, 19 Aug 2026 12:15:43 +0200 Subject: [PATCH 10/12] docs: fix napari package name and update spatialdataR package name on landing page (#1179) docs: fix package name and R repository link on landing page - `napari-spatialdata-repo` -> `napari-spatialdata`: the bullet label had leaked the Markdown link-reference name into the rendered package name. - The R implementation has been renamed from `SpatialData` to `spatialdataR`; update both the displayed name and the repository link, which previously only resolved via GitHub's rename redirect. --- docs/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index afb23a4bf..9ad5ca49d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,8 +12,8 @@ This page provides documentation on how to install, use, and extend the core `sp - `spatialdata-io`: load data from common spatial omics technologies into `spatialdata` ([repository][spatialdata-io-repo], [documentation][spatialdata-io-docs]). - `spatialdata-plot`: Static plotting library for `spatialdata` ([repository][spatialdata-plot-repo], [documentation][spatialdata-plot-docs]). -- `napari-spatialdata-repo`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]). -- `SpatialData` (R): R implementation of the SpatialData framework ([repository][spatialdata-r-repo]). +- `napari-spatialdata`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]). +- `spatialdataR`: R implementation of the SpatialData framework ([repository][spatialdata-r-repo]). - `SpatialData.js`: JavaScript/TypeScript implementation of the SpatialData framework ([repository][spatialdata-js-repo]). Please see our publication {cite}`marconatoSpatialDataOpenUniversal2024` for citation and to learn more. @@ -124,5 +124,5 @@ references.md [napari-spatialdata-docs]: https://spatialdata.scverse.org/projects/napari/en/stable/notebooks/spatialdata.html [spatialdata-io-docs]: https://spatialdata.scverse.org/projects/io/en/stable/ [spatialdata-plot-docs]: https://spatialdata.scverse.org/projects/plot/en/stable/api.html -[spatialdata-r-repo]: https://github.com/HelenaLC/SpatialData +[spatialdata-r-repo]: https://github.com/HelenaLC/spatialdataR [spatialdata-js-repo]: https://github.com/Taylor-CCB-Group/SpatialData.js From d66cde93ba1a8754c3773efc30e2a878ff42d501 Mon Sep 17 00:00:00 2001 From: Tomaz-Vieira Date: Wed, 19 Aug 2026 13:20:00 +0200 Subject: [PATCH 11/12] Fix numba JIT disable patch targeting wrong config attribute (#1181) Test suite tried to force-disable numba JIT when already imported by patching `NUMBA_DISABLE_JIT` on numba.core.config, but numba internally reads the env var into an attribute named `DISABLE_JIT` (no prefix). We should find a less hacky solution to this, but for now, running the tests sequentially (`pytest -x`) always pass. Co-authored-by: Claude Sonnet 5 Co-authored-by: LucaMarconato <2664412+LucaMarconato@users.noreply.github.com> --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 871b4280f..3aa3e0024 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,7 +8,7 @@ os.environ["NUMBA_DISABLE_JIT"] = "1" # If a pytest plugin already imported numba before this conftest ran, patch the cached config value too. if "numba.core.config" in sys.modules: - sys.modules["numba.core.config"].NUMBA_DISABLE_JIT = 1 + sys.modules["numba.core.config"].DISABLE_JIT = 1 import copy as _copy from collections.abc import Callable, Sequence From 79c30c519e457dbb363eafcc62bbad99c039925f Mon Sep 17 00:00:00 2001 From: Taimoor Qadir Date: Wed, 19 Aug 2026 13:37:48 +0200 Subject: [PATCH 12/12] =?UTF-8?q?docs:=20add=20docstrings=20to=20BaseTrans?= =?UTF-8?q?formation.inverse()=20and=20to=5Faffine=5Fma=E2=80=A6=20(#1169)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: add docstrings to BaseTransformation.inverse() and to_affine_matrix() Addresses issue #836 - adds missing docstrings to abstract methods in BaseTransformation * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * docs: small fixes to docstrings --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: LucaMarconato <2664412+LucaMarconato@users.noreply.github.com> --- .../transformations/transformations.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/spatialdata/transformations/transformations.py b/src/spatialdata/transformations/transformations.py index deace0ae1..b06e4319e 100644 --- a/src/spatialdata/transformations/transformations.py +++ b/src/spatialdata/transformations/transformations.py @@ -110,6 +110,15 @@ def _get_default_coordinate_system( @abstractmethod def inverse(self) -> BaseTransformation: + """ + Return the inverse of the transformation. + + Returns + ------- + BaseTransformation + A new transformation that is the inverse of this one, such that applying + both in sequence yields the identity transformation. + """ pass # @abstractmethod @@ -118,6 +127,22 @@ def inverse(self) -> BaseTransformation: @abstractmethod def to_affine_matrix(self, input_axes: tuple[ValidAxis_t, ...], output_axes: tuple[ValidAxis_t, ...]) -> ArrayLike: + """ + Return the affine matrix representation of the transformation. + + Parameters + ---------- + input_axes + The axes of the input coordinate system, e.g. ``("x", "y")`` or ``("c", "z", "y", "x")``. + output_axes + The axes of the output coordinate system. + + Returns + ------- + ArrayLike + A homogeneous affine matrix of shape ``(len(output_axes) + 1, len(input_axes) + 1)``. + The last row is always ``[0, 0, ..., 1]`` (homogeneity). + """ pass def to_affine(self, input_axes: tuple[ValidAxis_t, ...], output_axes: tuple[ValidAxis_t, ...]) -> Affine: