fix: point table validation to public sanitizer

.github/workflows/test.yaml

@@ -59,7 +59,7 @@ jobs:
                   PLATFORM: ${{ matrix.os }}
                   DISPLAY: :42
               run: |
-                  uv run pytest --cov --color=yes --cov-report=xml -n auto --dist worksteal
+                  uv run pytest --run-network --cov --color=yes --cov-report=xml -n auto --dist worksteal
             - name: Upload coverage to Codecov
               uses: codecov/codecov-action@v6
               with:

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "docs/tutorials/notebooks"]
 	path = docs/tutorials/notebooks
-	url = https://github.com/scverse/spatialdata-notebooks
+	url = https://github.com/scverse/spatialdata-tutorials

.pre-commit-config.yaml

@@ -9,18 +9,18 @@ ci:
     skip: []
 repos:
     - repo: https://github.com/rbubley/mirrors-prettier
-      rev: v3.9.4
+      rev: v3.9.6
       hooks:
           - id: prettier
             exclude: ^.github/workflows/test.yaml
     - repo: https://github.com/pre-commit/mirrors-mypy
-      rev: v2.1.0
+      rev: v2.3.1
       hooks:
           - id: mypy
             additional_dependencies: [numpy, types-requests]
             exclude: tests/|docs/
     - repo: https://github.com/astral-sh/ruff-pre-commit
-      rev: v0.15.20
+      rev: v0.16.3
       hooks:
           - id: ruff
             args: [--fix, --exit-non-zero-on-fix]

docs/api/datasets.md

@@ -7,5 +7,6 @@ Convenience small datasets
 
 .. autofunction:: blobs
 .. autofunction:: blobs_annotating_element
+.. autofunction:: cells
 .. autofunction:: raccoon
 ```

docs/contributing.md

@@ -13,8 +13,7 @@ In addition to the packages needed to _use_ this package, you need additional py
 the documentation_. It's easy to install them using `pip`:
 
 ```bash
-cd spatialdata-io
-pip install -e ".[dev,test,doc]"
+pip install -e . --group dev --group test --group docs
 ```
 
 ## Code-style
@@ -51,7 +50,7 @@ and [prettier][prettier-editors].
 ## Writing tests
 
 ```{note}
-Remember to first install the package with `pip install '-e[dev,test]'`
+Remember to first install the package with `pip install -e . --group dev --group test`
 ```
 
 This package uses [pytest][] for automated testing. Please [write tests][scanpy-test-docs] for every function added to the package.

docs/index.md

@@ -12,8 +12,8 @@ This page provides documentation on how to install, use, and extend the core `sp
 
 - `spatialdata-io`: load data from common spatial omics technologies into `spatialdata` ([repository][spatialdata-io-repo], [documentation][spatialdata-io-docs]).
 - `spatialdata-plot`: Static plotting library for `spatialdata` ([repository][spatialdata-plot-repo], [documentation][spatialdata-plot-docs]).
-- `napari-spatialdata-repo`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
-- `SpatialData` (R): R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
+- `napari-spatialdata`: napari plugin for interactive exploration and annotation of `spatialdata` ([repository][napari-spatialdata-repo], [documentation][napari-spatialdata-docs]).
+- `spatialdataR`: R implementation of the SpatialData framework ([repository][spatialdata-r-repo]).
 - `SpatialData.js`: JavaScript/TypeScript implementation of the SpatialData framework ([repository][spatialdata-js-repo]).
 
 Please see our publication {cite}`marconatoSpatialDataOpenUniversal2024` for citation and to learn more.
@@ -110,6 +110,7 @@ tutorials/notebooks/notebooks.md
 tutorials/notebooks/datasets/README.md
 glossary.md
 design_doc.md
+interoperability.md
 contributing.md
 changelog.md
 references.md
@@ -123,5 +124,5 @@ references.md
 [napari-spatialdata-docs]: https://spatialdata.scverse.org/projects/napari/en/stable/notebooks/spatialdata.html
 [spatialdata-io-docs]: https://spatialdata.scverse.org/projects/io/en/stable/
 [spatialdata-plot-docs]: https://spatialdata.scverse.org/projects/plot/en/stable/api.html
-[spatialdata-r-repo]: https://github.com/HelenaLC/SpatialData
+[spatialdata-r-repo]: https://github.com/HelenaLC/spatialdataR
 [spatialdata-js-repo]: https://github.com/Taylor-CCB-Group/SpatialData.js

docs/interoperability.md

@@ -0,0 +1,16 @@
+# Interoperability
+
+The on-disk representation of SpatialData can be read from other languages. Here we list interfaces for working with SpatialData from your language of choice:
+
+## R
+
+- [spatialdataR](https://helenalc.github.io/spatialdataR/) provides an R implementation of the `SpatialData` object, with out-of-memory images and labels, `duckdb`-backed points and shapes, and tables represented as `SingleCellExperiment` objects.
+
+## JavaScript and TypeScript
+
+- [SpatialData.js](https://github.com/Taylor-CCB-Group/SpatialData.js) provides a TypeScript and JavaScript library for interfacing with SpatialData stores.
+- [Vitessce](https://vitessce.io/docs/data-file-types/#spatialdatazarr) reads `spatialdata.zarr` stores directly and uses them for interactive visualization.
+
+## File format
+
+The SpatialData on-disk format builds on [OME-NGFF](https://ngff.openmicroscopy.org/latest/). See the [design document](design_doc.md) for details of the current on-disk layout.

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
     "spatial_image>=1.2.3",
     "scikit-image",
     "scipy!=1.17.0",
+    "scverse-misc[datasets]>=0.1.0",
     "typing_extensions>=4.8.0",
     "universal_pathlib>=0.2.6",
     "xarray>=2024.10.0",
@@ -108,6 +109,7 @@ addopts = [
 # These are all markers coming from xarray, dask or anndata. Added here to silence warnings.
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+    "network: marks tests that require network access; skipped by default, run with '--run-network'",
     "gpu: run test on GPU using CuPY.",
     "array_api: used by anndata.tests.helpers, not us",
     "skip_with_pyarrow_strings: skipwhen pyarrow string conversion is turned on",
@@ -244,3 +246,38 @@ memray-flame = "memray flamegraph --temporal"
 
 [tool.pixi.environments]
 profiling = { features = ["profiling"], solve-group = "default" }
+
+[tool.hatch.envs.test]
+dependency-groups = ["test"]
+
+[tool.hatch.envs.test-anndata-pandas]
+template = "test"
+extra-dependencies = ["zarr>=3"]
+scripts.test = ["pip list|grep anndata && pip list|grep pandas && pytest {args}"]
+scripts.test-readwrite = ["pip list|grep anndata && pip list|grep pandas && pytest tests/io/test_readwrite.py"]
+scripts.test-all = ["pip list|grep anndata && pip list|grep pandas && pytest ."]
+
+[[tool.hatch.envs.test-anndata-pandas.matrix]]
+anndata-pandas = [
+    "0.13-2",
+    "0.13-3",
+    "0.12-2"
+    # no "0.12-3": support for pandas>=3 is available only in anndata>=0.13
+]
+
+[tool.hatch.envs.test-anndata-pandas.overrides]
+matrix.anndata-pandas.extra-dependencies = [
+    # every option where the if-condition is True gets included
+
+    # anndata 0.13
+    {value="anndata~=0.13", if = ["0.13-2", "0.13-3"]},
+
+    # anndata 0.12
+    {value="anndata>=0.12,<0.13", if = ["0.12-2"]},
+
+    # pandas 2
+    {value="pandas>=2.3,<3", if = ["0.13-2", "0.12-2"]},
+
+    # pandas 3
+    {value="pandas~=3.0", if = ["0.13-3"]},
+]

src/spatialdata/_core/spatialdata.py

@@ -107,7 +107,7 @@ class SpatialData:
         - the table are stored as :class:`anndata.AnnData` objects,  with the spatial coordinates stored in the obsm
             slot.
 
-    The table can annotate regions (shapesor labels) and can be used to store additional information.
+    The table can annotate regions (shapes or labels) and can be used to store additional information.
     Points are not regions but 0-dimensional locations. They can't be annotated by a table, but they can store
     annotation directly.
     """
@@ -1114,6 +1114,7 @@ def write(
         sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
         shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
         raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+        convert_table_strings_to_categoricals: bool = False,
     ) -> None:
         """
         Write the `SpatialData` object to a Zarr store.
@@ -1166,6 +1167,9 @@ def write(
             compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
             supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
             ordered for more efficient compression.
+        convert_table_strings_to_categoricals
+            If True, convert string columns of all tables to categoricals before writing.
+            Note that this will have a side effect of modifying string columns into categoricals in place.
         """
         from spatialdata._io._utils import _resolve_zarr_store, _validate_compressor_args
         from spatialdata._io.format import _parse_formats
@@ -1194,6 +1198,7 @@ def write(
                 parsed_formats=parsed,
                 shapes_geometry_encoding=shapes_geometry_encoding,
                 raster_compressor=raster_compressor,
+                convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
             )
 
         if self.path != file_path and update_sdata_path:
@@ -1212,6 +1217,7 @@ def _write_element(
         parsed_formats: dict[str, SpatialDataFormatType] | None = None,
         shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
         raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+        convert_table_strings_to_categoricals: bool = False,
     ) -> None:
         from spatialdata._io.io_zarr import _get_groups_for_element
 
@@ -1279,6 +1285,7 @@ def _write_element(
                 group=element_type_group,
                 name=element_name,
                 element_format=parsed_formats["tables"],
+                convert_strings_to_categoricals=convert_table_strings_to_categoricals,
             )
         else:
             raise ValueError(f"Unknown element type: {element_type}")
@@ -1290,6 +1297,7 @@ def write_element(
         sdata_formats: SpatialDataFormatType | list[SpatialDataFormatType] | None = None,
         shapes_geometry_encoding: Literal["WKB", "geoarrow"] | None = None,
         raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
+        convert_table_strings_to_categoricals: bool = False,
     ) -> None:
         """
         Write a single element, or a list of elements, to the Zarr store used for backing.
@@ -1308,11 +1316,14 @@ def write_element(
         shapes_geometry_encoding
             Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet`
             for details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
-         raster_compressor
+        raster_compressor
             A lenght-1 dictionary with as key the type of compression to use for images and labels and as value the
             compression level which should be inclusive between 0 and 9. For compression, `lz4` and `zstd` are
             supported. If not specified, the compression will be `lz4` with compression level 5. Bytes are automatically
             ordered for more efficient compression.
+        convert_table_strings_to_categoricals
+            If True, and if element to be written is a table, convert string columns to categoricals before writing.
+            Note that this will have a side effect of modifying string columns into categoricals in place.
 
         Notes
         -----
@@ -1332,6 +1343,7 @@ def write_element(
                     sdata_formats=sdata_formats,
                     shapes_geometry_encoding=shapes_geometry_encoding,
                     raster_compressor=raster_compressor,
+                    convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
                 )
             return
 
@@ -1368,6 +1380,7 @@ def write_element(
             parsed_formats=parsed_formats,
             shapes_geometry_encoding=shapes_geometry_encoding,
             raster_compressor=raster_compressor,
+            convert_table_strings_to_categoricals=convert_table_strings_to_categoricals,
         )
         # After every write, metadata should be consolidated, otherwise this can lead to IO problems like when deleting.
         if self.has_consolidated_metadata():

src/spatialdata/_core/validation.py

@@ -388,7 +388,7 @@ def __exit__(
         # Exceptions were collected that we want to raise as a combined validation error.
         if self._collector.errors:
             raise ValidationError(
-                title=self._message + "\nTo fix, run `spatialdata.utils.sanitize_table(adata)`.",
+                title=self._message + "\nTo fix, run `spatialdata.sanitize_table(adata)`.",
                 errors=self._collector.errors,
             )
         return True

src/spatialdata/_io/exceptions.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from ome_zarr.format import Format
+
+
+class FormatVersionUnknownError(ValueError):
+    """Exception raised when an unknown element format is encountered."""
+
+    def __init__(self, element_type: str, version_encountered: Format):
+        self.element_type = element_type
+        self.version_encountered = version_encountered
+        self.message = (
+            f"Encountered unknown element format version "
+            f"`{self.version_encountered}` for element of type `{self.element_type}`"
+        )
+        super().__init__(self.message)
+
+
+class WritingToZarrV2DeprecationWarning(DeprecationWarning):
+    """Warning raised when writing to zarr v2 format."""
+
+    message = (
+        "Writing to zarr v2 format is currently deprecated in spatialdata "
+        "and will be removed in a future version. "
+        "Please consider writing to zarr v3."
+    )

src/spatialdata/_io/io_points.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 
 import zarr
@@ -12,6 +13,7 @@
     _write_metadata,
     overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import CurrentPointsFormat, PointsFormats, _parse_version
 from spatialdata.models import get_axes_names
 from spatialdata.transformations._utils import (
@@ -65,6 +67,10 @@ def write_points(
     element_format
         The format of the points element used to store it.
     """
+    if element_format.zarr_format == 2:
+        warnings.warn(
+            message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+        )
     axes = get_axes_names(points)
     transformations = _get_transformations(points)
     assert transformations is not None  # mypy: validate_element() in _write_element guarantees this

src/spatialdata/_io/io_raster.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from collections.abc import Sequence
 from pathlib import Path
 from typing import Any, Literal, TypeGuard, cast
@@ -23,6 +24,7 @@
     overwrite_channel_names,
     overwrite_coordinate_transformations_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
     CurrentRasterFormat,
     RasterFormatType,
@@ -581,6 +583,11 @@ def write_image(
     raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
     **metadata: str | JSONDict | list[JSONDict],
 ) -> None:
+    if element_format.zarr_format == 2:
+        warnings.warn(
+            message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+        )
+
     _write_raster(
         raster_type="image",
         raster_data=image,
@@ -603,6 +610,11 @@ def write_labels(
     raster_compressor: dict[Literal["lz4", "zstd"], int] | None = None,
     **metadata: JSONDict,
 ) -> None:
+    if element_format.zarr_format == 2:
+        warnings.warn(
+            message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+        )
+
     _write_raster(
         raster_type="labels",
         raster_data=labels,

src/spatialdata/_io/io_shapes.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from pathlib import Path
 from typing import Any, Literal
 
@@ -15,6 +16,7 @@
     _write_metadata,
     overwrite_coordinate_transformations_non_raster,
 )
+from spatialdata._io.exceptions import WritingToZarrV2DeprecationWarning
 from spatialdata._io.format import (
     CurrentShapesFormat,
     ShapesFormats,
@@ -93,6 +95,11 @@ def write_shapes(
         Whether to use the WKB or geoarrow encoding for GeoParquet. See :meth:`geopandas.GeoDataFrame.to_parquet` for
         details. If None, uses the value from :attr:`spatialdata.settings.shapes_geometry_encoding`.
     """
+    if element_format.zarr_format == 2:
+        warnings.warn(
+            message=WritingToZarrV2DeprecationWarning.message, category=WritingToZarrV2DeprecationWarning, stacklevel=2
+        )
+
     from spatialdata.config import settings
 
     if geometry_encoding is None: