diff --git a/changes/4159.doc.md b/changes/4159.doc.md new file mode 100644 index 0000000000..32738d3635 --- /dev/null +++ b/changes/4159.doc.md @@ -0,0 +1 @@ +Clarify how Zarr format 3 codec pipeline roles map to `filters`, `serializer`, and `compressors` when creating arrays. diff --git a/docs/user-guide/arrays.md b/docs/user-guide/arrays.md index a192845f9e..036c7fea24 100644 --- a/docs/user-guide/arrays.md +++ b/docs/user-guide/arrays.md @@ -195,13 +195,48 @@ arr_f = arr.with_config({"order": "F"}) print(arr_f.config) ``` +## Zarr format 3 codec pipeline + +Zarr format 3 stores a single ordered `codecs` pipeline in array metadata, but +Zarr-Python's array creation functions expose that pipeline through three +role-specific parameters: + +- `filters`: [`zarr.abc.codec.ArrayArrayCodec`][] instances. These transform chunk + arrays into chunk arrays before serialization. +- `serializer`: one [`zarr.abc.codec.ArrayBytesCodec`][] instance. This transforms + a chunk array into bytes. Every Zarr format 3 array needs exactly one + array-to-bytes codec, either supplied explicitly or chosen by default. +- `compressors`: [`zarr.abc.codec.BytesBytesCodec`][] instances. These transform + bytes into bytes after serialization. + +The `compressors` parameter is only for bytes-to-bytes codecs. If a codec is an +`ArrayBytesCodec`, pass it with `serializer`, not `compressors`. For example, the +built-in [`zarr.codecs.BytesCodec`][] can be supplied explicitly as the serializer: + +```python exec="true" session="arrays" source="above" result="ansi" +serializer = zarr.codecs.BytesCodec(endian="little") +z_explicit_serializer = zarr.create_array( + store="data/example-explicit-serializer.zarr", + shape=(100,), + chunks=(10,), + dtype="int32", + serializer=serializer, + compressors=None, +) +print(z_explicit_serializer.serializer) +print(f"Compressors: {z_explicit_serializer.compressors}") +``` + +The same rule applies to third-party Zarr format 3 codecs: if the codec is +documented as an `ArrayBytesCodec`, provide an instance as `serializer=...`. + ## Compressors A number of different compressors can be used with Zarr. Zarr includes Blosc, Zstandard and Gzip compressors. Additional compressors are available through a separate package called [NumCodecs](https://numcodecs.readthedocs.io/en/stable/) which provides various compressor libraries including LZ4, Zlib, BZ2 and LZMA. -Different compressors can be provided via the `compressors` keyword +Different bytes-to-bytes compressors can be provided via the `compressors` keyword argument accepted by all array creation functions. For example: ```python exec="true" session="arrays" source="above" result="ansi" diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 3bdc254ea5..f4af245ece 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -1020,8 +1020,12 @@ async def create( codecs : Sequence of Codecs or dicts, optional An iterable of Codec or dict serializations of Codecs. Zarr V3 only. - The elements of `codecs` specify the transformation from array values to stored bytes. - Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. + The elements of ``codecs`` specify the transformation from array values to stored bytes. + Zarr format 3 only. Zarr format 2 arrays should use ``filters`` and ``compressor`` instead. + In order, a Zarr format 3 pipeline contains zero or more + [`zarr.abc.codec.ArrayArrayCodec`][] filters, exactly one + [`zarr.abc.codec.ArrayBytesCodec`][] serializer, and zero or more + [`zarr.abc.codec.BytesBytesCodec`][] compressors. If no codecs are provided, default codecs will be used based on the data type of the array. For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`; diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 11afb4a85c..ac08d71a4e 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -758,8 +758,12 @@ def create( codecs : Sequence of Codecs or dicts, optional An iterable of Codec or dict serializations of Codecs. Zarr V3 only. - The elements of `codecs` specify the transformation from array values to stored bytes. - Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. + The elements of ``codecs`` specify the transformation from array values to stored bytes. + Zarr format 3 only. Zarr format 2 arrays should use ``filters`` and ``compressor`` instead. + In order, a Zarr format 3 pipeline contains zero or more + [`zarr.abc.codec.ArrayArrayCodec`][] filters, exactly one + [`zarr.abc.codec.ArrayBytesCodec`][] serializer, and zero or more + [`zarr.abc.codec.BytesBytesCodec`][] compressors. If no codecs are provided, default codecs will be used based on the data type of the array. For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`; @@ -892,9 +896,13 @@ def create_array( filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` + returns another bytestream. These values must be instances of + [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of + [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided + for Zarr format 3. Codecs that take an array and return bytes are serializers + and must be supplied with ``serializer`` instead. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` in [`zarr.config`][zarr.config]. Use `None` to omit default compressors. @@ -906,8 +914,10 @@ def create_array( serializer : dict[str, JSON] | ArrayBytesCodec, optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` + Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be + supplied here, not with ``compressors``. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` in [`zarr.config`][zarr.config]. fill_value : Any, optional Fill value for the array. diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 903f7b8f26..32244be0f8 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -4643,9 +4643,13 @@ async def create_array( filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` + returns another bytestream. These values must be instances of + [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of + [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided + for Zarr format 3. Codecs that take an array and return bytes are serializers + and must be supplied with ``serializer`` instead. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` in [`zarr.config`][zarr.config]. Use `None` to omit default compressors. @@ -4657,8 +4661,10 @@ async def create_array( serializer : dict[str, JSON] | ArrayBytesCodec, optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` + Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be + supplied here, not with ``compressors``. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` in [`zarr.config`][zarr.config]. fill_value : Any, optional Fill value for the array.