From 1713fd899b6c76edd64f127bc3141e4287fdd716 Mon Sep 17 00:00:00 2001 From: Jonathan Amponsah <82057176+mgalore@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:08:49 +0000 Subject: [PATCH 1/2] docs: clarify Zarr v3 codec pipeline roles --- changes/4159.doc.md | 1 + docs/user-guide/arrays.md | 37 +++++++++++++++++++++++++++++++++++- src/zarr/api/asynchronous.py | 8 ++++++-- src/zarr/api/synchronous.py | 24 ++++++++++++++++------- 4 files changed, 60 insertions(+), 10 deletions(-) create mode 100644 changes/4159.doc.md diff --git a/changes/4159.doc.md b/changes/4159.doc.md new file mode 100644 index 0000000000..32738d3635 --- /dev/null +++ b/changes/4159.doc.md @@ -0,0 +1 @@ +Clarify how Zarr format 3 codec pipeline roles map to `filters`, `serializer`, and `compressors` when creating arrays. diff --git a/docs/user-guide/arrays.md b/docs/user-guide/arrays.md index a192845f9e..036c7fea24 100644 --- a/docs/user-guide/arrays.md +++ b/docs/user-guide/arrays.md @@ -195,13 +195,48 @@ arr_f = arr.with_config({"order": "F"}) print(arr_f.config) ``` +## Zarr format 3 codec pipeline + +Zarr format 3 stores a single ordered `codecs` pipeline in array metadata, but +Zarr-Python's array creation functions expose that pipeline through three +role-specific parameters: + +- `filters`: [`zarr.abc.codec.ArrayArrayCodec`][] instances. These transform chunk + arrays into chunk arrays before serialization. +- `serializer`: one [`zarr.abc.codec.ArrayBytesCodec`][] instance. This transforms + a chunk array into bytes. Every Zarr format 3 array needs exactly one + array-to-bytes codec, either supplied explicitly or chosen by default. +- `compressors`: [`zarr.abc.codec.BytesBytesCodec`][] instances. These transform + bytes into bytes after serialization. + +The `compressors` parameter is only for bytes-to-bytes codecs. If a codec is an +`ArrayBytesCodec`, pass it with `serializer`, not `compressors`. For example, the +built-in [`zarr.codecs.BytesCodec`][] can be supplied explicitly as the serializer: + +```python exec="true" session="arrays" source="above" result="ansi" +serializer = zarr.codecs.BytesCodec(endian="little") +z_explicit_serializer = zarr.create_array( + store="data/example-explicit-serializer.zarr", + shape=(100,), + chunks=(10,), + dtype="int32", + serializer=serializer, + compressors=None, +) +print(z_explicit_serializer.serializer) +print(f"Compressors: {z_explicit_serializer.compressors}") +``` + +The same rule applies to third-party Zarr format 3 codecs: if the codec is +documented as an `ArrayBytesCodec`, provide an instance as `serializer=...`. + ## Compressors A number of different compressors can be used with Zarr. Zarr includes Blosc, Zstandard and Gzip compressors. Additional compressors are available through a separate package called [NumCodecs](https://numcodecs.readthedocs.io/en/stable/) which provides various compressor libraries including LZ4, Zlib, BZ2 and LZMA. -Different compressors can be provided via the `compressors` keyword +Different bytes-to-bytes compressors can be provided via the `compressors` keyword argument accepted by all array creation functions. For example: ```python exec="true" session="arrays" source="above" result="ansi" diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 3bdc254ea5..f4af245ece 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -1020,8 +1020,12 @@ async def create( codecs : Sequence of Codecs or dicts, optional An iterable of Codec or dict serializations of Codecs. Zarr V3 only. - The elements of `codecs` specify the transformation from array values to stored bytes. - Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. + The elements of ``codecs`` specify the transformation from array values to stored bytes. + Zarr format 3 only. Zarr format 2 arrays should use ``filters`` and ``compressor`` instead. + In order, a Zarr format 3 pipeline contains zero or more + [`zarr.abc.codec.ArrayArrayCodec`][] filters, exactly one + [`zarr.abc.codec.ArrayBytesCodec`][] serializer, and zero or more + [`zarr.abc.codec.BytesBytesCodec`][] compressors. If no codecs are provided, default codecs will be used based on the data type of the array. For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`; diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 11afb4a85c..ac08d71a4e 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -758,8 +758,12 @@ def create( codecs : Sequence of Codecs or dicts, optional An iterable of Codec or dict serializations of Codecs. Zarr V3 only. - The elements of `codecs` specify the transformation from array values to stored bytes. - Zarr format 3 only. Zarr format 2 arrays should use `filters` and `compressor` instead. + The elements of ``codecs`` specify the transformation from array values to stored bytes. + Zarr format 3 only. Zarr format 2 arrays should use ``filters`` and ``compressor`` instead. + In order, a Zarr format 3 pipeline contains zero or more + [`zarr.abc.codec.ArrayArrayCodec`][] filters, exactly one + [`zarr.abc.codec.ArrayBytesCodec`][] serializer, and zero or more + [`zarr.abc.codec.BytesBytesCodec`][] compressors. If no codecs are provided, default codecs will be used based on the data type of the array. For most data types, the default codecs are the tuple `(BytesCodec(), ZstdCodec())`; @@ -892,9 +896,13 @@ def create_array( filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` + returns another bytestream. These values must be instances of + [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of + [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided + for Zarr format 3. Codecs that take an array and return bytes are serializers + and must be supplied with ``serializer`` instead. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` in [`zarr.config`][zarr.config]. Use `None` to omit default compressors. @@ -906,8 +914,10 @@ def create_array( serializer : dict[str, JSON] | ArrayBytesCodec, optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` + Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be + supplied here, not with ``compressors``. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` in [`zarr.config`][zarr.config]. fill_value : Any, optional Fill value for the array. From a5f91c4ce5755bcf5c1d4bd603e3ef97263434d8 Mon Sep 17 00:00:00 2001 From: Jonathan Amponsah <82057176+mgalore@users.noreply.github.com> Date: Mon, 20 Jul 2026 18:07:25 +0000 Subject: [PATCH 2/2] docs: align async codec parameter docs --- src/zarr/core/array.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 903f7b8f26..32244be0f8 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -4643,9 +4643,13 @@ async def create_array( filters are applied (if any are specified) and the data is serialized into bytes. For Zarr format 3, a "compressor" is a codec that takes a bytestream, and - returns another bytestream. Multiple compressors may be provided for Zarr format 3. - If no `compressors` are provided, a default set of compressors will be used. - These defaults can be changed by modifying the value of `array.v3_default_compressors` + returns another bytestream. These values must be instances of + [`zarr.abc.codec.BytesBytesCodec`][], or dict representations of + [`zarr.abc.codec.BytesBytesCodec`][]. Multiple compressors may be provided + for Zarr format 3. Codecs that take an array and return bytes are serializers + and must be supplied with ``serializer`` instead. + If no ``compressors`` are provided, a default set of compressors will be used. + These defaults can be changed by modifying the value of ``array.v3_default_compressors`` in [`zarr.config`][zarr.config]. Use `None` to omit default compressors. @@ -4657,8 +4661,10 @@ async def create_array( serializer : dict[str, JSON] | ArrayBytesCodec, optional Array-to-bytes codec to use for encoding the array data. Zarr format 3 only. Zarr format 2 arrays use implicit array-to-bytes conversion. - If no `serializer` is provided, a default serializer will be used. - These defaults can be changed by modifying the value of `array.v3_default_serializer` + Codecs that are instances of [`zarr.abc.codec.ArrayBytesCodec`][] must be + supplied here, not with ``compressors``. + If no ``serializer`` is provided, a default serializer will be used. + These defaults can be changed by modifying the value of ``array.v3_default_serializer`` in [`zarr.config`][zarr.config]. fill_value : Any, optional Fill value for the array.