diff --git a/changes/457.bugfix.md b/changes/457.bugfix.md new file mode 100644 index 0000000000..b4a5a4642f --- /dev/null +++ b/changes/457.bugfix.md @@ -0,0 +1 @@ +`zarr.codecs.numcodecs.Blosc` now compresses with the item size of the chunk's data type as its typesize when `typesize` is not configured, as numcodecs did for Zarr V2 arrays. Previously it always used a typesize of 1, so the default byte shuffle had no effect. Existing data reads unchanged, because each chunk records its typesize. diff --git a/src/zarr/codecs/numcodecs/_codecs.py b/src/zarr/codecs/numcodecs/_codecs.py index 5681c16c54..d58241d54b 100644 --- a/src/zarr/codecs/numcodecs/_codecs.py +++ b/src/zarr/codecs/numcodecs/_codecs.py @@ -41,6 +41,7 @@ from zarr.abc.metadata import Metadata from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON, parse_named_configuration, product +from zarr.core.dtype.common import HasItemSize from zarr.dtype import UInt8, ZDType, parse_dtype from zarr.registry import get_numcodec @@ -203,7 +204,18 @@ async def _decode_single(self, chunk_data: Buffer, chunk_spec: ArraySpec) -> NDB # bytes-to-bytes codecs class Blosc(_NumcodecsBytesBytesCodec, codec_name="blosc"): - pass + def _encode_sync(self, chunk_data: Buffer, chunk_spec: ArraySpec) -> Buffer: + if "typesize" in self.codec_config: + return super()._encode_sync(chunk_data, chunk_spec) + # numcodecs.Blosc takes its typesize from the item size of the array it encodes, + # as it did for Zarr V2 arrays. Passing raw bytes would always mean a typesize + # of 1, which makes byte shuffling a no-op, so view the bytes with the item size + # of the chunk's data type when the bytes divide into whole items. + data = chunk_data.as_numpy_array() + item_size = chunk_spec.dtype.item_size if isinstance(chunk_spec.dtype, HasItemSize) else 1 + if item_size > 1 and data.nbytes % item_size == 0: + data = data.view(f"V{item_size}") + return chunk_spec.prototype.buffer.from_bytes(self._codec.encode(data)) class LZ4(_NumcodecsBytesBytesCodec, codec_name="lz4"): diff --git a/tests/test_codecs/test_numcodecs.py b/tests/test_codecs/test_numcodecs.py index d41ba3073d..ae265e4f93 100644 --- a/tests/test_codecs/test_numcodecs.py +++ b/tests/test_codecs/test_numcodecs.py @@ -4,9 +4,11 @@ import pickle from typing import TYPE_CHECKING, Any +import numcodecs import numpy as np import pytest from numcodecs import GZip +from packaging.version import Version from zarr import config, create_array, open_array from zarr.abc.numcodec import _is_numcodec, _is_numcodec_cls @@ -124,6 +126,50 @@ def test_generic_compressor(codec_class: type[_numcodecs._NumcodecsBytesBytesCod np.testing.assert_array_equal(data, a[:, :]) +@pytest.mark.parametrize( + ("dtype", "compressors", "typesize"), + [ + ("uint8", [_numcodecs.Blosc()], 1), + ("uint16", [_numcodecs.Blosc()], 2), + ("float64", [_numcodecs.Blosc()], 8), + # a preceding checksum leaves bytes that do not divide into float64 items + ("float64", [_numcodecs.CRC32(location="end"), _numcodecs.Blosc()], 1), + pytest.param( + "uint16", + [_numcodecs.Blosc(typesize=4)], + 4, + marks=pytest.mark.skipif( + Version(numcodecs.__version__) < Version("0.16.0"), + reason="numcodecs.Blosc accepts typesize from numcodecs 0.16", + ), + ), + ], + ids=["uint8", "uint16", "float64", "after-crc32", "configured"], +) +def test_blosc_typesize( + dtype: str, compressors: list[_numcodecs._NumcodecsBytesBytesCodec], typesize: int +) -> None: + """ + numcodecs.blosc compresses with the item size of the chunk's data type as its + typesize unless the configuration sets one, and the data round-trips. + """ + data = np.arange(256, dtype=dtype).reshape((16, 16)) + store: dict[str, Any] = {} + a = create_array( + store, + shape=data.shape, + chunks=(16, 16), + dtype=data.dtype, + fill_value=0, + compressors=compressors, + ) + a[:, :] = data + chunk = store["c/0/0"].to_bytes() + # byte 3 of a blosc1 chunk header holds the typesize + assert chunk[3] == typesize + np.testing.assert_array_equal(open_array(store, mode="r")[:, :], data) + + @pytest.mark.parametrize( ("codec_class", "codec_config"), [