Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/457.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`zarr.codecs.numcodecs.Blosc` now compresses with the item size of the chunk's data type as its typesize when `typesize` is not configured, as numcodecs did for Zarr V2 arrays. Previously it always used a typesize of 1, so the default byte shuffle had no effect. Existing data reads unchanged, because each chunk records its typesize.
14 changes: 13 additions & 1 deletion src/zarr/codecs/numcodecs/_codecs.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
from zarr.abc.metadata import Metadata
from zarr.core.buffer.cpu import as_numpy_array_wrapper
from zarr.core.common import JSON, parse_named_configuration, product
from zarr.core.dtype.common import HasItemSize
from zarr.dtype import UInt8, ZDType, parse_dtype
from zarr.registry import get_numcodec

Expand Down Expand Up @@ -203,7 +204,18 @@ async def _decode_single(self, chunk_data: Buffer, chunk_spec: ArraySpec) -> NDB

# bytes-to-bytes codecs
class Blosc(_NumcodecsBytesBytesCodec, codec_name="blosc"):
pass
def _encode_sync(self, chunk_data: Buffer, chunk_spec: ArraySpec) -> Buffer:
if "typesize" in self.codec_config:
return super()._encode_sync(chunk_data, chunk_spec)
# numcodecs.Blosc takes its typesize from the item size of the array it encodes,
# as it did for Zarr V2 arrays. Passing raw bytes would always mean a typesize
# of 1, which makes byte shuffling a no-op, so view the bytes with the item size
# of the chunk's data type when the bytes divide into whole items.
data = chunk_data.as_numpy_array()
item_size = chunk_spec.dtype.item_size if isinstance(chunk_spec.dtype, HasItemSize) else 1
if item_size > 1 and data.nbytes % item_size == 0:
data = data.view(f"V{item_size}")
return chunk_spec.prototype.buffer.from_bytes(self._codec.encode(data))


class LZ4(_NumcodecsBytesBytesCodec, codec_name="lz4"):
Expand Down
46 changes: 46 additions & 0 deletions tests/test_codecs/test_numcodecs.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@
import pickle
from typing import TYPE_CHECKING, Any

import numcodecs
import numpy as np
import pytest
from numcodecs import GZip
from packaging.version import Version

from zarr import config, create_array, open_array
from zarr.abc.numcodec import _is_numcodec, _is_numcodec_cls
Expand Down Expand Up @@ -124,6 +126,50 @@ def test_generic_compressor(codec_class: type[_numcodecs._NumcodecsBytesBytesCod
np.testing.assert_array_equal(data, a[:, :])


@pytest.mark.parametrize(
("dtype", "compressors", "typesize"),
[
("uint8", [_numcodecs.Blosc()], 1),
("uint16", [_numcodecs.Blosc()], 2),
("float64", [_numcodecs.Blosc()], 8),
# a preceding checksum leaves bytes that do not divide into float64 items
("float64", [_numcodecs.CRC32(location="end"), _numcodecs.Blosc()], 1),
pytest.param(
"uint16",
[_numcodecs.Blosc(typesize=4)],
4,
marks=pytest.mark.skipif(
Version(numcodecs.__version__) < Version("0.16.0"),
reason="numcodecs.Blosc accepts typesize from numcodecs 0.16",
),
),
],
ids=["uint8", "uint16", "float64", "after-crc32", "configured"],
)
def test_blosc_typesize(
dtype: str, compressors: list[_numcodecs._NumcodecsBytesBytesCodec], typesize: int
) -> None:
"""
numcodecs.blosc compresses with the item size of the chunk's data type as its
typesize unless the configuration sets one, and the data round-trips.
"""
data = np.arange(256, dtype=dtype).reshape((16, 16))
store: dict[str, Any] = {}
a = create_array(
store,
shape=data.shape,
chunks=(16, 16),
dtype=data.dtype,
fill_value=0,
compressors=compressors,
)
a[:, :] = data
chunk = store["c/0/0"].to_bytes()
# byte 3 of a blosc1 chunk header holds the typesize
assert chunk[3] == typesize
np.testing.assert_array_equal(open_array(store, mode="r")[:, :], data)


@pytest.mark.parametrize(
("codec_class", "codec_config"),
[
Expand Down
Loading