Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/456.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Arrays that apply a `zarr.codecs.numcodecs.Quantize` filter whose `astype` differs from the array data type can now be read back. Previously writing succeeded, but reading failed with `ValueError: cannot reshape array`.
24 changes: 16 additions & 8 deletions src/zarr/codecs/numcodecs/_codecs.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,12 +239,20 @@ def evolve_from_array_spec(self, array_spec: ArraySpec) -> Shuffle:


# array-to-array codecs ("filters")
def _resolve_astype(codec_config: dict[str, JSON], chunk_spec: ArraySpec) -> ArraySpec:
"""
Return the spec of a filter's output when its ``astype`` parameter sets the encoded
data type, and ``chunk_spec`` unchanged when ``astype`` is absent.
"""
if astype := codec_config.get("astype"):
dtype = parse_dtype(np.dtype(astype), zarr_format=3) # type: ignore[call-overload]
return replace(chunk_spec, dtype=dtype)
return chunk_spec


class Delta(_NumcodecsArrayArrayCodec, codec_name="delta"):
def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
if astype := self.codec_config.get("astype"):
dtype = parse_dtype(np.dtype(astype), zarr_format=3) # type: ignore[call-overload]
return replace(chunk_spec, dtype=dtype)
return chunk_spec
return _resolve_astype(self.codec_config, chunk_spec)


class BitRound(_NumcodecsArrayArrayCodec, codec_name="bitround"):
Expand All @@ -253,10 +261,7 @@ class BitRound(_NumcodecsArrayArrayCodec, codec_name="bitround"):

class FixedScaleOffset(_NumcodecsArrayArrayCodec, codec_name="fixedscaleoffset"):
def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
if astype := self.codec_config.get("astype"):
dtype = parse_dtype(np.dtype(astype), zarr_format=3) # type: ignore[call-overload]
return replace(chunk_spec, dtype=dtype)
return chunk_spec
return _resolve_astype(self.codec_config, chunk_spec)

def evolve_from_array_spec(self, array_spec: ArraySpec) -> FixedScaleOffset:
if self.codec_config.get("dtype") is None:
Expand All @@ -269,6 +274,9 @@ class Quantize(_NumcodecsArrayArrayCodec, codec_name="quantize"):
def __init__(self, **codec_config: JSON) -> None:
super().__init__(**codec_config)

def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec:
return _resolve_astype(self.codec_config, chunk_spec)

def evolve_from_array_spec(self, array_spec: ArraySpec) -> Quantize:
if self.codec_config.get("dtype") is None:
dtype = array_spec.dtype.to_native_dtype()
Expand Down
21 changes: 17 additions & 4 deletions tests/test_codecs/test_numcodecs.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import pickle
from typing import TYPE_CHECKING, Any

import numcodecs
import numpy as np
import pytest
from numcodecs import GZip
Expand Down Expand Up @@ -271,21 +272,33 @@ def test_generic_filter_bitround() -> None:
assert np.allclose(data, b[:, :], atol=0.1)


def test_generic_filter_quantize() -> None:
data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16))
@pytest.mark.parametrize(
("dtype", "astype"),
[("float32", None), ("float64", "float32"), ("float32", "float16"), ("float32", "float64")],
)
def test_generic_filter_quantize(dtype: str, astype: str | None) -> None:
"""
Quantize round-trips with and without an ``astype`` that differs from the array data type.
"""
data = np.linspace(0, 10, 256, dtype=dtype).reshape((16, 16))
codec_config: dict[str, Any] = {"digits": 3}
if astype is not None:
codec_config["astype"] = astype

a = create_array(
{},
shape=data.shape,
chunks=(16, 16),
dtype=data.dtype,
fill_value=0,
filters=[_numcodecs.Quantize(digits=3)],
filters=[_numcodecs.Quantize(**codec_config)],
)

a[:, :] = data.copy()
b = open_array(a.store, mode="r")
assert np.allclose(data, b[:, :], atol=0.001)
reference = numcodecs.Quantize(dtype=dtype, **codec_config)
expected = reference.decode(reference.encode(data)).reshape(data.shape)
np.testing.assert_array_equal(b[:, :], expected)


def test_generic_filter_packbits() -> None:
Expand Down
Loading