diff --git a/changes/456.bugfix.md b/changes/456.bugfix.md new file mode 100644 index 0000000000..248e6ad000 --- /dev/null +++ b/changes/456.bugfix.md @@ -0,0 +1 @@ +Arrays that apply a `zarr.codecs.numcodecs.Quantize` filter whose `astype` differs from the array data type can now be read back. Previously writing succeeded, but reading failed with `ValueError: cannot reshape array`. diff --git a/src/zarr/codecs/numcodecs/_codecs.py b/src/zarr/codecs/numcodecs/_codecs.py index 5681c16c54..6b503c53e7 100644 --- a/src/zarr/codecs/numcodecs/_codecs.py +++ b/src/zarr/codecs/numcodecs/_codecs.py @@ -239,12 +239,20 @@ def evolve_from_array_spec(self, array_spec: ArraySpec) -> Shuffle: # array-to-array codecs ("filters") +def _resolve_astype(codec_config: dict[str, JSON], chunk_spec: ArraySpec) -> ArraySpec: + """ + Return the spec of a filter's output when its ``astype`` parameter sets the encoded + data type, and ``chunk_spec`` unchanged when ``astype`` is absent. + """ + if astype := codec_config.get("astype"): + dtype = parse_dtype(np.dtype(astype), zarr_format=3) # type: ignore[call-overload] + return replace(chunk_spec, dtype=dtype) + return chunk_spec + + class Delta(_NumcodecsArrayArrayCodec, codec_name="delta"): def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: - if astype := self.codec_config.get("astype"): - dtype = parse_dtype(np.dtype(astype), zarr_format=3) # type: ignore[call-overload] - return replace(chunk_spec, dtype=dtype) - return chunk_spec + return _resolve_astype(self.codec_config, chunk_spec) class BitRound(_NumcodecsArrayArrayCodec, codec_name="bitround"): @@ -253,10 +261,7 @@ class BitRound(_NumcodecsArrayArrayCodec, codec_name="bitround"): class FixedScaleOffset(_NumcodecsArrayArrayCodec, codec_name="fixedscaleoffset"): def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: - if astype := self.codec_config.get("astype"): - dtype = parse_dtype(np.dtype(astype), zarr_format=3) # type: ignore[call-overload] - return replace(chunk_spec, dtype=dtype) - return chunk_spec + return _resolve_astype(self.codec_config, chunk_spec) def evolve_from_array_spec(self, array_spec: ArraySpec) -> FixedScaleOffset: if self.codec_config.get("dtype") is None: @@ -269,6 +274,9 @@ class Quantize(_NumcodecsArrayArrayCodec, codec_name="quantize"): def __init__(self, **codec_config: JSON) -> None: super().__init__(**codec_config) + def resolve_metadata(self, chunk_spec: ArraySpec) -> ArraySpec: + return _resolve_astype(self.codec_config, chunk_spec) + def evolve_from_array_spec(self, array_spec: ArraySpec) -> Quantize: if self.codec_config.get("dtype") is None: dtype = array_spec.dtype.to_native_dtype() diff --git a/tests/test_codecs/test_numcodecs.py b/tests/test_codecs/test_numcodecs.py index d41ba3073d..8e507f92d8 100644 --- a/tests/test_codecs/test_numcodecs.py +++ b/tests/test_codecs/test_numcodecs.py @@ -4,6 +4,7 @@ import pickle from typing import TYPE_CHECKING, Any +import numcodecs import numpy as np import pytest from numcodecs import GZip @@ -271,8 +272,18 @@ def test_generic_filter_bitround() -> None: assert np.allclose(data, b[:, :], atol=0.1) -def test_generic_filter_quantize() -> None: - data = np.linspace(0, 10, 256, dtype="float32").reshape((16, 16)) +@pytest.mark.parametrize( + ("dtype", "astype"), + [("float32", None), ("float64", "float32"), ("float32", "float16"), ("float32", "float64")], +) +def test_generic_filter_quantize(dtype: str, astype: str | None) -> None: + """ + Quantize round-trips with and without an ``astype`` that differs from the array data type. + """ + data = np.linspace(0, 10, 256, dtype=dtype).reshape((16, 16)) + codec_config: dict[str, Any] = {"digits": 3} + if astype is not None: + codec_config["astype"] = astype a = create_array( {}, @@ -280,12 +291,14 @@ def test_generic_filter_quantize() -> None: chunks=(16, 16), dtype=data.dtype, fill_value=0, - filters=[_numcodecs.Quantize(digits=3)], + filters=[_numcodecs.Quantize(**codec_config)], ) a[:, :] = data.copy() b = open_array(a.store, mode="r") - assert np.allclose(data, b[:, :], atol=0.001) + reference = numcodecs.Quantize(dtype=dtype, **codec_config) + expected = reference.decode(reference.encode(data)).reshape(data.shape) + np.testing.assert_array_equal(b[:, :], expected) def test_generic_filter_packbits() -> None: