diff --git a/changes/455.bugfix.md b/changes/455.bugfix.md new file mode 100644 index 0000000000..4ba384b2cb --- /dev/null +++ b/changes/455.bugfix.md @@ -0,0 +1 @@ +The `crc32c` codec now stores its checksum as a little-endian unsigned 32-bit integer on every host, as the crc32c codec specification requires. Previously it used the host byte order, so on big-endian hosts zarr-python wrote and expected checksums that other implementations reject. diff --git a/src/zarr/codecs/crc32c_.py b/src/zarr/codecs/crc32c_.py index 7d41e11637..c9c9fcb211 100644 --- a/src/zarr/codecs/crc32c_.py +++ b/src/zarr/codecs/crc32c_.py @@ -2,10 +2,11 @@ from collections.abc import Buffer as ABCBuffer from dataclasses import dataclass -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Any, cast import google_crc32c import numpy as np +import numpy.typing as npt from zarr.abc.codec import BytesBytesCodec from zarr.core.common import JSON, parse_named_configuration @@ -17,6 +18,17 @@ from zarr.core.buffer import Buffer +def _checksum_bytes(data: npt.NDArray[Any]) -> bytes: + """ + Return the CRC32C checksum of ``data`` as the 4 bytes the crc32c codec stores. + + The crc32c codec specification stores the checksum as a little-endian + unsigned 32-bit integer, independent of the byte order of the host. + """ + checksum: int = google_crc32c.value(cast(ABCBuffer, data)) + return checksum.to_bytes(4, "little") + + @dataclass(frozen=True) class Crc32cCodec(BytesBytesCodec): """crc32c codec""" @@ -40,8 +52,7 @@ def _decode_sync( crc32_bytes = data[-4:] inner_bytes = data[:-4] - # Need to do a manual cast until https://github.com/numpy/numpy/issues/26783 is resolved - computed_checksum = np.uint32(google_crc32c.value(cast(ABCBuffer, inner_bytes))).tobytes() + computed_checksum = _checksum_bytes(inner_bytes) stored_checksum = bytes(crc32_bytes) if computed_checksum != stored_checksum: raise ValueError( @@ -62,10 +73,8 @@ def _encode_sync( chunk_spec: ArraySpec, ) -> Buffer | None: data = chunk_bytes.as_numpy_array() - # Calculate the checksum and "cast" it to a numpy array - checksum = np.array([google_crc32c.value(cast(ABCBuffer, data))], dtype=np.uint32) - # Append the checksum (as bytes) to the data - return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum.view("B"))) + checksum = np.frombuffer(_checksum_bytes(data), dtype=np.uint8) + return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum)) async def _encode_single( self, diff --git a/tests/test_codecs/test_crc32c.py b/tests/test_codecs/test_crc32c.py index 3ab1070f60..f5099423bd 100644 --- a/tests/test_codecs/test_crc32c.py +++ b/tests/test_codecs/test_crc32c.py @@ -1,6 +1,7 @@ from __future__ import annotations import numpy as np +import pytest from zarr.abc.codec import SupportsSyncCodec from zarr.codecs.crc32c_ import Crc32cCodec @@ -31,3 +32,49 @@ def test_crc32c_codec_sync_roundtrip() -> None: decoded = codec._decode_sync(encoded, spec) result = np.frombuffer(decoded.as_numpy_array(), dtype="float64") np.testing.assert_array_equal(arr, result) + + +def _bytes_spec(nbytes: int) -> ArraySpec: + zdtype = get_data_type_from_native_dtype(np.dtype("uint8")) + return ArraySpec( + shape=(nbytes,), + dtype=zdtype, + fill_value=zdtype.cast_scalar(0), + config=ArrayConfig(order="C", write_empty_chunks=True), + prototype=default_buffer_prototype(), + ) + + +@pytest.mark.parametrize( + ("data", "checksum"), + [ + # check value of CRC-32C (RFC 3720) + (b"123456789", bytes.fromhex("839206e3")), + (b"", bytes.fromhex("00000000")), + (b"\x00" * 32, bytes.fromhex("aa36918a")), + ], +) +def test_crc32c_encoded_bytes(data: bytes, checksum: bytes) -> None: + """ + The crc32c codec appends the checksum as a little-endian uint32, whatever the host + byte order, and decoding returns the original bytes. + """ + codec = Crc32cCodec() + spec = _bytes_spec(len(data)) + encoded = codec._encode_sync(default_buffer_prototype().buffer.from_bytes(data), spec) + assert encoded is not None + assert encoded.to_bytes() == data + checksum + assert codec._decode_sync(encoded, spec).to_bytes() == data + + +def test_crc32c_decode_rejects_mismatched_checksum() -> None: + """ + Decoding raises when the stored checksum does not match the data, including when + the right checksum is stored in big-endian byte order. + """ + codec = Crc32cCodec() + data = b"123456789" + spec = _bytes_spec(len(data)) + stored = data + bytes.fromhex("e3069283") + with pytest.raises(ValueError, match="checksum do not match"): + codec._decode_sync(default_buffer_prototype().buffer.from_bytes(stored), spec)