Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/455.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
The `crc32c` codec now stores its checksum as a little-endian unsigned 32-bit integer on every host, as the crc32c codec specification requires. Previously it used the host byte order, so on big-endian hosts zarr-python wrote and expected checksums that other implementations reject.
23 changes: 16 additions & 7 deletions src/zarr/codecs/crc32c_.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@

from collections.abc import Buffer as ABCBuffer
from dataclasses import dataclass
from typing import TYPE_CHECKING, cast
from typing import TYPE_CHECKING, Any, cast

import google_crc32c
import numpy as np
import numpy.typing as npt

from zarr.abc.codec import BytesBytesCodec
from zarr.core.common import JSON, parse_named_configuration
Expand All @@ -17,6 +18,17 @@
from zarr.core.buffer import Buffer


def _checksum_bytes(data: npt.NDArray[Any]) -> bytes:
"""
Return the CRC32C checksum of ``data`` as the 4 bytes the crc32c codec stores.

The crc32c codec specification stores the checksum as a little-endian
unsigned 32-bit integer, independent of the byte order of the host.
"""
checksum: int = google_crc32c.value(cast(ABCBuffer, data))
return checksum.to_bytes(4, "little")


@dataclass(frozen=True)
class Crc32cCodec(BytesBytesCodec):
"""crc32c codec"""
Expand All @@ -40,8 +52,7 @@ def _decode_sync(
crc32_bytes = data[-4:]
inner_bytes = data[:-4]

# Need to do a manual cast until https://github.com/numpy/numpy/issues/26783 is resolved
computed_checksum = np.uint32(google_crc32c.value(cast(ABCBuffer, inner_bytes))).tobytes()
computed_checksum = _checksum_bytes(inner_bytes)
stored_checksum = bytes(crc32_bytes)
if computed_checksum != stored_checksum:
raise ValueError(
Expand All @@ -62,10 +73,8 @@ def _encode_sync(
chunk_spec: ArraySpec,
) -> Buffer | None:
data = chunk_bytes.as_numpy_array()
# Calculate the checksum and "cast" it to a numpy array
checksum = np.array([google_crc32c.value(cast(ABCBuffer, data))], dtype=np.uint32)
# Append the checksum (as bytes) to the data
return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum.view("B")))
checksum = np.frombuffer(_checksum_bytes(data), dtype=np.uint8)
return chunk_spec.prototype.buffer.from_array_like(np.append(data, checksum))

async def _encode_single(
self,
Expand Down
47 changes: 47 additions & 0 deletions tests/test_codecs/test_crc32c.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

import numpy as np
import pytest

from zarr.abc.codec import SupportsSyncCodec
from zarr.codecs.crc32c_ import Crc32cCodec
Expand Down Expand Up @@ -31,3 +32,49 @@ def test_crc32c_codec_sync_roundtrip() -> None:
decoded = codec._decode_sync(encoded, spec)
result = np.frombuffer(decoded.as_numpy_array(), dtype="float64")
np.testing.assert_array_equal(arr, result)


def _bytes_spec(nbytes: int) -> ArraySpec:
zdtype = get_data_type_from_native_dtype(np.dtype("uint8"))
return ArraySpec(
shape=(nbytes,),
dtype=zdtype,
fill_value=zdtype.cast_scalar(0),
config=ArrayConfig(order="C", write_empty_chunks=True),
prototype=default_buffer_prototype(),
)


@pytest.mark.parametrize(
("data", "checksum"),
[
# check value of CRC-32C (RFC 3720)
(b"123456789", bytes.fromhex("839206e3")),
(b"", bytes.fromhex("00000000")),
(b"\x00" * 32, bytes.fromhex("aa36918a")),
],
)
def test_crc32c_encoded_bytes(data: bytes, checksum: bytes) -> None:
"""
The crc32c codec appends the checksum as a little-endian uint32, whatever the host
byte order, and decoding returns the original bytes.
"""
codec = Crc32cCodec()
spec = _bytes_spec(len(data))
encoded = codec._encode_sync(default_buffer_prototype().buffer.from_bytes(data), spec)
assert encoded is not None
assert encoded.to_bytes() == data + checksum
assert codec._decode_sync(encoded, spec).to_bytes() == data


def test_crc32c_decode_rejects_mismatched_checksum() -> None:
"""
Decoding raises when the stored checksum does not match the data, including when
the right checksum is stored in big-endian byte order.
"""
codec = Crc32cCodec()
data = b"123456789"
spec = _bytes_spec(len(data))
stored = data + bytes.fromhex("e3069283")
with pytest.raises(ValueError, match="checksum do not match"):
codec._decode_sync(default_buffer_prototype().buffer.from_bytes(stored), spec)
Loading