Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/4633.bugfix.1.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Creating a sharded array no longer fails when the codecs inside the shard cannot encode the data type's default value, for example a `scale_offset` filter that maps only part of the data type's range. The inner codecs are now checked against the array's fill value, which must still be encodable.
1 change: 1 addition & 0 deletions changes/4633.bugfix.2.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Creating an array no longer rejects a codec placed after a data-type-changing codec such as `cast_value` because its configuration would be invalid for the array's original data type. Each codec is now checked against the data type it receives.
9 changes: 8 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,12 @@ test = [
"pytest-codspeed==5.0.4",
"uv==0.13.7",
]
# tests/test_zarr_metadata_conformance.py draws array documents from zarr-metadata's
# strategies. Its own group, because zarr-metadata needs a newer typing-extensions than
# the `min_deps` env pins; the test skips where it is not installed.
conformance = [
"zarr-metadata[testing]==0.8.0",
]
remote-tests = [
{include-group = "test"},
"fsspec>=2023.10.0",
Expand Down Expand Up @@ -156,6 +162,7 @@ docs = [
]
dev = [
{include-group = "test"},
{include-group = "conformance"},
{include-group = "remote-tests"},
{include-group = "docs"},
"universal-pathlib",
Expand Down Expand Up @@ -188,7 +195,7 @@ hooks.vcs.version-file = "src/zarr/_version.py"
dependency-groups = ["dev"]

[tool.hatch.envs.test]
dependency-groups = ["test"]
dependency-groups = ["test", "conformance"]

[tool.hatch.envs.test.env-vars]

Expand Down
66 changes: 32 additions & 34 deletions src/zarr/codecs/sharding.py
Original file line number Diff line number Diff line change
Expand Up @@ -588,27 +588,47 @@ def to_dict(self) -> dict[str, JSON]:
}

def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
"""Thread the spec through the inner chain.
"""Evolve and validate the inner chain against the chunks it encodes.

Each codec is evolved against the spec produced by the previous one.
Evolving every codec against the same unthreaded spec is the bug shape that
strips `BytesCodec.endian` behind a dtype-changing codec β€” and this
method runs on the real array-creation path, baking the damaged chain
into the evolved instance before the transform builders ever run.
Each inner codec is evolved against, validated against, and resolved through
the spec the codecs before it leave, as array metadata does for its own codecs,
with a regular grid of one inner chunk as its chunk grid. A sharding codec among
them is thereby checked against the chunk it splits.

The spec is threaded for two reasons. A dtype-changing codec changes what the
codecs after it see: evolving every codec against the unthreaded spec strips
`BytesCodec.endian` behind one. And the fill value the inner chain must encode
is the array's fill value as the codecs before this one leave it, which is what
an unwritten inner chunk holds. Probing the chain with any other value, such as
the data type's default scalar, rejects chains that never see that value.

Parameters
----------
array_spec
The base spec to be evolved as we thread it.
The spec of the shards this codec encodes.

Returns
-------
This codec with the evolved code chain.
"""
from zarr.core.chunk_utils import evolve_codecs
This codec with the evolved codec chain.

shard_spec = self._get_chunk_spec(array_spec)
evolved_codecs = evolve_codecs(self.codecs, shard_spec)
Raises
------
ValueError
If an inner codec is incompatible with the chunk it encodes, or cannot
encode the fill value it receives.
"""
evolved: list[Codec] = []
spec = self._get_chunk_spec(array_spec)
for codec in self.codecs:
evolved_codec = codec.evolve_from_array_spec(spec)
evolved_codec.validate(
shape=spec.shape,
dtype=spec.dtype,
chunk_grid=RegularChunkGridMetadata(chunk_shape=spec.shape),
)
evolved.append(evolved_codec)
spec = evolved_codec.resolve_metadata(spec)
evolved_codecs = tuple(evolved)
if evolved_codecs != self.codecs:
return replace(self, codecs=evolved_codecs)
return self
Expand Down Expand Up @@ -642,28 +662,6 @@ def validate(
f"Chunk edge length {edge} in dimension {i} is not "
f"divisible by the shard's inner chunk size {inner}."
)
self._validate_inner_codecs(dtype)

def _validate_inner_codecs(self, dtype: ZDType[TBaseDType, TBaseScalar]) -> None:
"""Validate the codecs that encode each inner chunk, as array metadata validates
its codecs: each against the chunk the codecs before it leave it, a regular
grid of one inner chunk. A sharding codec among them is thereby checked against
the chunk it splits, which `validate` of the outer codec alone does not see.
"""
spec = ArraySpec(
shape=self.chunk_shape,
dtype=dtype,
fill_value=dtype.default_scalar(),
config=ArrayConfig.from_dict({}),
prototype=default_buffer_prototype(),
)
for codec in self.codecs:
codec.validate(
shape=spec.shape,
dtype=spec.dtype,
chunk_grid=RegularChunkGridMetadata(chunk_shape=spec.shape),
)
spec = codec.resolve_metadata(spec)

def _get_inner_chunk_transform(self, shard_spec: ArraySpec) -> Any:
"""The synchronous transform for the inner codec chain.
Expand Down
5 changes: 3 additions & 2 deletions src/zarr/core/metadata/v3.py
Original file line number Diff line number Diff line change
Expand Up @@ -604,8 +604,9 @@ def _validate_metadata(self) -> None:
)
if self.fill_value is None:
raise ValueError("`fill_value` is required.")
for codec in self.codecs:
codec.validate(shape=self.shape, dtype=self.data_type, chunk_grid=self.chunk_grid)
# The codecs were validated in `__init__`, each against the spec the codecs
# before it leave. Validating them again here against the array's own data type
# would reject a codec that only ever sees the output of a dtype-changing codec.

@property
def ndim(self) -> int:
Expand Down
23 changes: 22 additions & 1 deletion tests/test_codecs/test_cast_value.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

import zarr
from tests.conftest import Expect, ExpectFail
from zarr.codecs import BytesCodec, TransposeCodec
from zarr.codecs import BytesCodec, ScaleOffset, TransposeCodec
from zarr.codecs.cast_value import CAST_VALUE_RS_MIN_VERSION, CastValue
from zarr.storage import MemoryStore

Expand Down Expand Up @@ -610,3 +610,24 @@ def make_array(filters: list[Any]) -> Any:

array[:] = data
assert_array_equal(array[:], data)


@requires_cast_value_rs
@pytest.mark.parametrize("shards", [None, (8,)], ids=["unsharded", "sharded"])
def test_codec_validated_against_the_dtype_it_sees(shards: tuple[int, ...] | None) -> None:
"""A codec behind `cast_value` is validated against the cast's target data type,
not the array's: a `scale_offset` scale of 0.5 is representable in float32, which
the codec sees, though not in the array's int16."""
store = MemoryStore()
data = np.arange(8, dtype="int16")
arr = zarr.create_array(
store,
shape=data.shape,
dtype=data.dtype,
chunks=(4,),
shards=shards,
filters=[CastValue(data_type="float32"), ScaleOffset(scale=0.5)],
compressors=None,
)
arr[:] = data
assert_array_equal(zarr.open_array(store, mode="r")[:], data)
85 changes: 85 additions & 0 deletions tests/test_codecs/test_sharding.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
BytesCodec,
Crc32cCodec,
GzipCodec,
ScaleOffset,
ShardingCodec,
TransposeCodec,
)
Expand Down Expand Up @@ -1522,3 +1523,87 @@ def test_open_rejects_stored_nested_sharding_with_indivisible_inner_chunk_shape(
doc["codecs"][0]["configuration"]["codecs"][0]["configuration"]["chunk_shape"] = (4, 7)
with pytest.raises(ValueError, match="not divisible"):
ArrayV3Metadata.from_dict(doc)


# Each layout pairs codecs with a fill value the inner chain can encode, as the codecs
# before it leave that value, and one it cannot. The accepted fill value is chosen so
# that neither the data type's default scalar (0) nor, where a filter precedes the
# sharding codec, the array's fill value as given would pass the inner chain.
_INNER_CHAIN_FILL_LAYOUTS: dict[str, tuple[dict[str, Any], int, int]] = {
# `scale_offset` inside the shard: the inner chain encodes [100, 65535].
"inside-shard": (
{"chunks": (4,), "shards": (16,), "filters": [ScaleOffset(offset=100)]},
100,
0,
),
# `scale_offset` before the sharding codec doubles the fill value, so the inner
# chain sees 120 for an array fill value of 60, and 80 for 40. Its own
# `scale_offset` encodes [100, 65535].
"behind-filter": (
{
"chunks": (16,),
"filters": [ScaleOffset(scale=2)],
"serializer": ShardingCodec(
chunk_shape=(4,), codecs=[ScaleOffset(offset=100), BytesCodec()]
),
},
60,
40,
),
# The fill value reaches a sharding codec nested in another one.
"nested": (
{
"chunks": (16,),
"serializer": ShardingCodec(
chunk_shape=(8,),
codecs=[
ShardingCodec(chunk_shape=(4,), codecs=[ScaleOffset(offset=100), BytesCodec()])
],
),
},
100,
0,
),
}


@pytest.mark.filterwarnings(
"ignore:Combining a `sharding_indexed` codec:zarr.errors.ZarrUserWarning"
)
@pytest.mark.parametrize("layout", _INNER_CHAIN_FILL_LAYOUTS)
def test_sharding_inner_chain_probed_with_array_fill_value(layout: str) -> None:
"""A sharding codec's inner chain is validated against the array's fill value, as
the codecs before it leave it, not against the data type's default scalar
(https://github.com/zarr-developers/zarr-python/issues/4511). The array is created,
reads its fill value before any write, and round-trips a partial write."""
codecs, fill_value, _ = _INNER_CHAIN_FILL_LAYOUTS[layout]
store = MemoryStore()
arr = zarr.create_array(
store, shape=(16,), dtype="uint16", fill_value=fill_value, compressors=None, **codecs
)
expected = np.full(16, fill_value, dtype="uint16")
np.testing.assert_array_equal(arr[:], expected)
written = np.arange(110, 114, dtype="uint16")
arr[4:8] = written
expected[4:8] = written
np.testing.assert_array_equal(zarr.open_array(store, mode="r")[:], expected)


@pytest.mark.filterwarnings(
"ignore:Combining a `sharding_indexed` codec:zarr.errors.ZarrUserWarning"
)
@pytest.mark.parametrize("layout", _INNER_CHAIN_FILL_LAYOUTS)
def test_sharding_rejects_fill_value_inner_chain_cannot_encode(layout: str) -> None:
"""An array whose fill value the inner chain cannot encode is rejected: an
unwritten inner chunk holds the fill value, so the array could not be read before
its first write."""
codecs, _, fill_value = _INNER_CHAIN_FILL_LAYOUTS[layout]
with pytest.raises(ValueError, match="outside the range"):
zarr.create_array(
MemoryStore(),
shape=(16,),
dtype="uint16",
fill_value=fill_value,
compressors=None,
**codecs,
)
Loading
Loading