From 7f38776668743ef3deefc13a5e3b8c22378f9cf1 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 13 Nov 2026 21:55:47 +0100 Subject: [PATCH 1/5] fix(sharding): validate the inner codec chain against the array's fill value Creating a sharded array failed when the inner codec chain could not encode the data type's default scalar, e.g. a `scale_offset` filter that maps only a subrange of the data type, even when the array's own fill value encodes fine (#4511). `ShardingCodec.validate` probed the inner chain with `dtype.default_scalar()`, since `validate` never receives the fill value. Run the inner-chain check from `ShardingCodec.evolve_from_array_spec` instead, which receives the threaded spec: each inner codec is evolved, validated and resolved against the spec the codecs before it leave, including the fill value as those codecs encode it. An array whose fill value the inner chain cannot encode is still rejected, because an unwritten chunk holds the fill value. Also drop the per-codec `validate` loop from `ArrayV3Metadata._validate_metadata`. `__init__` has already validated each codec against the threaded spec; the loop re-validated them against the array's own data type, which rejected valid chains such as int16 -> `cast_value` float32 -> `scale_offset` with a scale of 0.5. Assisted-by: ClaudeCode:claude-opus-5-5 --- src/zarr/codecs/sharding.py | 66 +++++++++++---------- src/zarr/core/metadata/v3.py | 5 +- tests/test_codecs/test_cast_value.py | 23 +++++++- tests/test_codecs/test_sharding.py | 85 ++++++++++++++++++++++++++++ 4 files changed, 142 insertions(+), 37 deletions(-) diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index d4ccf71cb9..e156d94c57 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -588,27 +588,47 @@ def to_dict(self) -> dict[str, JSON]: } def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: - """Thread the spec through the inner chain. + """Evolve and validate the inner chain against the chunks it encodes. - Each codec is evolved against the spec produced by the previous one. - Evolving every codec against the same unthreaded spec is the bug shape that - strips `BytesCodec.endian` behind a dtype-changing codec — and this - method runs on the real array-creation path, baking the damaged chain - into the evolved instance before the transform builders ever run. + Each inner codec is evolved against, validated against, and resolved through + the spec the codecs before it leave, as array metadata does for its own codecs, + with a regular grid of one inner chunk as its chunk grid. A sharding codec among + them is thereby checked against the chunk it splits. + + The spec is threaded for two reasons. A dtype-changing codec changes what the + codecs after it see: evolving every codec against the unthreaded spec strips + `BytesCodec.endian` behind one. And the fill value the inner chain must encode + is the array's fill value as the codecs before this one leave it, which is what + an unwritten inner chunk holds. Probing the chain with any other value, such as + the data type's default scalar, rejects chains that never see that value. Parameters ---------- array_spec - The base spec to be evolved as we thread it. + The spec of the shards this codec encodes. Returns ------- - This codec with the evolved code chain. - """ - from zarr.core.chunk_utils import evolve_codecs + This codec with the evolved codec chain. - shard_spec = self._get_chunk_spec(array_spec) - evolved_codecs = evolve_codecs(self.codecs, shard_spec) + Raises + ------ + ValueError + If an inner codec is incompatible with the chunk it encodes, or cannot + encode the fill value it receives. + """ + evolved: list[Codec] = [] + spec = self._get_chunk_spec(array_spec) + for codec in self.codecs: + evolved_codec = codec.evolve_from_array_spec(spec) + evolved_codec.validate( + shape=spec.shape, + dtype=spec.dtype, + chunk_grid=RegularChunkGridMetadata(chunk_shape=spec.shape), + ) + evolved.append(evolved_codec) + spec = evolved_codec.resolve_metadata(spec) + evolved_codecs = tuple(evolved) if evolved_codecs != self.codecs: return replace(self, codecs=evolved_codecs) return self @@ -642,28 +662,6 @@ def validate( f"Chunk edge length {edge} in dimension {i} is not " f"divisible by the shard's inner chunk size {inner}." ) - self._validate_inner_codecs(dtype) - - def _validate_inner_codecs(self, dtype: ZDType[TBaseDType, TBaseScalar]) -> None: - """Validate the codecs that encode each inner chunk, as array metadata validates - its codecs: each against the chunk the codecs before it leave it, a regular - grid of one inner chunk. A sharding codec among them is thereby checked against - the chunk it splits, which `validate` of the outer codec alone does not see. - """ - spec = ArraySpec( - shape=self.chunk_shape, - dtype=dtype, - fill_value=dtype.default_scalar(), - config=ArrayConfig.from_dict({}), - prototype=default_buffer_prototype(), - ) - for codec in self.codecs: - codec.validate( - shape=spec.shape, - dtype=spec.dtype, - chunk_grid=RegularChunkGridMetadata(chunk_shape=spec.shape), - ) - spec = codec.resolve_metadata(spec) def _get_inner_chunk_transform(self, shard_spec: ArraySpec) -> Any: """The synchronous transform for the inner codec chain. diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index 63d525ba2d..aa611d4289 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -604,8 +604,9 @@ def _validate_metadata(self) -> None: ) if self.fill_value is None: raise ValueError("`fill_value` is required.") - for codec in self.codecs: - codec.validate(shape=self.shape, dtype=self.data_type, chunk_grid=self.chunk_grid) + # The codecs were validated in `__init__`, each against the spec the codecs + # before it leave. Validating them again here against the array's own data type + # would reject a codec that only ever sees the output of a dtype-changing codec. @property def ndim(self) -> int: diff --git a/tests/test_codecs/test_cast_value.py b/tests/test_codecs/test_cast_value.py index d6c703767f..37d1cb385a 100644 --- a/tests/test_codecs/test_cast_value.py +++ b/tests/test_codecs/test_cast_value.py @@ -8,7 +8,7 @@ import zarr from tests.conftest import Expect, ExpectFail -from zarr.codecs import BytesCodec, TransposeCodec +from zarr.codecs import BytesCodec, ScaleOffset, TransposeCodec from zarr.codecs.cast_value import CAST_VALUE_RS_MIN_VERSION, CastValue from zarr.storage import MemoryStore @@ -610,3 +610,24 @@ def make_array(filters: list[Any]) -> Any: array[:] = data assert_array_equal(array[:], data) + + +@requires_cast_value_rs +@pytest.mark.parametrize("shards", [None, (8,)], ids=["unsharded", "sharded"]) +def test_codec_validated_against_the_dtype_it_sees(shards: tuple[int, ...] | None) -> None: + """A codec behind `cast_value` is validated against the cast's target data type, + not the array's: a `scale_offset` scale of 0.5 is representable in float32, which + the codec sees, though not in the array's int16.""" + store = MemoryStore() + data = np.arange(8, dtype="int16") + arr = zarr.create_array( + store, + shape=data.shape, + dtype=data.dtype, + chunks=(4,), + shards=shards, + filters=[CastValue(data_type="float32"), ScaleOffset(scale=0.5)], + compressors=None, + ) + arr[:] = data + assert_array_equal(zarr.open_array(store, mode="r")[:], data) diff --git a/tests/test_codecs/test_sharding.py b/tests/test_codecs/test_sharding.py index 5712c9576d..52e23164ad 100644 --- a/tests/test_codecs/test_sharding.py +++ b/tests/test_codecs/test_sharding.py @@ -19,6 +19,7 @@ BytesCodec, Crc32cCodec, GzipCodec, + ScaleOffset, ShardingCodec, TransposeCodec, ) @@ -1522,3 +1523,87 @@ def test_open_rejects_stored_nested_sharding_with_indivisible_inner_chunk_shape( doc["codecs"][0]["configuration"]["codecs"][0]["configuration"]["chunk_shape"] = (4, 7) with pytest.raises(ValueError, match="not divisible"): ArrayV3Metadata.from_dict(doc) + + +# Each layout pairs codecs with a fill value the inner chain can encode, as the codecs +# before it leave that value, and one it cannot. The accepted fill value is chosen so +# that neither the data type's default scalar (0) nor, where a filter precedes the +# sharding codec, the array's fill value as given would pass the inner chain. +_INNER_CHAIN_FILL_LAYOUTS: dict[str, tuple[dict[str, Any], int, int]] = { + # `scale_offset` inside the shard: the inner chain encodes [100, 65535]. + "inside-shard": ( + {"chunks": (4,), "shards": (16,), "filters": [ScaleOffset(offset=100)]}, + 100, + 0, + ), + # `scale_offset` before the sharding codec doubles the fill value, so the inner + # chain sees 120 for an array fill value of 60, and 80 for 40. Its own + # `scale_offset` encodes [100, 65535]. + "behind-filter": ( + { + "chunks": (16,), + "filters": [ScaleOffset(scale=2)], + "serializer": ShardingCodec( + chunk_shape=(4,), codecs=[ScaleOffset(offset=100), BytesCodec()] + ), + }, + 60, + 40, + ), + # The fill value reaches a sharding codec nested in another one. + "nested": ( + { + "chunks": (16,), + "serializer": ShardingCodec( + chunk_shape=(8,), + codecs=[ + ShardingCodec(chunk_shape=(4,), codecs=[ScaleOffset(offset=100), BytesCodec()]) + ], + ), + }, + 100, + 0, + ), +} + + +@pytest.mark.filterwarnings( + "ignore:Combining a `sharding_indexed` codec:zarr.errors.ZarrUserWarning" +) +@pytest.mark.parametrize("layout", _INNER_CHAIN_FILL_LAYOUTS) +def test_sharding_inner_chain_probed_with_array_fill_value(layout: str) -> None: + """A sharding codec's inner chain is validated against the array's fill value, as + the codecs before it leave it, not against the data type's default scalar + (https://github.com/zarr-developers/zarr-python/issues/4511). The array is created, + reads its fill value before any write, and round-trips a partial write.""" + codecs, fill_value, _ = _INNER_CHAIN_FILL_LAYOUTS[layout] + store = MemoryStore() + arr = zarr.create_array( + store, shape=(16,), dtype="uint16", fill_value=fill_value, compressors=None, **codecs + ) + expected = np.full(16, fill_value, dtype="uint16") + np.testing.assert_array_equal(arr[:], expected) + written = np.arange(110, 114, dtype="uint16") + arr[4:8] = written + expected[4:8] = written + np.testing.assert_array_equal(zarr.open_array(store, mode="r")[:], expected) + + +@pytest.mark.filterwarnings( + "ignore:Combining a `sharding_indexed` codec:zarr.errors.ZarrUserWarning" +) +@pytest.mark.parametrize("layout", _INNER_CHAIN_FILL_LAYOUTS) +def test_sharding_rejects_fill_value_inner_chain_cannot_encode(layout: str) -> None: + """An array whose fill value the inner chain cannot encode is rejected: an + unwritten inner chunk holds the fill value, so the array could not be read before + its first write.""" + codecs, _, fill_value = _INNER_CHAIN_FILL_LAYOUTS[layout] + with pytest.raises(ValueError, match="outside the range"): + zarr.create_array( + MemoryStore(), + shape=(16,), + dtype="uint16", + fill_value=fill_value, + compressors=None, + **codecs, + ) From 9942717067473b6b00df365fdf4ac27501f00617 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 13 Nov 2026 21:56:25 +0100 Subject: [PATCH 2/5] docs: add changelog fragments for #446 Assisted-by: ClaudeCode:claude-opus-5-5 --- changes/446.bugfix.1.md | 1 + changes/446.bugfix.2.md | 1 + 2 files changed, 2 insertions(+) create mode 100644 changes/446.bugfix.1.md create mode 100644 changes/446.bugfix.2.md diff --git a/changes/446.bugfix.1.md b/changes/446.bugfix.1.md new file mode 100644 index 0000000000..a8293c2cd6 --- /dev/null +++ b/changes/446.bugfix.1.md @@ -0,0 +1 @@ +Creating a sharded array no longer fails when the codecs inside the shard cannot encode the data type's default value, for example a `scale_offset` filter that maps only part of the data type's range. The inner codecs are now checked against the array's fill value, which must still be encodable. diff --git a/changes/446.bugfix.2.md b/changes/446.bugfix.2.md new file mode 100644 index 0000000000..ca87b8037a --- /dev/null +++ b/changes/446.bugfix.2.md @@ -0,0 +1 @@ +Creating an array no longer rejects a codec placed after a data-type-changing codec such as `cast_value` because its configuration would be invalid for the array's original data type. Each codec is now checked against the data type it receives. From 39eba055f645d832dd2a2b5c102eb1e5e36f8f49 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 11:18:45 +0100 Subject: [PATCH 3/5] docs: renumber changelog fragments for #4633 Assisted-by: ClaudeCode:claude-opus-5-5 --- changes/{446.bugfix.1.md => 4633.bugfix.1.md} | 0 changes/{446.bugfix.2.md => 4633.bugfix.2.md} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename changes/{446.bugfix.1.md => 4633.bugfix.1.md} (100%) rename changes/{446.bugfix.2.md => 4633.bugfix.2.md} (100%) diff --git a/changes/446.bugfix.1.md b/changes/4633.bugfix.1.md similarity index 100% rename from changes/446.bugfix.1.md rename to changes/4633.bugfix.1.md diff --git a/changes/446.bugfix.2.md b/changes/4633.bugfix.2.md similarity index 100% rename from changes/446.bugfix.2.md rename to changes/4633.bugfix.2.md From 8d801dd87a1451e962fb6577af4b21edc2fa41ca Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 13:27:05 +0100 Subject: [PATCH 4/5] test: check zarr-python's verdict on zarr-metadata's array documents Draw array documents from zarr-metadata's hypothesis strategies and hand each to `ArrayV3Metadata.from_dict`. zarr-python may refuse a valid document only for a fill value its codecs cannot encode, and must then refuse it too with each shard's codecs moved out of the shard. Two slices run: every definition, and the integer/float data types with the value-transforming codecs, where fill value refusals are common. Before this branch's fix the property fails for both bugs it fixes: the unthreaded re-validation in `_validate_metadata`, and the sharding probe with the data type's default scalar. Known divergences are left out of the draw where possible and each is pinned by a strict xfail test: object-spelled data types, `r*` raw bits, integer-array `bytes` fill values, sharding behind `transpose`, hex `cast_value` scalar map values, and zarr-metadata accepting empty rectilinear edge lists (barred since zarr-extensions#78). Adds `zarr-metadata[testing]==0.8.0` to the test dependency group. Assisted-by: ClaudeCode:claude-opus-5-5 --- pyproject.toml | 2 + tests/test_zarr_metadata_conformance.py | 315 ++++++++++++++++++++++++ uv.lock | 24 ++ 3 files changed, 341 insertions(+) create mode 100644 tests/test_zarr_metadata_conformance.py diff --git a/pyproject.toml b/pyproject.toml index 839b5fdef8..633fcec4d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,6 +122,8 @@ test = [ "pytest-benchmark==5.3.1", "pytest-codspeed==5.0.4", "uv==0.13.7", + # tests/test_zarr_metadata_conformance.py draws array documents from its strategies. + "zarr-metadata[testing]==0.8.0", ] remote-tests = [ {include-group = "test"}, diff --git a/tests/test_zarr_metadata_conformance.py b/tests/test_zarr_metadata_conformance.py new file mode 100644 index 0000000000..5c0339238f --- /dev/null +++ b/tests/test_zarr_metadata_conformance.py @@ -0,0 +1,315 @@ +"""zarr-python's verdict on the array documents zarr-metadata's strategies draw. + +zarr-metadata's strategies draw documents that the spec allows. Each codec in them +is drawn for the data type and the chunk the codecs before it hand on, so new codecs +and nestings are covered as soon as zarr-metadata defines them. + +zarr-python refuses one kind of valid document on purpose: one whose fill value the +codecs cannot encode, because an unwritten chunk holds the fill value. A document +refused for any other reason is a divergence from the spec. + +`_DIVERGENCES` lists the known divergences. The strategies here leave out each one +they can, and a strict xfail test below pins each one, so that fixing it fails that +test and prompts removing the exclusion. +""" + +from __future__ import annotations + +import base64 +import copy +import dataclasses +import importlib.util +import json +import traceback +import warnings +from functools import partial +from typing import TYPE_CHECKING, Any, Final + +import pytest +from hypothesis import event, given, reject +from hypothesis import strategies as st +from zarr_metadata.model import read_array_metadata_v3 +from zarr_metadata.testing.strategies import ( + DEFAULT_STRATEGIES, + ChunkGridStrategy, + DataTypeStrategy, + array_documents, +) +from zarr_metadata.v3.chunk_grid import RECTILINEAR_CHUNK_GRID +from zarr_metadata.v3.codec import CAST_VALUE_CODEC +from zarr_metadata.v3.data_type import BYTES_DATA_TYPE, RAW_BYTES_DATA_TYPE +from zarr_metadata.v3.definition import ChunkGridDefinition, CodecDefinition, DataTypeDefinition + +import zarr +from zarr.abc.codec import ArrayArrayCodec +from zarr.core.metadata.v3 import ArrayV3Metadata +from zarr.registry import get_codec_class + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + + from zarr_metadata.testing.strategies import Configurations, Strategies + +_HAS_CAST_VALUE_RS: Final = importlib.util.find_spec("cast_value_rs") is not None + +_DIVERGENCES: Final = { + "data type spellings": ( + "A data type written as an object, such as `{'name': 'int8'}` or one with " + "`must_understand`, is refused. Documents are drawn in canonical spelling." + ), + "raw bits": "The core `r*` data types are not implemented. None are drawn.", + "bytes fill value": ( + "A `bytes` fill value written as an array of integers is refused. Only the " + "base64 spelling is drawn." + ), + "sharding behind transpose": ( + "A sharding codec after a `transpose` is validated against the untransposed " + "chunk. Such documents are rejected from the draw." + ), + "cast_value hex scalar map": ( + "A `scalar_map` value written as a hex string is refused while the fill value " + "is encoded, so the property counts it as a fill value rejection. Not excluded." + ), + # zarr-metadata's side: it follows the spec as of zarr-extensions 4da7b37a, before + # https://github.com/zarr-developers/zarr-extensions/pull/78 required at least one + # edge. zarr-python refuses an empty list, as the spec now says. + "empty rectilinear edges": ( + "zarr-metadata accepts an empty list of edge lengths for an axis of length 0. " + "No such grid is drawn." + ), +} + + +def _base64_fill_values( + _configuration: Mapping[str, Any], _strategies: Strategies +) -> st.SearchStrategy[str]: + return st.binary(max_size=16).map(lambda data: base64.standard_b64encode(data).decode()) + + +def _has_edges(configuration: Mapping[str, Any]) -> bool: + return all(dimension != () for dimension in configuration["chunk_shapes"]) + + +def _with_edges( + configurations: Callable[[tuple[int, ...] | None, Strategies], Configurations], + shape: tuple[int, ...] | None, + strategies: Strategies, +) -> Configurations: + return configurations(shape, strategies).filter(_has_edges) + + +def _supported(strategies: Strategies) -> Strategies: + """`strategies` without the known divergences in `_DIVERGENCES`.""" + out = { + definition: strategy + for definition, strategy in strategies.items() + if definition is not RAW_BYTES_DATA_TYPE + and (_HAS_CAST_VALUE_RS or definition is not CAST_VALUE_CODEC) + } + bytes_strategy = out.get(BYTES_DATA_TYPE) + if isinstance(bytes_strategy, DataTypeStrategy): + out[BYTES_DATA_TYPE] = dataclasses.replace(bytes_strategy, fill_values=_base64_fill_values) + grid_strategy = out.get(RECTILINEAR_CHUNK_GRID) + if isinstance(grid_strategy, ChunkGridStrategy): + out[RECTILINEAR_CHUNK_GRID] = dataclasses.replace( + grid_strategy, configurations=partial(_with_edges, grid_strategy.configurations) + ) + return out + + +_VALUE_DATA_TYPES: Final = frozenset( + {"int8", "uint8", "int16", "uint16", "int32", "uint32", "float32", "float64"} +) +_VALUE_CODECS: Final = frozenset( + {"bytes", "cast_value", "scale_offset", "sharding_indexed", "transpose"} +) + + +def _value_transforming(strategies: Strategies) -> Strategies: + """Integer and float data types, regular grids, and the codecs that change values. + + In this slice a fill value is often one the codecs cannot encode, which is where + zarr-python's one deliberate refusal is decided. + """ + return { + definition: strategy + for definition, strategy in strategies.items() + if not ( + isinstance(definition, DataTypeDefinition) and definition.name not in _VALUE_DATA_TYPES + ) + and not (isinstance(definition, CodecDefinition) and definition.name not in _VALUE_CODECS) + and not (isinstance(definition, ChunkGridDefinition) and definition.name != "regular") + } + + +_DOCUMENTS: Final = { + "every-definition": array_documents( + spellings="canonical", strategies=_supported(DEFAULT_STRATEGIES) + ), + "value-transforming": array_documents( + shapes=st.lists(st.integers(1, 16), min_size=1, max_size=2).map(tuple), + spellings="canonical", + strategies=_value_transforming(_supported(DEFAULT_STRATEGIES)), + ), +} + + +def _codec_name(codec: str | Mapping[str, Any]) -> str: + return codec if isinstance(codec, str) else codec["name"] + + +def _is_array_array(codec: str | Mapping[str, Any]) -> bool: + return issubclass(get_codec_class(_codec_name(codec)), ArrayArrayCodec) + + +def _shard_behind_transpose(codecs: tuple[Any, ...]) -> bool: + """Whether the array's own codecs hold a transpose and a sharding codec, which follows it. + + Only the array's own codecs: inside a shard, each codec is validated against the + chunk the codecs before it leave. + """ + names = [_codec_name(codec) for codec in codecs] + return "transpose" in names and "sharding_indexed" in names + + +def _spliced(codecs: tuple[Any, ...]) -> list[Any]: + """The codecs up to the array -> bytes codec, each shard replaced by the codecs it holds. + + The codecs inside a shard see the fill value as the codecs before the shard leave + it, so these codecs can encode the same fill values as `codecs`. + """ + out: list[Any] = [] + for codec in codecs: + if _is_array_array(codec): + out.append(codec) + elif _codec_name(codec) == "sharding_indexed": + return out + _spliced(codec["configuration"]["codecs"]) + else: + return [*out, codec] + return out + + +def _verdict(document: dict[str, Any]) -> Exception | None: + """The error zarr-python refuses `document` with, or None if it accepts it.""" + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + ArrayV3Metadata.from_dict(document) + except Exception as error: # noqa: BLE001 + return error + return None + + +def _raised_encoding_fill_value(error: Exception) -> bool: + """Whether `error` was raised by a codec encoding the fill value.""" + frames = traceback.extract_tb(error.__traceback__) + return isinstance(error, ValueError) and any(f.name == "resolve_metadata" for f in frames) + + +@pytest.mark.parametrize("slice_", list(_DOCUMENTS)) +@given(data=st.data()) +def test_refuses_only_fill_values_the_codecs_cannot_encode( + slice_: str, data: st.DataObject +) -> None: + """zarr-python accepts each document, or refuses its fill value as the codecs + cannot encode it, and then refuses it too with the shards' codecs moved out of + the shards.""" + # As a reader sees it: zarr-metadata draws JSON arrays as tuples. + document = json.loads(json.dumps(data.draw(_DOCUMENTS[slice_]))) + if _shard_behind_transpose(document["codecs"]): + reject() # "sharding behind transpose" in _DIVERGENCES + with zarr.config.set({"array.rectilinear_chunks": True}): + error = _verdict(document) + if error is None: + event("accepted") + return + event("refused") + assert _raised_encoding_fill_value(error), ( + f"zarr-python refuses a document the spec allows: {error!r}" + ) + spliced = copy.deepcopy(document) + spliced["codecs"] = _spliced(document["codecs"]) + assert _verdict(spliced) is not None, ( + f"zarr-python refuses the fill value inside a shard ({error!r}) but accepts it " + "with the shard's codecs outside the shard" + ) + + +def _document( + data_type: Any, fill_value: Any, codecs: list[Any], chunk_shape: tuple[int, ...] = (4,) +) -> dict[str, Any]: + return { + "zarr_format": 3, + "node_type": "array", + "shape": chunk_shape, + "data_type": data_type, + "fill_value": fill_value, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": chunk_shape}}, + "chunk_key_encoding": {"name": "default"}, + "codecs": codecs, + } + + +@pytest.mark.xfail(strict=True, raises=ValueError, reason=_DIVERGENCES["data type spellings"]) +@pytest.mark.parametrize( + "data_type", + [ + {"name": "int8"}, + {"name": "int8", "configuration": {}}, + {"name": "int8", "must_understand": True}, + ], +) +def test_data_type_object_spelling(data_type: dict[str, Any]) -> None: + ArrayV3Metadata.from_dict(_document(data_type, 0, [{"name": "bytes"}])) + + +@pytest.mark.xfail(strict=True, raises=ValueError, reason=_DIVERGENCES["raw bits"]) +def test_raw_bits_data_type() -> None: + ArrayV3Metadata.from_dict(_document("r16", [0, 0], [{"name": "bytes"}])) + + +@pytest.mark.xfail(strict=True, raises=TypeError, reason=_DIVERGENCES["bytes fill value"]) +def test_bytes_fill_value_as_integers() -> None: + ArrayV3Metadata.from_dict(_document("bytes", [1, 2], [{"name": "vlen-bytes"}])) + + +@pytest.mark.xfail(strict=True, raises=ValueError, reason=_DIVERGENCES["sharding behind transpose"]) +def test_sharding_behind_transpose() -> None: + # The transposed chunk is (4, 2), which inner chunks of (4, 1) divide. + sharding = { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (4, 1), + "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], + "index_codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], + }, + } + transpose = {"name": "transpose", "configuration": {"order": (1, 0)}} + ArrayV3Metadata.from_dict(_document("uint8", 0, [transpose, sharding], chunk_shape=(2, 4))) + + +@pytest.mark.skipif(not _HAS_CAST_VALUE_RS, reason="cast-value-rs not installed") +@pytest.mark.xfail(strict=True, raises=ValueError, reason=_DIVERGENCES["cast_value hex scalar map"]) +def test_cast_value_scalar_map_hex_float() -> None: + cast = { + "name": "cast_value", + "configuration": { + "data_type": "float32", + "scalar_map": {"encode": [[1, "0x7fc00000"]]}, + }, + } + ArrayV3Metadata.from_dict( + _document("uint8", 0, [cast, {"name": "bytes", "configuration": {"endian": "little"}}]) + ) + + +@pytest.mark.xfail( + strict=True, raises=AssertionError, reason=_DIVERGENCES["empty rectilinear edges"] +) +def test_zarr_metadata_refuses_empty_rectilinear_edges() -> None: + document = _document("bool", False, [{"name": "bytes"}], chunk_shape=(0,)) + document["chunk_grid"] = { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": [[]]}, + } + assert len(read_array_metadata_v3(document).problems) != 0 diff --git a/uv.lock b/uv.lock index 8dbf06aab4..fc63b6245b 100644 --- a/uv.lock +++ b/uv.lock @@ -3862,6 +3862,7 @@ dev = [ { name = "towncrier" }, { name = "universal-pathlib" }, { name = "uv" }, + { name = "zarr-metadata", extra = ["testing"] }, ] docs = [ { name = "astroid" }, @@ -3900,6 +3901,7 @@ remote-tests = [ { name = "requests" }, { name = "s3fs" }, { name = "uv" }, + { name = "zarr-metadata", extra = ["testing"] }, ] test = [ { name = "coverage" }, @@ -3913,6 +3915,7 @@ test = [ { name = "pytest-reportlog" }, { name = "pytest-xdist" }, { name = "uv" }, + { name = "zarr-metadata", extra = ["testing"] }, ] [package.metadata] @@ -3966,6 +3969,7 @@ dev = [ { name = "towncrier", specifier = "==26.9.0" }, { name = "universal-pathlib" }, { name = "uv", specifier = "==0.13.7" }, + { name = "zarr-metadata", extras = ["testing"], specifier = "==0.8.0" }, ] docs = [ { name = "astroid", specifier = "==4.3.4" }, @@ -4002,6 +4006,7 @@ remote-tests = [ { name = "requests", specifier = "==2.34.2" }, { name = "s3fs", specifier = ">=2023.10.0" }, { name = "uv", specifier = "==0.13.7" }, + { name = "zarr-metadata", extras = ["testing"], specifier = "==0.8.0" }, ] test = [ { name = "coverage", specifier = "==7.17.2" }, @@ -4015,4 +4020,23 @@ test = [ { name = "pytest-reportlog", specifier = "==1.0.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, { name = "uv", specifier = "==0.13.7" }, + { name = "zarr-metadata", extras = ["testing"], specifier = "==0.8.0" }, +] + +[[package]] +name = "zarr-metadata" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/e2/8e1f15171c0a21e9fda74e17ba6ed5820dc2ee10e500d2594ceec4eaf1ff/zarr_metadata-0.8.0.tar.gz", hash = "sha256:cd91855317118014f84c670a6f73c568cafdc781750e4f071651084a73b5c71f", size = 607925, upload-time = "2026-11-13T08:15:50.448Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/da/f7d2d72d44a6bd4f361d9e6b4f81166fed0254ff3856573ca2949c0eb150/zarr_metadata-0.8.0-py3-none-any.whl", hash = "sha256:6cb20d7c96665dbf5ba2eaebe49f9a2f416a73b8f8dbb2e0ed95ece67cf5a0b6", size = 336024, upload-time = "2026-11-13T08:15:49.029Z" }, +] + +[package.optional-dependencies] +testing = [ + { name = "hypothesis" }, ] From a5497b81bfe9773233bed69640b1bceb9982a8e3 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 14:07:19 +0100 Subject: [PATCH 5/5] test: bound drawn rectilinear runs and keep zarr-metadata out of min_deps The conformance test exhausted memory on CI: zarr-python expands each run-length pair of rectilinear edges into a list, and zarr-metadata draws runs of up to millions of chunks. Draw no grid listing more than 10,000 chunks along an axis, and pin the divergence with a strict xfail test. The min_deps env could not resolve: zarr-metadata 0.8.0 needs typing-extensions>=4.16, and min_deps pins 4.14. Move zarr-metadata into its own `conformance` dependency group, which the test envs include and min_deps does not, and skip the module where it is not installed. Assisted-by: ClaudeCode:claude-opus-5-5 --- pyproject.toml | 9 +++-- tests/test_zarr_metadata_conformance.py | 46 ++++++++++++++++++++++--- uv.lock | 8 ++--- 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 633fcec4d4..8dce367304 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,7 +122,11 @@ test = [ "pytest-benchmark==5.3.1", "pytest-codspeed==5.0.4", "uv==0.13.7", - # tests/test_zarr_metadata_conformance.py draws array documents from its strategies. +] +# tests/test_zarr_metadata_conformance.py draws array documents from zarr-metadata's +# strategies. Its own group, because zarr-metadata needs a newer typing-extensions than +# the `min_deps` env pins; the test skips where it is not installed. +conformance = [ "zarr-metadata[testing]==0.8.0", ] remote-tests = [ @@ -158,6 +162,7 @@ docs = [ ] dev = [ {include-group = "test"}, + {include-group = "conformance"}, {include-group = "remote-tests"}, {include-group = "docs"}, "universal-pathlib", @@ -190,7 +195,7 @@ hooks.vcs.version-file = "src/zarr/_version.py" dependency-groups = ["dev"] [tool.hatch.envs.test] -dependency-groups = ["test"] +dependency-groups = ["test", "conformance"] [tool.hatch.envs.test.env-vars] diff --git a/tests/test_zarr_metadata_conformance.py b/tests/test_zarr_metadata_conformance.py index 5c0339238f..765eccb574 100644 --- a/tests/test_zarr_metadata_conformance.py +++ b/tests/test_zarr_metadata_conformance.py @@ -26,6 +26,10 @@ from typing import TYPE_CHECKING, Any, Final import pytest + +# Not installed in the `min_deps` env: see the `conformance` dependency group. +pytest.importorskip("zarr_metadata.testing") + from hypothesis import event, given, reject from hypothesis import strategies as st from zarr_metadata.model import read_array_metadata_v3 @@ -77,8 +81,15 @@ "zarr-metadata accepts an empty list of edge lengths for an axis of length 0. " "No such grid is drawn." ), + "expanded rectilinear runs": ( + "zarr-python expands each run-length pair of rectilinear edges into a list, so a " + "valid run of many chunks exhausts memory. No grid listing more than " + "`_MAX_LISTED_CHUNKS` chunks along an axis is drawn." + ), } +_MAX_LISTED_CHUNKS: Final = 10_000 + def _base64_fill_values( _configuration: Mapping[str, Any], _strategies: Strategies @@ -86,16 +97,27 @@ def _base64_fill_values( return st.binary(max_size=16).map(lambda data: base64.standard_b64encode(data).decode()) -def _has_edges(configuration: Mapping[str, Any]) -> bool: - return all(dimension != () for dimension in configuration["chunk_shapes"]) +def _listed_chunks(dimension: Any) -> int: + """How many chunks a dimension's edges list, counting each run; 0 for a bare length.""" + if isinstance(dimension, int): + return 0 + return sum(edge[1] if isinstance(edge, tuple | list) else 1 for edge in dimension) -def _with_edges( +def _readable(configuration: Mapping[str, Any]) -> bool: + """Whether zarr-python reads the rectilinear grid without a known divergence.""" + return all( + dimension != () and _listed_chunks(dimension) <= _MAX_LISTED_CHUNKS + for dimension in configuration["chunk_shapes"] + ) + + +def _readable_grids( configurations: Callable[[tuple[int, ...] | None, Strategies], Configurations], shape: tuple[int, ...] | None, strategies: Strategies, ) -> Configurations: - return configurations(shape, strategies).filter(_has_edges) + return configurations(shape, strategies).filter(_readable) def _supported(strategies: Strategies) -> Strategies: @@ -112,7 +134,7 @@ def _supported(strategies: Strategies) -> Strategies: grid_strategy = out.get(RECTILINEAR_CHUNK_GRID) if isinstance(grid_strategy, ChunkGridStrategy): out[RECTILINEAR_CHUNK_GRID] = dataclasses.replace( - grid_strategy, configurations=partial(_with_edges, grid_strategy.configurations) + grid_strategy, configurations=partial(_readable_grids, grid_strategy.configurations) ) return out @@ -313,3 +335,17 @@ def test_zarr_metadata_refuses_empty_rectilinear_edges() -> None: "configuration": {"kind": "inline", "chunk_shapes": [[]]}, } assert len(read_array_metadata_v3(document).problems) != 0 + + +@pytest.mark.xfail( + strict=True, raises=MemoryError, reason=_DIVERGENCES["expanded rectilinear runs"] +) +def test_rectilinear_run_is_not_expanded() -> None: + """A run of 2**62 chunks of length 2 is read without listing them.""" + document = _document("uint8", 0, [{"name": "bytes"}], chunk_shape=(2**63 - 1,)) + document["chunk_grid"] = { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": [[[2, 2**62]]]}, + } + with zarr.config.set({"array.rectilinear_chunks": True}): + ArrayV3Metadata.from_dict(document) diff --git a/uv.lock b/uv.lock index fc63b6245b..c111f2bddc 100644 --- a/uv.lock +++ b/uv.lock @@ -3830,6 +3830,9 @@ remote = [ ] [package.dev-dependencies] +conformance = [ + { name = "zarr-metadata", extra = ["testing"] }, +] dev = [ { name = "astroid" }, { name = "botocore" }, @@ -3901,7 +3904,6 @@ remote-tests = [ { name = "requests" }, { name = "s3fs" }, { name = "uv" }, - { name = "zarr-metadata", extra = ["testing"] }, ] test = [ { name = "coverage" }, @@ -3915,7 +3917,6 @@ test = [ { name = "pytest-reportlog" }, { name = "pytest-xdist" }, { name = "uv" }, - { name = "zarr-metadata", extra = ["testing"] }, ] [package.metadata] @@ -3937,6 +3938,7 @@ requires-dist = [ provides-extras = ["cast-value-rs", "cli", "gpu", "optional", "remote"] [package.metadata.requires-dev] +conformance = [{ name = "zarr-metadata", extras = ["testing"], specifier = "==0.8.0" }] dev = [ { name = "astroid", specifier = "==4.3.4" }, { name = "botocore" }, @@ -4006,7 +4008,6 @@ remote-tests = [ { name = "requests", specifier = "==2.34.2" }, { name = "s3fs", specifier = ">=2023.10.0" }, { name = "uv", specifier = "==0.13.7" }, - { name = "zarr-metadata", extras = ["testing"], specifier = "==0.8.0" }, ] test = [ { name = "coverage", specifier = "==7.17.2" }, @@ -4020,7 +4021,6 @@ test = [ { name = "pytest-reportlog", specifier = "==1.0.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, { name = "uv", specifier = "==0.13.7" }, - { name = "zarr-metadata", extras = ["testing"], specifier = "==0.8.0" }, ] [[package]]