diff --git a/changes/4633.bugfix.1.md b/changes/4633.bugfix.1.md new file mode 100644 index 0000000000..a8293c2cd6 --- /dev/null +++ b/changes/4633.bugfix.1.md @@ -0,0 +1 @@ +Creating a sharded array no longer fails when the codecs inside the shard cannot encode the data type's default value, for example a `scale_offset` filter that maps only part of the data type's range. The inner codecs are now checked against the array's fill value, which must still be encodable. diff --git a/changes/4633.bugfix.2.md b/changes/4633.bugfix.2.md new file mode 100644 index 0000000000..ca87b8037a --- /dev/null +++ b/changes/4633.bugfix.2.md @@ -0,0 +1 @@ +Creating an array no longer rejects a codec placed after a data-type-changing codec such as `cast_value` because its configuration would be invalid for the array's original data type. Each codec is now checked against the data type it receives. diff --git a/pyproject.toml b/pyproject.toml index 839b5fdef8..8dce367304 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,6 +123,12 @@ test = [ "pytest-codspeed==5.0.4", "uv==0.13.7", ] +# tests/test_zarr_metadata_conformance.py draws array documents from zarr-metadata's +# strategies. Its own group, because zarr-metadata needs a newer typing-extensions than +# the `min_deps` env pins; the test skips where it is not installed. +conformance = [ + "zarr-metadata[testing]==0.8.0", +] remote-tests = [ {include-group = "test"}, "fsspec>=2023.10.0", @@ -156,6 +162,7 @@ docs = [ ] dev = [ {include-group = "test"}, + {include-group = "conformance"}, {include-group = "remote-tests"}, {include-group = "docs"}, "universal-pathlib", @@ -188,7 +195,7 @@ hooks.vcs.version-file = "src/zarr/_version.py" dependency-groups = ["dev"] [tool.hatch.envs.test] -dependency-groups = ["test"] +dependency-groups = ["test", "conformance"] [tool.hatch.envs.test.env-vars] diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index d4ccf71cb9..e156d94c57 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -588,27 +588,47 @@ def to_dict(self) -> dict[str, JSON]: } def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: - """Thread the spec through the inner chain. + """Evolve and validate the inner chain against the chunks it encodes. - Each codec is evolved against the spec produced by the previous one. - Evolving every codec against the same unthreaded spec is the bug shape that - strips `BytesCodec.endian` behind a dtype-changing codec — and this - method runs on the real array-creation path, baking the damaged chain - into the evolved instance before the transform builders ever run. + Each inner codec is evolved against, validated against, and resolved through + the spec the codecs before it leave, as array metadata does for its own codecs, + with a regular grid of one inner chunk as its chunk grid. A sharding codec among + them is thereby checked against the chunk it splits. + + The spec is threaded for two reasons. A dtype-changing codec changes what the + codecs after it see: evolving every codec against the unthreaded spec strips + `BytesCodec.endian` behind one. And the fill value the inner chain must encode + is the array's fill value as the codecs before this one leave it, which is what + an unwritten inner chunk holds. Probing the chain with any other value, such as + the data type's default scalar, rejects chains that never see that value. Parameters ---------- array_spec - The base spec to be evolved as we thread it. + The spec of the shards this codec encodes. Returns ------- - This codec with the evolved code chain. - """ - from zarr.core.chunk_utils import evolve_codecs + This codec with the evolved codec chain. - shard_spec = self._get_chunk_spec(array_spec) - evolved_codecs = evolve_codecs(self.codecs, shard_spec) + Raises + ------ + ValueError + If an inner codec is incompatible with the chunk it encodes, or cannot + encode the fill value it receives. + """ + evolved: list[Codec] = [] + spec = self._get_chunk_spec(array_spec) + for codec in self.codecs: + evolved_codec = codec.evolve_from_array_spec(spec) + evolved_codec.validate( + shape=spec.shape, + dtype=spec.dtype, + chunk_grid=RegularChunkGridMetadata(chunk_shape=spec.shape), + ) + evolved.append(evolved_codec) + spec = evolved_codec.resolve_metadata(spec) + evolved_codecs = tuple(evolved) if evolved_codecs != self.codecs: return replace(self, codecs=evolved_codecs) return self @@ -642,28 +662,6 @@ def validate( f"Chunk edge length {edge} in dimension {i} is not " f"divisible by the shard's inner chunk size {inner}." ) - self._validate_inner_codecs(dtype) - - def _validate_inner_codecs(self, dtype: ZDType[TBaseDType, TBaseScalar]) -> None: - """Validate the codecs that encode each inner chunk, as array metadata validates - its codecs: each against the chunk the codecs before it leave it, a regular - grid of one inner chunk. A sharding codec among them is thereby checked against - the chunk it splits, which `validate` of the outer codec alone does not see. - """ - spec = ArraySpec( - shape=self.chunk_shape, - dtype=dtype, - fill_value=dtype.default_scalar(), - config=ArrayConfig.from_dict({}), - prototype=default_buffer_prototype(), - ) - for codec in self.codecs: - codec.validate( - shape=spec.shape, - dtype=spec.dtype, - chunk_grid=RegularChunkGridMetadata(chunk_shape=spec.shape), - ) - spec = codec.resolve_metadata(spec) def _get_inner_chunk_transform(self, shard_spec: ArraySpec) -> Any: """The synchronous transform for the inner codec chain. diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index 63d525ba2d..aa611d4289 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -604,8 +604,9 @@ def _validate_metadata(self) -> None: ) if self.fill_value is None: raise ValueError("`fill_value` is required.") - for codec in self.codecs: - codec.validate(shape=self.shape, dtype=self.data_type, chunk_grid=self.chunk_grid) + # The codecs were validated in `__init__`, each against the spec the codecs + # before it leave. Validating them again here against the array's own data type + # would reject a codec that only ever sees the output of a dtype-changing codec. @property def ndim(self) -> int: diff --git a/tests/test_codecs/test_cast_value.py b/tests/test_codecs/test_cast_value.py index d6c703767f..37d1cb385a 100644 --- a/tests/test_codecs/test_cast_value.py +++ b/tests/test_codecs/test_cast_value.py @@ -8,7 +8,7 @@ import zarr from tests.conftest import Expect, ExpectFail -from zarr.codecs import BytesCodec, TransposeCodec +from zarr.codecs import BytesCodec, ScaleOffset, TransposeCodec from zarr.codecs.cast_value import CAST_VALUE_RS_MIN_VERSION, CastValue from zarr.storage import MemoryStore @@ -610,3 +610,24 @@ def make_array(filters: list[Any]) -> Any: array[:] = data assert_array_equal(array[:], data) + + +@requires_cast_value_rs +@pytest.mark.parametrize("shards", [None, (8,)], ids=["unsharded", "sharded"]) +def test_codec_validated_against_the_dtype_it_sees(shards: tuple[int, ...] | None) -> None: + """A codec behind `cast_value` is validated against the cast's target data type, + not the array's: a `scale_offset` scale of 0.5 is representable in float32, which + the codec sees, though not in the array's int16.""" + store = MemoryStore() + data = np.arange(8, dtype="int16") + arr = zarr.create_array( + store, + shape=data.shape, + dtype=data.dtype, + chunks=(4,), + shards=shards, + filters=[CastValue(data_type="float32"), ScaleOffset(scale=0.5)], + compressors=None, + ) + arr[:] = data + assert_array_equal(zarr.open_array(store, mode="r")[:], data) diff --git a/tests/test_codecs/test_sharding.py b/tests/test_codecs/test_sharding.py index 5712c9576d..52e23164ad 100644 --- a/tests/test_codecs/test_sharding.py +++ b/tests/test_codecs/test_sharding.py @@ -19,6 +19,7 @@ BytesCodec, Crc32cCodec, GzipCodec, + ScaleOffset, ShardingCodec, TransposeCodec, ) @@ -1522,3 +1523,87 @@ def test_open_rejects_stored_nested_sharding_with_indivisible_inner_chunk_shape( doc["codecs"][0]["configuration"]["codecs"][0]["configuration"]["chunk_shape"] = (4, 7) with pytest.raises(ValueError, match="not divisible"): ArrayV3Metadata.from_dict(doc) + + +# Each layout pairs codecs with a fill value the inner chain can encode, as the codecs +# before it leave that value, and one it cannot. The accepted fill value is chosen so +# that neither the data type's default scalar (0) nor, where a filter precedes the +# sharding codec, the array's fill value as given would pass the inner chain. +_INNER_CHAIN_FILL_LAYOUTS: dict[str, tuple[dict[str, Any], int, int]] = { + # `scale_offset` inside the shard: the inner chain encodes [100, 65535]. + "inside-shard": ( + {"chunks": (4,), "shards": (16,), "filters": [ScaleOffset(offset=100)]}, + 100, + 0, + ), + # `scale_offset` before the sharding codec doubles the fill value, so the inner + # chain sees 120 for an array fill value of 60, and 80 for 40. Its own + # `scale_offset` encodes [100, 65535]. + "behind-filter": ( + { + "chunks": (16,), + "filters": [ScaleOffset(scale=2)], + "serializer": ShardingCodec( + chunk_shape=(4,), codecs=[ScaleOffset(offset=100), BytesCodec()] + ), + }, + 60, + 40, + ), + # The fill value reaches a sharding codec nested in another one. + "nested": ( + { + "chunks": (16,), + "serializer": ShardingCodec( + chunk_shape=(8,), + codecs=[ + ShardingCodec(chunk_shape=(4,), codecs=[ScaleOffset(offset=100), BytesCodec()]) + ], + ), + }, + 100, + 0, + ), +} + + +@pytest.mark.filterwarnings( + "ignore:Combining a `sharding_indexed` codec:zarr.errors.ZarrUserWarning" +) +@pytest.mark.parametrize("layout", _INNER_CHAIN_FILL_LAYOUTS) +def test_sharding_inner_chain_probed_with_array_fill_value(layout: str) -> None: + """A sharding codec's inner chain is validated against the array's fill value, as + the codecs before it leave it, not against the data type's default scalar + (https://github.com/zarr-developers/zarr-python/issues/4511). The array is created, + reads its fill value before any write, and round-trips a partial write.""" + codecs, fill_value, _ = _INNER_CHAIN_FILL_LAYOUTS[layout] + store = MemoryStore() + arr = zarr.create_array( + store, shape=(16,), dtype="uint16", fill_value=fill_value, compressors=None, **codecs + ) + expected = np.full(16, fill_value, dtype="uint16") + np.testing.assert_array_equal(arr[:], expected) + written = np.arange(110, 114, dtype="uint16") + arr[4:8] = written + expected[4:8] = written + np.testing.assert_array_equal(zarr.open_array(store, mode="r")[:], expected) + + +@pytest.mark.filterwarnings( + "ignore:Combining a `sharding_indexed` codec:zarr.errors.ZarrUserWarning" +) +@pytest.mark.parametrize("layout", _INNER_CHAIN_FILL_LAYOUTS) +def test_sharding_rejects_fill_value_inner_chain_cannot_encode(layout: str) -> None: + """An array whose fill value the inner chain cannot encode is rejected: an + unwritten inner chunk holds the fill value, so the array could not be read before + its first write.""" + codecs, _, fill_value = _INNER_CHAIN_FILL_LAYOUTS[layout] + with pytest.raises(ValueError, match="outside the range"): + zarr.create_array( + MemoryStore(), + shape=(16,), + dtype="uint16", + fill_value=fill_value, + compressors=None, + **codecs, + ) diff --git a/tests/test_zarr_metadata_conformance.py b/tests/test_zarr_metadata_conformance.py new file mode 100644 index 0000000000..765eccb574 --- /dev/null +++ b/tests/test_zarr_metadata_conformance.py @@ -0,0 +1,351 @@ +"""zarr-python's verdict on the array documents zarr-metadata's strategies draw. + +zarr-metadata's strategies draw documents that the spec allows. Each codec in them +is drawn for the data type and the chunk the codecs before it hand on, so new codecs +and nestings are covered as soon as zarr-metadata defines them. + +zarr-python refuses one kind of valid document on purpose: one whose fill value the +codecs cannot encode, because an unwritten chunk holds the fill value. A document +refused for any other reason is a divergence from the spec. + +`_DIVERGENCES` lists the known divergences. The strategies here leave out each one +they can, and a strict xfail test below pins each one, so that fixing it fails that +test and prompts removing the exclusion. +""" + +from __future__ import annotations + +import base64 +import copy +import dataclasses +import importlib.util +import json +import traceback +import warnings +from functools import partial +from typing import TYPE_CHECKING, Any, Final + +import pytest + +# Not installed in the `min_deps` env: see the `conformance` dependency group. +pytest.importorskip("zarr_metadata.testing") + +from hypothesis import event, given, reject +from hypothesis import strategies as st +from zarr_metadata.model import read_array_metadata_v3 +from zarr_metadata.testing.strategies import ( + DEFAULT_STRATEGIES, + ChunkGridStrategy, + DataTypeStrategy, + array_documents, +) +from zarr_metadata.v3.chunk_grid import RECTILINEAR_CHUNK_GRID +from zarr_metadata.v3.codec import CAST_VALUE_CODEC +from zarr_metadata.v3.data_type import BYTES_DATA_TYPE, RAW_BYTES_DATA_TYPE +from zarr_metadata.v3.definition import ChunkGridDefinition, CodecDefinition, DataTypeDefinition + +import zarr +from zarr.abc.codec import ArrayArrayCodec +from zarr.core.metadata.v3 import ArrayV3Metadata +from zarr.registry import get_codec_class + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + + from zarr_metadata.testing.strategies import Configurations, Strategies + +_HAS_CAST_VALUE_RS: Final = importlib.util.find_spec("cast_value_rs") is not None + +_DIVERGENCES: Final = { + "data type spellings": ( + "A data type written as an object, such as `{'name': 'int8'}` or one with " + "`must_understand`, is refused. Documents are drawn in canonical spelling." + ), + "raw bits": "The core `r*` data types are not implemented. None are drawn.", + "bytes fill value": ( + "A `bytes` fill value written as an array of integers is refused. Only the " + "base64 spelling is drawn." + ), + "sharding behind transpose": ( + "A sharding codec after a `transpose` is validated against the untransposed " + "chunk. Such documents are rejected from the draw." + ), + "cast_value hex scalar map": ( + "A `scalar_map` value written as a hex string is refused while the fill value " + "is encoded, so the property counts it as a fill value rejection. Not excluded." + ), + # zarr-metadata's side: it follows the spec as of zarr-extensions 4da7b37a, before + # https://github.com/zarr-developers/zarr-extensions/pull/78 required at least one + # edge. zarr-python refuses an empty list, as the spec now says. + "empty rectilinear edges": ( + "zarr-metadata accepts an empty list of edge lengths for an axis of length 0. " + "No such grid is drawn." + ), + "expanded rectilinear runs": ( + "zarr-python expands each run-length pair of rectilinear edges into a list, so a " + "valid run of many chunks exhausts memory. No grid listing more than " + "`_MAX_LISTED_CHUNKS` chunks along an axis is drawn." + ), +} + +_MAX_LISTED_CHUNKS: Final = 10_000 + + +def _base64_fill_values( + _configuration: Mapping[str, Any], _strategies: Strategies +) -> st.SearchStrategy[str]: + return st.binary(max_size=16).map(lambda data: base64.standard_b64encode(data).decode()) + + +def _listed_chunks(dimension: Any) -> int: + """How many chunks a dimension's edges list, counting each run; 0 for a bare length.""" + if isinstance(dimension, int): + return 0 + return sum(edge[1] if isinstance(edge, tuple | list) else 1 for edge in dimension) + + +def _readable(configuration: Mapping[str, Any]) -> bool: + """Whether zarr-python reads the rectilinear grid without a known divergence.""" + return all( + dimension != () and _listed_chunks(dimension) <= _MAX_LISTED_CHUNKS + for dimension in configuration["chunk_shapes"] + ) + + +def _readable_grids( + configurations: Callable[[tuple[int, ...] | None, Strategies], Configurations], + shape: tuple[int, ...] | None, + strategies: Strategies, +) -> Configurations: + return configurations(shape, strategies).filter(_readable) + + +def _supported(strategies: Strategies) -> Strategies: + """`strategies` without the known divergences in `_DIVERGENCES`.""" + out = { + definition: strategy + for definition, strategy in strategies.items() + if definition is not RAW_BYTES_DATA_TYPE + and (_HAS_CAST_VALUE_RS or definition is not CAST_VALUE_CODEC) + } + bytes_strategy = out.get(BYTES_DATA_TYPE) + if isinstance(bytes_strategy, DataTypeStrategy): + out[BYTES_DATA_TYPE] = dataclasses.replace(bytes_strategy, fill_values=_base64_fill_values) + grid_strategy = out.get(RECTILINEAR_CHUNK_GRID) + if isinstance(grid_strategy, ChunkGridStrategy): + out[RECTILINEAR_CHUNK_GRID] = dataclasses.replace( + grid_strategy, configurations=partial(_readable_grids, grid_strategy.configurations) + ) + return out + + +_VALUE_DATA_TYPES: Final = frozenset( + {"int8", "uint8", "int16", "uint16", "int32", "uint32", "float32", "float64"} +) +_VALUE_CODECS: Final = frozenset( + {"bytes", "cast_value", "scale_offset", "sharding_indexed", "transpose"} +) + + +def _value_transforming(strategies: Strategies) -> Strategies: + """Integer and float data types, regular grids, and the codecs that change values. + + In this slice a fill value is often one the codecs cannot encode, which is where + zarr-python's one deliberate refusal is decided. + """ + return { + definition: strategy + for definition, strategy in strategies.items() + if not ( + isinstance(definition, DataTypeDefinition) and definition.name not in _VALUE_DATA_TYPES + ) + and not (isinstance(definition, CodecDefinition) and definition.name not in _VALUE_CODECS) + and not (isinstance(definition, ChunkGridDefinition) and definition.name != "regular") + } + + +_DOCUMENTS: Final = { + "every-definition": array_documents( + spellings="canonical", strategies=_supported(DEFAULT_STRATEGIES) + ), + "value-transforming": array_documents( + shapes=st.lists(st.integers(1, 16), min_size=1, max_size=2).map(tuple), + spellings="canonical", + strategies=_value_transforming(_supported(DEFAULT_STRATEGIES)), + ), +} + + +def _codec_name(codec: str | Mapping[str, Any]) -> str: + return codec if isinstance(codec, str) else codec["name"] + + +def _is_array_array(codec: str | Mapping[str, Any]) -> bool: + return issubclass(get_codec_class(_codec_name(codec)), ArrayArrayCodec) + + +def _shard_behind_transpose(codecs: tuple[Any, ...]) -> bool: + """Whether the array's own codecs hold a transpose and a sharding codec, which follows it. + + Only the array's own codecs: inside a shard, each codec is validated against the + chunk the codecs before it leave. + """ + names = [_codec_name(codec) for codec in codecs] + return "transpose" in names and "sharding_indexed" in names + + +def _spliced(codecs: tuple[Any, ...]) -> list[Any]: + """The codecs up to the array -> bytes codec, each shard replaced by the codecs it holds. + + The codecs inside a shard see the fill value as the codecs before the shard leave + it, so these codecs can encode the same fill values as `codecs`. + """ + out: list[Any] = [] + for codec in codecs: + if _is_array_array(codec): + out.append(codec) + elif _codec_name(codec) == "sharding_indexed": + return out + _spliced(codec["configuration"]["codecs"]) + else: + return [*out, codec] + return out + + +def _verdict(document: dict[str, Any]) -> Exception | None: + """The error zarr-python refuses `document` with, or None if it accepts it.""" + try: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + ArrayV3Metadata.from_dict(document) + except Exception as error: # noqa: BLE001 + return error + return None + + +def _raised_encoding_fill_value(error: Exception) -> bool: + """Whether `error` was raised by a codec encoding the fill value.""" + frames = traceback.extract_tb(error.__traceback__) + return isinstance(error, ValueError) and any(f.name == "resolve_metadata" for f in frames) + + +@pytest.mark.parametrize("slice_", list(_DOCUMENTS)) +@given(data=st.data()) +def test_refuses_only_fill_values_the_codecs_cannot_encode( + slice_: str, data: st.DataObject +) -> None: + """zarr-python accepts each document, or refuses its fill value as the codecs + cannot encode it, and then refuses it too with the shards' codecs moved out of + the shards.""" + # As a reader sees it: zarr-metadata draws JSON arrays as tuples. + document = json.loads(json.dumps(data.draw(_DOCUMENTS[slice_]))) + if _shard_behind_transpose(document["codecs"]): + reject() # "sharding behind transpose" in _DIVERGENCES + with zarr.config.set({"array.rectilinear_chunks": True}): + error = _verdict(document) + if error is None: + event("accepted") + return + event("refused") + assert _raised_encoding_fill_value(error), ( + f"zarr-python refuses a document the spec allows: {error!r}" + ) + spliced = copy.deepcopy(document) + spliced["codecs"] = _spliced(document["codecs"]) + assert _verdict(spliced) is not None, ( + f"zarr-python refuses the fill value inside a shard ({error!r}) but accepts it " + "with the shard's codecs outside the shard" + ) + + +def _document( + data_type: Any, fill_value: Any, codecs: list[Any], chunk_shape: tuple[int, ...] = (4,) +) -> dict[str, Any]: + return { + "zarr_format": 3, + "node_type": "array", + "shape": chunk_shape, + "data_type": data_type, + "fill_value": fill_value, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": chunk_shape}}, + "chunk_key_encoding": {"name": "default"}, + "codecs": codecs, + } + + +@pytest.mark.xfail(strict=True, raises=ValueError, reason=_DIVERGENCES["data type spellings"]) +@pytest.mark.parametrize( + "data_type", + [ + {"name": "int8"}, + {"name": "int8", "configuration": {}}, + {"name": "int8", "must_understand": True}, + ], +) +def test_data_type_object_spelling(data_type: dict[str, Any]) -> None: + ArrayV3Metadata.from_dict(_document(data_type, 0, [{"name": "bytes"}])) + + +@pytest.mark.xfail(strict=True, raises=ValueError, reason=_DIVERGENCES["raw bits"]) +def test_raw_bits_data_type() -> None: + ArrayV3Metadata.from_dict(_document("r16", [0, 0], [{"name": "bytes"}])) + + +@pytest.mark.xfail(strict=True, raises=TypeError, reason=_DIVERGENCES["bytes fill value"]) +def test_bytes_fill_value_as_integers() -> None: + ArrayV3Metadata.from_dict(_document("bytes", [1, 2], [{"name": "vlen-bytes"}])) + + +@pytest.mark.xfail(strict=True, raises=ValueError, reason=_DIVERGENCES["sharding behind transpose"]) +def test_sharding_behind_transpose() -> None: + # The transposed chunk is (4, 2), which inner chunks of (4, 1) divide. + sharding = { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": (4, 1), + "codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], + "index_codecs": [{"name": "bytes", "configuration": {"endian": "little"}}], + }, + } + transpose = {"name": "transpose", "configuration": {"order": (1, 0)}} + ArrayV3Metadata.from_dict(_document("uint8", 0, [transpose, sharding], chunk_shape=(2, 4))) + + +@pytest.mark.skipif(not _HAS_CAST_VALUE_RS, reason="cast-value-rs not installed") +@pytest.mark.xfail(strict=True, raises=ValueError, reason=_DIVERGENCES["cast_value hex scalar map"]) +def test_cast_value_scalar_map_hex_float() -> None: + cast = { + "name": "cast_value", + "configuration": { + "data_type": "float32", + "scalar_map": {"encode": [[1, "0x7fc00000"]]}, + }, + } + ArrayV3Metadata.from_dict( + _document("uint8", 0, [cast, {"name": "bytes", "configuration": {"endian": "little"}}]) + ) + + +@pytest.mark.xfail( + strict=True, raises=AssertionError, reason=_DIVERGENCES["empty rectilinear edges"] +) +def test_zarr_metadata_refuses_empty_rectilinear_edges() -> None: + document = _document("bool", False, [{"name": "bytes"}], chunk_shape=(0,)) + document["chunk_grid"] = { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": [[]]}, + } + assert len(read_array_metadata_v3(document).problems) != 0 + + +@pytest.mark.xfail( + strict=True, raises=MemoryError, reason=_DIVERGENCES["expanded rectilinear runs"] +) +def test_rectilinear_run_is_not_expanded() -> None: + """A run of 2**62 chunks of length 2 is read without listing them.""" + document = _document("uint8", 0, [{"name": "bytes"}], chunk_shape=(2**63 - 1,)) + document["chunk_grid"] = { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": [[[2, 2**62]]]}, + } + with zarr.config.set({"array.rectilinear_chunks": True}): + ArrayV3Metadata.from_dict(document) diff --git a/uv.lock b/uv.lock index 8dbf06aab4..c111f2bddc 100644 --- a/uv.lock +++ b/uv.lock @@ -3830,6 +3830,9 @@ remote = [ ] [package.dev-dependencies] +conformance = [ + { name = "zarr-metadata", extra = ["testing"] }, +] dev = [ { name = "astroid" }, { name = "botocore" }, @@ -3862,6 +3865,7 @@ dev = [ { name = "towncrier" }, { name = "universal-pathlib" }, { name = "uv" }, + { name = "zarr-metadata", extra = ["testing"] }, ] docs = [ { name = "astroid" }, @@ -3934,6 +3938,7 @@ requires-dist = [ provides-extras = ["cast-value-rs", "cli", "gpu", "optional", "remote"] [package.metadata.requires-dev] +conformance = [{ name = "zarr-metadata", extras = ["testing"], specifier = "==0.8.0" }] dev = [ { name = "astroid", specifier = "==4.3.4" }, { name = "botocore" }, @@ -3966,6 +3971,7 @@ dev = [ { name = "towncrier", specifier = "==26.9.0" }, { name = "universal-pathlib" }, { name = "uv", specifier = "==0.13.7" }, + { name = "zarr-metadata", extras = ["testing"], specifier = "==0.8.0" }, ] docs = [ { name = "astroid", specifier = "==4.3.4" }, @@ -4016,3 +4022,21 @@ test = [ { name = "pytest-xdist", specifier = "==3.8.0" }, { name = "uv", specifier = "==0.13.7" }, ] + +[[package]] +name = "zarr-metadata" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/e2/8e1f15171c0a21e9fda74e17ba6ed5820dc2ee10e500d2594ceec4eaf1ff/zarr_metadata-0.8.0.tar.gz", hash = "sha256:cd91855317118014f84c670a6f73c568cafdc781750e4f071651084a73b5c71f", size = 607925, upload-time = "2026-11-13T08:15:50.448Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/da/f7d2d72d44a6bd4f361d9e6b4f81166fed0254ff3856573ca2949c0eb150/zarr_metadata-0.8.0-py3-none-any.whl", hash = "sha256:6cb20d7c96665dbf5ba2eaebe49f9a2f416a73b8f8dbb2e0ed95ece67cf5a0b6", size = 336024, upload-time = "2026-11-13T08:15:49.029Z" }, +] + +[package.optional-dependencies] +testing = [ + { name = "hypothesis" }, +]