From 6af625bbe6cb728bf74ea83ffe88349800f7d196 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 17 Nov 2026 22:27:20 +0100 Subject: [PATCH 1/3] fix(zarr-metadata): simplify a codec for its chunk in the spelling its canonical gives chunk_canonical received the configuration as written, and canonical ran afterwards. A codec whose canonical output its chunk_canonical reduces further then had two keys for one meaning: array models that differed only in such spellings were unequal, and the simplest spelling was not a fixed point. _for_chunk now hands chunk_canonical the configuration as canonical spells it, and checks that chunk_canonical has nothing left to simplify in its own result. A definition that leaves chunk_canonical at its default is skipped without calling either. Assisted-by: ClaudeCode:claude-fable-5-2 --- .../src/zarr_metadata/v3/_definition.py | 9 ++- .../src/zarr_metadata/v3/_pipeline.py | 79 +++++++++++++++---- .../zarr-metadata/tests/v3/test_pipelines.py | 59 +++++++++++++- 3 files changed, 126 insertions(+), 21 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index dc4a3135b1..34525f27c9 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -434,8 +434,11 @@ class ArrayCodecDefinition(CodecDefinition[C]): It simplifies only the codec's own members, and it must keep each nested declaration as it is, because each codec of a pipeline the codec contains is simplified for the chunk it receives in turn. What - does not depend on the chunk is left to `canonical`, which simplifies - the result as it simplifies any configuration. A chunk rule checks + does not depend on the chunk is left to `canonical`: `chunk_canonical` + receives the configuration as `canonical` spells it, and `canonical` + simplifies the result as it simplifies any configuration. The result + must be the simplest spelling, so `chunk_canonical` must change + nothing when it is asked about its own result. A chunk rule checks only the parts of a chunk that are known, and `chunk_canonical` simplifies only by them: a chunk with nothing known about it, `Chunk()`, leaves every configuration as it is. `codec_for_chunk` @@ -445,7 +448,7 @@ class ArrayCodecDefinition(CodecDefinition[C]): chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules """The spec violations for this codec with a given input chunk, located in the configuration. A problem at `()` is about the codec as a whole, such as an input it cannot encode, and is reported at the codec.""" chunk_canonical: Callable[[C, Nested, Chunk], C] = unchanged_for_chunk - """A configuration the chunk rules allow for a given input chunk, in the simplest spelling that means the same for that chunk.""" + """A configuration the chunk rules allow for a given input chunk, as `canonical` spells it, in the simplest spelling that means the same for that chunk.""" pipelines: Callable[[C, Nested, Chunk], Mapping[str, Chunk]] = no_pipelines """The chunk each contained pipeline receives, keyed by the configuration member that holds the pipeline.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index 9842d827da..6c3351a0b6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -58,6 +58,7 @@ Chunk, CodecDefinition, CodecKind, + unchanged_for_chunk, ) if TYPE_CHECKING: @@ -349,6 +350,9 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: codec of each pipeline it contains, for the chunk that codec receives, and then its own configuration, as its definition's `chunk_canonical` simplifies it for the chunk in `stage`. + `chunk_canonical` receives the configuration as the definition's + `canonical` spells it, so two spellings that `canonical` reduces to + one are simplified for the chunk in the same way. A codec the scope did not read is returned as it is. A codec's own configuration is kept as it is when the codec receives bytes, and @@ -360,9 +364,10 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: `chunk_canonical` is the extension author's code, so its result is checked: the definition must allow it, the chunk rules must find no - problem with it, and each nested declaration must be where it was. A - result that fails a check raises `ValueError`, which indicates a bug - in the definition, not in the codec. + problem with it, each nested declaration must be where it was, and + `chunk_canonical` must have nothing left to simplify in it. A result + that fails a check raises `ValueError`, which indicates a bug in the + definition, not in the codec. """ codec = stage.codec if not isinstance(codec, AcceptedDeclaration): @@ -382,25 +387,26 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: def _for_chunk( codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None ) -> Mapping[str, JSONValue] | None: - """The configuration of `codec` as its definition's `chunk_canonical` simplifies it for `chunk`, checked; None when nothing is simplified.""" + """The configuration of `codec` as its definition's `chunk_canonical` simplifies it for `chunk`, checked; None when nothing is simplified. + + `chunk_canonical` receives the configuration as the definition's + `canonical` spells it. So every spelling that `canonical` reduces to + one is simplified for the chunk in the same way, and `chunk_canonical` + does not have to know each of them. None is returned when + `chunk_canonical` changes nothing in that spelling, and the codec + then stays as written, for `canonical` to simplify as it simplifies + any configuration. + """ definition = codec.definition if chunk is None or not isinstance(definition, ArrayCodecDefinition): return None + if definition.chunk_canonical is unchanged_for_chunk: + return None configuration, nested = codec.configuration, codec.nested if len(_chunk_problems(definition, configuration, nested, chunk, ())) != 0: return None - given: object = asked( - definition, - "chunk_canonical", - lambda: cast("object", definition.chunk_canonical(read_only(configuration), nested, chunk)), - ) - if given is configuration: - return None - if not is_object(given): - msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}" - raise TypeError(msg) - simplified = cast("dict[str, JSONValue]", copied(cast("JSONValue", dict(given)))) - if json_text(simplified) == json_text(cast("JSONValue", configuration)): + simplified = _asked_for_chunk(definition, configuration, nested, chunk) + if simplified is None: return None _, refused = definition.read_configuration(simplified) found = ( @@ -420,6 +426,47 @@ def _for_chunk( f"{list(found)!r}" ) raise ValueError(msg) + # The simplest spelling has nothing left to simplify. Otherwise a + # document written from it would read as another codec. + further = _asked_for_chunk(definition, simplified, nested, chunk) + if further is not None: + msg = ( + f"{definition.name!r}: its chunk_canonical gave {simplified!r}, which is not the " + f"simplest spelling: asked again, it gave {further!r}" + ) + raise ValueError(msg) + return simplified + + +def _asked_for_chunk( + definition: ArrayCodecDefinition[Any], + configuration: Mapping[str, JSONValue], + nested: Nested, + chunk: Chunk, +) -> dict[str, JSONValue] | None: + """What `definition`'s `chunk_canonical` gives for `configuration` as its `canonical` spells it; None when it changes nothing in that spelling.""" + spelled = read_only( + asked( + definition, + "canonical", + lambda: dict( + cast("Mapping[str, JSONValue]", definition.canonical(read_only(configuration))) + ), + ) + ) + given: object = asked( + definition, + "chunk_canonical", + lambda: cast("object", definition.chunk_canonical(spelled, nested, chunk)), + ) + if given is spelled: + return None + if not is_object(given): + msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}" + raise TypeError(msg) + simplified = cast("dict[str, JSONValue]", copied(cast("JSONValue", dict(given)))) + if json_text(simplified) == json_text(cast("JSONValue", dict(spelled))): + return None return simplified diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index fe477dc8f0..920940fd92 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -9,7 +9,7 @@ from __future__ import annotations import dataclasses -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, NotRequired, cast import pytest from typing_extensions import TypedDict @@ -73,7 +73,39 @@ def _tiled(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) -> C ) """An array -> array codec that does not declare its output chunk.""" -CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(codecs=[ACME_TILE, ACME_SHUFFLE])) + +class AcmePackConfiguration(TypedDict, closed=True): + mode: NotRequired[str] + + +def _lower_mode(configuration: AcmePackConfiguration) -> AcmePackConfiguration: + """`mode` in lower case, since `acme.pack` reads it whatever its case.""" + if "mode" not in configuration: + return configuration + return {"mode": configuration["mode"].lower()} + + +def _mode_for_chunk( + configuration: AcmePackConfiguration, nested: Nested, chunk: Chunk +) -> AcmePackConfiguration: + """Without a `mode` of `"auto"`, which is what `acme.pack` does without a `mode` for a data type the scope read.""" + if isinstance(chunk.data_type, AcceptedDeclaration) and configuration.get("mode") == "auto": + return {} + return configuration + + +ACME_PACK = ArrayBytesCodecDefinition( + name="acme.pack", + configuration=AcmePackConfiguration, + size="static", + canonical=_lower_mode, + chunk_canonical=_mode_for_chunk, +) +"""An array -> bytes codec with both a `canonical` and a `chunk_canonical`: the second simplifies what the first writes.""" + +CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union( + ZarrV3Context(codecs=[ACME_TILE, ACME_SHUFFLE, ACME_PACK]) +) UINT8 = resolve("uint8", DataTypeDefinition, CONTEXT)[0] @@ -573,6 +605,10 @@ def _shard(*codecs: JSONValue, index_codecs: tuple[JSONValue, ...] = (LITTLE, CR } +def _pack(mode: str) -> JSONValue: + return {"name": "acme.pack", "configuration": {"mode": mode}} + + def _simplest(codecs: list[JSONValue], chunk: Chunk) -> list[JSONValue]: """Each codec of a pipeline that receives `chunk`, in its simplest spelling for the chunk it receives.""" stages, problems = _read(codecs, chunk) @@ -607,6 +643,11 @@ def _simplest(codecs: list[JSONValue], chunk: Chunk) -> list[JSONValue]: ("acme.t", [LITTLE], [LITTLE]), ("uint8", ["acme.shuffle", LITTLE], [{"name": "acme.shuffle"}, LITTLE]), ("uint8", ["acme.unclaimed", LITTLE], [{"name": "acme.unclaimed"}, LITTLE]), + # A codec is simplified for its chunk in the spelling its `canonical` gives it. + ("uint8", [_pack("auto")], [{"name": "acme.pack"}]), + ("uint8", [_pack("AUTO")], [{"name": "acme.pack"}]), + ("uint8", [_pack("FAST")], [_pack("fast")]), + ("acme.t", [_pack("AUTO")], [_pack("auto")]), ], ) def test_a_codec_is_written_in_the_simplest_spelling_for_the_chunk_it_receives( @@ -655,6 +696,20 @@ def test_error_a_chunk_canonical_the_chunk_rules_do_not_allow() -> None: codec_for_chunk(stage) +def test_error_a_chunk_canonical_with_something_left_to_simplify() -> None: + # Asked about `little` it gives `big`, and asked about `big` it gives `little`. + def chunk_canonical( + configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk + ) -> BytesCodecConfiguration: + return {"endian": "big" if configuration.get("endian") == "little" else "little"} + + stage = _stage(_bytes_with(chunk_canonical), CHUNK) + with pytest.raises( + ValueError, match="'acme.bytes': its chunk_canonical gave .* not the simplest" + ): + codec_for_chunk(stage) + + def test_error_a_chunk_canonical_that_gives_something_else() -> None: stage = _stage(_bytes_with(lambda configuration, nested, chunk: "little"), CHUNK) with pytest.raises(TypeError, match="'acme.bytes': its chunk_canonical gives a configuration"): From c5f7a4b9b0b004902076e1b9435bde5db474e13c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 17 Nov 2026 22:28:02 +0100 Subject: [PATCH 2/3] docs(zarr-metadata): add the changelog fragment Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/462.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/zarr-metadata/changes/462.bugfix.md diff --git a/packages/zarr-metadata/changes/462.bugfix.md b/packages/zarr-metadata/changes/462.bugfix.md new file mode 100644 index 0000000000..42ce128632 --- /dev/null +++ b/packages/zarr-metadata/changes/462.bugfix.md @@ -0,0 +1 @@ +A v3 codec's `chunk_canonical` receives the configuration as the codec's `canonical` spells it, so two spellings that `canonical` reduces to one are simplified for a chunk in the same way. Before, it received the configuration as written, and array models that differed only in such spellings were unequal. The result of `chunk_canonical` is also checked to have nothing left to simplify: a `chunk_canonical` that changes its own result raises `ValueError`. From cb67d4b33ec75a9a40f9ce80bede7123b7c54bc2 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 17 Nov 2026 23:27:27 +0100 Subject: [PATCH 3/3] fix(zarr-metadata): check what a codec's chunk_canonical gives as the document it becomes A review of #4640 found gaps in how _for_chunk checks the result of chunk_canonical: - a result that added a nested declaration passed, and the respelled codec then held a declaration its nested mapping did not; - a result that was not JSON raised an error that did not name the definition; - an array written as a list was reported as a changed nested declaration; - a problem inside a nested declaration, which is the document's, was held against the result. The result is now refined as any JSON is, its declaration locations are compared with the codec's, and problems under a nested location are left out. respelled refuses a location that holds no nested declaration with the TypeError its docstring promises. Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/462.bugfix.md | 2 +- packages/zarr-metadata/docs/models.md | 4 +- .../zarr_metadata/definition/_definition.py | 12 +- .../src/zarr_metadata/v3/_pipeline.py | 38 +++++-- .../tests/v3/test_definitions.py | 106 +++++++++++++++++- .../zarr-metadata/tests/v3/test_pipelines.py | 76 ++++++++++++- 6 files changed, 221 insertions(+), 17 deletions(-) diff --git a/packages/zarr-metadata/changes/462.bugfix.md b/packages/zarr-metadata/changes/462.bugfix.md index 42ce128632..e75570df6e 100644 --- a/packages/zarr-metadata/changes/462.bugfix.md +++ b/packages/zarr-metadata/changes/462.bugfix.md @@ -1 +1 @@ -A v3 codec's `chunk_canonical` receives the configuration as the codec's `canonical` spells it, so two spellings that `canonical` reduces to one are simplified for a chunk in the same way. Before, it received the configuration as written, and array models that differed only in such spellings were unequal. The result of `chunk_canonical` is also checked to have nothing left to simplify: a `chunk_canonical` that changes its own result raises `ValueError`. +A v3 codec's `chunk_canonical` receives the configuration as the codec's `canonical` spells it, so two spellings that `canonical` reduces to one are simplified for a chunk in the same way. Before, it received the configuration as written, and array models that differed only in such spellings were unequal. Its result is checked more closely: one that it would simplify again, or that holds a declaration the codec did not hold, raises `ValueError`, and one that is not JSON raises `TypeError` naming the definition. A result may write an array as a list, and a problem inside a nested declaration is no longer held against it. A codec that leaves `chunk_canonical` at its default is no longer copied each time an array model is compared or hashed. diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index ad2a700f6c..6f3c7f29b8 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -226,7 +226,9 @@ These are each declaration, and the fill value read against its data type. So in a `uint8` array, a `bytes` codec is the same codec with either `endian`, or with none. The codec is compared for the chunk it receives: after a `cast_value` to `float32`, the two `endian` values - are two codecs. + are two codecs. `codec_for_chunk` returns a codec in the spelling that + is compared, which `canonical_of` cannot give, since it reads the + codec alone. A v2 document's [dtype](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#data-type-encoding), diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index f59fea98a4..a8e08e673e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1291,11 +1291,15 @@ def respelled( where it is. `nested` holds only the nested declarations to replace, each one accepted or unclaimed, and each is written into the configuration as its `to_json` writes it. A location in `nested` that - does not lead into `configuration` raises `TypeError`. The result is + is not the location of a nested declaration of `declaration`, or that + does not lead into `configuration`, raises `TypeError`. The result is written as `to_json` writes it, since no document wrote it. """ spelled: JSONValue = dict(configuration) for loc, inner in nested.items(): + if loc not in declaration.nested or item_at(spelled, loc) is UNSET: + msg = f"{loc!r} is not the location of a nested declaration in {configuration!r}" + raise TypeError(msg) spelled = _replaced(spelled, loc, written_json(held(inner))) kind = declaration.read_as name = declaration.name @@ -1309,6 +1313,11 @@ def respelled( ) +def declaration_locations(definition: Definition[Any], configuration: JSONValue) -> tuple[Loc, ...]: + """The location of each declaration that `configuration` holds, where `definition`'s TypedDict types a member with a declaration alias, in order. `configuration` is JSON with arrays as tuples.""" + return tuple(found.loc for found in _typed(definition.configuration, configuration, ())[2]) + + def item_at(value: JSONValue, path: Loc) -> JSONValue | UNSET: """The item of `value` at `path`; `UNSET` when `path` does not lead into `value`.""" for step in path: @@ -1957,6 +1966,7 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "configuration_of", "declaration_key", "declaration_kind", + "declaration_locations", "declarations_of", "document_json", "filed_under", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index 6c3351a0b6..db1bf1ff9b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -35,10 +35,10 @@ from zarr_metadata._json import ( ValidationProblem, - copied, is_object, is_tuple, json_text, + refine_user_data, with_input, ) from zarr_metadata._sentinel import UNSET @@ -46,6 +46,7 @@ AcceptedDeclaration, ResolvedDeclaration, asked, + declaration_locations, in_document, item_at, read_only, @@ -364,10 +365,12 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: `chunk_canonical` is the extension author's code, so its result is checked: the definition must allow it, the chunk rules must find no - problem with it, each nested declaration must be where it was, and - `chunk_canonical` must have nothing left to simplify in it. A result - that fails a check raises `ValueError`, which indicates a bug in the - definition, not in the codec. + problem with it, it must hold the nested declarations the codec + holds, each as it was, and `chunk_canonical` must have nothing left + to simplify in it. A result that fails a check raises `ValueError`, + which indicates a bug in the definition, not in the codec. A problem + inside a nested declaration is the document's, and is not held + against the result. """ codec = stage.codec if not isinstance(codec, AcceptedDeclaration): @@ -409,8 +412,11 @@ def _for_chunk( if simplified is None: return None _, refused = definition.read_configuration(simplified) + held_at = frozenset(declaration_locations(definition, simplified)) found = ( - *refused, + # A problem inside a nested declaration is the document's: the + # declaration is checked to be as it was written. + *(problem for problem in refused if not _inside(problem.loc, nested)), *_chunk_problems(definition, simplified, nested, chunk, ()), *( ValidationProblem( @@ -419,6 +425,12 @@ def _for_chunk( for loc in nested if _text_at(simplified, loc) != _text_at(cast("JSONValue", configuration), loc) ), + *( + ValidationProblem( + loc, "expected no declaration that the codec did not hold", "invalid_value" + ) + for loc in sorted(held_at.difference(nested), key=repr) + ), ) if len(found) != 0: msg = ( @@ -438,13 +450,18 @@ def _for_chunk( return simplified +def _inside(loc: Loc, nested: Nested) -> bool: + """Whether `loc` is at or under the location of a declaration in `nested`.""" + return any(loc[: len(held_at)] == held_at for held_at in nested) + + def _asked_for_chunk( definition: ArrayCodecDefinition[Any], configuration: Mapping[str, JSONValue], nested: Nested, chunk: Chunk, ) -> dict[str, JSONValue] | None: - """What `definition`'s `chunk_canonical` gives for `configuration` as its `canonical` spells it; None when it changes nothing in that spelling.""" + """What `definition`'s `chunk_canonical` gives for `configuration` as its `canonical` spells it, as JSON with arrays as tuples; None when it changes nothing in that spelling.""" spelled = read_only( asked( definition, @@ -461,10 +478,13 @@ def _asked_for_chunk( ) if given is spelled: return None - if not is_object(given): + # The result is the extension author's value, so it is read as any JSON + # is: an array may be a list, and what is not JSON is refused here, + # naming the definition. `read_configuration` checks the rest. + simplified, not_json = refine_user_data(dict(given)) if is_object(given) else (None, ()) + if not isinstance(simplified, dict) or len(not_json) != 0: msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}" raise TypeError(msg) - simplified = cast("dict[str, JSONValue]", copied(cast("JSONValue", dict(given)))) if json_text(simplified) == json_text(cast("JSONValue", dict(spelled))): return None return simplified diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index 359275a18f..bcc2f3c9be 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -21,7 +21,7 @@ JSONValue, ValidationProblem, ) -from zarr_metadata._json import refine_json +from zarr_metadata._json import json_text, refine_json from zarr_metadata.definition import ( AcceptedDeclaration, Context, @@ -34,7 +34,7 @@ configuration_of, resolve, ) -from zarr_metadata.definition._definition import resolve_refined +from zarr_metadata.definition._definition import item_at, resolve_refined, respelled from zarr_metadata.model import ( UNSET, ZarrV3ArrayMetadata, @@ -1824,3 +1824,105 @@ def test_error_resolve_refuses_a_kind_of_the_other_format_whatever_the_declarati """`resolve` with a kind of one format and a scope of the other raises one `TypeError`, before it reads the declaration. So the message is the same for a declaration with a name that a scope could look up and for a value with no name.""" with pytest.raises(TypeError, match=f"^{re.escape(message)}$"): resolve(declaration, kind, context) + + +def _accepted(declaration: JSONValue, kind: type[Definition[Any]]) -> AcceptedDeclaration[Any]: + read, problems = resolve(declaration, kind, CORE_AND_EXTENSIONS_CONTEXT) + assert isinstance(read, AcceptedDeclaration) + assert problems == () + return read + + +def _sharding(**members: JSONValue) -> JSONValue: + return {"name": "sharding_indexed", "configuration": {"chunk_shape": [2], **members}} + + +@pytest.mark.parametrize( + ("declaration", "kind", "configuration", "nested", "written"), + [ + # A member of its own left out, and a nested declaration replaced. + ( + _sharding(codecs=[LE], index_codecs=[LE, CRC32C], index_location="end"), + CodecDefinition, + {"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, + {("codecs", 0): {"name": "bytes"}}, + _sharding(codecs=[{"name": "bytes"}], index_codecs=[LE, CRC32C]), + ), + # Nothing nested to replace. + ( + {"name": "gzip", "configuration": {"level": 1}}, + CodecDefinition, + {"level": 5}, + {}, + {"name": "gzip", "configuration": {"level": 5}}, + ), + # A name that carries its configuration is written alone. + ("r16", DataTypeDefinition, {"bits": 16}, {}, "r16"), + ], + ids=["shard", "gzip", "name-carries-configuration"], +) +def test_respelled_is_the_declaration_that_the_other_spelling_reads_as( + declaration: JSONValue, + kind: type[Definition[Any]], + configuration: JSONValue, + nested: Mapping[tuple[str | int, ...], JSONValue], + written: JSONValue, +) -> None: + """`respelled` returns what a read of the other spelling returns: the same JSON, name, configuration, nested declarations and key. The declaration it was given is left as it was.""" + read = _accepted(declaration, kind) + before = json_text(read.to_json()) + refined = cast("Mapping[str, JSONValue]", refine_json(configuration)[0]) + replacements = {loc: _accepted(inner, CodecDefinition) for loc, inner in nested.items()} + result = respelled(read, refined, replacements) + again = _accepted(written, kind) + assert json_text(result.json) == json_text(again.json) + assert json_text(result.to_json()) == json_text(again.to_json()) + assert result.name == again.name + assert json_text(cast("JSONValue", result.configuration)) == json_text( + cast("JSONValue", again.configuration) + ) + assert dict(result.nested) == dict(again.nested) + assert result == again + assert hash(result) == hash(again) + assert json_text(read.to_json()) == before + + +@pytest.mark.parametrize( + ("configuration", "loc"), + [ + ({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ("codecs", 5)), + ({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ("codecs", -1)), + ({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ("nope", 0)), + ({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ()), + # A location of a nested declaration that the new configuration does not have. + ({"chunk_shape": [2], "codecs": [], "index_codecs": [LE, CRC32C]}, ("codecs", 0)), + ], + ids=["past-the-end", "negative", "no-such-member", "the-configuration", "left-out"], +) +def test_error_respelled_at_a_location_that_holds_no_nested_declaration( + configuration: JSONValue, loc: tuple[str | int, ...] +) -> None: + read = _accepted(_sharding(codecs=[LE], index_codecs=[LE, CRC32C]), CodecDefinition) + refined = cast("Mapping[str, JSONValue]", refine_json(configuration)[0]) + with pytest.raises(TypeError, match="is not the location of a nested declaration"): + respelled(read, refined, {loc: _accepted({"name": "bytes"}, CodecDefinition)}) + + +@pytest.mark.parametrize( + ("path", "item"), + [ + ((), {"a": ({"b": 1}, 2)}), + (("a",), ({"b": 1}, 2)), + (("a", 0, "b"), 1), + (("a", 1), 2), + (("a", 2), UNSET), + (("a", -1), UNSET), + (("b",), UNSET), + (("a", "b"), UNSET), + ((0,), UNSET), + (("a", 1, "b"), UNSET), + ], +) +def test_item_at_follows_a_location_into_json(path: tuple[str | int, ...], item: object) -> None: + """`item_at` returns the item at a location, and `UNSET` when the location does not lead into the value: a key an object lacks, an index outside an array, or a step into a scalar.""" + assert item_at({"a": ({"b": 1}, 2)}, path) == item diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index 920940fd92..0eda1926b6 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -663,11 +663,64 @@ def test_a_codec_is_written_in_the_simplest_spelling_for_the_chunk_it_receives( def test_a_codec_with_a_problem_for_its_chunk_is_not_simplified() -> None: - """A codec the chunk rules find a problem with is returned as it is, as is a codec the scope refused.""" + """A codec the chunk rules find a problem with keeps its own configuration, and its `chunk_canonical` is not asked. A codec the scope refused is returned as it is.""" # `bytes` does not encode values that vary in size, and `gzip` requires a level. stages, problems = _read([LITTLE, {"name": "gzip", "configuration": {}}], _of("string")) assert [problem.loc for problem in problems] == [(1, "configuration", "level"), (0,)] assert all(codec_for_chunk(stage) is stage.codec for stage in stages) + # A `chunk_canonical` that would drop the `endian` whatever the chunk is not asked either. + eager = _bytes_with(lambda configuration, nested, chunk: {}) + stages, problems = read_pipeline([eager], _of("string")) + assert [problem.loc for problem in problems] == [(0,)] + assert codec_for_chunk(stages[0]) is eager + + +class AcmeLeveledConfiguration(TypedDict, closed=True): + codecs: tuple[ZarrV3CodecDeclaration, ...] + level: NotRequired[int] + + +def _leveled(chunk_canonical: object, *codecs: JSONValue) -> Stage: + """The stage, for `CHUNK`, of a codec `acme.leveled` that holds `codecs` and a `level` of 3, and whose `chunk_canonical` is `chunk_canonical`. It declares no pipelines, so the codecs it holds are left as they are.""" + leveled = ArrayBytesCodecDefinition( + name="acme.leveled", + configuration=AcmeLeveledConfiguration, + size="dynamic", + chunk_canonical=chunk_canonical, # pyright: ignore[reportArgumentType] + ) + declaration = {"name": "acme.leveled", "configuration": {"codecs": list(codecs), "level": 3}} + codec = resolve(declaration, CodecDefinition, CONTEXT.union(ZarrV3Context(codecs=[leveled])))[0] + return read_pipeline([codec], CHUNK)[0][0] + + +REFUSED_GZIP: JSONValue = {"name": "gzip", "configuration": {"level": 99}, "must_understand": False} +"""A codec with problems of its own: no such level, and a codec may not waive `must_understand`. A configuration that holds it keeps it as written.""" + + +@pytest.mark.parametrize( + ("codecs", "chunk_canonical"), + [ + # An array of the result may be a list. + ( + [LITTLE, GZIP], + lambda configuration, nested, chunk: {"codecs": list(configuration["codecs"])}, + ), + # A problem inside a nested declaration is the document's, not the result's. + ( + [LITTLE, REFUSED_GZIP], + lambda configuration, nested, chunk: {"codecs": configuration["codecs"]}, + ), + ], + ids=["an-array-as-a-list", "a-nested-declaration-with-a-problem"], +) +def test_a_chunk_canonical_that_simplifies_its_own_members_is_taken_at_its_word( + codecs: list[JSONValue], chunk_canonical: object +) -> None: + """A result that drops the codec's own `level` and keeps each nested declaration as it was written passes every check, however it spells its arrays and whatever problems the nested declarations have.""" + found = codec_for_chunk(_leveled(chunk_canonical, *codecs)) + assert isinstance(found, AcceptedDeclaration) + assert json_text(cast("JSONValue", found.configuration)) == json_text({"codecs": codecs}) + assert list(found.nested) == [("codecs", 0), ("codecs", 1)] def _bytes_with(chunk_canonical: object) -> ResolvedDeclaration[CodecDefinition[Any]]: @@ -710,12 +763,29 @@ def chunk_canonical( codec_for_chunk(stage) -def test_error_a_chunk_canonical_that_gives_something_else() -> None: - stage = _stage(_bytes_with(lambda configuration, nested, chunk: "little"), CHUNK) +@pytest.mark.parametrize( + "given", ["little", {"endian": {"little"}}, {1: "little"}], ids=["a-string", "a-set", "a-key"] +) +def test_error_a_chunk_canonical_that_gives_something_else(given: object) -> None: + """A result that is not an object of JSON is refused by name, before anything reads it.""" + stage = _stage(_bytes_with(lambda configuration, nested, chunk: given), CHUNK) with pytest.raises(TypeError, match="'acme.bytes': its chunk_canonical gives a configuration"): codec_for_chunk(stage) +def test_error_a_chunk_canonical_that_adds_a_nested_declaration() -> None: + # The codec it adds is one no scope read, so the result would hold a + # declaration that its `nested` does not. + def chunk_canonical( + configuration: AcmeLeveledConfiguration, nested: Nested, chunk: Chunk + ) -> AcmeLeveledConfiguration: + added = cast("ZarrV3CodecDeclaration", {"name": "crc32c"}) + return {"codecs": (*configuration["codecs"], added)} + + with pytest.raises(ValueError, match="expected no declaration that the codec did not hold"): + codec_for_chunk(_leveled(chunk_canonical, LITTLE, GZIP)) + + def test_error_a_chunk_canonical_that_raises_says_whose_it_is() -> None: def chunk_canonical( configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk