From e3e1c420a7d12c1eccef19ff64a994dcaff38661 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 15:28:15 +0100 Subject: [PATCH 1/2] feat(zarr-metadata): compare each codec in its simplest spelling for the chunk it receives A definition's `canonical` reduces two spellings of a configuration to one when they mean the same thing wherever the declaration is used. It receives the configuration alone, so it cannot reduce spellings that mean the same thing only for some chunks. The `endian` of a `bytes` codec is one: it means nothing when each value takes one byte, and is required when a value takes several. `ArrayCodecDefinition` gains `chunk_canonical`, a function of the configuration, the nested declarations and the chunk, and `codec_for_chunk` returns the codec of a `Stage` in that spelling, with each codec of the pipelines it contains simplified for the chunk that one receives. The result is a declaration, so `declaration_key`, `canonical_of` and `refines` work on it as on any other. Its configuration is checked again: the definition must allow it, the chunk rules must find no problem with it, and each nested declaration must stay as it is. `bytes` declares a `chunk_canonical` that drops `endian` for a data type stored in single bytes, and a v3 array model compares, hashes and refines by its codecs as `codec_for_chunk` returns them. So an `int8` array is one array with either `endian` or with none, as zarr-python 3.0 and 3.1 write it, while a `bytes` codec after a `cast_value` to a multi-byte data type is still compared by its `endian`. Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/450.feature.md | 1 + packages/zarr-metadata/docs/extensions.md | 4 +- packages/zarr-metadata/docs/models.md | 7 + .../src/zarr_metadata/definition/__init__.py | 10 +- .../zarr_metadata/definition/_definition.py | 48 ++++++ .../src/zarr_metadata/model/_array_v3.py | 19 ++- .../src/zarr_metadata/v3/_definition.py | 28 +++ .../src/zarr_metadata/v3/_pipeline.py | 107 +++++++++++- .../src/zarr_metadata/v3/codec/bytes.py | 27 +++ .../src/zarr_metadata/v3/definition.py | 11 +- .../zarr-metadata/tests/model/test_array.py | 53 ++++++ .../tests/v3/test_definitions.py | 1 + .../zarr-metadata/tests/v3/test_pipelines.py | 159 +++++++++++++++++- 13 files changed, 462 insertions(+), 13 deletions(-) create mode 100644 packages/zarr-metadata/changes/450.feature.md diff --git a/packages/zarr-metadata/changes/450.feature.md b/packages/zarr-metadata/changes/450.feature.md new file mode 100644 index 0000000000..c314478857 --- /dev/null +++ b/packages/zarr-metadata/changes/450.feature.md @@ -0,0 +1 @@ +A v3 codec that receives an array can declare its simplest spelling for the chunk it receives, with `chunk_canonical` on `ArrayCodecDefinition`. It is a function of the configuration, the nested declarations and the `Chunk`, for two spellings that mean the same thing for one chunk and not for another, which `canonical` cannot tell from the configuration alone. `codec_for_chunk` in `zarr_metadata.v3.definition` returns the codec of a `Stage` in that spelling, with each codec of the pipelines it contains simplified for the chunk that one receives. The `bytes` codec declares one: it drops an `endian` when each value takes one byte. So two `ZarrV3ArrayMetadata` models that differ only in the `endian` of a `bytes` codec that receives values of one byte (`bool`, `int8`, `uint8` or `r8`) are now equal, hash alike and refine each other. A `bytes` codec that receives values of several bytes, as it does after a `cast_value` to `float32`, is compared by its `endian` as before. diff --git a/packages/zarr-metadata/docs/extensions.md b/packages/zarr-metadata/docs/extensions.md index fc314760c8..0aa9332c56 100644 --- a/packages/zarr-metadata/docs/extensions.md +++ b/packages/zarr-metadata/docs/extensions.md @@ -122,7 +122,9 @@ Other kinds declare more: - A codec is defined by the class of its kind, `ArrayArrayCodecDefinition`, `ArrayBytesCodecDefinition` or `BytesBytesCodecDefinition`. A codec that receives an array declares what it checks about the chunk it receives - (`chunk_rules`) and any pipelines it contains (`pipelines`), and an + (`chunk_rules`), its simplest spelling for that chunk + (`chunk_canonical`, as `bytes` drops an `endian` when each value takes + one byte) and any pipelines it contains (`pipelines`), and an array -> array codec also declares what it hands on (`transition`). A bytes -> bytes codec receives bytes, so it declares none of these. diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index ea1a4ed294..ad2a700f6c 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -220,6 +220,13 @@ These are each declaration, and the fill value read against its data type. - A v3 blosc codec with `noshuffle` [ignores `typesize`](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/blosc/index.html#configuration-parameters). So it is the same codec with or without one. +- A v3 `bytes` codec's `endian` orders the bytes of each value, and the + encoding of a value that takes one byte + [does not depend on it](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/bytes/index.html#configuration-parameters). + So in a `uint8` array, a `bytes` codec is the same codec with either + `endian`, or with none. The codec is compared for the chunk it + receives: after a `cast_value` to `float32`, the two `endian` values + are two codecs. A v2 document's [dtype](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#data-type-encoding), diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index f259f3d7fa..df2f4905a7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -214,7 +214,11 @@ def acme_lz4_rules( equivalent spelling, and declarations are compared by it. Each nested declaration is written in its own simplest spelling, then the definition's `canonical` is applied (blosc drops a `typesize` that `noshuffle` ignores, and a -rectilinear grid run-length encodes its chunk shapes). The envelope +rectilinear grid run-length encodes its chunk shapes). A v3 codec that +receives an array can have a simpler spelling still for the chunk it +receives, which its `chunk_canonical` declares, and an array model +compares its codecs in that spelling (see `zarr_metadata.v3.definition`). +The envelope uses the shortest form every reader accepts: a data type with nothing to configure is its bare name, and any other declaration is an object, `{"name": ...}`, because a Zarr v3.0 reader does not accept a short-hand @@ -259,8 +263,8 @@ def acme_lz4_rules( that has a member no checker reads (the error names the TypedDict that contains it); a `name` that is not a string; an attribute declared as a function (`rules`, `canonical`, `fill_value_rules`, `storage`, -`shape_rules`, `chunk_lengths`, `chunk_rules`, `transition`, -`pipelines`) that is not a function; a data type's `fill_value` that no +`shape_rules`, `chunk_lengths`, `chunk_rules`, `chunk_canonical`, +`transition`, `pipelines`) that is not a function; a data type's `fill_value` that no checker reads, or that contains a declaration, which a value of the data type never does; a codec `kind` that is not one of the three, or a `size` that is not `"static"` or `"dynamic"`; a function that is never diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 05eefcf761..8d3a30344e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1259,6 +1259,52 @@ def written_json(declaration: AcceptedDeclaration[Any] | UnclaimedDeclaration) - return kind._envelope_json(declaration.name, declaration.configuration) +def respelled( + declaration: AcceptedDeclaration[D], + configuration: Mapping[str, JSONValue], + nested: Nested, +) -> AcceptedDeclaration[D]: + """`declaration` with `configuration` as its configuration, and each declaration in `nested` in place of the nested declaration at its location. + + This builds the declaration that another spelling of `declaration` + reads as, without reading it again. The caller answers for the + spelling: `configuration` must be one that the definition allows and + that means the same thing, and it must keep each nested declaration + where it is. `nested` holds only the nested declarations to replace, + each one accepted or unclaimed, and each is written into the + configuration as its `to_json` writes it. A location in `nested` that + does not lead into `configuration` raises `TypeError`. The result is + written as `to_json` writes it, since no document wrote it. + """ + spelled: JSONValue = dict(configuration) + for loc, inner in nested.items(): + spelled = _replaced(spelled, loc, written_json(held(inner))) + kind = declaration.read_as + name = declaration.name + carried = kind._filed_under(name)[1] is not None + return AcceptedDeclaration( + json=kind._envelope_json(name, {} if carried else cast("Mapping[str, JSONValue]", spelled)), + name=name, + definition=declaration.definition, + configuration=cast("Mapping[str, JSONValue]", spelled), + nested={**declaration.nested, **nested}, + ) + + +def item_at(value: JSONValue, path: Loc) -> JSONValue | UNSET: + """The item of `value` at `path`; `UNSET` when `path` does not lead into `value`.""" + for step in path: + if isinstance(step, str): + if not isinstance(value, Mapping) or step not in value: + return UNSET + value = value[step] + else: + if not isinstance(value, tuple) or not 0 <= step < len(value): + return UNSET + value = value[step] + return value + + Nested: TypeAlias = Mapping[Loc, ResolvedDeclaration[Any]] """The declarations in a configuration, each as the scope read it, keyed by location in the configuration.""" @@ -1902,6 +1948,7 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "held", "in_document", "is_declaration", + "item_at", "kind_of", "named_configuration", "no_rules", @@ -1909,6 +1956,7 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "read_only", "resolve", "resolve_refined", + "respelled", "ruled", "spelled_canonically", "unchanged", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 627545859c..2ea2af7ccb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -78,7 +78,7 @@ chunk_grid_lengths, default_array_bytes_codec, ) -from zarr_metadata.v3._pipeline import Stage, read_pipeline +from zarr_metadata.v3._pipeline import Stage, codec_for_chunk, read_pipeline from zarr_metadata.v3.array import ( ZARR_V3_ARRAY_METADATA_STORE_KEY, ZarrV3ArrayMetadataJSON, @@ -780,7 +780,7 @@ def refines(self, other: ZarrV3ArrayMetadata) -> bool: (self.data_type, other.data_type), (self.chunk_grid, other.chunk_grid), (self.chunk_key_encoding, other.chunk_key_encoding), - *zip(self.codecs, other.codecs, strict=True), + *zip(codecs_for_chunks(self._reading), codecs_for_chunks(other._reading), strict=True), *zip(self.storage_transformers, other.storage_transformers, strict=True), *((self.extra_fields[key], other.extra_fields[key]) for key in self.extra_fields), ) @@ -926,8 +926,10 @@ def array_key_of( The key holds each declaration's `declaration_key`, and the fill value as JSON text: in its canonical spelling when a definition in scope read the - data type, and as written otherwise. Every other member is included - as it is, with JSON members as text. + data type, and as written otherwise. A codec is compared as + `codec_for_chunk` returns it for the chunk it receives, so two + spellings that mean the same for that chunk are one codec. Every + other member is included as it is, with JSON members as text. """ data_type = held(reading.data_type) fill_value = ( @@ -940,7 +942,7 @@ def array_key_of( json_text(fill_value), declaration_key(data_type), declaration_key(held(reading.chunk_grid)), - tuple(declaration_key(held(stage.codec)) for stage in reading.pipeline), + tuple(declaration_key(codec) for codec in codecs_for_chunks(reading)), declaration_key(held(reading.chunk_key_encoding)), members.dimension_names, json_text(members.attributes), @@ -949,6 +951,13 @@ def array_key_of( ) +def codecs_for_chunks( + reading: ZarrV3ArrayMetadataReading, +) -> tuple[AcceptedDeclaration[CodecDefinition[Any]] | UnclaimedDeclaration, ...]: + """The codecs of `reading`, a read that found no problems, in pipeline order: each as `codec_for_chunk` returns it for the chunk it receives.""" + return tuple(held(codec_for_chunk(stage)) for stage in reading.pipeline) + + def extra_fields_key(extra_fields: Mapping[str, ResolvedDeclaration[Any]]) -> tuple[object, ...]: """A key for what a document's extra fields mean: each key with its declaration's `declaration_key`, sorted by key.""" return tuple( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index 0d734d81e1..dc4a3135b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -56,6 +56,7 @@ Nested, Problems, ResolvedDeclaration, + T, WithFillValue, asked, is_declaration, @@ -94,6 +95,11 @@ def no_pipelines(*_: object) -> Mapping[str, Chunk]: return {} +def unchanged_for_chunk(configuration: T, nested: object, chunk: object) -> T: + """The simplest spelling of a codec whose configuration means the same for every chunk it receives: the configuration itself.""" + return configuration + + StorageClass = Literal["single_byte", "multi_byte", "variable_length"] """How a data type's values are stored: in single bytes, in several bytes at a time, or each value in as many bytes as it needs. @@ -415,10 +421,31 @@ class ArrayCodecDefinition(CodecDefinition[C]): inner codecs receive its inner chunks, and its index codecs receive the shard index. It is called whatever the chunk rules found, and must return only what is true in either case. + + Two spellings of a configuration can mean the same thing for one + chunk and different things for another. For example, the `endian` of + a `bytes` codec changes nothing when each value takes one byte, and + is required when a value takes several. `canonical` cannot reduce + such spellings to one, because it receives the configuration alone. + `chunk_canonical` can: it receives the configuration, the nested + declarations as the scope read them, and the chunk, and returns the + simplest configuration that means the same for that chunk. It is + called only for a codec that the chunk rules found no problem with. + It simplifies only the codec's own members, and it must keep each + nested declaration as it is, because each codec of a pipeline the + codec contains is simplified for the chunk it receives in turn. What + does not depend on the chunk is left to `canonical`, which simplifies + the result as it simplifies any configuration. A chunk rule checks + only the parts of a chunk that are known, and `chunk_canonical` + simplifies only by them: a chunk with nothing known about it, + `Chunk()`, leaves every configuration as it is. `codec_for_chunk` + returns the codec of a `Stage` in that spelling. """ chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules """The spec violations for this codec with a given input chunk, located in the configuration. A problem at `()` is about the codec as a whole, such as an input it cannot encode, and is reported at the codec.""" + chunk_canonical: Callable[[C, Nested, Chunk], C] = unchanged_for_chunk + """A configuration the chunk rules allow for a given input chunk, in the simplest spelling that means the same for that chunk.""" pipelines: Callable[[C, Nested, Chunk], Mapping[str, Chunk]] = no_pipelines """The chunk each contained pipeline receives, keyed by the configuration member that holds the pipeline.""" @@ -879,6 +906,7 @@ def chunk_grid_lengths( "no_pipelines", "single_byte", "storage_of", + "unchanged_for_chunk", "unknown_chunk", "unknown_lengths", "unknown_storage", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index 592bb18a5f..9842d827da 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -33,13 +33,23 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, TypeGuard, cast -from zarr_metadata._json import ValidationProblem, is_object, is_tuple, with_input +from zarr_metadata._json import ( + ValidationProblem, + copied, + is_object, + is_tuple, + json_text, + with_input, +) +from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._definition import ( AcceptedDeclaration, ResolvedDeclaration, asked, in_document, + item_at, read_only, + respelled, ruled, ) from zarr_metadata.v3._definition import ( @@ -53,6 +63,7 @@ if TYPE_CHECKING: from collections.abc import Iterator, Mapping, Sequence + from zarr_metadata._common import JSONValue from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import ( Nested, @@ -326,4 +337,96 @@ def _handed_on( return given -__all__ = ["Stage", "read_pipeline"] +def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: + """The codec of `stage`, in the simplest spelling that means the same for the chunk it receives. + + A codec's own `canonical` reduces the spellings that mean the same + thing wherever the codec is used. Some spellings mean the same thing + only for some chunks. For example, a `bytes` codec with and without + `endian` is one codec for values that take one byte, and two for + values that take several. So a codec is compared, and written in its + simplest spelling, as the codec this function returns: first each + codec of each pipeline it contains, for the chunk that codec + receives, and then its own configuration, as its definition's + `chunk_canonical` simplifies it for the chunk in `stage`. + + A codec the scope did not read is returned as it is. A codec's own + configuration is kept as it is when the codec receives bytes, and + when its chunk rules find a problem with it, because a codec with a + problem has no simpler spelling. When nothing is simplified, the + result is the codec itself. Otherwise it is a new declaration, read + by the same definition. `canonical_of` writes it in its simplest + spelling, and `==` compares it. + + `chunk_canonical` is the extension author's code, so its result is + checked: the definition must allow it, the chunk rules must find no + problem with it, and each nested declaration must be where it was. A + result that fails a check raises `ValueError`, which indicates a bug + in the definition, not in the codec. + """ + codec = stage.codec + if not isinstance(codec, AcceptedDeclaration): + return codec + inner: dict[Loc, ResolvedDeclaration[Any]] = {} + for member, stages in stage.inner.items(): + for index, each in enumerate(stages): + simpler = codec_for_chunk(each) + if simpler is not each.codec: + inner[(member, index)] = simpler + configuration = _for_chunk(codec, stage.incoming) + if configuration is None and len(inner) == 0: + return codec + return respelled(codec, codec.configuration if configuration is None else configuration, inner) + + +def _for_chunk( + codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None +) -> Mapping[str, JSONValue] | None: + """The configuration of `codec` as its definition's `chunk_canonical` simplifies it for `chunk`, checked; None when nothing is simplified.""" + definition = codec.definition + if chunk is None or not isinstance(definition, ArrayCodecDefinition): + return None + configuration, nested = codec.configuration, codec.nested + if len(_chunk_problems(definition, configuration, nested, chunk, ())) != 0: + return None + given: object = asked( + definition, + "chunk_canonical", + lambda: cast("object", definition.chunk_canonical(read_only(configuration), nested, chunk)), + ) + if given is configuration: + return None + if not is_object(given): + msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}" + raise TypeError(msg) + simplified = cast("dict[str, JSONValue]", copied(cast("JSONValue", dict(given)))) + if json_text(simplified) == json_text(cast("JSONValue", configuration)): + return None + _, refused = definition.read_configuration(simplified) + found = ( + *refused, + *_chunk_problems(definition, simplified, nested, chunk, ()), + *( + ValidationProblem( + loc, "expected the nested declaration to stay as it is", "invalid_value" + ) + for loc in nested + if _text_at(simplified, loc) != _text_at(cast("JSONValue", configuration), loc) + ), + ) + if len(found) != 0: + msg = ( + f"{definition.name!r}: its chunk_canonical gave {simplified!r}, which does not hold: " + f"{list(found)!r}" + ) + raise ValueError(msg) + return simplified + + +def _text_at(configuration: JSONValue, loc: Loc) -> str | None: + """The JSON text of the item of `configuration` at `loc`; None when `loc` does not lead into it.""" + item = item_at(configuration, loc) + return None if item is UNSET else json_text(item) + + +__all__ = ["Stage", "codec_for_chunk", "read_pipeline"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 5b08cf3b0a..ca30fa46e2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -64,11 +64,38 @@ def _chunk_rules( ) +def _chunk_canonical( + configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk +) -> BytesCodecConfiguration: + """Remove `endian` when each value takes one byte, because it has no meaning there. + + `endian` is "Required for data types for which endianness is + applicable" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L64-L69), + and the encoding of a value that takes one byte does "not depend on + endian" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L84-L101). + So for such a data type, a `bytes` codec with either `endian`, or + with none, encodes the same bytes, and the three spellings describe + the same codec. zarr-python 3.0 wrote `endian` for these data types, + and zarr-python 3.1 does not. The storage is the one the data type + declares. If it is unknown, the configuration is kept as written. + """ + if chunk.data_type is None or "endian" not in configuration: + return configuration + if storage_of(chunk.data_type) != "single_byte": + return configuration + simpler = configuration.copy() + del simpler["endian"] + return simpler + + BYTES_CODEC: Final = ArrayBytesCodecDefinition( name=BYTES_CODEC_NAME, configuration=BytesCodecConfiguration, size="static", chunk_rules=_chunk_rules, + chunk_canonical=_chunk_canonical, ) """The `bytes` codec.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py index c98d6460c1..9c20fac4a5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -90,6 +90,14 @@ pipeline. For example, a shard's inner codecs receive its inner chunks, and its index codecs receive the shard index. Each pipeline is read the same way, and its stages are stored as the codec's `Stage.inner`. +Two spellings of a codec can mean the same thing for one chunk and not +for another, which `canonical` cannot tell, since it receives the +configuration alone. Such a codec declares its simplest spelling for a +given chunk with `chunk_canonical`. For example, `bytes` drops an +`endian` when each value takes one byte. `codec_for_chunk` returns the +codec of a `Stage` in that spelling, with each codec of the pipelines +it contains simplified for the chunk that one receives. An array model +compares its codecs this way. The reader does not guess. After a codec the scope did not read, or after a codec that does not declare its output, the next codec receives a chunk with nothing known about it, `Chunk()`, which never causes a @@ -147,7 +155,7 @@ default_array_bytes_codec, storage_of, ) -from zarr_metadata.v3._pipeline import Stage +from zarr_metadata.v3._pipeline import Stage, codec_for_chunk __all__ = [ "CORE_AND_EXTENSIONS_CONTEXT", @@ -181,6 +189,7 @@ "ZarrV3Kind", "ZarrV3StaticCodecDeclaration", "ZarrV3StorageTransformerDeclaration", + "codec_for_chunk", "default_array_bytes_codec", "storage_of", ] diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 7fd62c5442..2d2ae6cf27 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -805,6 +805,59 @@ def test_two_models_are_one_array_when_their_declarations_read_the_same( assert models[0].to_json() != models[1].to_json() +_BIG = {"name": "bytes", "configuration": {"endian": "big"}} +_TO_UINT8 = {"name": "cast_value", "configuration": {"data_type": "uint8"}} +_TO_FLOAT32 = {"name": "cast_value", "configuration": {"data_type": "float32"}} + + +def _shard_of(*codecs: object) -> dict[str, object]: + return { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [1], + "codecs": list(codecs), + "index_codecs": [_LITTLE, {"name": "crc32c"}], + }, + } + + +@pytest.mark.parametrize( + ("data_type", "left", "right", "same"), + [ + ("uint8", [_LITTLE], [{"name": "bytes"}], True), + ("uint8", [_LITTLE], [_BIG], True), + ("uint16", [_LITTLE], [_BIG], False), + ("uint8", [_TO_FLOAT32, _LITTLE], [_TO_FLOAT32, _BIG], False), + ("uint16", [_TO_UINT8, _LITTLE], [_TO_UINT8, {"name": "bytes"}], True), + ("uint8", [_shard_of(_LITTLE)], [_shard_of({"name": "bytes"})], True), + ("uint16", [_shard_of(_LITTLE)], [_shard_of(_BIG)], False), + ], + ids=[ + "one-byte-endian-or-none", + "one-byte-either-endian", + "two-bytes-either-endian", + "cast-to-four-bytes", + "cast-to-one-byte", + "shard-of-one-byte", + "shard-of-two-bytes", + ], +) +def test_two_models_are_one_array_when_each_codec_means_the_same_for_its_chunk( + data_type: str, left: object, right: object, same: bool +) -> None: + """A codec is compared for the chunk it receives, as `codec_for_chunk` returns it: the `endian` of a `bytes` codec tells two arrays apart only where a value takes several bytes. `refines` agrees with `==`, and `to_json` writes each codec as given.""" + document = ZarrV3ArrayMetadata.create_default(shape=(2,), data_type=data_type).to_json() + models = [ + ZarrV3ArrayMetadata.from_json({**document, "codecs": codecs}) for codecs in (left, right) + ] + assert (models[0] == models[1]) is same + assert models[0].refines(models[1]) is same + assert models[1].refines(models[0]) is same + assert models[0].to_json() != models[1].to_json() + if same: + assert hash(models[0]) == hash(models[1]) + + @pytest.mark.parametrize( ("left", "right", "same"), [ diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index d88dcf639e..70ac04d5eb 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -1473,6 +1473,7 @@ def test_error_a_data_type_fill_value_holding_a_declaration() -> None: (ChunkGridDefinition, "shape_rules"), (ChunkGridDefinition, "chunk_lengths"), (ArrayArrayCodecDefinition, "chunk_rules"), + (ArrayArrayCodecDefinition, "chunk_canonical"), (ArrayArrayCodecDefinition, "transition"), (ArrayArrayCodecDefinition, "pipelines"), ], diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index 7fcab1c1d2..fe477dc8f0 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -8,15 +8,19 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +import dataclasses +from typing import TYPE_CHECKING, Any, cast import pytest from typing_extensions import TypedDict +from zarr_metadata._json import json_text from zarr_metadata.definition import ( + AcceptedDeclaration, EmptyConfiguration, Nested, ResolvedDeclaration, + canonical_of, resolve, ) from zarr_metadata.model import ( @@ -26,6 +30,7 @@ from zarr_metadata.v3._pipeline import ( read_pipeline, ) +from zarr_metadata.v3.codec.bytes import BYTES_CODEC, BytesCodecConfiguration from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, ArrayArrayCodecDefinition, @@ -37,6 +42,7 @@ ZarrV3CodecDeclaration, ZarrV3Context, ZarrV3DataTypeDeclaration, + codec_for_chunk, ) if TYPE_CHECKING: @@ -422,6 +428,7 @@ def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) ("kind", "attribute"), [ (BytesBytesCodecDefinition, "chunk_rules"), + (BytesBytesCodecDefinition, "chunk_canonical"), (BytesBytesCodecDefinition, "transition"), (BytesBytesCodecDefinition, "pipelines"), (ArrayBytesCodecDefinition, "transition"), @@ -545,3 +552,153 @@ def test_error_a_stage_s_inner_pipelines_are_read_only() -> None: stage.inner["codecs"] = () # pyright: ignore[reportIndexIssue] for again in (pickle.loads(pickle.dumps(stages)), copy.deepcopy(stages)): assert again == stages + + +BIG: JSONValue = {"name": "bytes", "configuration": {"endian": "big"}} + +BARE: JSONValue = {"name": "bytes"} + +CRC32C: JSONValue = {"name": "crc32c"} + + +def _shard(*codecs: JSONValue, index_codecs: tuple[JSONValue, ...] = (LITTLE, CRC32C)) -> JSONValue: + """A shard of inner chunks of 2 by 1 that encodes them with `codecs`, and its index with `index_codecs`.""" + return { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [2, 1], + "codecs": list(codecs), + "index_codecs": list(index_codecs), + }, + } + + +def _simplest(codecs: list[JSONValue], chunk: Chunk) -> list[JSONValue]: + """Each codec of a pipeline that receives `chunk`, in its simplest spelling for the chunk it receives.""" + stages, problems = _read(codecs, chunk) + assert problems == () + return [canonical_of(codec_for_chunk(stage), ()) for stage in stages] + + +@pytest.mark.parametrize( + ("data_type", "codecs", "simplest"), + [ + # Each value takes one byte, so `endian` means nothing. + ("uint8", [LITTLE], [BARE]), + ("int8", [BIG], [BARE]), + ("bool", [LITTLE], [BARE]), + ("r8", [LITTLE], [BARE]), + ("uint8", [BARE], [BARE]), + ("uint8", [_transpose(1, 0), LITTLE, GZIP], [_transpose(1, 0), BARE, GZIP]), + # Each value takes several bytes, so `endian` says how they are ordered. + ("uint16", [LITTLE], [LITTLE]), + ("float32", [BIG], [BIG]), + ("r16", [LITTLE], [LITTLE]), + # The codec is simplified for the chunk it receives, not for the array's data type. + ("uint8", [_cast("float32"), LITTLE], [_cast("float32"), LITTLE]), + ("uint16", [_cast("uint8"), BIG], [_cast("uint8"), BARE]), + # A shard's inner codecs receive the shard's data type, and its index codecs `uint64`. + ("uint8", [_shard(LITTLE)], [_shard(BARE)]), + ("uint16", [_shard(LITTLE)], [_shard(LITTLE)]), + ("uint8", [_shard(_shard(BIG))], [_shard(_shard(BARE))]), + ("uint8", [_cast("uint16"), _shard(LITTLE)], [_cast("uint16"), _shard(LITTLE)]), + ("uint16", [_shard(_cast("uint8"), LITTLE)], [_shard(_cast("uint8"), BARE)]), + # Nothing is known of the values, so nothing is simplified. + ("acme.t", [LITTLE], [LITTLE]), + ("uint8", ["acme.shuffle", LITTLE], [{"name": "acme.shuffle"}, LITTLE]), + ("uint8", ["acme.unclaimed", LITTLE], [{"name": "acme.unclaimed"}, LITTLE]), + ], +) +def test_a_codec_is_written_in_the_simplest_spelling_for_the_chunk_it_receives( + data_type: JSONValue, codecs: list[JSONValue], simplest: list[JSONValue] +) -> None: + """`codec_for_chunk` returns each codec as its `chunk_canonical` simplifies it for the chunk it receives, and each codec of a pipeline it contains for the chunk that one receives. A codec with nothing to simplify is returned as it is.""" + found = _simplest(codecs, _of(data_type)) + assert json_text(found) == json_text(simplest) + # The simplest spelling reads without a problem, and has nothing left to simplify. + assert json_text(_simplest(found, _of(data_type))) == json_text(simplest) + stages, _ = _read(simplest, _of(data_type)) + assert all(codec_for_chunk(stage) is stage.codec for stage in stages) + + +def test_a_codec_with_a_problem_for_its_chunk_is_not_simplified() -> None: + """A codec the chunk rules find a problem with is returned as it is, as is a codec the scope refused.""" + # `bytes` does not encode values that vary in size, and `gzip` requires a level. + stages, problems = _read([LITTLE, {"name": "gzip", "configuration": {}}], _of("string")) + assert [problem.loc for problem in problems] == [(1, "configuration", "level"), (0,)] + assert all(codec_for_chunk(stage) is stage.codec for stage in stages) + + +def _bytes_with(chunk_canonical: object) -> ResolvedDeclaration[CodecDefinition[Any]]: + """A `bytes` codec with `endian`, read by a definition whose `chunk_canonical` is `chunk_canonical`.""" + lying = dataclasses.replace(BYTES_CODEC, name="acme.bytes", chunk_canonical=chunk_canonical) + declaration = {"name": "acme.bytes", "configuration": {"endian": "little"}} + return resolve(declaration, CodecDefinition, CONTEXT.union(ZarrV3Context(codecs=[lying])))[0] + + +def _stage(codec: ResolvedDeclaration[CodecDefinition[Any]], chunk: Chunk) -> Stage: + stages, problems = read_pipeline([codec], chunk) + assert problems == () + return stages[0] + + +def test_error_a_chunk_canonical_the_definition_does_not_allow() -> None: + stage = _stage(_bytes_with(lambda configuration, nested, chunk: {"endian": "middle"}), CHUNK) + with pytest.raises(ValueError, match="'acme.bytes': its chunk_canonical gave"): + codec_for_chunk(stage) + + +def test_error_a_chunk_canonical_the_chunk_rules_do_not_allow() -> None: + # A value of `uint16` takes two bytes, so the chunk rules require `endian`. + stage = _stage(_bytes_with(lambda configuration, nested, chunk: {}), _of("uint16")) + with pytest.raises(ValueError, match="'acme.bytes': its chunk_canonical gave {}, which"): + codec_for_chunk(stage) + + +def test_error_a_chunk_canonical_that_gives_something_else() -> None: + stage = _stage(_bytes_with(lambda configuration, nested, chunk: "little"), CHUNK) + with pytest.raises(TypeError, match="'acme.bytes': its chunk_canonical gives a configuration"): + codec_for_chunk(stage) + + +def test_error_a_chunk_canonical_that_raises_says_whose_it_is() -> None: + def chunk_canonical( + configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk + ) -> BytesCodecConfiguration: + raise KeyError("endian") + + with pytest.raises(KeyError) as raised: + codec_for_chunk(_stage(_bytes_with(chunk_canonical), CHUNK)) + assert raised.value.__notes__ == ["raised by the chunk_canonical of 'acme.bytes'"] + + +def test_error_a_chunk_canonical_that_writes_to_the_configuration() -> None: + def chunk_canonical( + configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk + ) -> BytesCodecConfiguration: + del configuration["endian"] + return configuration + + with pytest.raises(TypeError, match="does not support item deletion"): + codec_for_chunk(_stage(_bytes_with(chunk_canonical), CHUNK)) + + +def test_error_a_chunk_canonical_that_changes_a_nested_declaration() -> None: + # Each codec of a pipeline the codec contains is simplified for the + # chunk it receives, so the codec's own `chunk_canonical` leaves it alone. + def chunk_canonical( + configuration: AcmeHolderConfiguration, nested: Nested, chunk: Chunk + ) -> AcmeHolderConfiguration: + return {"codecs": (cast("ZarrV3CodecDeclaration", BARE),), "types": configuration["types"]} + + holder = ArrayBytesCodecDefinition( + name="acme.holder", + configuration=AcmeHolderConfiguration, + size="dynamic", + chunk_canonical=chunk_canonical, + ) + declaration = {"name": "acme.holder", "configuration": {"codecs": [LITTLE], "types": ["uint8"]}} + codec = resolve(declaration, CodecDefinition, CONTEXT.union(ZarrV3Context(codecs=[holder])))[0] + assert isinstance(codec, AcceptedDeclaration) + with pytest.raises(ValueError, match="expected the nested declaration to stay as it is"): + codec_for_chunk(_stage(codec, CHUNK)) From 070840fc5f72fbcf01edefc5dc638461566771af Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 16:29:23 +0100 Subject: [PATCH 2/2] docs(zarr-metadata): name the changelog fragment after the upstream PR Assisted-by: ClaudeCode:claude-fable-5-2 --- .../zarr-metadata/changes/{450.feature.md => 4640.feature.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{450.feature.md => 4640.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/450.feature.md b/packages/zarr-metadata/changes/4640.feature.md similarity index 100% rename from packages/zarr-metadata/changes/450.feature.md rename to packages/zarr-metadata/changes/4640.feature.md