From e3e1c420a7d12c1eccef19ff64a994dcaff38661 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 15:28:15 +0100 Subject: [PATCH 1/6] feat(zarr-metadata): compare each codec in its simplest spelling for the chunk it receives A definition's `canonical` reduces two spellings of a configuration to one when they mean the same thing wherever the declaration is used. It receives the configuration alone, so it cannot reduce spellings that mean the same thing only for some chunks. The `endian` of a `bytes` codec is one: it means nothing when each value takes one byte, and is required when a value takes several. `ArrayCodecDefinition` gains `chunk_canonical`, a function of the configuration, the nested declarations and the chunk, and `codec_for_chunk` returns the codec of a `Stage` in that spelling, with each codec of the pipelines it contains simplified for the chunk that one receives. The result is a declaration, so `declaration_key`, `canonical_of` and `refines` work on it as on any other. Its configuration is checked again: the definition must allow it, the chunk rules must find no problem with it, and each nested declaration must stay as it is. `bytes` declares a `chunk_canonical` that drops `endian` for a data type stored in single bytes, and a v3 array model compares, hashes and refines by its codecs as `codec_for_chunk` returns them. So an `int8` array is one array with either `endian` or with none, as zarr-python 3.0 and 3.1 write it, while a `bytes` codec after a `cast_value` to a multi-byte data type is still compared by its `endian`. Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/450.feature.md | 1 + packages/zarr-metadata/docs/extensions.md | 4 +- packages/zarr-metadata/docs/models.md | 7 + .../src/zarr_metadata/definition/__init__.py | 10 +- .../zarr_metadata/definition/_definition.py | 48 ++++++ .../src/zarr_metadata/model/_array_v3.py | 19 ++- .../src/zarr_metadata/v3/_definition.py | 28 +++ .../src/zarr_metadata/v3/_pipeline.py | 107 +++++++++++- .../src/zarr_metadata/v3/codec/bytes.py | 27 +++ .../src/zarr_metadata/v3/definition.py | 11 +- .../zarr-metadata/tests/model/test_array.py | 53 ++++++ .../tests/v3/test_definitions.py | 1 + .../zarr-metadata/tests/v3/test_pipelines.py | 159 +++++++++++++++++- 13 files changed, 462 insertions(+), 13 deletions(-) create mode 100644 packages/zarr-metadata/changes/450.feature.md diff --git a/packages/zarr-metadata/changes/450.feature.md b/packages/zarr-metadata/changes/450.feature.md new file mode 100644 index 0000000000..c314478857 --- /dev/null +++ b/packages/zarr-metadata/changes/450.feature.md @@ -0,0 +1 @@ +A v3 codec that receives an array can declare its simplest spelling for the chunk it receives, with `chunk_canonical` on `ArrayCodecDefinition`. It is a function of the configuration, the nested declarations and the `Chunk`, for two spellings that mean the same thing for one chunk and not for another, which `canonical` cannot tell from the configuration alone. `codec_for_chunk` in `zarr_metadata.v3.definition` returns the codec of a `Stage` in that spelling, with each codec of the pipelines it contains simplified for the chunk that one receives. The `bytes` codec declares one: it drops an `endian` when each value takes one byte. So two `ZarrV3ArrayMetadata` models that differ only in the `endian` of a `bytes` codec that receives values of one byte (`bool`, `int8`, `uint8` or `r8`) are now equal, hash alike and refine each other. A `bytes` codec that receives values of several bytes, as it does after a `cast_value` to `float32`, is compared by its `endian` as before. diff --git a/packages/zarr-metadata/docs/extensions.md b/packages/zarr-metadata/docs/extensions.md index fc314760c8..0aa9332c56 100644 --- a/packages/zarr-metadata/docs/extensions.md +++ b/packages/zarr-metadata/docs/extensions.md @@ -122,7 +122,9 @@ Other kinds declare more: - A codec is defined by the class of its kind, `ArrayArrayCodecDefinition`, `ArrayBytesCodecDefinition` or `BytesBytesCodecDefinition`. A codec that receives an array declares what it checks about the chunk it receives - (`chunk_rules`) and any pipelines it contains (`pipelines`), and an + (`chunk_rules`), its simplest spelling for that chunk + (`chunk_canonical`, as `bytes` drops an `endian` when each value takes + one byte) and any pipelines it contains (`pipelines`), and an array -> array codec also declares what it hands on (`transition`). A bytes -> bytes codec receives bytes, so it declares none of these. diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index ea1a4ed294..ad2a700f6c 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -220,6 +220,13 @@ These are each declaration, and the fill value read against its data type. - A v3 blosc codec with `noshuffle` [ignores `typesize`](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/blosc/index.html#configuration-parameters). So it is the same codec with or without one. +- A v3 `bytes` codec's `endian` orders the bytes of each value, and the + encoding of a value that takes one byte + [does not depend on it](https://zarr-specs.readthedocs.io/en/v3.1.0/v3/codecs/bytes/index.html#configuration-parameters). + So in a `uint8` array, a `bytes` codec is the same codec with either + `endian`, or with none. The codec is compared for the chunk it + receives: after a `cast_value` to `float32`, the two `endian` values + are two codecs. A v2 document's [dtype](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#data-type-encoding), diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index f259f3d7fa..df2f4905a7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -214,7 +214,11 @@ def acme_lz4_rules( equivalent spelling, and declarations are compared by it. Each nested declaration is written in its own simplest spelling, then the definition's `canonical` is applied (blosc drops a `typesize` that `noshuffle` ignores, and a -rectilinear grid run-length encodes its chunk shapes). The envelope +rectilinear grid run-length encodes its chunk shapes). A v3 codec that +receives an array can have a simpler spelling still for the chunk it +receives, which its `chunk_canonical` declares, and an array model +compares its codecs in that spelling (see `zarr_metadata.v3.definition`). +The envelope uses the shortest form every reader accepts: a data type with nothing to configure is its bare name, and any other declaration is an object, `{"name": ...}`, because a Zarr v3.0 reader does not accept a short-hand @@ -259,8 +263,8 @@ def acme_lz4_rules( that has a member no checker reads (the error names the TypedDict that contains it); a `name` that is not a string; an attribute declared as a function (`rules`, `canonical`, `fill_value_rules`, `storage`, -`shape_rules`, `chunk_lengths`, `chunk_rules`, `transition`, -`pipelines`) that is not a function; a data type's `fill_value` that no +`shape_rules`, `chunk_lengths`, `chunk_rules`, `chunk_canonical`, +`transition`, `pipelines`) that is not a function; a data type's `fill_value` that no checker reads, or that contains a declaration, which a value of the data type never does; a codec `kind` that is not one of the three, or a `size` that is not `"static"` or `"dynamic"`; a function that is never diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 05eefcf761..8d3a30344e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1259,6 +1259,52 @@ def written_json(declaration: AcceptedDeclaration[Any] | UnclaimedDeclaration) - return kind._envelope_json(declaration.name, declaration.configuration) +def respelled( + declaration: AcceptedDeclaration[D], + configuration: Mapping[str, JSONValue], + nested: Nested, +) -> AcceptedDeclaration[D]: + """`declaration` with `configuration` as its configuration, and each declaration in `nested` in place of the nested declaration at its location. + + This builds the declaration that another spelling of `declaration` + reads as, without reading it again. The caller answers for the + spelling: `configuration` must be one that the definition allows and + that means the same thing, and it must keep each nested declaration + where it is. `nested` holds only the nested declarations to replace, + each one accepted or unclaimed, and each is written into the + configuration as its `to_json` writes it. A location in `nested` that + does not lead into `configuration` raises `TypeError`. The result is + written as `to_json` writes it, since no document wrote it. + """ + spelled: JSONValue = dict(configuration) + for loc, inner in nested.items(): + spelled = _replaced(spelled, loc, written_json(held(inner))) + kind = declaration.read_as + name = declaration.name + carried = kind._filed_under(name)[1] is not None + return AcceptedDeclaration( + json=kind._envelope_json(name, {} if carried else cast("Mapping[str, JSONValue]", spelled)), + name=name, + definition=declaration.definition, + configuration=cast("Mapping[str, JSONValue]", spelled), + nested={**declaration.nested, **nested}, + ) + + +def item_at(value: JSONValue, path: Loc) -> JSONValue | UNSET: + """The item of `value` at `path`; `UNSET` when `path` does not lead into `value`.""" + for step in path: + if isinstance(step, str): + if not isinstance(value, Mapping) or step not in value: + return UNSET + value = value[step] + else: + if not isinstance(value, tuple) or not 0 <= step < len(value): + return UNSET + value = value[step] + return value + + Nested: TypeAlias = Mapping[Loc, ResolvedDeclaration[Any]] """The declarations in a configuration, each as the scope read it, keyed by location in the configuration.""" @@ -1902,6 +1948,7 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "held", "in_document", "is_declaration", + "item_at", "kind_of", "named_configuration", "no_rules", @@ -1909,6 +1956,7 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue: "read_only", "resolve", "resolve_refined", + "respelled", "ruled", "spelled_canonically", "unchanged", diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 627545859c..2ea2af7ccb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -78,7 +78,7 @@ chunk_grid_lengths, default_array_bytes_codec, ) -from zarr_metadata.v3._pipeline import Stage, read_pipeline +from zarr_metadata.v3._pipeline import Stage, codec_for_chunk, read_pipeline from zarr_metadata.v3.array import ( ZARR_V3_ARRAY_METADATA_STORE_KEY, ZarrV3ArrayMetadataJSON, @@ -780,7 +780,7 @@ def refines(self, other: ZarrV3ArrayMetadata) -> bool: (self.data_type, other.data_type), (self.chunk_grid, other.chunk_grid), (self.chunk_key_encoding, other.chunk_key_encoding), - *zip(self.codecs, other.codecs, strict=True), + *zip(codecs_for_chunks(self._reading), codecs_for_chunks(other._reading), strict=True), *zip(self.storage_transformers, other.storage_transformers, strict=True), *((self.extra_fields[key], other.extra_fields[key]) for key in self.extra_fields), ) @@ -926,8 +926,10 @@ def array_key_of( The key holds each declaration's `declaration_key`, and the fill value as JSON text: in its canonical spelling when a definition in scope read the - data type, and as written otherwise. Every other member is included - as it is, with JSON members as text. + data type, and as written otherwise. A codec is compared as + `codec_for_chunk` returns it for the chunk it receives, so two + spellings that mean the same for that chunk are one codec. Every + other member is included as it is, with JSON members as text. """ data_type = held(reading.data_type) fill_value = ( @@ -940,7 +942,7 @@ def array_key_of( json_text(fill_value), declaration_key(data_type), declaration_key(held(reading.chunk_grid)), - tuple(declaration_key(held(stage.codec)) for stage in reading.pipeline), + tuple(declaration_key(codec) for codec in codecs_for_chunks(reading)), declaration_key(held(reading.chunk_key_encoding)), members.dimension_names, json_text(members.attributes), @@ -949,6 +951,13 @@ def array_key_of( ) +def codecs_for_chunks( + reading: ZarrV3ArrayMetadataReading, +) -> tuple[AcceptedDeclaration[CodecDefinition[Any]] | UnclaimedDeclaration, ...]: + """The codecs of `reading`, a read that found no problems, in pipeline order: each as `codec_for_chunk` returns it for the chunk it receives.""" + return tuple(held(codec_for_chunk(stage)) for stage in reading.pipeline) + + def extra_fields_key(extra_fields: Mapping[str, ResolvedDeclaration[Any]]) -> tuple[object, ...]: """A key for what a document's extra fields mean: each key with its declaration's `declaration_key`, sorted by key.""" return tuple( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py index 0d734d81e1..dc4a3135b1 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_definition.py @@ -56,6 +56,7 @@ Nested, Problems, ResolvedDeclaration, + T, WithFillValue, asked, is_declaration, @@ -94,6 +95,11 @@ def no_pipelines(*_: object) -> Mapping[str, Chunk]: return {} +def unchanged_for_chunk(configuration: T, nested: object, chunk: object) -> T: + """The simplest spelling of a codec whose configuration means the same for every chunk it receives: the configuration itself.""" + return configuration + + StorageClass = Literal["single_byte", "multi_byte", "variable_length"] """How a data type's values are stored: in single bytes, in several bytes at a time, or each value in as many bytes as it needs. @@ -415,10 +421,31 @@ class ArrayCodecDefinition(CodecDefinition[C]): inner codecs receive its inner chunks, and its index codecs receive the shard index. It is called whatever the chunk rules found, and must return only what is true in either case. + + Two spellings of a configuration can mean the same thing for one + chunk and different things for another. For example, the `endian` of + a `bytes` codec changes nothing when each value takes one byte, and + is required when a value takes several. `canonical` cannot reduce + such spellings to one, because it receives the configuration alone. + `chunk_canonical` can: it receives the configuration, the nested + declarations as the scope read them, and the chunk, and returns the + simplest configuration that means the same for that chunk. It is + called only for a codec that the chunk rules found no problem with. + It simplifies only the codec's own members, and it must keep each + nested declaration as it is, because each codec of a pipeline the + codec contains is simplified for the chunk it receives in turn. What + does not depend on the chunk is left to `canonical`, which simplifies + the result as it simplifies any configuration. A chunk rule checks + only the parts of a chunk that are known, and `chunk_canonical` + simplifies only by them: a chunk with nothing known about it, + `Chunk()`, leaves every configuration as it is. `codec_for_chunk` + returns the codec of a `Stage` in that spelling. """ chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules """The spec violations for this codec with a given input chunk, located in the configuration. A problem at `()` is about the codec as a whole, such as an input it cannot encode, and is reported at the codec.""" + chunk_canonical: Callable[[C, Nested, Chunk], C] = unchanged_for_chunk + """A configuration the chunk rules allow for a given input chunk, in the simplest spelling that means the same for that chunk.""" pipelines: Callable[[C, Nested, Chunk], Mapping[str, Chunk]] = no_pipelines """The chunk each contained pipeline receives, keyed by the configuration member that holds the pipeline.""" @@ -879,6 +906,7 @@ def chunk_grid_lengths( "no_pipelines", "single_byte", "storage_of", + "unchanged_for_chunk", "unknown_chunk", "unknown_lengths", "unknown_storage", diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index 592bb18a5f..9842d827da 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -33,13 +33,23 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, TypeGuard, cast -from zarr_metadata._json import ValidationProblem, is_object, is_tuple, with_input +from zarr_metadata._json import ( + ValidationProblem, + copied, + is_object, + is_tuple, + json_text, + with_input, +) +from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._definition import ( AcceptedDeclaration, ResolvedDeclaration, asked, in_document, + item_at, read_only, + respelled, ruled, ) from zarr_metadata.v3._definition import ( @@ -53,6 +63,7 @@ if TYPE_CHECKING: from collections.abc import Iterator, Mapping, Sequence + from zarr_metadata._common import JSONValue from zarr_metadata._typed_json import Loc from zarr_metadata.definition._definition import ( Nested, @@ -326,4 +337,96 @@ def _handed_on( return given -__all__ = ["Stage", "read_pipeline"] +def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: + """The codec of `stage`, in the simplest spelling that means the same for the chunk it receives. + + A codec's own `canonical` reduces the spellings that mean the same + thing wherever the codec is used. Some spellings mean the same thing + only for some chunks. For example, a `bytes` codec with and without + `endian` is one codec for values that take one byte, and two for + values that take several. So a codec is compared, and written in its + simplest spelling, as the codec this function returns: first each + codec of each pipeline it contains, for the chunk that codec + receives, and then its own configuration, as its definition's + `chunk_canonical` simplifies it for the chunk in `stage`. + + A codec the scope did not read is returned as it is. A codec's own + configuration is kept as it is when the codec receives bytes, and + when its chunk rules find a problem with it, because a codec with a + problem has no simpler spelling. When nothing is simplified, the + result is the codec itself. Otherwise it is a new declaration, read + by the same definition. `canonical_of` writes it in its simplest + spelling, and `==` compares it. + + `chunk_canonical` is the extension author's code, so its result is + checked: the definition must allow it, the chunk rules must find no + problem with it, and each nested declaration must be where it was. A + result that fails a check raises `ValueError`, which indicates a bug + in the definition, not in the codec. + """ + codec = stage.codec + if not isinstance(codec, AcceptedDeclaration): + return codec + inner: dict[Loc, ResolvedDeclaration[Any]] = {} + for member, stages in stage.inner.items(): + for index, each in enumerate(stages): + simpler = codec_for_chunk(each) + if simpler is not each.codec: + inner[(member, index)] = simpler + configuration = _for_chunk(codec, stage.incoming) + if configuration is None and len(inner) == 0: + return codec + return respelled(codec, codec.configuration if configuration is None else configuration, inner) + + +def _for_chunk( + codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None +) -> Mapping[str, JSONValue] | None: + """The configuration of `codec` as its definition's `chunk_canonical` simplifies it for `chunk`, checked; None when nothing is simplified.""" + definition = codec.definition + if chunk is None or not isinstance(definition, ArrayCodecDefinition): + return None + configuration, nested = codec.configuration, codec.nested + if len(_chunk_problems(definition, configuration, nested, chunk, ())) != 0: + return None + given: object = asked( + definition, + "chunk_canonical", + lambda: cast("object", definition.chunk_canonical(read_only(configuration), nested, chunk)), + ) + if given is configuration: + return None + if not is_object(given): + msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}" + raise TypeError(msg) + simplified = cast("dict[str, JSONValue]", copied(cast("JSONValue", dict(given)))) + if json_text(simplified) == json_text(cast("JSONValue", configuration)): + return None + _, refused = definition.read_configuration(simplified) + found = ( + *refused, + *_chunk_problems(definition, simplified, nested, chunk, ()), + *( + ValidationProblem( + loc, "expected the nested declaration to stay as it is", "invalid_value" + ) + for loc in nested + if _text_at(simplified, loc) != _text_at(cast("JSONValue", configuration), loc) + ), + ) + if len(found) != 0: + msg = ( + f"{definition.name!r}: its chunk_canonical gave {simplified!r}, which does not hold: " + f"{list(found)!r}" + ) + raise ValueError(msg) + return simplified + + +def _text_at(configuration: JSONValue, loc: Loc) -> str | None: + """The JSON text of the item of `configuration` at `loc`; None when `loc` does not lead into it.""" + item = item_at(configuration, loc) + return None if item is UNSET else json_text(item) + + +__all__ = ["Stage", "codec_for_chunk", "read_pipeline"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py index 5b08cf3b0a..ca30fa46e2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/bytes.py @@ -64,11 +64,38 @@ def _chunk_rules( ) +def _chunk_canonical( + configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk +) -> BytesCodecConfiguration: + """Remove `endian` when each value takes one byte, because it has no meaning there. + + `endian` is "Required for data types for which endianness is + applicable" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L64-L69), + and the encoding of a value that takes one byte does "not depend on + endian" + (https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/codecs/bytes/index.rst#L84-L101). + So for such a data type, a `bytes` codec with either `endian`, or + with none, encodes the same bytes, and the three spellings describe + the same codec. zarr-python 3.0 wrote `endian` for these data types, + and zarr-python 3.1 does not. The storage is the one the data type + declares. If it is unknown, the configuration is kept as written. + """ + if chunk.data_type is None or "endian" not in configuration: + return configuration + if storage_of(chunk.data_type) != "single_byte": + return configuration + simpler = configuration.copy() + del simpler["endian"] + return simpler + + BYTES_CODEC: Final = ArrayBytesCodecDefinition( name=BYTES_CODEC_NAME, configuration=BytesCodecConfiguration, size="static", chunk_rules=_chunk_rules, + chunk_canonical=_chunk_canonical, ) """The `bytes` codec.""" diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py index c98d6460c1..9c20fac4a5 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -90,6 +90,14 @@ pipeline. For example, a shard's inner codecs receive its inner chunks, and its index codecs receive the shard index. Each pipeline is read the same way, and its stages are stored as the codec's `Stage.inner`. +Two spellings of a codec can mean the same thing for one chunk and not +for another, which `canonical` cannot tell, since it receives the +configuration alone. Such a codec declares its simplest spelling for a +given chunk with `chunk_canonical`. For example, `bytes` drops an +`endian` when each value takes one byte. `codec_for_chunk` returns the +codec of a `Stage` in that spelling, with each codec of the pipelines +it contains simplified for the chunk that one receives. An array model +compares its codecs this way. The reader does not guess. After a codec the scope did not read, or after a codec that does not declare its output, the next codec receives a chunk with nothing known about it, `Chunk()`, which never causes a @@ -147,7 +155,7 @@ default_array_bytes_codec, storage_of, ) -from zarr_metadata.v3._pipeline import Stage +from zarr_metadata.v3._pipeline import Stage, codec_for_chunk __all__ = [ "CORE_AND_EXTENSIONS_CONTEXT", @@ -181,6 +189,7 @@ "ZarrV3Kind", "ZarrV3StaticCodecDeclaration", "ZarrV3StorageTransformerDeclaration", + "codec_for_chunk", "default_array_bytes_codec", "storage_of", ] diff --git a/packages/zarr-metadata/tests/model/test_array.py b/packages/zarr-metadata/tests/model/test_array.py index 7fd62c5442..2d2ae6cf27 100644 --- a/packages/zarr-metadata/tests/model/test_array.py +++ b/packages/zarr-metadata/tests/model/test_array.py @@ -805,6 +805,59 @@ def test_two_models_are_one_array_when_their_declarations_read_the_same( assert models[0].to_json() != models[1].to_json() +_BIG = {"name": "bytes", "configuration": {"endian": "big"}} +_TO_UINT8 = {"name": "cast_value", "configuration": {"data_type": "uint8"}} +_TO_FLOAT32 = {"name": "cast_value", "configuration": {"data_type": "float32"}} + + +def _shard_of(*codecs: object) -> dict[str, object]: + return { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [1], + "codecs": list(codecs), + "index_codecs": [_LITTLE, {"name": "crc32c"}], + }, + } + + +@pytest.mark.parametrize( + ("data_type", "left", "right", "same"), + [ + ("uint8", [_LITTLE], [{"name": "bytes"}], True), + ("uint8", [_LITTLE], [_BIG], True), + ("uint16", [_LITTLE], [_BIG], False), + ("uint8", [_TO_FLOAT32, _LITTLE], [_TO_FLOAT32, _BIG], False), + ("uint16", [_TO_UINT8, _LITTLE], [_TO_UINT8, {"name": "bytes"}], True), + ("uint8", [_shard_of(_LITTLE)], [_shard_of({"name": "bytes"})], True), + ("uint16", [_shard_of(_LITTLE)], [_shard_of(_BIG)], False), + ], + ids=[ + "one-byte-endian-or-none", + "one-byte-either-endian", + "two-bytes-either-endian", + "cast-to-four-bytes", + "cast-to-one-byte", + "shard-of-one-byte", + "shard-of-two-bytes", + ], +) +def test_two_models_are_one_array_when_each_codec_means_the_same_for_its_chunk( + data_type: str, left: object, right: object, same: bool +) -> None: + """A codec is compared for the chunk it receives, as `codec_for_chunk` returns it: the `endian` of a `bytes` codec tells two arrays apart only where a value takes several bytes. `refines` agrees with `==`, and `to_json` writes each codec as given.""" + document = ZarrV3ArrayMetadata.create_default(shape=(2,), data_type=data_type).to_json() + models = [ + ZarrV3ArrayMetadata.from_json({**document, "codecs": codecs}) for codecs in (left, right) + ] + assert (models[0] == models[1]) is same + assert models[0].refines(models[1]) is same + assert models[1].refines(models[0]) is same + assert models[0].to_json() != models[1].to_json() + if same: + assert hash(models[0]) == hash(models[1]) + + @pytest.mark.parametrize( ("left", "right", "same"), [ diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index d88dcf639e..70ac04d5eb 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -1473,6 +1473,7 @@ def test_error_a_data_type_fill_value_holding_a_declaration() -> None: (ChunkGridDefinition, "shape_rules"), (ChunkGridDefinition, "chunk_lengths"), (ArrayArrayCodecDefinition, "chunk_rules"), + (ArrayArrayCodecDefinition, "chunk_canonical"), (ArrayArrayCodecDefinition, "transition"), (ArrayArrayCodecDefinition, "pipelines"), ], diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index 7fcab1c1d2..fe477dc8f0 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -8,15 +8,19 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +import dataclasses +from typing import TYPE_CHECKING, Any, cast import pytest from typing_extensions import TypedDict +from zarr_metadata._json import json_text from zarr_metadata.definition import ( + AcceptedDeclaration, EmptyConfiguration, Nested, ResolvedDeclaration, + canonical_of, resolve, ) from zarr_metadata.model import ( @@ -26,6 +30,7 @@ from zarr_metadata.v3._pipeline import ( read_pipeline, ) +from zarr_metadata.v3.codec.bytes import BYTES_CODEC, BytesCodecConfiguration from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, ArrayArrayCodecDefinition, @@ -37,6 +42,7 @@ ZarrV3CodecDeclaration, ZarrV3Context, ZarrV3DataTypeDeclaration, + codec_for_chunk, ) if TYPE_CHECKING: @@ -422,6 +428,7 @@ def transition(configuration: EmptyConfiguration, nested: Nested, chunk: Chunk) ("kind", "attribute"), [ (BytesBytesCodecDefinition, "chunk_rules"), + (BytesBytesCodecDefinition, "chunk_canonical"), (BytesBytesCodecDefinition, "transition"), (BytesBytesCodecDefinition, "pipelines"), (ArrayBytesCodecDefinition, "transition"), @@ -545,3 +552,153 @@ def test_error_a_stage_s_inner_pipelines_are_read_only() -> None: stage.inner["codecs"] = () # pyright: ignore[reportIndexIssue] for again in (pickle.loads(pickle.dumps(stages)), copy.deepcopy(stages)): assert again == stages + + +BIG: JSONValue = {"name": "bytes", "configuration": {"endian": "big"}} + +BARE: JSONValue = {"name": "bytes"} + +CRC32C: JSONValue = {"name": "crc32c"} + + +def _shard(*codecs: JSONValue, index_codecs: tuple[JSONValue, ...] = (LITTLE, CRC32C)) -> JSONValue: + """A shard of inner chunks of 2 by 1 that encodes them with `codecs`, and its index with `index_codecs`.""" + return { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [2, 1], + "codecs": list(codecs), + "index_codecs": list(index_codecs), + }, + } + + +def _simplest(codecs: list[JSONValue], chunk: Chunk) -> list[JSONValue]: + """Each codec of a pipeline that receives `chunk`, in its simplest spelling for the chunk it receives.""" + stages, problems = _read(codecs, chunk) + assert problems == () + return [canonical_of(codec_for_chunk(stage), ()) for stage in stages] + + +@pytest.mark.parametrize( + ("data_type", "codecs", "simplest"), + [ + # Each value takes one byte, so `endian` means nothing. + ("uint8", [LITTLE], [BARE]), + ("int8", [BIG], [BARE]), + ("bool", [LITTLE], [BARE]), + ("r8", [LITTLE], [BARE]), + ("uint8", [BARE], [BARE]), + ("uint8", [_transpose(1, 0), LITTLE, GZIP], [_transpose(1, 0), BARE, GZIP]), + # Each value takes several bytes, so `endian` says how they are ordered. + ("uint16", [LITTLE], [LITTLE]), + ("float32", [BIG], [BIG]), + ("r16", [LITTLE], [LITTLE]), + # The codec is simplified for the chunk it receives, not for the array's data type. + ("uint8", [_cast("float32"), LITTLE], [_cast("float32"), LITTLE]), + ("uint16", [_cast("uint8"), BIG], [_cast("uint8"), BARE]), + # A shard's inner codecs receive the shard's data type, and its index codecs `uint64`. + ("uint8", [_shard(LITTLE)], [_shard(BARE)]), + ("uint16", [_shard(LITTLE)], [_shard(LITTLE)]), + ("uint8", [_shard(_shard(BIG))], [_shard(_shard(BARE))]), + ("uint8", [_cast("uint16"), _shard(LITTLE)], [_cast("uint16"), _shard(LITTLE)]), + ("uint16", [_shard(_cast("uint8"), LITTLE)], [_shard(_cast("uint8"), BARE)]), + # Nothing is known of the values, so nothing is simplified. + ("acme.t", [LITTLE], [LITTLE]), + ("uint8", ["acme.shuffle", LITTLE], [{"name": "acme.shuffle"}, LITTLE]), + ("uint8", ["acme.unclaimed", LITTLE], [{"name": "acme.unclaimed"}, LITTLE]), + ], +) +def test_a_codec_is_written_in_the_simplest_spelling_for_the_chunk_it_receives( + data_type: JSONValue, codecs: list[JSONValue], simplest: list[JSONValue] +) -> None: + """`codec_for_chunk` returns each codec as its `chunk_canonical` simplifies it for the chunk it receives, and each codec of a pipeline it contains for the chunk that one receives. A codec with nothing to simplify is returned as it is.""" + found = _simplest(codecs, _of(data_type)) + assert json_text(found) == json_text(simplest) + # The simplest spelling reads without a problem, and has nothing left to simplify. + assert json_text(_simplest(found, _of(data_type))) == json_text(simplest) + stages, _ = _read(simplest, _of(data_type)) + assert all(codec_for_chunk(stage) is stage.codec for stage in stages) + + +def test_a_codec_with_a_problem_for_its_chunk_is_not_simplified() -> None: + """A codec the chunk rules find a problem with is returned as it is, as is a codec the scope refused.""" + # `bytes` does not encode values that vary in size, and `gzip` requires a level. + stages, problems = _read([LITTLE, {"name": "gzip", "configuration": {}}], _of("string")) + assert [problem.loc for problem in problems] == [(1, "configuration", "level"), (0,)] + assert all(codec_for_chunk(stage) is stage.codec for stage in stages) + + +def _bytes_with(chunk_canonical: object) -> ResolvedDeclaration[CodecDefinition[Any]]: + """A `bytes` codec with `endian`, read by a definition whose `chunk_canonical` is `chunk_canonical`.""" + lying = dataclasses.replace(BYTES_CODEC, name="acme.bytes", chunk_canonical=chunk_canonical) + declaration = {"name": "acme.bytes", "configuration": {"endian": "little"}} + return resolve(declaration, CodecDefinition, CONTEXT.union(ZarrV3Context(codecs=[lying])))[0] + + +def _stage(codec: ResolvedDeclaration[CodecDefinition[Any]], chunk: Chunk) -> Stage: + stages, problems = read_pipeline([codec], chunk) + assert problems == () + return stages[0] + + +def test_error_a_chunk_canonical_the_definition_does_not_allow() -> None: + stage = _stage(_bytes_with(lambda configuration, nested, chunk: {"endian": "middle"}), CHUNK) + with pytest.raises(ValueError, match="'acme.bytes': its chunk_canonical gave"): + codec_for_chunk(stage) + + +def test_error_a_chunk_canonical_the_chunk_rules_do_not_allow() -> None: + # A value of `uint16` takes two bytes, so the chunk rules require `endian`. + stage = _stage(_bytes_with(lambda configuration, nested, chunk: {}), _of("uint16")) + with pytest.raises(ValueError, match="'acme.bytes': its chunk_canonical gave {}, which"): + codec_for_chunk(stage) + + +def test_error_a_chunk_canonical_that_gives_something_else() -> None: + stage = _stage(_bytes_with(lambda configuration, nested, chunk: "little"), CHUNK) + with pytest.raises(TypeError, match="'acme.bytes': its chunk_canonical gives a configuration"): + codec_for_chunk(stage) + + +def test_error_a_chunk_canonical_that_raises_says_whose_it_is() -> None: + def chunk_canonical( + configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk + ) -> BytesCodecConfiguration: + raise KeyError("endian") + + with pytest.raises(KeyError) as raised: + codec_for_chunk(_stage(_bytes_with(chunk_canonical), CHUNK)) + assert raised.value.__notes__ == ["raised by the chunk_canonical of 'acme.bytes'"] + + +def test_error_a_chunk_canonical_that_writes_to_the_configuration() -> None: + def chunk_canonical( + configuration: BytesCodecConfiguration, nested: Nested, chunk: Chunk + ) -> BytesCodecConfiguration: + del configuration["endian"] + return configuration + + with pytest.raises(TypeError, match="does not support item deletion"): + codec_for_chunk(_stage(_bytes_with(chunk_canonical), CHUNK)) + + +def test_error_a_chunk_canonical_that_changes_a_nested_declaration() -> None: + # Each codec of a pipeline the codec contains is simplified for the + # chunk it receives, so the codec's own `chunk_canonical` leaves it alone. + def chunk_canonical( + configuration: AcmeHolderConfiguration, nested: Nested, chunk: Chunk + ) -> AcmeHolderConfiguration: + return {"codecs": (cast("ZarrV3CodecDeclaration", BARE),), "types": configuration["types"]} + + holder = ArrayBytesCodecDefinition( + name="acme.holder", + configuration=AcmeHolderConfiguration, + size="dynamic", + chunk_canonical=chunk_canonical, + ) + declaration = {"name": "acme.holder", "configuration": {"codecs": [LITTLE], "types": ["uint8"]}} + codec = resolve(declaration, CodecDefinition, CONTEXT.union(ZarrV3Context(codecs=[holder])))[0] + assert isinstance(codec, AcceptedDeclaration) + with pytest.raises(ValueError, match="expected the nested declaration to stay as it is"): + codec_for_chunk(_stage(codec, CHUNK)) From 1be8677c36104d59ce3d89ccc86075d4a5454006 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 15:42:36 +0100 Subject: [PATCH 2/6] feat(zarr-metadata): return each model with its document in its simplest equivalent spelling Two models are equal when their documents mean the same thing, and `to_json` writes each document as it was given, so equal models can write different documents. A consumer that needs one spelling for all of them, to compare documents field by field or to hash them, had to normalize each member itself. Each model gains `canonicalize`: the v3 and v2 array, group and consolidated metadata models. It returns the model read from its document in its simplest equivalent spelling, written from the parts that `==` compares: each declaration as `canonical_of` writes it, each v3 codec as `codec_for_chunk` returns it for the chunk it receives, and the fill value in the canonical spelling of its data type. Empty `attributes`, empty v3 `storage_transformers` and a v2 `dimension_separator` of "." are left out, since a document without them means the same. Everything else is kept as written. The result equals the model, and equal models in one scope give the same document apart from the order of the members of an object, which a `JSONWriter(sort_keys=True)` sorts. A group keeps the documents its `consolidated_metadata` lists as written, since it does not read them; the consolidated metadata models canonicalize each node they list. Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/changes/452.feature.md | 1 + packages/zarr-metadata/docs/models.md | 75 ++++ .../src/zarr_metadata/model/__init__.py | 5 +- .../src/zarr_metadata/model/_array_v2.py | 35 ++ .../src/zarr_metadata/model/_array_v3.py | 50 +++ .../src/zarr_metadata/model/_group_v2.py | 31 ++ .../src/zarr_metadata/model/_group_v3.py | 37 ++ .../src/zarr_metadata/model/_shared.py | 46 ++ .../tests/model/test_canonicalize.py | 401 ++++++++++++++++++ 9 files changed, 680 insertions(+), 1 deletion(-) create mode 100644 packages/zarr-metadata/changes/452.feature.md create mode 100644 packages/zarr-metadata/tests/model/test_canonicalize.py diff --git a/packages/zarr-metadata/changes/452.feature.md b/packages/zarr-metadata/changes/452.feature.md new file mode 100644 index 0000000000..a79317f00b --- /dev/null +++ b/packages/zarr-metadata/changes/452.feature.md @@ -0,0 +1 @@ +Each model has a `canonicalize` method that returns the model with its document in its simplest equivalent spelling: `ZarrV3ArrayMetadata`, `ZarrV3GroupMetadata`, `ZarrV3ConsolidatedMetadata`, `ZarrV2ArrayMetadata`, `ZarrV2GroupMetadata` and `ZarrV2ConsolidatedMetadata`. The result equals the model. Every model equal to it, in the same scope, gives the same document, apart from the order of the members of an object, so `to_key_value` with `JSONWriter(sort_keys=True)` writes the same bytes for equal models. The document is written from the parts `==` compares: each declaration as `canonical_of` writes it, each v3 codec as `codec_for_chunk` returns it for the chunk it receives, and the fill value as `canonical_fill_value` writes it. Empty `attributes`, empty v3 `storage_transformers` and a v2 `dimension_separator` of `"."` are left out, and everything else is kept as written. diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index ad2a700f6c..16c4e4378a 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -130,6 +130,8 @@ To change a model, use one of these methods: - `refined_in` also reads the document in another context. The new context must claim what this context left unclaimed and contradict nothing. Otherwise it raises `ScopeConflictError`. +- `canonicalize` returns the model with its document in its simplest + equivalent spelling. See [Canonical documents](#canonical-documents). ## Writing store bytes @@ -264,3 +266,76 @@ equal to itself. Equal models have equal hashes. They may still write different documents, because `to_json` writes each document as it was given. + +## Canonical documents + +`canonicalize` returns a model with its document in its simplest +equivalent spelling. The result equals the model it came from. Every model +equal to that one, read in the same context, gives the same document. + +```python test="true" session="models" +from zarr_metadata.model import ZarrV3ArrayMetadata + +written = ZarrV3ArrayMetadata.from_json( + { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, + "fill_value": 0, + "codecs": [ + {"name": "bytes", "configuration": {"endian": "little"}}, + {"name": "zstd", "configuration": {"level": 1, "checksum": False}}, + ], + "attributes": {}, + } +) +canonical = written.canonicalize() +assert canonical == written +print(canonical.to_json()["chunk_key_encoding"]) +#> {'name': 'default'} +print(canonical.to_json()["codecs"]) +#> ({'name': 'bytes'}, {'name': 'zstd', 'configuration': {'level': 1}}) +print("attributes" in canonical.to_json()) +#> False +``` + +The canonical document is written from the same parts that `==` compares: + +- Each declaration is written as `canonical_of` writes it. +- Each v3 codec is written for the chunk it receives, as `codec_for_chunk` + returns it. Above, each `uint8` value takes one byte, so the `endian` + says nothing. +- The fill value is written as `canonical_fill_value` writes it for the + data type. +- A member that says what its absence says is left out. These are empty + `attributes`, empty v3 `storage_transformers`, and a v2 + `dimension_separator` of `"."`. +- Everything else is kept as written. This includes the attributes, and + the configuration of a declaration that nothing in the context claims, + because only its own definition could simplify it. + +A v2 codec is written with each numcodecs default it leaves out, as its +definition's `canonical` writes it. So a canonical v2 document can be +longer than the one it came from. + +The order of the members of an object is not part of the spelling, because +JSON gives it no meaning. Two equal models can give canonical documents +whose members are in different orders. A `JSONWriter` that sorts them +writes the same bytes for both, so those bytes are a key for what the +document means: + +```python test="true" session="models" +import hashlib + +stored = canonical.to_key_value({"sort_keys": True})["zarr.json"] +digest = hashlib.sha256(stored).hexdigest() +``` + +The other models work the same way. A v3 group does not read the documents +its `consolidated_metadata` lists, so its `canonicalize` keeps them as +written. `ZarrV3ConsolidatedMetadata.canonicalize` and +`ZarrV2ConsolidatedMetadata.canonicalize` write each node they list in its +simplest spelling. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 39f4ca5e58..b18d1eb092 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -42,7 +42,10 @@ scope and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as written. `replace` reads new members in the model's own scope. `with_context` and -`refined_in` read the document in another scope. `to_key_value` writes +`refined_in` read the document in another scope. `canonicalize` returns +the model with its document in its simplest equivalent spelling, the one +that every equal model gives, apart from the order of the members of an +object. `to_key_value` writes the model as it is, with a `JSONWriter`, or a mapping of its settings, deciding how the bytes are spelled (indentation, key order, ASCII escaping, trailing newline); the default is that of `json.dumps`. A v3 group reads its diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index 2e5ba2930e..dc39bdfc6f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -50,6 +50,7 @@ Derived, StoreKey, attributes_of, + canonical_members, check_literal, created, dimension_lengths, @@ -59,6 +60,7 @@ missing_keys, other_members, reading_of, + simplest, unclaimed_declarations_of, ) @@ -772,6 +774,39 @@ def __replace__(self, /, **members: Unpack[ZarrV2ArrayMetadataUpdate]) -> ZarrV2 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV2ArrayMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same document: `to_json` of each result is + the same JSON, apart from the order of the members of an object, + which `to_key_value` sorts with `JSONWriter(sort_keys=True)`. So + the bytes that writer writes are the same for equal models, and + are a key for what the document means. + + The dtype, the compressor and each filter are written as + `canonical_of` writes them, and the fill value as + `canonical_fill_value` writes it for the dtype. A + `dimension_separator` of `"."` is left out, and so are empty + `attributes`, because a document without them means the same. + Everything else is kept as written: `shape`, `chunks`, `order`, + the attributes, and whatever nothing in scope claims. + """ + compressor, filters = self.compressor, self.filters + spelled: dict[str, JSONValue] = { + "dtype": simplest(self.dtype), + "fill_value": spelled_canonically(self.dtype, self._members.fill_value), + "compressor": None if compressor is None else simplest(compressor), + "filters": None if filters is None else tuple(simplest(entry) for entry in filters), + } + document = canonical_members( + self._document, + spelled, + empty=("attributes",), + defaults={"dimension_separator": ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR}, + ) + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 2ea2af7ccb..1d15ef74fb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -52,6 +52,7 @@ Derived, StoreKey, attributes_of, + canonical_members, check_literal, created, dimension_lengths, @@ -60,6 +61,8 @@ member_changes, missing_keys, reading_of, + simplest, + simplest_members, unclaimed_declarations_of, ) from zarr_metadata.v3._common import ZarrV3DeclarationJSON, ZarrV3ExtensionField @@ -734,6 +737,29 @@ def __replace__(self, /, **members: Unpack[ZarrV3ArrayMetadataUpdate]) -> ZarrV3 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV3ArrayMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same document: `to_json` of each result is + the same JSON, apart from the order of the members of an object, + which `to_key_value` sorts with `JSONWriter(sort_keys=True)`. So + the bytes that writer writes are the same for equal models, and + are a key for what the document means. + + Each declaration is written as `canonical_of` writes it, each + codec as `codec_for_chunk` returns it for the chunk it receives, + and the fill value as `canonical_fill_value` writes it for the + data type. `attributes` and `storage_transformers` are left out + when empty, because a document without them means the same. + Everything else is kept as written: `shape`, `dimension_names`, + the attributes, and whatever nothing in scope claims, which only + its own definition could simplify. A document that is already in + this spelling is returned as it is, in a new model. + """ + document = canonical_array_document_v3(self._reading, self._members) + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. @@ -951,6 +977,30 @@ def array_key_of( ) +def canonical_array_document_v3( + reading: ZarrV3ArrayMetadataReading, members: ArrayMembersV3 +) -> dict[str, JSONValue]: + """A v3 array document in its simplest equivalent spelling, from its `reading` and refined `members`, as `ZarrV3ArrayMetadata.canonicalize` reads it. + + It is written from the parts that `array_key_of` compares, so two + documents with one key are written the same way, apart from the order + of the members of an object. + """ + data_type = held(reading.data_type) + spelled: dict[str, JSONValue] = { + "data_type": simplest(data_type), + "chunk_grid": simplest(reading.chunk_grid), + "chunk_key_encoding": simplest(reading.chunk_key_encoding), + "fill_value": spelled_canonically(data_type, members.fill_value), + "codecs": tuple(simplest(codec) for codec in codecs_for_chunks(reading)), + "storage_transformers": tuple(simplest(entry) for entry in reading.storage_transformers), + **simplest_members(reading.extra_fields), + } + return canonical_members( + members.document, spelled, empty=("attributes", "storage_transformers") + ) + + def codecs_for_chunks( reading: ZarrV3ArrayMetadataReading, ) -> tuple[AcceptedDeclaration[CodecDefinition[Any]] | UnclaimedDeclaration, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index cbd34fb85f..3f185a0f9d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -48,6 +48,7 @@ from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, + canonical_members, check_literal, load_store_json, located_conflicts, @@ -378,6 +379,11 @@ def __replace__(self, /, **members: Unpack[ZarrV2GroupMetadataUpdate]) -> ZarrV2 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV2GroupMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope: without `attributes` when they are empty, because a group with no `.zattrs` means the same. The result equals this model, and every model equal to this one gives the same document, apart from the order of the members of an object, which `to_key_value` sorts with `JSONWriter(sort_keys=True)`.""" + document = canonical_members(self._document, {}, empty=("attributes",)) + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: """Return this document read in `context`: the same group, holding that scope.""" context = scoped(context, CORE_V2_CONTEXT) @@ -586,6 +592,31 @@ def _located_declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]] def __reduce__(self) -> tuple[type[ZarrV2ConsolidatedMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) + def canonicalize(self) -> ZarrV2ConsolidatedMetadata: + """Return this document with each node in its simplest equivalent spelling, as that node's `canonicalize` writes it, read in this model's own scope. + + Each node's `.zarray` or `.zgroup` entry is written without the + attributes, and its `.zattrs` entry holds them, or is left out + when they are empty. Every other entry is kept as written. The + result equals this model. Every model equal to this one, in this + scope, gives the same document, apart from the order of the + members of an object, which a `JSONWriter(sort_keys=True)` sorts. + """ + entries = dict(object_at(self._document, "metadata")) + for path, node in self._metadata.items(): + names = self._files[path] + written = cast("dict[str, JSONValue]", node.canonicalize().to_json()) + attributes = written.pop("attributes", None) + for name, key in names.items(): + if name != ZARR_V2_ATTRIBUTES_STORE_KEY: + entries[key] = written + elif attributes is None: + del entries[key] + else: + entries[key] = attributes + document: dict[str, JSONValue] = {**self._document, "metadata": entries} + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: """Return this document with every node read in `context`, whatever that changes. Raises `MetadataValidationError` when a node has a problem in `context`.""" context = scoped(context, CORE_V2_CONTEXT) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 9c5d9dc006..33e4cd82ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -67,12 +67,14 @@ from zarr_metadata.model._shared import ( StoreKey, attributes_of, + canonical_members, check_literal, load_store_json, located_conflicts, member_changes, missing_keys, reading_of, + simplest_members, unclaimed_declarations_of, unexpected_keys, ) @@ -392,6 +394,27 @@ def __replace__(self, /, **members: Unpack[ZarrV3GroupMetadataUpdate]) -> ZarrV3 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV3GroupMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same document: `to_json` of each result is + the same JSON, apart from the order of the members of an object, + which `to_key_value` sorts with `JSONWriter(sort_keys=True)`. + + `attributes` is left out when empty, because a document without + it means the same, and each extra field a definition in scope + claims is written as `canonical_of` writes it. Everything else is + kept as written. That includes the documents that + `consolidated_metadata` lists, which a group does not read: + `consolidated().canonicalize()` returns them in their simplest + spelling, and `replace(consolidated_metadata=...)` writes them + back into the group. + """ + spelled = simplest_members(self._reading.extra_fields) + document = canonical_members(self._document, spelled, empty=("attributes",)) + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: """Return this document read in `context`, whatever that changes. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically.""" context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) @@ -595,6 +618,20 @@ def _key_of(self) -> tuple[object, ...]: def __reduce__(self) -> tuple[type[ZarrV3ConsolidatedMetadata], tuple[object, ZarrV3Context]]: return type(self), (self._document, self._context) + def canonicalize(self) -> ZarrV3ConsolidatedMetadata: + """Return this member with each document it lists in its simplest equivalent spelling, as that node's `canonicalize` writes it, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same member, apart from the order of the + members of an object, which a `JSONWriter(sort_keys=True)` sorts. + """ + metadata: dict[str, JSONValue] = { + path: cast("JSONValue", node.canonicalize().to_json()) + for path, node in self._metadata.items() + } + document: dict[str, JSONValue] = {**self._document, "metadata": metadata} + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: """Return this member with every document read in `context`, whatever that changes. Raises `MetadataValidationError` when a document has a problem in `context`. The existing readings are reused when `context` reads every claim of every document identically.""" context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py index a946c4a608..2bb83aa52e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py @@ -24,6 +24,8 @@ from zarr_metadata.definition._definition import ( RefusedDeclaration, UnclaimedDeclaration, + canonical_of, + held, is_declaration, ) from zarr_metadata.definition._scope import Conflict, claim_key @@ -204,6 +206,50 @@ def created( raise +def simplest(declaration: ResolvedDeclaration[Any] | UNSET) -> JSONValue: + """`declaration`, from a read that found no problems, in its simplest equivalent spelling, as `canonical_of` writes it. Raises `TypeError` for a declaration with none, which such a read cannot produce.""" + spelled = canonical_of(held(declaration), ()) + if spelled is None: + msg = f"expected a declaration a read found nothing wrong with, got {declaration!r}" + raise TypeError(msg) + return spelled + + +def simplest_members(extra_fields: Mapping[str, ResolvedDeclaration[Any]]) -> dict[str, JSONValue]: + """The extra fields of a document that a read found no problems with, by key, each in its simplest equivalent spelling.""" + spelled: dict[str, JSONValue] = {} + for extra in extra_fields.values(): + # An extra field is written as the member it is: its key, and its value. + spelled.update(cast("Mapping[str, JSONValue]", simplest(extra))) + return spelled + + +def canonical_members( + document: Mapping[str, JSONValue], + spelled: Mapping[str, JSONValue], + *, + empty: Iterable[str] = (), + defaults: Mapping[str, JSONValue] | None = None, +) -> dict[str, JSONValue]: + """`document`, the document a model holds, with each member in `spelled` written as `spelled` writes it, and without the members that say what their absence says. + + A member in `empty` is left out when it is empty, and a member in + `defaults` when it has the value given there. The members keep the + order `document` writes them in. A member of `spelled` that + `document` does not write is not added. + """ + canonical = {key: spelled.get(key, value) for key, value in document.items()} + for key in empty: + value = canonical.get(key) + if (is_array(value) or is_object(value)) and len(value) == 0: + del canonical[key] + if defaults is not None: + for key, default in defaults.items(): + if key in canonical and canonical[key] == default: + del canonical[key] + return canonical + + def unclaimed_declarations_of( declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], waived: Iterable[str] ) -> dict[Loc, UnclaimedDeclaration]: diff --git a/packages/zarr-metadata/tests/model/test_canonicalize.py b/packages/zarr-metadata/tests/model/test_canonicalize.py new file mode 100644 index 0000000000..ff7b92cabd --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_canonicalize.py @@ -0,0 +1,401 @@ +"""`canonicalize`: each model with its document in its simplest equivalent spelling. + +Two models are equal when their documents mean the same thing, and +`to_json` writes each document as it was given. `canonicalize` returns the +one document that every model equal to a given one writes, apart from the +order of the members of an object, which a `JSONWriter` sorts. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, cast + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from zarr_metadata._json import json_text +from zarr_metadata.model import ( + JSONWriter, + ZarrV2ArrayMetadata, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, +) +from zarr_metadata.testing.strategies import ( + Spellings, + array_documents, + consolidated_group_documents, + group_documents, +) +from zarr_metadata.v3.definition import ZARR_PYTHON_CONTEXT + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata import JSONValue + +Model = ( + ZarrV2ArrayMetadata + | ZarrV2ConsolidatedMetadata + | ZarrV2GroupMetadata + | ZarrV3ArrayMetadata + | ZarrV3ConsolidatedMetadata + | ZarrV3GroupMetadata +) + +_LITTLE: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "little"}} +_BIG: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "big"}} +_NOSHUFFLE = {"cname": "lz4", "clevel": 5, "shuffle": "noshuffle", "blocksize": 0} +_WAIVED = {"name": "acme.note", "must_understand": False, "configuration": {"b": 1, "a": 2}} + +_V3_ARRAY: dict[str, Any] = { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default"}, + "fill_value": 0, + "codecs": [{"name": "bytes"}], +} +"""A v3 array document in its simplest spelling.""" + +_V3_ARRAY_SPELLED_OUT: dict[str, Any] = { + "attributes": {}, + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": {"name": "uint8", "configuration": {}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, + "fill_value": 0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [2], + "index_location": "end", + "codecs": [_BIG, {"name": "blosc", "configuration": {**_NOSHUFFLE, "typesize": 4}}], + "index_codecs": [_LITTLE, {"name": "crc32c", "configuration": {}}], + }, + } + ], + "storage_transformers": [], + "dimension_names": [None], + "acme": _WAIVED, +} +"""A v3 array document that says in more words what `_V3_ARRAY_SIMPLEST` says.""" + +_V3_ARRAY_SIMPLEST: dict[str, Any] = { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default"}, + "fill_value": 0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [2], + "codecs": [{"name": "bytes"}, {"name": "blosc", "configuration": _NOSHUFFLE}], + "index_codecs": [_LITTLE, {"name": "crc32c"}], + }, + } + ], + "dimension_names": [None], + "acme": _WAIVED, +} + +_V3_FLOATS: dict[str, Any] = {**_V3_ARRAY, "data_type": "float32", "codecs": [_LITTLE]} + +_V3_UNCLAIMED: dict[str, Any] = { + **_V3_ARRAY, + "data_type": {"name": "acme.t", "configuration": {"bits": 8, "signed": False}}, + "fill_value": "0x00", + "codecs": [{"name": "acme.codec", "configuration": {"level": 1}}, _LITTLE], + "attributes": {"b": [1.0, {}], "a": None}, +} +"""A v3 array document that holds names nothing in scope claims. Their configurations are kept as written, since only their own definitions could simplify them, and so is the `endian` of the `bytes` codec, which receives values nothing is known of.""" + +_V3_GROUP: dict[str, Any] = {"zarr_format": 3, "node_type": "group"} + + +def _v3_listing(*documents: dict[str, Any]) -> dict[str, Any]: + return { + "kind": "inline", + "must_understand": False, + "metadata": {f"n{index}": document for index, document in enumerate(documents)}, + } + + +_V2_ARRAY: dict[str, Any] = { + "zarr_format": 2, + "shape": [4], + "chunks": [2], + "dtype": "|i1", + "compressor": {"id": "zstd", "level": 0, "checksum": False}, + "fill_value": 0, + "order": "C", + "filters": [{"id": "delta", "dtype": "|i1", "astype": "|i1"}], +} +"""A v2 array document in its canonical spelling, in which a codec writes each numcodecs default.""" + +_V2_ARRAY_SPELLED_OTHERWISE: dict[str, Any] = { + **_V2_ARRAY, + "dtype": " dict[str, Any]: + return {key: value for key, value in document.items() if key != "attributes"} + + +@pytest.mark.parametrize( + ("model", "document", "canonical"), + [ + (ZarrV3ArrayMetadata, _V3_ARRAY, _V3_ARRAY), + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _V3_ARRAY_SIMPLEST), + ( + ZarrV3ArrayMetadata, + {**_V3_FLOATS, "fill_value": "0x7fc00000"}, + {**_V3_FLOATS, "fill_value": "NaN"}, + ), + (ZarrV3ArrayMetadata, _V3_UNCLAIMED, _V3_UNCLAIMED), + (ZarrV3GroupMetadata, _V3_GROUP, _V3_GROUP), + ( + ZarrV3GroupMetadata, + {"attributes": {}, **_V3_GROUP, "acme": _WAIVED}, + {**_V3_GROUP, "acme": _WAIVED}, + ), + ( + ZarrV3GroupMetadata, + {**_V3_GROUP, "attributes": {"a": 1}}, + {**_V3_GROUP, "attributes": {"a": 1}}, + ), + # A group does not read the documents its consolidated metadata lists. + ( + ZarrV3GroupMetadata, + {**_V3_GROUP, "consolidated_metadata": _v3_listing(_V3_ARRAY_SPELLED_OUT)}, + {**_V3_GROUP, "consolidated_metadata": _v3_listing(_V3_ARRAY_SPELLED_OUT)}, + ), + ( + ZarrV3ConsolidatedMetadata, + _v3_listing(_V3_ARRAY_SPELLED_OUT, {**_V3_GROUP, "attributes": {}}), + _v3_listing(_V3_ARRAY_SIMPLEST, _V3_GROUP), + ), + (ZarrV2ArrayMetadata, _V2_ARRAY, _V2_ARRAY), + (ZarrV2ArrayMetadata, _V2_ARRAY_SPELLED_OTHERWISE, _V2_ARRAY), + (ZarrV2ArrayMetadata, _V2_FLOATS, _V2_FLOATS), + (ZarrV2GroupMetadata, _V2_GROUP, _V2_GROUP), + (ZarrV2GroupMetadata, {**_V2_GROUP, "attributes": {}}, _V2_GROUP), + ( + ZarrV2GroupMetadata, + {**_V2_GROUP, "attributes": {"a": 1}}, + {**_V2_GROUP, "attributes": {"a": 1}}, + ), + ( + ZarrV2ConsolidatedMetadata, + { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": _V2_GROUP, + ".zattrs": {}, + "a/.zarray": _v2_array_entry(_V2_ARRAY_SPELLED_OTHERWISE), + "a/.zattrs": {"x": 1}, + "b/.zarray": _v2_array_entry(_V2_ARRAY_SPELLED_OTHERWISE), + "b/.zattrs": {}, + "c/.zattrs": {}, + "notes": [1], + }, + }, + { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": _V2_GROUP, + "a/.zarray": _V2_ARRAY, + "a/.zattrs": {"x": 1}, + "b/.zarray": _V2_ARRAY, + # No node is read from these entries, so they are kept as written. + "c/.zattrs": {}, + "notes": [1], + }, + }, + ), + ], + ids=[ + "v3-array-simplest", + "v3-array-spelled-out", + "v3-array-fill-value", + "v3-array-unclaimed", + "v3-group-simplest", + "v3-group-empty-attributes", + "v3-group-attributes", + "v3-group-consolidated", + "v3-consolidated", + "v2-array-canonical", + "v2-array-spelled-otherwise", + "v2-array-nothing-to-simplify", + "v2-group-simplest", + "v2-group-empty-attributes", + "v2-group-attributes", + "v2-consolidated", + ], +) +def test_a_model_canonicalizes_to_the_simplest_spelling_of_its_document( + model: type[Model], document: dict[str, Any], canonical: dict[str, Any] +) -> None: + """`canonicalize` returns an equal model of the same class, in the same scope, whose document is in its simplest equivalent spelling, with the members that remain in the order they were written. A document already in that spelling is returned as it is.""" + read = model.from_json(document) + result = read.canonicalize() + assert json.dumps(result.to_json()) == json.dumps(canonical) + assert type(result) is type(read) + assert result.context is read.context + assert result == read + assert hash(result) == hash(read) + assert json.dumps(result.canonicalize().to_json()) == json.dumps(canonical) + + +def _reordered(document: dict[str, Any]) -> dict[str, Any]: + """`document` with its members in the reverse order, and those of each object in its `attributes` and in each member the spec does not define.""" + + def reverse(value: object) -> object: + if isinstance(value, dict): + held = cast("dict[str, object]", value) + return {key: reverse(held[key]) for key in reversed(held)} + if isinstance(value, list): + return [reverse(item) for item in cast("list[object]", value)] + return value + + return { + key: reverse(document[key]) if key in ("attributes", "acme") else document[key] + for key in reversed(document) + } + + +@pytest.mark.parametrize( + ("model", "left", "right"), + [ + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _V3_ARRAY_SIMPLEST), + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _reordered(_V3_ARRAY_SPELLED_OUT)), + (ZarrV3ArrayMetadata, _V3_UNCLAIMED, _reordered(_V3_UNCLAIMED)), + (ZarrV3ArrayMetadata, {**_V3_ARRAY, "codecs": [_LITTLE]}, {**_V3_ARRAY, "codecs": [_BIG]}), + (ZarrV3GroupMetadata, {**_V3_GROUP, "attributes": {}}, _V3_GROUP), + ( + ZarrV3ConsolidatedMetadata, + _v3_listing(_V3_ARRAY_SPELLED_OUT, _V3_GROUP), + _v3_listing(_reordered(_V3_ARRAY_SIMPLEST), {**_V3_GROUP, "attributes": {}}), + ), + (ZarrV2ArrayMetadata, _V2_ARRAY_SPELLED_OTHERWISE, _reordered(_V2_ARRAY)), + (ZarrV2GroupMetadata, {**_V2_GROUP, "attributes": {}}, _V2_GROUP), + ], +) +def test_equal_models_canonicalize_to_one_document( + model: type[Model], left: dict[str, Any], right: dict[str, Any] +) -> None: + """Models that are equal give the same canonical document, apart from the order of the members of an object: a `JSONWriter` that sorts them writes the same bytes for each.""" + models = [model.from_json(document) for document in (left, right)] + assert models[0] == models[1] + canonical = [each.canonicalize() for each in models] + assert json_text(_json_of(canonical[0])) == json_text(_json_of(canonical[1])) + assert _stored(canonical[0]) == _stored(canonical[1]) + + +@pytest.mark.parametrize( + ("model", "left", "right"), + [ + ( + ZarrV3ArrayMetadata, + {**_V3_ARRAY, "data_type": "uint16", "codecs": [_LITTLE]}, + {**_V3_ARRAY, "data_type": "uint16", "codecs": [_BIG]}, + ), + (ZarrV3ArrayMetadata, _V3_ARRAY, {**_V3_ARRAY, "dimension_names": [None]}), + ( + ZarrV3ArrayMetadata, + {**_V3_FLOATS, "fill_value": 0.0}, + {**_V3_FLOATS, "fill_value": -0.0}, + ), + (ZarrV3GroupMetadata, _V3_GROUP, {**_V3_GROUP, "attributes": {"a": 1}}), + (ZarrV2ArrayMetadata, _V2_ARRAY, {**_V2_ARRAY, "dimension_separator": "/"}), + (ZarrV2ArrayMetadata, {**_V2_ARRAY, "filters": None}, {**_V2_ARRAY, "filters": []}), + ], +) +def test_models_that_differ_canonicalize_to_different_documents( + model: type[Model], left: dict[str, Any], right: dict[str, Any] +) -> None: + """Only spellings that mean the same thing are reduced to one: models that are not equal keep documents that differ.""" + models = [model.from_json(document) for document in (left, right)] + assert models[0] != models[1] + written = [json_text(_json_of(each.canonicalize())) for each in models] + assert written[0] != written[1] + + +def _json_of(model: Model) -> JSONValue: + document: Any = model.to_json() + return document + + +def _stored(model: Model) -> object: + """The bytes a store holds for `model`, with the members of every object sorted; None for a v3 listing, which a group's document carries.""" + if isinstance(model, ZarrV3ConsolidatedMetadata): + return None + return model.to_key_value(JSONWriter(sort_keys=True)) + + +def _assert_canonicalizes( + model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata | ZarrV3ConsolidatedMetadata, +) -> None: + """`model` equals its canonical model, whose document has nothing left to simplify and reads as the same model from the JSON text a store holds.""" + canonical = model.canonicalize() + assert canonical == model + assert hash(canonical) == hash(model) + written = json.dumps(canonical.to_json()) + assert json.dumps(canonical.canonicalize().to_json()) == written + assert type(model).from_json(json.loads(written), context=ZARR_PYTHON_CONTEXT) == model + + +@pytest.mark.parametrize("spellings", ["all", "canonical"]) +@given(data=st.data()) +def test_every_array_document_canonicalizes(spellings: Spellings, data: st.DataObject) -> None: + document = data.draw(array_documents(spellings=spellings)) + _assert_canonicalizes(ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT)) + + +@given(document=group_documents()) +def test_every_group_document_canonicalizes(document: Mapping[str, Any]) -> None: + _assert_canonicalizes(ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT)) + + +@given(document=consolidated_group_documents()) +def test_every_consolidated_group_document_canonicalizes( + document: Mapping[str, Any], +) -> None: + group = ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT) + _assert_canonicalizes(group) + listing = group.consolidated() + assert listing is not None + _assert_canonicalizes(listing) + # The listing in its simplest spelling can be written back into the group. + _assert_canonicalizes(group.replace(consolidated_metadata=listing.canonicalize())) From 070840fc5f72fbcf01edefc5dc638461566771af Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 16:29:23 +0100 Subject: [PATCH 3/6] docs(zarr-metadata): name the changelog fragment after the upstream PR Assisted-by: ClaudeCode:claude-fable-5-2 --- .../zarr-metadata/changes/{450.feature.md => 4640.feature.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{450.feature.md => 4640.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/450.feature.md b/packages/zarr-metadata/changes/4640.feature.md similarity index 100% rename from packages/zarr-metadata/changes/450.feature.md rename to packages/zarr-metadata/changes/4640.feature.md From fa27e3807bed26454f3ea1825397beabe81ab623 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 16:29:55 +0100 Subject: [PATCH 4/6] docs(zarr-metadata): name the changelog fragment after the upstream PR Assisted-by: ClaudeCode:claude-fable-5-2 --- .../zarr-metadata/changes/{452.feature.md => 4642.feature.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{452.feature.md => 4642.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/452.feature.md b/packages/zarr-metadata/changes/4642.feature.md similarity index 100% rename from packages/zarr-metadata/changes/452.feature.md rename to packages/zarr-metadata/changes/4642.feature.md From 64c89a7648625f1fc92e91861257e6ad3885930f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 17 Nov 2026 22:33:09 +0100 Subject: [PATCH 5/6] test(zarr-metadata): check what canonicalize simplifies, in scopes that are not the default A review found that the Hypothesis tests passed with a canonicalize that returned the model's own document, that no test noticed a model losing its scope, and that four behaviours had no case: a claimed extra field, a v2 fill value, a chunk grid and a storage transformer that are respelled. The parametrized cases are now read in scopes that are not the default ones. The generated documents are compared with the same document reordered at every depth and, for arrays, with each declaration in its own simplest spelling. The reordering helper reverses every object, which the fix in #4641 allows. Assisted-by: ClaudeCode:claude-fable-5-2 --- .../tests/model/test_canonicalize.py | 206 ++++++++++++++++-- 1 file changed, 189 insertions(+), 17 deletions(-) diff --git a/packages/zarr-metadata/tests/model/test_canonicalize.py b/packages/zarr-metadata/tests/model/test_canonicalize.py index ff7b92cabd..25dda00891 100644 --- a/packages/zarr-metadata/tests/model/test_canonicalize.py +++ b/packages/zarr-metadata/tests/model/test_canonicalize.py @@ -9,13 +9,15 @@ from __future__ import annotations import json -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, NotRequired, TypeVar, cast import pytest from hypothesis import given from hypothesis import strategies as st +from typing_extensions import TypedDict, TypeIs from zarr_metadata._json import json_text +from zarr_metadata.definition._definition import canonicalize from zarr_metadata.model import ( JSONWriter, ZarrV2ArrayMetadata, @@ -31,7 +33,19 @@ consolidated_group_documents, group_documents, ) -from zarr_metadata.v3.definition import ZARR_PYTHON_CONTEXT +from zarr_metadata.v2.definition import CORE_V2_CONTEXT, ZarrV2Context +from zarr_metadata.v3.definition import ( + CORE_AND_EXTENSIONS_CONTEXT, + ZARR_PYTHON_CONTEXT, + ArrayExtraFieldDefinition, + ChunkGridDefinition, + ChunkKeyEncodingDefinition, + CodecDefinition, + DataTypeDefinition, + GroupExtraFieldDefinition, + StorageTransformerDefinition, + ZarrV3Context, +) if TYPE_CHECKING: from collections.abc import Mapping @@ -47,6 +61,50 @@ | ZarrV3GroupMetadata ) + +class AcmeLevelConfiguration(TypedDict, closed=True): + level: NotRequired[int] + + +class AcmeNoteConfiguration(TypedDict, closed=True): + must_understand: NotRequired[bool] + level: NotRequired[int] + + +C = TypeVar("C", AcmeLevelConfiguration, AcmeNoteConfiguration) + + +def _without_level_0(configuration: C) -> C: + """Without a `level` of 0, which is what both definitions here do without a `level`.""" + if configuration.get("level") != 0: + return configuration + return cast("C", {key: value for key, value in configuration.items() if key != "level"}) + + +_V3_CONTEXT = CORE_AND_EXTENSIONS_CONTEXT.union( + ZarrV3Context( + array_extra_fields=[ + ArrayExtraFieldDefinition( + name="note", configuration=AcmeNoteConfiguration, canonical=_without_level_0 + ) + ], + group_extra_fields=[ + GroupExtraFieldDefinition( + name="note", configuration=AcmeNoteConfiguration, canonical=_without_level_0 + ) + ], + storage_transformers=[ + StorageTransformerDefinition( + name="acme.st", configuration=AcmeLevelConfiguration, canonical=_without_level_0 + ) + ], + ) +) +"""The scope the v3 documents here are read in. It is not the default scope: it also claims an extra field `note` and a storage transformer `acme.st`, each with a `canonical`.""" + +_V2_CONTEXT = CORE_V2_CONTEXT.union(ZarrV2Context()) +"""The scope the v2 documents here are read in: the default scope as another object, which tells whether a model kept the scope it was read in.""" + _LITTLE: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "little"}} _BIG: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "big"}} _NOSHUFFLE = {"cname": "lz4", "clevel": 5, "shuffle": "noshuffle", "blocksize": 0} @@ -123,6 +181,39 @@ } """A v3 array document that holds names nothing in scope claims. Their configurations are kept as written, since only their own definitions could simplify them, and so is the `endian` of the `bytes` codec, which receives values nothing is known of.""" +_V3_UNCLAIMED_ENVELOPES: dict[str, Any] = { + **_V3_ARRAY, + "data_type": {"name": "acme.t", "configuration": {}}, + "fill_value": "0x00", + "codecs": [ + {"name": "acme.codec", "configuration": {}}, + "acme.other", + {"name": "acme.third", "must_understand": True, "configuration": {"b": 1, "a": {}}}, + _LITTLE, + ], +} +"""A v3 array document that holds names nothing in scope claims, each in an envelope that has a shorter form.""" + +_V3_UNCLAIMED_SHORTEST: dict[str, Any] = { + **_V3_ARRAY, + "data_type": "acme.t", + "fill_value": "0x00", + "codecs": [ + {"name": "acme.codec"}, + {"name": "acme.other"}, + {"name": "acme.third", "configuration": {"b": 1, "a": {}}}, + _LITTLE, + ], +} + + +def _rectilinear(*chunk_shapes: list[int] | list[list[int]]) -> dict[str, Any]: + return { + "name": "rectilinear", + "configuration": {"kind": "inline", "chunk_shapes": list(chunk_shapes)}, + } + + _V3_GROUP: dict[str, Any] = {"zarr_format": 3, "node_type": "group"} @@ -184,6 +275,21 @@ def _v2_array_entry(document: dict[str, Any]) -> dict[str, Any]: {**_V3_FLOATS, "fill_value": "NaN"}, ), (ZarrV3ArrayMetadata, _V3_UNCLAIMED, _V3_UNCLAIMED), + (ZarrV3ArrayMetadata, _V3_UNCLAIMED_ENVELOPES, _V3_UNCLAIMED_SHORTEST), + ( + ZarrV3ArrayMetadata, + {**_V3_ARRAY, "shape": [6], "chunk_grid": _rectilinear([2, 2, 2])}, + {**_V3_ARRAY, "shape": [6], "chunk_grid": _rectilinear([[2, 3]])}, + ), + ( + ZarrV3ArrayMetadata, + { + **_V3_ARRAY, + "storage_transformers": [{"name": "acme.st", "configuration": {"level": 0}}], + }, + {**_V3_ARRAY, "storage_transformers": [{"name": "acme.st"}]}, + ), + (ZarrV3ArrayMetadata, {**_V3_ARRAY, "note": {"level": 0}}, {**_V3_ARRAY, "note": {}}), (ZarrV3GroupMetadata, _V3_GROUP, _V3_GROUP), ( ZarrV3GroupMetadata, @@ -195,6 +301,7 @@ def _v2_array_entry(document: dict[str, Any]) -> dict[str, Any]: {**_V3_GROUP, "attributes": {"a": 1}}, {**_V3_GROUP, "attributes": {"a": 1}}, ), + (ZarrV3GroupMetadata, {**_V3_GROUP, "note": {"level": 0}}, {**_V3_GROUP, "note": {}}), # A group does not read the documents its consolidated metadata lists. ( ZarrV3GroupMetadata, @@ -209,6 +316,12 @@ def _v2_array_entry(document: dict[str, Any]) -> dict[str, Any]: (ZarrV2ArrayMetadata, _V2_ARRAY, _V2_ARRAY), (ZarrV2ArrayMetadata, _V2_ARRAY_SPELLED_OTHERWISE, _V2_ARRAY), (ZarrV2ArrayMetadata, _V2_FLOATS, _V2_FLOATS), + (ZarrV2ArrayMetadata, {**_V2_FLOATS, "fill_value": 0}, {**_V2_FLOATS, "fill_value": 0.0}), + ( + ZarrV2ArrayMetadata, + {**_V2_FLOATS, "fill_value": 1e39}, + {**_V2_FLOATS, "fill_value": "Infinity"}, + ), (ZarrV2GroupMetadata, _V2_GROUP, _V2_GROUP), (ZarrV2GroupMetadata, {**_V2_GROUP, "attributes": {}}, _V2_GROUP), ( @@ -250,14 +363,21 @@ def _v2_array_entry(document: dict[str, Any]) -> dict[str, Any]: "v3-array-spelled-out", "v3-array-fill-value", "v3-array-unclaimed", + "v3-array-unclaimed-envelopes", + "v3-array-chunk-grid", + "v3-array-storage-transformer", + "v3-array-claimed-extra-field", "v3-group-simplest", "v3-group-empty-attributes", "v3-group-attributes", + "v3-group-claimed-extra-field", "v3-group-consolidated", "v3-consolidated", "v2-array-canonical", "v2-array-spelled-otherwise", "v2-array-nothing-to-simplify", + "v2-array-fill-value", + "v2-array-fill-value-out-of-range", "v2-group-simplest", "v2-group-empty-attributes", "v2-group-attributes", @@ -268,18 +388,32 @@ def test_a_model_canonicalizes_to_the_simplest_spelling_of_its_document( model: type[Model], document: dict[str, Any], canonical: dict[str, Any] ) -> None: """`canonicalize` returns an equal model of the same class, in the same scope, whose document is in its simplest equivalent spelling, with the members that remain in the order they were written. A document already in that spelling is returned as it is.""" - read = model.from_json(document) + read = _read(model, document) result = read.canonicalize() assert json.dumps(result.to_json()) == json.dumps(canonical) assert type(result) is type(read) - assert result.context is read.context + # Neither scope here is the default one, so a model that lost its scope would show. + assert result.context is (_V2_CONTEXT if _is_v2(model) else _V3_CONTEXT) assert result == read assert hash(result) == hash(read) assert json.dumps(result.canonicalize().to_json()) == json.dumps(canonical) -def _reordered(document: dict[str, Any]) -> dict[str, Any]: - """`document` with its members in the reverse order, and those of each object in its `attributes` and in each member the spec does not define.""" +def _is_v2( + model: type[Model], +) -> TypeIs[type[ZarrV2ArrayMetadata | ZarrV2ConsolidatedMetadata | ZarrV2GroupMetadata]]: + return issubclass(model, (ZarrV2ArrayMetadata, ZarrV2ConsolidatedMetadata, ZarrV2GroupMetadata)) + + +def _read(model: type[Model], document: Mapping[str, Any]) -> Model: + """The model of `document`, read in `_V2_CONTEXT` or `_V3_CONTEXT`.""" + if _is_v2(model): + return model.from_json(document, context=_V2_CONTEXT) + return model.from_json(document, context=_V3_CONTEXT) + + +def _reordered(document: Mapping[str, Any]) -> dict[str, Any]: + """`document` with the members of every object it holds, at any depth, in the reverse order. The items of an array keep their order.""" def reverse(value: object) -> object: if isinstance(value, dict): @@ -289,18 +423,20 @@ def reverse(value: object) -> object: return [reverse(item) for item in cast("list[object]", value)] return value - return { - key: reverse(document[key]) if key in ("attributes", "acme") else document[key] - for key in reversed(document) - } + return cast("dict[str, Any]", reverse(dict(document))) @pytest.mark.parametrize( ("model", "left", "right"), [ (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _V3_ARRAY_SIMPLEST), - (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _reordered(_V3_ARRAY_SPELLED_OUT)), - (ZarrV3ArrayMetadata, _V3_UNCLAIMED, _reordered(_V3_UNCLAIMED)), + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _reordered(_V3_ARRAY_SIMPLEST)), + (ZarrV3ArrayMetadata, _V3_UNCLAIMED_ENVELOPES, _reordered(_V3_UNCLAIMED_SHORTEST)), + ( + ZarrV3ArrayMetadata, + {**_V3_UNCLAIMED, "storage_transformers": []}, + _reordered(_V3_UNCLAIMED), + ), (ZarrV3ArrayMetadata, {**_V3_ARRAY, "codecs": [_LITTLE]}, {**_V3_ARRAY, "codecs": [_BIG]}), (ZarrV3GroupMetadata, {**_V3_GROUP, "attributes": {}}, _V3_GROUP), ( @@ -316,7 +452,7 @@ def test_equal_models_canonicalize_to_one_document( model: type[Model], left: dict[str, Any], right: dict[str, Any] ) -> None: """Models that are equal give the same canonical document, apart from the order of the members of an object: a `JSONWriter` that sorts them writes the same bytes for each.""" - models = [model.from_json(document) for document in (left, right)] + models = [_read(model, document) for document in (left, right)] assert models[0] == models[1] canonical = [each.canonicalize() for each in models] assert json_text(_json_of(canonical[0])) == json_text(_json_of(canonical[1])) @@ -346,7 +482,7 @@ def test_models_that_differ_canonicalize_to_different_documents( model: type[Model], left: dict[str, Any], right: dict[str, Any] ) -> None: """Only spellings that mean the same thing are reduced to one: models that are not equal keep documents that differ.""" - models = [model.from_json(document) for document in (left, right)] + models = [_read(model, document) for document in (left, right)] assert models[0] != models[1] written = [json_text(_json_of(each.canonicalize())) for each in models] assert written[0] != written[1] @@ -358,22 +494,33 @@ def _json_of(model: Model) -> JSONValue: def _stored(model: Model) -> object: - """The bytes a store holds for `model`, with the members of every object sorted; None for a v3 listing, which a group's document carries.""" + """The bytes a store holds for `model`, with the members of every object sorted. A v3 listing is stored in the document of a group.""" if isinstance(model, ZarrV3ConsolidatedMetadata): - return None + group = ZarrV3GroupMetadata.from_json(_V3_GROUP, context=model.context) + model = group.replace(consolidated_metadata=model) return model.to_key_value(JSONWriter(sort_keys=True)) def _assert_canonicalizes( model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata | ZarrV3ConsolidatedMetadata, ) -> None: - """`model` equals its canonical model, whose document has nothing left to simplify and reads as the same model from the JSON text a store holds.""" + """`model` equals its canonical model, which is in the same scope, has nothing left to simplify and reads as the same model from the JSON text a store holds. The document of `model` with the members of every object in the reverse order gives the same canonical document.""" canonical = model.canonicalize() assert canonical == model + assert canonical.context is model.context assert hash(canonical) == hash(model) written = json.dumps(canonical.to_json()) assert json.dumps(canonical.canonicalize().to_json()) == written assert type(model).from_json(json.loads(written), context=ZARR_PYTHON_CONTEXT) == model + if not isinstance(canonical, ZarrV3ConsolidatedMetadata): + document = cast("Mapping[str, Any]", _json_of(canonical)) + for member in ("attributes", "storage_transformers"): + assert member not in document or len(document[member]) != 0 + reordered = type(model).from_json( + _reordered(json.loads(json.dumps(model.to_json()))), context=ZARR_PYTHON_CONTEXT + ) + assert reordered == model + assert json_text(_json_of(reordered.canonicalize())) == json_text(_json_of(canonical)) @pytest.mark.parametrize("spellings", ["all", "canonical"]) @@ -383,6 +530,31 @@ def test_every_array_document_canonicalizes(spellings: Spellings, data: st.DataO _assert_canonicalizes(ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT)) +@given(document=array_documents(spellings="all")) +def test_every_array_document_canonicalizes_as_its_declarations_do( + document: Mapping[str, Any], +) -> None: + """A document, and the same document with each declaration in its own simplest spelling, are one array, and give one canonical document.""" + simpler = dict(document) + for member, kind in ( + ("data_type", DataTypeDefinition), + ("chunk_grid", ChunkGridDefinition), + ("chunk_key_encoding", ChunkKeyEncodingDefinition), + ): + simpler[member], problems = canonicalize(document[member], kind, ZARR_PYTHON_CONTEXT) + assert problems == () + simpler["codecs"] = [ + canonicalize(codec, CodecDefinition, ZARR_PYTHON_CONTEXT)[0] for codec in document["codecs"] + ] + models = [ + ZarrV3ArrayMetadata.from_json(each, context=ZARR_PYTHON_CONTEXT) + for each in (document, simpler) + ] + assert models[0] == models[1] + canonical = [json_text(_json_of(each.canonicalize())) for each in models] + assert canonical[0] == canonical[1] + + @given(document=group_documents()) def test_every_group_document_canonicalizes(document: Mapping[str, Any]) -> None: _assert_canonicalizes(ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT)) From 34c3378aa2365763c0c25ab84b47316fc7e58dbc Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 17 Nov 2026 22:33:10 +0100 Subject: [PATCH 6/6] docs(zarr-metadata): say how canonicalize writes a declaration nothing claims Its configuration is kept, but its envelope is written in the shortest form: an empty configuration and a must_understand of true are left out. Assisted-by: ClaudeCode:claude-fable-5-2 --- packages/zarr-metadata/docs/models.md | 4 +++- .../zarr-metadata/src/zarr_metadata/model/_array_v3.py | 9 ++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index 16c4e4378a..4e7d3ac3c8 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -315,7 +315,9 @@ The canonical document is written from the same parts that `==` compares: `dimension_separator` of `"."`. - Everything else is kept as written. This includes the attributes, and the configuration of a declaration that nothing in the context claims, - because only its own definition could simplify it. + because only its own definition could simplify it. Such a declaration + is still written in the shortest form every reader accepts, without an + empty `configuration` or a `must_understand` of `true`. A v2 codec is written with each numcodecs default it leaves out, as its definition's `canonical` writes it. So a canonical v2 document can be diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 1d15ef74fb..c0fa12a402 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -753,9 +753,12 @@ def canonicalize(self) -> ZarrV3ArrayMetadata: data type. `attributes` and `storage_transformers` are left out when empty, because a document without them means the same. Everything else is kept as written: `shape`, `dimension_names`, - the attributes, and whatever nothing in scope claims, which only - its own definition could simplify. A document that is already in - this spelling is returned as it is, in a new model. + the attributes, and the configuration of a declaration nothing in + scope claims, which only its own definition could simplify. Such + a declaration is still written in the shortest form every reader + accepts, without an empty `configuration` or a `must_understand` + of `true`. A document that is already in this spelling is + returned as it is, in a new model. """ document = canonical_array_document_v3(self._reading, self._members) return type(self)(document, context=self._context)