diff --git a/packages/zarr-metadata/changes/4642.feature.md b/packages/zarr-metadata/changes/4642.feature.md new file mode 100644 index 0000000000..a79317f00b --- /dev/null +++ b/packages/zarr-metadata/changes/4642.feature.md @@ -0,0 +1 @@ +Each model has a `canonicalize` method that returns the model with its document in its simplest equivalent spelling: `ZarrV3ArrayMetadata`, `ZarrV3GroupMetadata`, `ZarrV3ConsolidatedMetadata`, `ZarrV2ArrayMetadata`, `ZarrV2GroupMetadata` and `ZarrV2ConsolidatedMetadata`. The result equals the model. Every model equal to it, in the same scope, gives the same document, apart from the order of the members of an object, so `to_key_value` with `JSONWriter(sort_keys=True)` writes the same bytes for equal models. The document is written from the parts `==` compares: each declaration as `canonical_of` writes it, each v3 codec as `codec_for_chunk` returns it for the chunk it receives, and the fill value as `canonical_fill_value` writes it. Empty `attributes`, empty v3 `storage_transformers` and a v2 `dimension_separator` of `"."` are left out, and everything else is kept as written. diff --git a/packages/zarr-metadata/docs/models.md b/packages/zarr-metadata/docs/models.md index ad2a700f6c..16c4e4378a 100644 --- a/packages/zarr-metadata/docs/models.md +++ b/packages/zarr-metadata/docs/models.md @@ -130,6 +130,8 @@ To change a model, use one of these methods: - `refined_in` also reads the document in another context. The new context must claim what this context left unclaimed and contradict nothing. Otherwise it raises `ScopeConflictError`. +- `canonicalize` returns the model with its document in its simplest + equivalent spelling. See [Canonical documents](#canonical-documents). ## Writing store bytes @@ -264,3 +266,76 @@ equal to itself. Equal models have equal hashes. They may still write different documents, because `to_json` writes each document as it was given. + +## Canonical documents + +`canonicalize` returns a model with its document in its simplest +equivalent spelling. The result equals the model it came from. Every model +equal to that one, read in the same context, gives the same document. + +```python test="true" session="models" +from zarr_metadata.model import ZarrV3ArrayMetadata + +written = ZarrV3ArrayMetadata.from_json( + { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, + "fill_value": 0, + "codecs": [ + {"name": "bytes", "configuration": {"endian": "little"}}, + {"name": "zstd", "configuration": {"level": 1, "checksum": False}}, + ], + "attributes": {}, + } +) +canonical = written.canonicalize() +assert canonical == written +print(canonical.to_json()["chunk_key_encoding"]) +#> {'name': 'default'} +print(canonical.to_json()["codecs"]) +#> ({'name': 'bytes'}, {'name': 'zstd', 'configuration': {'level': 1}}) +print("attributes" in canonical.to_json()) +#> False +``` + +The canonical document is written from the same parts that `==` compares: + +- Each declaration is written as `canonical_of` writes it. +- Each v3 codec is written for the chunk it receives, as `codec_for_chunk` + returns it. Above, each `uint8` value takes one byte, so the `endian` + says nothing. +- The fill value is written as `canonical_fill_value` writes it for the + data type. +- A member that says what its absence says is left out. These are empty + `attributes`, empty v3 `storage_transformers`, and a v2 + `dimension_separator` of `"."`. +- Everything else is kept as written. This includes the attributes, and + the configuration of a declaration that nothing in the context claims, + because only its own definition could simplify it. + +A v2 codec is written with each numcodecs default it leaves out, as its +definition's `canonical` writes it. So a canonical v2 document can be +longer than the one it came from. + +The order of the members of an object is not part of the spelling, because +JSON gives it no meaning. Two equal models can give canonical documents +whose members are in different orders. A `JSONWriter` that sorts them +writes the same bytes for both, so those bytes are a key for what the +document means: + +```python test="true" session="models" +import hashlib + +stored = canonical.to_key_value({"sort_keys": True})["zarr.json"] +digest = hashlib.sha256(stored).hexdigest() +``` + +The other models work the same way. A v3 group does not read the documents +its `consolidated_metadata` lists, so its `canonicalize` keeps them as +written. `ZarrV3ConsolidatedMetadata.canonicalize` and +`ZarrV2ConsolidatedMetadata.canonicalize` write each node they list in its +simplest spelling. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py index 39f4ca5e58..b18d1eb092 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/__init__.py @@ -42,7 +42,10 @@ scope and raises `MetadataValidationError` with every problem, so an invalid model is never built. `to_json` returns the document as written. `replace` reads new members in the model's own scope. `with_context` and -`refined_in` read the document in another scope. `to_key_value` writes +`refined_in` read the document in another scope. `canonicalize` returns +the model with its document in its simplest equivalent spelling, the one +that every equal model gives, apart from the order of the members of an +object. `to_key_value` writes the model as it is, with a `JSONWriter`, or a mapping of its settings, deciding how the bytes are spelled (indentation, key order, ASCII escaping, trailing newline); the default is that of `json.dumps`. A v3 group reads its diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py index 2e5ba2930e..dc39bdfc6f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py @@ -50,6 +50,7 @@ Derived, StoreKey, attributes_of, + canonical_members, check_literal, created, dimension_lengths, @@ -59,6 +60,7 @@ missing_keys, other_members, reading_of, + simplest, unclaimed_declarations_of, ) @@ -772,6 +774,39 @@ def __replace__(self, /, **members: Unpack[ZarrV2ArrayMetadataUpdate]) -> ZarrV2 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV2ArrayMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same document: `to_json` of each result is + the same JSON, apart from the order of the members of an object, + which `to_key_value` sorts with `JSONWriter(sort_keys=True)`. So + the bytes that writer writes are the same for equal models, and + are a key for what the document means. + + The dtype, the compressor and each filter are written as + `canonical_of` writes them, and the fill value as + `canonical_fill_value` writes it for the dtype. A + `dimension_separator` of `"."` is left out, and so are empty + `attributes`, because a document without them means the same. + Everything else is kept as written: `shape`, `chunks`, `order`, + the attributes, and whatever nothing in scope claims. + """ + compressor, filters = self.compressor, self.filters + spelled: dict[str, JSONValue] = { + "dtype": simplest(self.dtype), + "fill_value": spelled_canonically(self.dtype, self._members.fill_value), + "compressor": None if compressor is None else simplest(compressor), + "filters": None if filters is None else tuple(simplest(entry) for entry in filters), + } + document = canonical_members( + self._document, + spelled, + empty=("attributes",), + defaults={"dimension_separator": ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR}, + ) + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 2ea2af7ccb..1d15ef74fb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -52,6 +52,7 @@ Derived, StoreKey, attributes_of, + canonical_members, check_literal, created, dimension_lengths, @@ -60,6 +61,8 @@ member_changes, missing_keys, reading_of, + simplest, + simplest_members, unclaimed_declarations_of, ) from zarr_metadata.v3._common import ZarrV3DeclarationJSON, ZarrV3ExtensionField @@ -734,6 +737,29 @@ def __replace__(self, /, **members: Unpack[ZarrV3ArrayMetadataUpdate]) -> ZarrV3 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV3ArrayMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same document: `to_json` of each result is + the same JSON, apart from the order of the members of an object, + which `to_key_value` sorts with `JSONWriter(sort_keys=True)`. So + the bytes that writer writes are the same for equal models, and + are a key for what the document means. + + Each declaration is written as `canonical_of` writes it, each + codec as `codec_for_chunk` returns it for the chunk it receives, + and the fill value as `canonical_fill_value` writes it for the + data type. `attributes` and `storage_transformers` are left out + when empty, because a document without them means the same. + Everything else is kept as written: `shape`, `dimension_names`, + the attributes, and whatever nothing in scope claims, which only + its own definition could simplify. A document that is already in + this spelling is returned as it is, in a new model. + """ + document = canonical_array_document_v3(self._reading, self._members) + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadata: """Return this document read in `context`, whatever that changes: a gain, a loss, or a conflict. @@ -951,6 +977,30 @@ def array_key_of( ) +def canonical_array_document_v3( + reading: ZarrV3ArrayMetadataReading, members: ArrayMembersV3 +) -> dict[str, JSONValue]: + """A v3 array document in its simplest equivalent spelling, from its `reading` and refined `members`, as `ZarrV3ArrayMetadata.canonicalize` reads it. + + It is written from the parts that `array_key_of` compares, so two + documents with one key are written the same way, apart from the order + of the members of an object. + """ + data_type = held(reading.data_type) + spelled: dict[str, JSONValue] = { + "data_type": simplest(data_type), + "chunk_grid": simplest(reading.chunk_grid), + "chunk_key_encoding": simplest(reading.chunk_key_encoding), + "fill_value": spelled_canonically(data_type, members.fill_value), + "codecs": tuple(simplest(codec) for codec in codecs_for_chunks(reading)), + "storage_transformers": tuple(simplest(entry) for entry in reading.storage_transformers), + **simplest_members(reading.extra_fields), + } + return canonical_members( + members.document, spelled, empty=("attributes", "storage_transformers") + ) + + def codecs_for_chunks( reading: ZarrV3ArrayMetadataReading, ) -> tuple[AcceptedDeclaration[CodecDefinition[Any]] | UnclaimedDeclaration, ...]: diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py index cbd34fb85f..3f185a0f9d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v2.py @@ -48,6 +48,7 @@ from zarr_metadata.model._keyed import Keyed from zarr_metadata.model._shared import ( StoreKey, + canonical_members, check_literal, load_store_json, located_conflicts, @@ -378,6 +379,11 @@ def __replace__(self, /, **members: Unpack[ZarrV2GroupMetadataUpdate]) -> ZarrV2 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV2GroupMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope: without `attributes` when they are empty, because a group with no `.zattrs` means the same. The result equals this model, and every model equal to this one gives the same document, apart from the order of the members of an object, which `to_key_value` sorts with `JSONWriter(sort_keys=True)`.""" + document = canonical_members(self._document, {}, empty=("attributes",)) + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2GroupMetadata: """Return this document read in `context`: the same group, holding that scope.""" context = scoped(context, CORE_V2_CONTEXT) @@ -586,6 +592,31 @@ def _located_declarations(self) -> Iterator[tuple[Loc, ResolvedDeclaration[Any]] def __reduce__(self) -> tuple[type[ZarrV2ConsolidatedMetadata], tuple[object, ZarrV2Context]]: return type(self), (self._document, self._context) + def canonicalize(self) -> ZarrV2ConsolidatedMetadata: + """Return this document with each node in its simplest equivalent spelling, as that node's `canonicalize` writes it, read in this model's own scope. + + Each node's `.zarray` or `.zgroup` entry is written without the + attributes, and its `.zattrs` entry holds them, or is left out + when they are empty. Every other entry is kept as written. The + result equals this model. Every model equal to this one, in this + scope, gives the same document, apart from the order of the + members of an object, which a `JSONWriter(sort_keys=True)` sorts. + """ + entries = dict(object_at(self._document, "metadata")) + for path, node in self._metadata.items(): + names = self._files[path] + written = cast("dict[str, JSONValue]", node.canonicalize().to_json()) + attributes = written.pop("attributes", None) + for name, key in names.items(): + if name != ZARR_V2_ATTRIBUTES_STORE_KEY: + entries[key] = written + elif attributes is None: + del entries[key] + else: + entries[key] = attributes + document: dict[str, JSONValue] = {**self._document, "metadata": entries} + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV2Context | None = None) -> ZarrV2ConsolidatedMetadata: """Return this document with every node read in `context`, whatever that changes. Raises `MetadataValidationError` when a node has a problem in `context`.""" context = scoped(context, CORE_V2_CONTEXT) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py index 9c5d9dc006..33e4cd82ab 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py @@ -67,12 +67,14 @@ from zarr_metadata.model._shared import ( StoreKey, attributes_of, + canonical_members, check_literal, load_store_json, located_conflicts, member_changes, missing_keys, reading_of, + simplest_members, unclaimed_declarations_of, unexpected_keys, ) @@ -392,6 +394,27 @@ def __replace__(self, /, **members: Unpack[ZarrV3GroupMetadataUpdate]) -> ZarrV3 """Return `self.replace(**members)`, which `copy.replace(model, **members)` (Python 3.13) calls. It calls `replace` rather than being it, so a subclass that overrides `replace` changes what `copy.replace` does too.""" return self.replace(**members) + def canonicalize(self) -> ZarrV3GroupMetadata: + """Return this model with its document in its simplest equivalent spelling, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same document: `to_json` of each result is + the same JSON, apart from the order of the members of an object, + which `to_key_value` sorts with `JSONWriter(sort_keys=True)`. + + `attributes` is left out when empty, because a document without + it means the same, and each extra field a definition in scope + claims is written as `canonical_of` writes it. Everything else is + kept as written. That includes the documents that + `consolidated_metadata` lists, which a group does not read: + `consolidated().canonicalize()` returns them in their simplest + spelling, and `replace(consolidated_metadata=...)` writes them + back into the group. + """ + spelled = simplest_members(self._reading.extra_fields) + document = canonical_members(self._document, spelled, empty=("attributes",)) + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3GroupMetadata: """Return this document read in `context`, whatever that changes. Raises `MetadataValidationError` when the document has a problem in `context`. The existing reading is reused when `context` reads every claim identically.""" context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) @@ -595,6 +618,20 @@ def _key_of(self) -> tuple[object, ...]: def __reduce__(self) -> tuple[type[ZarrV3ConsolidatedMetadata], tuple[object, ZarrV3Context]]: return type(self), (self._document, self._context) + def canonicalize(self) -> ZarrV3ConsolidatedMetadata: + """Return this member with each document it lists in its simplest equivalent spelling, as that node's `canonicalize` writes it, read in this model's own scope. + + The result equals this model. Every model equal to this one, in + this scope, gives the same member, apart from the order of the + members of an object, which a `JSONWriter(sort_keys=True)` sorts. + """ + metadata: dict[str, JSONValue] = { + path: cast("JSONValue", node.canonicalize().to_json()) + for path, node in self._metadata.items() + } + document: dict[str, JSONValue] = {**self._document, "metadata": metadata} + return type(self)(document, context=self._context) + def with_context(self, context: ZarrV3Context | None = None) -> ZarrV3ConsolidatedMetadata: """Return this member with every document read in `context`, whatever that changes. Raises `MetadataValidationError` when a document has a problem in `context`. The existing readings are reused when `context` reads every claim of every document identically.""" context = scoped(context, CORE_AND_EXTENSIONS_CONTEXT) diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py index a946c4a608..2bb83aa52e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_shared.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_shared.py @@ -24,6 +24,8 @@ from zarr_metadata.definition._definition import ( RefusedDeclaration, UnclaimedDeclaration, + canonical_of, + held, is_declaration, ) from zarr_metadata.definition._scope import Conflict, claim_key @@ -204,6 +206,50 @@ def created( raise +def simplest(declaration: ResolvedDeclaration[Any] | UNSET) -> JSONValue: + """`declaration`, from a read that found no problems, in its simplest equivalent spelling, as `canonical_of` writes it. Raises `TypeError` for a declaration with none, which such a read cannot produce.""" + spelled = canonical_of(held(declaration), ()) + if spelled is None: + msg = f"expected a declaration a read found nothing wrong with, got {declaration!r}" + raise TypeError(msg) + return spelled + + +def simplest_members(extra_fields: Mapping[str, ResolvedDeclaration[Any]]) -> dict[str, JSONValue]: + """The extra fields of a document that a read found no problems with, by key, each in its simplest equivalent spelling.""" + spelled: dict[str, JSONValue] = {} + for extra in extra_fields.values(): + # An extra field is written as the member it is: its key, and its value. + spelled.update(cast("Mapping[str, JSONValue]", simplest(extra))) + return spelled + + +def canonical_members( + document: Mapping[str, JSONValue], + spelled: Mapping[str, JSONValue], + *, + empty: Iterable[str] = (), + defaults: Mapping[str, JSONValue] | None = None, +) -> dict[str, JSONValue]: + """`document`, the document a model holds, with each member in `spelled` written as `spelled` writes it, and without the members that say what their absence says. + + A member in `empty` is left out when it is empty, and a member in + `defaults` when it has the value given there. The members keep the + order `document` writes them in. A member of `spelled` that + `document` does not write is not added. + """ + canonical = {key: spelled.get(key, value) for key, value in document.items()} + for key in empty: + value = canonical.get(key) + if (is_array(value) or is_object(value)) and len(value) == 0: + del canonical[key] + if defaults is not None: + for key, default in defaults.items(): + if key in canonical and canonical[key] == default: + del canonical[key] + return canonical + + def unclaimed_declarations_of( declarations: Iterable[tuple[Loc, ResolvedDeclaration[Any]]], waived: Iterable[str] ) -> dict[Loc, UnclaimedDeclaration]: diff --git a/packages/zarr-metadata/tests/model/test_canonicalize.py b/packages/zarr-metadata/tests/model/test_canonicalize.py new file mode 100644 index 0000000000..ff7b92cabd --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_canonicalize.py @@ -0,0 +1,401 @@ +"""`canonicalize`: each model with its document in its simplest equivalent spelling. + +Two models are equal when their documents mean the same thing, and +`to_json` writes each document as it was given. `canonicalize` returns the +one document that every model equal to a given one writes, apart from the +order of the members of an object, which a `JSONWriter` sorts. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, cast + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from zarr_metadata._json import json_text +from zarr_metadata.model import ( + JSONWriter, + ZarrV2ArrayMetadata, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, +) +from zarr_metadata.testing.strategies import ( + Spellings, + array_documents, + consolidated_group_documents, + group_documents, +) +from zarr_metadata.v3.definition import ZARR_PYTHON_CONTEXT + +if TYPE_CHECKING: + from collections.abc import Mapping + + from zarr_metadata import JSONValue + +Model = ( + ZarrV2ArrayMetadata + | ZarrV2ConsolidatedMetadata + | ZarrV2GroupMetadata + | ZarrV3ArrayMetadata + | ZarrV3ConsolidatedMetadata + | ZarrV3GroupMetadata +) + +_LITTLE: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "little"}} +_BIG: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "big"}} +_NOSHUFFLE = {"cname": "lz4", "clevel": 5, "shuffle": "noshuffle", "blocksize": 0} +_WAIVED = {"name": "acme.note", "must_understand": False, "configuration": {"b": 1, "a": 2}} + +_V3_ARRAY: dict[str, Any] = { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default"}, + "fill_value": 0, + "codecs": [{"name": "bytes"}], +} +"""A v3 array document in its simplest spelling.""" + +_V3_ARRAY_SPELLED_OUT: dict[str, Any] = { + "attributes": {}, + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": {"name": "uint8", "configuration": {}}, + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default", "configuration": {"separator": "/"}}, + "fill_value": 0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [2], + "index_location": "end", + "codecs": [_BIG, {"name": "blosc", "configuration": {**_NOSHUFFLE, "typesize": 4}}], + "index_codecs": [_LITTLE, {"name": "crc32c", "configuration": {}}], + }, + } + ], + "storage_transformers": [], + "dimension_names": [None], + "acme": _WAIVED, +} +"""A v3 array document that says in more words what `_V3_ARRAY_SIMPLEST` says.""" + +_V3_ARRAY_SIMPLEST: dict[str, Any] = { + "zarr_format": 3, + "node_type": "array", + "shape": [4], + "data_type": "uint8", + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": [4]}}, + "chunk_key_encoding": {"name": "default"}, + "fill_value": 0, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [2], + "codecs": [{"name": "bytes"}, {"name": "blosc", "configuration": _NOSHUFFLE}], + "index_codecs": [_LITTLE, {"name": "crc32c"}], + }, + } + ], + "dimension_names": [None], + "acme": _WAIVED, +} + +_V3_FLOATS: dict[str, Any] = {**_V3_ARRAY, "data_type": "float32", "codecs": [_LITTLE]} + +_V3_UNCLAIMED: dict[str, Any] = { + **_V3_ARRAY, + "data_type": {"name": "acme.t", "configuration": {"bits": 8, "signed": False}}, + "fill_value": "0x00", + "codecs": [{"name": "acme.codec", "configuration": {"level": 1}}, _LITTLE], + "attributes": {"b": [1.0, {}], "a": None}, +} +"""A v3 array document that holds names nothing in scope claims. Their configurations are kept as written, since only their own definitions could simplify them, and so is the `endian` of the `bytes` codec, which receives values nothing is known of.""" + +_V3_GROUP: dict[str, Any] = {"zarr_format": 3, "node_type": "group"} + + +def _v3_listing(*documents: dict[str, Any]) -> dict[str, Any]: + return { + "kind": "inline", + "must_understand": False, + "metadata": {f"n{index}": document for index, document in enumerate(documents)}, + } + + +_V2_ARRAY: dict[str, Any] = { + "zarr_format": 2, + "shape": [4], + "chunks": [2], + "dtype": "|i1", + "compressor": {"id": "zstd", "level": 0, "checksum": False}, + "fill_value": 0, + "order": "C", + "filters": [{"id": "delta", "dtype": "|i1", "astype": "|i1"}], +} +"""A v2 array document in its canonical spelling, in which a codec writes each numcodecs default.""" + +_V2_ARRAY_SPELLED_OTHERWISE: dict[str, Any] = { + **_V2_ARRAY, + "dtype": " dict[str, Any]: + return {key: value for key, value in document.items() if key != "attributes"} + + +@pytest.mark.parametrize( + ("model", "document", "canonical"), + [ + (ZarrV3ArrayMetadata, _V3_ARRAY, _V3_ARRAY), + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _V3_ARRAY_SIMPLEST), + ( + ZarrV3ArrayMetadata, + {**_V3_FLOATS, "fill_value": "0x7fc00000"}, + {**_V3_FLOATS, "fill_value": "NaN"}, + ), + (ZarrV3ArrayMetadata, _V3_UNCLAIMED, _V3_UNCLAIMED), + (ZarrV3GroupMetadata, _V3_GROUP, _V3_GROUP), + ( + ZarrV3GroupMetadata, + {"attributes": {}, **_V3_GROUP, "acme": _WAIVED}, + {**_V3_GROUP, "acme": _WAIVED}, + ), + ( + ZarrV3GroupMetadata, + {**_V3_GROUP, "attributes": {"a": 1}}, + {**_V3_GROUP, "attributes": {"a": 1}}, + ), + # A group does not read the documents its consolidated metadata lists. + ( + ZarrV3GroupMetadata, + {**_V3_GROUP, "consolidated_metadata": _v3_listing(_V3_ARRAY_SPELLED_OUT)}, + {**_V3_GROUP, "consolidated_metadata": _v3_listing(_V3_ARRAY_SPELLED_OUT)}, + ), + ( + ZarrV3ConsolidatedMetadata, + _v3_listing(_V3_ARRAY_SPELLED_OUT, {**_V3_GROUP, "attributes": {}}), + _v3_listing(_V3_ARRAY_SIMPLEST, _V3_GROUP), + ), + (ZarrV2ArrayMetadata, _V2_ARRAY, _V2_ARRAY), + (ZarrV2ArrayMetadata, _V2_ARRAY_SPELLED_OTHERWISE, _V2_ARRAY), + (ZarrV2ArrayMetadata, _V2_FLOATS, _V2_FLOATS), + (ZarrV2GroupMetadata, _V2_GROUP, _V2_GROUP), + (ZarrV2GroupMetadata, {**_V2_GROUP, "attributes": {}}, _V2_GROUP), + ( + ZarrV2GroupMetadata, + {**_V2_GROUP, "attributes": {"a": 1}}, + {**_V2_GROUP, "attributes": {"a": 1}}, + ), + ( + ZarrV2ConsolidatedMetadata, + { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": _V2_GROUP, + ".zattrs": {}, + "a/.zarray": _v2_array_entry(_V2_ARRAY_SPELLED_OTHERWISE), + "a/.zattrs": {"x": 1}, + "b/.zarray": _v2_array_entry(_V2_ARRAY_SPELLED_OTHERWISE), + "b/.zattrs": {}, + "c/.zattrs": {}, + "notes": [1], + }, + }, + { + "zarr_consolidated_format": 1, + "metadata": { + ".zgroup": _V2_GROUP, + "a/.zarray": _V2_ARRAY, + "a/.zattrs": {"x": 1}, + "b/.zarray": _V2_ARRAY, + # No node is read from these entries, so they are kept as written. + "c/.zattrs": {}, + "notes": [1], + }, + }, + ), + ], + ids=[ + "v3-array-simplest", + "v3-array-spelled-out", + "v3-array-fill-value", + "v3-array-unclaimed", + "v3-group-simplest", + "v3-group-empty-attributes", + "v3-group-attributes", + "v3-group-consolidated", + "v3-consolidated", + "v2-array-canonical", + "v2-array-spelled-otherwise", + "v2-array-nothing-to-simplify", + "v2-group-simplest", + "v2-group-empty-attributes", + "v2-group-attributes", + "v2-consolidated", + ], +) +def test_a_model_canonicalizes_to_the_simplest_spelling_of_its_document( + model: type[Model], document: dict[str, Any], canonical: dict[str, Any] +) -> None: + """`canonicalize` returns an equal model of the same class, in the same scope, whose document is in its simplest equivalent spelling, with the members that remain in the order they were written. A document already in that spelling is returned as it is.""" + read = model.from_json(document) + result = read.canonicalize() + assert json.dumps(result.to_json()) == json.dumps(canonical) + assert type(result) is type(read) + assert result.context is read.context + assert result == read + assert hash(result) == hash(read) + assert json.dumps(result.canonicalize().to_json()) == json.dumps(canonical) + + +def _reordered(document: dict[str, Any]) -> dict[str, Any]: + """`document` with its members in the reverse order, and those of each object in its `attributes` and in each member the spec does not define.""" + + def reverse(value: object) -> object: + if isinstance(value, dict): + held = cast("dict[str, object]", value) + return {key: reverse(held[key]) for key in reversed(held)} + if isinstance(value, list): + return [reverse(item) for item in cast("list[object]", value)] + return value + + return { + key: reverse(document[key]) if key in ("attributes", "acme") else document[key] + for key in reversed(document) + } + + +@pytest.mark.parametrize( + ("model", "left", "right"), + [ + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _V3_ARRAY_SIMPLEST), + (ZarrV3ArrayMetadata, _V3_ARRAY_SPELLED_OUT, _reordered(_V3_ARRAY_SPELLED_OUT)), + (ZarrV3ArrayMetadata, _V3_UNCLAIMED, _reordered(_V3_UNCLAIMED)), + (ZarrV3ArrayMetadata, {**_V3_ARRAY, "codecs": [_LITTLE]}, {**_V3_ARRAY, "codecs": [_BIG]}), + (ZarrV3GroupMetadata, {**_V3_GROUP, "attributes": {}}, _V3_GROUP), + ( + ZarrV3ConsolidatedMetadata, + _v3_listing(_V3_ARRAY_SPELLED_OUT, _V3_GROUP), + _v3_listing(_reordered(_V3_ARRAY_SIMPLEST), {**_V3_GROUP, "attributes": {}}), + ), + (ZarrV2ArrayMetadata, _V2_ARRAY_SPELLED_OTHERWISE, _reordered(_V2_ARRAY)), + (ZarrV2GroupMetadata, {**_V2_GROUP, "attributes": {}}, _V2_GROUP), + ], +) +def test_equal_models_canonicalize_to_one_document( + model: type[Model], left: dict[str, Any], right: dict[str, Any] +) -> None: + """Models that are equal give the same canonical document, apart from the order of the members of an object: a `JSONWriter` that sorts them writes the same bytes for each.""" + models = [model.from_json(document) for document in (left, right)] + assert models[0] == models[1] + canonical = [each.canonicalize() for each in models] + assert json_text(_json_of(canonical[0])) == json_text(_json_of(canonical[1])) + assert _stored(canonical[0]) == _stored(canonical[1]) + + +@pytest.mark.parametrize( + ("model", "left", "right"), + [ + ( + ZarrV3ArrayMetadata, + {**_V3_ARRAY, "data_type": "uint16", "codecs": [_LITTLE]}, + {**_V3_ARRAY, "data_type": "uint16", "codecs": [_BIG]}, + ), + (ZarrV3ArrayMetadata, _V3_ARRAY, {**_V3_ARRAY, "dimension_names": [None]}), + ( + ZarrV3ArrayMetadata, + {**_V3_FLOATS, "fill_value": 0.0}, + {**_V3_FLOATS, "fill_value": -0.0}, + ), + (ZarrV3GroupMetadata, _V3_GROUP, {**_V3_GROUP, "attributes": {"a": 1}}), + (ZarrV2ArrayMetadata, _V2_ARRAY, {**_V2_ARRAY, "dimension_separator": "/"}), + (ZarrV2ArrayMetadata, {**_V2_ARRAY, "filters": None}, {**_V2_ARRAY, "filters": []}), + ], +) +def test_models_that_differ_canonicalize_to_different_documents( + model: type[Model], left: dict[str, Any], right: dict[str, Any] +) -> None: + """Only spellings that mean the same thing are reduced to one: models that are not equal keep documents that differ.""" + models = [model.from_json(document) for document in (left, right)] + assert models[0] != models[1] + written = [json_text(_json_of(each.canonicalize())) for each in models] + assert written[0] != written[1] + + +def _json_of(model: Model) -> JSONValue: + document: Any = model.to_json() + return document + + +def _stored(model: Model) -> object: + """The bytes a store holds for `model`, with the members of every object sorted; None for a v3 listing, which a group's document carries.""" + if isinstance(model, ZarrV3ConsolidatedMetadata): + return None + return model.to_key_value(JSONWriter(sort_keys=True)) + + +def _assert_canonicalizes( + model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata | ZarrV3ConsolidatedMetadata, +) -> None: + """`model` equals its canonical model, whose document has nothing left to simplify and reads as the same model from the JSON text a store holds.""" + canonical = model.canonicalize() + assert canonical == model + assert hash(canonical) == hash(model) + written = json.dumps(canonical.to_json()) + assert json.dumps(canonical.canonicalize().to_json()) == written + assert type(model).from_json(json.loads(written), context=ZARR_PYTHON_CONTEXT) == model + + +@pytest.mark.parametrize("spellings", ["all", "canonical"]) +@given(data=st.data()) +def test_every_array_document_canonicalizes(spellings: Spellings, data: st.DataObject) -> None: + document = data.draw(array_documents(spellings=spellings)) + _assert_canonicalizes(ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT)) + + +@given(document=group_documents()) +def test_every_group_document_canonicalizes(document: Mapping[str, Any]) -> None: + _assert_canonicalizes(ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT)) + + +@given(document=consolidated_group_documents()) +def test_every_consolidated_group_document_canonicalizes( + document: Mapping[str, Any], +) -> None: + group = ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT) + _assert_canonicalizes(group) + listing = group.consolidated() + assert listing is not None + _assert_canonicalizes(listing) + # The listing in its simplest spelling can be written back into the group. + _assert_canonicalizes(group.replace(consolidated_metadata=listing.canonicalize()))