diff --git a/packages/zarr-metadata/changes/4641.bugfix.md b/packages/zarr-metadata/changes/4641.bugfix.md new file mode 100644 index 0000000000..8904ec277c --- /dev/null +++ b/packages/zarr-metadata/changes/4641.bugfix.md @@ -0,0 +1 @@ +Two declarations that hold other declarations, such as two `sharding_indexed` codecs, are now equal when they differ only in the order their configurations write their members in. The declarations a configuration holds were compared in the order it writes them, so a shard that writes `index_codecs` before `codecs` was not equal to one that writes them the other way round, and neither were the array models that hold them, although the order of the members of a JSON object means nothing. The items of an array, such as the codecs of a pipeline, are still compared in order. diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 05eefcf761..19e2af9ea3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1234,8 +1234,26 @@ def own_key(declaration: AcceptedDeclaration[Any]) -> tuple[object, ...]: def _nested_key(nested: Nested) -> tuple[tuple[Loc, tuple[object, ...]], ...]: - """The key of each nested declaration, paired with its location.""" - return tuple((loc, declaration_key(inner)) for loc, inner in nested.items()) + """The key of each nested declaration, paired with its location, in the order of the locations. + + `nested` holds the declarations in the order the configuration writes + them, and the order of the members of a JSON object means nothing. + So the key is sorted by location, and a shard that writes + `index_codecs` before `codecs` has the same key as one that writes + them the other way round. The items of an array keep their order, + since their locations are their indices. + """ + return tuple( + sorted( + ((loc, declaration_key(inner)) for loc, inner in nested.items()), + key=lambda item: _location_order(item[0]), + ) + ) + + +def _location_order(loc: Loc) -> tuple[tuple[bool, str | int], ...]: + """A sort key for a location: a key of an object and an index of an array are never compared with each other.""" + return tuple((isinstance(step, str), step) for step in loc) def document_json(declaration: ResolvedDeclaration[Any]) -> JSONValue | UNSET: diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index d88dcf639e..a1d929f388 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -1182,6 +1182,7 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: LE: Final = {"name": "bytes", "configuration": {"endian": "little"}} SHARD: Final = {"chunk_shape": [2], "codecs": [LE], "index_location": "end"} NOSHUFFLE: Final = {"cname": "lz4", "clevel": 5, "shuffle": "noshuffle", "blocksize": 0} +CRC32C: Final = {"name": "crc32c"} @pytest.mark.parametrize( @@ -1221,6 +1222,29 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: False, ), ("int8", "uint8", DataTypeDefinition, False), + # The order of the members of an object means nothing, in a + # configuration that holds declarations or one that holds none. + ( + {"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [LE]}}, + { + "configuration": {"index_codecs": [LE], **dict(reversed(SHARD.items()))}, + "name": "sharding_indexed", + }, + CodecDefinition, + True, + ), + ( + {"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [LE, CRC32C]}}, + {"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [CRC32C, LE]}}, + CodecDefinition, + False, + ), + ( + {"name": "zstd", "configuration": {"level": 1, "checksum": True}}, + {"configuration": {"checksum": True, "level": 1}, "name": "zstd"}, + CodecDefinition, + True, + ), # The spec's equivalences, which each definition's `canonical` # normalizes: a member at its default, a run-length encoding, and # leading zeros in a size. @@ -1296,6 +1320,9 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: "another-configuration", "unclaimed-another-configuration", "another-name", + "members-holding-declarations-in-another-order", + "declarations-in-another-order", + "members-in-another-order", "blosc-typesize-noshuffle", "default-separator", "default-other-separator",