Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/zarr-metadata/changes/4641.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Two declarations that hold other declarations, such as two `sharding_indexed` codecs, are now equal when they differ only in the order their configurations write their members in. The declarations a configuration holds were compared in the order it writes them, so a shard that writes `index_codecs` before `codecs` was not equal to one that writes them the other way round, and neither were the array models that hold them, although the order of the members of a JSON object means nothing. The items of an array, such as the codecs of a pipeline, are still compared in order.
22 changes: 20 additions & 2 deletions packages/zarr-metadata/src/zarr_metadata/definition/_definition.py
Original file line number Diff line number Diff line change
Expand Up @@ -1234,8 +1234,26 @@ def own_key(declaration: AcceptedDeclaration[Any]) -> tuple[object, ...]:


def _nested_key(nested: Nested) -> tuple[tuple[Loc, tuple[object, ...]], ...]:
"""The key of each nested declaration, paired with its location."""
return tuple((loc, declaration_key(inner)) for loc, inner in nested.items())
"""The key of each nested declaration, paired with its location, in the order of the locations.

`nested` holds the declarations in the order the configuration writes
them, and the order of the members of a JSON object means nothing.
So the key is sorted by location, and a shard that writes
`index_codecs` before `codecs` has the same key as one that writes
them the other way round. The items of an array keep their order,
since their locations are their indices.
"""
return tuple(
sorted(
((loc, declaration_key(inner)) for loc, inner in nested.items()),
key=lambda item: _location_order(item[0]),
)
)


def _location_order(loc: Loc) -> tuple[tuple[bool, str | int], ...]:
"""A sort key for a location: a key of an object and an index of an array are never compared with each other."""
return tuple((isinstance(step, str), step) for step in loc)


def document_json(declaration: ResolvedDeclaration[Any]) -> JSONValue | UNSET:
Expand Down
27 changes: 27 additions & 0 deletions packages/zarr-metadata/tests/v3/test_definitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -1182,6 +1182,7 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None:
LE: Final = {"name": "bytes", "configuration": {"endian": "little"}}
SHARD: Final = {"chunk_shape": [2], "codecs": [LE], "index_location": "end"}
NOSHUFFLE: Final = {"cname": "lz4", "clevel": 5, "shuffle": "noshuffle", "blocksize": 0}
CRC32C: Final = {"name": "crc32c"}


@pytest.mark.parametrize(
Expand Down Expand Up @@ -1221,6 +1222,29 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None:
False,
),
("int8", "uint8", DataTypeDefinition, False),
# The order of the members of an object means nothing, in a
# configuration that holds declarations or one that holds none.
(
{"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [LE]}},
{
"configuration": {"index_codecs": [LE], **dict(reversed(SHARD.items()))},
"name": "sharding_indexed",
},
CodecDefinition,
True,
),
(
{"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [LE, CRC32C]}},
{"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [CRC32C, LE]}},
CodecDefinition,
False,
),
(
{"name": "zstd", "configuration": {"level": 1, "checksum": True}},
{"configuration": {"checksum": True, "level": 1}, "name": "zstd"},
CodecDefinition,
True,
),
# The spec's equivalences, which each definition's `canonical`
# normalizes: a member at its default, a run-length encoding, and
# leading zeros in a size.
Expand Down Expand Up @@ -1296,6 +1320,9 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None:
"another-configuration",
"unclaimed-another-configuration",
"another-name",
"members-holding-declarations-in-another-order",
"declarations-in-another-order",
"members-in-another-order",
"blosc-typesize-noshuffle",
"default-separator",
"default-other-separator",
Expand Down
Loading