Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/zarr-metadata/changes/463.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`ZarrV3ArrayMetadata.refines` compares each codec for the chunk it receives in the model that refines, so a document read in a richer scope refines the same document read in a poorer one, whatever the richer scope learns of the chunks. Before, it did not when the richer scope simplified a codec for its chunk: `refined_in` of a `uint8` array whose `bytes` codec writes an `endian`, read in a scope that claims no `bytes` codec, did not refine the model it came from. `codec_refines` in `zarr_metadata.v3.definition` makes the comparison for one codec.
10 changes: 9 additions & 1 deletion packages/zarr-metadata/src/zarr_metadata/definition/_scope.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,8 +171,15 @@ def refines(declaration: ResolvedDeclaration[Any], other: ResolvedDeclaration[An

def _gained(declaration: AcceptedDeclaration[Any], other: UnclaimedDeclaration) -> bool:
"""Whether reading `other` in the scope formed by `declaration`'s own claims produces `declaration`. This is the test for a gain."""
return read_as_claimed(other, declaration) == declaration


def read_as_claimed(
other: UnclaimedDeclaration, declaration: AcceptedDeclaration[D]
) -> ResolvedDeclaration[D]:
"""`other`, a declaration nothing claimed, read in the scope formed by `declaration`'s own claims: its definition and the definitions of the declarations nested in it. A gain is judged by what this returns."""
again, _ = resolve(other.json, declaration.read_as, _Claimed.of(declaration))
return again == declaration
return cast("ResolvedDeclaration[D]", again)


@dataclass(frozen=True, slots=True)
Expand Down Expand Up @@ -240,5 +247,6 @@ def disagreements_of(
"claims_of",
"disagreements_of",
"kind_name",
"read_as_claimed",
"refines",
]
12 changes: 9 additions & 3 deletions packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@
chunk_grid_lengths,
default_array_bytes_codec,
)
from zarr_metadata.v3._pipeline import Stage, codec_for_chunk, read_pipeline
from zarr_metadata.v3._pipeline import Stage, codec_for_chunk, codec_refines, read_pipeline
from zarr_metadata.v3.array import (
ZARR_V3_ARRAY_METADATA_STORE_KEY,
ZarrV3ArrayMetadataJSON,
Expand Down Expand Up @@ -767,7 +767,7 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadat
return self.with_context(context)

def refines(self, other: ZarrV3ArrayMetadata) -> bool:
"""Whether this model holds everything `other` holds. Each declaration must refine its counterpart, in the order `refines` defines for declarations (including the declarations nested in each declaration). Every other member must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement."""
"""Whether this model holds everything `other` holds. Each declaration must refine its counterpart, in the order `refines` defines for declarations (including the declarations nested in each declaration). A codec is compared as `codec_refines` compares it, for the chunk it receives in this model, so a document read in a richer scope refines the same document read in a poorer one, whatever the richer scope learns of the chunks. Every other member must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement."""
if type(other) is not type(self):
return False
if len(self.codecs) != len(other.codecs) or len(self.storage_transformers) != len(
Expand All @@ -780,12 +780,18 @@ def refines(self, other: ZarrV3ArrayMetadata) -> bool:
(self.data_type, other.data_type),
(self.chunk_grid, other.chunk_grid),
(self.chunk_key_encoding, other.chunk_key_encoding),
*zip(codecs_for_chunks(self._reading), codecs_for_chunks(other._reading), strict=True),
*zip(self.storage_transformers, other.storage_transformers, strict=True),
*((self.extra_fields[key], other.extra_fields[key]) for key in self.extra_fields),
)
if not all(refines_declaration(mine, theirs) for mine, theirs in pairs):
return False
# Each codec is compared for the chunk it receives in this model,
# which knows at least as much of it as `other` does.
if not all(
codec_refines(mine, theirs.codec)
for mine, theirs in zip(self._reading.pipeline, other._reading.pipeline, strict=True)
):
return False
if len(fill_value_problems(self.data_type, other._members.fill_value)) != 0:
return False
return self._plain_key(self.data_type) == other._plain_key(self.data_type)
Expand Down
85 changes: 78 additions & 7 deletions packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,19 @@
from zarr_metadata._sentinel import UNSET
from zarr_metadata.definition._definition import (
AcceptedDeclaration,
RefusedDeclaration,
ResolvedDeclaration,
UnclaimedDeclaration,
asked,
declaration_locations,
in_document,
item_at,
own_key,
read_only,
respelled,
ruled,
)
from zarr_metadata.definition._scope import claim_key, read_as_claimed, refines
from zarr_metadata.v3._definition import (
ArrayArrayCodecDefinition,
ArrayCodecDefinition,
Expand Down Expand Up @@ -372,21 +376,88 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]:
inside a nested declaration is the document's, and is not held
against the result.
"""
codec = stage.codec
return _for_stage(stage.codec, stage)


def _for_stage(
codec: ResolvedDeclaration[CodecDefinition[Any]], stage: Stage
) -> ResolvedDeclaration[CodecDefinition[Any]]:
"""`codec`, a reading of the codec at the place of `stage` in its pipeline, in the simplest spelling for the chunks that `stage` and the stages it contains receive.

`codec` is the codec of `stage` itself, or the same codec as another
scope read it. Each codec it holds is simplified for the chunk that
the stage at its location receives.
"""
if not isinstance(codec, AcceptedDeclaration):
return codec
inner: dict[Loc, ResolvedDeclaration[Any]] = {}
for member, stages in stage.inner.items():
for index, each in enumerate(stages):
simpler = codec_for_chunk(each)
if simpler is not each.codec:
inner[(member, index)] = simpler
for loc, each in _inner_stages(stage).items():
held_there = codec.nested.get(loc)
if held_there is None:
continue
simpler = _for_stage(held_there, each)
if simpler is not held_there:
inner[loc] = simpler
configuration = _for_chunk(codec, stage.incoming)
if configuration is None and len(inner) == 0:
return codec
return respelled(codec, codec.configuration if configuration is None else configuration, inner)


def _inner_stages(stage: Stage) -> dict[Loc, Stage]:
"""Each stage of the pipelines the codec of `stage` contains, by the location of its codec in the configuration."""
return {
(member, index): each
for member, stages in stage.inner.items()
for index, each in enumerate(stages)
}


def codec_refines(stage: Stage, other: ResolvedDeclaration[CodecDefinition[Any]]) -> bool:
"""Whether the codec of `stage` refines `other`, another reading of the codec at that place in the pipeline.

This is the order that `refines` defines for declarations, with each
codec compared as `codec_for_chunk` compares it: in its simplest
spelling for the chunk it receives. The chunk is the one `stage`
knows. A scope that read less may know less of the chunk: when it
claims no data type, or no codec that comes before, a `bytes` codec
keeps an `endian` that `stage` knows to mean nothing. So `other` is
simplified for the chunks of `stage` before the two are compared, and
a document read in a richer scope refines the same document read in
a poorer one.
"""
mine = stage.codec
if not isinstance(mine, AcceptedDeclaration) or isinstance(other, RefusedDeclaration):
# Nothing is simplified for a codec the scope did not read.
return refines(mine, other)
if isinstance(other, UnclaimedDeclaration):
# A gain, when what `other` reads as with the claims of `mine` is `mine`.
if claim_key(mine) != claim_key(other):
return False
return _for_stage(read_as_claimed(other, mine), stage) == codec_for_chunk(stage)
if mine.definition != other.definition or set(mine.nested) != set(other.nested):
return False
if own_key(_own_for_chunk(mine, stage.incoming)) != own_key(
_own_for_chunk(other, stage.incoming)
):
return False
inner = _inner_stages(stage)
return all(
codec_refines(inner[loc], other.nested[loc])
if loc in inner
else refines(mine.nested[loc], other.nested[loc])
for loc in mine.nested
)


def _own_for_chunk(
codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None
) -> AcceptedDeclaration[CodecDefinition[Any]]:
"""`codec` with its own configuration as its `chunk_canonical` simplifies it for `chunk`, and each nested declaration as it is."""
configuration = _for_chunk(codec, chunk)
return codec if configuration is None else respelled(codec, configuration, {})


def _for_chunk(
codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None
) -> Mapping[str, JSONValue] | None:
Expand Down Expand Up @@ -496,4 +567,4 @@ def _text_at(configuration: JSONValue, loc: Loc) -> str | None:
return None if item is UNSET else json_text(item)


__all__ = ["Stage", "codec_for_chunk", "read_pipeline"]
__all__ = ["Stage", "codec_for_chunk", "codec_refines", "read_pipeline"]
7 changes: 5 additions & 2 deletions packages/zarr-metadata/src/zarr_metadata/v3/definition.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,9 @@
`endian` when each value takes one byte. `codec_for_chunk` returns the
codec of a `Stage` in that spelling, with each codec of the pipelines
it contains simplified for the chunk that one receives. An array model
compares its codecs this way.
compares its codecs this way. `codec_refines` tells whether the codec of
a `Stage` refines another reading of it, such as the same document read
in a scope that knew less of the chunk.
The reader does not guess. After a codec the scope did not read, or
after a codec that does not declare its output, the next codec receives
a chunk with nothing known about it, `Chunk()`, which never causes a
Expand Down Expand Up @@ -155,7 +157,7 @@
default_array_bytes_codec,
storage_of,
)
from zarr_metadata.v3._pipeline import Stage, codec_for_chunk
from zarr_metadata.v3._pipeline import Stage, codec_for_chunk, codec_refines

__all__ = [
"CORE_AND_EXTENSIONS_CONTEXT",
Expand Down Expand Up @@ -190,6 +192,7 @@
"ZarrV3StaticCodecDeclaration",
"ZarrV3StorageTransformerDeclaration",
"codec_for_chunk",
"codec_refines",
"default_array_bytes_codec",
"storage_of",
]
Expand Down
96 changes: 96 additions & 0 deletions packages/zarr-metadata/tests/model/test_invariants.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
"""What holds for every v3 document the strategies generate: how a model compares with the same document read in another scope, or written in another order."""

from __future__ import annotations

from collections.abc import Mapping
from typing import Any

import pytest
from hypothesis import given

from zarr_metadata.model import ZarrV3ArrayMetadata, ZarrV3GroupMetadata
from zarr_metadata.testing.strategies import (
array_documents,
consolidated_group_documents,
group_documents,
)
from zarr_metadata.v3.definition import CORE_CONTEXT, ZARR_PYTHON_CONTEXT, ZarrV3Context


@pytest.mark.parametrize(
"poorer",
[
ZarrV3Context(),
CORE_CONTEXT,
ZARR_PYTHON_CONTEXT.copy(update={"data_types": {}}),
ZARR_PYTHON_CONTEXT.copy(update={"codecs": {}}),
],
ids=["empty", "core", "no-data-types", "no-codecs"],
)
@given(document=array_documents())
def test_every_array_read_in_a_richer_scope_refines_the_one_a_poorer_scope_read(
poorer: ZarrV3Context, document: Mapping[str, Any]
) -> None:
"""`refined_in` returns the model that the richer scope reads, and that model refines the one it came from, whatever the richer scope learns of the chunk each codec receives."""
low = ZarrV3ArrayMetadata.from_json(document, context=poorer)
high = low.refined_in(ZARR_PYTHON_CONTEXT)
assert high == ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT)
assert high.refines(low)
assert low.refines(low)


def _reordered(value: object) -> Any: # noqa: ANN401
"""`value` with the members of every object it holds, at any depth, in the reverse order. The items of an array keep their order."""
if isinstance(value, Mapping):
return {key: _reordered(value[key]) for key in reversed(value)}
if isinstance(value, (list, tuple)):
return [_reordered(item) for item in value]
return value


def _assert_one_model(
model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata,
reordered: ZarrV3ArrayMetadata | ZarrV3GroupMetadata,
) -> None:
assert reordered == model
assert hash(reordered) == hash(model)
assert reordered.refines(model) # pyright: ignore[reportArgumentType]
assert model.refines(reordered) # pyright: ignore[reportArgumentType]


@given(document=array_documents())
def test_every_array_reads_as_one_model_whatever_order_its_members_are_in(
document: Mapping[str, Any],
) -> None:
"""The order of the members of an object means nothing, at any depth: in the document, in a configuration, and in a configuration that holds declarations."""
_assert_one_model(
ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT),
ZarrV3ArrayMetadata.from_json(_reordered(document), context=ZARR_PYTHON_CONTEXT),
)


@given(document=group_documents())
def test_every_group_reads_as_one_model_whatever_order_its_members_are_in(
document: Mapping[str, Any],
) -> None:
_assert_one_model(
ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT),
ZarrV3GroupMetadata.from_json(_reordered(document), context=ZARR_PYTHON_CONTEXT),
)


@given(document=consolidated_group_documents())
def test_every_listing_reads_as_one_model_whatever_order_its_members_are_in(
document: Mapping[str, Any],
) -> None:
"""A group lists the documents of its nodes. Each is read, so the listing is one model whatever order the members of those documents are in."""
listings = [
ZarrV3GroupMetadata.from_json(each, context=ZARR_PYTHON_CONTEXT).consolidated()
for each in (document, _reordered(document))
]
assert listings[0] is not None
assert listings[1] is not None
assert listings[1] == listings[0]
assert hash(listings[1]) == hash(listings[0])
assert listings[1].refines(listings[0])
assert listings[0].refines(listings[1])
Loading
Loading