Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/zarr-metadata/changes/462.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
A v3 codec's `chunk_canonical` receives the configuration as the codec's `canonical` spells it, so two spellings that `canonical` reduces to one are simplified for a chunk in the same way. Before, it received the configuration as written, and array models that differed only in such spellings were unequal. Its result is checked more closely: one that it would simplify again, or that holds a declaration the codec did not hold, raises `ValueError`, and one that is not JSON raises `TypeError` naming the definition. A result may write an array as a list, and a problem inside a nested declaration is no longer held against it. A codec that leaves `chunk_canonical` at its default is no longer copied each time an array model is compared or hashed.
4 changes: 3 additions & 1 deletion packages/zarr-metadata/docs/models.md
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,9 @@ These are each declaration, and the fill value read against its data type.
So in a `uint8` array, a `bytes` codec is the same codec with either
`endian`, or with none. The codec is compared for the chunk it
receives: after a `cast_value` to `float32`, the two `endian` values
are two codecs.
are two codecs. `codec_for_chunk` returns a codec in the spelling that
is compared, which `canonical_of` cannot give, since it reads the
codec alone.

A v2 document's
[dtype](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html#data-type-encoding),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1291,11 +1291,15 @@ def respelled(
where it is. `nested` holds only the nested declarations to replace,
each one accepted or unclaimed, and each is written into the
configuration as its `to_json` writes it. A location in `nested` that
does not lead into `configuration` raises `TypeError`. The result is
is not the location of a nested declaration of `declaration`, or that
does not lead into `configuration`, raises `TypeError`. The result is
written as `to_json` writes it, since no document wrote it.
"""
spelled: JSONValue = dict(configuration)
for loc, inner in nested.items():
if loc not in declaration.nested or item_at(spelled, loc) is UNSET:
msg = f"{loc!r} is not the location of a nested declaration in {configuration!r}"
raise TypeError(msg)
spelled = _replaced(spelled, loc, written_json(held(inner)))
kind = declaration.read_as
name = declaration.name
Expand All @@ -1309,6 +1313,11 @@ def respelled(
)


def declaration_locations(definition: Definition[Any], configuration: JSONValue) -> tuple[Loc, ...]:
"""The location of each declaration that `configuration` holds, where `definition`'s TypedDict types a member with a declaration alias, in order. `configuration` is JSON with arrays as tuples."""
return tuple(found.loc for found in _typed(definition.configuration, configuration, ())[2])


def item_at(value: JSONValue, path: Loc) -> JSONValue | UNSET:
"""The item of `value` at `path`; `UNSET` when `path` does not lead into `value`."""
for step in path:
Expand Down Expand Up @@ -1957,6 +1966,7 @@ def _replaced(value: JSONValue, path: Loc, new: JSONValue) -> JSONValue:
"configuration_of",
"declaration_key",
"declaration_kind",
"declaration_locations",
"declarations_of",
"document_json",
"filed_under",
Expand Down
9 changes: 6 additions & 3 deletions packages/zarr-metadata/src/zarr_metadata/v3/_definition.py
Original file line number Diff line number Diff line change
Expand Up @@ -434,8 +434,11 @@ class ArrayCodecDefinition(CodecDefinition[C]):
It simplifies only the codec's own members, and it must keep each
nested declaration as it is, because each codec of a pipeline the
codec contains is simplified for the chunk it receives in turn. What
does not depend on the chunk is left to `canonical`, which simplifies
the result as it simplifies any configuration. A chunk rule checks
does not depend on the chunk is left to `canonical`: `chunk_canonical`
receives the configuration as `canonical` spells it, and `canonical`
simplifies the result as it simplifies any configuration. The result
must be the simplest spelling, so `chunk_canonical` must change
nothing when it is asked about its own result. A chunk rule checks
only the parts of a chunk that are known, and `chunk_canonical`
simplifies only by them: a chunk with nothing known about it,
`Chunk()`, leaves every configuration as it is. `codec_for_chunk`
Expand All @@ -445,7 +448,7 @@ class ArrayCodecDefinition(CodecDefinition[C]):
chunk_rules: Callable[[C, Nested, Chunk], Iterable[ValidationProblem]] = no_rules
"""The spec violations for this codec with a given input chunk, located in the configuration. A problem at `()` is about the codec as a whole, such as an input it cannot encode, and is reported at the codec."""
chunk_canonical: Callable[[C, Nested, Chunk], C] = unchanged_for_chunk
"""A configuration the chunk rules allow for a given input chunk, in the simplest spelling that means the same for that chunk."""
"""A configuration the chunk rules allow for a given input chunk, as `canonical` spells it, in the simplest spelling that means the same for that chunk."""
pipelines: Callable[[C, Nested, Chunk], Mapping[str, Chunk]] = no_pipelines
"""The chunk each contained pipeline receives, keyed by the configuration member that holds the pipeline."""

Expand Down
103 changes: 85 additions & 18 deletions packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,18 @@

from zarr_metadata._json import (
ValidationProblem,
copied,
is_object,
is_tuple,
json_text,
refine_user_data,
with_input,
)
from zarr_metadata._sentinel import UNSET
from zarr_metadata.definition._definition import (
AcceptedDeclaration,
ResolvedDeclaration,
asked,
declaration_locations,
in_document,
item_at,
read_only,
Expand All @@ -58,6 +59,7 @@
Chunk,
CodecDefinition,
CodecKind,
unchanged_for_chunk,
)

if TYPE_CHECKING:
Expand Down Expand Up @@ -349,6 +351,9 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]:
codec of each pipeline it contains, for the chunk that codec
receives, and then its own configuration, as its definition's
`chunk_canonical` simplifies it for the chunk in `stage`.
`chunk_canonical` receives the configuration as the definition's
`canonical` spells it, so two spellings that `canonical` reduces to
one are simplified for the chunk in the same way.

A codec the scope did not read is returned as it is. A codec's own
configuration is kept as it is when the codec receives bytes, and
Expand All @@ -360,9 +365,12 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]:

`chunk_canonical` is the extension author's code, so its result is
checked: the definition must allow it, the chunk rules must find no
problem with it, and each nested declaration must be where it was. A
result that fails a check raises `ValueError`, which indicates a bug
in the definition, not in the codec.
problem with it, it must hold the nested declarations the codec
holds, each as it was, and `chunk_canonical` must have nothing left
to simplify in it. A result that fails a check raises `ValueError`,
which indicates a bug in the definition, not in the codec. A problem
inside a nested declaration is the document's, and is not held
against the result.
"""
codec = stage.codec
if not isinstance(codec, AcceptedDeclaration):
Expand All @@ -382,29 +390,33 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]:
def _for_chunk(
codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None
) -> Mapping[str, JSONValue] | None:
"""The configuration of `codec` as its definition's `chunk_canonical` simplifies it for `chunk`, checked; None when nothing is simplified."""
"""The configuration of `codec` as its definition's `chunk_canonical` simplifies it for `chunk`, checked; None when nothing is simplified.

`chunk_canonical` receives the configuration as the definition's
`canonical` spells it. So every spelling that `canonical` reduces to
one is simplified for the chunk in the same way, and `chunk_canonical`
does not have to know each of them. None is returned when
`chunk_canonical` changes nothing in that spelling, and the codec
then stays as written, for `canonical` to simplify as it simplifies
any configuration.
"""
definition = codec.definition
if chunk is None or not isinstance(definition, ArrayCodecDefinition):
return None
if definition.chunk_canonical is unchanged_for_chunk:
return None
configuration, nested = codec.configuration, codec.nested
if len(_chunk_problems(definition, configuration, nested, chunk, ())) != 0:
return None
given: object = asked(
definition,
"chunk_canonical",
lambda: cast("object", definition.chunk_canonical(read_only(configuration), nested, chunk)),
)
if given is configuration:
return None
if not is_object(given):
msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}"
raise TypeError(msg)
simplified = cast("dict[str, JSONValue]", copied(cast("JSONValue", dict(given))))
if json_text(simplified) == json_text(cast("JSONValue", configuration)):
simplified = _asked_for_chunk(definition, configuration, nested, chunk)
if simplified is None:
return None
_, refused = definition.read_configuration(simplified)
held_at = frozenset(declaration_locations(definition, simplified))
found = (
*refused,
# A problem inside a nested declaration is the document's: the
# declaration is checked to be as it was written.
*(problem for problem in refused if not _inside(problem.loc, nested)),
*_chunk_problems(definition, simplified, nested, chunk, ()),
*(
ValidationProblem(
Expand All @@ -413,13 +425,68 @@ def _for_chunk(
for loc in nested
if _text_at(simplified, loc) != _text_at(cast("JSONValue", configuration), loc)
),
*(
ValidationProblem(
loc, "expected no declaration that the codec did not hold", "invalid_value"
)
for loc in sorted(held_at.difference(nested), key=repr)
),
)
if len(found) != 0:
msg = (
f"{definition.name!r}: its chunk_canonical gave {simplified!r}, which does not hold: "
f"{list(found)!r}"
)
raise ValueError(msg)
# The simplest spelling has nothing left to simplify. Otherwise a
# document written from it would read as another codec.
further = _asked_for_chunk(definition, simplified, nested, chunk)
if further is not None:
msg = (
f"{definition.name!r}: its chunk_canonical gave {simplified!r}, which is not the "
f"simplest spelling: asked again, it gave {further!r}"
)
raise ValueError(msg)
return simplified


def _inside(loc: Loc, nested: Nested) -> bool:
"""Whether `loc` is at or under the location of a declaration in `nested`."""
return any(loc[: len(held_at)] == held_at for held_at in nested)


def _asked_for_chunk(
definition: ArrayCodecDefinition[Any],
configuration: Mapping[str, JSONValue],
nested: Nested,
chunk: Chunk,
) -> dict[str, JSONValue] | None:
"""What `definition`'s `chunk_canonical` gives for `configuration` as its `canonical` spells it, as JSON with arrays as tuples; None when it changes nothing in that spelling."""
spelled = read_only(
asked(
definition,
"canonical",
lambda: dict(
cast("Mapping[str, JSONValue]", definition.canonical(read_only(configuration)))
),
)
)
given: object = asked(
definition,
"chunk_canonical",
lambda: cast("object", definition.chunk_canonical(spelled, nested, chunk)),
)
if given is spelled:
return None
# The result is the extension author's value, so it is read as any JSON
# is: an array may be a list, and what is not JSON is refused here,
# naming the definition. `read_configuration` checks the rest.
simplified, not_json = refine_user_data(dict(given)) if is_object(given) else (None, ())
if not isinstance(simplified, dict) or len(not_json) != 0:
msg = f"{definition.name!r}: its chunk_canonical gives a configuration, got {given!r}"
raise TypeError(msg)
if json_text(simplified) == json_text(cast("JSONValue", dict(spelled))):
return None
return simplified


Expand Down
106 changes: 104 additions & 2 deletions packages/zarr-metadata/tests/v3/test_definitions.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
JSONValue,
ValidationProblem,
)
from zarr_metadata._json import refine_json
from zarr_metadata._json import json_text, refine_json
from zarr_metadata.definition import (
AcceptedDeclaration,
Context,
Expand All @@ -34,7 +34,7 @@
configuration_of,
resolve,
)
from zarr_metadata.definition._definition import resolve_refined
from zarr_metadata.definition._definition import item_at, resolve_refined, respelled
from zarr_metadata.model import (
UNSET,
ZarrV3ArrayMetadata,
Expand Down Expand Up @@ -1824,3 +1824,105 @@ def test_error_resolve_refuses_a_kind_of_the_other_format_whatever_the_declarati
"""`resolve` with a kind of one format and a scope of the other raises one `TypeError`, before it reads the declaration. So the message is the same for a declaration with a name that a scope could look up and for a value with no name."""
with pytest.raises(TypeError, match=f"^{re.escape(message)}$"):
resolve(declaration, kind, context)


def _accepted(declaration: JSONValue, kind: type[Definition[Any]]) -> AcceptedDeclaration[Any]:
read, problems = resolve(declaration, kind, CORE_AND_EXTENSIONS_CONTEXT)
assert isinstance(read, AcceptedDeclaration)
assert problems == ()
return read


def _sharding(**members: JSONValue) -> JSONValue:
return {"name": "sharding_indexed", "configuration": {"chunk_shape": [2], **members}}


@pytest.mark.parametrize(
("declaration", "kind", "configuration", "nested", "written"),
[
# A member of its own left out, and a nested declaration replaced.
(
_sharding(codecs=[LE], index_codecs=[LE, CRC32C], index_location="end"),
CodecDefinition,
{"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]},
{("codecs", 0): {"name": "bytes"}},
_sharding(codecs=[{"name": "bytes"}], index_codecs=[LE, CRC32C]),
),
# Nothing nested to replace.
(
{"name": "gzip", "configuration": {"level": 1}},
CodecDefinition,
{"level": 5},
{},
{"name": "gzip", "configuration": {"level": 5}},
),
# A name that carries its configuration is written alone.
("r16", DataTypeDefinition, {"bits": 16}, {}, "r16"),
],
ids=["shard", "gzip", "name-carries-configuration"],
)
def test_respelled_is_the_declaration_that_the_other_spelling_reads_as(
declaration: JSONValue,
kind: type[Definition[Any]],
configuration: JSONValue,
nested: Mapping[tuple[str | int, ...], JSONValue],
written: JSONValue,
) -> None:
"""`respelled` returns what a read of the other spelling returns: the same JSON, name, configuration, nested declarations and key. The declaration it was given is left as it was."""
read = _accepted(declaration, kind)
before = json_text(read.to_json())
refined = cast("Mapping[str, JSONValue]", refine_json(configuration)[0])
replacements = {loc: _accepted(inner, CodecDefinition) for loc, inner in nested.items()}
result = respelled(read, refined, replacements)
again = _accepted(written, kind)
assert json_text(result.json) == json_text(again.json)
assert json_text(result.to_json()) == json_text(again.to_json())
assert result.name == again.name
assert json_text(cast("JSONValue", result.configuration)) == json_text(
cast("JSONValue", again.configuration)
)
assert dict(result.nested) == dict(again.nested)
assert result == again
assert hash(result) == hash(again)
assert json_text(read.to_json()) == before


@pytest.mark.parametrize(
("configuration", "loc"),
[
({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ("codecs", 5)),
({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ("codecs", -1)),
({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ("nope", 0)),
({"chunk_shape": [2], "codecs": [LE], "index_codecs": [LE, CRC32C]}, ()),
# A location of a nested declaration that the new configuration does not have.
({"chunk_shape": [2], "codecs": [], "index_codecs": [LE, CRC32C]}, ("codecs", 0)),
],
ids=["past-the-end", "negative", "no-such-member", "the-configuration", "left-out"],
)
def test_error_respelled_at_a_location_that_holds_no_nested_declaration(
configuration: JSONValue, loc: tuple[str | int, ...]
) -> None:
read = _accepted(_sharding(codecs=[LE], index_codecs=[LE, CRC32C]), CodecDefinition)
refined = cast("Mapping[str, JSONValue]", refine_json(configuration)[0])
with pytest.raises(TypeError, match="is not the location of a nested declaration"):
respelled(read, refined, {loc: _accepted({"name": "bytes"}, CodecDefinition)})


@pytest.mark.parametrize(
("path", "item"),
[
((), {"a": ({"b": 1}, 2)}),
(("a",), ({"b": 1}, 2)),
(("a", 0, "b"), 1),
(("a", 1), 2),
(("a", 2), UNSET),
(("a", -1), UNSET),
(("b",), UNSET),
(("a", "b"), UNSET),
((0,), UNSET),
(("a", 1, "b"), UNSET),
],
)
def test_item_at_follows_a_location_into_json(path: tuple[str | int, ...], item: object) -> None:
"""`item_at` returns the item at a location, and `UNSET` when the location does not lead into the value: a key an object lacks, an index outside an array, or a step into a scalar."""
assert item_at({"a": ({"b": 1}, 2)}, path) == item
Loading
Loading