diff --git a/packages/zarr-metadata/changes/463.bugfix.md b/packages/zarr-metadata/changes/463.bugfix.md new file mode 100644 index 0000000000..480b5c6762 --- /dev/null +++ b/packages/zarr-metadata/changes/463.bugfix.md @@ -0,0 +1 @@ +`ZarrV3ArrayMetadata.refines` compares each codec for the chunk it receives in the model that refines, so a document read in a richer scope refines the same document read in a poorer one, whatever the richer scope learns of the chunks. Before, it did not when the richer scope simplified a codec for its chunk: `refined_in` of a `uint8` array whose `bytes` codec writes an `endian`, read in a scope that claims no `bytes` codec, did not refine the model it came from. `codec_refines` in `zarr_metadata.v3.definition` makes the comparison for one codec. diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py index 9a28bc2a32..34c2254759 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_scope.py @@ -171,8 +171,15 @@ def refines(declaration: ResolvedDeclaration[Any], other: ResolvedDeclaration[An def _gained(declaration: AcceptedDeclaration[Any], other: UnclaimedDeclaration) -> bool: """Whether reading `other` in the scope formed by `declaration`'s own claims produces `declaration`. This is the test for a gain.""" + return read_as_claimed(other, declaration) == declaration + + +def read_as_claimed( + other: UnclaimedDeclaration, declaration: AcceptedDeclaration[D] +) -> ResolvedDeclaration[D]: + """`other`, a declaration nothing claimed, read in the scope formed by `declaration`'s own claims: its definition and the definitions of the declarations nested in it. A gain is judged by what this returns.""" again, _ = resolve(other.json, declaration.read_as, _Claimed.of(declaration)) - return again == declaration + return cast("ResolvedDeclaration[D]", again) @dataclass(frozen=True, slots=True) @@ -240,5 +247,6 @@ def disagreements_of( "claims_of", "disagreements_of", "kind_name", + "read_as_claimed", "refines", ] diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 2ea2af7ccb..6c8b858521 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -78,7 +78,7 @@ chunk_grid_lengths, default_array_bytes_codec, ) -from zarr_metadata.v3._pipeline import Stage, codec_for_chunk, read_pipeline +from zarr_metadata.v3._pipeline import Stage, codec_for_chunk, codec_refines, read_pipeline from zarr_metadata.v3.array import ( ZARR_V3_ARRAY_METADATA_STORE_KEY, ZarrV3ArrayMetadataJSON, @@ -767,7 +767,7 @@ def refined_in(self, context: ZarrV3Context | None = None) -> ZarrV3ArrayMetadat return self.with_context(context) def refines(self, other: ZarrV3ArrayMetadata) -> bool: - """Whether this model holds everything `other` holds. Each declaration must refine its counterpart, in the order `refines` defines for declarations (including the declarations nested in each declaration). Every other member must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" + """Whether this model holds everything `other` holds. Each declaration must refine its counterpart, in the order `refines` defines for declarations (including the declarations nested in each declaration). A codec is compared as `codec_refines` compares it, for the chunk it receives in this model, so a document read in a richer scope refines the same document read in a poorer one, whatever the richer scope learns of the chunks. Every other member must be the same, with the fill value compared in the spelling of the more informed data type. If that data type rejects the other model's fill value, this is not a refinement.""" if type(other) is not type(self): return False if len(self.codecs) != len(other.codecs) or len(self.storage_transformers) != len( @@ -780,12 +780,18 @@ def refines(self, other: ZarrV3ArrayMetadata) -> bool: (self.data_type, other.data_type), (self.chunk_grid, other.chunk_grid), (self.chunk_key_encoding, other.chunk_key_encoding), - *zip(codecs_for_chunks(self._reading), codecs_for_chunks(other._reading), strict=True), *zip(self.storage_transformers, other.storage_transformers, strict=True), *((self.extra_fields[key], other.extra_fields[key]) for key in self.extra_fields), ) if not all(refines_declaration(mine, theirs) for mine, theirs in pairs): return False + # Each codec is compared for the chunk it receives in this model, + # which knows at least as much of it as `other` does. + if not all( + codec_refines(mine, theirs.codec) + for mine, theirs in zip(self._reading.pipeline, other._reading.pipeline, strict=True) + ): + return False if len(fill_value_problems(self.data_type, other._members.fill_value)) != 0: return False return self._plain_key(self.data_type) == other._plain_key(self.data_type) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py index db1bf1ff9b..3edf1df097 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_pipeline.py @@ -44,15 +44,19 @@ from zarr_metadata._sentinel import UNSET from zarr_metadata.definition._definition import ( AcceptedDeclaration, + RefusedDeclaration, ResolvedDeclaration, + UnclaimedDeclaration, asked, declaration_locations, in_document, item_at, + own_key, read_only, respelled, ruled, ) +from zarr_metadata.definition._scope import claim_key, read_as_claimed, refines from zarr_metadata.v3._definition import ( ArrayArrayCodecDefinition, ArrayCodecDefinition, @@ -372,21 +376,88 @@ def codec_for_chunk(stage: Stage) -> ResolvedDeclaration[CodecDefinition[Any]]: inside a nested declaration is the document's, and is not held against the result. """ - codec = stage.codec + return _for_stage(stage.codec, stage) + + +def _for_stage( + codec: ResolvedDeclaration[CodecDefinition[Any]], stage: Stage +) -> ResolvedDeclaration[CodecDefinition[Any]]: + """`codec`, a reading of the codec at the place of `stage` in its pipeline, in the simplest spelling for the chunks that `stage` and the stages it contains receive. + + `codec` is the codec of `stage` itself, or the same codec as another + scope read it. Each codec it holds is simplified for the chunk that + the stage at its location receives. + """ if not isinstance(codec, AcceptedDeclaration): return codec inner: dict[Loc, ResolvedDeclaration[Any]] = {} - for member, stages in stage.inner.items(): - for index, each in enumerate(stages): - simpler = codec_for_chunk(each) - if simpler is not each.codec: - inner[(member, index)] = simpler + for loc, each in _inner_stages(stage).items(): + held_there = codec.nested.get(loc) + if held_there is None: + continue + simpler = _for_stage(held_there, each) + if simpler is not held_there: + inner[loc] = simpler configuration = _for_chunk(codec, stage.incoming) if configuration is None and len(inner) == 0: return codec return respelled(codec, codec.configuration if configuration is None else configuration, inner) +def _inner_stages(stage: Stage) -> dict[Loc, Stage]: + """Each stage of the pipelines the codec of `stage` contains, by the location of its codec in the configuration.""" + return { + (member, index): each + for member, stages in stage.inner.items() + for index, each in enumerate(stages) + } + + +def codec_refines(stage: Stage, other: ResolvedDeclaration[CodecDefinition[Any]]) -> bool: + """Whether the codec of `stage` refines `other`, another reading of the codec at that place in the pipeline. + + This is the order that `refines` defines for declarations, with each + codec compared as `codec_for_chunk` compares it: in its simplest + spelling for the chunk it receives. The chunk is the one `stage` + knows. A scope that read less may know less of the chunk: when it + claims no data type, or no codec that comes before, a `bytes` codec + keeps an `endian` that `stage` knows to mean nothing. So `other` is + simplified for the chunks of `stage` before the two are compared, and + a document read in a richer scope refines the same document read in + a poorer one. + """ + mine = stage.codec + if not isinstance(mine, AcceptedDeclaration) or isinstance(other, RefusedDeclaration): + # Nothing is simplified for a codec the scope did not read. + return refines(mine, other) + if isinstance(other, UnclaimedDeclaration): + # A gain, when what `other` reads as with the claims of `mine` is `mine`. + if claim_key(mine) != claim_key(other): + return False + return _for_stage(read_as_claimed(other, mine), stage) == codec_for_chunk(stage) + if mine.definition != other.definition or set(mine.nested) != set(other.nested): + return False + if own_key(_own_for_chunk(mine, stage.incoming)) != own_key( + _own_for_chunk(other, stage.incoming) + ): + return False + inner = _inner_stages(stage) + return all( + codec_refines(inner[loc], other.nested[loc]) + if loc in inner + else refines(mine.nested[loc], other.nested[loc]) + for loc in mine.nested + ) + + +def _own_for_chunk( + codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None +) -> AcceptedDeclaration[CodecDefinition[Any]]: + """`codec` with its own configuration as its `chunk_canonical` simplifies it for `chunk`, and each nested declaration as it is.""" + configuration = _for_chunk(codec, chunk) + return codec if configuration is None else respelled(codec, configuration, {}) + + def _for_chunk( codec: AcceptedDeclaration[CodecDefinition[Any]], chunk: Chunk | None ) -> Mapping[str, JSONValue] | None: @@ -496,4 +567,4 @@ def _text_at(configuration: JSONValue, loc: Loc) -> str | None: return None if item is UNSET else json_text(item) -__all__ = ["Stage", "codec_for_chunk", "read_pipeline"] +__all__ = ["Stage", "codec_for_chunk", "codec_refines", "read_pipeline"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py index 9c20fac4a5..fc7e8a498e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/definition.py @@ -97,7 +97,9 @@ `endian` when each value takes one byte. `codec_for_chunk` returns the codec of a `Stage` in that spelling, with each codec of the pipelines it contains simplified for the chunk that one receives. An array model -compares its codecs this way. +compares its codecs this way. `codec_refines` tells whether the codec of +a `Stage` refines another reading of it, such as the same document read +in a scope that knew less of the chunk. The reader does not guess. After a codec the scope did not read, or after a codec that does not declare its output, the next codec receives a chunk with nothing known about it, `Chunk()`, which never causes a @@ -155,7 +157,7 @@ default_array_bytes_codec, storage_of, ) -from zarr_metadata.v3._pipeline import Stage, codec_for_chunk +from zarr_metadata.v3._pipeline import Stage, codec_for_chunk, codec_refines __all__ = [ "CORE_AND_EXTENSIONS_CONTEXT", @@ -190,6 +192,7 @@ "ZarrV3StaticCodecDeclaration", "ZarrV3StorageTransformerDeclaration", "codec_for_chunk", + "codec_refines", "default_array_bytes_codec", "storage_of", ] diff --git a/packages/zarr-metadata/tests/model/test_invariants.py b/packages/zarr-metadata/tests/model/test_invariants.py new file mode 100644 index 0000000000..84e11290ca --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_invariants.py @@ -0,0 +1,96 @@ +"""What holds for every v3 document the strategies generate: how a model compares with the same document read in another scope, or written in another order.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +import pytest +from hypothesis import given + +from zarr_metadata.model import ZarrV3ArrayMetadata, ZarrV3GroupMetadata +from zarr_metadata.testing.strategies import ( + array_documents, + consolidated_group_documents, + group_documents, +) +from zarr_metadata.v3.definition import CORE_CONTEXT, ZARR_PYTHON_CONTEXT, ZarrV3Context + + +@pytest.mark.parametrize( + "poorer", + [ + ZarrV3Context(), + CORE_CONTEXT, + ZARR_PYTHON_CONTEXT.copy(update={"data_types": {}}), + ZARR_PYTHON_CONTEXT.copy(update={"codecs": {}}), + ], + ids=["empty", "core", "no-data-types", "no-codecs"], +) +@given(document=array_documents()) +def test_every_array_read_in_a_richer_scope_refines_the_one_a_poorer_scope_read( + poorer: ZarrV3Context, document: Mapping[str, Any] +) -> None: + """`refined_in` returns the model that the richer scope reads, and that model refines the one it came from, whatever the richer scope learns of the chunk each codec receives.""" + low = ZarrV3ArrayMetadata.from_json(document, context=poorer) + high = low.refined_in(ZARR_PYTHON_CONTEXT) + assert high == ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT) + assert high.refines(low) + assert low.refines(low) + + +def _reordered(value: object) -> Any: # noqa: ANN401 + """`value` with the members of every object it holds, at any depth, in the reverse order. The items of an array keep their order.""" + if isinstance(value, Mapping): + return {key: _reordered(value[key]) for key in reversed(value)} + if isinstance(value, (list, tuple)): + return [_reordered(item) for item in value] + return value + + +def _assert_one_model( + model: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, + reordered: ZarrV3ArrayMetadata | ZarrV3GroupMetadata, +) -> None: + assert reordered == model + assert hash(reordered) == hash(model) + assert reordered.refines(model) # pyright: ignore[reportArgumentType] + assert model.refines(reordered) # pyright: ignore[reportArgumentType] + + +@given(document=array_documents()) +def test_every_array_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + """The order of the members of an object means nothing, at any depth: in the document, in a configuration, and in a configuration that holds declarations.""" + _assert_one_model( + ZarrV3ArrayMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT), + ZarrV3ArrayMetadata.from_json(_reordered(document), context=ZARR_PYTHON_CONTEXT), + ) + + +@given(document=group_documents()) +def test_every_group_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + _assert_one_model( + ZarrV3GroupMetadata.from_json(document, context=ZARR_PYTHON_CONTEXT), + ZarrV3GroupMetadata.from_json(_reordered(document), context=ZARR_PYTHON_CONTEXT), + ) + + +@given(document=consolidated_group_documents()) +def test_every_listing_reads_as_one_model_whatever_order_its_members_are_in( + document: Mapping[str, Any], +) -> None: + """A group lists the documents of its nodes. Each is read, so the listing is one model whatever order the members of those documents are in.""" + listings = [ + ZarrV3GroupMetadata.from_json(each, context=ZARR_PYTHON_CONTEXT).consolidated() + for each in (document, _reordered(document)) + ] + assert listings[0] is not None + assert listings[1] is not None + assert listings[1] == listings[0] + assert hash(listings[1]) == hash(listings[0]) + assert listings[1].refines(listings[0]) + assert listings[0].refines(listings[1]) diff --git a/packages/zarr-metadata/tests/model/test_pair.py b/packages/zarr-metadata/tests/model/test_pair.py index 998a45e3f2..0cc7df2335 100644 --- a/packages/zarr-metadata/tests/model/test_pair.py +++ b/packages/zarr-metadata/tests/model/test_pair.py @@ -265,6 +265,40 @@ def test_error_replace_refuses_a_document_with_a_problem() -> None: ZarrV3ArrayMetadata(ARRAY).replace(dimension_names=("x", "y")) +def _core_without_codecs(*names: str) -> ZarrV3Context: + """`CORE_CONTEXT` without the codecs it files under `names`.""" + kept = {name: codec for name, codec in CORE_CONTEXT.codecs.items() if name not in names} + return CORE_CONTEXT.copy(update={"codecs": kept}) + + +def _core_without_data_types(*names: str) -> ZarrV3Context: + """`CORE_CONTEXT` without the data types it files under `names`.""" + kept = {name: each for name, each in CORE_CONTEXT.data_types.items() if name not in names} + return CORE_CONTEXT.copy(update={"data_types": kept}) + + +LITTLE: dict[str, Any] = {"name": "bytes", "configuration": {"endian": "little"}} +ENDIAN: dict[str, Any] = {**ARRAY, "codecs": [LITTLE]} +"""A `uint8` array whose `bytes` codec writes an `endian`, which means nothing for values of one byte. A scope that reads both the codec and the data type compares the codec without it.""" +TRANSPOSED: dict[str, Any] = { + **ARRAY, + "codecs": [{"name": "transpose", "configuration": {"order": [0]}}, LITTLE], +} +SHARDED_ENDIAN: dict[str, Any] = { + **ARRAY, + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [1], + "codecs": [LITTLE], + "index_codecs": [LITTLE, "crc32c"], + }, + } + ], +} + + @pytest.mark.parametrize( ("model", "context", "gained"), [ @@ -272,8 +306,48 @@ def test_error_replace_refuses_a_document_with_a_problem() -> None: (ZarrV3ArrayMetadata(ARRAY, context=CORE_CONTEXT), CORE_AND_EXTENSIONS_CONTEXT, False), (ZarrV3ArrayMetadata(FLOAT, context=ZarrV3Context()), CORE_CONTEXT, True), (ZarrV3ArrayMetadata(SHARDED, context=ZarrV3Context()), CORE_CONTEXT, True), + # What the new scope claims tells it more of the chunk a codec + # receives, so it compares that codec in a simpler spelling than + # the one the document writes. + (ZarrV3ArrayMetadata(ENDIAN, context=ZarrV3Context()), CORE_CONTEXT, True), + ( + ZarrV3ArrayMetadata(ENDIAN, context=_core_without_codecs("bytes")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(ENDIAN, context=_core_without_data_types("uint8")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(TRANSPOSED, context=_core_without_codecs("transpose")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(SHARDED_ENDIAN, context=_core_without_codecs("sharding_indexed")), + CORE_CONTEXT, + True, + ), + ( + ZarrV3ArrayMetadata(SHARDED_ENDIAN, context=_core_without_codecs("bytes")), + CORE_CONTEXT, + True, + ), + ], + ids=[ + "gain", + "nothing-to-gain", + "gain-data-type-with-fill-value", + "gain-of-a-shard", + "gain-of-everything-with-an-endian-that-means-nothing", + "gain-of-a-codec-simplified-for-its-chunk", + "gain-of-the-data-type-a-codec-is-simplified-by", + "gain-of-a-codec-before-one-simplified-for-its-chunk", + "gain-of-a-shard-whose-codecs-are-simplified", + "gain-of-a-codec-in-a-shard", ], - ids=["gain", "nothing-to-gain", "gain-data-type-with-fill-value", "gain-of-a-shard"], ) def test_refined_in_moves_a_model_up_the_order( model: ZarrV3ArrayMetadata, context: ZarrV3Context, gained: bool diff --git a/packages/zarr-metadata/tests/v3/test_definitions.py b/packages/zarr-metadata/tests/v3/test_definitions.py index bcc2f3c9be..adfa8ecc35 100644 --- a/packages/zarr-metadata/tests/v3/test_definitions.py +++ b/packages/zarr-metadata/tests/v3/test_definitions.py @@ -1185,6 +1185,14 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: CRC32C: Final = {"name": "crc32c"} +def _shard_of(codec: JSONValue, *, index_first: bool = False, chunk_shape: int = 2) -> JSONValue: + """A shard that holds `codec`, with its `index_codecs` written before its `codecs` when `index_first`.""" + members: dict[str, JSONValue] = {"codecs": [codec], "index_codecs": [LE, CRC32C]} + if index_first: + members = dict(reversed(members.items())) + return {"name": "sharding_indexed", "configuration": {"chunk_shape": [chunk_shape], **members}} + + @pytest.mark.parametrize( ("one", "other", "kind", "equal"), [ @@ -1233,6 +1241,12 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: CodecDefinition, True, ), + ( + _shard_of(_shard_of(LE)), + _shard_of(_shard_of(LE, index_first=True), index_first=True), + CodecDefinition, + True, + ), ( {"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [LE, CRC32C]}}, {"name": "sharding_indexed", "configuration": {**SHARD, "index_codecs": [CRC32C, LE]}}, @@ -1321,6 +1335,7 @@ def test_a_scope_pickles_as_its_definitions_and_copies_as_itself() -> None: "unclaimed-another-configuration", "another-name", "members-holding-declarations-in-another-order", + "members-holding-declarations-in-another-order-two-deep", "declarations-in-another-order", "members-in-another-order", "blosc-typesize-noshuffle", @@ -1348,6 +1363,20 @@ def test_two_declarations_are_equal_when_they_read_the_same( assert hash(first) == hash(second) +def test_two_refused_declarations_are_equal_whatever_order_their_members_are_in() -> None: + """A refused declaration is compared by what was written and by the declarations it holds, in the order of their locations, not of the members that hold them.""" + # A shard of inner chunks with no elements is refused, and still holds its codecs. + first, _ = resolve(_shard_of(LE, chunk_shape=0), CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT) + second, _ = resolve( + _shard_of(LE, chunk_shape=0, index_first=True), CodecDefinition, CORE_AND_EXTENSIONS_CONTEXT + ) + assert isinstance(first, RefusedDeclaration) + assert isinstance(second, RefusedDeclaration) + assert list(first.nested) != list(second.nested) + assert first == second + assert hash(first) == hash(second) + + def test_a_declaration_copied_or_pickled_is_read_by_a_definition_equal_to_its_own() -> None: declaration, _ = resolve( {"name": "gzip", "configuration": {"level": 1}}, CodecDefinition, CORE_CONTEXT diff --git a/packages/zarr-metadata/tests/v3/test_pipelines.py b/packages/zarr-metadata/tests/v3/test_pipelines.py index 0eda1926b6..12ed0eacd1 100644 --- a/packages/zarr-metadata/tests/v3/test_pipelines.py +++ b/packages/zarr-metadata/tests/v3/test_pipelines.py @@ -23,11 +23,13 @@ canonical_of, resolve, ) +from zarr_metadata.definition._definition import held from zarr_metadata.model import ( ZarrV3ArrayMetadata, read_array_metadata_v3, ) from zarr_metadata.v3._pipeline import ( + codec_refines, read_pipeline, ) from zarr_metadata.v3.codec.bytes import BYTES_CODEC, BytesCodecConfiguration @@ -723,6 +725,58 @@ def test_a_chunk_canonical_that_simplifies_its_own_members_is_taken_at_its_word( assert list(found.nested) == [("codecs", 0), ("codecs", 1)] +@pytest.mark.parametrize( + ("data_type", "mine", "theirs", "without", "refines"), + [ + # The same scope: the order is equality, for the chunk each codec receives. + ("uint8", [LITTLE, GZIP], [BIG, GZIP], (), [True, True]), + ("uint16", [LITTLE, GZIP], [BIG, GZIP], (), [False, True]), + ("uint8", [_shard(BARE)], [_shard(LITTLE)], (), [True]), + # A gain: what the poorer scope left unclaimed reads as the codec, for the chunk it receives. + ("uint8", [LITTLE], [LITTLE], ("bytes",), [True]), + ("uint8", [LITTLE], [BIG], ("bytes",), [True]), + ("uint16", [LITTLE], [BIG], ("bytes",), [False]), + ("uint8", [BARE, GZIP], [LITTLE, GZIP], ("bytes", "gzip"), [True, True]), + # A gain of a codec that holds codecs, and a gain of a codec inside one. + ("uint8", [_shard(BARE)], [_shard(LITTLE)], ("sharding_indexed",), [True]), + ("uint8", [_shard(LITTLE)], [_shard(BIG)], ("bytes",), [True]), + ("uint16", [_shard(LITTLE)], [_shard(BIG)], ("bytes",), [False]), + # The poorer scope knew less of the chunk: it did not read the cast before. + ( + "uint16", + [_cast("uint8"), BARE], + [_cast("uint8"), LITTLE], + ("cast_value",), + [True, True], + ), + # Another codec under the same name is not a gain. + ("uint8", [LITTLE, GZIP], [LITTLE, {"name": "zstd"}], ("zstd",), [True, False]), + ], +) +def test_a_codec_refines_another_reading_of_it_for_the_chunk_it_receives( + data_type: JSONValue, + mine: list[JSONValue], + theirs: list[JSONValue], + without: tuple[str, ...], + refines: list[bool], +) -> None: + """`codec_refines` is the order `refines` defines for declarations, with each codec in its simplest spelling for the chunk its stage receives. `theirs` is read in a scope without the codecs named in `without`, which may leave it knowing less of the chunk.""" + kept = {name: codec for name, codec in CONTEXT.codecs.items() if name not in without} + poorer = CONTEXT.copy(update={"codecs": kept}) + stages, problems = _read(mine, _of(data_type)) + assert problems == () + others = [resolve(codec, CodecDefinition, poorer)[0] for codec in theirs] + found = [codec_refines(stage, other) for stage, other in zip(stages, others, strict=True)] + assert found == refines + # With nothing left out, each codec refines the other exactly when the two are equal. + if without == (): + back, _ = _read(theirs, _of(data_type)) + assert [ + codec_refines(stage, held(other.codec)) + for stage, other in zip(back, stages, strict=True) + ] == refines + + def _bytes_with(chunk_canonical: object) -> ResolvedDeclaration[CodecDefinition[Any]]: """A `bytes` codec with `endian`, read by a definition whose `chunk_canonical` is `chunk_canonical`.""" lying = dataclasses.replace(BYTES_CODEC, name="acme.bytes", chunk_canonical=chunk_canonical)