From 56729b616ab8937d4c306e651a341ef78883cb4f Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 17:24:36 +0100 Subject: [PATCH 01/13] fix(zarr-metadata): draw the same codecs on every derandomized run `_codec` drew from `st.sampled_from` over `(definition, configurations)` pairs whose configurations are strategies built during the draw. Hypothesis labels such a `sampled_from` by hashing its elements, and a strategy hashes by address, so these spans were labeled by memory layout. Hypothesis's mutator copies choices between spans that share a label, so which spans it grouped, and so which examples a derandomized run drew, changed with whatever moved the allocator: another process, machine load. `_codec` now samples an index into the options, labeled by a `range`. The share tests measured this drift. `examples_of` now draws twice and fails if the runs differ, which catches an address-labeled strategy in the same process; the array-document targets share one draw. Bounds are reset from the shares of 100 seeds: the "no axes" ceiling (0.25) sat under the seed maximum (0.303), and the "axis past 32 bits" floor (0.1) over the seed minimum (0.06). Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../src/zarr_metadata/testing/_draw.py | 7 +- .../zarr-metadata/tests/testing/_shares.py | 68 +++++++++++++++---- .../tests/testing/test_documents.py | 26 ++++--- .../tests/testing/test_strategies.py | 8 +-- 4 files changed, 82 insertions(+), 27 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py index f2c0ec3c0b..e63d4afe1c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py @@ -278,7 +278,12 @@ def pipelines_for( def _codec( draw: st.DrawFn, options: Sequence[tuple[K, Configurations]] ) -> tuple[K, ZarrV3DeclarationJSON]: - definition, configurations = draw(st.sampled_from(options)) + # An index, not the options: Hypothesis labels a `sampled_from` by + # hashing its elements, and each option holds a strategy built for this + # draw, whose hash is its address. Labels that follow addresses send + # Hypothesis's mutator down different paths from one run to the next, so + # a derandomized run would not draw the same examples twice. + definition, configurations = options[draw(st.sampled_from(range(len(options))))] return definition, draw(any_spelling(definition, configurations)) diff --git a/packages/zarr-metadata/tests/testing/_shares.py b/packages/zarr-metadata/tests/testing/_shares.py index 27517e2653..3d70bf11aa 100644 --- a/packages/zarr-metadata/tests/testing/_shares.py +++ b/packages/zarr-metadata/tests/testing/_shares.py @@ -2,9 +2,15 @@ A reach test asks whether a target can be drawn at all, in as many examples as it takes, which says nothing of whether a test, at the 100 examples it -runs by default, ever meets it. Runs are derandomized, which keeps most shares -fixed, but a share can still move a few points between processes, so a bound -needs room: this is no place to bound what is drawn a few percent of the time. +runs by default, ever meets it. Runs are derandomized, so a share is a fixed +number for a given strategy. A change to the strategy draws other examples, +as another seed would, so a bound sits where the share stays across seeds: +this is no place to bound what is drawn a few percent of the time. + +A bound is set from the shares of 100 seeds, each run as `examples_of` +runs, but with `derandomize=False` under `hypothesis.seed`. It lies past +the most extreme share on its side by a quarter of that share's distance +from the median, rounded outward to a hundredth. """ from __future__ import annotations @@ -14,24 +20,58 @@ from hypothesis import HealthCheck, given, settings if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from hypothesis.strategies import SearchStrategy T = TypeVar("T") +_DERANDOMIZED = settings( + database=None, + derandomize=True, + deadline=None, + suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], +) + + +def examples_of(strategy: SearchStrategy[T], *, examples: int = 300) -> list[T]: + """The examples a derandomized run of `strategy` draws, checked to be the same in a second run. + + Hypothesis's mutator copies spans between the parts of an example that + share a label, and a label can follow an object's address: a + `sampled_from` of elements that hold a strategy built for the draw is + labeled by that strategy's `id`. Such a strategy draws other examples + from run to run, and from machine to machine, however derandomized. A + second run in the same process lays objects out at other addresses, so + it catches this. + """ + first = _draw(strategy, examples) + second = _draw(strategy, examples) + if [repr(value) for value in first] != [repr(value) for value in second]: + raise AssertionError( + f"{strategy!r} drew other examples on a second derandomized run. " + "A strategy built during a draw is likely in a `sampled_from`, which " + "labels it by its address; sample an index into the options instead." + ) + return first + + +def share(values: Sequence[T], counts: Callable[[T], bool]) -> float: + """The share of `values` that `counts` holds of.""" + return sum(counts(value) for value in values) / len(values) + def share_of( strategy: SearchStrategy[T], counts: Callable[[T], bool], *, examples: int = 300 ) -> float: """The share of the examples of `strategy` that `counts` holds of.""" - tally: list[bool] = [] - run = settings( - max_examples=examples, - database=None, - derandomize=True, - deadline=None, - suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], - )(given(strategy)(lambda value: tally.append(counts(value)))) - run() - return sum(tally) / len(tally) + return share(examples_of(strategy, examples=examples), counts) + + +def _draw(strategy: SearchStrategy[T], examples: int) -> list[T]: + drawn: list[T] = [] + # A lambda, not `drawn.append`: Hypothesis cannot wrap a builtin method, + # and derandomizes from the test function's source, so this line is the seed. + run = given(strategy)(lambda value: drawn.append(value)) + settings(_DERANDOMIZED, max_examples=examples)(run)() + return drawn diff --git a/packages/zarr-metadata/tests/testing/test_documents.py b/packages/zarr-metadata/tests/testing/test_documents.py index 6c20a3c29f..53cd047724 100644 --- a/packages/zarr-metadata/tests/testing/test_documents.py +++ b/packages/zarr-metadata/tests/testing/test_documents.py @@ -8,6 +8,7 @@ from __future__ import annotations +import functools import json from typing import TYPE_CHECKING, Any, cast @@ -15,7 +16,7 @@ from hypothesis import find, given, settings from hypothesis import strategies as st -from tests.testing._shares import share_of +from tests.testing._shares import examples_of, share from zarr_metadata.definition import AcceptedDeclaration, Definition, resolve from zarr_metadata.definition._definition import canonicalize from zarr_metadata.model import ( @@ -340,19 +341,28 @@ def test_a_data_type_no_codec_takes_is_not_drawn(document: Mapping[str, Any]) -> _DOCUMENT_SHARES: dict[str, tuple[Callable[[Mapping[str, Any]], bool], float, float]] = { - "no axes": (lambda d: d["shape"] == (), 0.03, 0.25), - "more axes than five": (lambda d: len(d["shape"]) > 5, 0.1, 1), - "a zero-length axis": (lambda d: 0 in d["shape"], 0.1, 1), - "an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.1, 1), - "the first data type": (lambda d: _name(d["data_type"]) == "bool", 0, 0.12), + "no axes": (lambda d: d["shape"] == (), 0.04, 0.34), + "more axes than five": (lambda d: len(d["shape"]) > 5, 0.15, 1), + "a zero-length axis": (lambda d: 0 in d["shape"], 0.44, 1), + "an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.03, 1), + "the first data type": (lambda d: _name(d["data_type"]) == "bool", 0, 0.18), } -"""What a share of array documents holds, as a low and a high bound: what a test running only a few hundred examples meets.""" +"""What a share of array documents holds, as a low and a high bound: what a test running only a few hundred examples meets. + +Each bound is set as `_shares` says, from 100 seeds. +""" + + +@functools.cache +def _array_document_examples() -> list[Mapping[str, Any]]: + """Drawn once for every target, as each counts the same examples.""" + return examples_of(array_documents()) @pytest.mark.parametrize("target", _DOCUMENT_SHARES) def test_array_documents_share(target: str) -> None: counts, low, high = _DOCUMENT_SHARES[target] - assert low <= share_of(array_documents(), counts) <= high + assert low <= share(_array_document_examples(), counts) <= high def _empty_listings(document: Mapping[str, Any]) -> list[Any]: diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index 8ef6023265..4d4a6d78d0 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -1057,7 +1057,7 @@ def _float_class(value: object) -> str | None: ) -@pytest.mark.parametrize(("kind", "at_least"), [("normal", 0.2), ("NaN", 0.1), ("infinity", 0.01)]) +@pytest.mark.parametrize(("kind", "at_least"), [("normal", 0.2), ("NaN", 0.26), ("infinity", 0.01)]) def test_hex_fill_values_are_not_only_small_numbers(kind: str, at_least: float) -> None: """Bits drawn as an integer alone are nearly all subnormal.""" assert share_of(_HEX_FILL_VALUES, lambda value: _float_class(value) == kind) >= at_least @@ -1097,7 +1097,7 @@ def _shard_inner_lengths(draw: st.DrawFn) -> tuple[int, ...]: def test_shards_of_a_rectilinear_grid_are_often_divided() -> None: """Edges drawn apart rarely share a factor, which leaves a shard only inner chunks of length 1.""" - assert share_of(_shard_inner_lengths(), lambda inner: any(n > 1 for n in inner)) >= 0.4 + assert share_of(_shard_inner_lengths(), lambda inner: any(n > 1 for n in inner)) >= 0.39 def _edge_factor(configuration: Mapping[str, Any]) -> int: @@ -1145,7 +1145,7 @@ def test_a_long_dimension_keeps_its_edge_factor() -> None: edges = chunk_grid_configurations(RECTILINEAR_CHUNK_GRID, (2**16 + 1,)).filter( lambda configuration: not isinstance(configuration["chunk_shapes"][0], int) ) - assert share_of(edges, lambda configuration: _edge_factor(configuration) > 1) >= 0.3 + assert share_of(edges, lambda configuration: _edge_factor(configuration) > 1) >= 0.32 def test_divisors_of_a_very_long_shard_are_found_quickly() -> None: @@ -1168,4 +1168,4 @@ def _field_types(declaration: Any) -> list[str]: # noqa: ANN401 def test_struct_fields_do_not_favor_the_first_data_type() -> None: """A type drawn after a field's name would be `bool` a fifth of the time.""" - assert share_of(declarations(STRUCT_DATA_TYPE), lambda d: _field_types(d)[0] == "bool") <= 0.12 + assert share_of(declarations(STRUCT_DATA_TYPE), lambda d: _field_types(d)[0] == "bool") <= 0.16 From 6ecbaebd520800c8b2e018d142e3f512f6f96fa5 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 18:18:52 +0100 Subject: [PATCH 02/13] docs(zarr-metadata): add changelog fragment for #4635 Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/changes/4635.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/zarr-metadata/changes/4635.bugfix.md diff --git a/packages/zarr-metadata/changes/4635.bugfix.md b/packages/zarr-metadata/changes/4635.bugfix.md new file mode 100644 index 0000000000..d92c9b5193 --- /dev/null +++ b/packages/zarr-metadata/changes/4635.bugfix.md @@ -0,0 +1 @@ +Strategies in `zarr_metadata.testing.strategies` that draw codec pipelines, such as `array_documents` and `pipelines`, now draw the same examples on every derandomized Hypothesis run. A codec was sampled from options that hold strategies built during the draw, which Hypothesis labels by memory address, so the examples its mutator produced varied from one process to the next. From 42a175971fa4794d2af36a60e7f5316df7901ce0 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 14:47:20 +0100 Subject: [PATCH 03/13] fix(zarr-metadata): refuse an empty rectilinear edge list zarr-extensions#78 (cadd1dab6) made the array form of a dimension's chunk edge lengths non-empty, even for an axis of length 0, which writes a bare integer to have no chunks. The rectilinear chunk grid's rules now report `chunk_shapes: [[]]`, its JSON Schema says `minItems: 1` for the list, and `chunk_grid_configurations` always draws at least one edge. Spec links are re-pinned to cadd1dab6. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/changes/4635.bugfix.md | 1 + .../src/zarr_metadata/testing/_chunk_grid.py | 11 +++---- .../src/zarr_metadata/v3/_json_schema.py | 33 +++++++++++++++++++ .../src/zarr_metadata/v3/_types/chunk_grid.py | 16 +++++---- .../v3/chunk_grid/rectilinear.py | 29 ++++++++++++++-- .../tests/testing/test_strategies.py | 11 ++++++- .../tests/v3/test_grid_shapes.py | 16 +++++++-- 7 files changed, 96 insertions(+), 21 deletions(-) create mode 100644 packages/zarr-metadata/changes/4635.bugfix.md diff --git a/packages/zarr-metadata/changes/4635.bugfix.md b/packages/zarr-metadata/changes/4635.bugfix.md new file mode 100644 index 0000000000..ac16d99ecc --- /dev/null +++ b/packages/zarr-metadata/changes/4635.bugfix.md @@ -0,0 +1 @@ +The `rectilinear` chunk grid now refuses an empty list of chunk edge lengths, such as `"chunk_shapes": [[]]`, even along a dimension of length 0. zarr-extensions#78 made the list form non-empty; a dimension with no chunks writes a bare integer instead. The JSON Schema says `minItems: 1` for the list, and `chunk_grid_configurations` no longer draws an empty list. diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py index a987f241c8..6a88f813ea 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py @@ -89,17 +89,14 @@ def _edges(draw: st.DrawFn, extent: int | None) -> tuple[int | tuple[int, int], can be long, since a pair costs the same however many chunks it counts, which is what the spelling is for. Its edges often share a factor, as they must for a shard to divide them into inner chunks - longer than 1; edges drawn apart rarely do. A - dimension of length zero, or of a length not known, may be covered by - no edges at all. + longer than 1; edges drawn apart rarely do. There is always at least + one edge, even along a dimension of length zero: the spec allows no + empty list, and a dimension with no chunks writes a bare length instead. """ - min_runs = 0 if extent is None or extent == 0 else 1 factor = draw(st.just(1) | st.integers(2, _MAX_FACTOR)) sizes = [ factor * size - for size in draw( - st.lists(st.integers(1, _MAX_LENGTH), min_size=min_runs, max_size=_MAX_RUNS) - ) + for size in draw(st.lists(st.integers(1, _MAX_LENGTH), min_size=1, max_size=_MAX_RUNS)) ] runs = [(size, draw(_count(extent, size))) for size in sizes] covered = sum(size * count for size, count in runs) diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py b/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py index f647895154..3530618288 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py @@ -22,6 +22,7 @@ ExtraFieldDefinition, GroupExtraFieldDefinition, ) +from zarr_metadata.v3._types.chunk_grid import RectilinearChunkGridConfiguration from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON @@ -82,6 +83,10 @@ def declaration_schemas(context: ZarrV3Context) -> SchemaLeaf: """The schema leaf that `declaration_json_schema_v3` uses. It writes each declaration alias as a declaration of its kind, as `context` reads it.""" def leaf(annotation: object, schemas: Schemas) -> JSONSchema | None: + if annotation is RectilinearChunkGridConfiguration: + return schemas.defined( + annotation, annotation.__name__, lambda: _rectilinear_configuration(schemas) + ) kind = declaration_kind(annotation) if kind is None or not is_alias(annotation): return None @@ -273,6 +278,34 @@ def _member_object(schema: Mapping[str, JSONValue], key: str) -> Mapping[str, JS return found +def _rectilinear_configuration(schemas: Schemas) -> JSONSchema: + """The rectilinear chunk grid's configuration: its TypedDict, with at least one edge length in each dimension's list. + + The TypedDict cannot declare a minimum length, so the schema adds it, + as the grid's rules check it: the spec requires "a non-empty array", + even for a dimension of length 0. + """ + schema = schemas.object_of(RectilinearChunkGridConfiguration) + properties = _member_object(schema, "properties") + chunk_shapes = _member_object(properties, "chunk_shapes") + dimension = _member_object(chunk_shapes, "items") + branches = dimension.get("anyOf") + if not isinstance(branches, list): + msg = f"the schema of a rectilinear dimension is a union, got {dimension!r}" + raise TypeError(msg) + nonempty: list[JSONValue] = [ + {**branch, "minItems": 1} + if isinstance(branch, Mapping) and branch.get("type") == "array" + else branch + for branch in branches + ] + items: JSONSchema = {**dimension, "anyOf": nonempty} + return { + **schema, + "properties": {**properties, "chunk_shapes": {**chunk_shapes, "items": items}}, + } + + def _array(context: ZarrV3Context, schemas: Schemas) -> JSONSchema: """An array document: its TypedDict, at least one codec, and a constraint on its fill value for each data type in scope, which applies when the document names that data type. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py index ba34cde633..4e74edccf8 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py @@ -67,7 +67,7 @@ class RegularChunkGridObject(TypedDict, closed=True): """ -# rectilinear: https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md +# rectilinear: https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md RECTILINEAR_CHUNK_GRID_NAME: Final = "rectilinear" """The `name` field value of the rectilinear chunk grid.""" @@ -81,8 +81,10 @@ class RegularChunkGridObject(TypedDict, closed=True): """JSON shape for one dimension's rectilinear spec. Either a bare integer (uniform shorthand for a regular dimension within -a rectilinear grid), or a tuple of integers and/or `[value, count]` RLE -pairs. Every extent, and every run's length and count, is at least 1. +a rectilinear grid), or a non-empty tuple of integers and/or +`[value, count]` RLE pairs. Every extent, and every run's length and +count, is at least 1. The type cannot declare that the tuple is +non-empty; the rectilinear grid's rules check it. """ @@ -95,9 +97,9 @@ class RectilinearChunkGridConfiguration(TypedDict, closed=True): """The chunk edge lengths along each dimension of the array, one entry per dimension. An entry is a bare integer, a step repeated until it covers the - dimension, or a list of edge lengths and `[length, count]` runs, whose - lengths must sum to at least the dimension's length. Every length and - count is at least 1. + dimension, or a non-empty list of edge lengths and `[length, count]` + runs, whose lengths must sum to at least the dimension's length. Every + length and count is at least 1. """ @@ -121,7 +123,7 @@ class RectilinearChunkGridObject(TypedDict, closed=True): `kind` and `chunk_shapes` are required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this grid. - https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md#L59-L62 + https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md#L59-L62 https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 0b6db11ac1..e512001b48 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -1,7 +1,7 @@ """ Rectilinear chunk grid (zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md +See https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md """ from collections.abc import Iterator @@ -72,6 +72,28 @@ def _canonical( ) +def _rules( + configuration: RectilinearChunkGridConfiguration, nested: Nested +) -> Iterator[ValidationProblem]: + """Check that each dimension's list of chunk edge lengths has at least one. + + "as a non-empty array", and "For an axis of length 0, the array form + still requires at least one edge length" + (https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md?plain=1#L89-L95). + A dimension that has no chunks, because its length is 0, writes a bare + integer instead. The type cannot declare a minimum length, so this rule + checks it. + """ + for axis, spec in enumerate(configuration["chunk_shapes"]): + if isinstance(spec, tuple) and len(spec) == 0: + yield ValidationProblem( + ("chunk_shapes", axis), + "expected at least one chunk edge length, got an empty list", + "invalid_value", + ctx={"min_length": 1}, + ) + + def _shape_rules( configuration: RectilinearChunkGridConfiguration, nested: Nested, shape: tuple[int, ...] ) -> Iterator[ValidationProblem]: @@ -79,7 +101,7 @@ def _shape_rules( "The length of `chunk_shapes` MUST match the number of dimensions of the array", and "The sum of the edge lengths MUST equal or exceed `L`" - (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md?plain=1#L62-L91). + (https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md?plain=1#L62-L95). A bare integer repeats until it covers the dimension, so it always passes this check. """ @@ -112,7 +134,7 @@ def _chunk_lengths( A bare integer is the length of every chunk along its axis. A list gives the length of each chunk, and a `[length, count]` pair gives `count` chunks of that length - (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md?plain=1#L62-L91). + (https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md?plain=1#L62-L95). Every listed length counts, including the length of a chunk past the array's edge, which the array grows into when it is resized. """ @@ -127,6 +149,7 @@ def _chunk_lengths( RECTILINEAR_CHUNK_GRID: Final = ChunkGridDefinition( name=RECTILINEAR_CHUNK_GRID_NAME, configuration=RectilinearChunkGridConfiguration, + rules=_rules, canonical=_canonical, shape_rules=_shape_rules, chunk_lengths=_chunk_lengths, diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index 8ef6023265..bd16ca573f 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -670,7 +670,6 @@ def _is_pair(entry: object) -> bool: _RECTILINEAR_SPELLINGS: dict[str, tuple[tuple[int, ...] | None, Callable[[object], bool]]] = { "a bare length": (None, lambda dimension: isinstance(dimension, int)), - "no edges": (None, lambda dimension: dimension == ()), "only written-out lengths": ( None, lambda dimension: ( @@ -741,6 +740,16 @@ def test_rectilinear_dimensions_reach(spelling: str) -> None: ) +@given(data=st.data(), shape=st.none() | st.lists(st.integers(0, 2), max_size=3).map(tuple)) +def test_rectilinear_dimensions_are_never_empty( + data: st.DataObject, shape: tuple[int, ...] | None +) -> None: + """A list of edges has at least one, even along a dimension of length 0 or of a length not known.""" + configuration = data.draw(chunk_grid_configurations(RECTILINEAR_CHUNK_GRID, shape)) + assert () not in configuration["chunk_shapes"] + assert list(RECTILINEAR_CHUNK_GRID.rules(configuration, {})) == [] + + @given(data=st.data(), shape=st.lists(st.integers(0, 2**63 - 1), max_size=3).map(tuple)) def test_chunk_grids_fit_a_long_shape(data: st.DataObject, shape: tuple[int, ...]) -> None: """However long an axis is, a grid covers it without a list as long as the axis.""" diff --git a/packages/zarr-metadata/tests/v3/test_grid_shapes.py b/packages/zarr-metadata/tests/v3/test_grid_shapes.py index 40887f37bb..d56dc4d366 100644 --- a/packages/zarr-metadata/tests/v3/test_grid_shapes.py +++ b/packages/zarr-metadata/tests/v3/test_grid_shapes.py @@ -64,8 +64,9 @@ def _problems(grid: JSONValue, shape: tuple[int, ...]) -> list[tuple[tuple[str | (_rectilinear([4, 4, 4]), (10,), ({4},)), (_rectilinear([4, 4, 2, 7]), (10,), ({4, 2, 7},)), (_rectilinear([[4, 2], 2]), (10,), ({4, 2},)), - # A dimension of length 0 and no chunks. - (_rectilinear([]), (0,), (set(),)), + # A dimension of length 0: a bare integer gives it no chunks, and a + # list still lists at least one. + (_rectilinear(4, [3]), (0, 0), ({4}, {3})), (_rectilinear(2, [3, 3]), (7, 6), ({2}, {3})), # A grid that nothing in scope claims, or one that is not read, is # not checked, and its lengths are unknown along each axis. @@ -95,13 +96,22 @@ def test_error_a_rectilinear_grid_of_another_rank(grid: JSONValue, shape: tuple[ assert _problems(grid, shape) == [(("configuration", "chunk_shapes"), "invalid_value")] -@pytest.mark.parametrize("spec", [[4, 4], [[4, 2]], []]) +@pytest.mark.parametrize("spec", [[4, 4], [[4, 2]]]) def test_error_rectilinear_chunks_that_do_not_cover_their_dimension(spec: JSONValue) -> None: assert _problems(_rectilinear(2, spec), (4, 10)) == [ (("configuration", "chunk_shapes", 1), "invalid_value") ] +def test_error_rectilinear_empty_edge_list() -> None: + # The spec requires a non-empty list even along a dimension of length 0, + # which writes a bare integer to have no chunks; no shape is needed to tell. + _, found = resolve(_rectilinear(2, []), ChunkGridDefinition, CORE_AND_EXTENSIONS_CONTEXT) + assert [(problem.loc, problem.kind) for problem in found] == [ + (("configuration", "chunk_shapes", 1), "invalid_value") + ] + + def test_a_grid_that_says_nothing_of_the_shape_fits_every_one_its_lengths_unknown() -> None: lenient = ChunkGridDefinition(name="acme.grid", configuration=EmptyConfiguration) context = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(chunk_grids=[lenient])) From 077376ca90ba448f98e1c5a90e77fc55255ce243 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 17:17:20 +0100 Subject: [PATCH 04/13] feat(zarr-metadata): read MinLen and MaxLen on arrays in the TypedDict checker The checker, the JSON Schema writer and the pydantic schema now read annotated-types' `MinLen`, `MaxLen` and `Len` on an array, as pydantic does. One table of constraints now drives parsing, the shape check and the schema keywords, in place of five parallel tables. An array outside its bounds is an `invalid_value` problem whose `ctx` holds `min_length` or `max_length`. A length bound on anything but an array raises `TypeError`. `codecs` and each rectilinear dimension's edge list now declare `MinLen(1)` on their types. This removes the two hand-patched `minItems: 1` schema fragments and the rectilinear `_rules` function. `chunk_shapes: [[]]` and `codecs: []` are still refused at the same `loc`. `check` now also refuses `codecs: []`. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../zarr-metadata/changes/4635.feature.md | 1 + packages/zarr-metadata/docs/extensions.md | 5 +- .../src/zarr_metadata/_pydantic_schema.py | 7 +- .../src/zarr_metadata/_typed_json.py | 224 +++++++++++------- .../src/zarr_metadata/definition/__init__.py | 6 +- .../src/zarr_metadata/model/_array_v3.py | 8 +- .../src/zarr_metadata/typed_json.py | 7 +- .../src/zarr_metadata/v3/_json_schema.py | 43 +--- .../src/zarr_metadata/v3/_types/chunk_grid.py | 13 +- .../src/zarr_metadata/v3/array.py | 4 +- .../v3/chunk_grid/rectilinear.py | 23 -- .../zarr-metadata/tests/test_json_schema.py | 29 ++- .../zarr-metadata/tests/test_typed_json.py | 154 ++++++++++-- 13 files changed, 332 insertions(+), 192 deletions(-) create mode 100644 packages/zarr-metadata/changes/4635.feature.md diff --git a/packages/zarr-metadata/changes/4635.feature.md b/packages/zarr-metadata/changes/4635.feature.md new file mode 100644 index 0000000000..df9a1fc974 --- /dev/null +++ b/packages/zarr-metadata/changes/4635.feature.md @@ -0,0 +1 @@ +A TypedDict read by `zarr_metadata.typed_json.check`, or used as a definition's configuration, may now bound the length of an array with annotated-types' `MinLen`, `MaxLen` and `Len`, as pydantic reads them: `Annotated[tuple[int, ...], MinLen(1)]` is a non-empty array. An array outside its bounds is an `invalid_value` problem whose `ctx` holds `min_length` or `max_length`, and the JSON Schema writes `minItems` and `maxItems`. A length bound on anything but an array raises `TypeError`. An array document's `codecs` and each list in a `rectilinear` chunk grid's `chunk_shapes` now say on their types that they are never empty, so `check` also refuses `"codecs": []`. diff --git a/packages/zarr-metadata/docs/extensions.md b/packages/zarr-metadata/docs/extensions.md index 57acdeffbd..bfc3c39753 100644 --- a/packages/zarr-metadata/docs/extensions.md +++ b/packages/zarr-metadata/docs/extensions.md @@ -82,8 +82,9 @@ names `acme.lz4` is then checked against this definition. - a `NewType` or a type alias. A number may carry bounds from annotated-types: `Gt`, `Ge`, `Lt`, `Le` or -`Interval`. Any other `Annotated` metadata is refused when the definition -is built. +`Interval`. An array may bound its length with `MinLen`, `MaxLen` or +`Len`, so `Annotated[tuple[int, ...], MinLen(1)]` is never empty. Any +other `Annotated` metadata is refused when the definition is built. **The rules function** diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py index c7b2074ea2..6e97d43d8d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -1,8 +1,8 @@ """Private input types used only to generate accurate Pydantic JSON schemas. These repeat the document TypedDicts in a stricter form: extension names -match the spec's pattern, `must_understand` is `true` at an extension -point, and `codecs` is not empty. They exist because pydantic builds a +match the spec's pattern, and `must_understand` is `true` at an extension +point. They exist because pydantic builds a schema from Python types and cannot embed a JSON Schema that has its own `$defs`, such as the ones `zarr_metadata.json_schema` writes; the v3 document schemas are recursive, so they cannot be inlined either. @@ -16,6 +16,7 @@ from collections.abc import Mapping # resolved by Pydantic at runtime from typing import Annotated, Literal, NotRequired +from annotated_types import MinLen from pydantic import Field from typing_extensions import TypedDict @@ -57,7 +58,7 @@ class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): ZarrV3DeclarationJSON = ExtensionName | ZarrV3NamedConfigJSON ZarrV3MandatoryDeclarationJSON = ExtensionName | ZarrV3MandatoryNamedConfigJSON -ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MandatoryDeclarationJSON, ...], Field(min_length=1)] +ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MandatoryDeclarationJSON, ...], MinLen(1)] ZarrV2FilterPipelineJSON = tuple[ZarrV2CodecMetadata, ...] diff --git a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py index e9574b46c4..3469b14d45 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py @@ -9,9 +9,10 @@ `Mapping[str, V]`, a `NewType` (parsed as its supertype), and a type alias (parsed as the type it stands for). This short list keeps the module small. Any other annotation has no parser. A number type may carry bounds, -written with annotated-types as pydantic reads them, for example -`Annotated[int, Interval(ge=0, le=9)]`. A value outside the bounds is a -problem. `constraints_of` lists the bounds the checker supports. +and an array type bounds on its length, written with annotated-types as +pydantic reads them, for example `Annotated[int, Interval(ge=0, le=9)]` or +`Annotated[tuple[int, ...], MinLen(1)]`. A value outside the bounds is a +problem. `constraints_of` lists the constraints the checker supports. A TypedDict is read as the typing spec defines it. This is not always what its runtime attributes say. `typeddict_keys` reads which keys the @@ -838,47 +839,66 @@ def parse(candidate: object, loc: Loc) -> Parsed: # --- constraints --------------------------------------------------------- Constraints: TypeAlias = Mapping[str, int | float] -"""The bounds a type carries, keyed by the names pydantic uses: `{"ge": 0, "le": 9}`.""" +"""The constraints a type carries, keyed by the names pydantic uses: `{"ge": 0, "le": 9}`, `{"min_length": 1}`.""" -_BOUNDS: Final[tuple[tuple[type, str], ...]] = ( - (annotated_types.Gt, "gt"), - (annotated_types.Ge, "ge"), - (annotated_types.Lt, "lt"), - (annotated_types.Le, "le"), -) -"""The annotated-types constraints the checker supports, each with its name.""" -_FROM_BELOW: Final = frozenset({"gt", "ge"}) -"""The lower bounds.""" +@dataclass(frozen=True, slots=True) +class _Constraint: + """A constraint the checker reads: its annotated-types marker, and how it is checked and written.""" + + marker: type + """The annotated-types class. Its attribute named as the constraint holds the bound.""" + keyword: str + """JSON Schema's keyword for the constraint.""" + holds: Callable[[float, float], bool] + """Whether a measured value satisfies the bound.""" + side: Literal["below", "above"] + """The side the bound limits. A type takes one bound from each side.""" + of: Literal["number", "length"] + """What the bound limits: a number, or the length of an array.""" + + +_CONSTRAINTS: Final[Mapping[str, _Constraint]] = { + "gt": _Constraint(annotated_types.Gt, "exclusiveMinimum", operator.gt, "below", "number"), + "ge": _Constraint(annotated_types.Ge, "minimum", operator.ge, "below", "number"), + "lt": _Constraint(annotated_types.Lt, "exclusiveMaximum", operator.lt, "above", "number"), + "le": _Constraint(annotated_types.Le, "maximum", operator.le, "above", "number"), + "min_length": _Constraint(annotated_types.MinLen, "minItems", operator.ge, "below", "length"), + "max_length": _Constraint(annotated_types.MaxLen, "maxItems", operator.le, "above", "length"), +} +"""The annotated-types constraints the checker reads, by the name pydantic uses, which is also the attribute of the marker that holds the bound.""" -_FROM_ABOVE: Final = frozenset({"lt", "le"}) -"""The upper bounds.""" +_NAMES: Final = tuple(each.marker.__name__ for each in _CONSTRAINTS.values()) +"""The names of the markers the checker reads, in the order of `_CONSTRAINTS`.""" -_EXCLUSIVE: Final = frozenset({"gt", "lt"}) -"""The exclusive bounds: a value may not equal them.""" +_MARKERS: Final = f"{', '.join(_NAMES[:-1])} and {_NAMES[-1]}" +"""The markers the checker reads, as an error message lists them: "Gt, Ge, Lt, Le, MinLen and MaxLen".""" -_HOLDS: Final[Mapping[str, Callable[[float, float], bool]]] = { - "gt": operator.gt, - "ge": operator.ge, - "lt": operator.lt, - "le": operator.le, +_MEASURED: Final[Mapping[str, tuple[frozenset[str | None], str]]] = { + "number": (frozenset({"int", "number"}), "a number"), + "length": (frozenset({"tuple"}), "an array"), } -"""For each bound name, a test of whether a number satisfies that bound.""" +"""For what each constraint limits, the shapes (from `shape_of`) it applies to, and their name in an error message.""" + +_EXCLUSIVE: Final = frozenset({"gt", "lt"}) +"""The exclusive bounds: a value may not equal them.""" def constraints_of(metadata: Sequence[object]) -> dict[str, int | float]: - """The bounds in an `Annotated` type's metadata, by name: at most one lower and one upper bound. + """The constraints in an `Annotated` type's metadata, by name: at most one lower and one upper bound. - The supported bounds are annotated-types' `Gt`, `Ge`, `Lt` and `Le`, - and an `Interval`, which unpacks to them, as pydantic reads them. A - note (a string or a `Doc`) does not constrain the value and is - skipped. Anything else raises `TypeError`: + The supported constraints are annotated-types' `Gt`, `Ge`, `Lt` and + `Le`, which bound a number, `MinLen` and `MaxLen`, which bound the + length of an array, and an `Interval` or a `Len`, which unpack to + them, as pydantic reads them. A note (a string or a `Doc`) does not + constrain the value and is skipped. Anything else raises `TypeError`: - - a constraint the checker does not support, such as `MinLen`, + - a constraint the checker does not support, such as `MultipleOf`, `Predicate`, or pydantic's `Field`. Otherwise the type would declare a rule that the checker does not enforce. - a second bound on the same side. pydantic uses the last one. - - a bound that is not a finite number. + - a bound that is not a finite number, or a length bound that is not + a non-negative integer. A bound is stored as the number it equals, as an integer when it is one. `Ge(True)`, `Ge(1.0)` and `Ge(1)` are the same bound, because @@ -889,16 +909,23 @@ def constraints_of(metadata: Sequence[object]) -> dict[str, int | float]: for item in _unpacked(metadata): if isinstance(item, (str, typing_extensions.Doc)): continue - name = next((name for kind, name in _BOUNDS if isinstance(item, kind)), None) + name = next( + (name for name, each in _CONSTRAINTS.items() if isinstance(item, each.marker)), None + ) if name is None: - msg = f"{item!r} is not a constraint the checker reads, which are Gt, Ge, Lt and Le" + msg = f"{item!r} is not a constraint the checker reads, which are {_MARKERS}" raise UnreadConstraintError(msg) - side = _FROM_BELOW if name in _FROM_BELOW else _FROM_ABOVE - if not side.isdisjoint(said): - where = "below" if side is _FROM_BELOW else "above" - msg = f"{item!r} is a second bound from {where}; a type takes one from each side" + constraint = _CONSTRAINTS[name] + if any( + _CONSTRAINTS[held].side == constraint.side and _CONSTRAINTS[held].of == constraint.of + for held in said + ): + msg = ( + f"{item!r} is a second bound from {constraint.side}; " + "a type takes one from each side" + ) raise TypeError(msg) - said[name] = _bound(item, name) + said[name] = _bound(item, name) if constraint.of == "number" else _length(item, name) return said @@ -922,32 +949,50 @@ def _bound(item: object, name: str) -> int | float: raise TypeError(msg) +def _length(item: object, name: str) -> int: + """The length of length bound `item` named `name`; `TypeError` if it is not a non-negative integer.""" + bound = _bound(item, name) + if isinstance(bound, int) and bound >= 0: + return bound + msg = f"{item!r}: a length bound is a non-negative integer" + raise TypeError(msg) + + def _constrainable(inner: object, said: Constraints) -> None: - """Raise `TypeError` for bounds on a type that is not a number: a string, an array, a value of any type.""" - if len(said) != 0 and shape_of(inner) not in ("int", "number"): - msg = f"{', '.join(said)}: a bound is on a number, and {describe(inner)} is not one" - raise TypeError(msg) + """Raise `TypeError` for a constraint on a type it does not limit: a bound on what is not a number, or a length bound on what is not an array.""" + shape = shape_of(inner) + for of, (shapes, noun) in _MEASURED.items(): + names = [name for name in said if _CONSTRAINTS[name].of == of] + if len(names) != 0 and shape not in shapes: + msg = f"{', '.join(names)}: a bound is on {noun}, and {describe(inner)} is not one" + raise TypeError(msg) def constrained(parse: Parser, description: str, said: Constraints) -> Parser: - """Wrap `parse` to also check the bounds its type carries: one `invalid_value` problem if any bound fails. - - The message states what the type allows, for example "expected an - integer in [0, 9], got 12", and the problem's `ctx` holds the bounds. - The bounds are checked only when `parse` reports no problem, in the - same way that rules run only on a value of their type. + """Wrap `parse` to also check the constraints its type carries: one `invalid_value` problem if any fails. + + A bound limits a number, and a length bound the number of elements in + an array. The message states what the type allows, for example + "expected an integer in [0, 9], got 12" or "expected an array of + integers with at least 1 element, got []", and the problem's `ctx` + holds the constraints, as pydantic names them. The constraints are + checked only when `parse` reports no problem, in the same way that + rules run only on a value of their type. """ expected = f"expected {description} {_admitted(said)}" ctx: dict[str, JSONValue] = dict(said) - holds = tuple((_HOLDS[name], bound) for name, bound in said.items()) + checks = tuple( + (_CONSTRAINTS[name].holds, bound, _CONSTRAINTS[name].of == "length") + for name, bound in said.items() + ) def parse_constrained(value: object, loc: Loc) -> Parsed: typed, found = parse(value, loc) if len(found) != 0: return typed, found - number = cast("float", typed) - for keeps, bound in holds: - if not keeps(number, bound): + for keeps, bound, length in checks: + measured = len(cast("tuple[object, ...]", typed)) if length else cast("float", typed) + if not keeps(measured, bound): message = f"{expected}, got {shown(value)}" return typed, (ValidationProblem(loc, message, "invalid_value", ctx=ctx),) return typed, () @@ -956,9 +1001,15 @@ def parse_constrained(value: object, loc: Loc) -> Parsed: def _admitted(said: Constraints) -> str: - """The range the bounds allow, as written after the type in a message: "in [0, 9]", "in (0, 1)", ">= 1".""" - low = next(((name, bound) for name, bound in said.items() if name in _FROM_BELOW), None) - high = next(((name, bound) for name, bound in said.items() if name in _FROM_ABOVE), None) + """What the constraints allow, as written after the type in a message: "in [0, 9]", "in (0, 1)", ">= 1", "with at least 1 element".""" + if any(_CONSTRAINTS[name].of == "length" for name in said): + return _admitted_lengths(said.get("min_length"), said.get("max_length")) + low = next( + ((name, bound) for name, bound in said.items() if _CONSTRAINTS[name].side == "below"), None + ) + high = next( + ((name, bound) for name, bound in said.items() if _CONSTRAINTS[name].side == "above"), None + ) if low is not None and high is not None: opening = "(" if low[0] in _EXCLUSIVE else "[" closing = ")" if high[0] in _EXCLUSIVE else "]" @@ -969,6 +1020,22 @@ def _admitted(said: Constraints) -> str: return f"{'<' if name in _EXCLUSIVE else '<='} {shown(bound)}" +def _admitted_lengths(low: float | None, high: float | None) -> str: + """The lengths that length bounds allow, as written after the type in a message: "with at least 1 element", "with 1 to 3 elements".""" + if low is not None and high is not None: + if low == high: + return f"with {_elements(low)}" + return f"with {shown(low)} to {_elements(high)}" + if low is not None: + return f"with at least {_elements(low)}" + return f"with at most {_elements(cast('float', high))}" + + +def _elements(count: float) -> str: + """A number of elements, as a message writes it: "1 element", "3 elements".""" + return f"{shown(count)} element{'' if count == 1 else 's'}" + + # --- the compiler -------------------------------------------------------- _Building: TypeAlias = dict[object, Parser] @@ -1314,7 +1381,7 @@ def _typeddicts_in(annotation: object, seen: frozenset[object]) -> tuple[type, . def _metadata_problem(inner: object, metadata: tuple[object, ...]) -> TypeError | None: - """The error the checker raises for `metadata` on a member of type `inner`: a constraint it does not read, or a bound on a type that is not a number; None when it reads the metadata, as it does a note.""" + """The error the checker raises for `metadata` on a member of type `inner`: a constraint it does not read, or a constraint on a type it does not limit; None when it reads the metadata, as it does a note.""" try: _constrainable(inner, constraints_of(metadata)) except TypeError as error: @@ -1382,22 +1449,6 @@ def check( DIALECT: Final = "https://json-schema.org/draft/2020-12/schema" """The dialect of every schema: JSON Schema draft 2020-12, the same dialect pydantic and zod use.""" -_KEYWORDS: Final[Mapping[str, str]] = { - "gt": "exclusiveMinimum", - "ge": "minimum", - "lt": "exclusiveMaximum", - "le": "maximum", -} -"""JSON Schema's keyword for each bound.""" - -_STRICTER: Final[Mapping[str, Callable[[float, float], float]]] = { - "exclusiveMinimum": max, - "minimum": max, - "exclusiveMaximum": min, - "maximum": min, -} -"""For each keyword, the function that picks the stricter of two bounds.""" - def no_schema_leaf(annotation: object, schemas: Schemas) -> JSONSchema | None: """The schema leaf for a caller with no shapes of its own.""" @@ -1424,15 +1475,16 @@ def __init__(self, leaf: SchemaLeaf = no_schema_leaf) -> None: self._uses: dict[str, int] = {} def of(self, annotation: object) -> JSONSchema: - """The schema of `annotation` as the checker reads it: its type, with the bounds and notes from `Annotated`. - - Each bound becomes the matching JSON Schema keyword (`Ge(0)` becomes - `minimum`), and a `Doc` becomes the `description`. The schema keeps - both the type's bounds and the bounds of its `NewType`, because the - checker applies both. When both set the same keyword, the stricter - bound is kept. An annotation the checker supports is written. Any - other raises `TypeError`, which cannot happen if the caller already - checked the annotation with `parser`. + """The schema of `annotation` as the checker reads it: its type, with the constraints and notes from `Annotated`. + + Each constraint becomes the matching JSON Schema keyword (`Ge(0)` + becomes `minimum`, `MinLen(1)` becomes `minItems`), and a `Doc` + becomes the `description`. The schema keeps both the type's bounds + and the bounds of its `NewType`, or of the fixed length of its + tuple, because the checker applies both. When both set the same + keyword, the stricter bound is kept. An annotation the checker + supports is written. Any other raises `TypeError`, which cannot + happen if the caller already checked the annotation with `parser`. """ inner, metadata = strip_annotation(annotation) schema = self._type(inner) @@ -1446,9 +1498,11 @@ def of(self, annotation: object) -> JSONSchema: if len(notes) != 0: schema = {**schema, "description": "\n\n".join(notes)} for name, bound in constraints_of(metadata).items(): - keyword = _KEYWORDS[name] - held = cast("int | float | None", schema.get(keyword)) - schema = {**schema, keyword: bound if held is None else _STRICTER[keyword](held, bound)} + constraint = _CONSTRAINTS[name] + held = cast("int | float | None", schema.get(constraint.keyword)) + stricter = max if constraint.side == "below" else min + kept = bound if held is None else stricter(held, bound) + schema = {**schema, constraint.keyword: kept} return schema def object_of(self, typeddict: type) -> JSONSchema: @@ -1591,8 +1645,8 @@ def json_schema(shape: type) -> JSONSchema: - A TypedDict becomes an object with its keys, its required keys, and what any other key may hold (nothing, for a closed TypedDict). - - A bound becomes the matching JSON Schema keyword: `Ge(0)` becomes - `minimum`. + - A constraint becomes the matching JSON Schema keyword: `Ge(0)` + becomes `minimum`, and `MinLen(1)` becomes `minItems`. - A `Doc` becomes the `description`. Only a `Doc` produces a description; docstrings are for Python readers and are not used. This matches zod, which writes only what `.describe()` sets. diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 4dc31e9162..0fadbe02ef 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -163,14 +163,16 @@ def acme_lz4_rules( type may carry bounds, written with annotated-types and read the way pydantic reads them: `Gt`, `Ge`, `Lt`, `Le` and `Interval`, at most one from each side, at any depth. For example, -`tuple[Annotated[int, Ge(1)], ...]` bounds each element. A value out of +`tuple[Annotated[int, Ge(1)], ...]` bounds each element. An array's type +may bound its length with `MinLen`, `MaxLen` and `Len`, so +`Annotated[tuple[int, ...], MinLen(1)]` is never empty. A value out of bounds is an `invalid_value` problem. Its message says what the type admits ("expected an integer >= 1, got 0"), and its `ctx` holds the bounds. The rules run only on a configuration within its bounds, so a rule can rely on them, as pydantic's after-validators and zod's refinements do. Until a value out of bounds is fixed, it is the only problem reported for the configuration. `Annotated` may also carry a -note, as a string or a `Doc`. Any other metadata (a `MinLen`, a +note, as a string or a `Doc`. Any other metadata (a `MultipleOf`, a `Predicate`, pydantic's `Field`) is rejected when the definition is built, because a type whose constraint the checker does not enforce would give false information. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 22e4ecc466..627545859c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -27,7 +27,7 @@ within, ) from zarr_metadata._sentinel import UNSET -from zarr_metadata._typed_json import typeddict_keys +from zarr_metadata._typed_json import strip_annotation, typeddict_keys from zarr_metadata.definition._context import disagreements, scoped from zarr_metadata.definition._definition import ( AcceptedDeclaration, @@ -132,10 +132,10 @@ def members_past_the_levels( _EXTENSION_LISTS_V3: Final[tuple[tuple[str, type[Definition[Any]]], ...]] = tuple( (key, kind) for key, (annotation, _) in _MEMBERS_V3.items() - if get_origin(annotation) is tuple - and (kind := declaration_kind(get_args(annotation)[0])) is not None + if get_origin(listed := strip_annotation(annotation)[0]) is tuple + and (kind := declaration_kind(get_args(listed)[0])) is not None ) -"""The lists of extension points of a v3 array document, and the kind each entry is read as: each member that the TypedDict annotates as a tuple of a declaration alias.""" +"""The lists of extension points of a v3 array document, and the kind each entry is read as: each member that the TypedDict annotates as a tuple of a declaration alias, whatever constraints the tuple carries.""" def no_extra_fields() -> Mapping[str, ResolvedDeclaration[Any]]: diff --git a/packages/zarr-metadata/src/zarr_metadata/typed_json.py b/packages/zarr-metadata/src/zarr_metadata/typed_json.py index 982a3d4868..75480f378d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/typed_json.py @@ -41,8 +41,11 @@ - A number type may carry bounds, written with annotated-types as pydantic reads them: `Gt`, `Ge`, `Lt`, `Le` and `Interval`, at any depth. For example, `tuple[Annotated[int, Ge(1)], ...]` bounds each - element. A value outside the bounds is an `invalid_value` problem. Its - message states what the type allows, and its `ctx` holds the bounds. + element. An array type may bound its length the same way, with + `MinLen`, `MaxLen` and `Len`: `Annotated[tuple[int, ...], MinLen(1)]` + is a non-empty array. A value outside the bounds is an `invalid_value` + problem. Its message states what the type allows, and its `ctx` holds + the bounds, as pydantic names them (`ge`, `min_length`). `Annotated` may also carry a note (a string or a `Doc`). Any other metadata raises `TypeError`, because `check` would not enforce a constraint it does not support. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py b/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py index 3530618288..f4dd1e6f03 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py @@ -22,7 +22,6 @@ ExtraFieldDefinition, GroupExtraFieldDefinition, ) -from zarr_metadata.v3._types.chunk_grid import RectilinearChunkGridConfiguration from zarr_metadata.v3.array import ZarrV3ArrayMetadataJSON from zarr_metadata.v3.group import ZarrV3GroupMetadataJSON @@ -83,10 +82,6 @@ def declaration_schemas(context: ZarrV3Context) -> SchemaLeaf: """The schema leaf that `declaration_json_schema_v3` uses. It writes each declaration alias as a declaration of its kind, as `context` reads it.""" def leaf(annotation: object, schemas: Schemas) -> JSONSchema | None: - if annotation is RectilinearChunkGridConfiguration: - return schemas.defined( - annotation, annotation.__name__, lambda: _rectilinear_configuration(schemas) - ) kind = declaration_kind(annotation) if kind is None or not is_alias(annotation): return None @@ -278,45 +273,9 @@ def _member_object(schema: Mapping[str, JSONValue], key: str) -> Mapping[str, JS return found -def _rectilinear_configuration(schemas: Schemas) -> JSONSchema: - """The rectilinear chunk grid's configuration: its TypedDict, with at least one edge length in each dimension's list. - - The TypedDict cannot declare a minimum length, so the schema adds it, - as the grid's rules check it: the spec requires "a non-empty array", - even for a dimension of length 0. - """ - schema = schemas.object_of(RectilinearChunkGridConfiguration) - properties = _member_object(schema, "properties") - chunk_shapes = _member_object(properties, "chunk_shapes") - dimension = _member_object(chunk_shapes, "items") - branches = dimension.get("anyOf") - if not isinstance(branches, list): - msg = f"the schema of a rectilinear dimension is a union, got {dimension!r}" - raise TypeError(msg) - nonempty: list[JSONValue] = [ - {**branch, "minItems": 1} - if isinstance(branch, Mapping) and branch.get("type") == "array" - else branch - for branch in branches - ] - items: JSONSchema = {**dimension, "anyOf": nonempty} - return { - **schema, - "properties": {**properties, "chunk_shapes": {**chunk_shapes, "items": items}}, - } - - def _array(context: ZarrV3Context, schemas: Schemas) -> JSONSchema: - """An array document: its TypedDict, at least one codec, and a constraint on its fill value for each data type in scope, which applies when the document names that data type. - - The TypedDict cannot declare a minimum length, so the schema adds it: - a pipeline needs its array -> bytes codec, and the reader rejects an - empty `codecs`. - """ + """An array document: its TypedDict, and a constraint on its fill value for each data type in scope, which applies when the document names that data type.""" schema = schemas.object_of(ZarrV3ArrayMetadataJSON) - properties = _member_object(schema, "properties") - codecs = _member_object(properties, "codecs") - schema = {**schema, "properties": {**properties, "codecs": {**codecs, "minItems": 1}}} schema = _with_extra_fields(schema, context.array_extra_fields, schemas) held: list[JSONValue] = [] for definition in context.data_types.values(): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py index 4e74edccf8..16135ea84d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py @@ -10,7 +10,7 @@ from typing import Annotated, Final, Literal, NotRequired -from annotated_types import Ge +from annotated_types import Ge, MinLen from typing_extensions import TypedDict # regular: https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#regular-grids @@ -77,14 +77,19 @@ class RegularChunkGridObject(TypedDict, closed=True): _Positive = Annotated[int, Ge(1)] -RectilinearDimSpec = _Positive | tuple[_Positive | tuple[_Positive, _Positive], ...] +RectilinearDimSpec = ( + _Positive | Annotated[tuple[_Positive | tuple[_Positive, _Positive], ...], MinLen(1)] +) """JSON shape for one dimension's rectilinear spec. Either a bare integer (uniform shorthand for a regular dimension within a rectilinear grid), or a non-empty tuple of integers and/or `[value, count]` RLE pairs. Every extent, and every run's length and -count, is at least 1. The type cannot declare that the tuple is -non-empty; the rectilinear grid's rules check it. +count, is at least 1. The tuple is non-empty even for a dimension of +length 0, which writes a bare integer to have no chunks: "as a non-empty +array", and "For an axis of length 0, the array form still requires at +least one edge length" +(https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md?plain=1#L89-L95). """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index 2f256e219b..1d13e60f64 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -3,7 +3,7 @@ from collections.abc import Mapping from typing import Annotated, Final, Literal, NotRequired -from annotated_types import Ge +from annotated_types import Ge, MinLen from typing_extensions import TypedDict from zarr_metadata._common import JSONValue @@ -47,7 +47,7 @@ class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): Which JSON values it may take depends on `data_type`. """ - codecs: tuple[ZarrV3CodecDeclaration, ...] + codecs: Annotated[tuple[ZarrV3CodecDeclaration, ...], MinLen(1)] """The codecs that encode each chunk, in encoding order. Zero or more array -> array codecs, then exactly one array -> bytes codec, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index e512001b48..2d7b2808a6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -72,28 +72,6 @@ def _canonical( ) -def _rules( - configuration: RectilinearChunkGridConfiguration, nested: Nested -) -> Iterator[ValidationProblem]: - """Check that each dimension's list of chunk edge lengths has at least one. - - "as a non-empty array", and "For an axis of length 0, the array form - still requires at least one edge length" - (https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md?plain=1#L89-L95). - A dimension that has no chunks, because its length is 0, writes a bare - integer instead. The type cannot declare a minimum length, so this rule - checks it. - """ - for axis, spec in enumerate(configuration["chunk_shapes"]): - if isinstance(spec, tuple) and len(spec) == 0: - yield ValidationProblem( - ("chunk_shapes", axis), - "expected at least one chunk edge length, got an empty list", - "invalid_value", - ctx={"min_length": 1}, - ) - - def _shape_rules( configuration: RectilinearChunkGridConfiguration, nested: Nested, shape: tuple[int, ...] ) -> Iterator[ValidationProblem]: @@ -149,7 +127,6 @@ def _chunk_lengths( RECTILINEAR_CHUNK_GRID: Final = ChunkGridDefinition( name=RECTILINEAR_CHUNK_GRID_NAME, configuration=RectilinearChunkGridConfiguration, - rules=_rules, canonical=_canonical, shape_rules=_shape_rules, chunk_lengths=_chunk_lengths, diff --git a/packages/zarr-metadata/tests/test_json_schema.py b/packages/zarr-metadata/tests/test_json_schema.py index 48d4b6f2fb..ad70fb5902 100644 --- a/packages/zarr-metadata/tests/test_json_schema.py +++ b/packages/zarr-metadata/tests/test_json_schema.py @@ -15,7 +15,7 @@ from typing import Annotated, Any, Literal, NewType, NotRequired, cast import pytest -from annotated_types import Ge, Gt, Interval, Le, Lt, MinLen +from annotated_types import Ge, Gt, Interval, Le, Len, Lt, MaxLen, MinLen, MultipleOf from hypothesis import HealthCheck, event, given, settings from hypothesis import strategies as st from jsonschema import Draft202012Validator @@ -170,6 +170,31 @@ def _held(member: dict[str, Any], defs: dict[str, Any] | None = None) -> dict[st _holding(Annotated[int, Ge(0), Doc("a count")]), _held({"type": "integer", "description": "a count", "minimum": 0}), ), + ( + "non-empty-array", + _holding(Annotated[tuple[int, ...], MinLen(1)]), + _held({"type": "array", "items": {"type": "integer"}, "minItems": 1}), + ), + ( + "length-range", + _holding(Annotated[tuple[int, ...], Len(1, 3)]), + _held({"type": "array", "items": {"type": "integer"}, "minItems": 1, "maxItems": 3}), + ), + # A length bound and the length of a fixed-length array set the same + # keyword, and the stricter is kept. + ( + "a-length-on-a-pair", + _holding(Annotated[tuple[int, str], MinLen(1), MaxLen(3)]), + _held( + { + "type": "array", + "prefixItems": [{"type": "integer"}, {"type": "string"}], + "items": False, + "minItems": 2, + "maxItems": 3, + } + ), + ), # Metadata that is not a `Doc` adds nothing to the schema. ("note", _holding(Annotated[str, "a note"]), _held({"type": "string"})), # A value must meet its type's bounds and its `NewType`'s bounds. When @@ -334,7 +359,7 @@ def test_error_a_schema_that_fails_to_write_leaves_nothing_behind() -> None: def test_json_schema_refuses_metadata_check_does_not_hold_a_value_to() -> None: with pytest.raises(TypeError, match="is not a constraint the checker reads"): - json_schema(_holding(Annotated[tuple[int, ...], MinLen(1)])) + json_schema(_holding(Annotated[int, MultipleOf(2)])) _PROPERTY = settings(max_examples=300, deadline=None, suppress_health_check=[HealthCheck.too_slow]) diff --git a/packages/zarr-metadata/tests/test_typed_json.py b/packages/zarr-metadata/tests/test_typed_json.py index f399e4cf30..3762b817d7 100644 --- a/packages/zarr-metadata/tests/test_typed_json.py +++ b/packages/zarr-metadata/tests/test_typed_json.py @@ -14,6 +14,7 @@ import json import math import pkgutil +import re import sys import textwrap import types @@ -41,6 +42,7 @@ Le, Len, Lt, + MaxLen, MinLen, MultipleOf, Predicate, @@ -1042,6 +1044,72 @@ class Noted(TypedDict, extra_items=Annotated[object, "any JSON"]): (Digit | str, 10, [((), "expected an integer in [0, 9], got 10", {"ge": 0, "le": 9})]), (Digit, "nine", [((), 'expected an integer, got "nine"', {})]), (Annotated[int, "a note", Doc("another")], -1, []), + (Annotated[tuple[int, ...], MinLen(1)], [0], []), + ( + Annotated[tuple[int, ...], MinLen(1)], + [], + [ + ( + (), + "expected an array of integers with at least 1 element, got []", + {"min_length": 1}, + ) + ], + ), + ( + Annotated[tuple[int, ...], MaxLen(2)], + [0, 0, 0], + [ + ( + (), + "expected an array of integers with at most 2 elements, got [0, 0, 0]", + {"max_length": 2}, + ) + ], + ), + ( + Annotated[tuple[int, ...], Len(1, 3)], + [], + [ + ( + (), + "expected an array of integers with 1 to 3 elements, got []", + {"min_length": 1, "max_length": 3}, + ) + ], + ), + ( + Annotated[tuple[int, ...], Len(2, 2)], + [0], + [ + ( + (), + "expected an array of integers with 2 elements, got [0]", + {"min_length": 2, "max_length": 2}, + ) + ], + ), + ( + Annotated[tuple[int, int], MinLen(1)], + [0, 0], + [], + ), + ( + Annotated[tuple[Digit, ...], MinLen(1)], + [10], + [((0,), "expected an integer in [0, 9], got 10", {"ge": 0, "le": 9})], + ), + ( + Digit | Annotated[tuple[Digit, ...], MinLen(1)], + [], + [ + ( + (), + "expected an array of integers with at least 1 element, got []", + {"min_length": 1}, + ) + ], + ), ], ids=[ "within", @@ -1059,15 +1127,24 @@ class Noted(TypedDict, extra_items=Annotated[object, "any JSON"]): "the-branch-it-bounds", "a-value-of-another-type-is-that-problem-alone", "notes", + "a-length-within", + "a-length-below", + "a-length-above", + "a-length-range", + "an-exact-length", + "a-length-on-a-fixed-length-array", + "a-length-checked-only-once-the-elements-are-read", + "the-branch-it-bounds-the-length-of", ], ) def test_a_value_is_held_to_the_bounds_its_type_carries( annotation: object, value: object, found: list[tuple[Loc, str, dict[str, JSONValue]]] ) -> None: - # annotated-types bounds, read as pydantic reads them. A value outside - # any bound gives one problem that says what the type allows, with the - # bounds in the problem's `ctx`. Bounds are checked only for a value of - # the type. A bound is compared as the number it equals, whichever of + # annotated-types bounds, read as pydantic reads them: on a number's + # value, or on an array's length. A value outside any bound gives one + # problem that says what the type allows, with the bounds in the + # problem's `ctx`, named as pydantic names them. Bounds are checked only + # for a value of the type. A bound is compared as the number it equals, whichever of # the equal values Python's `Annotated` cache returns. _, problems = _read(annotation, value) assert [(problem.loc, problem.message, dict(problem.ctx)) for problem in problems] == found @@ -1101,6 +1178,28 @@ def test_error_a_bound_on_what_is_not_a_number() -> None: parser(Annotated[str, Ge(0)], no_leaf) +@pytest.mark.parametrize( + ("annotation", "named"), + [ + (Annotated[str, MinLen(1)], "min_length: a bound is on an array, and a string is not one"), + ( + Annotated[Mapping[str, int], MaxLen(1)], + "max_length: a bound is on an array, and an object is not one", + ), + ( + Annotated[int, Len(1, 2)], + "min_length, max_length: a bound is on an array, and an integer is not one", + ), + ], + ids=["min-len-on-a-string", "max-len-on-an-object", "len-on-a-number"], +) +def test_error_a_length_bound_on_what_is_not_an_array(annotation: object, named: str) -> None: + # pydantic also bounds the length of a string or an object; the checker + # bounds only an array's, so the type cannot promise more than it checks. + with pytest.raises(TypeError, match=re.escape(named)): + parser(annotation, no_leaf) + + def test_error_a_bound_on_the_values_an_open_typeddict_takes() -> None: class Loose(TypedDict, extra_items=Annotated[object, Ge(0)]): level: int @@ -1117,8 +1216,6 @@ class Later(BaseMetadata): @pytest.mark.parametrize( "constraint", [ - MinLen(1), - Len(1, 3), MultipleOf(2), Predicate(str.isdigit), Timezone(None), @@ -1127,8 +1224,6 @@ class Later(BaseMetadata): pydantic.StringConstraints(pattern="^[a-z]+$"), ], ids=[ - "min-len", - "len", "multiple-of", "predicate", "timezone", @@ -1314,7 +1409,7 @@ class _SelfReferentialReadable(TypedDict, closed=True): _ConstrainedMember, Unread( "_ConstrainedMember.a: MultipleOf(multiple_of=2) is not a constraint the checker " - "reads, which are Gt, Ge, Lt and Le", + "reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", constraint=True, ), ), @@ -1322,7 +1417,7 @@ class _SelfReferentialReadable(TypedDict, closed=True): _HoldsAConstrainedMember, Unread( "_HoldsAConstrainedMember.inner: _ConstrainedMember.a: MultipleOf(multiple_of=2) is " - "not a constraint the checker reads, which are Gt, Ge, Lt and Le", + "not a constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", constraint=True, ), ), @@ -1355,7 +1450,7 @@ class _SelfReferentialReadable(TypedDict, closed=True): _ConstraintOnAReadableMember, Unread( "_ConstraintOnAReadableMember.inner: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt and Le", + "constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", constraint=True, ), ), @@ -1364,7 +1459,7 @@ class _SelfReferentialReadable(TypedDict, closed=True): Unread( "_ConstraintOnAConstrainedMember.inner: _ConstrainedMember.a: " "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " - "Ge, Lt and Le", + "Ge, Lt, Le, MinLen and MaxLen", constraint=True, ), ), @@ -1372,7 +1467,7 @@ class _SelfReferentialReadable(TypedDict, closed=True): _SelfReferentialWithAConstraint, Unread( "_SelfReferentialWithAConstraint.a: MultipleOf(multiple_of=2) is not a constraint " - "the checker reads, which are Gt, Ge, Lt and Le", + "the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", constraint=True, ), ), @@ -1388,7 +1483,7 @@ class _SelfReferentialReadable(TypedDict, closed=True): _SelfReferentialConstraintOnItself, Unread( "_SelfReferentialConstraintOnItself.parent: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt and Le", + "constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", constraint=True, ), ), @@ -1404,7 +1499,7 @@ class _SelfReferentialReadable(TypedDict, closed=True): _OuterWithAConstraintAfterACycle, Unread( "_OuterWithAConstraintAfterACycle.a: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt and Le", + "constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", constraint=True, ), ), @@ -1420,7 +1515,7 @@ class _SelfReferentialReadable(TypedDict, closed=True): _OuterWithAConstraintOnTheCycle, Unread( "_OuterWithAConstraintOnTheCycle.inner: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt and Le", + "constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", constraint=True, ), ), @@ -1429,7 +1524,7 @@ class _SelfReferentialReadable(TypedDict, closed=True): Unread( "_SelfReferentialTwiceWithAConstraintOnTheSecond.constrained: " "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " - "Ge, Lt and Le", + "Ge, Lt, Le, MinLen and MaxLen", constraint=True, ), ), @@ -1437,7 +1532,7 @@ class _SelfReferentialReadable(TypedDict, closed=True): _DocumentedThenConstrainedBackReferences, Unread( "_DocumentedThenConstrainedBackReferences.constrained: MultipleOf(multiple_of=2) " - "is not a constraint the checker reads, which are Gt, Ge, Lt and Le", + "is not a constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", constraint=True, ), ), @@ -1445,7 +1540,7 @@ class _SelfReferentialReadable(TypedDict, closed=True): _OuterWithAConstraintAfterAnOptionalCycle, Unread( "_OuterWithAConstraintAfterAnOptionalCycle.a: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt and Le", + "constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", constraint=True, ), ), @@ -1463,7 +1558,7 @@ class _SelfReferentialReadable(TypedDict, closed=True): Unread( "_HoldsATupleOfAConstrainedMember.inner: _ConstrainedMember.a: " "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " - "Ge, Lt and Le", + "Ge, Lt, Le, MinLen and MaxLen", constraint=True, ), ), @@ -1562,8 +1657,9 @@ def test_unread_in_names_the_part_without_a_parser_and_whether_it_is_a_constrain Annotated[Digit, Ge(1)], Annotated[int, Ge(0), Gt(0)], Annotated[int, Interval(le=9), Lt(5)], + Annotated[tuple[int, ...], Len(1, 3), MinLen(2)], ], - ids=["narrowing-an-alias", "gt-and-ge", "interval-and-lt"], + ids=["narrowing-an-alias", "gt-and-ge", "interval-and-lt", "len-and-min-len"], ) def test_error_a_second_bound_from_one_side(annotation: object) -> None: # Pydantic uses the last bound given; write only one. @@ -1579,6 +1675,12 @@ def test_error_a_bound_that_is_not_a_finite_number(bound: object) -> None: parser(Annotated[int, Ge(bound)], no_leaf) # pyright: ignore[reportArgumentType] +@pytest.mark.parametrize("length", [-1, 1.5], ids=["negative", "fractional"]) +def test_error_a_length_bound_that_is_not_a_non_negative_integer(length: object) -> None: + with pytest.raises(TypeError, match="a length bound is a non-negative integer"): + parser(Annotated[tuple[int, ...], MinLen(length)], no_leaf) # pyright: ignore[reportArgumentType] + + # --- check: the public door ------------------------------------------------ _V3_DOCUMENT = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) @@ -1671,6 +1773,16 @@ def test_error_check_refuses_a_value_of_another_type() -> None: assert [(problem.loc, problem.kind) for problem in problems] == [(("level",), "invalid_type")] +def test_error_check_refuses_an_array_document_without_codecs() -> None: + # The document's type says `codecs` is never empty, so `check` refuses + # an empty list at `codecs`, as the reader and the JSON Schema do. + typed, problems = check({**_V3_DOCUMENT, "codecs": []}, ZarrV3ArrayMetadataJSON) + assert typed is None + assert [(problem.loc, problem.kind, dict(problem.ctx)) for problem in problems] == [ + (("codecs",), "invalid_value", {"min_length": 1}) + ] + + _NOT_DOCUMENTS: Final = frozenset({"ZarrV3ContextSlots", "ZarrV2ContextSlots"}) """TypedDicts that describe no document: the keyword arguments of a scope class, which hold definitions, not JSON.""" From 8d06cd369eb490aff35afa3f93280b53ad350f4a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 18:19:42 +0100 Subject: [PATCH 05/13] feat(zarr-metadata): declare sharding's codec pipelines non-empty The `sharding_indexed` codec's `codecs` and `index_codecs` each hold exactly one array -> bytes codec, so neither is ever empty. Both now declare `MinLen(1)` on their types, as the array document's `codecs` does. An empty one is reported by the type check at the same `loc` as before, with `ctx={"min_length": 1}`. Before, the pipeline's order check reported it as "expected an array -> bytes codec, got none". Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/changes/4635.feature.md | 2 +- packages/zarr-metadata/src/zarr_metadata/v3/_types/codec.py | 6 +++--- packages/zarr-metadata/tests/v3/test_sharding.py | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/zarr-metadata/changes/4635.feature.md b/packages/zarr-metadata/changes/4635.feature.md index df9a1fc974..e770632434 100644 --- a/packages/zarr-metadata/changes/4635.feature.md +++ b/packages/zarr-metadata/changes/4635.feature.md @@ -1 +1 @@ -A TypedDict read by `zarr_metadata.typed_json.check`, or used as a definition's configuration, may now bound the length of an array with annotated-types' `MinLen`, `MaxLen` and `Len`, as pydantic reads them: `Annotated[tuple[int, ...], MinLen(1)]` is a non-empty array. An array outside its bounds is an `invalid_value` problem whose `ctx` holds `min_length` or `max_length`, and the JSON Schema writes `minItems` and `maxItems`. A length bound on anything but an array raises `TypeError`. An array document's `codecs` and each list in a `rectilinear` chunk grid's `chunk_shapes` now say on their types that they are never empty, so `check` also refuses `"codecs": []`. +A TypedDict read by `zarr_metadata.typed_json.check`, or used as a definition's configuration, may now bound the length of an array with annotated-types' `MinLen`, `MaxLen` and `Len`, as pydantic reads them: `Annotated[tuple[int, ...], MinLen(1)]` is a non-empty array. An array outside its bounds is an `invalid_value` problem whose `ctx` holds `min_length` or `max_length`, and the JSON Schema writes `minItems` and `maxItems`. A length bound on anything but an array raises `TypeError`. An array document's `codecs`, the `sharding_indexed` codec's `codecs` and `index_codecs`, and each list in a `rectilinear` chunk grid's `chunk_shapes` now say on their types that they are never empty, so `check` also refuses `"codecs": []`. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/codec.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/codec.py index 00792264ea..806ce06c29 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_types/codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/codec.py @@ -12,7 +12,7 @@ from typing import Annotated, Final, Literal, NotRequired -from annotated_types import Ge, Interval +from annotated_types import Ge, Interval, MinLen from typing_extensions import TypedDict from zarr_metadata._common import EmptyConfiguration, JSONValue @@ -510,12 +510,12 @@ class ShardingIndexedCodecConfiguration(TypedDict, closed=True): One length, at least 1, per axis of the shard the codec is handed, each dividing the shard's length along that axis. """ - codecs: tuple[ZarrV3CodecDeclaration, ...] + codecs: Annotated[tuple[ZarrV3CodecDeclaration, ...], MinLen(1)] """The codec pipeline that encodes each inner chunk. Like an array's `codecs`, it holds exactly one array -> bytes codec. """ - index_codecs: tuple[ZarrV3StaticCodecDeclaration, ...] + index_codecs: Annotated[tuple[ZarrV3StaticCodecDeclaration, ...], MinLen(1)] """The codec pipeline that encodes the shard index. It holds exactly one array -> bytes codec, and only codecs whose output diff --git a/packages/zarr-metadata/tests/v3/test_sharding.py b/packages/zarr-metadata/tests/v3/test_sharding.py index dc3e81e69b..8585adf089 100644 --- a/packages/zarr-metadata/tests/v3/test_sharding.py +++ b/packages/zarr-metadata/tests/v3/test_sharding.py @@ -207,6 +207,7 @@ def test_error_an_inner_chunk_length_that_does_not_divide_the_shard( [ # Each pipeline is read as the array's pipeline is, at its own location. (_shard([4, 4], codecs=[]), [((0, "configuration", "codecs"), "invalid_value")]), + (_shard([4, 4], index=[]), [((0, "configuration", "index_codecs"), "invalid_value")]), ( _shard([4, 4], codecs=["crc32c", LITTLE]), [((0, "configuration", "codecs", 1), "invalid_value")], From 8158e267b1f3c2e4c976473825582a67b6b52daa Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 18:36:56 +0100 Subject: [PATCH 06/13] refactor(zarr-metadata): move constant data type rules onto their types `null_terminated_bytes` and zarr-python's `raw_bytes` checked `length_bytes >= 1` in a rule, and v3 `struct` checked that it has a field. The types now declare `Ge(1)` and `MinLen(1)`, the two rules that did only this are gone, and the struct rule keeps its name and field-type checks. The problems keep their `loc` and kind. The JSON Schema of each type now says so too: it accepted `length_bytes: 0` and `fields: []`, which the reader refuses. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/changes/4635.feature.md | 2 +- .../src/zarr_metadata/v3/_types/data_type.py | 18 ++++++++++++------ .../v3/data_type/null_terminated_bytes.py | 12 ------------ .../src/zarr_metadata/v3/data_type/struct.py | 4 +--- .../v3/data_type/zarr_python_raw_bytes.py | 12 ------------ 5 files changed, 14 insertions(+), 34 deletions(-) diff --git a/packages/zarr-metadata/changes/4635.feature.md b/packages/zarr-metadata/changes/4635.feature.md index e770632434..a99c327d13 100644 --- a/packages/zarr-metadata/changes/4635.feature.md +++ b/packages/zarr-metadata/changes/4635.feature.md @@ -1 +1 @@ -A TypedDict read by `zarr_metadata.typed_json.check`, or used as a definition's configuration, may now bound the length of an array with annotated-types' `MinLen`, `MaxLen` and `Len`, as pydantic reads them: `Annotated[tuple[int, ...], MinLen(1)]` is a non-empty array. An array outside its bounds is an `invalid_value` problem whose `ctx` holds `min_length` or `max_length`, and the JSON Schema writes `minItems` and `maxItems`. A length bound on anything but an array raises `TypeError`. An array document's `codecs`, the `sharding_indexed` codec's `codecs` and `index_codecs`, and each list in a `rectilinear` chunk grid's `chunk_shapes` now say on their types that they are never empty, so `check` also refuses `"codecs": []`. +A TypedDict read by `zarr_metadata.typed_json.check`, or used as a definition's configuration, may now bound the length of an array with annotated-types' `MinLen`, `MaxLen` and `Len`, as pydantic reads them: `Annotated[tuple[int, ...], MinLen(1)]` is a non-empty array. An array outside its bounds is an `invalid_value` problem whose `ctx` holds `min_length` or `max_length`, and the JSON Schema writes `minItems` and `maxItems`. A length bound on anything but an array raises `TypeError`. An array document's `codecs`, the `sharding_indexed` codec's `codecs` and `index_codecs`, and each list in a `rectilinear` chunk grid's `chunk_shapes` now say on their types that they are never empty, so `check` also refuses `"codecs": []`. The `null_terminated_bytes` and zarr-python `raw_bytes` data types declare `length_bytes` at least 1, and `struct` declares at least one field, on their types, so their JSON Schemas now say so too. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py index 3c8b19d703..b800fd452c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py @@ -13,7 +13,7 @@ from collections.abc import Mapping from typing import Annotated, Final, Literal, NewType, NotRequired, get_args -from annotated_types import Interval +from annotated_types import Ge, Interval, MinLen from typing_extensions import ReadOnly, TypedDict from zarr_metadata._common import JSONValue @@ -447,8 +447,11 @@ class FixedLengthUtf32(TypedDict, closed=True): class NullTerminatedBytesConfiguration(TypedDict, closed=True): """Configuration for the `null_terminated_bytes` data type.""" - length_bytes: ReadOnly[int] - """The size in bytes of every value, at least 1.""" + length_bytes: ReadOnly[Annotated[int, Ge(1)]] + """The size in bytes of every value, at least 1. + + zarr-python refuses a length below 1, since such a type holds no data. + """ class NullTerminatedBytes(TypedDict, closed=True): @@ -482,8 +485,11 @@ class NullTerminatedBytes(TypedDict, closed=True): class ZarrPythonRawBytesConfiguration(TypedDict, closed=True): """Configuration for zarr-python's `raw_bytes` data type.""" - length_bytes: ReadOnly[int] - """The size in bytes of every value, at least 1.""" + length_bytes: ReadOnly[Annotated[int, Ge(1)]] + """The size in bytes of every value, at least 1. + + zarr-python refuses a length below 1, since such a type holds no data. + """ class ZarrPythonRawBytes(TypedDict, closed=True): @@ -626,7 +632,7 @@ class StructField(TypedDict, closed=True): class StructConfiguration(TypedDict, closed=True): """Configuration for the `struct` data type.""" - fields: ReadOnly[tuple[StructField, ...]] + fields: ReadOnly[Annotated[tuple[StructField, ...], MinLen(1)]] """The record's fields, in the order they are encoded. At least one. The order is significant: it is the order in which the diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py index 333f362cb5..7217f5eb1d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py @@ -25,17 +25,6 @@ from zarr_metadata.v3.data_type._byte import base64_decoded, base64_problems, base64_spelling -def _rules( - configuration: NullTerminatedBytesConfiguration, nested: Nested -) -> Iterator[ValidationProblem]: - """Check that `length_bytes` is positive. zarr-python refuses a length below 1, since such a type holds no data.""" - length_bytes = configuration["length_bytes"] - if length_bytes < 1: - yield ValidationProblem( - ("length_bytes",), f"expected a positive size, got {length_bytes}", "invalid_value" - ) - - def _fill_value_rules( configuration: NullTerminatedBytesConfiguration, nested: Nested, @@ -75,7 +64,6 @@ def _fill_value_canonical( NULL_TERMINATED_BYTES_DATA_TYPE: Final = DataTypeDefinition( name=NULL_TERMINATED_BYTES_DATA_TYPE_NAME, configuration=NullTerminatedBytesConfiguration, - rules=_rules, fill_value=NullTerminatedBytesFillValue, fill_value_default=ConstantFillValue(""), fill_value_rules=_fill_value_rules, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index af9fbc0038..653d5f33d6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -32,7 +32,7 @@ def _rules(configuration: StructConfiguration, nested: Nested) -> Iterator[ValidationProblem]: - """Check that there is at least one field, that field names are non-empty and distinct, and that field types have a fixed size. + """Check that field names are non-empty and distinct, and that field types have a fixed size. Names must be distinct because a fill value addresses fields by name. "Variable-length data types @@ -43,8 +43,6 @@ def _rules(configuration: StructConfiguration, nested: Nested) -> Iterator[Valid whose storage is unknown is not checked. """ fields = configuration["fields"] - if len(fields) == 0: - yield ValidationProblem(("fields",), "expected at least one struct field", "invalid_value") seen: dict[str, int] = {} for index, member in enumerate(fields): name = member["name"] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py index e1f41efcc9..cc9aa3bccb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py @@ -29,17 +29,6 @@ from zarr_metadata.v3.data_type._byte import base64_decoded, base64_problems, base64_spelling -def _rules( - configuration: ZarrPythonRawBytesConfiguration, nested: Nested -) -> Iterator[ValidationProblem]: - """Check that `length_bytes` is positive. zarr-python refuses a length below 1, since such a type holds no data.""" - length_bytes = configuration["length_bytes"] - if length_bytes < 1: - yield ValidationProblem( - ("length_bytes",), f"expected a positive size, got {length_bytes}", "invalid_value" - ) - - def _fill_value_rules( configuration: ZarrPythonRawBytesConfiguration, nested: Nested, @@ -82,7 +71,6 @@ def _fill_value_default(configuration: ZarrPythonRawBytesConfiguration, nested: ZARR_PYTHON_RAW_BYTES_DATA_TYPE: Final = DataTypeDefinition( name=ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME, configuration=ZarrPythonRawBytesConfiguration, - rules=_rules, fill_value=ZarrPythonRawBytesFillValue, fill_value_default=_fill_value_default, fill_value_rules=_fill_value_rules, From 08b58273e55bc0280ca54a399a35f51b453408b8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 18:46:56 +0100 Subject: [PATCH 07/13] feat(zarr-metadata): read MultipleOf on integers in the TypedDict checker The checker, the JSON Schema writer and the pydantic schema now read annotated-types' `MultipleOf` on an integer, as pydantic does under `multiple_of`. JSON Schema's keyword is `multipleOf`. The step must be a positive integer, and a type takes one. Two steps on a NewType and its annotation merge to their least common multiple. A float is refused, because validators compare floats inexactly. Each constraint in the table now names the shapes it applies to, with the JSON Schema keyword for each, and a slot (below, above or step) in place of a side. `fixed_length_utf32`'s `length_bytes` and raw `r` `bits` now declare `Gt(0), MultipleOf(8 or 4)` on their types. The two rules that checked only this are gone. A test whose own `r*` definition accepts any size now uses its own configuration type. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../zarr-metadata/changes/4635.feature.md | 2 +- packages/zarr-metadata/docs/extensions.md | 7 +- .../src/zarr_metadata/_typed_json.py | 186 +++++++++++++----- .../src/zarr_metadata/definition/__init__.py | 9 +- .../src/zarr_metadata/typed_json.py | 8 +- .../src/zarr_metadata/v3/_types/data_type.py | 15 +- .../v3/data_type/fixed_length_utf32.py | 14 -- .../src/zarr_metadata/v3/data_type/raw.py | 12 -- .../zarr-metadata/tests/test_json_schema.py | 11 +- .../zarr-metadata/tests/test_typed_json.py | 128 ++++++++---- .../tests/v3/test_every_definition.py | 11 +- 11 files changed, 260 insertions(+), 143 deletions(-) diff --git a/packages/zarr-metadata/changes/4635.feature.md b/packages/zarr-metadata/changes/4635.feature.md index a99c327d13..65b16a4559 100644 --- a/packages/zarr-metadata/changes/4635.feature.md +++ b/packages/zarr-metadata/changes/4635.feature.md @@ -1 +1 @@ -A TypedDict read by `zarr_metadata.typed_json.check`, or used as a definition's configuration, may now bound the length of an array with annotated-types' `MinLen`, `MaxLen` and `Len`, as pydantic reads them: `Annotated[tuple[int, ...], MinLen(1)]` is a non-empty array. An array outside its bounds is an `invalid_value` problem whose `ctx` holds `min_length` or `max_length`, and the JSON Schema writes `minItems` and `maxItems`. A length bound on anything but an array raises `TypeError`. An array document's `codecs`, the `sharding_indexed` codec's `codecs` and `index_codecs`, and each list in a `rectilinear` chunk grid's `chunk_shapes` now say on their types that they are never empty, so `check` also refuses `"codecs": []`. The `null_terminated_bytes` and zarr-python `raw_bytes` data types declare `length_bytes` at least 1, and `struct` declares at least one field, on their types, so their JSON Schemas now say so too. +`zarr_metadata.typed_json.check` and definitions' configurations now read more annotated-types constraints, as pydantic does: `MinLen`, `MaxLen` and `Len` on an array, and `MultipleOf` on an integer. A value outside them is an `invalid_value` problem whose `ctx` uses pydantic's names (`min_length`, `multiple_of`), and the JSON Schema writes `minItems`, `maxItems` and `multipleOf`. A constraint on a type it does not apply to raises `TypeError`. Constant rules now live on the types, so the JSON Schemas say them too: non-empty `codecs`, sharding `codecs` and `index_codecs`, rectilinear edge lists and `struct` fields; `length_bytes` of at least 1 for `null_terminated_bytes` and zarr-python `raw_bytes`; and positive multiples of 4 for `fixed_length_utf32`'s `length_bytes` and of 8 for raw `r` bits. `check` now also refuses `"codecs": []`. diff --git a/packages/zarr-metadata/docs/extensions.md b/packages/zarr-metadata/docs/extensions.md index bfc3c39753..cbe9466ba0 100644 --- a/packages/zarr-metadata/docs/extensions.md +++ b/packages/zarr-metadata/docs/extensions.md @@ -82,9 +82,10 @@ names `acme.lz4` is then checked against this definition. - a `NewType` or a type alias. A number may carry bounds from annotated-types: `Gt`, `Ge`, `Lt`, `Le` or -`Interval`. An array may bound its length with `MinLen`, `MaxLen` or -`Len`, so `Annotated[tuple[int, ...], MinLen(1)]` is never empty. Any -other `Annotated` metadata is refused when the definition is built. +`Interval`, and an integer a step, `MultipleOf`. An array may bound its +length with `MinLen`, `MaxLen` or `Len`, so +`Annotated[tuple[int, ...], MinLen(1)]` is never empty. Any other +`Annotated` metadata is refused when the definition is built. **The rules function** diff --git a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py index 3469b14d45..93779de1d3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py @@ -9,8 +9,9 @@ `Mapping[str, V]`, a `NewType` (parsed as its supertype), and a type alias (parsed as the type it stands for). This short list keeps the module small. Any other annotation has no parser. A number type may carry bounds, -and an array type bounds on its length, written with annotated-types as -pydantic reads them, for example `Annotated[int, Interval(ge=0, le=9)]` or +an integer type a step, and an array type bounds on its length, written +with annotated-types as pydantic reads them, for example +`Annotated[int, Interval(ge=0, le=9)]`, `Annotated[int, MultipleOf(4)]` or `Annotated[tuple[int, ...], MinLen(1)]`. A value outside the bounds is a problem. `constraints_of` lists the constraints the checker supports. @@ -848,23 +849,50 @@ class _Constraint: marker: type """The annotated-types class. Its attribute named as the constraint holds the bound.""" - keyword: str - """JSON Schema's keyword for the constraint.""" + keywords: Mapping[str | None, str] + """For each shape (from `shape_of`) the constraint applies to, JSON Schema's keyword for it there.""" holds: Callable[[float, float], bool] """Whether a measured value satisfies the bound.""" - side: Literal["below", "above"] - """The side the bound limits. A type takes one bound from each side.""" - of: Literal["number", "length"] - """What the bound limits: a number, or the length of an array.""" + slot: Literal["below", "above", "step"] + """Which of a type's constraints it is: a lower bound, an upper bound, or a step. A type takes one of each.""" + of: Literal["number", "integer", "length"] + """What the bound limits: a number, an integer, or the length of an array.""" _CONSTRAINTS: Final[Mapping[str, _Constraint]] = { - "gt": _Constraint(annotated_types.Gt, "exclusiveMinimum", operator.gt, "below", "number"), - "ge": _Constraint(annotated_types.Ge, "minimum", operator.ge, "below", "number"), - "lt": _Constraint(annotated_types.Lt, "exclusiveMaximum", operator.lt, "above", "number"), - "le": _Constraint(annotated_types.Le, "maximum", operator.le, "above", "number"), - "min_length": _Constraint(annotated_types.MinLen, "minItems", operator.ge, "below", "length"), - "max_length": _Constraint(annotated_types.MaxLen, "maxItems", operator.le, "above", "length"), + "gt": _Constraint( + annotated_types.Gt, + {"int": "exclusiveMinimum", "number": "exclusiveMinimum"}, + operator.gt, + "below", + "number", + ), + "ge": _Constraint( + annotated_types.Ge, {"int": "minimum", "number": "minimum"}, operator.ge, "below", "number" + ), + "lt": _Constraint( + annotated_types.Lt, + {"int": "exclusiveMaximum", "number": "exclusiveMaximum"}, + operator.lt, + "above", + "number", + ), + "le": _Constraint( + annotated_types.Le, {"int": "maximum", "number": "maximum"}, operator.le, "above", "number" + ), + "multiple_of": _Constraint( + annotated_types.MultipleOf, + {"int": "multipleOf"}, + lambda value, step: value % step == 0, + "step", + "integer", + ), + "min_length": _Constraint( + annotated_types.MinLen, {"tuple": "minItems"}, operator.ge, "below", "length" + ), + "max_length": _Constraint( + annotated_types.MaxLen, {"tuple": "maxItems"}, operator.le, "above", "length" + ), } """The annotated-types constraints the checker reads, by the name pydantic uses, which is also the attribute of the marker that holds the bound.""" @@ -872,13 +900,28 @@ class _Constraint: """The names of the markers the checker reads, in the order of `_CONSTRAINTS`.""" _MARKERS: Final = f"{', '.join(_NAMES[:-1])} and {_NAMES[-1]}" -"""The markers the checker reads, as an error message lists them: "Gt, Ge, Lt, Le, MinLen and MaxLen".""" +"""The markers the checker reads, as an error message lists them: "Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen".""" -_MEASURED: Final[Mapping[str, tuple[frozenset[str | None], str]]] = { - "number": (frozenset({"int", "number"}), "a number"), - "length": (frozenset({"tuple"}), "an array"), +_MEASURED: Final[Mapping[str, str]] = { + "number": "a bound is on a number", + "integer": "a step is on an integer", + "length": "a bound is on an array", } -"""For what each constraint limits, the shapes (from `shape_of`) it applies to, and their name in an error message.""" +"""For what each constraint limits, what an error message says it applies to.""" + +_SECOND: Final[Mapping[str, str]] = { + "below": "bound from below", + "above": "bound from above", + "step": "step", +} +"""For each slot, what an error message calls a second constraint in it.""" + +_STRICTER: Final[Mapping[str, Callable[[int | float, int | float], int | float]]] = { + "below": max, + "above": min, + "step": lambda one, other: math.lcm(int(one), int(other)), +} +"""For each slot, the constraint a value meets exactly when it meets both of two: the stricter bound, or the least common multiple of two steps.""" _EXCLUSIVE: Final = frozenset({"gt", "lt"}) """The exclusive bounds: a value may not equal them.""" @@ -888,17 +931,19 @@ def constraints_of(metadata: Sequence[object]) -> dict[str, int | float]: """The constraints in an `Annotated` type's metadata, by name: at most one lower and one upper bound. The supported constraints are annotated-types' `Gt`, `Ge`, `Lt` and - `Le`, which bound a number, `MinLen` and `MaxLen`, which bound the - length of an array, and an `Interval` or a `Len`, which unpack to - them, as pydantic reads them. A note (a string or a `Doc`) does not - constrain the value and is skipped. Anything else raises `TypeError`: - - - a constraint the checker does not support, such as `MultipleOf`, - `Predicate`, or pydantic's `Field`. Otherwise the type would declare - a rule that the checker does not enforce. - - a second bound on the same side. pydantic uses the last one. - - a bound that is not a finite number, or a length bound that is not - a non-negative integer. + `Le`, which bound a number, `MultipleOf`, which is a step an integer + is a multiple of, `MinLen` and `MaxLen`, which bound the length of an + array, and an `Interval` or a `Len`, which unpack to them, as pydantic + reads them. A note (a string or a `Doc`) does not constrain the value + and is skipped. Anything else raises `TypeError`: + + - a constraint the checker does not support, such as `Predicate` or + pydantic's `Field`. Otherwise the type would declare a rule that the + checker does not enforce. + - a second bound on the same side, or a second step. pydantic uses the + last one. + - a bound that is not a finite number, a step that is not a positive + integer, or a length bound that is not a non-negative integer. A bound is stored as the number it equals, as an integer when it is one. `Ge(True)`, `Ge(1.0)` and `Ge(1)` are the same bound, because @@ -917,15 +962,12 @@ def constraints_of(metadata: Sequence[object]) -> dict[str, int | float]: raise UnreadConstraintError(msg) constraint = _CONSTRAINTS[name] if any( - _CONSTRAINTS[held].side == constraint.side and _CONSTRAINTS[held].of == constraint.of + _CONSTRAINTS[held].slot == constraint.slot and _CONSTRAINTS[held].of == constraint.of for held in said ): - msg = ( - f"{item!r} is a second bound from {constraint.side}; " - "a type takes one from each side" - ) + msg = f"{item!r} is a second {_SECOND[constraint.slot]}; a type takes one of each" raise TypeError(msg) - said[name] = _bound(item, name) if constraint.of == "number" else _length(item, name) + said[name] = _read_bound(item, name, constraint.of) return said @@ -949,6 +991,24 @@ def _bound(item: object, name: str) -> int | float: raise TypeError(msg) +def _read_bound(item: object, name: str, of: str) -> int | float: + """The bound of `item` named `name`, as a constraint on what `of` names reads it; `TypeError` if it cannot be one.""" + if of == "number": + return _bound(item, name) + if of == "integer": + return _step(item, name) + return _length(item, name) + + +def _step(item: object, name: str) -> int: + """The step of `item` named `name`; `TypeError` if it is not a positive integer, as JSON Schema requires of `multipleOf`.""" + bound = _bound(item, name) + if isinstance(bound, int) and bound > 0: + return bound + msg = f"{item!r}: a step is a positive integer" + raise TypeError(msg) + + def _length(item: object, name: str) -> int: """The length of length bound `item` named `name`; `TypeError` if it is not a non-negative integer.""" bound = _bound(item, name) @@ -959,22 +1019,24 @@ def _length(item: object, name: str) -> int: def _constrainable(inner: object, said: Constraints) -> None: - """Raise `TypeError` for a constraint on a type it does not limit: a bound on what is not a number, or a length bound on what is not an array.""" + """Raise `TypeError` for a constraint on a type it does not limit: a bound on what is not a number, a step on what is not an integer, or a length bound on what is not an array.""" shape = shape_of(inner) - for of, (shapes, noun) in _MEASURED.items(): + for of, applies in _MEASURED.items(): names = [name for name in said if _CONSTRAINTS[name].of == of] - if len(names) != 0 and shape not in shapes: - msg = f"{', '.join(names)}: a bound is on {noun}, and {describe(inner)} is not one" + if any(shape not in _CONSTRAINTS[name].keywords for name in names): + msg = f"{', '.join(names)}: {applies}, and {describe(inner)} is not one" raise TypeError(msg) def constrained(parse: Parser, description: str, said: Constraints) -> Parser: """Wrap `parse` to also check the constraints its type carries: one `invalid_value` problem if any fails. - A bound limits a number, and a length bound the number of elements in - an array. The message states what the type allows, for example - "expected an integer in [0, 9], got 12" or "expected an array of - integers with at least 1 element, got []", and the problem's `ctx` + A bound limits a number, a step what an integer is a multiple of, and + a length bound the number of elements in an array. The message states + what the type allows, for example "expected an integer in [0, 9], got + 12", "expected an integer > 0 that is a multiple of 4, got 6" or + "expected an array of integers with at least 1 element, got []", and + the problem's `ctx` holds the constraints, as pydantic names them. The constraints are checked only when `parse` reports no problem, in the same way that rules run only on a value of their type. @@ -1001,23 +1063,36 @@ def parse_constrained(value: object, loc: Loc) -> Parsed: def _admitted(said: Constraints) -> str: - """What the constraints allow, as written after the type in a message: "in [0, 9]", "in (0, 1)", ">= 1", "with at least 1 element".""" + """What the constraints allow, as written after the type in a message: "in [0, 9]", ">= 1", "> 0 that is a multiple of 4", "with at least 1 element".""" if any(_CONSTRAINTS[name].of == "length" for name in said): return _admitted_lengths(said.get("min_length"), said.get("max_length")) low = next( - ((name, bound) for name, bound in said.items() if _CONSTRAINTS[name].side == "below"), None + ((name, bound) for name, bound in said.items() if _CONSTRAINTS[name].slot == "below"), None ) high = next( - ((name, bound) for name, bound in said.items() if _CONSTRAINTS[name].side == "above"), None + ((name, bound) for name, bound in said.items() if _CONSTRAINTS[name].slot == "above"), None ) + bounded = _admitted_range(low, high) + step = said.get("multiple_of") + if step is None: + return bounded + multiple = f"that is a multiple of {shown(step)}" + return multiple if bounded == "" else f"{bounded} {multiple}" + + +def _admitted_range( + low: tuple[str, int | float] | None, high: tuple[str, int | float] | None +) -> str: + """The range a lower and an upper bound allow, each named, as written after the type in a message: "in [0, 9]", "in (0, 1)", ">= 1"; empty without either.""" if low is not None and high is not None: opening = "(" if low[0] in _EXCLUSIVE else "[" closing = ")" if high[0] in _EXCLUSIVE else "]" return f"in {opening}{shown(low[1])}, {shown(high[1])}{closing}" if low is not None: return f"{'>' if low[0] in _EXCLUSIVE else '>='} {shown(low[1])}" - name, bound = cast("tuple[str, int | float]", high) - return f"{'<' if name in _EXCLUSIVE else '<='} {shown(bound)}" + if high is not None: + return f"{'<' if high[0] in _EXCLUSIVE else '<='} {shown(high[1])}" + return "" def _admitted_lengths(low: float | None, high: float | None) -> str: @@ -1497,12 +1572,15 @@ def of(self, annotation: object) -> JSONSchema: ] if len(notes) != 0: schema = {**schema, "description": "\n\n".join(notes)} - for name, bound in constraints_of(metadata).items(): + said = constraints_of(metadata) + _constrainable(inner, said) + shape = shape_of(inner) + for name, bound in said.items(): constraint = _CONSTRAINTS[name] - held = cast("int | float | None", schema.get(constraint.keyword)) - stricter = max if constraint.side == "below" else min - kept = bound if held is None else stricter(held, bound) - schema = {**schema, constraint.keyword: kept} + keyword = constraint.keywords[shape] + held = cast("int | float | None", schema.get(keyword)) + kept = bound if held is None else _STRICTER[constraint.slot](held, bound) + schema = {**schema, keyword: kept} return schema def object_of(self, typeddict: type) -> JSONSchema: diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 0fadbe02ef..5172f33443 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -163,8 +163,9 @@ def acme_lz4_rules( type may carry bounds, written with annotated-types and read the way pydantic reads them: `Gt`, `Ge`, `Lt`, `Le` and `Interval`, at most one from each side, at any depth. For example, -`tuple[Annotated[int, Ge(1)], ...]` bounds each element. An array's type -may bound its length with `MinLen`, `MaxLen` and `Len`, so +`tuple[Annotated[int, Ge(1)], ...]` bounds each element. An integer's +type may carry a step, `MultipleOf`, at most one. An array's type may +bound its length with `MinLen`, `MaxLen` and `Len`, so `Annotated[tuple[int, ...], MinLen(1)]` is never empty. A value out of bounds is an `invalid_value` problem. Its message says what the type admits ("expected an integer >= 1, got 0"), and its `ctx` holds the @@ -172,8 +173,8 @@ def acme_lz4_rules( rule can rely on them, as pydantic's after-validators and zod's refinements do. Until a value out of bounds is fixed, it is the only problem reported for the configuration. `Annotated` may also carry a -note, as a string or a `Doc`. Any other metadata (a `MultipleOf`, a -`Predicate`, pydantic's `Field`) is rejected when the definition is +note, as a string or a `Doc`. Any other metadata (a `Predicate`, +pydantic's `Field`) is rejected when the definition is built, because a type whose constraint the checker does not enforce would give false information. diff --git a/packages/zarr-metadata/src/zarr_metadata/typed_json.py b/packages/zarr-metadata/src/zarr_metadata/typed_json.py index 75480f378d..b18c4a2f4e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/typed_json.py @@ -41,9 +41,11 @@ - A number type may carry bounds, written with annotated-types as pydantic reads them: `Gt`, `Ge`, `Lt`, `Le` and `Interval`, at any depth. For example, `tuple[Annotated[int, Ge(1)], ...]` bounds each - element. An array type may bound its length the same way, with - `MinLen`, `MaxLen` and `Len`: `Annotated[tuple[int, ...], MinLen(1)]` - is a non-empty array. A value outside the bounds is an `invalid_value` + element. An integer type may also carry a step, `MultipleOf`: + `Annotated[int, Gt(0), MultipleOf(4)]` is a positive multiple of 4. An + array type may bound its length the same way, with `MinLen`, `MaxLen` + and `Len`: `Annotated[tuple[int, ...], MinLen(1)]` is a non-empty + array. A value outside the bounds is an `invalid_value` problem. Its message states what the type allows, and its `ctx` holds the bounds, as pydantic names them (`ge`, `min_length`). `Annotated` may also carry a note (a string or a `Doc`). Any other diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py index b800fd452c..dd3bed411b 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py @@ -13,7 +13,7 @@ from collections.abc import Mapping from typing import Annotated, Final, Literal, NewType, NotRequired, get_args -from annotated_types import Ge, Interval, MinLen +from annotated_types import Ge, Gt, Interval, MinLen, MultipleOf from typing_extensions import ReadOnly, TypedDict from zarr_metadata._common import JSONValue @@ -352,10 +352,11 @@ class RawBytesConfiguration(TypedDict, closed=True): back as the name. """ - bits: int + bits: Annotated[int, Gt(0), MultipleOf(8)] """The size in bits that the name writes, `16` for `r16`. - A positive multiple of 8. + A positive multiple of 8: "raw bits, variable size given by *, limited + to be a multiple of 8". Zero bits is not a valid type. """ @@ -411,9 +412,11 @@ class RawBytesConfiguration(TypedDict, closed=True): class FixedLengthUtf32Configuration(TypedDict, closed=True): """Configuration for the `fixed_length_utf32` data type.""" - length_bytes: ReadOnly[int] - """The size in bytes of every value: a positive multiple of 4, since - each code point takes 4 bytes.""" + length_bytes: ReadOnly[Annotated[int, Gt(0), MultipleOf(4)]] + """The size in bytes of every value: a "positive integer, multiple of `4`", since each code point takes 4 bytes. + + https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/data-types/fixed_length_utf32/README.md?plain=1#L41 + """ class FixedLengthUtf32(TypedDict, closed=True): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py index 7bb8c135e6..9c585d20ba 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py @@ -21,19 +21,6 @@ _CODE_POINT_BYTES: Final = 4 -def _rules( - configuration: FixedLengthUtf32Configuration, nested: Nested -) -> Iterator[ValidationProblem]: - """Check that `length_bytes` is a "positive integer, multiple of `4`" (https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/data-types/fixed_length_utf32/README.md?plain=1#L41).""" - length_bytes = configuration["length_bytes"] - if length_bytes <= 0 or length_bytes % _CODE_POINT_BYTES != 0: - yield ValidationProblem( - ("length_bytes",), - f"expected a positive multiple of {_CODE_POINT_BYTES}, got {length_bytes}", - "invalid_value", - ) - - def _fill_value_rules( configuration: FixedLengthUtf32Configuration, nested: Nested, value: FixedLengthUtf32FillValue ) -> Iterator[ValidationProblem]: @@ -73,7 +60,6 @@ def _fill_value_canonical( FIXED_LENGTH_UTF32_DATA_TYPE: Final = DataTypeDefinition( name=FIXED_LENGTH_UTF32_DATA_TYPE_NAME, configuration=FixedLengthUtf32Configuration, - rules=_rules, fill_value=FixedLengthUtf32FillValue, fill_value_default=ConstantFillValue(""), fill_value_rules=_fill_value_rules, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 63fc15e878..8af68ee617 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -41,17 +41,6 @@ def raw_bytes_dtype_name(value: str) -> RawBytesDataTypeName: return RawBytesDataTypeName(value) -def _rules(configuration: RawBytesConfiguration, nested: Nested) -> Iterator[ValidationProblem]: - """Check the size: "raw bits, variable size given by *, limited to be a multiple of 8". Zero bits is not a valid type.""" - bits = configuration["bits"] - if bits == 0 or bits % 8 != 0: - yield ValidationProblem( - ("bits",), - f"expected a size in bits that is a positive multiple of 8, got {bits}", - "invalid_value", - ) - - def _fill_value_rules( configuration: RawBytesConfiguration, nested: Nested, value: RawBytesFillValue ) -> Iterator[ValidationProblem]: @@ -88,7 +77,6 @@ def _storage(configuration: RawBytesConfiguration, nested: Nested) -> StorageCla RAW_BYTES_DATA_TYPE: Final = DataTypeDefinition( name=RAW_BYTES_NAME, configuration=RawBytesConfiguration, - rules=_rules, fill_value=RawBytesFillValue, fill_value_default=_fill_value_default, fill_value_rules=_fill_value_rules, diff --git a/packages/zarr-metadata/tests/test_json_schema.py b/packages/zarr-metadata/tests/test_json_schema.py index ad70fb5902..368d6c1e3d 100644 --- a/packages/zarr-metadata/tests/test_json_schema.py +++ b/packages/zarr-metadata/tests/test_json_schema.py @@ -15,7 +15,7 @@ from typing import Annotated, Any, Literal, NewType, NotRequired, cast import pytest -from annotated_types import Ge, Gt, Interval, Le, Len, Lt, MaxLen, MinLen, MultipleOf +from annotated_types import Ge, Gt, Interval, Le, Len, Lt, MaxLen, MinLen, MultipleOf, Predicate from hypothesis import HealthCheck, event, given, settings from hypothesis import strategies as st from jsonschema import Draft202012Validator @@ -170,6 +170,11 @@ def _held(member: dict[str, Any], defs: dict[str, Any] | None = None) -> dict[st _holding(Annotated[int, Ge(0), Doc("a count")]), _held({"type": "integer", "description": "a count", "minimum": 0}), ), + ( + "a-step", + _holding(Annotated[int, Gt(0), MultipleOf(4)]), + _held({"type": "integer", "exclusiveMinimum": 0, "multipleOf": 4}), + ), ( "non-empty-array", _holding(Annotated[tuple[int, ...], MinLen(1)]), @@ -359,7 +364,7 @@ def test_error_a_schema_that_fails_to_write_leaves_nothing_behind() -> None: def test_json_schema_refuses_metadata_check_does_not_hold_a_value_to() -> None: with pytest.raises(TypeError, match="is not a constraint the checker reads"): - json_schema(_holding(Annotated[int, MultipleOf(2)])) + json_schema(_holding(Annotated[int, Predicate(bool)])) _PROPERTY = settings(max_examples=300, deadline=None, suppress_health_check=[HealthCheck.too_slow]) @@ -975,6 +980,7 @@ def test_every_schema_is_a_json_schema_written_the_same_each_time(context: ZarrV (Le(9), Le(5), {"type": "integer", "maximum": 5}), (Ge(5), Ge(9), {"type": "integer", "minimum": 9}), (Ge(9), Ge(5), {"type": "integer", "minimum": 9}), + (MultipleOf(4), MultipleOf(6), {"type": "integer", "multipleOf": 12}), ], ids=[ "gt-looser-inside", @@ -985,6 +991,7 @@ def test_every_schema_is_a_json_schema_written_the_same_each_time(context: ZarrV "le-stricter-inside", "ge", "ge-stricter-inside", + "two-steps-are-their-least-common-multiple", ], ) def test_the_stricter_of_two_bounds_of_one_keyword_is_kept( diff --git a/packages/zarr-metadata/tests/test_typed_json.py b/packages/zarr-metadata/tests/test_typed_json.py index 3762b817d7..09a2b23572 100644 --- a/packages/zarr-metadata/tests/test_typed_json.py +++ b/packages/zarr-metadata/tests/test_typed_json.py @@ -1044,6 +1044,34 @@ class Noted(TypedDict, extra_items=Annotated[object, "any JSON"]): (Digit | str, 10, [((), "expected an integer in [0, 9], got 10", {"ge": 0, "le": 9})]), (Digit, "nine", [((), 'expected an integer, got "nine"', {})]), (Annotated[int, "a note", Doc("another")], -1, []), + (Annotated[int, MultipleOf(4)], 8, []), + ( + Annotated[int, MultipleOf(4)], + 6, + [((), "expected an integer that is a multiple of 4, got 6", {"multiple_of": 4})], + ), + ( + Annotated[int, Gt(0), MultipleOf(4)], + 0, + [ + ( + (), + "expected an integer > 0 that is a multiple of 4, got 0", + {"gt": 0, "multiple_of": 4}, + ) + ], + ), + ( + Annotated[int, Interval(ge=0, le=9), MultipleOf(2)], + 3, + [ + ( + (), + "expected an integer in [0, 9] that is a multiple of 2, got 3", + {"ge": 0, "le": 9, "multiple_of": 2}, + ) + ], + ), (Annotated[tuple[int, ...], MinLen(1)], [0], []), ( Annotated[tuple[int, ...], MinLen(1)], @@ -1127,6 +1155,10 @@ class Noted(TypedDict, extra_items=Annotated[object, "any JSON"]): "the-branch-it-bounds", "a-value-of-another-type-is-that-problem-alone", "notes", + "a-multiple", + "not-a-multiple", + "a-multiple-out-of-bounds", + "bounded-and-not-a-multiple", "a-length-within", "a-length-below", "a-length-above", @@ -1141,7 +1173,7 @@ def test_a_value_is_held_to_the_bounds_its_type_carries( annotation: object, value: object, found: list[tuple[Loc, str, dict[str, JSONValue]]] ) -> None: # annotated-types bounds, read as pydantic reads them: on a number's - # value, or on an array's length. A value outside any bound gives one + # value, on what an integer is a multiple of, or on an array's length. A value outside any bound gives one # problem that says what the type allows, with the bounds in the # problem's `ctx`, named as pydantic names them. Bounds are checked only # for a value of the type. A bound is compared as the number it equals, whichever of @@ -1216,7 +1248,6 @@ class Later(BaseMetadata): @pytest.mark.parametrize( "constraint", [ - MultipleOf(2), Predicate(str.isdigit), Timezone(None), Later(), @@ -1224,7 +1255,6 @@ class Later(BaseMetadata): pydantic.StringConstraints(pattern="^[a-z]+$"), ], ids=[ - "multiple-of", "predicate", "timezone", "one-a-later-release-adds", @@ -1240,7 +1270,7 @@ def test_error_a_constraint_the_checker_does_not_read(constraint: object) -> Non class _ConstrainedMember(TypedDict, closed=True): - a: Annotated[int, MultipleOf(2)] + a: Annotated[int, Later()] class _HoldsAConstrainedMember(TypedDict, closed=True): @@ -1269,16 +1299,16 @@ class _HoldsAnUnresolvableTypedDict(TypedDict, closed=True): class _ConstraintOnAReadableMember(TypedDict, closed=True): - inner: Annotated[_Readable, MultipleOf(2)] + inner: Annotated[_Readable, Later()] class _ConstraintOnAConstrainedMember(TypedDict, closed=True): - inner: Annotated[_ConstrainedMember, MultipleOf(2)] + inner: Annotated[_ConstrainedMember, Later()] class _SelfReferentialWithAConstraint(TypedDict, closed=True): parent: NotRequired[_SelfReferentialWithAConstraint] - a: Annotated[int, MultipleOf(2)] + a: Annotated[int, Later()] class _SelfReferentialNotAJSONShape(TypedDict, closed=True): @@ -1287,7 +1317,7 @@ class _SelfReferentialNotAJSONShape(TypedDict, closed=True): class _SelfReferentialConstraintOnItself(TypedDict, closed=True): - parent: NotRequired[Annotated[_SelfReferentialConstraintOnItself, MultipleOf(2)]] + parent: NotRequired[Annotated[_SelfReferentialConstraintOnItself, Later()]] a: int @@ -1302,7 +1332,7 @@ class _CycleBack(TypedDict, closed=True): class _OuterWithAConstraintAfterACycle(TypedDict, closed=True): inner: _InnerLeadingBackToAConstraint - a: Annotated[int, MultipleOf(2)] + a: Annotated[int, Later()] class _InnerLeadingBackToAConstraint(TypedDict, closed=True): @@ -1319,7 +1349,7 @@ class _InnerLeadingBackToAShape(TypedDict, closed=True): class _OuterWithAConstraintOnTheCycle(TypedDict, closed=True): - inner: Annotated[_InnerLeadingBackToTheConstraint, MultipleOf(2)] + inner: Annotated[_InnerLeadingBackToTheConstraint, Later()] a: int @@ -1329,19 +1359,17 @@ class _InnerLeadingBackToTheConstraint(TypedDict, closed=True): class _SelfReferentialTwiceWithAConstraintOnTheSecond(TypedDict, closed=True): plain: NotRequired[_SelfReferentialTwiceWithAConstraintOnTheSecond] - constrained: NotRequired[ - Annotated[_SelfReferentialTwiceWithAConstraintOnTheSecond, MultipleOf(2)] - ] + constrained: NotRequired[Annotated[_SelfReferentialTwiceWithAConstraintOnTheSecond, Later()]] class _DocumentedThenConstrainedBackReferences(TypedDict, closed=True): documented: NotRequired[Annotated[_DocumentedThenConstrainedBackReferences, Doc("parent")]] - constrained: NotRequired[Annotated[_DocumentedThenConstrainedBackReferences, MultipleOf(2)]] + constrained: NotRequired[Annotated[_DocumentedThenConstrainedBackReferences, Later()]] class _OuterWithAConstraintAfterAnOptionalCycle(TypedDict, closed=True): inner: _InnerLeadingBackOptionally - a: Annotated[int, MultipleOf(2)] + a: Annotated[int, Later()] class _InnerLeadingBackOptionally(TypedDict, closed=True): @@ -1357,7 +1385,7 @@ class _HoldsATupleOfAConstrainedMember(TypedDict, closed=True): class _ConstraintOnANotAJSONShapeMember(TypedDict, closed=True): - inner: Annotated[_NotAJSONShape, MultipleOf(2)] + inner: Annotated[_NotAJSONShape, Later()] _Children = TypeAliasType("_Children", "tuple[_SelfReferentialThroughAnAlias, ...]") @@ -1408,16 +1436,16 @@ class _SelfReferentialReadable(TypedDict, closed=True): ( _ConstrainedMember, Unread( - "_ConstrainedMember.a: MultipleOf(multiple_of=2) is not a constraint the checker " - "reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", + "_ConstrainedMember.a: Later() is not a constraint the checker " + "reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), ( _HoldsAConstrainedMember, Unread( - "_HoldsAConstrainedMember.inner: _ConstrainedMember.a: MultipleOf(multiple_of=2) is " - "not a constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", + "_HoldsAConstrainedMember.inner: _ConstrainedMember.a: Later() is " + "not a constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1449,8 +1477,8 @@ class _SelfReferentialReadable(TypedDict, closed=True): ( _ConstraintOnAReadableMember, Unread( - "_ConstraintOnAReadableMember.inner: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", + "_ConstraintOnAReadableMember.inner: Later() is not a " + "constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1458,16 +1486,16 @@ class _SelfReferentialReadable(TypedDict, closed=True): _ConstraintOnAConstrainedMember, Unread( "_ConstraintOnAConstrainedMember.inner: _ConstrainedMember.a: " - "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " - "Ge, Lt, Le, MinLen and MaxLen", + "Later() is not a constraint the checker reads, which are Gt, " + "Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), ( _SelfReferentialWithAConstraint, Unread( - "_SelfReferentialWithAConstraint.a: MultipleOf(multiple_of=2) is not a constraint " - "the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", + "_SelfReferentialWithAConstraint.a: Later() is not a constraint " + "the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1482,8 +1510,8 @@ class _SelfReferentialReadable(TypedDict, closed=True): ( _SelfReferentialConstraintOnItself, Unread( - "_SelfReferentialConstraintOnItself.parent: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", + "_SelfReferentialConstraintOnItself.parent: Later() is not a " + "constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1498,8 +1526,8 @@ class _SelfReferentialReadable(TypedDict, closed=True): ( _OuterWithAConstraintAfterACycle, Unread( - "_OuterWithAConstraintAfterACycle.a: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", + "_OuterWithAConstraintAfterACycle.a: Later() is not a " + "constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1514,8 +1542,8 @@ class _SelfReferentialReadable(TypedDict, closed=True): ( _OuterWithAConstraintOnTheCycle, Unread( - "_OuterWithAConstraintOnTheCycle.inner: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", + "_OuterWithAConstraintOnTheCycle.inner: Later() is not a " + "constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1523,24 +1551,24 @@ class _SelfReferentialReadable(TypedDict, closed=True): _SelfReferentialTwiceWithAConstraintOnTheSecond, Unread( "_SelfReferentialTwiceWithAConstraintOnTheSecond.constrained: " - "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " - "Ge, Lt, Le, MinLen and MaxLen", + "Later() is not a constraint the checker reads, which are Gt, " + "Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), ( _DocumentedThenConstrainedBackReferences, Unread( - "_DocumentedThenConstrainedBackReferences.constrained: MultipleOf(multiple_of=2) " - "is not a constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", + "_DocumentedThenConstrainedBackReferences.constrained: Later() " + "is not a constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), ( _OuterWithAConstraintAfterAnOptionalCycle, Unread( - "_OuterWithAConstraintAfterAnOptionalCycle.a: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt, Le, MinLen and MaxLen", + "_OuterWithAConstraintAfterAnOptionalCycle.a: Later() is not a " + "constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1557,8 +1585,8 @@ class _SelfReferentialReadable(TypedDict, closed=True): _HoldsATupleOfAConstrainedMember, Unread( "_HoldsATupleOfAConstrainedMember.inner: _ConstrainedMember.a: " - "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " - "Ge, Lt, Le, MinLen and MaxLen", + "Later() is not a constraint the checker reads, which are Gt, " + "Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1658,12 +1686,13 @@ def test_unread_in_names_the_part_without_a_parser_and_whether_it_is_a_constrain Annotated[int, Ge(0), Gt(0)], Annotated[int, Interval(le=9), Lt(5)], Annotated[tuple[int, ...], Len(1, 3), MinLen(2)], + Annotated[int, MultipleOf(2), MultipleOf(3)], ], - ids=["narrowing-an-alias", "gt-and-ge", "interval-and-lt", "len-and-min-len"], + ids=["narrowing-an-alias", "gt-and-ge", "interval-and-lt", "len-and-min-len", "two-steps"], ) def test_error_a_second_bound_from_one_side(annotation: object) -> None: # Pydantic uses the last bound given; write only one. - with pytest.raises(TypeError, match="is a second bound from (below|above)"): + with pytest.raises(TypeError, match="is a second (bound from below|bound from above|step)"): parser(annotation, no_leaf) @@ -1675,6 +1704,21 @@ def test_error_a_bound_that_is_not_a_finite_number(bound: object) -> None: parser(Annotated[int, Ge(bound)], no_leaf) # pyright: ignore[reportArgumentType] +def test_error_a_step_on_what_is_not_an_integer() -> None: + # pydantic also takes a step on a float; JSON Schema validators compare + # floats inexactly, so the checker takes one on an integer only. + with pytest.raises( + TypeError, match="multiple_of: a step is on an integer, and a number is not one" + ): + parser(Annotated[float, MultipleOf(2)], no_leaf) + + +@pytest.mark.parametrize("step", [0, -2, 1.5], ids=["zero", "negative", "fractional"]) +def test_error_a_step_that_is_not_a_positive_integer(step: object) -> None: + with pytest.raises(TypeError, match="a step is a positive integer"): + parser(Annotated[int, MultipleOf(step)], no_leaf) # pyright: ignore[reportArgumentType] + + @pytest.mark.parametrize("length", [-1, 1.5], ids=["negative", "fractional"]) def test_error_a_length_bound_that_is_not_a_non_negative_integer(length: object) -> None: with pytest.raises(TypeError, match="a length bound is a non-negative integer"): diff --git a/packages/zarr-metadata/tests/v3/test_every_definition.py b/packages/zarr-metadata/tests/v3/test_every_definition.py index 3090024c53..86a3e4a2e2 100644 --- a/packages/zarr-metadata/tests/v3/test_every_definition.py +++ b/packages/zarr-metadata/tests/v3/test_every_definition.py @@ -12,6 +12,7 @@ from typing import Any import pytest +from typing_extensions import TypedDict from zarr_metadata import ValidationProblem from zarr_metadata._json import value_at @@ -26,7 +27,7 @@ ) from zarr_metadata.definition._definition import canonicalize from zarr_metadata.v3.codec.gzip import GZIP_CODEC -from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE, RawBytesConfiguration +from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, @@ -385,11 +386,17 @@ def test_raw_bits_read_as_r_star_with_the_size_their_name_carries( ) +class AnyRawSize(TypedDict, closed=True): + """A raw-bits configuration of any size, which the package's own type does not allow.""" + + bits: int + + def test_a_reader_reads_raw_bits_its_own_way_by_defining_r_star() -> None: # This definition accepts any size. Filed under `r*` in place of the # package's own, it reads every raw-bits name. Filed back, the package's # definition reads them again. - mine = DataTypeDefinition(name="r*", configuration=RawBytesConfiguration) + mine = DataTypeDefinition(name="r*", configuration=AnyRawSize) context = CORE_AND_EXTENSIONS_CONTEXT.copy( update={"data_types": {**CORE_AND_EXTENSIONS_CONTEXT.data_types, "r*": mine}} ) From 13ae83fba87345c2595595c97bd5535fd8d0aee6 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 18:54:15 +0100 Subject: [PATCH 08/13] feat(zarr-metadata): read MinLen and MaxLen on strings in the TypedDict checker `MinLen`, `MaxLen` and `Len` now also bound a string, as pydantic's `min_length` does. The JSON Schema writes `minLength` and `maxLength`. Python and JSON Schema both count code points, so the checker and the schema agree on what a string's length is. Messages count characters: "expected a string with at least 1 character, got \"\"". A length bound on an object still raises `TypeError`. The v3 `struct` field name now declares `MinLen(1)` on its type, and the struct rule no longer checks for an empty name. The problem keeps its `loc` and kind. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../zarr-metadata/changes/4635.feature.md | 2 +- packages/zarr-metadata/docs/extensions.md | 4 +- .../src/zarr_metadata/_typed_json.py | 67 ++++++++++++------- .../src/zarr_metadata/definition/__init__.py | 4 +- .../src/zarr_metadata/typed_json.py | 7 +- .../src/zarr_metadata/v3/_types/data_type.py | 4 +- .../src/zarr_metadata/v3/data_type/struct.py | 6 +- .../zarr-metadata/tests/test_json_schema.py | 5 ++ .../zarr-metadata/tests/test_typed_json.py | 43 ++++++++++-- 9 files changed, 95 insertions(+), 47 deletions(-) diff --git a/packages/zarr-metadata/changes/4635.feature.md b/packages/zarr-metadata/changes/4635.feature.md index 65b16a4559..5ff0bcb2e5 100644 --- a/packages/zarr-metadata/changes/4635.feature.md +++ b/packages/zarr-metadata/changes/4635.feature.md @@ -1 +1 @@ -`zarr_metadata.typed_json.check` and definitions' configurations now read more annotated-types constraints, as pydantic does: `MinLen`, `MaxLen` and `Len` on an array, and `MultipleOf` on an integer. A value outside them is an `invalid_value` problem whose `ctx` uses pydantic's names (`min_length`, `multiple_of`), and the JSON Schema writes `minItems`, `maxItems` and `multipleOf`. A constraint on a type it does not apply to raises `TypeError`. Constant rules now live on the types, so the JSON Schemas say them too: non-empty `codecs`, sharding `codecs` and `index_codecs`, rectilinear edge lists and `struct` fields; `length_bytes` of at least 1 for `null_terminated_bytes` and zarr-python `raw_bytes`; and positive multiples of 4 for `fixed_length_utf32`'s `length_bytes` and of 8 for raw `r` bits. `check` now also refuses `"codecs": []`. +`zarr_metadata.typed_json.check` and definitions' configurations now read more annotated-types constraints, as pydantic does: `MinLen`, `MaxLen` and `Len` on an array or a string, and `MultipleOf` on an integer. A value outside them is an `invalid_value` problem whose `ctx` uses pydantic's names (`min_length`, `multiple_of`), and the JSON Schema writes the matching keyword, such as `minItems`, `minLength` or `multipleOf`. A constraint on a type it does not apply to raises `TypeError`. Constant rules now live on the types, so the JSON Schemas say them too: non-empty `codecs`, sharding `codecs` and `index_codecs`, rectilinear edge lists, `struct` fields and v3 `struct` field names; `length_bytes` of at least 1 for `null_terminated_bytes` and zarr-python `raw_bytes`; and positive multiples of 4 for `fixed_length_utf32`'s `length_bytes` and of 8 for raw `r` bits. `check` now also refuses `"codecs": []`. diff --git a/packages/zarr-metadata/docs/extensions.md b/packages/zarr-metadata/docs/extensions.md index cbe9466ba0..fc314760c8 100644 --- a/packages/zarr-metadata/docs/extensions.md +++ b/packages/zarr-metadata/docs/extensions.md @@ -82,8 +82,8 @@ names `acme.lz4` is then checked against this definition. - a `NewType` or a type alias. A number may carry bounds from annotated-types: `Gt`, `Ge`, `Lt`, `Le` or -`Interval`, and an integer a step, `MultipleOf`. An array may bound its -length with `MinLen`, `MaxLen` or `Len`, so +`Interval`, and an integer a step, `MultipleOf`. An array or a string may +bound its length with `MinLen`, `MaxLen` or `Len`, so `Annotated[tuple[int, ...], MinLen(1)]` is never empty. Any other `Annotated` metadata is refused when the definition is built. diff --git a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py index 93779de1d3..516a41e149 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py @@ -9,7 +9,7 @@ `Mapping[str, V]`, a `NewType` (parsed as its supertype), and a type alias (parsed as the type it stands for). This short list keeps the module small. Any other annotation has no parser. A number type may carry bounds, -an integer type a step, and an array type bounds on its length, written +an integer type a step, and an array or a string bounds on its length, written with annotated-types as pydantic reads them, for example `Annotated[int, Interval(ge=0, le=9)]`, `Annotated[int, MultipleOf(4)]` or `Annotated[tuple[int, ...], MinLen(1)]`. A value outside the bounds is a @@ -88,6 +88,8 @@ ) if TYPE_CHECKING: + from collections.abc import Sized + from zarr_metadata._json import ProblemKind @@ -856,7 +858,7 @@ class _Constraint: slot: Literal["below", "above", "step"] """Which of a type's constraints it is: a lower bound, an upper bound, or a step. A type takes one of each.""" of: Literal["number", "integer", "length"] - """What the bound limits: a number, an integer, or the length of an array.""" + """What the bound limits: a number, an integer, or the length of an array or a string.""" _CONSTRAINTS: Final[Mapping[str, _Constraint]] = { @@ -888,10 +890,18 @@ class _Constraint: "integer", ), "min_length": _Constraint( - annotated_types.MinLen, {"tuple": "minItems"}, operator.ge, "below", "length" + annotated_types.MinLen, + {"tuple": "minItems", "str": "minLength"}, + operator.ge, + "below", + "length", ), "max_length": _Constraint( - annotated_types.MaxLen, {"tuple": "maxItems"}, operator.le, "above", "length" + annotated_types.MaxLen, + {"tuple": "maxItems", "str": "maxLength"}, + operator.le, + "above", + "length", ), } """The annotated-types constraints the checker reads, by the name pydantic uses, which is also the attribute of the marker that holds the bound.""" @@ -905,7 +915,7 @@ class _Constraint: _MEASURED: Final[Mapping[str, str]] = { "number": "a bound is on a number", "integer": "a step is on an integer", - "length": "a bound is on an array", + "length": "a bound is on an array or a string", } """For what each constraint limits, what an error message says it applies to.""" @@ -933,7 +943,7 @@ def constraints_of(metadata: Sequence[object]) -> dict[str, int | float]: The supported constraints are annotated-types' `Gt`, `Ge`, `Lt` and `Le`, which bound a number, `MultipleOf`, which is a step an integer is a multiple of, `MinLen` and `MaxLen`, which bound the length of an - array, and an `Interval` or a `Len`, which unpack to them, as pydantic + array or a string, and an `Interval` or a `Len`, which unpack to them, as pydantic reads them. A note (a string or a `Doc`) does not constrain the value and is skipped. Anything else raises `TypeError`: @@ -1019,7 +1029,7 @@ def _length(item: object, name: str) -> int: def _constrainable(inner: object, said: Constraints) -> None: - """Raise `TypeError` for a constraint on a type it does not limit: a bound on what is not a number, a step on what is not an integer, or a length bound on what is not an array.""" + """Raise `TypeError` for a constraint on a type it does not limit: a bound on what is not a number, a step on what is not an integer, or a length bound on what is not an array or a string.""" shape = shape_of(inner) for of, applies in _MEASURED.items(): names = [name for name in said if _CONSTRAINTS[name].of == of] @@ -1028,11 +1038,15 @@ def _constrainable(inner: object, said: Constraints) -> None: raise TypeError(msg) -def constrained(parse: Parser, description: str, said: Constraints) -> Parser: +def constrained( + parse: Parser, description: str, said: Constraints, unit: str = "element" +) -> Parser: """Wrap `parse` to also check the constraints its type carries: one `invalid_value` problem if any fails. A bound limits a number, a step what an integer is a multiple of, and - a length bound the number of elements in an array. The message states + a length bound the number of `unit`s in the value: the elements of an + array, or the characters of a string, which are its code points, as + JSON Schema counts them. The message states what the type allows, for example "expected an integer in [0, 9], got 12", "expected an integer > 0 that is a multiple of 4, got 6" or "expected an array of integers with at least 1 element, got []", and @@ -1041,7 +1055,7 @@ def constrained(parse: Parser, description: str, said: Constraints) -> Parser: checked only when `parse` reports no problem, in the same way that rules run only on a value of their type. """ - expected = f"expected {description} {_admitted(said)}" + expected = f"expected {description} {_admitted(said, unit)}" ctx: dict[str, JSONValue] = dict(said) checks = tuple( (_CONSTRAINTS[name].holds, bound, _CONSTRAINTS[name].of == "length") @@ -1053,7 +1067,7 @@ def parse_constrained(value: object, loc: Loc) -> Parsed: if len(found) != 0: return typed, found for keeps, bound, length in checks: - measured = len(cast("tuple[object, ...]", typed)) if length else cast("float", typed) + measured = len(cast("Sized", typed)) if length else cast("float", typed) if not keeps(measured, bound): message = f"{expected}, got {shown(value)}" return typed, (ValidationProblem(loc, message, "invalid_value", ctx=ctx),) @@ -1062,10 +1076,10 @@ def parse_constrained(value: object, loc: Loc) -> Parsed: return parse_constrained -def _admitted(said: Constraints) -> str: +def _admitted(said: Constraints, unit: str) -> str: """What the constraints allow, as written after the type in a message: "in [0, 9]", ">= 1", "> 0 that is a multiple of 4", "with at least 1 element".""" if any(_CONSTRAINTS[name].of == "length" for name in said): - return _admitted_lengths(said.get("min_length"), said.get("max_length")) + return _admitted_lengths(said.get("min_length"), said.get("max_length"), unit) low = next( ((name, bound) for name, bound in said.items() if _CONSTRAINTS[name].slot == "below"), None ) @@ -1095,20 +1109,20 @@ def _admitted_range( return "" -def _admitted_lengths(low: float | None, high: float | None) -> str: - """The lengths that length bounds allow, as written after the type in a message: "with at least 1 element", "with 1 to 3 elements".""" +def _admitted_lengths(low: float | None, high: float | None, unit: str) -> str: + """The lengths that length bounds allow, counted in `unit`s, as written after the type in a message: "with at least 1 element", "with 1 to 3 characters".""" if low is not None and high is not None: if low == high: - return f"with {_elements(low)}" - return f"with {shown(low)} to {_elements(high)}" + return f"with {_counted(low, unit)}" + return f"with {shown(low)} to {_counted(high, unit)}" if low is not None: - return f"with at least {_elements(low)}" - return f"with at most {_elements(cast('float', high))}" + return f"with at least {_counted(low, unit)}" + return f"with at most {_counted(cast('float', high), unit)}" -def _elements(count: float) -> str: - """A number of elements, as a message writes it: "1 element", "3 elements".""" - return f"{shown(count)} element{'' if count == 1 else 's'}" +def _counted(count: float, unit: str) -> str: + """A number of `unit`s, as a message writes it: "1 element", "3 characters".""" + return f"{shown(count)} {unit}{'' if count == 1 else 's'}" # --- the compiler -------------------------------------------------------- @@ -1279,7 +1293,8 @@ def _compile(annotation: object, leaf: Leaf, building: _Building) -> Parser | No if len(said) == 0: return parse _constrainable(inner, said) - return constrained(parse, describe(inner), said) + unit = "character" if shape_of(inner) == "str" else "element" + return constrained(parse, describe(inner), said, unit) def _compile_type(inner: object, leaf: Leaf, building: _Building) -> Parser | None: @@ -1553,7 +1568,8 @@ def of(self, annotation: object) -> JSONSchema: """The schema of `annotation` as the checker reads it: its type, with the constraints and notes from `Annotated`. Each constraint becomes the matching JSON Schema keyword (`Ge(0)` - becomes `minimum`, `MinLen(1)` becomes `minItems`), and a `Doc` + becomes `minimum`, `MinLen(1)` becomes `minItems` on an array and + `minLength` on a string), and a `Doc` becomes the `description`. The schema keeps both the type's bounds and the bounds of its `NewType`, or of the fixed length of its tuple, because the checker applies both. When both set the same @@ -1724,7 +1740,8 @@ def json_schema(shape: type) -> JSONSchema: - A TypedDict becomes an object with its keys, its required keys, and what any other key may hold (nothing, for a closed TypedDict). - A constraint becomes the matching JSON Schema keyword: `Ge(0)` - becomes `minimum`, and `MinLen(1)` becomes `minItems`. + becomes `minimum`, and `MinLen(1)` becomes `minItems` on an array + and `minLength` on a string. - A `Doc` becomes the `description`. Only a `Doc` produces a description; docstrings are for Python readers and are not used. This matches zod, which writes only what `.describe()` sets. diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 5172f33443..f259f3d7fa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -164,8 +164,8 @@ def acme_lz4_rules( pydantic reads them: `Gt`, `Ge`, `Lt`, `Le` and `Interval`, at most one from each side, at any depth. For example, `tuple[Annotated[int, Ge(1)], ...]` bounds each element. An integer's -type may carry a step, `MultipleOf`, at most one. An array's type may -bound its length with `MinLen`, `MaxLen` and `Len`, so +type may carry a step, `MultipleOf`, at most one. An array's or a +string's type may bound its length with `MinLen`, `MaxLen` and `Len`, so `Annotated[tuple[int, ...], MinLen(1)]` is never empty. A value out of bounds is an `invalid_value` problem. Its message says what the type admits ("expected an integer >= 1, got 0"), and its `ctx` holds the diff --git a/packages/zarr-metadata/src/zarr_metadata/typed_json.py b/packages/zarr-metadata/src/zarr_metadata/typed_json.py index b18c4a2f4e..b98d392927 100644 --- a/packages/zarr-metadata/src/zarr_metadata/typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/typed_json.py @@ -43,9 +43,10 @@ depth. For example, `tuple[Annotated[int, Ge(1)], ...]` bounds each element. An integer type may also carry a step, `MultipleOf`: `Annotated[int, Gt(0), MultipleOf(4)]` is a positive multiple of 4. An - array type may bound its length the same way, with `MinLen`, `MaxLen` - and `Len`: `Annotated[tuple[int, ...], MinLen(1)]` is a non-empty - array. A value outside the bounds is an `invalid_value` + array or a string type may bound its length the same way, with + `MinLen`, `MaxLen` and `Len`: `Annotated[tuple[int, ...], MinLen(1)]` + is a non-empty array. A string's length counts its code points, as + JSON Schema counts them. A value outside the bounds is an `invalid_value` problem. Its message states what the type allows, and its `ctx` holds the bounds, as pydantic names them (`ge`, `min_length`). `Annotated` may also carry a note (a string or a `Doc`). Any other diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py index dd3bed411b..69f304bb0c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py @@ -622,8 +622,8 @@ class NumpyTimedelta64(TypedDict, closed=True): class StructField(TypedDict, closed=True): """A single field entry inside a structured dtype.""" - name: ReadOnly[str] - """The field name (must be unique within a struct and non-empty).""" + name: ReadOnly[Annotated[str, MinLen(1)]] + """The field name: non-empty, and unique within the struct.""" data_type: ReadOnly[ZarrV3DataTypeDeclaration] """The field's data type. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 653d5f33d6..9034b80443 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -32,7 +32,7 @@ def _rules(configuration: StructConfiguration, nested: Nested) -> Iterator[ValidationProblem]: - """Check that field names are non-empty and distinct, and that field types have a fixed size. + """Check that field names are distinct, and that field types have a fixed size. Names must be distinct because a fill value addresses fields by name. "Variable-length data types @@ -46,10 +46,6 @@ def _rules(configuration: StructConfiguration, nested: Nested) -> Iterator[Valid seen: dict[str, int] = {} for index, member in enumerate(fields): name = member["name"] - if name == "": - yield ValidationProblem( - ("fields", index, "name"), "expected a non-empty field name", "invalid_value" - ) first = seen.setdefault(name, index) if first != index: yield ValidationProblem( diff --git a/packages/zarr-metadata/tests/test_json_schema.py b/packages/zarr-metadata/tests/test_json_schema.py index 368d6c1e3d..5e7f50c240 100644 --- a/packages/zarr-metadata/tests/test_json_schema.py +++ b/packages/zarr-metadata/tests/test_json_schema.py @@ -175,6 +175,11 @@ def _held(member: dict[str, Any], defs: dict[str, Any] | None = None) -> dict[st _holding(Annotated[int, Gt(0), MultipleOf(4)]), _held({"type": "integer", "exclusiveMinimum": 0, "multipleOf": 4}), ), + ( + "non-empty-string", + _holding(Annotated[str, MinLen(1)]), + _held({"type": "string", "minLength": 1}), + ), ( "non-empty-array", _holding(Annotated[tuple[int, ...], MinLen(1)]), diff --git a/packages/zarr-metadata/tests/test_typed_json.py b/packages/zarr-metadata/tests/test_typed_json.py index 09a2b23572..a5c7beb1f3 100644 --- a/packages/zarr-metadata/tests/test_typed_json.py +++ b/packages/zarr-metadata/tests/test_typed_json.py @@ -1138,6 +1138,26 @@ class Noted(TypedDict, extra_items=Annotated[object, "any JSON"]): ) ], ), + (Annotated[str, MinLen(1)], "a", []), + ( + Annotated[str, MinLen(1)], + "", + [((), 'expected a string with at least 1 character, got ""', {"min_length": 1})], + ), + ( + Annotated[str, Len(2, 2)], + "abc", + [ + ( + (), + 'expected a string with 2 characters, got "abc"', + {"min_length": 2, "max_length": 2}, + ) + ], + ), + # One code point outside the Basic Multilingual Plane is one + # character, as JSON Schema's `maxLength` counts it. + (Annotated[str, MaxLen(1)], "\U0001d11e", []), ], ids=[ "within", @@ -1167,6 +1187,10 @@ class Noted(TypedDict, extra_items=Annotated[object, "any JSON"]): "a-length-on-a-fixed-length-array", "a-length-checked-only-once-the-elements-are-read", "the-branch-it-bounds-the-length-of", + "a-string-length-within", + "a-string-length-below", + "an-exact-string-length", + "a-string-length-counts-code-points", ], ) def test_a_value_is_held_to_the_bounds_its_type_carries( @@ -1213,21 +1237,26 @@ def test_error_a_bound_on_what_is_not_a_number() -> None: @pytest.mark.parametrize( ("annotation", "named"), [ - (Annotated[str, MinLen(1)], "min_length: a bound is on an array, and a string is not one"), + ( + Annotated[bool, MinLen(1)], + "min_length: a bound is on an array or a string, and a boolean is not one", + ), ( Annotated[Mapping[str, int], MaxLen(1)], - "max_length: a bound is on an array, and an object is not one", + "max_length: a bound is on an array or a string, and an object is not one", ), ( Annotated[int, Len(1, 2)], - "min_length, max_length: a bound is on an array, and an integer is not one", + "min_length, max_length: a bound is on an array or a string, and an integer is not one", ), ], - ids=["min-len-on-a-string", "max-len-on-an-object", "len-on-a-number"], + ids=["min-len-on-a-boolean", "max-len-on-an-object", "len-on-a-number"], ) -def test_error_a_length_bound_on_what_is_not_an_array(annotation: object, named: str) -> None: - # pydantic also bounds the length of a string or an object; the checker - # bounds only an array's, so the type cannot promise more than it checks. +def test_error_a_length_bound_on_what_is_not_an_array_or_a_string( + annotation: object, named: str +) -> None: + # pydantic also bounds the number of an object's keys; the checker does + # not, so the type cannot promise more than it checks. with pytest.raises(TypeError, match=re.escape(named)): parser(annotation, no_leaf) From 4ad0230af31f4c1367de3002602211d64a357a20 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 19:37:30 +0100 Subject: [PATCH 09/13] chore(zarr-metadata): number the changelog fragments by this PR Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/changes/{4635.bugfix.md => 4636.bugfix.md} | 0 .../zarr-metadata/changes/{4635.feature.md => 4636.feature.md} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{4635.bugfix.md => 4636.bugfix.md} (100%) rename packages/zarr-metadata/changes/{4635.feature.md => 4636.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/4635.bugfix.md b/packages/zarr-metadata/changes/4636.bugfix.md similarity index 100% rename from packages/zarr-metadata/changes/4635.bugfix.md rename to packages/zarr-metadata/changes/4636.bugfix.md diff --git a/packages/zarr-metadata/changes/4635.feature.md b/packages/zarr-metadata/changes/4636.feature.md similarity index 100% rename from packages/zarr-metadata/changes/4635.feature.md rename to packages/zarr-metadata/changes/4636.feature.md From e0e6bd4a2377ce34aee55393fb865eb96cc48f99 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 14:47:37 +0100 Subject: [PATCH 10/13] test(zarr-metadata): loosen the floor on documents with an axis past 32 bits The derandomized share stream changes with any change to what the strategies draw, and with machine load: base main measures 0.103 against a floor of 0.1 under load, and the rectilinear fix measures 0.047 to 0.133 across runs. Over seeds 0-5 at 500 examples both trees average about 0.17, from 0.07 to 0.29. A floor of 0.03 still asks that a few hundred examples meet several such axes. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/tests/testing/test_documents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/zarr-metadata/tests/testing/test_documents.py b/packages/zarr-metadata/tests/testing/test_documents.py index 6c20a3c29f..e9393c549d 100644 --- a/packages/zarr-metadata/tests/testing/test_documents.py +++ b/packages/zarr-metadata/tests/testing/test_documents.py @@ -343,7 +343,7 @@ def test_a_data_type_no_codec_takes_is_not_drawn(document: Mapping[str, Any]) -> "no axes": (lambda d: d["shape"] == (), 0.03, 0.25), "more axes than five": (lambda d: len(d["shape"]) > 5, 0.1, 1), "a zero-length axis": (lambda d: 0 in d["shape"], 0.1, 1), - "an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.1, 1), + "an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.03, 1), "the first data type": (lambda d: _name(d["data_type"]) == "bool", 0, 0.12), } """What a share of array documents holds, as a low and a high bound: what a test running only a few hundred examples meets.""" From 93a92106e18b5a7aa6a33083ea429c7b8917fa2a Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 20:02:58 +0100 Subject: [PATCH 11/13] fix(zarr-metadata): hold a length bound beside an unknown key `constrained` skipped a type's constraints whenever the inner parse reported any problem. An `unknown_key` problem leaves the value readable, because the key is dropped and the value is still returned. So `check` returned an array of closed TypedDicts longer than its `MaxLen`, and a definition's rules ran on it, although the rules run only on a configuration within its bounds. The constraints are now skipped only for a problem other than an unknown key. Number bounds were never affected, because a number holds no keys. The docs now list `multiple_of`, `min_length` and `max_length` among the `ctx` keys, and `constraints_of` says a type takes one step too. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../zarr-metadata/src/zarr_metadata/_json.py | 8 +++--- .../src/zarr_metadata/_typed_json.py | 25 ++++++++++--------- .../zarr-metadata/tests/test_typed_json.py | 22 ++++++++++++++++ 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/_json.py b/packages/zarr-metadata/src/zarr_metadata/_json.py index d5de1537a2..f55081b1f6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_json.py @@ -127,9 +127,11 @@ class ValidationProblem: document, `input` shows the change. `ctx` holds what was expected, when that is more than a type: - - `gt`, `ge`, `lt` and `le`: the bounds from the value's type or from - a rule, with the names pydantic uses. For example `{"ge": 0, "le": 9}` - for a gzip `level`, whose type is `Annotated[int, Interval(ge=0, le=9)]`. + - `gt`, `ge`, `lt`, `le`, `multiple_of`, `min_length` and + `max_length`: the constraints from the value's type or from a rule, + with the names pydantic uses. For example `{"ge": 0, "le": 9}` for a + gzip `level`, whose type is `Annotated[int, Interval(ge=0, le=9)]`, + or `{"min_length": 1}` for an array document's `codecs`. - `expected`: the values of a closed set, like zod's `values`, in the order the message lists them. For example the values of a `Literal`, of `node_type`, or of `zarr_format`. diff --git a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py index 516a41e149..b774ddd586 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py @@ -938,7 +938,7 @@ class _Constraint: def constraints_of(metadata: Sequence[object]) -> dict[str, int | float]: - """The constraints in an `Annotated` type's metadata, by name: at most one lower and one upper bound. + """The constraints in an `Annotated` type's metadata, by name: at most one lower bound, one upper bound and one step. The supported constraints are annotated-types' `Gt`, `Ge`, `Lt` and `Le`, which bound a number, `MultipleOf`, which is a step an integer @@ -1046,14 +1046,15 @@ def constrained( A bound limits a number, a step what an integer is a multiple of, and a length bound the number of `unit`s in the value: the elements of an array, or the characters of a string, which are its code points, as - JSON Schema counts them. The message states - what the type allows, for example "expected an integer in [0, 9], got - 12", "expected an integer > 0 that is a multiple of 4, got 6" or - "expected an array of integers with at least 1 element, got []", and - the problem's `ctx` - holds the constraints, as pydantic names them. The constraints are - checked only when `parse` reports no problem, in the same way that - rules run only on a value of their type. + JSON Schema counts them. The message states what the type allows, for + example "expected an integer in [0, 9], got 12", "expected an integer + > 0 that is a multiple of 4, got 6" or "expected an array of integers + with at least 1 element, got []", and the problem's `ctx` holds the + constraints, as pydantic names them. The constraints are checked only + on a value of the type, in the same way that rules are: when `parse` + reports no problem, or only keys that a closed TypedDict does not + declare. Such a key is left out and the value is still returned, so + the value it returns is held to the constraints too. """ expected = f"expected {description} {_admitted(said, unit)}" ctx: dict[str, JSONValue] = dict(said) @@ -1064,14 +1065,14 @@ def constrained( def parse_constrained(value: object, loc: Loc) -> Parsed: typed, found = parse(value, loc) - if len(found) != 0: + if any(each.kind != "unknown_key" for each in found): return typed, found for keeps, bound, length in checks: measured = len(cast("Sized", typed)) if length else cast("float", typed) if not keeps(measured, bound): message = f"{expected}, got {shown(value)}" - return typed, (ValidationProblem(loc, message, "invalid_value", ctx=ctx),) - return typed, () + return typed, (*found, ValidationProblem(loc, message, "invalid_value", ctx=ctx)) + return typed, found return parse_constrained diff --git a/packages/zarr-metadata/tests/test_typed_json.py b/packages/zarr-metadata/tests/test_typed_json.py index a5c7beb1f3..7684d07287 100644 --- a/packages/zarr-metadata/tests/test_typed_json.py +++ b/packages/zarr-metadata/tests/test_typed_json.py @@ -1014,6 +1014,10 @@ class Noted(TypedDict, extra_items=Annotated[object, "any JSON"]): level: int +class _Item(TypedDict, closed=True): + a: int + + @pytest.mark.parametrize( ("annotation", "value", "found"), [ @@ -1155,6 +1159,23 @@ class Noted(TypedDict, extra_items=Annotated[object, "any JSON"]): ) ], ), + # A key the element's closed TypedDict does not declare is left out + # and the value is still returned, so its length is held too. + ( + Annotated[tuple[_Item, ...], MaxLen(1)], + [{"a": 1, "zzz": 0}, {"a": 2}], + [ + ((0, "zzz"), "unexpected key 'zzz'", {}), + ( + (), + ( + "expected an array of objects with at most 1 element, got " + '[{"a": 1, "zzz": 0}, {"a": 2}]' + ), + {"max_length": 1}, + ), + ], + ), # One code point outside the Basic Multilingual Plane is one # character, as JSON Schema's `maxLength` counts it. (Annotated[str, MaxLen(1)], "\U0001d11e", []), @@ -1190,6 +1211,7 @@ class Noted(TypedDict, extra_items=Annotated[object, "any JSON"]): "a-string-length-within", "a-string-length-below", "an-exact-string-length", + "a-length-held-beside-an-unknown-key", "a-string-length-counts-code-points", ], ) From 5294e1771b1795bc9e8c8b8ef07045d86eb4f778 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 20:03:00 +0100 Subject: [PATCH 12/13] fix(zarr-metadata): name the member a carried name's type check refuses A raw `r` name's size is now checked by its type. For `"data_type": "r12"` that gave "expected an integer > 0 that is a multiple of 8, got 12", about a string the document wrote, with no mention of bits. A type-check problem with a configuration that a name encodes now names the member and the name: `bits in "r12": ...`. A rule's message is left as written, because such rules already describe the name, as the v2 typestr rules do. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../zarr_metadata/definition/_definition.py | 21 ++++++++++++++++++- .../zarr-metadata/tests/test_problem_data.py | 5 +++-- .../tests/v3/test_every_definition.py | 17 +++++++++++---- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 933f2f7f55..05eefcf761 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1717,6 +1717,10 @@ def _read_carried( declared = {key: value for key, value in carried.items() if key in members} undeclared = [key for key in carried if key not in members] configuration, judged = definition.read_configuration(declared) + # The type check describes a configuration member, which the document + # never wrote, so its messages name the member and the name it was read + # from. A rule of such a definition writes its messages for the name. + _, typed = definition._check_configuration(declared) # A problem with the configuration encoded in the name belongs to the # declaration. It is located at the declaration, where the name is, and the # `input` and `ctx` that described a configuration member do not apply @@ -1725,7 +1729,14 @@ def _read_carried( problems = ( *beside, *( - dataclasses.replace(found, loc=loc, input=UNSET, ctx={}, kind="invalid_value") + dataclasses.replace( + found, + loc=loc, + input=UNSET, + ctx={}, + kind="invalid_value", + message=_read_from(name, found) if found in typed else found.message, + ) for found in judged ), ) @@ -1741,6 +1752,14 @@ def _read_carried( ), problems +def _read_from(name: str, found: ValidationProblem) -> str: + """The message of `found`, a problem the type check found in the configuration that `name` encodes, naming the member it is about and the name: `bits in "r12": expected ...`.""" + if len(found.loc) == 0: + return f"{shown(name)}: {found.message}" + member = ".".join(str(part) for part in found.loc) + return f"{member} in {shown(name)}: {found.message}" + + def _sized(declaration: _NestedDeclaration, inner: ResolvedDeclaration[Any]) -> Problems: """The problems with the definition that read `inner` in the member that holds `declaration`, as the kind of `declaration` reports them for its alias. diff --git a/packages/zarr-metadata/tests/test_problem_data.py b/packages/zarr-metadata/tests/test_problem_data.py index 91d9d01412..34d2d79799 100644 --- a/packages/zarr-metadata/tests/test_problem_data.py +++ b/packages/zarr-metadata/tests/test_problem_data.py @@ -315,7 +315,8 @@ def test_a_problem_with_what_a_name_carries_is_the_declaration_s() -> None: # `r24` carries `{"bits": 24}`, which this scope's raw-bits definition # refuses. The problem is located at the declaration, because the name is what # the document wrote there. The bound on `bits` is not a bound on the - # name, so the problem's `ctx` is empty. + # name, so the problem's `ctx` is empty, and its message names the + # member and the name it was read from. context = CORE_AND_EXTENSIONS_CONTEXT.copy( update={ "data_types": { @@ -326,7 +327,7 @@ def test_a_problem_with_what_a_name_carries_is_the_declaration_s() -> None: ) _, problems = resolve("r24", DataTypeDefinition, context, ("data_type",)) assert [(p.loc, p.input, dict(p.ctx)) for p in problems] == [(("data_type",), "r24", {})] - assert problems[0].message == "expected an integer <= 16, got 24" + assert problems[0].message == 'bits in "r24": expected an integer <= 16, got 24' class Ordered(TypedDict, closed=True): diff --git a/packages/zarr-metadata/tests/v3/test_every_definition.py b/packages/zarr-metadata/tests/v3/test_every_definition.py index 86a3e4a2e2..c72161d85c 100644 --- a/packages/zarr-metadata/tests/v3/test_every_definition.py +++ b/packages/zarr-metadata/tests/v3/test_every_definition.py @@ -907,12 +907,21 @@ def test_error_struct_field_type_is_judged_where_it_sits() -> None: ] -@pytest.mark.parametrize("name", ["r12", "r0"]) -def test_error_a_raw_bytes_width_is_not_a_positive_multiple_of_8(name: str) -> None: +@pytest.mark.parametrize( + ("name", "message"), + [ + ("r12", 'bits in "r12": expected an integer > 0 that is a multiple of 8, got 12'), + ("r0", 'bits in "r0": expected an integer > 0 that is a multiple of 8, got 0'), + ], +) +def test_error_a_raw_bytes_width_is_not_a_positive_multiple_of_8(name: str, message: str) -> None: # Raw bits of a size the spec does not allow are reported, not left # unchecked. The document wrote a name, so the problem is located at - # the declaration. - assert _problems("data_type:r*", name) == [((), "invalid_value")] + # the declaration, and its message names the size the name spells. + _, problems = resolve(name, DataTypeDefinition, ZARR_PYTHON_CONTEXT) + assert [(found.loc, found.kind, found.message) for found in problems] == [ + ((), "invalid_value", message) + ] def test_error_a_configuration_written_beside_a_raw_bits_name() -> None: From 95eaf0141fc700d3e06d10b0ac435d6b9c4360e8 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 14 Nov 2026 20:03:01 +0100 Subject: [PATCH 13/13] test(zarr-metadata): pin that the types refuse empty pipelines and edge lists The sharding tests for an empty `codecs` or `index_codecs` checked only the `loc` and kind, which the pipeline's order check also reports. They now also assert `ctx={"min_length": 1}`, which only the type check gives. The rectilinear strategy test checked the grid's rules, which no longer look at empty lists. It now reads the configuration with the definition, which checks the type. The changelog no longer says the JSON Schema of a raw `r` name carries its multiple of 8: that schema is the name pattern. Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- packages/zarr-metadata/changes/4636.feature.md | 2 +- .../tests/testing/test_strategies.py | 4 +++- packages/zarr-metadata/tests/v3/test_sharding.py | 16 ++++++++++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/zarr-metadata/changes/4636.feature.md b/packages/zarr-metadata/changes/4636.feature.md index 5ff0bcb2e5..aed68bd9d0 100644 --- a/packages/zarr-metadata/changes/4636.feature.md +++ b/packages/zarr-metadata/changes/4636.feature.md @@ -1 +1 @@ -`zarr_metadata.typed_json.check` and definitions' configurations now read more annotated-types constraints, as pydantic does: `MinLen`, `MaxLen` and `Len` on an array or a string, and `MultipleOf` on an integer. A value outside them is an `invalid_value` problem whose `ctx` uses pydantic's names (`min_length`, `multiple_of`), and the JSON Schema writes the matching keyword, such as `minItems`, `minLength` or `multipleOf`. A constraint on a type it does not apply to raises `TypeError`. Constant rules now live on the types, so the JSON Schemas say them too: non-empty `codecs`, sharding `codecs` and `index_codecs`, rectilinear edge lists, `struct` fields and v3 `struct` field names; `length_bytes` of at least 1 for `null_terminated_bytes` and zarr-python `raw_bytes`; and positive multiples of 4 for `fixed_length_utf32`'s `length_bytes` and of 8 for raw `r` bits. `check` now also refuses `"codecs": []`. +`zarr_metadata.typed_json.check` and definitions' configurations now read more annotated-types constraints, as pydantic does: `MinLen`, `MaxLen` and `Len` on an array or a string, and `MultipleOf` on an integer. A value outside them is an `invalid_value` problem whose `ctx` uses pydantic's names (`min_length`, `multiple_of`), and the JSON Schema writes the matching keyword, such as `minItems`, `minLength` or `multipleOf`. A constraint on a type it does not apply to raises `TypeError`. Constant rules now live on the types, so the JSON Schemas say them too: non-empty `codecs`, sharding `codecs` and `index_codecs`, rectilinear edge lists, `struct` fields and v3 `struct` field names; `length_bytes` of at least 1 for `null_terminated_bytes` and zarr-python `raw_bytes`, and a positive multiple of 4 for `fixed_length_utf32`. Raw `r` declares its bits a positive multiple of 8 on its type, and a refused name's message now names them: `bits in "r12": ...`. `check` now also refuses `"codecs": []`. diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index bd16ca573f..4c38a4a104 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -746,8 +746,10 @@ def test_rectilinear_dimensions_are_never_empty( ) -> None: """A list of edges has at least one, even along a dimension of length 0 or of a length not known.""" configuration = data.draw(chunk_grid_configurations(RECTILINEAR_CHUNK_GRID, shape)) + # The type declares each list non-empty, so the definition's read of + # the configuration, which checks the type, is what would refuse one. assert () not in configuration["chunk_shapes"] - assert list(RECTILINEAR_CHUNK_GRID.rules(configuration, {})) == [] + assert RECTILINEAR_CHUNK_GRID.read_configuration(configuration)[1] == () @given(data=st.data(), shape=st.lists(st.integers(0, 2**63 - 1), max_size=3).map(tuple)) diff --git a/packages/zarr-metadata/tests/v3/test_sharding.py b/packages/zarr-metadata/tests/v3/test_sharding.py index 8585adf089..ae6336a979 100644 --- a/packages/zarr-metadata/tests/v3/test_sharding.py +++ b/packages/zarr-metadata/tests/v3/test_sharding.py @@ -206,8 +206,6 @@ def test_error_an_inner_chunk_length_that_does_not_divide_the_shard( ("shard", "found"), [ # Each pipeline is read as the array's pipeline is, at its own location. - (_shard([4, 4], codecs=[]), [((0, "configuration", "codecs"), "invalid_value")]), - (_shard([4, 4], index=[]), [((0, "configuration", "index_codecs"), "invalid_value")]), ( _shard([4, 4], codecs=["crc32c", LITTLE]), [((0, "configuration", "codecs", 1), "invalid_value")], @@ -234,6 +232,20 @@ def test_error_a_pipeline_a_shard_holds_that_does_not_fit_what_it_is_handed( assert _problems([shard], _chunk({8}, {8})) == found +@pytest.mark.parametrize( + ("shard", "key"), + [(_shard([4, 4], codecs=[]), "codecs"), (_shard([4, 4], index=[]), "index_codecs")], + ids=["codecs", "index-codecs"], +) +def test_error_a_shard_s_empty_pipeline_is_refused_by_its_type(shard: JSONValue, key: str) -> None: + # Each pipeline's type declares it non-empty, so the type check reports + # an empty one, before the pipeline's order check would. + problems = _read([shard], _chunk({8}, {8}))[1] + assert [(p.loc, p.kind, dict(p.ctx)) for p in problems] == [ + ((0, "configuration", key), "invalid_value", {"min_length": 1}) + ] + + def test_error_an_array_document_s_shard_is_read_where_it_sits() -> None: document = dict(ZarrV3ArrayMetadata.create_default(shape=(16, 16)).to_json()) document["chunk_grid"] = {"name": "regular", "configuration": {"chunk_shape": [8, 6]}}