diff --git a/packages/zarr-metadata/changes/4635.bugfix.md b/packages/zarr-metadata/changes/4635.bugfix.md new file mode 100644 index 0000000000..d92c9b5193 --- /dev/null +++ b/packages/zarr-metadata/changes/4635.bugfix.md @@ -0,0 +1 @@ +Strategies in `zarr_metadata.testing.strategies` that draw codec pipelines, such as `array_documents` and `pipelines`, now draw the same examples on every derandomized Hypothesis run. A codec was sampled from options that hold strategies built during the draw, which Hypothesis labels by memory address, so the examples its mutator produced varied from one process to the next. diff --git a/packages/zarr-metadata/changes/4636.bugfix.md b/packages/zarr-metadata/changes/4636.bugfix.md new file mode 100644 index 0000000000..ac16d99ecc --- /dev/null +++ b/packages/zarr-metadata/changes/4636.bugfix.md @@ -0,0 +1 @@ +The `rectilinear` chunk grid now refuses an empty list of chunk edge lengths, such as `"chunk_shapes": [[]]`, even along a dimension of length 0. zarr-extensions#78 made the list form non-empty; a dimension with no chunks writes a bare integer instead. The JSON Schema says `minItems: 1` for the list, and `chunk_grid_configurations` no longer draws an empty list. diff --git a/packages/zarr-metadata/changes/4636.feature.md b/packages/zarr-metadata/changes/4636.feature.md new file mode 100644 index 0000000000..aed68bd9d0 --- /dev/null +++ b/packages/zarr-metadata/changes/4636.feature.md @@ -0,0 +1 @@ +`zarr_metadata.typed_json.check` and definitions' configurations now read more annotated-types constraints, as pydantic does: `MinLen`, `MaxLen` and `Len` on an array or a string, and `MultipleOf` on an integer. A value outside them is an `invalid_value` problem whose `ctx` uses pydantic's names (`min_length`, `multiple_of`), and the JSON Schema writes the matching keyword, such as `minItems`, `minLength` or `multipleOf`. A constraint on a type it does not apply to raises `TypeError`. Constant rules now live on the types, so the JSON Schemas say them too: non-empty `codecs`, sharding `codecs` and `index_codecs`, rectilinear edge lists, `struct` fields and v3 `struct` field names; `length_bytes` of at least 1 for `null_terminated_bytes` and zarr-python `raw_bytes`, and a positive multiple of 4 for `fixed_length_utf32`. Raw `r` declares its bits a positive multiple of 8 on its type, and a refused name's message now names them: `bits in "r12": ...`. `check` now also refuses `"codecs": []`. diff --git a/packages/zarr-metadata/docs/extensions.md b/packages/zarr-metadata/docs/extensions.md index 57acdeffbd..fc314760c8 100644 --- a/packages/zarr-metadata/docs/extensions.md +++ b/packages/zarr-metadata/docs/extensions.md @@ -82,8 +82,10 @@ names `acme.lz4` is then checked against this definition. - a `NewType` or a type alias. A number may carry bounds from annotated-types: `Gt`, `Ge`, `Lt`, `Le` or -`Interval`. Any other `Annotated` metadata is refused when the definition -is built. +`Interval`, and an integer a step, `MultipleOf`. An array or a string may +bound its length with `MinLen`, `MaxLen` or `Len`, so +`Annotated[tuple[int, ...], MinLen(1)]` is never empty. Any other +`Annotated` metadata is refused when the definition is built. **The rules function** diff --git a/packages/zarr-metadata/src/zarr_metadata/_json.py b/packages/zarr-metadata/src/zarr_metadata/_json.py index d5de1537a2..f55081b1f6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_json.py @@ -127,9 +127,11 @@ class ValidationProblem: document, `input` shows the change. `ctx` holds what was expected, when that is more than a type: - - `gt`, `ge`, `lt` and `le`: the bounds from the value's type or from - a rule, with the names pydantic uses. For example `{"ge": 0, "le": 9}` - for a gzip `level`, whose type is `Annotated[int, Interval(ge=0, le=9)]`. + - `gt`, `ge`, `lt`, `le`, `multiple_of`, `min_length` and + `max_length`: the constraints from the value's type or from a rule, + with the names pydantic uses. For example `{"ge": 0, "le": 9}` for a + gzip `level`, whose type is `Annotated[int, Interval(ge=0, le=9)]`, + or `{"min_length": 1}` for an array document's `codecs`. - `expected`: the values of a closed set, like zod's `values`, in the order the message lists them. For example the values of a `Literal`, of `node_type`, or of `zarr_format`. diff --git a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py index c7b2074ea2..6e97d43d8d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/_pydantic_schema.py @@ -1,8 +1,8 @@ """Private input types used only to generate accurate Pydantic JSON schemas. These repeat the document TypedDicts in a stricter form: extension names -match the spec's pattern, `must_understand` is `true` at an extension -point, and `codecs` is not empty. They exist because pydantic builds a +match the spec's pattern, and `must_understand` is `true` at an extension +point. They exist because pydantic builds a schema from Python types and cannot embed a JSON Schema that has its own `$defs`, such as the ones `zarr_metadata.json_schema` writes; the v3 document schemas are recursive, so they cannot be inlined either. @@ -16,6 +16,7 @@ from collections.abc import Mapping # resolved by Pydantic at runtime from typing import Annotated, Literal, NotRequired +from annotated_types import MinLen from pydantic import Field from typing_extensions import TypedDict @@ -57,7 +58,7 @@ class ZarrV3MandatoryNamedConfigJSON(TypedDict, closed=True): ZarrV3DeclarationJSON = ExtensionName | ZarrV3NamedConfigJSON ZarrV3MandatoryDeclarationJSON = ExtensionName | ZarrV3MandatoryNamedConfigJSON -ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MandatoryDeclarationJSON, ...], Field(min_length=1)] +ZarrV3CodecPipelineJSON = Annotated[tuple[ZarrV3MandatoryDeclarationJSON, ...], MinLen(1)] ZarrV2FilterPipelineJSON = tuple[ZarrV2CodecMetadata, ...] diff --git a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py index e9574b46c4..b774ddd586 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/_typed_json.py @@ -9,9 +9,11 @@ `Mapping[str, V]`, a `NewType` (parsed as its supertype), and a type alias (parsed as the type it stands for). This short list keeps the module small. Any other annotation has no parser. A number type may carry bounds, -written with annotated-types as pydantic reads them, for example -`Annotated[int, Interval(ge=0, le=9)]`. A value outside the bounds is a -problem. `constraints_of` lists the bounds the checker supports. +an integer type a step, and an array or a string bounds on its length, written +with annotated-types as pydantic reads them, for example +`Annotated[int, Interval(ge=0, le=9)]`, `Annotated[int, MultipleOf(4)]` or +`Annotated[tuple[int, ...], MinLen(1)]`. A value outside the bounds is a +problem. `constraints_of` lists the constraints the checker supports. A TypedDict is read as the typing spec defines it. This is not always what its runtime attributes say. `typeddict_keys` reads which keys the @@ -86,6 +88,8 @@ ) if TYPE_CHECKING: + from collections.abc import Sized + from zarr_metadata._json import ProblemKind @@ -838,47 +842,118 @@ def parse(candidate: object, loc: Loc) -> Parsed: # --- constraints --------------------------------------------------------- Constraints: TypeAlias = Mapping[str, int | float] -"""The bounds a type carries, keyed by the names pydantic uses: `{"ge": 0, "le": 9}`.""" +"""The constraints a type carries, keyed by the names pydantic uses: `{"ge": 0, "le": 9}`, `{"min_length": 1}`.""" -_BOUNDS: Final[tuple[tuple[type, str], ...]] = ( - (annotated_types.Gt, "gt"), - (annotated_types.Ge, "ge"), - (annotated_types.Lt, "lt"), - (annotated_types.Le, "le"), -) -"""The annotated-types constraints the checker supports, each with its name.""" -_FROM_BELOW: Final = frozenset({"gt", "ge"}) -"""The lower bounds.""" +@dataclass(frozen=True, slots=True) +class _Constraint: + """A constraint the checker reads: its annotated-types marker, and how it is checked and written.""" + + marker: type + """The annotated-types class. Its attribute named as the constraint holds the bound.""" + keywords: Mapping[str | None, str] + """For each shape (from `shape_of`) the constraint applies to, JSON Schema's keyword for it there.""" + holds: Callable[[float, float], bool] + """Whether a measured value satisfies the bound.""" + slot: Literal["below", "above", "step"] + """Which of a type's constraints it is: a lower bound, an upper bound, or a step. A type takes one of each.""" + of: Literal["number", "integer", "length"] + """What the bound limits: a number, an integer, or the length of an array or a string.""" + + +_CONSTRAINTS: Final[Mapping[str, _Constraint]] = { + "gt": _Constraint( + annotated_types.Gt, + {"int": "exclusiveMinimum", "number": "exclusiveMinimum"}, + operator.gt, + "below", + "number", + ), + "ge": _Constraint( + annotated_types.Ge, {"int": "minimum", "number": "minimum"}, operator.ge, "below", "number" + ), + "lt": _Constraint( + annotated_types.Lt, + {"int": "exclusiveMaximum", "number": "exclusiveMaximum"}, + operator.lt, + "above", + "number", + ), + "le": _Constraint( + annotated_types.Le, {"int": "maximum", "number": "maximum"}, operator.le, "above", "number" + ), + "multiple_of": _Constraint( + annotated_types.MultipleOf, + {"int": "multipleOf"}, + lambda value, step: value % step == 0, + "step", + "integer", + ), + "min_length": _Constraint( + annotated_types.MinLen, + {"tuple": "minItems", "str": "minLength"}, + operator.ge, + "below", + "length", + ), + "max_length": _Constraint( + annotated_types.MaxLen, + {"tuple": "maxItems", "str": "maxLength"}, + operator.le, + "above", + "length", + ), +} +"""The annotated-types constraints the checker reads, by the name pydantic uses, which is also the attribute of the marker that holds the bound.""" -_FROM_ABOVE: Final = frozenset({"lt", "le"}) -"""The upper bounds.""" +_NAMES: Final = tuple(each.marker.__name__ for each in _CONSTRAINTS.values()) +"""The names of the markers the checker reads, in the order of `_CONSTRAINTS`.""" -_EXCLUSIVE: Final = frozenset({"gt", "lt"}) -"""The exclusive bounds: a value may not equal them.""" +_MARKERS: Final = f"{', '.join(_NAMES[:-1])} and {_NAMES[-1]}" +"""The markers the checker reads, as an error message lists them: "Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen".""" -_HOLDS: Final[Mapping[str, Callable[[float, float], bool]]] = { - "gt": operator.gt, - "ge": operator.ge, - "lt": operator.lt, - "le": operator.le, +_MEASURED: Final[Mapping[str, str]] = { + "number": "a bound is on a number", + "integer": "a step is on an integer", + "length": "a bound is on an array or a string", } -"""For each bound name, a test of whether a number satisfies that bound.""" +"""For what each constraint limits, what an error message says it applies to.""" +_SECOND: Final[Mapping[str, str]] = { + "below": "bound from below", + "above": "bound from above", + "step": "step", +} +"""For each slot, what an error message calls a second constraint in it.""" -def constraints_of(metadata: Sequence[object]) -> dict[str, int | float]: - """The bounds in an `Annotated` type's metadata, by name: at most one lower and one upper bound. +_STRICTER: Final[Mapping[str, Callable[[int | float, int | float], int | float]]] = { + "below": max, + "above": min, + "step": lambda one, other: math.lcm(int(one), int(other)), +} +"""For each slot, the constraint a value meets exactly when it meets both of two: the stricter bound, or the least common multiple of two steps.""" - The supported bounds are annotated-types' `Gt`, `Ge`, `Lt` and `Le`, - and an `Interval`, which unpacks to them, as pydantic reads them. A - note (a string or a `Doc`) does not constrain the value and is - skipped. Anything else raises `TypeError`: +_EXCLUSIVE: Final = frozenset({"gt", "lt"}) +"""The exclusive bounds: a value may not equal them.""" - - a constraint the checker does not support, such as `MinLen`, - `Predicate`, or pydantic's `Field`. Otherwise the type would declare - a rule that the checker does not enforce. - - a second bound on the same side. pydantic uses the last one. - - a bound that is not a finite number. + +def constraints_of(metadata: Sequence[object]) -> dict[str, int | float]: + """The constraints in an `Annotated` type's metadata, by name: at most one lower bound, one upper bound and one step. + + The supported constraints are annotated-types' `Gt`, `Ge`, `Lt` and + `Le`, which bound a number, `MultipleOf`, which is a step an integer + is a multiple of, `MinLen` and `MaxLen`, which bound the length of an + array or a string, and an `Interval` or a `Len`, which unpack to them, as pydantic + reads them. A note (a string or a `Doc`) does not constrain the value + and is skipped. Anything else raises `TypeError`: + + - a constraint the checker does not support, such as `Predicate` or + pydantic's `Field`. Otherwise the type would declare a rule that the + checker does not enforce. + - a second bound on the same side, or a second step. pydantic uses the + last one. + - a bound that is not a finite number, a step that is not a positive + integer, or a length bound that is not a non-negative integer. A bound is stored as the number it equals, as an integer when it is one. `Ge(True)`, `Ge(1.0)` and `Ge(1)` are the same bound, because @@ -889,16 +964,20 @@ def constraints_of(metadata: Sequence[object]) -> dict[str, int | float]: for item in _unpacked(metadata): if isinstance(item, (str, typing_extensions.Doc)): continue - name = next((name for kind, name in _BOUNDS if isinstance(item, kind)), None) + name = next( + (name for name, each in _CONSTRAINTS.items() if isinstance(item, each.marker)), None + ) if name is None: - msg = f"{item!r} is not a constraint the checker reads, which are Gt, Ge, Lt and Le" + msg = f"{item!r} is not a constraint the checker reads, which are {_MARKERS}" raise UnreadConstraintError(msg) - side = _FROM_BELOW if name in _FROM_BELOW else _FROM_ABOVE - if not side.isdisjoint(said): - where = "below" if side is _FROM_BELOW else "above" - msg = f"{item!r} is a second bound from {where}; a type takes one from each side" + constraint = _CONSTRAINTS[name] + if any( + _CONSTRAINTS[held].slot == constraint.slot and _CONSTRAINTS[held].of == constraint.of + for held in said + ): + msg = f"{item!r} is a second {_SECOND[constraint.slot]}; a type takes one of each" raise TypeError(msg) - said[name] = _bound(item, name) + said[name] = _read_bound(item, name, constraint.of) return said @@ -922,51 +1001,129 @@ def _bound(item: object, name: str) -> int | float: raise TypeError(msg) -def _constrainable(inner: object, said: Constraints) -> None: - """Raise `TypeError` for bounds on a type that is not a number: a string, an array, a value of any type.""" - if len(said) != 0 and shape_of(inner) not in ("int", "number"): - msg = f"{', '.join(said)}: a bound is on a number, and {describe(inner)} is not one" - raise TypeError(msg) +def _read_bound(item: object, name: str, of: str) -> int | float: + """The bound of `item` named `name`, as a constraint on what `of` names reads it; `TypeError` if it cannot be one.""" + if of == "number": + return _bound(item, name) + if of == "integer": + return _step(item, name) + return _length(item, name) -def constrained(parse: Parser, description: str, said: Constraints) -> Parser: - """Wrap `parse` to also check the bounds its type carries: one `invalid_value` problem if any bound fails. +def _step(item: object, name: str) -> int: + """The step of `item` named `name`; `TypeError` if it is not a positive integer, as JSON Schema requires of `multipleOf`.""" + bound = _bound(item, name) + if isinstance(bound, int) and bound > 0: + return bound + msg = f"{item!r}: a step is a positive integer" + raise TypeError(msg) + + +def _length(item: object, name: str) -> int: + """The length of length bound `item` named `name`; `TypeError` if it is not a non-negative integer.""" + bound = _bound(item, name) + if isinstance(bound, int) and bound >= 0: + return bound + msg = f"{item!r}: a length bound is a non-negative integer" + raise TypeError(msg) + + +def _constrainable(inner: object, said: Constraints) -> None: + """Raise `TypeError` for a constraint on a type it does not limit: a bound on what is not a number, a step on what is not an integer, or a length bound on what is not an array or a string.""" + shape = shape_of(inner) + for of, applies in _MEASURED.items(): + names = [name for name in said if _CONSTRAINTS[name].of == of] + if any(shape not in _CONSTRAINTS[name].keywords for name in names): + msg = f"{', '.join(names)}: {applies}, and {describe(inner)} is not one" + raise TypeError(msg) + - The message states what the type allows, for example "expected an - integer in [0, 9], got 12", and the problem's `ctx` holds the bounds. - The bounds are checked only when `parse` reports no problem, in the - same way that rules run only on a value of their type. +def constrained( + parse: Parser, description: str, said: Constraints, unit: str = "element" +) -> Parser: + """Wrap `parse` to also check the constraints its type carries: one `invalid_value` problem if any fails. + + A bound limits a number, a step what an integer is a multiple of, and + a length bound the number of `unit`s in the value: the elements of an + array, or the characters of a string, which are its code points, as + JSON Schema counts them. The message states what the type allows, for + example "expected an integer in [0, 9], got 12", "expected an integer + > 0 that is a multiple of 4, got 6" or "expected an array of integers + with at least 1 element, got []", and the problem's `ctx` holds the + constraints, as pydantic names them. The constraints are checked only + on a value of the type, in the same way that rules are: when `parse` + reports no problem, or only keys that a closed TypedDict does not + declare. Such a key is left out and the value is still returned, so + the value it returns is held to the constraints too. """ - expected = f"expected {description} {_admitted(said)}" + expected = f"expected {description} {_admitted(said, unit)}" ctx: dict[str, JSONValue] = dict(said) - holds = tuple((_HOLDS[name], bound) for name, bound in said.items()) + checks = tuple( + (_CONSTRAINTS[name].holds, bound, _CONSTRAINTS[name].of == "length") + for name, bound in said.items() + ) def parse_constrained(value: object, loc: Loc) -> Parsed: typed, found = parse(value, loc) - if len(found) != 0: + if any(each.kind != "unknown_key" for each in found): return typed, found - number = cast("float", typed) - for keeps, bound in holds: - if not keeps(number, bound): + for keeps, bound, length in checks: + measured = len(cast("Sized", typed)) if length else cast("float", typed) + if not keeps(measured, bound): message = f"{expected}, got {shown(value)}" - return typed, (ValidationProblem(loc, message, "invalid_value", ctx=ctx),) - return typed, () + return typed, (*found, ValidationProblem(loc, message, "invalid_value", ctx=ctx)) + return typed, found return parse_constrained -def _admitted(said: Constraints) -> str: - """The range the bounds allow, as written after the type in a message: "in [0, 9]", "in (0, 1)", ">= 1".""" - low = next(((name, bound) for name, bound in said.items() if name in _FROM_BELOW), None) - high = next(((name, bound) for name, bound in said.items() if name in _FROM_ABOVE), None) +def _admitted(said: Constraints, unit: str) -> str: + """What the constraints allow, as written after the type in a message: "in [0, 9]", ">= 1", "> 0 that is a multiple of 4", "with at least 1 element".""" + if any(_CONSTRAINTS[name].of == "length" for name in said): + return _admitted_lengths(said.get("min_length"), said.get("max_length"), unit) + low = next( + ((name, bound) for name, bound in said.items() if _CONSTRAINTS[name].slot == "below"), None + ) + high = next( + ((name, bound) for name, bound in said.items() if _CONSTRAINTS[name].slot == "above"), None + ) + bounded = _admitted_range(low, high) + step = said.get("multiple_of") + if step is None: + return bounded + multiple = f"that is a multiple of {shown(step)}" + return multiple if bounded == "" else f"{bounded} {multiple}" + + +def _admitted_range( + low: tuple[str, int | float] | None, high: tuple[str, int | float] | None +) -> str: + """The range a lower and an upper bound allow, each named, as written after the type in a message: "in [0, 9]", "in (0, 1)", ">= 1"; empty without either.""" if low is not None and high is not None: opening = "(" if low[0] in _EXCLUSIVE else "[" closing = ")" if high[0] in _EXCLUSIVE else "]" return f"in {opening}{shown(low[1])}, {shown(high[1])}{closing}" if low is not None: return f"{'>' if low[0] in _EXCLUSIVE else '>='} {shown(low[1])}" - name, bound = cast("tuple[str, int | float]", high) - return f"{'<' if name in _EXCLUSIVE else '<='} {shown(bound)}" + if high is not None: + return f"{'<' if high[0] in _EXCLUSIVE else '<='} {shown(high[1])}" + return "" + + +def _admitted_lengths(low: float | None, high: float | None, unit: str) -> str: + """The lengths that length bounds allow, counted in `unit`s, as written after the type in a message: "with at least 1 element", "with 1 to 3 characters".""" + if low is not None and high is not None: + if low == high: + return f"with {_counted(low, unit)}" + return f"with {shown(low)} to {_counted(high, unit)}" + if low is not None: + return f"with at least {_counted(low, unit)}" + return f"with at most {_counted(cast('float', high), unit)}" + + +def _counted(count: float, unit: str) -> str: + """A number of `unit`s, as a message writes it: "1 element", "3 characters".""" + return f"{shown(count)} {unit}{'' if count == 1 else 's'}" # --- the compiler -------------------------------------------------------- @@ -1137,7 +1294,8 @@ def _compile(annotation: object, leaf: Leaf, building: _Building) -> Parser | No if len(said) == 0: return parse _constrainable(inner, said) - return constrained(parse, describe(inner), said) + unit = "character" if shape_of(inner) == "str" else "element" + return constrained(parse, describe(inner), said, unit) def _compile_type(inner: object, leaf: Leaf, building: _Building) -> Parser | None: @@ -1314,7 +1472,7 @@ def _typeddicts_in(annotation: object, seen: frozenset[object]) -> tuple[type, . def _metadata_problem(inner: object, metadata: tuple[object, ...]) -> TypeError | None: - """The error the checker raises for `metadata` on a member of type `inner`: a constraint it does not read, or a bound on a type that is not a number; None when it reads the metadata, as it does a note.""" + """The error the checker raises for `metadata` on a member of type `inner`: a constraint it does not read, or a constraint on a type it does not limit; None when it reads the metadata, as it does a note.""" try: _constrainable(inner, constraints_of(metadata)) except TypeError as error: @@ -1382,22 +1540,6 @@ def check( DIALECT: Final = "https://json-schema.org/draft/2020-12/schema" """The dialect of every schema: JSON Schema draft 2020-12, the same dialect pydantic and zod use.""" -_KEYWORDS: Final[Mapping[str, str]] = { - "gt": "exclusiveMinimum", - "ge": "minimum", - "lt": "exclusiveMaximum", - "le": "maximum", -} -"""JSON Schema's keyword for each bound.""" - -_STRICTER: Final[Mapping[str, Callable[[float, float], float]]] = { - "exclusiveMinimum": max, - "minimum": max, - "exclusiveMaximum": min, - "maximum": min, -} -"""For each keyword, the function that picks the stricter of two bounds.""" - def no_schema_leaf(annotation: object, schemas: Schemas) -> JSONSchema | None: """The schema leaf for a caller with no shapes of its own.""" @@ -1424,15 +1566,17 @@ def __init__(self, leaf: SchemaLeaf = no_schema_leaf) -> None: self._uses: dict[str, int] = {} def of(self, annotation: object) -> JSONSchema: - """The schema of `annotation` as the checker reads it: its type, with the bounds and notes from `Annotated`. - - Each bound becomes the matching JSON Schema keyword (`Ge(0)` becomes - `minimum`), and a `Doc` becomes the `description`. The schema keeps - both the type's bounds and the bounds of its `NewType`, because the - checker applies both. When both set the same keyword, the stricter - bound is kept. An annotation the checker supports is written. Any - other raises `TypeError`, which cannot happen if the caller already - checked the annotation with `parser`. + """The schema of `annotation` as the checker reads it: its type, with the constraints and notes from `Annotated`. + + Each constraint becomes the matching JSON Schema keyword (`Ge(0)` + becomes `minimum`, `MinLen(1)` becomes `minItems` on an array and + `minLength` on a string), and a `Doc` + becomes the `description`. The schema keeps both the type's bounds + and the bounds of its `NewType`, or of the fixed length of its + tuple, because the checker applies both. When both set the same + keyword, the stricter bound is kept. An annotation the checker + supports is written. Any other raises `TypeError`, which cannot + happen if the caller already checked the annotation with `parser`. """ inner, metadata = strip_annotation(annotation) schema = self._type(inner) @@ -1445,10 +1589,15 @@ def of(self, annotation: object) -> JSONSchema: ] if len(notes) != 0: schema = {**schema, "description": "\n\n".join(notes)} - for name, bound in constraints_of(metadata).items(): - keyword = _KEYWORDS[name] + said = constraints_of(metadata) + _constrainable(inner, said) + shape = shape_of(inner) + for name, bound in said.items(): + constraint = _CONSTRAINTS[name] + keyword = constraint.keywords[shape] held = cast("int | float | None", schema.get(keyword)) - schema = {**schema, keyword: bound if held is None else _STRICTER[keyword](held, bound)} + kept = bound if held is None else _STRICTER[constraint.slot](held, bound) + schema = {**schema, keyword: kept} return schema def object_of(self, typeddict: type) -> JSONSchema: @@ -1591,8 +1740,9 @@ def json_schema(shape: type) -> JSONSchema: - A TypedDict becomes an object with its keys, its required keys, and what any other key may hold (nothing, for a closed TypedDict). - - A bound becomes the matching JSON Schema keyword: `Ge(0)` becomes - `minimum`. + - A constraint becomes the matching JSON Schema keyword: `Ge(0)` + becomes `minimum`, and `MinLen(1)` becomes `minItems` on an array + and `minLength` on a string. - A `Doc` becomes the `description`. Only a `Doc` produces a description; docstrings are for Python readers and are not used. This matches zod, which writes only what `.describe()` sets. diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py index 4dc31e9162..f259f3d7fa 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/__init__.py @@ -163,15 +163,18 @@ def acme_lz4_rules( type may carry bounds, written with annotated-types and read the way pydantic reads them: `Gt`, `Ge`, `Lt`, `Le` and `Interval`, at most one from each side, at any depth. For example, -`tuple[Annotated[int, Ge(1)], ...]` bounds each element. A value out of +`tuple[Annotated[int, Ge(1)], ...]` bounds each element. An integer's +type may carry a step, `MultipleOf`, at most one. An array's or a +string's type may bound its length with `MinLen`, `MaxLen` and `Len`, so +`Annotated[tuple[int, ...], MinLen(1)]` is never empty. A value out of bounds is an `invalid_value` problem. Its message says what the type admits ("expected an integer >= 1, got 0"), and its `ctx` holds the bounds. The rules run only on a configuration within its bounds, so a rule can rely on them, as pydantic's after-validators and zod's refinements do. Until a value out of bounds is fixed, it is the only problem reported for the configuration. `Annotated` may also carry a -note, as a string or a `Doc`. Any other metadata (a `MinLen`, a -`Predicate`, pydantic's `Field`) is rejected when the definition is +note, as a string or a `Doc`. Any other metadata (a `Predicate`, +pydantic's `Field`) is rejected when the definition is built, because a type whose constraint the checker does not enforce would give false information. diff --git a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py index 933f2f7f55..05eefcf761 100644 --- a/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py +++ b/packages/zarr-metadata/src/zarr_metadata/definition/_definition.py @@ -1717,6 +1717,10 @@ def _read_carried( declared = {key: value for key, value in carried.items() if key in members} undeclared = [key for key in carried if key not in members] configuration, judged = definition.read_configuration(declared) + # The type check describes a configuration member, which the document + # never wrote, so its messages name the member and the name it was read + # from. A rule of such a definition writes its messages for the name. + _, typed = definition._check_configuration(declared) # A problem with the configuration encoded in the name belongs to the # declaration. It is located at the declaration, where the name is, and the # `input` and `ctx` that described a configuration member do not apply @@ -1725,7 +1729,14 @@ def _read_carried( problems = ( *beside, *( - dataclasses.replace(found, loc=loc, input=UNSET, ctx={}, kind="invalid_value") + dataclasses.replace( + found, + loc=loc, + input=UNSET, + ctx={}, + kind="invalid_value", + message=_read_from(name, found) if found in typed else found.message, + ) for found in judged ), ) @@ -1741,6 +1752,14 @@ def _read_carried( ), problems +def _read_from(name: str, found: ValidationProblem) -> str: + """The message of `found`, a problem the type check found in the configuration that `name` encodes, naming the member it is about and the name: `bits in "r12": expected ...`.""" + if len(found.loc) == 0: + return f"{shown(name)}: {found.message}" + member = ".".join(str(part) for part in found.loc) + return f"{member} in {shown(name)}: {found.message}" + + def _sized(declaration: _NestedDeclaration, inner: ResolvedDeclaration[Any]) -> Problems: """The problems with the definition that read `inner` in the member that holds `declaration`, as the kind of `declaration` reports them for its alias. diff --git a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py index 22e4ecc466..627545859c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py +++ b/packages/zarr-metadata/src/zarr_metadata/model/_array_v3.py @@ -27,7 +27,7 @@ within, ) from zarr_metadata._sentinel import UNSET -from zarr_metadata._typed_json import typeddict_keys +from zarr_metadata._typed_json import strip_annotation, typeddict_keys from zarr_metadata.definition._context import disagreements, scoped from zarr_metadata.definition._definition import ( AcceptedDeclaration, @@ -132,10 +132,10 @@ def members_past_the_levels( _EXTENSION_LISTS_V3: Final[tuple[tuple[str, type[Definition[Any]]], ...]] = tuple( (key, kind) for key, (annotation, _) in _MEMBERS_V3.items() - if get_origin(annotation) is tuple - and (kind := declaration_kind(get_args(annotation)[0])) is not None + if get_origin(listed := strip_annotation(annotation)[0]) is tuple + and (kind := declaration_kind(get_args(listed)[0])) is not None ) -"""The lists of extension points of a v3 array document, and the kind each entry is read as: each member that the TypedDict annotates as a tuple of a declaration alias.""" +"""The lists of extension points of a v3 array document, and the kind each entry is read as: each member that the TypedDict annotates as a tuple of a declaration alias, whatever constraints the tuple carries.""" def no_extra_fields() -> Mapping[str, ResolvedDeclaration[Any]]: diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py index a987f241c8..6a88f813ea 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_chunk_grid.py @@ -89,17 +89,14 @@ def _edges(draw: st.DrawFn, extent: int | None) -> tuple[int | tuple[int, int], can be long, since a pair costs the same however many chunks it counts, which is what the spelling is for. Its edges often share a factor, as they must for a shard to divide them into inner chunks - longer than 1; edges drawn apart rarely do. A - dimension of length zero, or of a length not known, may be covered by - no edges at all. + longer than 1; edges drawn apart rarely do. There is always at least + one edge, even along a dimension of length zero: the spec allows no + empty list, and a dimension with no chunks writes a bare length instead. """ - min_runs = 0 if extent is None or extent == 0 else 1 factor = draw(st.just(1) | st.integers(2, _MAX_FACTOR)) sizes = [ factor * size - for size in draw( - st.lists(st.integers(1, _MAX_LENGTH), min_size=min_runs, max_size=_MAX_RUNS) - ) + for size in draw(st.lists(st.integers(1, _MAX_LENGTH), min_size=1, max_size=_MAX_RUNS)) ] runs = [(size, draw(_count(extent, size))) for size in sizes] covered = sum(size * count for size, count in runs) diff --git a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py index f2c0ec3c0b..e63d4afe1c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py +++ b/packages/zarr-metadata/src/zarr_metadata/testing/_draw.py @@ -278,7 +278,12 @@ def pipelines_for( def _codec( draw: st.DrawFn, options: Sequence[tuple[K, Configurations]] ) -> tuple[K, ZarrV3DeclarationJSON]: - definition, configurations = draw(st.sampled_from(options)) + # An index, not the options: Hypothesis labels a `sampled_from` by + # hashing its elements, and each option holds a strategy built for this + # draw, whose hash is its address. Labels that follow addresses send + # Hypothesis's mutator down different paths from one run to the next, so + # a derandomized run would not draw the same examples twice. + definition, configurations = options[draw(st.sampled_from(range(len(options))))] return definition, draw(any_spelling(definition, configurations)) diff --git a/packages/zarr-metadata/src/zarr_metadata/typed_json.py b/packages/zarr-metadata/src/zarr_metadata/typed_json.py index 982a3d4868..b98d392927 100644 --- a/packages/zarr-metadata/src/zarr_metadata/typed_json.py +++ b/packages/zarr-metadata/src/zarr_metadata/typed_json.py @@ -41,8 +41,14 @@ - A number type may carry bounds, written with annotated-types as pydantic reads them: `Gt`, `Ge`, `Lt`, `Le` and `Interval`, at any depth. For example, `tuple[Annotated[int, Ge(1)], ...]` bounds each - element. A value outside the bounds is an `invalid_value` problem. Its - message states what the type allows, and its `ctx` holds the bounds. + element. An integer type may also carry a step, `MultipleOf`: + `Annotated[int, Gt(0), MultipleOf(4)]` is a positive multiple of 4. An + array or a string type may bound its length the same way, with + `MinLen`, `MaxLen` and `Len`: `Annotated[tuple[int, ...], MinLen(1)]` + is a non-empty array. A string's length counts its code points, as + JSON Schema counts them. A value outside the bounds is an `invalid_value` + problem. Its message states what the type allows, and its `ctx` holds + the bounds, as pydantic names them (`ge`, `min_length`). `Annotated` may also carry a note (a string or a `Doc`). Any other metadata raises `TypeError`, because `check` would not enforce a constraint it does not support. diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py b/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py index f647895154..f4dd1e6f03 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_json_schema.py @@ -274,16 +274,8 @@ def _member_object(schema: Mapping[str, JSONValue], key: str) -> Mapping[str, JS def _array(context: ZarrV3Context, schemas: Schemas) -> JSONSchema: - """An array document: its TypedDict, at least one codec, and a constraint on its fill value for each data type in scope, which applies when the document names that data type. - - The TypedDict cannot declare a minimum length, so the schema adds it: - a pipeline needs its array -> bytes codec, and the reader rejects an - empty `codecs`. - """ + """An array document: its TypedDict, and a constraint on its fill value for each data type in scope, which applies when the document names that data type.""" schema = schemas.object_of(ZarrV3ArrayMetadataJSON) - properties = _member_object(schema, "properties") - codecs = _member_object(properties, "codecs") - schema = {**schema, "properties": {**properties, "codecs": {**codecs, "minItems": 1}}} schema = _with_extra_fields(schema, context.array_extra_fields, schemas) held: list[JSONValue] = [] for definition in context.data_types.values(): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py index ba34cde633..16135ea84d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/chunk_grid.py @@ -10,7 +10,7 @@ from typing import Annotated, Final, Literal, NotRequired -from annotated_types import Ge +from annotated_types import Ge, MinLen from typing_extensions import TypedDict # regular: https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html#regular-grids @@ -67,7 +67,7 @@ class RegularChunkGridObject(TypedDict, closed=True): """ -# rectilinear: https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md +# rectilinear: https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md RECTILINEAR_CHUNK_GRID_NAME: Final = "rectilinear" """The `name` field value of the rectilinear chunk grid.""" @@ -77,12 +77,19 @@ class RegularChunkGridObject(TypedDict, closed=True): _Positive = Annotated[int, Ge(1)] -RectilinearDimSpec = _Positive | tuple[_Positive | tuple[_Positive, _Positive], ...] +RectilinearDimSpec = ( + _Positive | Annotated[tuple[_Positive | tuple[_Positive, _Positive], ...], MinLen(1)] +) """JSON shape for one dimension's rectilinear spec. Either a bare integer (uniform shorthand for a regular dimension within -a rectilinear grid), or a tuple of integers and/or `[value, count]` RLE -pairs. Every extent, and every run's length and count, is at least 1. +a rectilinear grid), or a non-empty tuple of integers and/or +`[value, count]` RLE pairs. Every extent, and every run's length and +count, is at least 1. The tuple is non-empty even for a dimension of +length 0, which writes a bare integer to have no chunks: "as a non-empty +array", and "For an axis of length 0, the array form still requires at +least one edge length" +(https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md?plain=1#L89-L95). """ @@ -95,9 +102,9 @@ class RectilinearChunkGridConfiguration(TypedDict, closed=True): """The chunk edge lengths along each dimension of the array, one entry per dimension. An entry is a bare integer, a step repeated until it covers the - dimension, or a list of edge lengths and `[length, count]` runs, whose - lengths must sum to at least the dimension's length. Every length and - count is at least 1. + dimension, or a non-empty list of edge lengths and `[length, count]` + runs, whose lengths must sum to at least the dimension's length. Every + length and count is at least 1. """ @@ -121,7 +128,7 @@ class RectilinearChunkGridObject(TypedDict, closed=True): `kind` and `chunk_shapes` are required, so only the object form is valid; the short-hand-name form is not permitted by the spec for this grid. - https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md#L59-L62 + https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md#L59-L62 https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L1562-L1564 """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/codec.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/codec.py index 00792264ea..806ce06c29 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_types/codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/codec.py @@ -12,7 +12,7 @@ from typing import Annotated, Final, Literal, NotRequired -from annotated_types import Ge, Interval +from annotated_types import Ge, Interval, MinLen from typing_extensions import TypedDict from zarr_metadata._common import EmptyConfiguration, JSONValue @@ -510,12 +510,12 @@ class ShardingIndexedCodecConfiguration(TypedDict, closed=True): One length, at least 1, per axis of the shard the codec is handed, each dividing the shard's length along that axis. """ - codecs: tuple[ZarrV3CodecDeclaration, ...] + codecs: Annotated[tuple[ZarrV3CodecDeclaration, ...], MinLen(1)] """The codec pipeline that encodes each inner chunk. Like an array's `codecs`, it holds exactly one array -> bytes codec. """ - index_codecs: tuple[ZarrV3StaticCodecDeclaration, ...] + index_codecs: Annotated[tuple[ZarrV3StaticCodecDeclaration, ...], MinLen(1)] """The codec pipeline that encodes the shard index. It holds exactly one array -> bytes codec, and only codecs whose output diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py b/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py index 3c8b19d703..69f304bb0c 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/_types/data_type.py @@ -13,7 +13,7 @@ from collections.abc import Mapping from typing import Annotated, Final, Literal, NewType, NotRequired, get_args -from annotated_types import Interval +from annotated_types import Ge, Gt, Interval, MinLen, MultipleOf from typing_extensions import ReadOnly, TypedDict from zarr_metadata._common import JSONValue @@ -352,10 +352,11 @@ class RawBytesConfiguration(TypedDict, closed=True): back as the name. """ - bits: int + bits: Annotated[int, Gt(0), MultipleOf(8)] """The size in bits that the name writes, `16` for `r16`. - A positive multiple of 8. + A positive multiple of 8: "raw bits, variable size given by *, limited + to be a multiple of 8". Zero bits is not a valid type. """ @@ -411,9 +412,11 @@ class RawBytesConfiguration(TypedDict, closed=True): class FixedLengthUtf32Configuration(TypedDict, closed=True): """Configuration for the `fixed_length_utf32` data type.""" - length_bytes: ReadOnly[int] - """The size in bytes of every value: a positive multiple of 4, since - each code point takes 4 bytes.""" + length_bytes: ReadOnly[Annotated[int, Gt(0), MultipleOf(4)]] + """The size in bytes of every value: a "positive integer, multiple of `4`", since each code point takes 4 bytes. + + https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/data-types/fixed_length_utf32/README.md?plain=1#L41 + """ class FixedLengthUtf32(TypedDict, closed=True): @@ -447,8 +450,11 @@ class FixedLengthUtf32(TypedDict, closed=True): class NullTerminatedBytesConfiguration(TypedDict, closed=True): """Configuration for the `null_terminated_bytes` data type.""" - length_bytes: ReadOnly[int] - """The size in bytes of every value, at least 1.""" + length_bytes: ReadOnly[Annotated[int, Ge(1)]] + """The size in bytes of every value, at least 1. + + zarr-python refuses a length below 1, since such a type holds no data. + """ class NullTerminatedBytes(TypedDict, closed=True): @@ -482,8 +488,11 @@ class NullTerminatedBytes(TypedDict, closed=True): class ZarrPythonRawBytesConfiguration(TypedDict, closed=True): """Configuration for zarr-python's `raw_bytes` data type.""" - length_bytes: ReadOnly[int] - """The size in bytes of every value, at least 1.""" + length_bytes: ReadOnly[Annotated[int, Ge(1)]] + """The size in bytes of every value, at least 1. + + zarr-python refuses a length below 1, since such a type holds no data. + """ class ZarrPythonRawBytes(TypedDict, closed=True): @@ -613,8 +622,8 @@ class NumpyTimedelta64(TypedDict, closed=True): class StructField(TypedDict, closed=True): """A single field entry inside a structured dtype.""" - name: ReadOnly[str] - """The field name (must be unique within a struct and non-empty).""" + name: ReadOnly[Annotated[str, MinLen(1)]] + """The field name: non-empty, and unique within the struct.""" data_type: ReadOnly[ZarrV3DataTypeDeclaration] """The field's data type. @@ -626,7 +635,7 @@ class StructField(TypedDict, closed=True): class StructConfiguration(TypedDict, closed=True): """Configuration for the `struct` data type.""" - fields: ReadOnly[tuple[StructField, ...]] + fields: ReadOnly[Annotated[tuple[StructField, ...], MinLen(1)]] """The record's fields, in the order they are encoded. At least one. The order is significant: it is the order in which the diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index 2f256e219b..1d13e60f64 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -3,7 +3,7 @@ from collections.abc import Mapping from typing import Annotated, Final, Literal, NotRequired -from annotated_types import Ge +from annotated_types import Ge, MinLen from typing_extensions import TypedDict from zarr_metadata._common import JSONValue @@ -47,7 +47,7 @@ class ZarrV3ArrayMetadataJSON(TypedDict, extra_items=ZarrV3ExtensionField): Which JSON values it may take depends on `data_type`. """ - codecs: tuple[ZarrV3CodecDeclaration, ...] + codecs: Annotated[tuple[ZarrV3CodecDeclaration, ...], MinLen(1)] """The codecs that encode each chunk, in encoding order. Zero or more array -> array codecs, then exactly one array -> bytes codec, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py index 0b6db11ac1..2d7b2808a6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/chunk_grid/rectilinear.py @@ -1,7 +1,7 @@ """ Rectilinear chunk grid (zarr-extensions). -See https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md +See https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md """ from collections.abc import Iterator @@ -79,7 +79,7 @@ def _shape_rules( "The length of `chunk_shapes` MUST match the number of dimensions of the array", and "The sum of the edge lengths MUST equal or exceed `L`" - (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md?plain=1#L62-L91). + (https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md?plain=1#L62-L95). A bare integer repeats until it covers the dimension, so it always passes this check. """ @@ -112,7 +112,7 @@ def _chunk_lengths( A bare integer is the length of every chunk along its axis. A list gives the length of each chunk, and a `[length, count]` pair gives `count` chunks of that length - (https://github.com/zarr-developers/zarr-extensions/blob/4da7b37a84f76e660902f6d3de3eaef0e0febae6/chunk-grids/rectilinear/README.md?plain=1#L62-L91). + (https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/chunk-grids/rectilinear/README.md?plain=1#L62-L95). Every listed length counts, including the length of a chunk past the array's edge, which the array grows into when it is resized. """ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py index 7bb8c135e6..9c585d20ba 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/fixed_length_utf32.py @@ -21,19 +21,6 @@ _CODE_POINT_BYTES: Final = 4 -def _rules( - configuration: FixedLengthUtf32Configuration, nested: Nested -) -> Iterator[ValidationProblem]: - """Check that `length_bytes` is a "positive integer, multiple of `4`" (https://github.com/zarr-developers/zarr-extensions/blob/cadd1dab6c9aebafb66080b78b583f6507466b3d/data-types/fixed_length_utf32/README.md?plain=1#L41).""" - length_bytes = configuration["length_bytes"] - if length_bytes <= 0 or length_bytes % _CODE_POINT_BYTES != 0: - yield ValidationProblem( - ("length_bytes",), - f"expected a positive multiple of {_CODE_POINT_BYTES}, got {length_bytes}", - "invalid_value", - ) - - def _fill_value_rules( configuration: FixedLengthUtf32Configuration, nested: Nested, value: FixedLengthUtf32FillValue ) -> Iterator[ValidationProblem]: @@ -73,7 +60,6 @@ def _fill_value_canonical( FIXED_LENGTH_UTF32_DATA_TYPE: Final = DataTypeDefinition( name=FIXED_LENGTH_UTF32_DATA_TYPE_NAME, configuration=FixedLengthUtf32Configuration, - rules=_rules, fill_value=FixedLengthUtf32FillValue, fill_value_default=ConstantFillValue(""), fill_value_rules=_fill_value_rules, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py index 333f362cb5..7217f5eb1d 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/null_terminated_bytes.py @@ -25,17 +25,6 @@ from zarr_metadata.v3.data_type._byte import base64_decoded, base64_problems, base64_spelling -def _rules( - configuration: NullTerminatedBytesConfiguration, nested: Nested -) -> Iterator[ValidationProblem]: - """Check that `length_bytes` is positive. zarr-python refuses a length below 1, since such a type holds no data.""" - length_bytes = configuration["length_bytes"] - if length_bytes < 1: - yield ValidationProblem( - ("length_bytes",), f"expected a positive size, got {length_bytes}", "invalid_value" - ) - - def _fill_value_rules( configuration: NullTerminatedBytesConfiguration, nested: Nested, @@ -75,7 +64,6 @@ def _fill_value_canonical( NULL_TERMINATED_BYTES_DATA_TYPE: Final = DataTypeDefinition( name=NULL_TERMINATED_BYTES_DATA_TYPE_NAME, configuration=NullTerminatedBytesConfiguration, - rules=_rules, fill_value=NullTerminatedBytesFillValue, fill_value_default=ConstantFillValue(""), fill_value_rules=_fill_value_rules, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py index 63fc15e878..8af68ee617 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/raw.py @@ -41,17 +41,6 @@ def raw_bytes_dtype_name(value: str) -> RawBytesDataTypeName: return RawBytesDataTypeName(value) -def _rules(configuration: RawBytesConfiguration, nested: Nested) -> Iterator[ValidationProblem]: - """Check the size: "raw bits, variable size given by *, limited to be a multiple of 8". Zero bits is not a valid type.""" - bits = configuration["bits"] - if bits == 0 or bits % 8 != 0: - yield ValidationProblem( - ("bits",), - f"expected a size in bits that is a positive multiple of 8, got {bits}", - "invalid_value", - ) - - def _fill_value_rules( configuration: RawBytesConfiguration, nested: Nested, value: RawBytesFillValue ) -> Iterator[ValidationProblem]: @@ -88,7 +77,6 @@ def _storage(configuration: RawBytesConfiguration, nested: Nested) -> StorageCla RAW_BYTES_DATA_TYPE: Final = DataTypeDefinition( name=RAW_BYTES_NAME, configuration=RawBytesConfiguration, - rules=_rules, fill_value=RawBytesFillValue, fill_value_default=_fill_value_default, fill_value_rules=_fill_value_rules, diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index af9fbc0038..9034b80443 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -32,7 +32,7 @@ def _rules(configuration: StructConfiguration, nested: Nested) -> Iterator[ValidationProblem]: - """Check that there is at least one field, that field names are non-empty and distinct, and that field types have a fixed size. + """Check that field names are distinct, and that field types have a fixed size. Names must be distinct because a fill value addresses fields by name. "Variable-length data types @@ -43,15 +43,9 @@ def _rules(configuration: StructConfiguration, nested: Nested) -> Iterator[Valid whose storage is unknown is not checked. """ fields = configuration["fields"] - if len(fields) == 0: - yield ValidationProblem(("fields",), "expected at least one struct field", "invalid_value") seen: dict[str, int] = {} for index, member in enumerate(fields): name = member["name"] - if name == "": - yield ValidationProblem( - ("fields", index, "name"), "expected a non-empty field name", "invalid_value" - ) first = seen.setdefault(name, index) if first != index: yield ValidationProblem( diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py index e1f41efcc9..cc9aa3bccb 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/zarr_python_raw_bytes.py @@ -29,17 +29,6 @@ from zarr_metadata.v3.data_type._byte import base64_decoded, base64_problems, base64_spelling -def _rules( - configuration: ZarrPythonRawBytesConfiguration, nested: Nested -) -> Iterator[ValidationProblem]: - """Check that `length_bytes` is positive. zarr-python refuses a length below 1, since such a type holds no data.""" - length_bytes = configuration["length_bytes"] - if length_bytes < 1: - yield ValidationProblem( - ("length_bytes",), f"expected a positive size, got {length_bytes}", "invalid_value" - ) - - def _fill_value_rules( configuration: ZarrPythonRawBytesConfiguration, nested: Nested, @@ -82,7 +71,6 @@ def _fill_value_default(configuration: ZarrPythonRawBytesConfiguration, nested: ZARR_PYTHON_RAW_BYTES_DATA_TYPE: Final = DataTypeDefinition( name=ZARR_PYTHON_RAW_BYTES_DATA_TYPE_NAME, configuration=ZarrPythonRawBytesConfiguration, - rules=_rules, fill_value=ZarrPythonRawBytesFillValue, fill_value_default=_fill_value_default, fill_value_rules=_fill_value_rules, diff --git a/packages/zarr-metadata/tests/test_json_schema.py b/packages/zarr-metadata/tests/test_json_schema.py index 48d4b6f2fb..5e7f50c240 100644 --- a/packages/zarr-metadata/tests/test_json_schema.py +++ b/packages/zarr-metadata/tests/test_json_schema.py @@ -15,7 +15,7 @@ from typing import Annotated, Any, Literal, NewType, NotRequired, cast import pytest -from annotated_types import Ge, Gt, Interval, Le, Lt, MinLen +from annotated_types import Ge, Gt, Interval, Le, Len, Lt, MaxLen, MinLen, MultipleOf, Predicate from hypothesis import HealthCheck, event, given, settings from hypothesis import strategies as st from jsonschema import Draft202012Validator @@ -170,6 +170,41 @@ def _held(member: dict[str, Any], defs: dict[str, Any] | None = None) -> dict[st _holding(Annotated[int, Ge(0), Doc("a count")]), _held({"type": "integer", "description": "a count", "minimum": 0}), ), + ( + "a-step", + _holding(Annotated[int, Gt(0), MultipleOf(4)]), + _held({"type": "integer", "exclusiveMinimum": 0, "multipleOf": 4}), + ), + ( + "non-empty-string", + _holding(Annotated[str, MinLen(1)]), + _held({"type": "string", "minLength": 1}), + ), + ( + "non-empty-array", + _holding(Annotated[tuple[int, ...], MinLen(1)]), + _held({"type": "array", "items": {"type": "integer"}, "minItems": 1}), + ), + ( + "length-range", + _holding(Annotated[tuple[int, ...], Len(1, 3)]), + _held({"type": "array", "items": {"type": "integer"}, "minItems": 1, "maxItems": 3}), + ), + # A length bound and the length of a fixed-length array set the same + # keyword, and the stricter is kept. + ( + "a-length-on-a-pair", + _holding(Annotated[tuple[int, str], MinLen(1), MaxLen(3)]), + _held( + { + "type": "array", + "prefixItems": [{"type": "integer"}, {"type": "string"}], + "items": False, + "minItems": 2, + "maxItems": 3, + } + ), + ), # Metadata that is not a `Doc` adds nothing to the schema. ("note", _holding(Annotated[str, "a note"]), _held({"type": "string"})), # A value must meet its type's bounds and its `NewType`'s bounds. When @@ -334,7 +369,7 @@ def test_error_a_schema_that_fails_to_write_leaves_nothing_behind() -> None: def test_json_schema_refuses_metadata_check_does_not_hold_a_value_to() -> None: with pytest.raises(TypeError, match="is not a constraint the checker reads"): - json_schema(_holding(Annotated[tuple[int, ...], MinLen(1)])) + json_schema(_holding(Annotated[int, Predicate(bool)])) _PROPERTY = settings(max_examples=300, deadline=None, suppress_health_check=[HealthCheck.too_slow]) @@ -950,6 +985,7 @@ def test_every_schema_is_a_json_schema_written_the_same_each_time(context: ZarrV (Le(9), Le(5), {"type": "integer", "maximum": 5}), (Ge(5), Ge(9), {"type": "integer", "minimum": 9}), (Ge(9), Ge(5), {"type": "integer", "minimum": 9}), + (MultipleOf(4), MultipleOf(6), {"type": "integer", "multipleOf": 12}), ], ids=[ "gt-looser-inside", @@ -960,6 +996,7 @@ def test_every_schema_is_a_json_schema_written_the_same_each_time(context: ZarrV "le-stricter-inside", "ge", "ge-stricter-inside", + "two-steps-are-their-least-common-multiple", ], ) def test_the_stricter_of_two_bounds_of_one_keyword_is_kept( diff --git a/packages/zarr-metadata/tests/test_problem_data.py b/packages/zarr-metadata/tests/test_problem_data.py index 91d9d01412..34d2d79799 100644 --- a/packages/zarr-metadata/tests/test_problem_data.py +++ b/packages/zarr-metadata/tests/test_problem_data.py @@ -315,7 +315,8 @@ def test_a_problem_with_what_a_name_carries_is_the_declaration_s() -> None: # `r24` carries `{"bits": 24}`, which this scope's raw-bits definition # refuses. The problem is located at the declaration, because the name is what # the document wrote there. The bound on `bits` is not a bound on the - # name, so the problem's `ctx` is empty. + # name, so the problem's `ctx` is empty, and its message names the + # member and the name it was read from. context = CORE_AND_EXTENSIONS_CONTEXT.copy( update={ "data_types": { @@ -326,7 +327,7 @@ def test_a_problem_with_what_a_name_carries_is_the_declaration_s() -> None: ) _, problems = resolve("r24", DataTypeDefinition, context, ("data_type",)) assert [(p.loc, p.input, dict(p.ctx)) for p in problems] == [(("data_type",), "r24", {})] - assert problems[0].message == "expected an integer <= 16, got 24" + assert problems[0].message == 'bits in "r24": expected an integer <= 16, got 24' class Ordered(TypedDict, closed=True): diff --git a/packages/zarr-metadata/tests/test_typed_json.py b/packages/zarr-metadata/tests/test_typed_json.py index f399e4cf30..7684d07287 100644 --- a/packages/zarr-metadata/tests/test_typed_json.py +++ b/packages/zarr-metadata/tests/test_typed_json.py @@ -14,6 +14,7 @@ import json import math import pkgutil +import re import sys import textwrap import types @@ -41,6 +42,7 @@ Le, Len, Lt, + MaxLen, MinLen, MultipleOf, Predicate, @@ -1012,6 +1014,10 @@ class Noted(TypedDict, extra_items=Annotated[object, "any JSON"]): level: int +class _Item(TypedDict, closed=True): + a: int + + @pytest.mark.parametrize( ("annotation", "value", "found"), [ @@ -1042,6 +1048,137 @@ class Noted(TypedDict, extra_items=Annotated[object, "any JSON"]): (Digit | str, 10, [((), "expected an integer in [0, 9], got 10", {"ge": 0, "le": 9})]), (Digit, "nine", [((), 'expected an integer, got "nine"', {})]), (Annotated[int, "a note", Doc("another")], -1, []), + (Annotated[int, MultipleOf(4)], 8, []), + ( + Annotated[int, MultipleOf(4)], + 6, + [((), "expected an integer that is a multiple of 4, got 6", {"multiple_of": 4})], + ), + ( + Annotated[int, Gt(0), MultipleOf(4)], + 0, + [ + ( + (), + "expected an integer > 0 that is a multiple of 4, got 0", + {"gt": 0, "multiple_of": 4}, + ) + ], + ), + ( + Annotated[int, Interval(ge=0, le=9), MultipleOf(2)], + 3, + [ + ( + (), + "expected an integer in [0, 9] that is a multiple of 2, got 3", + {"ge": 0, "le": 9, "multiple_of": 2}, + ) + ], + ), + (Annotated[tuple[int, ...], MinLen(1)], [0], []), + ( + Annotated[tuple[int, ...], MinLen(1)], + [], + [ + ( + (), + "expected an array of integers with at least 1 element, got []", + {"min_length": 1}, + ) + ], + ), + ( + Annotated[tuple[int, ...], MaxLen(2)], + [0, 0, 0], + [ + ( + (), + "expected an array of integers with at most 2 elements, got [0, 0, 0]", + {"max_length": 2}, + ) + ], + ), + ( + Annotated[tuple[int, ...], Len(1, 3)], + [], + [ + ( + (), + "expected an array of integers with 1 to 3 elements, got []", + {"min_length": 1, "max_length": 3}, + ) + ], + ), + ( + Annotated[tuple[int, ...], Len(2, 2)], + [0], + [ + ( + (), + "expected an array of integers with 2 elements, got [0]", + {"min_length": 2, "max_length": 2}, + ) + ], + ), + ( + Annotated[tuple[int, int], MinLen(1)], + [0, 0], + [], + ), + ( + Annotated[tuple[Digit, ...], MinLen(1)], + [10], + [((0,), "expected an integer in [0, 9], got 10", {"ge": 0, "le": 9})], + ), + ( + Digit | Annotated[tuple[Digit, ...], MinLen(1)], + [], + [ + ( + (), + "expected an array of integers with at least 1 element, got []", + {"min_length": 1}, + ) + ], + ), + (Annotated[str, MinLen(1)], "a", []), + ( + Annotated[str, MinLen(1)], + "", + [((), 'expected a string with at least 1 character, got ""', {"min_length": 1})], + ), + ( + Annotated[str, Len(2, 2)], + "abc", + [ + ( + (), + 'expected a string with 2 characters, got "abc"', + {"min_length": 2, "max_length": 2}, + ) + ], + ), + # A key the element's closed TypedDict does not declare is left out + # and the value is still returned, so its length is held too. + ( + Annotated[tuple[_Item, ...], MaxLen(1)], + [{"a": 1, "zzz": 0}, {"a": 2}], + [ + ((0, "zzz"), "unexpected key 'zzz'", {}), + ( + (), + ( + "expected an array of objects with at most 1 element, got " + '[{"a": 1, "zzz": 0}, {"a": 2}]' + ), + {"max_length": 1}, + ), + ], + ), + # One code point outside the Basic Multilingual Plane is one + # character, as JSON Schema's `maxLength` counts it. + (Annotated[str, MaxLen(1)], "\U0001d11e", []), ], ids=[ "within", @@ -1059,15 +1196,33 @@ class Noted(TypedDict, extra_items=Annotated[object, "any JSON"]): "the-branch-it-bounds", "a-value-of-another-type-is-that-problem-alone", "notes", + "a-multiple", + "not-a-multiple", + "a-multiple-out-of-bounds", + "bounded-and-not-a-multiple", + "a-length-within", + "a-length-below", + "a-length-above", + "a-length-range", + "an-exact-length", + "a-length-on-a-fixed-length-array", + "a-length-checked-only-once-the-elements-are-read", + "the-branch-it-bounds-the-length-of", + "a-string-length-within", + "a-string-length-below", + "an-exact-string-length", + "a-length-held-beside-an-unknown-key", + "a-string-length-counts-code-points", ], ) def test_a_value_is_held_to_the_bounds_its_type_carries( annotation: object, value: object, found: list[tuple[Loc, str, dict[str, JSONValue]]] ) -> None: - # annotated-types bounds, read as pydantic reads them. A value outside - # any bound gives one problem that says what the type allows, with the - # bounds in the problem's `ctx`. Bounds are checked only for a value of - # the type. A bound is compared as the number it equals, whichever of + # annotated-types bounds, read as pydantic reads them: on a number's + # value, on what an integer is a multiple of, or on an array's length. A value outside any bound gives one + # problem that says what the type allows, with the bounds in the + # problem's `ctx`, named as pydantic names them. Bounds are checked only + # for a value of the type. A bound is compared as the number it equals, whichever of # the equal values Python's `Annotated` cache returns. _, problems = _read(annotation, value) assert [(problem.loc, problem.message, dict(problem.ctx)) for problem in problems] == found @@ -1101,6 +1256,33 @@ def test_error_a_bound_on_what_is_not_a_number() -> None: parser(Annotated[str, Ge(0)], no_leaf) +@pytest.mark.parametrize( + ("annotation", "named"), + [ + ( + Annotated[bool, MinLen(1)], + "min_length: a bound is on an array or a string, and a boolean is not one", + ), + ( + Annotated[Mapping[str, int], MaxLen(1)], + "max_length: a bound is on an array or a string, and an object is not one", + ), + ( + Annotated[int, Len(1, 2)], + "min_length, max_length: a bound is on an array or a string, and an integer is not one", + ), + ], + ids=["min-len-on-a-boolean", "max-len-on-an-object", "len-on-a-number"], +) +def test_error_a_length_bound_on_what_is_not_an_array_or_a_string( + annotation: object, named: str +) -> None: + # pydantic also bounds the number of an object's keys; the checker does + # not, so the type cannot promise more than it checks. + with pytest.raises(TypeError, match=re.escape(named)): + parser(annotation, no_leaf) + + def test_error_a_bound_on_the_values_an_open_typeddict_takes() -> None: class Loose(TypedDict, extra_items=Annotated[object, Ge(0)]): level: int @@ -1117,9 +1299,6 @@ class Later(BaseMetadata): @pytest.mark.parametrize( "constraint", [ - MinLen(1), - Len(1, 3), - MultipleOf(2), Predicate(str.isdigit), Timezone(None), Later(), @@ -1127,9 +1306,6 @@ class Later(BaseMetadata): pydantic.StringConstraints(pattern="^[a-z]+$"), ], ids=[ - "min-len", - "len", - "multiple-of", "predicate", "timezone", "one-a-later-release-adds", @@ -1145,7 +1321,7 @@ def test_error_a_constraint_the_checker_does_not_read(constraint: object) -> Non class _ConstrainedMember(TypedDict, closed=True): - a: Annotated[int, MultipleOf(2)] + a: Annotated[int, Later()] class _HoldsAConstrainedMember(TypedDict, closed=True): @@ -1174,16 +1350,16 @@ class _HoldsAnUnresolvableTypedDict(TypedDict, closed=True): class _ConstraintOnAReadableMember(TypedDict, closed=True): - inner: Annotated[_Readable, MultipleOf(2)] + inner: Annotated[_Readable, Later()] class _ConstraintOnAConstrainedMember(TypedDict, closed=True): - inner: Annotated[_ConstrainedMember, MultipleOf(2)] + inner: Annotated[_ConstrainedMember, Later()] class _SelfReferentialWithAConstraint(TypedDict, closed=True): parent: NotRequired[_SelfReferentialWithAConstraint] - a: Annotated[int, MultipleOf(2)] + a: Annotated[int, Later()] class _SelfReferentialNotAJSONShape(TypedDict, closed=True): @@ -1192,7 +1368,7 @@ class _SelfReferentialNotAJSONShape(TypedDict, closed=True): class _SelfReferentialConstraintOnItself(TypedDict, closed=True): - parent: NotRequired[Annotated[_SelfReferentialConstraintOnItself, MultipleOf(2)]] + parent: NotRequired[Annotated[_SelfReferentialConstraintOnItself, Later()]] a: int @@ -1207,7 +1383,7 @@ class _CycleBack(TypedDict, closed=True): class _OuterWithAConstraintAfterACycle(TypedDict, closed=True): inner: _InnerLeadingBackToAConstraint - a: Annotated[int, MultipleOf(2)] + a: Annotated[int, Later()] class _InnerLeadingBackToAConstraint(TypedDict, closed=True): @@ -1224,7 +1400,7 @@ class _InnerLeadingBackToAShape(TypedDict, closed=True): class _OuterWithAConstraintOnTheCycle(TypedDict, closed=True): - inner: Annotated[_InnerLeadingBackToTheConstraint, MultipleOf(2)] + inner: Annotated[_InnerLeadingBackToTheConstraint, Later()] a: int @@ -1234,19 +1410,17 @@ class _InnerLeadingBackToTheConstraint(TypedDict, closed=True): class _SelfReferentialTwiceWithAConstraintOnTheSecond(TypedDict, closed=True): plain: NotRequired[_SelfReferentialTwiceWithAConstraintOnTheSecond] - constrained: NotRequired[ - Annotated[_SelfReferentialTwiceWithAConstraintOnTheSecond, MultipleOf(2)] - ] + constrained: NotRequired[Annotated[_SelfReferentialTwiceWithAConstraintOnTheSecond, Later()]] class _DocumentedThenConstrainedBackReferences(TypedDict, closed=True): documented: NotRequired[Annotated[_DocumentedThenConstrainedBackReferences, Doc("parent")]] - constrained: NotRequired[Annotated[_DocumentedThenConstrainedBackReferences, MultipleOf(2)]] + constrained: NotRequired[Annotated[_DocumentedThenConstrainedBackReferences, Later()]] class _OuterWithAConstraintAfterAnOptionalCycle(TypedDict, closed=True): inner: _InnerLeadingBackOptionally - a: Annotated[int, MultipleOf(2)] + a: Annotated[int, Later()] class _InnerLeadingBackOptionally(TypedDict, closed=True): @@ -1262,7 +1436,7 @@ class _HoldsATupleOfAConstrainedMember(TypedDict, closed=True): class _ConstraintOnANotAJSONShapeMember(TypedDict, closed=True): - inner: Annotated[_NotAJSONShape, MultipleOf(2)] + inner: Annotated[_NotAJSONShape, Later()] _Children = TypeAliasType("_Children", "tuple[_SelfReferentialThroughAnAlias, ...]") @@ -1313,16 +1487,16 @@ class _SelfReferentialReadable(TypedDict, closed=True): ( _ConstrainedMember, Unread( - "_ConstrainedMember.a: MultipleOf(multiple_of=2) is not a constraint the checker " - "reads, which are Gt, Ge, Lt and Le", + "_ConstrainedMember.a: Later() is not a constraint the checker " + "reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), ( _HoldsAConstrainedMember, Unread( - "_HoldsAConstrainedMember.inner: _ConstrainedMember.a: MultipleOf(multiple_of=2) is " - "not a constraint the checker reads, which are Gt, Ge, Lt and Le", + "_HoldsAConstrainedMember.inner: _ConstrainedMember.a: Later() is " + "not a constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1354,8 +1528,8 @@ class _SelfReferentialReadable(TypedDict, closed=True): ( _ConstraintOnAReadableMember, Unread( - "_ConstraintOnAReadableMember.inner: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt and Le", + "_ConstraintOnAReadableMember.inner: Later() is not a " + "constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1363,16 +1537,16 @@ class _SelfReferentialReadable(TypedDict, closed=True): _ConstraintOnAConstrainedMember, Unread( "_ConstraintOnAConstrainedMember.inner: _ConstrainedMember.a: " - "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " - "Ge, Lt and Le", + "Later() is not a constraint the checker reads, which are Gt, " + "Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), ( _SelfReferentialWithAConstraint, Unread( - "_SelfReferentialWithAConstraint.a: MultipleOf(multiple_of=2) is not a constraint " - "the checker reads, which are Gt, Ge, Lt and Le", + "_SelfReferentialWithAConstraint.a: Later() is not a constraint " + "the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1387,8 +1561,8 @@ class _SelfReferentialReadable(TypedDict, closed=True): ( _SelfReferentialConstraintOnItself, Unread( - "_SelfReferentialConstraintOnItself.parent: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt and Le", + "_SelfReferentialConstraintOnItself.parent: Later() is not a " + "constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1403,8 +1577,8 @@ class _SelfReferentialReadable(TypedDict, closed=True): ( _OuterWithAConstraintAfterACycle, Unread( - "_OuterWithAConstraintAfterACycle.a: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt and Le", + "_OuterWithAConstraintAfterACycle.a: Later() is not a " + "constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1419,8 +1593,8 @@ class _SelfReferentialReadable(TypedDict, closed=True): ( _OuterWithAConstraintOnTheCycle, Unread( - "_OuterWithAConstraintOnTheCycle.inner: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt and Le", + "_OuterWithAConstraintOnTheCycle.inner: Later() is not a " + "constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1428,24 +1602,24 @@ class _SelfReferentialReadable(TypedDict, closed=True): _SelfReferentialTwiceWithAConstraintOnTheSecond, Unread( "_SelfReferentialTwiceWithAConstraintOnTheSecond.constrained: " - "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " - "Ge, Lt and Le", + "Later() is not a constraint the checker reads, which are Gt, " + "Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), ( _DocumentedThenConstrainedBackReferences, Unread( - "_DocumentedThenConstrainedBackReferences.constrained: MultipleOf(multiple_of=2) " - "is not a constraint the checker reads, which are Gt, Ge, Lt and Le", + "_DocumentedThenConstrainedBackReferences.constrained: Later() " + "is not a constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), ( _OuterWithAConstraintAfterAnOptionalCycle, Unread( - "_OuterWithAConstraintAfterAnOptionalCycle.a: MultipleOf(multiple_of=2) is not a " - "constraint the checker reads, which are Gt, Ge, Lt and Le", + "_OuterWithAConstraintAfterAnOptionalCycle.a: Later() is not a " + "constraint the checker reads, which are Gt, Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1462,8 +1636,8 @@ class _SelfReferentialReadable(TypedDict, closed=True): _HoldsATupleOfAConstrainedMember, Unread( "_HoldsATupleOfAConstrainedMember.inner: _ConstrainedMember.a: " - "MultipleOf(multiple_of=2) is not a constraint the checker reads, which are Gt, " - "Ge, Lt and Le", + "Later() is not a constraint the checker reads, which are Gt, " + "Ge, Lt, Le, MultipleOf, MinLen and MaxLen", constraint=True, ), ), @@ -1562,12 +1736,14 @@ def test_unread_in_names_the_part_without_a_parser_and_whether_it_is_a_constrain Annotated[Digit, Ge(1)], Annotated[int, Ge(0), Gt(0)], Annotated[int, Interval(le=9), Lt(5)], + Annotated[tuple[int, ...], Len(1, 3), MinLen(2)], + Annotated[int, MultipleOf(2), MultipleOf(3)], ], - ids=["narrowing-an-alias", "gt-and-ge", "interval-and-lt"], + ids=["narrowing-an-alias", "gt-and-ge", "interval-and-lt", "len-and-min-len", "two-steps"], ) def test_error_a_second_bound_from_one_side(annotation: object) -> None: # Pydantic uses the last bound given; write only one. - with pytest.raises(TypeError, match="is a second bound from (below|above)"): + with pytest.raises(TypeError, match="is a second (bound from below|bound from above|step)"): parser(annotation, no_leaf) @@ -1579,6 +1755,27 @@ def test_error_a_bound_that_is_not_a_finite_number(bound: object) -> None: parser(Annotated[int, Ge(bound)], no_leaf) # pyright: ignore[reportArgumentType] +def test_error_a_step_on_what_is_not_an_integer() -> None: + # pydantic also takes a step on a float; JSON Schema validators compare + # floats inexactly, so the checker takes one on an integer only. + with pytest.raises( + TypeError, match="multiple_of: a step is on an integer, and a number is not one" + ): + parser(Annotated[float, MultipleOf(2)], no_leaf) + + +@pytest.mark.parametrize("step", [0, -2, 1.5], ids=["zero", "negative", "fractional"]) +def test_error_a_step_that_is_not_a_positive_integer(step: object) -> None: + with pytest.raises(TypeError, match="a step is a positive integer"): + parser(Annotated[int, MultipleOf(step)], no_leaf) # pyright: ignore[reportArgumentType] + + +@pytest.mark.parametrize("length", [-1, 1.5], ids=["negative", "fractional"]) +def test_error_a_length_bound_that_is_not_a_non_negative_integer(length: object) -> None: + with pytest.raises(TypeError, match="a length bound is a non-negative integer"): + parser(Annotated[tuple[int, ...], MinLen(length)], no_leaf) # pyright: ignore[reportArgumentType] + + # --- check: the public door ------------------------------------------------ _V3_DOCUMENT = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) @@ -1671,6 +1868,16 @@ def test_error_check_refuses_a_value_of_another_type() -> None: assert [(problem.loc, problem.kind) for problem in problems] == [(("level",), "invalid_type")] +def test_error_check_refuses_an_array_document_without_codecs() -> None: + # The document's type says `codecs` is never empty, so `check` refuses + # an empty list at `codecs`, as the reader and the JSON Schema do. + typed, problems = check({**_V3_DOCUMENT, "codecs": []}, ZarrV3ArrayMetadataJSON) + assert typed is None + assert [(problem.loc, problem.kind, dict(problem.ctx)) for problem in problems] == [ + (("codecs",), "invalid_value", {"min_length": 1}) + ] + + _NOT_DOCUMENTS: Final = frozenset({"ZarrV3ContextSlots", "ZarrV2ContextSlots"}) """TypedDicts that describe no document: the keyword arguments of a scope class, which hold definitions, not JSON.""" diff --git a/packages/zarr-metadata/tests/testing/_shares.py b/packages/zarr-metadata/tests/testing/_shares.py index 27517e2653..3d70bf11aa 100644 --- a/packages/zarr-metadata/tests/testing/_shares.py +++ b/packages/zarr-metadata/tests/testing/_shares.py @@ -2,9 +2,15 @@ A reach test asks whether a target can be drawn at all, in as many examples as it takes, which says nothing of whether a test, at the 100 examples it -runs by default, ever meets it. Runs are derandomized, which keeps most shares -fixed, but a share can still move a few points between processes, so a bound -needs room: this is no place to bound what is drawn a few percent of the time. +runs by default, ever meets it. Runs are derandomized, so a share is a fixed +number for a given strategy. A change to the strategy draws other examples, +as another seed would, so a bound sits where the share stays across seeds: +this is no place to bound what is drawn a few percent of the time. + +A bound is set from the shares of 100 seeds, each run as `examples_of` +runs, but with `derandomize=False` under `hypothesis.seed`. It lies past +the most extreme share on its side by a quarter of that share's distance +from the median, rounded outward to a hundredth. """ from __future__ import annotations @@ -14,24 +20,58 @@ from hypothesis import HealthCheck, given, settings if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Sequence from hypothesis.strategies import SearchStrategy T = TypeVar("T") +_DERANDOMIZED = settings( + database=None, + derandomize=True, + deadline=None, + suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], +) + + +def examples_of(strategy: SearchStrategy[T], *, examples: int = 300) -> list[T]: + """The examples a derandomized run of `strategy` draws, checked to be the same in a second run. + + Hypothesis's mutator copies spans between the parts of an example that + share a label, and a label can follow an object's address: a + `sampled_from` of elements that hold a strategy built for the draw is + labeled by that strategy's `id`. Such a strategy draws other examples + from run to run, and from machine to machine, however derandomized. A + second run in the same process lays objects out at other addresses, so + it catches this. + """ + first = _draw(strategy, examples) + second = _draw(strategy, examples) + if [repr(value) for value in first] != [repr(value) for value in second]: + raise AssertionError( + f"{strategy!r} drew other examples on a second derandomized run. " + "A strategy built during a draw is likely in a `sampled_from`, which " + "labels it by its address; sample an index into the options instead." + ) + return first + + +def share(values: Sequence[T], counts: Callable[[T], bool]) -> float: + """The share of `values` that `counts` holds of.""" + return sum(counts(value) for value in values) / len(values) + def share_of( strategy: SearchStrategy[T], counts: Callable[[T], bool], *, examples: int = 300 ) -> float: """The share of the examples of `strategy` that `counts` holds of.""" - tally: list[bool] = [] - run = settings( - max_examples=examples, - database=None, - derandomize=True, - deadline=None, - suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.too_slow], - )(given(strategy)(lambda value: tally.append(counts(value)))) - run() - return sum(tally) / len(tally) + return share(examples_of(strategy, examples=examples), counts) + + +def _draw(strategy: SearchStrategy[T], examples: int) -> list[T]: + drawn: list[T] = [] + # A lambda, not `drawn.append`: Hypothesis cannot wrap a builtin method, + # and derandomizes from the test function's source, so this line is the seed. + run = given(strategy)(lambda value: drawn.append(value)) + settings(_DERANDOMIZED, max_examples=examples)(run)() + return drawn diff --git a/packages/zarr-metadata/tests/testing/test_documents.py b/packages/zarr-metadata/tests/testing/test_documents.py index 6c20a3c29f..53cd047724 100644 --- a/packages/zarr-metadata/tests/testing/test_documents.py +++ b/packages/zarr-metadata/tests/testing/test_documents.py @@ -8,6 +8,7 @@ from __future__ import annotations +import functools import json from typing import TYPE_CHECKING, Any, cast @@ -15,7 +16,7 @@ from hypothesis import find, given, settings from hypothesis import strategies as st -from tests.testing._shares import share_of +from tests.testing._shares import examples_of, share from zarr_metadata.definition import AcceptedDeclaration, Definition, resolve from zarr_metadata.definition._definition import canonicalize from zarr_metadata.model import ( @@ -340,19 +341,28 @@ def test_a_data_type_no_codec_takes_is_not_drawn(document: Mapping[str, Any]) -> _DOCUMENT_SHARES: dict[str, tuple[Callable[[Mapping[str, Any]], bool], float, float]] = { - "no axes": (lambda d: d["shape"] == (), 0.03, 0.25), - "more axes than five": (lambda d: len(d["shape"]) > 5, 0.1, 1), - "a zero-length axis": (lambda d: 0 in d["shape"], 0.1, 1), - "an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.1, 1), - "the first data type": (lambda d: _name(d["data_type"]) == "bool", 0, 0.12), + "no axes": (lambda d: d["shape"] == (), 0.04, 0.34), + "more axes than five": (lambda d: len(d["shape"]) > 5, 0.15, 1), + "a zero-length axis": (lambda d: 0 in d["shape"], 0.44, 1), + "an axis past 32 bits": (lambda d: any(n >= 2**32 for n in d["shape"]), 0.03, 1), + "the first data type": (lambda d: _name(d["data_type"]) == "bool", 0, 0.18), } -"""What a share of array documents holds, as a low and a high bound: what a test running only a few hundred examples meets.""" +"""What a share of array documents holds, as a low and a high bound: what a test running only a few hundred examples meets. + +Each bound is set as `_shares` says, from 100 seeds. +""" + + +@functools.cache +def _array_document_examples() -> list[Mapping[str, Any]]: + """Drawn once for every target, as each counts the same examples.""" + return examples_of(array_documents()) @pytest.mark.parametrize("target", _DOCUMENT_SHARES) def test_array_documents_share(target: str) -> None: counts, low, high = _DOCUMENT_SHARES[target] - assert low <= share_of(array_documents(), counts) <= high + assert low <= share(_array_document_examples(), counts) <= high def _empty_listings(document: Mapping[str, Any]) -> list[Any]: diff --git a/packages/zarr-metadata/tests/testing/test_strategies.py b/packages/zarr-metadata/tests/testing/test_strategies.py index 8ef6023265..6a7a66035e 100644 --- a/packages/zarr-metadata/tests/testing/test_strategies.py +++ b/packages/zarr-metadata/tests/testing/test_strategies.py @@ -670,7 +670,6 @@ def _is_pair(entry: object) -> bool: _RECTILINEAR_SPELLINGS: dict[str, tuple[tuple[int, ...] | None, Callable[[object], bool]]] = { "a bare length": (None, lambda dimension: isinstance(dimension, int)), - "no edges": (None, lambda dimension: dimension == ()), "only written-out lengths": ( None, lambda dimension: ( @@ -741,6 +740,18 @@ def test_rectilinear_dimensions_reach(spelling: str) -> None: ) +@given(data=st.data(), shape=st.none() | st.lists(st.integers(0, 2), max_size=3).map(tuple)) +def test_rectilinear_dimensions_are_never_empty( + data: st.DataObject, shape: tuple[int, ...] | None +) -> None: + """A list of edges has at least one, even along a dimension of length 0 or of a length not known.""" + configuration = data.draw(chunk_grid_configurations(RECTILINEAR_CHUNK_GRID, shape)) + # The type declares each list non-empty, so the definition's read of + # the configuration, which checks the type, is what would refuse one. + assert () not in configuration["chunk_shapes"] + assert RECTILINEAR_CHUNK_GRID.read_configuration(configuration)[1] == () + + @given(data=st.data(), shape=st.lists(st.integers(0, 2**63 - 1), max_size=3).map(tuple)) def test_chunk_grids_fit_a_long_shape(data: st.DataObject, shape: tuple[int, ...]) -> None: """However long an axis is, a grid covers it without a list as long as the axis.""" @@ -1057,7 +1068,7 @@ def _float_class(value: object) -> str | None: ) -@pytest.mark.parametrize(("kind", "at_least"), [("normal", 0.2), ("NaN", 0.1), ("infinity", 0.01)]) +@pytest.mark.parametrize(("kind", "at_least"), [("normal", 0.2), ("NaN", 0.26), ("infinity", 0.01)]) def test_hex_fill_values_are_not_only_small_numbers(kind: str, at_least: float) -> None: """Bits drawn as an integer alone are nearly all subnormal.""" assert share_of(_HEX_FILL_VALUES, lambda value: _float_class(value) == kind) >= at_least @@ -1097,7 +1108,7 @@ def _shard_inner_lengths(draw: st.DrawFn) -> tuple[int, ...]: def test_shards_of_a_rectilinear_grid_are_often_divided() -> None: """Edges drawn apart rarely share a factor, which leaves a shard only inner chunks of length 1.""" - assert share_of(_shard_inner_lengths(), lambda inner: any(n > 1 for n in inner)) >= 0.4 + assert share_of(_shard_inner_lengths(), lambda inner: any(n > 1 for n in inner)) >= 0.39 def _edge_factor(configuration: Mapping[str, Any]) -> int: @@ -1145,7 +1156,7 @@ def test_a_long_dimension_keeps_its_edge_factor() -> None: edges = chunk_grid_configurations(RECTILINEAR_CHUNK_GRID, (2**16 + 1,)).filter( lambda configuration: not isinstance(configuration["chunk_shapes"][0], int) ) - assert share_of(edges, lambda configuration: _edge_factor(configuration) > 1) >= 0.3 + assert share_of(edges, lambda configuration: _edge_factor(configuration) > 1) >= 0.32 def test_divisors_of_a_very_long_shard_are_found_quickly() -> None: @@ -1168,4 +1179,4 @@ def _field_types(declaration: Any) -> list[str]: # noqa: ANN401 def test_struct_fields_do_not_favor_the_first_data_type() -> None: """A type drawn after a field's name would be `bool` a fifth of the time.""" - assert share_of(declarations(STRUCT_DATA_TYPE), lambda d: _field_types(d)[0] == "bool") <= 0.12 + assert share_of(declarations(STRUCT_DATA_TYPE), lambda d: _field_types(d)[0] == "bool") <= 0.16 diff --git a/packages/zarr-metadata/tests/v3/test_every_definition.py b/packages/zarr-metadata/tests/v3/test_every_definition.py index 3090024c53..c72161d85c 100644 --- a/packages/zarr-metadata/tests/v3/test_every_definition.py +++ b/packages/zarr-metadata/tests/v3/test_every_definition.py @@ -12,6 +12,7 @@ from typing import Any import pytest +from typing_extensions import TypedDict from zarr_metadata import ValidationProblem from zarr_metadata._json import value_at @@ -26,7 +27,7 @@ ) from zarr_metadata.definition._definition import canonicalize from zarr_metadata.v3.codec.gzip import GZIP_CODEC -from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE, RawBytesConfiguration +from zarr_metadata.v3.data_type.raw import RAW_BYTES_DATA_TYPE from zarr_metadata.v3.definition import ( CORE_AND_EXTENSIONS_CONTEXT, CORE_CONTEXT, @@ -385,11 +386,17 @@ def test_raw_bits_read_as_r_star_with_the_size_their_name_carries( ) +class AnyRawSize(TypedDict, closed=True): + """A raw-bits configuration of any size, which the package's own type does not allow.""" + + bits: int + + def test_a_reader_reads_raw_bits_its_own_way_by_defining_r_star() -> None: # This definition accepts any size. Filed under `r*` in place of the # package's own, it reads every raw-bits name. Filed back, the package's # definition reads them again. - mine = DataTypeDefinition(name="r*", configuration=RawBytesConfiguration) + mine = DataTypeDefinition(name="r*", configuration=AnyRawSize) context = CORE_AND_EXTENSIONS_CONTEXT.copy( update={"data_types": {**CORE_AND_EXTENSIONS_CONTEXT.data_types, "r*": mine}} ) @@ -900,12 +907,21 @@ def test_error_struct_field_type_is_judged_where_it_sits() -> None: ] -@pytest.mark.parametrize("name", ["r12", "r0"]) -def test_error_a_raw_bytes_width_is_not_a_positive_multiple_of_8(name: str) -> None: +@pytest.mark.parametrize( + ("name", "message"), + [ + ("r12", 'bits in "r12": expected an integer > 0 that is a multiple of 8, got 12'), + ("r0", 'bits in "r0": expected an integer > 0 that is a multiple of 8, got 0'), + ], +) +def test_error_a_raw_bytes_width_is_not_a_positive_multiple_of_8(name: str, message: str) -> None: # Raw bits of a size the spec does not allow are reported, not left # unchecked. The document wrote a name, so the problem is located at - # the declaration. - assert _problems("data_type:r*", name) == [((), "invalid_value")] + # the declaration, and its message names the size the name spells. + _, problems = resolve(name, DataTypeDefinition, ZARR_PYTHON_CONTEXT) + assert [(found.loc, found.kind, found.message) for found in problems] == [ + ((), "invalid_value", message) + ] def test_error_a_configuration_written_beside_a_raw_bits_name() -> None: diff --git a/packages/zarr-metadata/tests/v3/test_grid_shapes.py b/packages/zarr-metadata/tests/v3/test_grid_shapes.py index 40887f37bb..d56dc4d366 100644 --- a/packages/zarr-metadata/tests/v3/test_grid_shapes.py +++ b/packages/zarr-metadata/tests/v3/test_grid_shapes.py @@ -64,8 +64,9 @@ def _problems(grid: JSONValue, shape: tuple[int, ...]) -> list[tuple[tuple[str | (_rectilinear([4, 4, 4]), (10,), ({4},)), (_rectilinear([4, 4, 2, 7]), (10,), ({4, 2, 7},)), (_rectilinear([[4, 2], 2]), (10,), ({4, 2},)), - # A dimension of length 0 and no chunks. - (_rectilinear([]), (0,), (set(),)), + # A dimension of length 0: a bare integer gives it no chunks, and a + # list still lists at least one. + (_rectilinear(4, [3]), (0, 0), ({4}, {3})), (_rectilinear(2, [3, 3]), (7, 6), ({2}, {3})), # A grid that nothing in scope claims, or one that is not read, is # not checked, and its lengths are unknown along each axis. @@ -95,13 +96,22 @@ def test_error_a_rectilinear_grid_of_another_rank(grid: JSONValue, shape: tuple[ assert _problems(grid, shape) == [(("configuration", "chunk_shapes"), "invalid_value")] -@pytest.mark.parametrize("spec", [[4, 4], [[4, 2]], []]) +@pytest.mark.parametrize("spec", [[4, 4], [[4, 2]]]) def test_error_rectilinear_chunks_that_do_not_cover_their_dimension(spec: JSONValue) -> None: assert _problems(_rectilinear(2, spec), (4, 10)) == [ (("configuration", "chunk_shapes", 1), "invalid_value") ] +def test_error_rectilinear_empty_edge_list() -> None: + # The spec requires a non-empty list even along a dimension of length 0, + # which writes a bare integer to have no chunks; no shape is needed to tell. + _, found = resolve(_rectilinear(2, []), ChunkGridDefinition, CORE_AND_EXTENSIONS_CONTEXT) + assert [(problem.loc, problem.kind) for problem in found] == [ + (("configuration", "chunk_shapes", 1), "invalid_value") + ] + + def test_a_grid_that_says_nothing_of_the_shape_fits_every_one_its_lengths_unknown() -> None: lenient = ChunkGridDefinition(name="acme.grid", configuration=EmptyConfiguration) context = CORE_AND_EXTENSIONS_CONTEXT.union(ZarrV3Context(chunk_grids=[lenient])) diff --git a/packages/zarr-metadata/tests/v3/test_sharding.py b/packages/zarr-metadata/tests/v3/test_sharding.py index dc3e81e69b..ae6336a979 100644 --- a/packages/zarr-metadata/tests/v3/test_sharding.py +++ b/packages/zarr-metadata/tests/v3/test_sharding.py @@ -206,7 +206,6 @@ def test_error_an_inner_chunk_length_that_does_not_divide_the_shard( ("shard", "found"), [ # Each pipeline is read as the array's pipeline is, at its own location. - (_shard([4, 4], codecs=[]), [((0, "configuration", "codecs"), "invalid_value")]), ( _shard([4, 4], codecs=["crc32c", LITTLE]), [((0, "configuration", "codecs", 1), "invalid_value")], @@ -233,6 +232,20 @@ def test_error_a_pipeline_a_shard_holds_that_does_not_fit_what_it_is_handed( assert _problems([shard], _chunk({8}, {8})) == found +@pytest.mark.parametrize( + ("shard", "key"), + [(_shard([4, 4], codecs=[]), "codecs"), (_shard([4, 4], index=[]), "index_codecs")], + ids=["codecs", "index-codecs"], +) +def test_error_a_shard_s_empty_pipeline_is_refused_by_its_type(shard: JSONValue, key: str) -> None: + # Each pipeline's type declares it non-empty, so the type check reports + # an empty one, before the pipeline's order check would. + problems = _read([shard], _chunk({8}, {8}))[1] + assert [(p.loc, p.kind, dict(p.ctx)) for p in problems] == [ + ((0, "configuration", key), "invalid_value", {"min_length": 1}) + ] + + def test_error_an_array_document_s_shard_is_read_where_it_sits() -> None: document = dict(ZarrV3ArrayMetadata.create_default(shape=(16, 16)).to_json()) document["chunk_grid"] = {"name": "regular", "configuration": {"chunk_shape": [8, 6]}}