Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions packages/zarr-metadata/changes/4638.feature.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
Add `ZarrV3NodeType` and `ZARR_V3_NODE_TYPE` for the `node_type` member of a
v3 document, and constants for the separator the spec assumes when one is
absent: `ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR` (`"."`),
`DEFAULT_CHUNK_KEY_ENCODING_DEFAULT_SEPARATOR` (`"/"`) and
`V2_CHUNK_KEY_ENCODING_DEFAULT_SEPARATOR` (`"."`). The `TypeIs` functions
`is_zarr_v2_array_order`, `is_zarr_v2_array_dimension_separator` (in
`zarr_metadata.v2.array`) and `is_zarr_v3_node_type` (in `zarr_metadata.v3`)
narrow any value to the matching `Literal` type, under every type checker.
16 changes: 15 additions & 1 deletion packages/zarr-metadata/src/zarr_metadata/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
from zarr_metadata.problems import MetadataValidationError, ProblemKind, ValidationProblem
from zarr_metadata.typed_json import JSONValue
from zarr_metadata.v2.array import (
ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR,
ZARR_V2_ARRAY_DIMENSION_SEPARATOR,
ZARR_V2_ARRAY_METADATA_STORE_KEY,
ZARR_V2_ARRAY_ORDER,
Expand Down Expand Up @@ -65,7 +66,13 @@
ZarrV2GroupMetadataStoreKey,
ZarrV2ZGroupJSON,
)
from zarr_metadata.v3 import ZarrV3DeclarationJSON, ZarrV3ExtensionField, ZarrV3NamedConfigJSON
from zarr_metadata.v3 import (
ZARR_V3_NODE_TYPE,
ZarrV3DeclarationJSON,
ZarrV3ExtensionField,
ZarrV3NamedConfigJSON,
ZarrV3NodeType,
)
from zarr_metadata.v3.array import (
ZARR_V3_ARRAY_METADATA_STORE_KEY,
ZarrV3ArrayMetadataJSON,
Expand All @@ -82,13 +89,15 @@
RegularChunkGridName,
)
from zarr_metadata.v3.chunk_key_encoding.default import (
DEFAULT_CHUNK_KEY_ENCODING_DEFAULT_SEPARATOR,
DEFAULT_CHUNK_KEY_ENCODING_NAME,
DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR,
DefaultChunkKeyEncodingMetadata,
DefaultChunkKeyEncodingName,
DefaultChunkKeyEncodingSeparator,
)
from zarr_metadata.v3.chunk_key_encoding.v2 import (
V2_CHUNK_KEY_ENCODING_DEFAULT_SEPARATOR,
V2_CHUNK_KEY_ENCODING_NAME,
V2_CHUNK_KEY_ENCODING_SEPARATOR,
V2ChunkKeyEncodingMetadata,
Expand Down Expand Up @@ -281,6 +290,7 @@
"COMPLEX64_DATA_TYPE_NAME",
"COMPLEX128_DATA_TYPE_NAME",
"CRC32C_CODEC_NAME",
"DEFAULT_CHUNK_KEY_ENCODING_DEFAULT_SEPARATOR",
"DEFAULT_CHUNK_KEY_ENCODING_NAME",
"DEFAULT_CHUNK_KEY_ENCODING_SEPARATOR",
"ENDIANNESS",
Expand Down Expand Up @@ -309,10 +319,12 @@
"UINT32_DATA_TYPE_NAME",
"UINT64_DATA_TYPE_NAME",
"UNSET",
"V2_CHUNK_KEY_ENCODING_DEFAULT_SEPARATOR",
"V2_CHUNK_KEY_ENCODING_NAME",
"V2_CHUNK_KEY_ENCODING_SEPARATOR",
"VLEN_BYTES_CODEC_NAME",
"VLEN_UTF8_CODEC_NAME",
"ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR",
"ZARR_V2_ARRAY_DIMENSION_SEPARATOR",
"ZARR_V2_ARRAY_METADATA_STORE_KEY",
"ZARR_V2_ARRAY_ORDER",
Expand All @@ -322,6 +334,7 @@
"ZARR_V3_ARRAY_METADATA_STORE_KEY",
"ZARR_V3_CONSOLIDATED_METADATA_KEY",
"ZARR_V3_GROUP_METADATA_STORE_KEY",
"ZARR_V3_NODE_TYPE",
"ZSTD_CODEC_NAME",
"BloscCName",
"BloscCodecMetadata",
Expand Down Expand Up @@ -444,6 +457,7 @@
"ZarrV3GroupMetadataUpdate",
"ZarrV3NamedConfigJSON",
"ZarrV3NodeMetadataInput",
"ZarrV3NodeType",
"ZstdCodecMetadata",
"ZstdCodecName",
"__version__",
Expand Down
41 changes: 27 additions & 14 deletions packages/zarr-metadata/src/zarr_metadata/model/_array_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,15 @@
from zarr_metadata.v2._context import ZarrV2Context # noqa: TC001
from zarr_metadata.v2._declaration_json import ZarrV2DataTypeMetadata
from zarr_metadata.v2.array import (
ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR,
ZARR_V2_ARRAY_DIMENSION_SEPARATOR,
ZARR_V2_ARRAY_METADATA_STORE_KEY,
ZARR_V2_ARRAY_ORDER,
ZarrV2ArrayDimensionSeparator,
ZarrV2ArrayMetadataJSON,
ZarrV2ArrayOrder,
is_zarr_v2_array_dimension_separator,
is_zarr_v2_array_order,
)
from zarr_metadata.v2.attributes import ZARR_V2_ATTRIBUTES_STORE_KEY
from zarr_metadata.v2.codec import ZarrV2CodecMetadata
Expand Down Expand Up @@ -239,13 +244,15 @@ def read_array_v2(
doc["dtype"], refined["dtype"], found, ZarrV2DataTypeDefinition, context, ("dtype",)
)
problems.extend(found)
order: ZarrV2ArrayOrder | None = None
if "order" in doc:
# Checked after refinement, so a value that only compares equal to
# "C" is refused, like every other value outside the set.
order, found = refine_json(doc["order"], ("order",))
refined["order"] = order
if len(found) != 0 or order not in ("C", "F"):
problems.append(outside_of(("order",), doc["order"], ("C", "F")))
refined["order"], found = refine_json(doc["order"], ("order",))
if len(found) == 0 and is_zarr_v2_array_order(refined["order"]):
order = refined["order"]
else:
problems.append(outside_of(("order",), doc["order"], ZARR_V2_ARRAY_ORDER))
compressor: ResolvedDeclaration[ZarrV2CodecDefinition[Any]] | UNSET | None = UNSET
if "compressor" in doc:
compressor = None
Expand Down Expand Up @@ -297,12 +304,20 @@ def read_array_v2(
problems.extend(found)
filters = tuple(read)
refined["filters"] = tuple(items)
dimension_separator: ZarrV2ArrayDimensionSeparator = ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR
if "dimension_separator" in doc:
separator, found = refine_json(doc["dimension_separator"], ("dimension_separator",))
refined["dimension_separator"] = separator
if len(found) != 0 or separator not in (".", "/"):
refined["dimension_separator"], found = refine_json(
doc["dimension_separator"], ("dimension_separator",)
)
if len(found) == 0 and is_zarr_v2_array_dimension_separator(refined["dimension_separator"]):
dimension_separator = refined["dimension_separator"]
else:
problems.append(
outside_of(("dimension_separator",), doc["dimension_separator"], (".", "/"))
outside_of(
("dimension_separator",),
doc["dimension_separator"],
ZARR_V2_ARRAY_DIMENSION_SEPARATOR,
)
)
fill_value: JSONValue = None
if "fill_value" in doc:
Expand All @@ -325,16 +340,14 @@ def read_array_v2(
filters=filters,
problems=within(with_input(problems, doc), at),
)
if len(problems) != 0 or shape is None or chunks is None or attributes is None:
if len(problems) != 0 or shape is None or chunks is None or order is None or attributes is None:
return reading, None
members = ArrayMembersV2(
shape=shape,
chunks=chunks,
fill_value=fill_value,
order=cast("ZarrV2ArrayOrder", refined["order"]),
dimension_separator=cast(
"ZarrV2ArrayDimensionSeparator", refined.get("dimension_separator", ".")
),
order=order,
dimension_separator=dimension_separator,
attributes=attributes,
extra_fields=extra_fields,
document={key: refined[key] for key in cast("Mapping[str, object]", doc)},
Expand Down Expand Up @@ -897,7 +910,7 @@ def from_key_value(
"order": "C",
"compressor": None,
"filters": None,
"dimension_separator": ".",
"dimension_separator": ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR,
}
)
"""The document of `ZarrV2ArrayMetadata.create_default` given nothing: a scalar `|u1` array, without its `zarr_format`."""
Expand Down
19 changes: 10 additions & 9 deletions packages/zarr-metadata/src/zarr_metadata/model/_group_v3.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,12 @@
unclaimed_declarations_of,
unexpected_keys,
)
from zarr_metadata.v3._common import ZarrV3ExtensionField
from zarr_metadata.v3._common import (
ZARR_V3_NODE_TYPE,
ZarrV3ExtensionField,
ZarrV3NodeType,
is_zarr_v3_node_type,
)

# A runtime import: `get_type_hints` resolves the public signatures.
from zarr_metadata.v3._context import ZarrV3Context # noqa: TC001
Expand Down Expand Up @@ -872,12 +877,8 @@ def _read_node_v3(
return ZarrV3UnknownNodeMetadataReading(problems), None


_NODE_TYPES: Final = ("array", "group")
"""The node types the spec defines."""


def _node_type(value: object) -> tuple[str | None, tuple[ValidationProblem, ...]]:
"""The node type `value` declares, one of `_NODE_TYPES`. None, with the problems, when it declares none of them or is not an object.
def _node_type(value: object) -> tuple[ZarrV3NodeType | None, tuple[ValidationProblem, ...]]:
"""The node type `value` declares, one of `ZARR_V3_NODE_TYPE`. None, with the problems, when it declares none of them or is not an object.

When a document declares no valid node type, its `zarr_format` is
checked too, so a document in another format is reported as not v3.
Expand All @@ -886,7 +887,7 @@ def _node_type(value: object) -> tuple[str | None, tuple[ValidationProblem, ...]
return None, not_an_object(value)
document = value
node_type = document.get("node_type")
if isinstance(node_type, str) and node_type in _NODE_TYPES:
if is_zarr_v3_node_type(node_type):
return node_type, ()
problems = [
*missing_keys(frozenset({"zarr_format"}), document),
Expand All @@ -895,7 +896,7 @@ def _node_type(value: object) -> tuple[str | None, tuple[ValidationProblem, ...]
if "node_type" not in document:
problems.append(ValidationProblem(("node_type",), "missing required key", "missing_key"))
else:
problems.append(outside_of(("node_type",), node_type, _NODE_TYPES))
problems.append(outside_of(("node_type",), node_type, ZARR_V3_NODE_TYPE))
return None, with_input(problems, document)


Expand Down
23 changes: 22 additions & 1 deletion packages/zarr-metadata/src/zarr_metadata/v2/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from typing import Annotated, Final, Literal, NotRequired

from annotated_types import Ge
from typing_extensions import TypedDict
from typing_extensions import TypedDict, TypeIs

from zarr_metadata._common import JSONValue
from zarr_metadata.v2._declaration_json import ZarrV2CodecMetadata, ZarrV2DataTypeMetadata
Expand All @@ -20,6 +20,12 @@
ZARR_V2_ARRAY_ORDER: Final = ("C", "F")
"""Tuple of permitted values for the `order` field of v2 array metadata."""


def is_zarr_v2_array_order(value: object) -> TypeIs[ZarrV2ArrayOrder]:
"""Whether `value` is one of `ZARR_V2_ARRAY_ORDER`, narrowing it to `ZarrV2ArrayOrder`."""
return value in ZARR_V2_ARRAY_ORDER


ZarrV2ArrayDimensionSeparator = Literal[".", "/"]
"""Literal type of permitted values for the `dimension_separator` field of v2 array metadata.

Expand All @@ -32,6 +38,18 @@
ZARR_V2_ARRAY_DIMENSION_SEPARATOR: Final = (".", "/")
"""Tuple of permitted values for the `dimension_separator` field of v2 array metadata."""

ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR: Final[ZarrV2ArrayDimensionSeparator] = "."
"""The `dimension_separator` a v2 array's metadata means when it has none.

"If the value is not set, then the default MUST be assumed to be `"."`"
(https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v2/v2.0.rst#L83-L86).
"""


def is_zarr_v2_array_dimension_separator(value: object) -> TypeIs[ZarrV2ArrayDimensionSeparator]:
"""Whether `value` is one of `ZARR_V2_ARRAY_DIMENSION_SEPARATOR`, narrowing it to `ZarrV2ArrayDimensionSeparator`."""
return value in ZARR_V2_ARRAY_DIMENSION_SEPARATOR


class ZarrV2ZArrayJSON(TypedDict):
"""
Expand Down Expand Up @@ -148,6 +166,7 @@ class ZarrV2ArrayMetadataJSON(TypedDict, extra_items=JSONValue):


__all__ = [
"ZARR_V2_ARRAY_DEFAULT_DIMENSION_SEPARATOR",
"ZARR_V2_ARRAY_DIMENSION_SEPARATOR",
"ZARR_V2_ARRAY_METADATA_STORE_KEY",
"ZARR_V2_ARRAY_ORDER",
Expand All @@ -156,4 +175,6 @@ class ZarrV2ArrayMetadataJSON(TypedDict, extra_items=JSONValue):
"ZarrV2ArrayMetadataStoreKey",
"ZarrV2ArrayOrder",
"ZarrV2ZArrayJSON",
"is_zarr_v2_array_dimension_separator",
"is_zarr_v2_array_order",
]
15 changes: 12 additions & 3 deletions packages/zarr-metadata/src/zarr_metadata/v3/__init__.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
"""Zarr v3 metadata types, and the front door to the v3 definitions.

This module holds the JSON of any declaration (a bare name or a
`ZarrV3NamedConfigJSON` envelope) and the node name and path
types. Each document type is in the submodule for its node: `array`,
`ZarrV3NamedConfigJSON` envelope), the node type, and the node name and
path types. Each document type is in the submodule for its node: `array`,
`group` and `consolidated`. The extensions are in `codec`, `data_type`,
`chunk_grid` and `chunk_key_encoding`, and the v3 kinds and scopes are in
`definition`.
Expand All @@ -18,7 +18,13 @@

from zarr_metadata._common import ZarrV3NamedConfigJSON
from zarr_metadata._renamed import renamed_scope
from zarr_metadata.v3._common import ZarrV3DeclarationJSON, ZarrV3ExtensionField
from zarr_metadata.v3._common import (
ZARR_V3_NODE_TYPE,
ZarrV3DeclarationJSON,
ZarrV3ExtensionField,
ZarrV3NodeType,
is_zarr_v3_node_type,
)
from zarr_metadata.v3._core import RENAMED_SCOPES
from zarr_metadata.v3._hierarchy import NodeName, NodePath
from zarr_metadata.v3.chunk_grid.rectilinear import RECTILINEAR_CHUNK_GRID
Expand Down Expand Up @@ -111,13 +117,16 @@
"VLEN_UTF8_CODEC",
"ZARR_PYTHON_CONTEXT",
"ZARR_PYTHON_RAW_BYTES_DATA_TYPE",
"ZARR_V3_NODE_TYPE",
"ZSTD_CODEC",
"NodeName",
"NodePath",
"ZarrV3Context",
"ZarrV3DeclarationJSON",
"ZarrV3ExtensionField",
"ZarrV3NamedConfigJSON",
"ZarrV3NodeType",
"is_zarr_v3_node_type",
]


Expand Down
29 changes: 26 additions & 3 deletions packages/zarr-metadata/src/zarr_metadata/v3/_common.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,38 @@
"""The v3 declaration: its JSON type, and the type aliases for members that hold a declaration.
"""The v3 node types, the v3 declaration's JSON type, and the type aliases for members that hold a declaration.

This module is private. It sits below both code paths that read a declaration:
the model, which checks the members of a document, and the definitions,
which read a declaration's configuration. Public consumers import
`ZarrV3NodeType`, `ZARR_V3_NODE_TYPE`, `is_zarr_v3_node_type` and
`ZarrV3DeclarationJSON` from `zarr_metadata.v3`, and the aliases from
`zarr_metadata.v3.definition`. One declaration is read with
`zarr_metadata.definition.resolve`.
"""

from typing import TypeAlias
from typing import Final, TypeAlias

from typing_extensions import TypeAliasType
from typing_extensions import TypeAliasType, TypeIs

from zarr_metadata._common import JSONValue, ZarrV3NamedConfigJSON
from zarr_metadata._hierarchy import NodeType

ZarrV3NodeType = NodeType
"""Literal type of permitted values for the `node_type` field of v3 metadata.

An array's document declares `"array"`
(https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L483-L487)
and a group's `"group"`
(https://github.com/zarr-developers/zarr-specs/blob/fc7dd9c9beb5a50b87f9b08b00bf50fc0048482f/docs/v3/core/index.rst#L760-L764).
"""

ZARR_V3_NODE_TYPE: Final = ("array", "group")
"""Tuple of permitted values for the `node_type` field of v3 metadata."""


def is_zarr_v3_node_type(value: object) -> TypeIs[ZarrV3NodeType]:
"""Whether `value` is one of `ZARR_V3_NODE_TYPE`, narrowing it to `ZarrV3NodeType`."""
return value in ZARR_V3_NODE_TYPE


ZarrV3ExtensionField: TypeAlias = JSONValue
"""The JSON value of an unknown top-level v3 metadata field.
Expand Down Expand Up @@ -59,12 +79,15 @@


__all__ = [
"ZARR_V3_NODE_TYPE",
"ZarrV3ChunkGridDeclaration",
"ZarrV3ChunkKeyEncodingDeclaration",
"ZarrV3CodecDeclaration",
"ZarrV3DataTypeDeclaration",
"ZarrV3DeclarationJSON",
"ZarrV3ExtensionField",
"ZarrV3NodeType",
"ZarrV3StaticCodecDeclaration",
"ZarrV3StorageTransformerDeclaration",
"is_zarr_v3_node_type",
]
Loading
Loading