From 2bf2534c85b09e53b1fe3d7b3b4cb842a92499c9 Mon Sep 17 00:00:00 2001 From: Janick Martinez Esturo Date: Sun, 11 Oct 2026 12:58:28 +0200 Subject: [PATCH 1/7] refactor(data): implement IndexedTarStore independently of zarr-python Separate the record storage of indexed tar archives (itar.py) from zarr-python, in preparation of supporting multiple zarr-python versions: - IndexedTarStore provides raw access to the records of an archive (mapping interface), stores.ZarrIndexedTarStore adapts it to zarr-python's store interface - Read local archives via positional reads (os.pread), which don't serialize reads of multiple threads, and close their file descriptors also when stores are dropped without closing them - Serve records starting within the index tail read from memory - Look up the record index decoder / encoder by index type, and share the CBOR + LZMA document encoding of the record index and the compressed consolidated metadata (encoding.py) Archives are byte-identical to archives written by ncore 19.8 (tested via the sha256 of a fixed hierarchy written by ncore 19.8). BREAKING CHANGE: IndexedTarStore is no longer a zarr-python store, wrap it via stores.ZarrIndexedTarStore to access zarr hierarchies of archives directly with zarr-python. --- ncore/impl/data/BUILD.bazel | 3 + ncore/impl/data/encoding.py | 45 +++ ncore/impl/data/itar.py | 339 +++++++++++++++++++++ ncore/impl/data/stores.py | 502 ++++--------------------------- ncore/impl/data/stores_test.py | 428 ++++++++++++++++---------- ncore/impl/data/v4/components.py | 23 +- 6 files changed, 728 insertions(+), 612 deletions(-) create mode 100644 ncore/impl/data/encoding.py create mode 100644 ncore/impl/data/itar.py diff --git a/ncore/impl/data/BUILD.bazel b/ncore/impl/data/BUILD.bazel index 5d7237a7..6bb60a4d 100644 --- a/ncore/impl/data/BUILD.bazel +++ b/ncore/impl/data/BUILD.bazel @@ -36,10 +36,13 @@ py_library( py_library( name = "pylib_stores", srcs = [ + "encoding.py", + "itar.py", "stores.py", ], deps = [ requirement("cbor2"), + requirement("numcodecs"), requirement("universal_pathlib"), requirement("zarr"), ], diff --git a/ncore/impl/data/encoding.py b/ncore/impl/data/encoding.py new file mode 100644 index 00000000..277221c4 --- /dev/null +++ b/ncore/impl/data/encoding.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compressed binary encodings of JSON-like documents (record indices of indexed tar archives, compressed consolidated +zarr metadata)""" + +from __future__ import annotations + +import io +import lzma + +from typing import Any + +import cbor2 + + +def encode_cbor_xz(document: Any) -> bytes: + """Encodes a document as CBOR + LZMA (xz container format)""" + with io.BytesIO() as buffer: + with lzma.open(buffer, "wb", format=lzma.FORMAT_XZ) as lzma_file: + cbor2.dump(document, lzma_file) + return buffer.getvalue() + + +def decode_cbor_xz(data: bytes) -> Any: + """Decodes a CBOR + LZMA (xz container format) encoded document""" + return cbor2.loads(lzma.LZMADecompressor().decompress(data)) + + +__all__ = [ + "decode_cbor_xz", + "encode_cbor_xz", +] diff --git a/ncore/impl/data/itar.py b/ncore/impl/data/itar.py new file mode 100644 index 00000000..d49e50c7 --- /dev/null +++ b/ncore/impl/data/itar.py @@ -0,0 +1,339 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Indexed tar archives (``.itar``), independent of zarr-python. + +An indexed tar archive is a regular tar archive with a compressed record index appended after the tar end-of-archive +blocks, enabling O(1) lookups of record payloads. :class:`IndexedTarStore` provides raw access to the records of such +archives; zarr-python version specific store adapters (see :mod:`ncore.impl.data.stores`) store zarr hierarchies in +them. + +Records are *write-once*: archives are append-only, and writing a record twice raises a ``ValueError``. +""" + +from __future__ import annotations + +import io +import logging +import os +import struct +import tarfile +import threading +import weakref + +from enum import IntEnum, unique +from pathlib import Path +from typing import IO, Any, Callable, ClassVar, Dict, Iterator, List, Literal, NamedTuple, Optional, Tuple, Union, cast + +from upath import UPath + +from ncore.impl.data.encoding import decode_cbor_xz, encode_cbor_xz + + +_logger = logging.getLogger(__name__) + + +class IndexedTarStore: + """The records of an indexed tar archive, opened for reading or (truncating) writing. + + Parameters + ---------- + itar_path : string, Path or UPath + Location of the archive (local or remote). + mode : string, optional, default 'r' + 'r' to read an existing archive, or 'w' to create (truncate) and write an archive. + index_tail_read_size : int or None, optional, default 1 << 20 (1 MiB) + Maximum bytes read from the end of the archive in a single I/O call to load the record index. Covers both the + small index header and (ideally) the compressed index. Values smaller than the tar block size (512 bytes), or + None, result in separate reads of the index header and index, which is more robust for large indices. + + Records are accessed via a mapping interface (``store[key]``, ``store[key] = value``, ``key in store``, + ``keys()``, ``len(store)``). Records of the index tail read are served from memory, so records stored near the end + of the archive don't repeat its high-latency I/O. Local archives are read via positional reads (``os.pread``), + which don't serialize reads of multiple threads. + + Archives opened for writing need to be closed (``close()``, or by using the store as context manager), which + appends the record index. + + All methods are thread-safe. + """ + + class Record(NamedTuple): + """The payload of a record within the archive file""" + + offset: int + size: int + + class TailBuffer(NamedTuple): + """Bytes read from the end of the archive file when loading the index""" + + start: int #: File offset of the first byte + data: bytes + + @unique + class IndexType(IntEnum): + """Serialization formats of the record index""" + + CBOR_LZMA_XZ_V1 = 1 + + #: Index header (little-endian): magic (4 bytes), index type (uint32), index offset (uint64), index size (uint32) + INDEX_HEADER_FORMAT: ClassVar[str] = "<4sIQI" + INDEX_HEADER_MAGIC: ClassVar[bytes] = b"itar" + + #: Index type of written archives + INDEX_TYPE: ClassVar[IndexType] = IndexType.CBOR_LZMA_XZ_V1 + + #: Decoders / encoders of the supported index types (the index is a structure of arrays: record keys, payload + #: offsets and payload sizes, ordered by payload offsets) + INDEX_CODECS: ClassVar[Dict[IndexType, Tuple[Callable[[bytes], Any], Callable[[Any], bytes]]]] = { + IndexType.CBOR_LZMA_XZ_V1: (decode_cbor_xz, encode_cbor_xz), + } + + def __init__( + self, + itar_path: Union[str, Path, UPath], + mode: Literal["r", "w"] = "r", + index_tail_read_size: Optional[int] = 1 << 20, # 1 MiB by default + ) -> None: + if mode not in ("r", "w"): + raise ValueError(f"IndexedTarStore: unsupported mode {mode!r}, supported are 'r' and 'w'") + self._mode: Literal["r", "w"] = mode + + # convert str / Path to an absolute UPath (with the UPath-internal `file://` protocol for local files) + itar_upath = UPath(itar_path) + if itar_upath.protocol == "": + itar_upath = UPath("file://" + str(itar_upath)) + self._itar_upath = itar_upath.absolute() + + # the tarfile module isn't thread-safe: writing and file-object based reading are serialized + self._lock = threading.RLock() + + self._records: Dict[str, IndexedTarStore.Record] = {} + self._tail: Optional[IndexedTarStore.TailBuffer] = None + self._tar_file: Optional[tarfile.TarFile] = None + self._file: IO[bytes] = self._open_file(truncate=True) + if mode == "r": + self._records, self._tail = self._load_index(self._file, index_tail_read_size) + else: + self._tar_file = tarfile.TarFile(fileobj=self._file, mode="w") + + self._pread_fd: Optional[int] = None + self._pread_finalizer: Optional[weakref.finalize] = None + self._open_positional_reads() + + @property + def mode(self) -> Literal["r", "w"]: + """Access mode of the archive ('r' for reading, 'w' for writing)""" + return self._mode + + @property + def itar_upath(self) -> UPath: + """Absolute location of the archive""" + return self._itar_upath + + def __eq__(self, value: object) -> bool: + return isinstance(value, IndexedTarStore) and (self._itar_upath, self._mode) == (value._itar_upath, value._mode) + + def __hash__(self) -> int: + return hash((self._itar_upath, self._mode)) + + def __repr__(self) -> str: + return f"IndexedTarStore('{self._itar_upath}', mode='{self._mode}')" + + # ------------------------------------------------------------------------- + # Records + # ------------------------------------------------------------------------- + + def keys(self) -> List[str]: + """Returns the keys of all records (in archive order)""" + with self._lock: + return list(self._records) + + def __iter__(self) -> Iterator[str]: + return iter(self.keys()) + + def __len__(self) -> int: + return len(self._records) + + def __contains__(self, key: object) -> bool: + return key in self._records + + def __getitem__(self, key: str) -> bytes: + """Reads the payload of a record (raises KeyError if it doesn't exist)""" + record = self._records[key] + + # (records end before the index, which ends the tail read: records starting within the tail read are part of it) + if (tail := self._tail) is not None and record.offset >= tail.start: + start = record.offset - tail.start + return tail.data[start : start + record.size] + + if self._pread_fd is not None: + return self._pread(self._pread_fd, record.offset, record.size) + + with self._lock: + # restore the position afterwards (the position to append records to when writing) + position = self._file.tell() + self._file.seek(record.offset) + value = self._file.read(record.size) + self._file.seek(position) + return value + + def __setitem__(self, key: str, value: Union[bytes, bytearray, memoryview]) -> None: + """Appends a record (requires write mode, raises ValueError if the record exists already)""" + if self._mode != "w": + raise ValueError("IndexedTarStore: archive is opened for reading and doesn't support writing") + payload = bytes(value) + + with self._lock: + if key in self._records: + raise ValueError(f"IndexedTarStore: record {key} already exists, records are write-once") + + # the header of a record can span multiple tar blocks (long keys), the payload is padded to full blocks + info = tarfile.TarInfo(key) + info.size = len(payload) + cast(tarfile.TarFile, self._tar_file).addfile(info, fileobj=io.BytesIO(payload)) + padded_size = -(-len(payload) // tarfile.BLOCKSIZE) * tarfile.BLOCKSIZE + self._records[key] = self.Record(self._file.tell() - padded_size, len(payload)) + + def __delitem__(self, key: str) -> None: + raise NotImplementedError("IndexedTarStore: records can't be deleted") + + # ------------------------------------------------------------------------- + # Lifecycle + # ------------------------------------------------------------------------- + + def __enter__(self) -> IndexedTarStore: + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + self.close() + + def close(self) -> None: + """Closes the archive (when writing: appends the record index, which is required to read the archive)""" + with self._lock: + if self._file.closed: + return + if self._mode == "w": + # closing the tar file appends the end-of-archive blocks, but keeps the file object open + cast(tarfile.TarFile, self._tar_file).close() + self._save_index(self._file, self._records) + self._file.close() + self._close_positional_reads() + + def reload_resources(self) -> None: + """Re-opens the file objects (required in processes forked after opening the archive)""" + with self._lock: + position = self._file.tell() + self._file.close() + self._file = self._open_file(truncate=False) + self._file.seek(position) + if self._tar_file is not None: + self._tar_file.fileobj = self._file + self._close_positional_reads() + self._open_positional_reads() + + def _open_file(self, truncate: bool) -> IO[bytes]: + if self._mode == "r": + return self._itar_upath.open("rb") + # writing requires a readable file (universal_pathlib <= 0.2.6, the last version supporting python 3.8, + # supports "wb+" / "rb+" without declaring them in its type annotations) + return self._itar_upath.open("wb+" if truncate else "rb+") # type: ignore[call-overload] + + def _open_positional_reads(self) -> None: + """Opens a file descriptor for positional reads of local archives opened for reading, which is closed with the + archive (or when the store is garbage-collected)""" + if self._mode == "r" and self._itar_upath.protocol in ("file", "local") and hasattr(os, "pread"): + self._pread_fd = os.open(self._itar_upath.path, os.O_RDONLY) + self._pread_finalizer = weakref.finalize(self, os.close, self._pread_fd) + + def _close_positional_reads(self) -> None: + if self._pread_finalizer is not None: + self._pread_finalizer() + self._pread_finalizer = None + self._pread_fd = None + + # ------------------------------------------------------------------------- + # Positional reads / index serialization + # ------------------------------------------------------------------------- + + @staticmethod + def _pread(fd: int, offset: int, size: int) -> bytes: + value = os.pread(fd, size, offset) + while len(value) < size: # positional reads may return fewer bytes than requested + if not (chunk := os.pread(fd, size - len(value), offset + len(value))): + raise EOFError(f"IndexedTarStore: archive ends within record at offset {offset}") + value += chunk + return value + + @classmethod + def _load_index( + cls, file: IO[bytes], tail_read_size: Optional[int] + ) -> Tuple[Dict[str, IndexedTarStore.Record], IndexedTarStore.TailBuffer]: + """Loads the record index from the end of an archive file (with a single read if the index lies within the tail + read), and returns it together with the tail read""" + file.seek(0, os.SEEK_END) + file_size = file.tell() + + tail_size = min(file_size, max(tarfile.BLOCKSIZE, tail_read_size or 0)) + file.seek(file_size - tail_size) + tail = cls.TailBuffer(file_size - tail_size, file.read(tail_size)) + + # the index header is stored in the last tar block + header_offset = len(tail.data) - tarfile.BLOCKSIZE + magic, index_type, index_offset, index_size = struct.unpack_from( + cls.INDEX_HEADER_FORMAT, tail.data, header_offset + ) + if magic != cls.INDEX_HEADER_MAGIC: + raise ValueError("IndexedTarStore: invalid index header, can't load indexed tar archive") + if index_type not in cls.INDEX_CODECS: + raise TypeError(f"IndexedTarStore: unsupported index type {index_type}") + decode_index = cls.INDEX_CODECS[cls.IndexType(index_type)][0] + + if index_offset >= tail.start: + index = tail.data[index_offset - tail.start : index_offset - tail.start + index_size] + else: + file.seek(index_offset) + index = file.read(index_size) + + table = decode_index(index) + _logger.debug(f"IndexedTarStore: loaded index of {len(table['items'])} records ({index_size} bytes)") + records = { + key: cls.Record(offset, size) + for key, offset, size in zip(table["items"], table["offset_datas"], table["sizes"]) + } + return records, tail + + @classmethod + def _save_index(cls, file: IO[bytes], records: Dict[str, IndexedTarStore.Record]) -> None: + """Appends the record index and its header to a finalized archive file (after the end-of-archive blocks)""" + + def pad_to_block() -> None: + if remainder := file.tell() % tarfile.BLOCKSIZE: + file.write(tarfile.NUL * (tarfile.BLOCKSIZE - remainder)) + + index_offset = file.tell() + assert index_offset % tarfile.BLOCKSIZE == 0, "IndexedTarStore: archive end not at a tar block boundary" + + table = sorted(((key, record.offset, record.size) for key, record in records.items()), key=lambda item: item[1]) + keys, offsets, sizes = (list(column) for column in zip(*table)) if table else ([], [], []) + index = cls.INDEX_CODECS[cls.INDEX_TYPE][1]({"items": keys, "offset_datas": offsets, "sizes": sizes}) + file.write(index) + pad_to_block() + + file.write( + struct.pack(cls.INDEX_HEADER_FORMAT, cls.INDEX_HEADER_MAGIC, cls.INDEX_TYPE, index_offset, len(index)) + ) + pad_to_block() + _logger.debug(f"IndexedTarStore: saved index of {len(records)} records ({len(index)} bytes)") diff --git a/ncore/impl/data/stores.py b/ncore/impl/data/stores.py index a7a588f1..4d80b0ff 100644 --- a/ncore/impl/data/stores.py +++ b/ncore/impl/data/stores.py @@ -13,494 +13,124 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import annotations +"""zarr stores of NCore data: indexed tar archives (``.itar``) and their compressed consolidated metadata""" -import io -import logging -import lzma -import os -import struct -import tarfile +from __future__ import annotations -from dataclasses import dataclass, field -from enum import IntEnum, auto, unique -from pathlib import Path -from threading import RLock -from typing import IO, Any, Dict, Iterator, Literal, NamedTuple, Optional, Union, cast +from typing import Iterator, List, Mapping, Union -import cbor2 import zarr from numcodecs import compat -from upath import UPath from zarr._storage.store import Store from zarr.util import json_loads +from ncore.impl.data.encoding import decode_cbor_xz, encode_cbor_xz +from ncore.impl.data.itar import IndexedTarStore -_logger = logging.getLogger(__name__) - - -class IndexedTarStore(Store): - """A zarr store over *indexed* tar files - - Parameters - ---------- - itar_path : string - Location of the tar file (needs to end with '.itar'). - mode : string, optional, default 'r' - One of 'r' to read an existing file, or 'w' to truncate and write a new - file. - index_tail_read_size : int or None, optional, default 1 << 20 (1 MiB) - Maximum bytes read from the tail of the file in a single I/O call to load the tar index. - Covers both the small index header and (ideally) the compressed index payload. - If not specified, perform separate reads for header and payload, - which is more robust for large indices, but requires two I/O calls. - Will be increased to at least one tar block size if specified smaller, - since the header is located in the last block. - Default 1 MiB size fits typical use cases, but may be increased if the compressed - index size is expected to be larger or decreased if remote access chunk sizes are smaller. - - Read mode retains the byte range loaded for the index (the tail read) as an in-memory - cache. Any store key whose payload lies entirely in that range is served from the cache - without an additional read, so overlapping zarr chunks near the end of the archive do - not repeat the same high-latency tail I/O. - - After modifying a IndexedTarStore, the ``close()`` method must be called, otherwise - essential data will not be written to the underlying files. The IndexedTarStore - class also supports the context manager protocol, which ensures the ``close()`` - method is called on leaving the context, e.g.:: - - >>> with IndexedTarStore('data/array.itar', mode='w') as store: - ... z = zarr.zeros((10, 10), chunks=(5, 5), store=store) - ... z[...] = 42 - ... # no need to call store.close() - - """ - - _erasable = False - - @dataclass - class TarRecord: - """A file record within a tar file""" - - offset_data: int - size: int - - @dataclass - class TarRecordIndex: - """All file records within a tar file""" - - records: Dict[str, IndexedTarStore.TarRecord] = field(default_factory=dict) - - class TailBuffer(NamedTuple): - """Cached byte range from the tail of the archive file. - Retains the raw bytes of the single tail I/O read performed when the - index is loaded so that payloads falling entirely within that range can - be served from memory without an additional file read. - """ +#: Default key of the compressed consolidated metadata of zarr hierarchies (CBOR + LZMA encoded ``.zmetadata``) +CONSOLIDATED_METADATA_KEY = ".zmetadata.cbor.xz" - start: int #: File byte offset where the tail read began - data: bytes #: Raw bytes read from *start* to the end of the file - def __init__( - self, - itar_path: Union[str, Path, UPath], - mode: Literal["r", "w"] = "r", - index_tail_read_size: Optional[int] = 1 << 20, # 1 MiB by default - ) -> None: - if mode not in ["r", "w"]: - raise ValueError("TarRecordIndex: only r/w modes supported") +class ZarrIndexedTarStore(Store): + """zarr-python store of the records of an :class:`IndexedTarStore`""" - # store properties - self.mode = mode - - # Current understanding is that tarfile module in stdlib is not thread-safe, - # and so locking is required for both read and write. However, this has not - # been investigated in detail, perhaps no lock is needed if mode='r'. - self.mutex = RLock() - - # convert str / Path to absolute UPath uncondtionally - itar_upath = UPath(itar_path) - if itar_upath.protocol == "": - # use UPath-internal `file://` protocol for local files - itar_upath = UPath("file://" + str(itar_upath)) - - self.itar_upath = itar_upath.absolute() + _erasable = False - self._tail_buffer: Optional[IndexedTarStore.TailBuffer] = None + def __init__(self, store: IndexedTarStore) -> None: + self.store = store - # open file object (require file to be both writeable and readable when writing) and - # tar file (only required for write mode, read mode directly uses file object) - self.tar_file_object: IO[Any] - self.tar_file: Optional[tarfile.TarFile] = None - if self.mode == "r": - self.tar_file_object = self.itar_upath.open("rb") - else: - # universal_path for Python 3.8 (<=0.2.6) doesn't expose a - # write/read mode in it's static type-hints, although "wb+" is still accepted - # if the FS supports it, so ignore type-checker here - self.tar_file_object = self.itar_upath.open("wb+") # type: ignore[call-overload] + def __getitem__(self, key: str) -> bytes: + return self.store[key] - self.tar_file = tarfile.TarFile(fileobj=self.tar_file_object, mode="w") + def __setitem__(self, key: str, value: object) -> None: + if self.store.mode != "w": + raise zarr.errors.ReadOnlyError() + self.store[key] = compat.ensure_bytes(value) - # init / load index table - if self.mode == "r": - self.index, self._tail_buffer = self._load_tar_index(self.tar_file_object, index_tail_read_size) - else: - self.index = self.TarRecordIndex() + def __delitem__(self, key: str) -> None: + raise NotImplementedError("IndexedTarStore: records can't be deleted") - def __delitem__(self, _: str) -> None: - raise NotImplementedError("Deleting items is not supported") + def __contains__(self, key: object) -> bool: + return key in self.store def __iter__(self) -> Iterator[str]: - with self.mutex: - return iter(self.index.records.keys()) + return iter(self.store.keys()) def __len__(self) -> int: - with self.mutex: - return len(self.index.records) - - def __contains__(self, item: object) -> bool: - with self.mutex: - return item in self.index.records - - def __getitem__(self, item: str) -> bytes: - with self.mutex: - # Query index for file record - record = self.index.records[item] # raises KeyError if not in archive - - # Check if payload is contained in tail buffer - if (tail := self._tail_buffer) is not None: - tail_start = tail.start - tail_end = tail_start + len(tail.data) - offset_data = record.offset_data - if offset_data >= tail_start and offset_data + record.size <= tail_end: - return tail.data[offset_data - tail_start : offset_data - tail_start + record.size] - - # Remember current tar file position - current_position = self.tar_file_object.tell() - - # Read the value - self.tar_file_object.seek(record.offset_data) - value = self.tar_file_object.read(record.size) - - # Return tar file to previous location - self.tar_file_object.seek(current_position) + return len(self.store) - return value + def __eq__(self, value: object) -> bool: + return isinstance(value, ZarrIndexedTarStore) and self.store == value.store - def __setitem__(self, item: str, value) -> None: - if self.mode != "w": - raise zarr.errors.ReadOnlyError + def __hash__(self) -> int: + return hash(self.store) - with self.mutex: - if item in self.index.records: - raise ValueError(f"{item} already exists, update is not supported") - - value_bytes: bytes = compat.ensure_bytes(value) - value_size: int = len(value_bytes) - - # Remember current tar file position, which is the start of the header - header_start_position = self.tar_file_object.tell() - - # Store value in tar-file (will pre-pend a potentially *multi*-block header depending on item path-lengths) - tarinfo = tarfile.TarInfo(item) - tarinfo.size = value_size - - cast(tarfile.TarFile, self.tar_file).addfile(tarinfo, fileobj=io.BytesIO(value_bytes)) - - # End position after writing both header and payload - end_position = self.tar_file_object.tell() - - # Determine the effective value's payload size as a multiple of blocksize - payload_size = value_size - if remainder := payload_size % tarfile.BLOCKSIZE: - payload_size += tarfile.BLOCKSIZE - remainder + def close(self) -> None: + self.store.close() - # Determine the effective header-size (can be multiple blocks for long path names) - header_size = end_position - header_start_position - payload_size - # Construct record from reconstructed size-information - record = self.TarRecord( - # Effective start of the data in the tar file (current tar file position + header-size) - header_start_position + header_size, - # Length of the data - value_size, - ) +StoreLike = Union[IndexedTarStore, zarr.storage.BaseStore] - self.index.records[item] = record - def __enter__(self) -> IndexedTarStore: - return self +def _zarr_store(store: StoreLike) -> zarr.storage.BaseStore: + """The zarr-python store of a store (wraps indexed tar stores)""" + return ZarrIndexedTarStore(store) if isinstance(store, IndexedTarStore) else store - def __exit__(self, exc_type, exc_value, traceback) -> None: - self.close() - def close(self) -> None: - """Needs to be called after finishing updating the store""" - with self.mutex: - if self.mode == "w": - # Closing the tar file appends two finishing blocks to the end of the file - # if in write mode, but doesn't close the internal file object yet - cast(tarfile.TarFile, self.tar_file).close() - - # Add index if writing - self._save_tar_index(self.tar_file_object, self.index) - - self.tar_file_object.close() - - def reload_resources(self) -> None: - """Reloads the tar file object *only* - useful to re-initialize the store in multi-process 'fork()' settings""" - with self.mutex: - # get current tar file path and seek positions, and close file object - current_position = self.tar_file_object.tell() - self.tar_file_object.close() - - # reload file object (require file to be both writeable and readable when writing) - if self.mode == "r": - self.tar_file_object = self.itar_upath.open("rb") - else: - # universal_path for Python 3.8 (<=0.2.6) doesn't expose a - # write/read mode in it's static type-hints, although "rb+" is still accepted - # if the FS supports it, so ignore type-checker here - self.tar_file_object = self.itar_upath.open("rb+") # type: ignore[call-overload] - - if self.mode == "w": - # assign the new file object to the tar file (only required for write mode) - cast(tarfile.TarFile, self.tar_file).fileobj = self.tar_file_object - - # seek to previous position - self.tar_file_object.seek(current_position) - - # Methods / constants for storing index header and payload - INDEX_HEADER_MAGIC = b"itar" - - # Index header binary format (20-bytes) - # - # tuple[TarRecordIndex, TailBuffer]: - """Loads a tar record index from the end of a tar file object. - - Returns the parsed index and a :class:`TailBuffer` representing the - single tail read used to locate the index. Callers may reuse the - buffer to satisfy reads for payloads that fall entirely within that range. - """ - # Determine tail read size - if index_tail_read_size is None or index_tail_read_size < tarfile.BLOCKSIZE: - index_tail_read_size = tarfile.BLOCKSIZE # explicit separate reads - - original_file_position = tar_file_object.tell() - - # Determine file size - tar_file_object.seek(0, os.SEEK_END) - file_size = tar_file_object.tell() - - # Read up to index_tail_read_size from the tail in one read call - tail_buffer_size = min(file_size, index_tail_read_size) - tar_file_object.seek(file_size - tail_buffer_size) - tail_buffer = tar_file_object.read(tail_buffer_size) - - # Extract the header from the last 512-byte block (it's guaranteed that the header fits into a single block) - header_binary_size = struct.calcsize(cls.INDEX_HEADER_FORMAT) - header_offset_in_tail_buffer = tail_buffer_size - tarfile.BLOCKSIZE - header = cls.IndexHeader._make( - struct.unpack( - cls.INDEX_HEADER_FORMAT, - tail_buffer[header_offset_in_tail_buffer : header_offset_in_tail_buffer + header_binary_size], - ) - ) - - if header.magic != cls.INDEX_HEADER_MAGIC: - raise ValueError("IndexedTarStore: invalid index header, can't load indexed tar file") - - # Try to extract the compressed index payload from the tail buffer - index_start_in_file = file_size - tail_buffer_size - if header.offset >= index_start_in_file: - # Payload is fully within the buffer we already read - index_start_in_tail_buffer = header.offset - index_start_in_file - index_binary = tail_buffer[index_start_in_tail_buffer : index_start_in_tail_buffer + header.size] - else: - # Rare: compressed index > index_tail_read_size — fall back to a second read - tar_file_object.seek(header.offset) - index_binary = tar_file_object.read(header.size) - - tar_file_object.seek(original_file_position) - - if header.type == cls.IndexType.CBOR_LZMA_XZ_V1.value: - _logger.debug(f"IndexedTarStore: lzma-compressed (xz archive format) index load size={len(index_binary)}") - - # load table (SOA) - table = cbor2.loads(lzma.LZMADecompressor().decompress(index_binary)) - items = table["items"] - offset_datas = table["offset_datas"] - sizes = table["sizes"] - else: - raise TypeError(f"IndexedTarStore: unsupported header type {header.type}") - - # Construct record index from loaded table - index = cls.TarRecordIndex({item: cls.TarRecord(offset_datas[i], sizes[i]) for i, item in enumerate(items)}) - return index, cls.TailBuffer(start=index_start_in_file, data=tail_buffer) - - @classmethod - def _save_tar_index(cls, tar_file_object: IO[Any], index: TarRecordIndex) -> None: - """Saves a tar record index at the end of a tar file object (needs to be finalized / have two empty blocks appended already)""" - - def fill_block() -> None: - # Fill up block with zeros - _, remainder = divmod(tar_file_object.tell(), tarfile.BLOCKSIZE) - if remainder > 0: - tar_file_object.write(tarfile.NUL * (tarfile.BLOCKSIZE - remainder)) - - assert tar_file_object.tell() % tarfile.BLOCKSIZE == 0, "Tar file not at block boundary" - - # Remember where we are storing the index - index_offset = tar_file_object.tell() - - assert index_offset % tarfile.BLOCKSIZE == 0, "Tar file not at block boundary" - - # Reformat index table as SOA (sorted by offset) - table = [(item, record.offset_data, record.size) for (item, record) in index.records.items()] - items, offset_datas, sizes = list(zip(*sorted(table, key=lambda data: data[1]))) if len(table) else ([], [], []) - - # Append compressed table to tar file - with io.BytesIO() as index_buffer: - # Compress table to in-memory buffer - with lzma.open(index_buffer, "wb", format=lzma.FORMAT_XZ) as lzma_file: - cbor2.dump({"items": items, "offset_datas": offset_datas, "sizes": sizes}, lzma_file) - - index_binary = index_buffer.getvalue() - index_size = len(index_binary) - - _logger.debug(f"IndexedTarStore lzma-compressed index store size={index_size}") - - # Append buffer to tar file - tar_file_object.write(index_binary) - - fill_block() - - # Create index header block - assert struct.calcsize(cls.INDEX_HEADER_FORMAT) <= tarfile.BLOCKSIZE, ( - "Index header larger than single block size" - ) - header_binary = struct.pack( - cls.INDEX_HEADER_FORMAT, - cls.INDEX_HEADER_MAGIC, - cls.IndexType.CBOR_LZMA_XZ_V1.value, - index_offset, - index_size, - ) - _logger.debug(f"IndexedTarStore: header store size={len(header_binary)}") - - # Append index header to tar file - tar_file_object.write(header_binary) - fill_block() - - -def consolidate_compressed_metadata(store: zarr.storage.BaseStore, metadata_key=".zmetadata.cbor.xz") -> None: - """Consolidate all metadata for groups and arrays within the given store - into a single compressed cbor resource and put it under the given key. +def consolidate_compressed_metadata(store: StoreLike, metadata_key: str = CONSOLIDATED_METADATA_KEY) -> None: + """Consolidates the metadata of all groups and arrays of a store into a single compressed record (CBOR + LZMA + encoded zarr consolidated metadata) See Also -------- zarr.consolidate_metadata """ - store = zarr.storage.normalize_store_arg(store, mode="w") - - version = store._store_version - - if version == 2: - - def is_zarr_key(key) -> bool: - return key.endswith(".zarray") or key.endswith(".zgroup") or key.endswith(".zattrs") - - else: + store = zarr.storage.normalize_store_arg(_zarr_store(store), mode="w") + if store._store_version != 2: raise NotImplementedError("Only supporting V2 stores") - # Collect all meta-data - out = { - "zarr_consolidated_format": 1, - "metadata": {key: json_loads(store[key]) for key in store if is_zarr_key(key)}, - } - - with io.BytesIO() as metadata_buffer: - # Compress meta-data to in-memory buffer - with lzma.open(metadata_buffer, "wb") as lzma_file: - cbor2.dump(out, lzma_file) - - store[metadata_key] = metadata_buffer.getvalue() + metadata = {key: json_loads(store[key]) for key in store if key.endswith((".zarray", ".zgroup", ".zattrs"))} + store[metadata_key] = encode_cbor_xz({"zarr_consolidated_format": 1, "metadata": metadata}) class ConsolidatedCompressedMetadataStore(zarr.storage.ConsolidatedMetadataStore): - """A layer over other storage, where the metadata has been consolidated into a single compressed key.""" - - # Overwrite constructor to perform decompression of metadata - def __init__(self, store: zarr.storage.StoreLike, metadata_key=".zmetadata.cbor.xz") -> None: - self.store = Store._ensure_store(store) + """A layer over a store, where the metadata of all nodes is read from the compressed consolidated metadata""" - # retrieve consolidated metadata - meta = cbor2.loads(lzma.LZMADecompressor().decompress(self.store[metadata_key])) + def __init__(self, store: StoreLike, metadata_key: str = CONSOLIDATED_METADATA_KEY) -> None: + self.store = Store._ensure_store(_zarr_store(store)) - # check format of consolidated metadata - consolidated_format = meta.get("zarr_consolidated_format", None) - if consolidated_format != 1: - raise zarr.MetadataError("unsupported zarr consolidated metadata format: %s" % consolidated_format) + consolidated = decode_cbor_xz(self.store[metadata_key]) + if (consolidated_format := consolidated.get("zarr_consolidated_format")) != 1: + raise zarr.MetadataError(f"unsupported zarr consolidated metadata format: {consolidated_format}") - # decode metadata - self.meta_store: zarr.storage.Store = zarr.KVStore(meta["metadata"]) + self.meta_store: Mapping[str, object] = zarr.KVStore(consolidated["metadata"]) def open_compressed_consolidated( - store: zarr.storage.StoreLike, metadata_key=".zmetadata.cbor.xz", mode="r+", **kwargs -) -> zarr.hierarchy.Group: - """Open group using metadata previously consolidated and compressed into a single key. + store: StoreLike, metadata_key: str = CONSOLIDATED_METADATA_KEY, mode: str = "r+", **kwargs +) -> zarr.Group: + """Opens a group of a store via its compressed consolidated metadata See Also -------- consolidate_compressed_metadata zarr.open_consolidated """ - - # normalize parameters - zarr_version = kwargs.get("zarr_version") - store = zarr.storage.normalize_store_arg( - store, storage_options=kwargs.get("storage_options"), mode=mode, zarr_version=zarr_version - ) if mode not in {"r", "r+"}: - raise ValueError("invalid mode, expected either 'r' or 'r+'; found {!r}".format(mode)) - - path = kwargs.pop("path", None) - if store._store_version == 2: - ConsolidatedStoreClass = ConsolidatedCompressedMetadataStore - else: - raise NotImplementedError("Only supporting V2 stores") - - # setup metadata store - meta_store = ConsolidatedStoreClass(store, metadata_key=metadata_key) - - # pass through - chunk_store = kwargs.pop("chunk_store", None) or store - return zarr.convenience.open(store=meta_store, chunk_store=chunk_store, mode=mode, path=path, **kwargs) + raise ValueError(f"invalid mode, expected either 'r' or 'r+'; found {mode!r}") + meta_store = ConsolidatedCompressedMetadataStore(store, metadata_key=metadata_key) + chunk_store = kwargs.pop("chunk_store", None) or meta_store.store + return zarr.open(store=meta_store, chunk_store=chunk_store, mode=mode, **kwargs) + + +__all__: List[str] = [ + "CONSOLIDATED_METADATA_KEY", + "ConsolidatedCompressedMetadataStore", + "IndexedTarStore", + "StoreLike", + "ZarrIndexedTarStore", + "consolidate_compressed_metadata", + "open_compressed_consolidated", +] diff --git a/ncore/impl/data/stores_test.py b/ncore/impl/data/stores_test.py index 030cb0c6..19c7d10c 100644 --- a/ncore/impl/data/stores_test.py +++ b/ncore/impl/data/stores_test.py @@ -13,195 +13,295 @@ # See the License for the specific language governing permissions and # limitations under the License. -import itertools +import gc +import hashlib +import os +import struct +import tarfile import tempfile import unittest from pathlib import Path -from typing import IO, Any, Optional +from typing import Any, Optional +from unittest import mock import numpy as np import parameterized import zarr -from ncore.impl.data.stores import IndexedTarStore, consolidate_compressed_metadata, open_compressed_consolidated +from ncore.impl.data.encoding import decode_cbor_xz +from ncore.impl.data.stores import ( + CONSOLIDATED_METADATA_KEY, + ConsolidatedCompressedMetadataStore, + IndexedTarStore, + ZarrIndexedTarStore, + consolidate_compressed_metadata, + open_compressed_consolidated, +) -COMPRESSED_CONSOLIDATED_VALUES = [False, True] -INDEX_TAIL_READ_SIZES = [ - None, - 1 << 10, - 1 << 20, - 1, - 512, - 513, -] # None (explicit separate reads), 0.5 MiB, 1 MiB (default), 1 byte / 512 / 513 bytes (edge cases resulting in separate reads) +#: None / values < 512 (separate reads of index header and index), 1 KiB, 1 MiB (default), edge cases around 512 +INDEX_TAIL_READ_SIZES = [None, 1, 512, 513, 1 << 10, 1 << 20] + + +def _fill_reference(root: zarr.Group) -> None: + rng = np.random.default_rng(0) + root.attrs.put({"some": "thing"}) + root.create_dataset("foo", data=rng.random((3, 3, 3)), chunks=(2, 3, 3)) + sub = root.create_group("subgroup") + sub.attrs.put({"other": [1, 2, 3]}) + sub.create_dataset("foo", data=rng.random((5, 5, 5))) + sub.create_dataset("blob", data=np.asarray(b"\x89PNG\x00some binary data"), compressor=None) + + +def _check_reference(test: unittest.TestCase, root: zarr.Group) -> None: + rng = np.random.default_rng(0) + np.testing.assert_array_equal(root["foo"][...], rng.random((3, 3, 3))) + np.testing.assert_array_equal(root["subgroup/foo"][...], rng.random((5, 5, 5))) + test.assertEqual(bytes(root["subgroup/blob"][()]), b"\x89PNG\x00some binary data") + test.assertEqual(root.attrs.asdict(), {"some": "thing"}) + test.assertEqual(root["subgroup"].attrs.asdict(), {"other": [1, 2, 3]}) + + +def _write_itar(path: str, consolidate: bool = True) -> None: + with IndexedTarStore(path, mode="w") as store: + zarr_store = ZarrIndexedTarStore(store) + _fill_reference(zarr.group(store=zarr_store)) + if consolidate: + consolidate_compressed_metadata(zarr_store) class TestIndexedTarStore(unittest.TestCase): - """Test to verify functionality of IndexedTarStore""" + """Records of indexed tar archives""" def setUp(self): - # Fill a reference group with an in-memory store - self.g_ref = zarr.open(store=zarr.MemoryStore()) - self.g_ref.create_dataset("foo", data=np.random.default_rng().random((3, 3, 3))) - self.g_ref.attrs.update({"some": "thing"}) - self.g_ref.require_group("subgroup").create_dataset("foo", data=np.random.default_rng().random((5, 5, 5))) - - def check_with_reference(self, group): - """Verifies all values of a group against the reference""" - self.assertIsNone(np.testing.assert_array_equal(self.g_ref["foo"][()], group["foo"][()])) - self.assertIsNone( - np.testing.assert_array_equal(self.g_ref["subgroup"]["foo"][()], group["subgroup"]["foo"][()]) - ) - self.assertDictEqual(self.g_ref.attrs.asdict(), group.attrs.asdict()) + self._tmp = tempfile.TemporaryDirectory() + self.path = os.path.join(self._tmp.name, "store.itar") + + def tearDown(self): + self._tmp.cleanup() + + def test_records(self): + records = {"a": b"0123456789", "dir/b": b"x" * 513, "empty": b"", "l" * 150 + "/long-key": bytes(range(256))} + with IndexedTarStore(self.path, mode="w") as store: + for key, value in records.items(): + store[key] = value + self.assertEqual(store["dir/b"], records["dir/b"]) # readable while writing + with self.assertRaises(ValueError): # write-once + store["a"] = b"again" + + store = IndexedTarStore(self.path) + self.assertEqual(store.keys(), list(records)) + self.assertEqual(len(store), len(records)) + self.assertEqual({key: store[key] for key in store}, records) + self.assertIn("a", store) + self.assertNotIn("missing", store) + with self.assertRaises(KeyError): + store["missing"] + with self.assertRaises(ValueError): # read-only + store["c"] = b"c" + with self.assertRaises(NotImplementedError): + del store["a"] + + # archives are regular tar archives + with tarfile.open(self.path) as tar: + self.assertEqual({m.name: unpack(tar.extractfile(m)) for m in tar.getmembers()}, records) + + @parameterized.parameterized.expand([(size,) for size in INDEX_TAIL_READ_SIZES]) + def test_index_tail_read_sizes(self, index_tail_read_size: Optional[int]): + records = {f"r{i}": bytes([i]) * (i * 97) for i in range(40)} + with IndexedTarStore(self.path, mode="w") as store: + for key, value in records.items(): + store[key] = value + + store = IndexedTarStore(self.path, index_tail_read_size=index_tail_read_size) + self.assertEqual({key: store[key] for key in store}, records) + + def test_records_within_the_index_tail_read_require_no_reads(self): + with IndexedTarStore(self.path, mode="w") as store: + store["first"] = b"1" * 1000 + store["last"] = b"2" * 1000 + + store = IndexedTarStore(self.path, index_tail_read_size=Path(self.path).stat().st_size) + with mock.patch("os.pread", side_effect=AssertionError("unexpected read")): + self.assertEqual(store["first"], b"1" * 1000) + self.assertEqual(store["last"], b"2" * 1000) + + def test_records_partially_within_the_index_tail_read(self): + """Records starting before the index tail read are read from the archive file""" + with IndexedTarStore(self.path, mode="w") as store: + store["large"] = bytes(range(256)) * 64 # 16 KiB + store["small"] = b"s" + + file_size = Path(self.path).stat().st_size + store = IndexedTarStore(self.path, index_tail_read_size=file_size - 1024) # ends within "large" + self.assertEqual(store["large"], bytes(range(256)) * 64) + self.assertEqual(store["small"], b"s") + + def test_positional_reads(self): + """Local archives are read via positional reads""" + with IndexedTarStore(self.path, mode="w") as store: + store["a"] = b"a" * 4096 + + store = IndexedTarStore(self.path, index_tail_read_size=None) + with mock.patch("os.pread", side_effect=os.pread) as pread: + self.assertEqual(store["a"], b"a" * 4096) + self.assertGreater(pread.call_count, 0) + + def test_dropped_stores_release_file_descriptors(self): + with IndexedTarStore(self.path, mode="w") as store: + store["a"] = b"a" + + def open_fds() -> int: + return len(os.listdir(f"/proc/{os.getpid()}/fd")) + + IndexedTarStore(self.path)["a"] + gc.collect() + before = open_fds() + for _ in range(20): + IndexedTarStore(self.path)["a"] + gc.collect() + self.assertEqual(open_fds(), before) + + def test_reload_resources(self): + """Reloading resources neither truncates archives opened for writing nor affects reads""" + with IndexedTarStore(self.path, mode="w") as store: + store["first"] = b"first payload" + store.reload_resources() + self.assertEqual(store["first"], b"first payload") + store["second"] = b"second payload" + + store = IndexedTarStore(self.path, index_tail_read_size=None) + store.reload_resources() + self.assertEqual(store["first"], b"first payload") + self.assertEqual(store["second"], b"second payload") + + def test_invalid_archives(self): + invalid_mode: Any = "a" + with self.assertRaises(ValueError): + IndexedTarStore(self.path, mode=invalid_mode) + Path(self.path).write_bytes(b"\0" * 2048) + with self.assertRaises(ValueError): + IndexedTarStore(self.path) + + def test_unsupported_index_types(self): + with IndexedTarStore(self.path, mode="w") as store: + store["a"] = b"a" + data = bytearray(Path(self.path).read_bytes()) + header = len(data) - tarfile.BLOCKSIZE # (the index header is stored in the last tar block) + magic, _, offset, size = struct.unpack_from(IndexedTarStore.INDEX_HEADER_FORMAT, data, header) + struct.pack_into(IndexedTarStore.INDEX_HEADER_FORMAT, data, header, magic, 99, offset, size) + Path(self.path).write_bytes(bytes(data)) + with self.assertRaisesRegex(TypeError, "unsupported index type 99"): + IndexedTarStore(self.path) + + def test_empty(self): + with IndexedTarStore(self.path, mode="w"): + pass + self.assertEqual(IndexedTarStore(self.path).keys(), []) + + +class TestZarrIndexedTarStore(unittest.TestCase): + """zarr hierarchies in indexed tar archives""" - @parameterized.parameterized.expand( - itertools.product( - INDEX_TAIL_READ_SIZES, - ) - ) - def test_reserialization(self, index_tail_read_size: Optional[int]): - """Make sure storing / loading of regular zarr data to .itar files works correctly""" + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.path = os.path.join(self._tmp.name, "store.itar") - # re-serialize to .itar archive - with tempfile.NamedTemporaryFile(suffix=".itar") as f: - with IndexedTarStore(f.name, mode="w") as s_itar_out: # closes file on exit - zarr.copy_store(self.g_ref.store, s_itar_out) + def tearDown(self): + self._tmp.cleanup() - # reload store from file - store = IndexedTarStore(f.name, index_tail_read_size=index_tail_read_size) - g_reload = zarr.open(store=store, mode="r") + @parameterized.parameterized.expand([(size,) for size in INDEX_TAIL_READ_SIZES]) + def test_reserialization(self, index_tail_read_size: Optional[int]): + _write_itar(self.path, consolidate=False) - # check all data was correctly serialized / deserialized - self.check_with_reference(g_reload) + store = IndexedTarStore(self.path, index_tail_read_size=index_tail_read_size) + root = zarr.open_group(store=ZarrIndexedTarStore(store), mode="r") + _check_reference(self, root) - # check reloading resources is functional - store.reload_resources() - self.check_with_reference(g_reload) + store.reload_resources() + _check_reference(self, root) - @parameterized.parameterized.expand( - itertools.product( - INDEX_TAIL_READ_SIZES, - ) - ) + @parameterized.parameterized.expand([(size,) for size in INDEX_TAIL_READ_SIZES]) def test_compressed_consolidated(self, index_tail_read_size: Optional[int]): - """Make sure compressed consolidated meta data is stored/loaded correctly""" - - # serialize to .itar archive (will also serialize compressed-consolidated meta-data) - with tempfile.NamedTemporaryFile(suffix=".itar") as f: - with IndexedTarStore(f.name, mode="w") as s_itar_out: # closes file on exit - zarr.copy_store(self.g_ref.store, s_itar_out) + _write_itar(self.path) + + store = IndexedTarStore(self.path, index_tail_read_size=index_tail_read_size) + with mock.patch.object( + IndexedTarStore, "__getitem__", autospec=True, side_effect=IndexedTarStore.__getitem__ + ) as getitem: + root = open_compressed_consolidated(store, mode="r") + _check_reference(self, root) + # node metadata is read from the compressed consolidated metadata only + read_keys = [call.args[1] for call in getitem.call_args_list] + self.assertIn(CONSOLIDATED_METADATA_KEY, read_keys) + self.assertEqual([key for key in read_keys if key.endswith((".zarray", ".zgroup", ".zattrs"))], []) + # (zarr group -> compressed consolidated metadata layer -> zarr-python adapter -> indexed tar store) + consolidated_store = root.store + assert isinstance(consolidated_store, ConsolidatedCompressedMetadataStore) + assert isinstance(zarr_store := consolidated_store.store, ZarrIndexedTarStore) + self.assertIs(zarr_store.store, store) + + store.reload_resources() + _check_reference(self, root) + + def test_records_match_zarr_directory_stores(self): + """Archives store the same records as zarr's directory stores (byte-identical)""" + _write_itar(self.path) + + directory = os.path.join(self._tmp.name, "store.zarr") + directory_store = zarr.storage.DirectoryStore(directory) + _fill_reference(zarr.group(store=directory_store)) + consolidate_compressed_metadata(directory_store) + + store = IndexedTarStore(self.path) + self.assertEqual(sorted(store.keys()), sorted(directory_store.keys())) + for key in store.keys(): + if key == CONSOLIDATED_METADATA_KEY: # (documents ordered by the key order of the stores) + self.assertEqual(decode_cbor_xz(store[key]), decode_cbor_xz(directory_store[key])) + else: + self.assertEqual(store[key], directory_store[key], key) + + def test_archives_of_ncore_19_8(self): + """Archives are byte-identical to archives written by ncore 19.8 (sha256 of a fixed hierarchy written by + ncore 19.8)""" + with IndexedTarStore(self.path, mode="w") as store: + zarr_store = ZarrIndexedTarStore(store) + root = zarr.group(store=zarr_store) + root.attrs.put({"some": "thing", "nested": {"a": [1, 2.5, None]}}) + root.create_dataset("foo", data=np.arange(27, dtype=np.float64).reshape(3, 3, 3), chunks=(2, 3, 3)) + sub = root.create_group("subgroup") + sub.attrs.put({"other": [1, 2, 3]}) + sub.create_dataset("bar", data=np.arange(10, dtype=np.uint16), compressor=None) + sub.create_dataset("blob", data=np.asarray(b"\x89PNG\x00data"), compressor=None) + consolidate_compressed_metadata(zarr_store) + + digest = hashlib.sha256(Path(self.path).read_bytes()).hexdigest() + self.assertIn( + digest, + { + # (python's tarfile module encodes the device numbers of tar headers differently before python 3.9) + "98852fe2314dc3c9680cd0997d6f914785f5f7d2e552c102ed71c9b400340226", # python 3.8 + "016af4d3f0d4964c025832a4864b45ffe13cd68e74436da331086d255592a7de", # python >= 3.9 + }, + ) - # consolidate compress meta-data - consolidate_compressed_metadata(s_itar_out) + def test_empty(self): + with IndexedTarStore(self.path, mode="w") as store: + consolidate_compressed_metadata(ZarrIndexedTarStore(store)) - # reload store from file with compressed consolidated meta-data - store = IndexedTarStore(f.name, index_tail_read_size=index_tail_read_size) - g_reload = open_compressed_consolidated(store=store, mode="r") + with self.assertRaises(zarr.errors.PathNotFoundError): + open_compressed_consolidated(IndexedTarStore(self.path), mode="r") + with self.assertRaises(zarr.errors.PathNotFoundError): + zarr.open(store=ZarrIndexedTarStore(IndexedTarStore(self.path)), mode="r") - # check all data was correctly serialized / deserialized - self.check_with_reference(g_reload) + def test_read_only(self): + _write_itar(self.path) + root = zarr.open_group(store=ZarrIndexedTarStore(IndexedTarStore(self.path)), mode="r") + with self.assertRaises(zarr.errors.ReadOnlyError): + root.create_group("new") - # check reloading resources is functional - store.reload_resources() - self.check_with_reference(g_reload) - @parameterized.parameterized.expand( - itertools.product( - COMPRESSED_CONSOLIDATED_VALUES, - INDEX_TAIL_READ_SIZES, - ) - ) - def test_empty(self, compressed_consolidate: bool, index_tail_read_size: Optional[int]): - """Verify edge case of serialization of empty store is possible without errors""" - with tempfile.NamedTemporaryFile(suffix=".itar") as f: - with IndexedTarStore(f.name, mode="w") as s_itar_out: # closes file on exit - # Don't write any zarr data (still serializes empty tar / seek tables) - - if compressed_consolidate: - consolidate_compressed_metadata(s_itar_out) - - with IndexedTarStore(f.name, index_tail_read_size=index_tail_read_size) as s_itar_in: - # Loading store should work without errors - - # But loading a non-existing group should then fail - with self.assertRaises(zarr.errors.PathNotFoundError): - if compressed_consolidate: - open_compressed_consolidated(s_itar_in, mode="r") - else: - zarr.open(store=s_itar_in, mode="r") - - def test_reload_resources_preserves_write_mode_archive(self): - """Verify writer resource reload does not truncate already-written archive data.""" - with tempfile.NamedTemporaryFile(suffix=".itar") as f: - with IndexedTarStore(f.name, mode="w") as store: - store["first"] = b"first payload" - store.tar_file_object.flush() - - size_before_reload = Path(f.name).stat().st_size - self.assertGreater(size_before_reload, 0) - - store.reload_resources() - - self.assertEqual(Path(f.name).stat().st_size, size_before_reload) - self.assertEqual(store["first"], b"first payload") - - store["second"] = b"second payload" - self.assertEqual(store["first"], b"first payload") - self.assertEqual(store["second"], b"second payload") - - with IndexedTarStore(f.name, mode="r") as store: - self.assertEqual(store["first"], b"first payload") - self.assertEqual(store["second"], b"second payload") - - def test_index_tail_cache_avoids_extra_file_reads(self): - """Payloads fully covered by the index tail read must not trigger further ``read()`` calls.""" - - class _CountingReader(IO[Any]): - __slots__ = ("_f", "read_calls") - - def __init__(self, f: IO[Any]): - self._f = f - self.read_calls = 0 - - def read(self, n=-1): - self.read_calls += 1 - return self._f.read(n) - - def seek(self, *args, **kwargs): - return self._f.seek(*args, **kwargs) - - def tell(self): - return self._f.tell() - - def close(self): - return self._f.close() - - def __getattr__(self, name): - return getattr(self._f, name) - - with tempfile.NamedTemporaryFile(suffix=".itar") as f: - g_ref = zarr.open(store=zarr.MemoryStore()) - g_ref.create_dataset("x", data=np.array([1.0, 2.0])) - with IndexedTarStore(f.name, mode="w") as s_out: - zarr.copy_store(g_ref.store, s_out) - - file_size = Path(f.name).stat().st_size - self.assertGreater(file_size, 0) - store = IndexedTarStore(f.name, index_tail_read_size=file_size) - - # Wrap the file object to intercept read counts - inner = store.tar_file_object - counter = _CountingReader(inner) - store.tar_file_object = counter - self.assertEqual(counter.read_calls, 0) - - # Read every key via the tail buffer and verify no extra file reads occurred - values = {key: store[key] for key in store} - self.assertEqual(counter.read_calls, 0) - - # Verify actual data was served from the store - self.assertGreater(len(values), 0) - g_reload = zarr.open(store=store, mode="r") - self.assertIsNone(np.testing.assert_array_equal(g_ref["x"][()], g_reload["x"][()])) +def unpack(file: Optional[Any]) -> bytes: + assert file is not None + return file.read() diff --git a/ncore/impl/data/v4/components.py b/ncore/impl/data/v4/components.py index bf95083d..9f7fe2f5 100644 --- a/ncore/impl/data/v4/components.py +++ b/ncore/impl/data/v4/components.py @@ -225,7 +225,7 @@ def get_base_group(self, component_group_name: Optional[str]) -> zarr.Group: if self._store_type == "itar": # container-based zarr stores ..zarr.itar store_path = self._output_dir_path / f"{self._store_base_name}.{store_name}.zarr.itar" - store = stores.IndexedTarStore(store_path, mode="w") + store = stores.ZarrIndexedTarStore(stores.IndexedTarStore(store_path, mode="w")) elif self._store_type == "directory": # directory-based zarr stores ..zarr.zarr store_path = self._output_dir_path / f"{self._store_base_name}.{store_name}.zarr" @@ -431,8 +431,10 @@ def thread_load_component_store(component_store_upath: UPath) -> Tuple[zarr.Grou f"Unsupported file-based store format {component_store_upath}, expected .zarr.itar" ) - component_store = stores.IndexedTarStore( - component_store_upath, mode="r", index_tail_read_size=itar_index_tail_read_size + component_store = stores.ZarrIndexedTarStore( + stores.IndexedTarStore( + component_store_upath, mode="r", index_tail_read_size=itar_index_tail_read_size + ) ) else: component_store = zarr.storage.DirectoryStore(component_store_upath) @@ -497,16 +499,13 @@ def thread_load_component_store(component_store_upath: UPath) -> Tuple[zarr.Grou def reload_resources(self) -> None: """Trigger a reload of each itar store - useful to re-initialize file objects in multi-process settings""" - component_store: Union[zarr.Group, stores.ConsolidatedCompressedMetadataStore] - for component_store, _ in self._component_stores.values(): + for component_root, _ in self._component_stores.values(): + store = component_root.store # unwind one layer of possible consolidated metadata store - if isinstance( - compressed_consolidated_store := component_store.store, stores.ConsolidatedCompressedMetadataStore - ): - component_store = compressed_consolidated_store - - if isinstance(store := component_store.store, stores.IndexedTarStore): - store.reload_resources() + if isinstance(store, stores.ConsolidatedCompressedMetadataStore): + store = store.store + if isinstance(store, stores.ZarrIndexedTarStore): + store.store.reload_resources() @property def sequence_id(self) -> str: From 7f4849a77671d712d0066a5681da6a82650eb241 Mon Sep 17 00:00:00 2001 From: Janick Martinez Esturo Date: Sun, 11 Oct 2026 13:55:55 +0200 Subject: [PATCH 2/7] feat(data)!: typed node API for component groups Access the zarr hierarchies of component stores via typed group / array wrappers instead of zarr-python APIs, as a prerequisite of supporting multiple zarr-python versions: - nodes.Group / nodes.Array: create and read (nested) groups, arrays, binary blobs and (typed) attributes; the wrapped zarr-python objects are accessible via .zarr - stores: open_directory_store, create_root_group, open_store, consolidate_store, close_store and group_store for indexed tar archives and directories with compressed consolidated metadata - Version specific operations are implemented by a zarr-python 2 backend (_zarr2.py) of a backend interface (_backend.py) - Port all components to the node API (single chunk arrays by default, blosc lz4 / bit-shuffle compression), sharing frame / point cloud path lookups Written stores are byte-identical to stores written by ncore 19.8. BREAKING CHANGE: The groups of component writers / readers (self._group) are nodes.Group instances (the wrapped zarr-python group is accessible via self._group.zarr). The stores functions consolidate_compressed_metadata / open_compressed_consolidated and the ConsolidatedCompressedMetadataStore class are replaced by consolidate_store / open_store. --- docs/data/formats.rst | 19 +- docs/data/storage_and_access.rst | 4 +- ncore/impl/data/BUILD.bazel | 14 + ncore/impl/data/_backend.py | 220 ++++++++ ncore/impl/data/_json.py | 35 ++ ncore/impl/data/_zarr2.py | 205 +++++++ ncore/impl/data/nodes.py | 242 ++++++++ ncore/impl/data/stores.py | 134 ++--- ncore/impl/data/stores_test.py | 289 +++++++--- ncore/impl/data/types.py | 12 +- ncore/impl/data/v4/components.py | 519 ++++++++---------- ncore/impl/data/v4/components_test.py | 34 +- .../argoverse2/converter_test.py | 4 +- .../data_converter/nuscenes/converter_test.py | 2 +- 14 files changed, 1223 insertions(+), 510 deletions(-) create mode 100644 ncore/impl/data/_backend.py create mode 100644 ncore/impl/data/_json.py create mode 100644 ncore/impl/data/_zarr2.py create mode 100644 ncore/impl/data/nodes.py diff --git a/docs/data/formats.rst b/docs/data/formats.rst index 125612f7..86d32841 100644 --- a/docs/data/formats.rst +++ b/docs/data/formats.rst @@ -107,7 +107,7 @@ might attach raw GPS/IMU measurements to the poses component: ├── {component_specific_data}... │ └── generic_data/ (optional, only if generic data was set) - ├── {dataset_name} [shape] dtype (lz4-compressed zarr dataset) + ├── {array_name} [shape] dtype (lz4-compressed array) └── ... Writers use :meth:`~ncore.impl.data.v4.components.ComponentWriter.set_generic_data` @@ -414,12 +414,12 @@ Supported categories: Encoding ^^^^^^^^ -* ``RAW`` -- Numpy array stored as a zarr dataset regular compression. Shape +* ``RAW`` -- Numpy array stored as an array with regular compression. Shape is ``[H, W] + shape_suffix`` (e.g., ``[H, W, 2]`` for optical flow). Transparent quantization of raw labels is supported optionally (e.g., float32 depth quantized to uint16 with scale/offset). * ``IMAGE_ENCODED`` -- Pre-encoded image bytes (PNG, JPEG) stored as a 1-D - zarr uint8 dataset with no compression. Consumers can call ``get_encoded_data()`` for raw + uint8 array with no compression. Consumers can call ``get_encoded_data()`` for raw bytes (GPU-based decoding) or ``get_data()`` for Pillow-decoded numpy arrays. Instance naming convention @@ -486,7 +486,7 @@ A minimal custom component looks like: ... # collect data def finalize(self): - ... # write zarr datasets to self._group + ... # write arrays to self._group, e.g. self._group.create_array("velocities", ...) class Reader(ComponentReader): @staticmethod @@ -498,7 +498,16 @@ A minimal custom component looks like: return version == "v1" def get_velocities(self): - return self._group["velocities"][:], self._group["timestamps_us"][:] + return ( + self._group.array("velocities").read(), + self._group.array("timestamps_us").read(), + ) + +The groups of component writers / readers (``self._group``) are +``ncore.impl.data.nodes.Group`` instances with a typed API to create and read groups, arrays +(``create_array`` / ``array(path).read()``), binary blobs (``create_bytes_array`` / +``read_bytes``), and attributes. The wrapped zarr-python group is accessible via +``self._group.zarr``. Writers must ensure that all stored timestamps fall within the sequence's ``sequence_timestamp_interval_us`` time range. Existing datasets can be extended diff --git a/docs/data/storage_and_access.rst b/docs/data/storage_and_access.rst index 91df6110..38d9911d 100644 --- a/docs/data/storage_and_access.rst +++ b/docs/data/storage_and_access.rst @@ -34,8 +34,8 @@ capability. appends a compressed index enabling O(1) key lookups and direct seeks to any chunk. -The ``.itar`` store implements the abstract zarr ``Store`` interface, so it can -be used as a drop-in replacement for directory stores in all NCore APIs. Via +The ``.itar`` store can be used as a drop-in replacement for directory stores in +all NCore APIs. Via `UPath `_, ``.itar`` containers can also be accessed transparently from cloud storage backends (e.g., S3, GCS) without requiring a local copy. diff --git a/ncore/impl/data/BUILD.bazel b/ncore/impl/data/BUILD.bazel index 6bb60a4d..85aea767 100644 --- a/ncore/impl/data/BUILD.bazel +++ b/ncore/impl/data/BUILD.bazel @@ -25,6 +25,7 @@ py_library( "types.py", ], deps = [ + ":pylib_json", ":pylib_util", "//ncore/impl/common:pylib_transformations", requirement("dataclasses_json"), @@ -33,16 +34,29 @@ py_library( ], ) +py_library( + name = "pylib_json", + srcs = [ + "_json.py", + ], +) + py_library( name = "pylib_stores", srcs = [ + "_backend.py", + "_zarr2.py", "encoding.py", "itar.py", + "nodes.py", "stores.py", ], deps = [ + ":pylib_json", requirement("cbor2"), requirement("numcodecs"), + requirement("numpy"), + requirement("typing_extensions"), requirement("universal_pathlib"), requirement("zarr"), ], diff --git a/ncore/impl/data/_backend.py b/ncore/impl/data/_backend.py new file mode 100644 index 00000000..2b124940 --- /dev/null +++ b/ncore/impl/data/_backend.py @@ -0,0 +1,220 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Interface of the zarr-python version specific backends of :mod:`ncore.impl.data.nodes` / :mod:`.stores`, and +types shared by the backends""" + +from __future__ import annotations + +import dataclasses +import sys +import typing + +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional, Tuple, Union + +import numcodecs +import zarr + +from ncore.impl.data.itar import IndexedTarStore + + +if sys.version_info >= (3, 10): + from types import EllipsisType +else: # pragma: no cover + EllipsisType = type(Ellipsis) + +if TYPE_CHECKING: + import numpy as np + import numpy.typing as npt # type: ignore[import-not-found] + + from typing_extensions import Protocol +else: + Protocol = object + + +#: Key of the compressed consolidated metadata of zarr format v2 hierarchies (CBOR + LZMA encoded ``.zmetadata``) +V2_CONSOLIDATED_METADATA_KEY = ".zmetadata.cbor.xz" + +#: Basic selections of array data (integers, slices and Ellipsis) +Selection = Union[int, slice, EllipsisType, Tuple[Union[int, slice, EllipsisType], ...]] + +#: Attributes of nodes (JSON-serializable values, also nested mappings / sequences: values are JSON-encoded by zarr-python, +#: see :data:`ncore.impl.data.types.JsonLike` for the attributes read back) +Attributes = Mapping[str, object] + + +def attributes_dict(attributes: Attributes) -> Dict[str, Any]: + """A (shallow) copy of attributes as dictionary, as accepted by zarr-python""" + return dict(attributes) + + +class ZarrStore: + """A zarr-python store of the installed zarr-python version (e.g., of a directory)""" + + +#: Stores of zarr hierarchies: indexed tar archives, or zarr-python stores +StoreLike = Union[IndexedTarStore, ZarrStore] + + +def is_v2_metadata_key(key: str) -> bool: + """True for keys of zarr format v2 node metadata""" + return key.endswith((".zarray", ".zgroup", ".zattrs")) + + +#: Blosc compressors supported by zarr-python 2 (numcodecs ``Blosc``) and zarr-python 3 (``BloscCodec``) +BloscCname = Literal["blosclz", "lz4", "lz4hc", "zlib", "zstd"] + +#: Blosc shuffle modes (numcodecs ``Blosc`` encodes them as 0 / 1 / 2) +BloscShuffle = Literal["noshuffle", "shuffle", "bitshuffle"] + +_BLOSC_SHUFFLES: List[BloscShuffle] = ["noshuffle", "shuffle", "bitshuffle"] + + +@dataclasses.dataclass(frozen=True) +class BloscCompression: + """Blosc compression of array chunks""" + + #: Compressor used by blosc + cname: BloscCname = "lz4" + #: Compression level (0: no compression, 9: maximum compression) + clevel: int = 5 + #: Shuffle filter applied before compression + shuffle: BloscShuffle = "bitshuffle" + #: Block size in bytes (0: automatic) + blocksize: int = 0 + + def __post_init__(self) -> None: + if self.cname not in typing.get_args(BloscCname): + raise ValueError(f"Unsupported blosc compressor {self.cname!r}, supported: {typing.get_args(BloscCname)}") + if not 0 <= self.clevel <= 9: + raise ValueError(f"Blosc compression level needs to be in [0, 9], got {self.clevel}") + if self.shuffle not in _BLOSC_SHUFFLES: + raise ValueError(f"Unsupported blosc shuffle {self.shuffle!r}, supported: {_BLOSC_SHUFFLES}") + if self.blocksize < 0: + raise ValueError(f"Blosc block size needs to be non-negative, got {self.blocksize}") + + def to_numcodecs(self) -> numcodecs.Blosc: + return numcodecs.Blosc( + cname=self.cname, + clevel=self.clevel, + shuffle=_BLOSC_SHUFFLES.index(self.shuffle), + blocksize=self.blocksize, + ) + + @staticmethod + def from_numcodecs(compressor: numcodecs.Blosc) -> BloscCompression: + config = compressor.get_config() + return BloscCompression( + cname=config["cname"], + clevel=config["clevel"], + shuffle=_BLOSC_SHUFFLES[config["shuffle"]], + blocksize=config["blocksize"], + ) + + +ZarrNode = Union[zarr.Group, zarr.Array] + + +class Backend(Protocol): + """zarr-python version specific operations of nodes and stores""" + + #: zarr formats supported by the zarr-python version + supported_zarr_formats: Tuple[int, ...] + + # -- nodes ----------------------------------------------------------------- + + def zarr_format(self, node: ZarrNode) -> int: + """The zarr format of a node""" + ... + + def open_member(self, group: zarr.Group, path: str) -> ZarrNode: + """Opens a (nested) member of a group (raises KeyError if it doesn't exist)""" + ... + + def member_names(self, group: zarr.Group) -> List[str]: + """Names of the direct members of a group""" + ... + + def has_member(self, group: zarr.Group, name: str) -> bool: + """True if a group has a direct member with the given name""" + ... + + def member_groups(self, group: zarr.Group) -> List[Tuple[str, zarr.Group]]: + """Direct member groups of a group""" + ... + + def create_group(self, group: zarr.Group, name: str, attributes: Optional[Attributes]) -> zarr.Group: + """Creates a member group (raises if a member with the name exists already)""" + ... + + def require_group(self, group: zarr.Group, name: str) -> zarr.Group: + """Returns a member group, creating it if it doesn't exist yet""" + ... + + def create_array( + self, + group: zarr.Group, + name: str, + data: npt.NDArray[np.generic], + chunks: Union[Tuple[int, ...], Literal["auto"]], + compression: Optional[BloscCompression], + attributes: Optional[Attributes], + ) -> zarr.Array: + """Creates a member array initialized with data""" + ... + + def read(self, array: zarr.Array, selection: Selection) -> object: + """Reads (a selection of) the data of an array (0-dimensional selections may result in numpy scalars)""" + ... + + def compression(self, array: zarr.Array) -> Optional[BloscCompression]: + """The blosc compression of an array (None if uncompressed, raises ValueError for other compressors)""" + ... + + # -- stores ---------------------------------------------------------------- + + def open_directory_store(self, path: str) -> ZarrStore: + """Opens a directory store (a directory per group / array, a file per record)""" + ... + + def create_root_group(self, store: StoreLike, zarr_format: Literal[2, 3], attributes: Attributes) -> zarr.Group: + """Creates the root group of a hierarchy in a store""" + ... + + def open_store(self, store: StoreLike, consolidated: bool) -> zarr.Group: + """Opens the root group of a hierarchy for reading, via its compressed consolidated metadata if consolidated + (raises KeyError if there is no compressed consolidated metadata)""" + ... + + def consolidate_store(self, store: StoreLike) -> None: + """Stores the compressed consolidated metadata of the hierarchy of a store""" + ... + + def group_store(self, group: zarr.Group) -> StoreLike: + """The store of a group (the indexed tar store for groups of indexed tar archives)""" + ... + + +__all__ = [ + "Attributes", + "attributes_dict", + "Backend", + "BloscCompression", + "Selection", + "StoreLike", + "V2_CONSOLIDATED_METADATA_KEY", + "ZarrStore", + "is_v2_metadata_key", +] diff --git a/ncore/impl/data/_json.py b/ncore/impl/data/_json.py new file mode 100644 index 00000000..d19174bf --- /dev/null +++ b/ncore/impl/data/_json.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""JSON-like values (independent of the other data modules)""" + +from __future__ import annotations + +from typing import Dict, List, Union + + +#: A JSON-serializable value +JsonLike = Union[ + Dict[str, "JsonLike"], + List["JsonLike"], + str, + int, + float, + bool, + None, +] + +#: A JSON-serializable document (a dictionary at the top level) +JsonDocument = Dict[str, JsonLike] diff --git a/ncore/impl/data/_zarr2.py b/ncore/impl/data/_zarr2.py new file mode 100644 index 00000000..41df494c --- /dev/null +++ b/ncore/impl/data/_zarr2.py @@ -0,0 +1,205 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""zarr-python 2 backend of :mod:`ncore.impl.data.nodes` and :mod:`ncore.impl.data.stores`""" + +from __future__ import annotations + +import json + +from typing import TYPE_CHECKING, Any, Iterator, List, Literal, Optional, Tuple, Union + +import numcodecs +import numpy as np +import zarr + +from numcodecs.compat import ensure_bytes + +from ncore.impl.data._backend import ( + V2_CONSOLIDATED_METADATA_KEY, + Attributes, + BloscCompression, + Selection, + StoreLike, + ZarrStore, + attributes_dict, + is_v2_metadata_key, +) +from ncore.impl.data.encoding import decode_cbor_xz, encode_cbor_xz +from ncore.impl.data.itar import IndexedTarStore + + +if TYPE_CHECKING: + import numpy.typing as npt # type: ignore[import-not-found] + + _StoreBase = object + _ConsolidatedMetadataStoreBase = object +else: + from zarr._storage.store import Store as _StoreBase + from zarr.storage import ConsolidatedMetadataStore as _ConsolidatedMetadataStoreBase + + +#: zarr-python 2 API (untyped, type checking may run against zarr-python 3) +_zarr: Any = zarr + + +def _z(node: object) -> Any: + """A zarr-python 2 group / array / store (untyped, see :data:`_zarr`)""" + return node + + +class _TarStore(_StoreBase): + """zarr-python 2 store of the records of an :class:`IndexedTarStore`""" + + _erasable = False + + def __init__(self, store: IndexedTarStore) -> None: + self.store = store + + def __getitem__(self, key: str) -> bytes: + return self.store[key] + + def __setitem__(self, key: str, value: object) -> None: + if self.store.mode != "w": + raise _zarr.errors.ReadOnlyError() + self.store[key] = ensure_bytes(value) + + def __delitem__(self, key: str) -> None: + raise NotImplementedError("IndexedTarStore: records can't be deleted") + + def __contains__(self, key: object) -> bool: + return key in self.store + + def __iter__(self) -> Iterator[str]: + return iter(self.store.keys()) + + def __len__(self) -> int: + return len(self.store) + + def __eq__(self, value: object) -> bool: + return isinstance(value, _TarStore) and self.store == value.store + + def __hash__(self) -> int: + return hash(self.store) + + def close(self) -> None: + self.store.close() + + +class _CompressedConsolidatedStore(_ConsolidatedMetadataStoreBase): + """A store serving all node metadata from the compressed consolidated metadata of the wrapped store""" + + def __init__(self, store: Any) -> None: + self.store = store + consolidated = decode_cbor_xz(store[V2_CONSOLIDATED_METADATA_KEY]) + if (consolidated_format := consolidated.get("zarr_consolidated_format")) != 1: + raise _zarr.errors.MetadataError(f"unsupported zarr consolidated metadata format: {consolidated_format}") + self.meta_store = _zarr.storage.KVStore(consolidated["metadata"]) + + +def _zarr_store(store: StoreLike) -> Any: + """The zarr-python 2 store of a store (wraps indexed tar stores)""" + return _TarStore(store) if isinstance(store, IndexedTarStore) else store + + +class Backend: + """zarr-python 2 implementation of the version specific operations of nodes and stores""" + + supported_zarr_formats: Tuple[int, ...] = (2,) + + # -- nodes ----------------------------------------------------------------- + + def zarr_format(self, node: Union[zarr.Group, zarr.Array]) -> int: + return 2 + + def open_member(self, group: zarr.Group, path: str) -> Union[zarr.Group, zarr.Array]: + return _z(group)[path] + + def member_names(self, group: zarr.Group) -> List[str]: + return list(_z(group).keys()) + + def has_member(self, group: zarr.Group, name: str) -> bool: + return name in _z(group) + + def member_groups(self, group: zarr.Group) -> List[Tuple[str, zarr.Group]]: + return list(_z(group).groups()) + + def create_group(self, group: zarr.Group, name: str, attributes: Optional[Attributes]) -> zarr.Group: + child = _z(group).create_group(name) + if attributes is not None: + child.attrs.update(attributes_dict(attributes)) # (writes .zattrs once) + return child + + def require_group(self, group: zarr.Group, name: str) -> zarr.Group: + return _z(group).require_group(name) + + def create_array( + self, + group: zarr.Group, + name: str, + data: npt.NDArray[np.generic], + chunks: Union[Tuple[int, ...], Literal["auto"]], + compression: Optional[BloscCompression], + attributes: Optional[Attributes], + ) -> zarr.Array: + array = _z(group).create_dataset( + name, + data=data, + chunks=True if chunks == "auto" else chunks, + compressor=None if compression is None else compression.to_numcodecs(), + ) + if attributes is not None: + array.attrs.update(attributes_dict(attributes)) + return array + + def read(self, array: zarr.Array, selection: Selection) -> object: + return _z(array)[selection] + + def compression(self, array: zarr.Array) -> Optional[BloscCompression]: + if (compressor := _z(array).compressor) is None: + return None + if not isinstance(compressor, numcodecs.Blosc): + raise ValueError(f"Unsupported compressor {compressor}") + return BloscCompression.from_numcodecs(compressor) + + # -- stores ---------------------------------------------------------------- + + def open_directory_store(self, path: str) -> ZarrStore: + return _zarr.storage.DirectoryStore(path) + + def create_root_group(self, store: StoreLike, zarr_format: Literal[2, 3], attributes: Attributes) -> zarr.Group: + root = _zarr.group(store=_zarr_store(store)) + root.attrs.put(attributes_dict(attributes)) + return root + + def open_store(self, store: StoreLike, consolidated: bool) -> zarr.Group: + zarr_store = _zarr_store(store) + if consolidated: + return _zarr.open_group(store=_CompressedConsolidatedStore(zarr_store), chunk_store=zarr_store, mode="r") + return _zarr.open_group(store=zarr_store, mode="r") + + def consolidate_store(self, store: StoreLike) -> None: + zarr_store = _zarr_store(store) + metadata = {key: json.loads(zarr_store[key]) for key in zarr_store if is_v2_metadata_key(key)} + zarr_store[V2_CONSOLIDATED_METADATA_KEY] = encode_cbor_xz({"zarr_consolidated_format": 1, "metadata": metadata}) + + def group_store(self, group: zarr.Group) -> StoreLike: + store = _z(group).store + if isinstance(store, _CompressedConsolidatedStore): + store = store.store + return store.store if isinstance(store, _TarStore) else store + + +backend = Backend() diff --git a/ncore/impl/data/nodes.py b/ncore/impl/data/nodes.py new file mode 100644 index 00000000..a7b47db7 --- /dev/null +++ b/ncore/impl/data/nodes.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Groups and arrays of NCore data stores. + +:class:`Group` and :class:`Array` wrap zarr-python groups and arrays with a restricted, typed API, implemented by a +backend of the installed zarr-python version (see :mod:`ncore.impl.data._backend`). The wrapped zarr-python objects +are accessible via the ``zarr`` properties (for APIs of the installed zarr-python version not exposed by NCore). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Literal, Optional, Tuple, Type, TypeVar, Union, cast + +import numpy as np + +from zarr import Array as ZarrArray +from zarr import Group as ZarrGroup + +from ncore.impl.data._backend import Attributes, Backend, BloscCompression, Selection, attributes_dict +from ncore.impl.data._json import JsonLike +from ncore.impl.data._zarr2 import backend as _zarr2_backend + + +if TYPE_CHECKING: + import numpy.typing as npt # type: ignore[import-not-found] + + +_backend: Backend = _zarr2_backend + +#: zarr formats supported by the installed zarr-python version +SUPPORTED_ZARR_FORMATS: Tuple[int, ...] = _backend.supported_zarr_formats + +T = TypeVar("T") + + +class _Node: + """Common functionality of groups and arrays""" + + def __init__(self, node: Union[ZarrGroup, ZarrArray]) -> None: + self._node = node + + @property + def path(self) -> str: + """The path of the node within its store ("" for root groups)""" + return self._node.path + + @property + def zarr_format(self) -> int: + """The zarr format of the node""" + return _backend.zarr_format(self._node) + + @property + def attrs(self) -> Dict[str, JsonLike]: + """A copy of the attributes of the node""" + return dict(self._node.attrs.asdict()) + + def attr_str(self, name: str) -> str: + """Returns a string attribute (raises KeyError if missing, TypeError for other types)""" + return _typed_attr(self, name, str) + + def attr_int(self, name: str) -> int: + """Returns an integer attribute (raises KeyError if missing, TypeError for other types)""" + return _typed_attr(self, name, int) + + def attr_dict(self, name: str) -> Dict[str, JsonLike]: + """Returns a dictionary attribute (raises KeyError if missing, TypeError for other types)""" + return _typed_attr(self, name, dict) + + def attr_list(self, name: str) -> List[JsonLike]: + """Returns a list attribute (raises KeyError if missing, TypeError for other types)""" + return _typed_attr(self, name, list) + + def update_attrs(self, attributes: Attributes) -> None: + """Updates (merges) attributes of the node""" + self._node.attrs.update(attributes_dict(attributes)) + + def __repr__(self) -> str: + return f"{type(self).__name__}({self.path!r}, zarr_format={self.zarr_format})" + + +class Array(_Node): + """An array of an NCore data store""" + + def __init__(self, array: ZarrArray) -> None: + super().__init__(array) + self._array = array + + @property + def zarr(self) -> ZarrArray: + """The wrapped zarr-python array""" + return self._array + + @property + def shape(self) -> Tuple[int, ...]: + return tuple(self._array.shape) + + @property + def dtype(self) -> np.dtype: + return np.dtype(self._array.dtype) + + @property + def compression(self) -> Optional[BloscCompression]: + """The blosc compression of the array (None if uncompressed, raises ValueError for other compressors)""" + return _backend.compression(self._array) + + def read(self, selection: Selection = Ellipsis) -> npt.NDArray[Any]: + """Reads (a selection of) the data of the array (0-dimensional selections result in 0-d arrays)""" + return np.asarray(_backend.read(self._array, selection)) + + def read_bytes(self) -> bytes: + """Reads a binary blob stored by :meth:`Group.create_bytes_array`""" + value = _backend.read(self._array, ()) + if isinstance(value, bytes): # fixed-length bytes scalar (trailing NUL bytes are stripped) + return bytes(value) + if isinstance(value, np.ndarray) and value.dtype == np.uint8: + return value.tobytes() + raise TypeError(f"Array {self.path!r} doesn't store a binary blob ({self.dtype})") + + +class Group(_Node): + """A group of an NCore data store""" + + def __init__(self, group: ZarrGroup) -> None: + super().__init__(group) + self._group = group + + @property + def zarr(self) -> ZarrGroup: + """The wrapped zarr-python group""" + return self._group + + # -- reading --------------------------------------------------------------- + + def node(self, path: str) -> Union[Group, Array]: + """Returns a (nested) member group or array (raises KeyError if it doesn't exist)""" + node = _backend.open_member(self._group, path) + return Group(node) if isinstance(node, ZarrGroup) else Array(node) + + def group(self, path: str) -> Group: + """Returns a (nested) member group (raises KeyError if it doesn't exist or isn't a group)""" + if not isinstance(node := _backend.open_member(self._group, path), ZarrGroup): + raise KeyError(f"{_join(self.path, path)} is not a group") + return Group(node) + + def array(self, path: str) -> Array: + """Returns a (nested) member array (raises KeyError if it doesn't exist or isn't an array)""" + if not isinstance(node := _backend.open_member(self._group, path), ZarrArray): + raise KeyError(f"{_join(self.path, path)} is not an array") + return Array(node) + + def members(self) -> List[str]: + """Names of all direct members""" + return _backend.member_names(self._group) + + def __contains__(self, name: str) -> bool: + """True if the group has a direct member with the given name""" + return _backend.has_member(self._group, name) + + def groups(self) -> Iterator[Tuple[str, Group]]: + """Iterates over the direct member groups (ordered by name)""" + for name, group in sorted(_backend.member_groups(self._group), key=lambda member: member[0]): + yield name, Group(group) + + # -- writing --------------------------------------------------------------- + + def create_group(self, name: str, *, attributes: Optional[Attributes] = None) -> Group: + """Creates a member group (raises if a member with the name exists already)""" + return Group(_backend.create_group(self._group, name, attributes)) + + def require_group(self, name: str) -> Group: + """Returns a member group, creating it if it doesn't exist yet""" + return Group(_backend.require_group(self._group, name)) + + def create_array( + self, + name: str, + data: npt.ArrayLike, + *, + dtype: Optional[npt.DTypeLike] = None, + chunks: Union[Tuple[int, ...], Literal["auto"], None] = None, + compression: Optional[BloscCompression] = BloscCompression(), + attributes: Optional[Attributes] = None, + ) -> Array: + """Creates a member array initialized with data. + + Parameters + ---------- + name + Name of the array within the group. + data + Data of the array (defines the shape of the array, and its dtype if ``dtype`` isn't given). + dtype + Optional dtype of the array (the data is converted to it). + chunks + Chunk shape, a single chunk by default (zero-sized dimensions are clamped to 1). "auto" uses the chunk + shape heuristic of zarr-python. + compression + Blosc compression of the chunks (default: lz4, level 5, bit-shuffle), or None for uncompressed chunks. + attributes + Optional attributes of the array. + """ + values = np.asarray(data, dtype=dtype) + if chunks != "auto": + chunks = tuple(max(1, int(size)) for size in (values.shape if chunks is None else chunks)) + return Array(_backend.create_array(self._group, name, values, chunks, compression, attributes)) + + def create_bytes_array(self, name: str, data: bytes, *, attributes: Optional[Attributes] = None) -> Array: + """Stores a binary blob (e.g., an encoded image) as uncompressed member array (0-dimensional fixed-length + bytes array for zarr format v2)""" + return self.create_array(name, np.asarray(data), compression=None, attributes=attributes) + + +def _typed_attr(node: _Node, name: str, expected: Type[T]) -> T: + value = node._node.attrs[name] + if not isinstance(value, expected) or (isinstance(value, bool) and expected is int): + raise TypeError(f"Attribute {name!r} of {node.path!r} is not of type {expected.__name__}: {value!r}") + return cast(T, value) + + +def _join(parent: str, path: str) -> str: + return f"{parent}/{path}" if parent else path + + +__all__ = [ + "Array", + "BloscCompression", + "Group", + "SUPPORTED_ZARR_FORMATS", +] diff --git a/ncore/impl/data/stores.py b/ncore/impl/data/stores.py index 4d80b0ff..f77795f8 100644 --- a/ncore/impl/data/stores.py +++ b/ncore/impl/data/stores.py @@ -13,124 +13,62 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""zarr stores of NCore data: indexed tar archives (``.itar``) and their compressed consolidated metadata""" +"""Stores of NCore data: zarr hierarchies in indexed tar archives (:class:`IndexedTarStore`) or directories, with +compressed consolidated metadata""" from __future__ import annotations -from typing import Iterator, List, Mapping, Union +from pathlib import Path +from typing import Union -import zarr +from upath import UPath -from numcodecs import compat -from zarr._storage.store import Store -from zarr.util import json_loads - -from ncore.impl.data.encoding import decode_cbor_xz, encode_cbor_xz +from ncore.impl.data._backend import V2_CONSOLIDATED_METADATA_KEY, Attributes, StoreLike, ZarrStore from ncore.impl.data.itar import IndexedTarStore +from ncore.impl.data.nodes import Group, _backend -#: Default key of the compressed consolidated metadata of zarr hierarchies (CBOR + LZMA encoded ``.zmetadata``) -CONSOLIDATED_METADATA_KEY = ".zmetadata.cbor.xz" - - -class ZarrIndexedTarStore(Store): - """zarr-python store of the records of an :class:`IndexedTarStore`""" - - _erasable = False - - def __init__(self, store: IndexedTarStore) -> None: - self.store = store - - def __getitem__(self, key: str) -> bytes: - return self.store[key] - - def __setitem__(self, key: str, value: object) -> None: - if self.store.mode != "w": - raise zarr.errors.ReadOnlyError() - self.store[key] = compat.ensure_bytes(value) - - def __delitem__(self, key: str) -> None: - raise NotImplementedError("IndexedTarStore: records can't be deleted") - - def __contains__(self, key: object) -> bool: - return key in self.store - - def __iter__(self) -> Iterator[str]: - return iter(self.store.keys()) - - def __len__(self) -> int: - return len(self.store) - - def __eq__(self, value: object) -> bool: - return isinstance(value, ZarrIndexedTarStore) and self.store == value.store - - def __hash__(self) -> int: - return hash(self.store) - - def close(self) -> None: - self.store.close() - - -StoreLike = Union[IndexedTarStore, zarr.storage.BaseStore] - - -def _zarr_store(store: StoreLike) -> zarr.storage.BaseStore: - """The zarr-python store of a store (wraps indexed tar stores)""" - return ZarrIndexedTarStore(store) if isinstance(store, IndexedTarStore) else store - - -def consolidate_compressed_metadata(store: StoreLike, metadata_key: str = CONSOLIDATED_METADATA_KEY) -> None: - """Consolidates the metadata of all groups and arrays of a store into a single compressed record (CBOR + LZMA - encoded zarr consolidated metadata) +def open_directory_store(path: Union[str, Path, UPath]) -> ZarrStore: + """Opens a directory store (a directory per group / array, a file per record) for reading or writing""" + return _backend.open_directory_store(str(path)) - See Also - -------- - zarr.consolidate_metadata - """ - store = zarr.storage.normalize_store_arg(_zarr_store(store), mode="w") - if store._store_version != 2: - raise NotImplementedError("Only supporting V2 stores") - metadata = {key: json_loads(store[key]) for key in store if key.endswith((".zarray", ".zgroup", ".zattrs"))} - store[metadata_key] = encode_cbor_xz({"zarr_consolidated_format": 1, "metadata": metadata}) +def create_root_group(store: StoreLike, attributes: Attributes) -> Group: + """Creates the root group of a hierarchy in a store""" + return Group(_backend.create_root_group(store, 2, attributes)) -class ConsolidatedCompressedMetadataStore(zarr.storage.ConsolidatedMetadataStore): - """A layer over a store, where the metadata of all nodes is read from the compressed consolidated metadata""" +def open_store(store: StoreLike, consolidated: bool = True) -> Group: + """Opens the root group of the hierarchy of a store for reading, via its compressed consolidated metadata if + consolidated (raises KeyError if the store has no compressed consolidated metadata)""" + return Group(_backend.open_store(store, consolidated)) - def __init__(self, store: StoreLike, metadata_key: str = CONSOLIDATED_METADATA_KEY) -> None: - self.store = Store._ensure_store(_zarr_store(store)) - consolidated = decode_cbor_xz(self.store[metadata_key]) - if (consolidated_format := consolidated.get("zarr_consolidated_format")) != 1: - raise zarr.MetadataError(f"unsupported zarr consolidated metadata format: {consolidated_format}") +def consolidate_store(store: StoreLike) -> None: + """Stores the compressed consolidated metadata of the hierarchy of a store (after writing all nodes)""" + _backend.consolidate_store(store) - self.meta_store: Mapping[str, object] = zarr.KVStore(consolidated["metadata"]) +def close_store(store: StoreLike) -> None: + """Closes a store (required for indexed tar stores opened for writing)""" + if isinstance(store, IndexedTarStore): + store.close() -def open_compressed_consolidated( - store: StoreLike, metadata_key: str = CONSOLIDATED_METADATA_KEY, mode: str = "r+", **kwargs -) -> zarr.Group: - """Opens a group of a store via its compressed consolidated metadata - See Also - -------- - consolidate_compressed_metadata - zarr.open_consolidated - """ - if mode not in {"r", "r+"}: - raise ValueError(f"invalid mode, expected either 'r' or 'r+'; found {mode!r}") - meta_store = ConsolidatedCompressedMetadataStore(store, metadata_key=metadata_key) - chunk_store = kwargs.pop("chunk_store", None) or meta_store.store - return zarr.open(store=meta_store, chunk_store=chunk_store, mode=mode, **kwargs) +def group_store(group: Group) -> StoreLike: + """The store of a group (the :class:`IndexedTarStore` of groups of indexed tar archives)""" + return _backend.group_store(group.zarr) -__all__: List[str] = [ - "CONSOLIDATED_METADATA_KEY", - "ConsolidatedCompressedMetadataStore", +__all__ = [ "IndexedTarStore", "StoreLike", - "ZarrIndexedTarStore", - "consolidate_compressed_metadata", - "open_compressed_consolidated", + "V2_CONSOLIDATED_METADATA_KEY", + "ZarrStore", + "close_store", + "consolidate_store", + "create_root_group", + "group_store", + "open_directory_store", + "open_store", ] diff --git a/ncore/impl/data/stores_test.py b/ncore/impl/data/stores_test.py index 19c7d10c..090e39d7 100644 --- a/ncore/impl/data/stores_test.py +++ b/ncore/impl/data/stores_test.py @@ -27,48 +27,64 @@ import numpy as np import parameterized -import zarr +from ncore.impl.data import nodes from ncore.impl.data.encoding import decode_cbor_xz from ncore.impl.data.stores import ( - CONSOLIDATED_METADATA_KEY, - ConsolidatedCompressedMetadataStore, + V2_CONSOLIDATED_METADATA_KEY, IndexedTarStore, - ZarrIndexedTarStore, - consolidate_compressed_metadata, - open_compressed_consolidated, + StoreLike, + close_store, + consolidate_store, + create_root_group, + group_store, + open_directory_store, + open_store, ) #: None / values < 512 (separate reads of index header and index), 1 KiB, 1 MiB (default), edge cases around 512 INDEX_TAIL_READ_SIZES = [None, 1, 512, 513, 1 << 10, 1 << 20] - -def _fill_reference(root: zarr.Group) -> None: - rng = np.random.default_rng(0) - root.attrs.put({"some": "thing"}) - root.create_dataset("foo", data=rng.random((3, 3, 3)), chunks=(2, 3, 3)) - sub = root.create_group("subgroup") - sub.attrs.put({"other": [1, 2, 3]}) - sub.create_dataset("foo", data=rng.random((5, 5, 5))) - sub.create_dataset("blob", data=np.asarray(b"\x89PNG\x00some binary data"), compressor=None) +_ROOT_ATTRIBUTES = {"some": "thing"} -def _check_reference(test: unittest.TestCase, root: zarr.Group) -> None: +def _fill_reference(root: nodes.Group) -> None: rng = np.random.default_rng(0) - np.testing.assert_array_equal(root["foo"][...], rng.random((3, 3, 3))) - np.testing.assert_array_equal(root["subgroup/foo"][...], rng.random((5, 5, 5))) - test.assertEqual(bytes(root["subgroup/blob"][()]), b"\x89PNG\x00some binary data") - test.assertEqual(root.attrs.asdict(), {"some": "thing"}) - test.assertEqual(root["subgroup"].attrs.asdict(), {"other": [1, 2, 3]}) + root.create_array("foo", rng.random((3, 3, 3)), chunks=(2, 3, 3)) + sub = root.create_group("subgroup", attributes={"other": [1, 2, 3]}) + sub.create_array("foo", rng.random((5, 5, 5)), attributes={"unit": "m"}) + sub.create_array("empty", np.zeros((0, 3), dtype=np.float32)) + sub.create_bytes_array("blob", b"\x89PNG\x00some binary data", attributes={"format": "png"}) -def _write_itar(path: str, consolidate: bool = True) -> None: - with IndexedTarStore(path, mode="w") as store: - zarr_store = ZarrIndexedTarStore(store) - _fill_reference(zarr.group(store=zarr_store)) - if consolidate: - consolidate_compressed_metadata(zarr_store) +def _check_reference(test: unittest.TestCase, root: nodes.Group) -> None: + rng = np.random.default_rng(0) + np.testing.assert_array_equal(root.array("foo").read(), rng.random((3, 3, 3))) + np.testing.assert_array_equal(root.array("subgroup/foo").read(), rng.random((5, 5, 5))) + test.assertEqual(root.array("subgroup/empty").shape, (0, 3)) + test.assertEqual(root.array("subgroup/blob").read_bytes(), b"\x89PNG\x00some binary data") + test.assertEqual(root.attrs, _ROOT_ATTRIBUTES) + test.assertEqual(root.group("subgroup").attrs, {"other": [1, 2, 3]}) + test.assertEqual(root.array("subgroup/foo").attrs, {"unit": "m"}) + test.assertEqual(sorted(root.members()), ["foo", "subgroup"]) + test.assertEqual(sorted(root.group("subgroup").members()), ["blob", "empty", "foo"]) + test.assertEqual([name for name, _ in root.groups()], ["subgroup"]) + test.assertIn("blob", root.group("subgroup")) + test.assertNotIn("missing", root.group("subgroup")) + with test.assertRaises(KeyError): + root.group("foo") + with test.assertRaises(KeyError): + root.array("subgroup") + with test.assertRaises(KeyError): + root.array("missing") + + +def _write(store: StoreLike, consolidate: bool = True) -> None: + _fill_reference(create_root_group(store, _ROOT_ATTRIBUTES)) + if consolidate: + consolidate_store(store) + close_store(store) class TestIndexedTarStore(unittest.TestCase): @@ -105,7 +121,7 @@ def test_records(self): # archives are regular tar archives with tarfile.open(self.path) as tar: - self.assertEqual({m.name: unpack(tar.extractfile(m)) for m in tar.getmembers()}, records) + self.assertEqual({m.name: _read_member(tar, m) for m in tar.getmembers()}, records) @parameterized.parameterized.expand([(size,) for size in INDEX_TAIL_READ_SIZES]) def test_index_tail_read_sizes(self, index_tail_read_size: Optional[int]): @@ -201,82 +217,100 @@ def test_empty(self): self.assertEqual(IndexedTarStore(self.path).keys(), []) -class TestZarrIndexedTarStore(unittest.TestCase): - """zarr hierarchies in indexed tar archives""" +class TestStores(unittest.TestCase): + """zarr hierarchies in indexed tar archives and directories""" def setUp(self): self._tmp = tempfile.TemporaryDirectory() - self.path = os.path.join(self._tmp.name, "store.itar") + self.itar_path = os.path.join(self._tmp.name, "store.itar") + self.directory_path = os.path.join(self._tmp.name, "store.zarr") def tearDown(self): self._tmp.cleanup() - @parameterized.parameterized.expand([(size,) for size in INDEX_TAIL_READ_SIZES]) - def test_reserialization(self, index_tail_read_size: Optional[int]): - _write_itar(self.path, consolidate=False) + def _store(self, store_type: str, mode: str) -> StoreLike: + if store_type == "itar": + return IndexedTarStore(self.itar_path, mode="w" if mode == "w" else "r") + return open_directory_store(self.directory_path) - store = IndexedTarStore(self.path, index_tail_read_size=index_tail_read_size) - root = zarr.open_group(store=ZarrIndexedTarStore(store), mode="r") - _check_reference(self, root) + @parameterized.parameterized.expand( + [(store_type, consolidated) for store_type in ("itar", "directory") for consolidated in (True, False)] + ) + def test_reserialization(self, store_type: str, consolidated: bool): + _write(self._store(store_type, "w")) - store.reload_resources() + store = self._store(store_type, "r") + root = open_store(store, consolidated) _check_reference(self, root) + if isinstance(store, IndexedTarStore): + self.assertIs(group_store(root), store) + self.assertIs(group_store(root.group("subgroup")), store) + store.reload_resources() + _check_reference(self, root) + @parameterized.parameterized.expand([(size,) for size in INDEX_TAIL_READ_SIZES]) - def test_compressed_consolidated(self, index_tail_read_size: Optional[int]): - _write_itar(self.path) + def test_consolidated_open_reads_no_node_metadata(self, index_tail_read_size: Optional[int]): + _write(self._store("itar", "w")) - store = IndexedTarStore(self.path, index_tail_read_size=index_tail_read_size) + store = IndexedTarStore(self.itar_path, index_tail_read_size=index_tail_read_size) with mock.patch.object( IndexedTarStore, "__getitem__", autospec=True, side_effect=IndexedTarStore.__getitem__ ) as getitem: - root = open_compressed_consolidated(store, mode="r") - _check_reference(self, root) - # node metadata is read from the compressed consolidated metadata only + _check_reference(self, open_store(store, consolidated=True)) read_keys = [call.args[1] for call in getitem.call_args_list] - self.assertIn(CONSOLIDATED_METADATA_KEY, read_keys) + self.assertIn(V2_CONSOLIDATED_METADATA_KEY, read_keys) self.assertEqual([key for key in read_keys if key.endswith((".zarray", ".zgroup", ".zattrs"))], []) - # (zarr group -> compressed consolidated metadata layer -> zarr-python adapter -> indexed tar store) - consolidated_store = root.store - assert isinstance(consolidated_store, ConsolidatedCompressedMetadataStore) - assert isinstance(zarr_store := consolidated_store.store, ZarrIndexedTarStore) - self.assertIs(zarr_store.store, store) - - store.reload_resources() - _check_reference(self, root) - def test_records_match_zarr_directory_stores(self): - """Archives store the same records as zarr's directory stores (byte-identical)""" - _write_itar(self.path) - - directory = os.path.join(self._tmp.name, "store.zarr") - directory_store = zarr.storage.DirectoryStore(directory) - _fill_reference(zarr.group(store=directory_store)) - consolidate_compressed_metadata(directory_store) - - store = IndexedTarStore(self.path) - self.assertEqual(sorted(store.keys()), sorted(directory_store.keys())) - for key in store.keys(): - if key == CONSOLIDATED_METADATA_KEY: # (documents ordered by the key order of the stores) - self.assertEqual(decode_cbor_xz(store[key]), decode_cbor_xz(directory_store[key])) + @parameterized.parameterized.expand([("itar",), ("directory",)]) + def test_missing_consolidated_metadata(self, store_type: str): + _write(self._store(store_type, "w"), consolidate=False) + with self.assertRaises(KeyError): + open_store(self._store(store_type, "r"), consolidated=True) + _check_reference(self, open_store(self._store(store_type, "r"), consolidated=False)) + + def test_stores_are_identical(self): + """Indexed tar archives and directories store the same records""" + _write(self._store("itar", "w")) + _write(self._store("directory", "w")) + + archive = IndexedTarStore(self.itar_path) + files = { + str(path.relative_to(self.directory_path)): path.read_bytes() + for path in Path(self.directory_path).rglob("*") + if path.is_file() + } + self.assertEqual(sorted(archive.keys()), sorted(files)) + for key in archive.keys(): + if key == V2_CONSOLIDATED_METADATA_KEY: # (ordered by the record order of the stores) + self.assertEqual(decode_cbor_xz(archive[key]), decode_cbor_xz(files[key])) else: - self.assertEqual(store[key], directory_store[key], key) + self.assertEqual(archive[key], files[key], key) + + def test_directory_stores_are_not_truncated(self): + Path(self.directory_path).mkdir() + (Path(self.directory_path) / "other").write_bytes(b"other") + _write(self._store("directory", "w")) + self.assertEqual((Path(self.directory_path) / "other").read_bytes(), b"other") def test_archives_of_ncore_19_8(self): """Archives are byte-identical to archives written by ncore 19.8 (sha256 of a fixed hierarchy written by ncore 19.8)""" - with IndexedTarStore(self.path, mode="w") as store: - zarr_store = ZarrIndexedTarStore(store) - root = zarr.group(store=zarr_store) - root.attrs.put({"some": "thing", "nested": {"a": [1, 2.5, None]}}) - root.create_dataset("foo", data=np.arange(27, dtype=np.float64).reshape(3, 3, 3), chunks=(2, 3, 3)) - sub = root.create_group("subgroup") - sub.attrs.put({"other": [1, 2, 3]}) - sub.create_dataset("bar", data=np.arange(10, dtype=np.uint16), compressor=None) - sub.create_dataset("blob", data=np.asarray(b"\x89PNG\x00data"), compressor=None) - consolidate_compressed_metadata(zarr_store) - - digest = hashlib.sha256(Path(self.path).read_bytes()).hexdigest() + store = IndexedTarStore(self.itar_path, mode="w") + root = create_root_group(store, {"some": "thing", "nested": {"a": [1, 2.5, None]}}) + root.create_array( + "foo", + np.arange(27, dtype=np.float64).reshape(3, 3, 3), + chunks=(2, 3, 3), + compression=nodes.BloscCompression(shuffle="shuffle"), + ) + sub = root.create_group("subgroup", attributes={"other": [1, 2, 3]}) + sub.create_array("bar", np.arange(10, dtype=np.uint16), compression=None) + sub.create_bytes_array("blob", b"\x89PNG\x00data") + consolidate_store(store) + close_store(store) + + digest = hashlib.sha256(Path(self.itar_path).read_bytes()).hexdigest() self.assertIn( digest, { @@ -287,21 +321,94 @@ def test_archives_of_ncore_19_8(self): ) def test_empty(self): - with IndexedTarStore(self.path, mode="w") as store: - consolidate_compressed_metadata(ZarrIndexedTarStore(store)) - - with self.assertRaises(zarr.errors.PathNotFoundError): - open_compressed_consolidated(IndexedTarStore(self.path), mode="r") - with self.assertRaises(zarr.errors.PathNotFoundError): - zarr.open(store=ZarrIndexedTarStore(IndexedTarStore(self.path)), mode="r") + """Stores without a root group can be consolidated, but not opened""" + store = IndexedTarStore(self.itar_path, mode="w") + consolidate_store(store) + close_store(store) + with self.assertRaises(Exception): # (zarr-python's error of missing groups) + open_store(IndexedTarStore(self.itar_path), consolidated=True) def test_read_only(self): - _write_itar(self.path) - root = zarr.open_group(store=ZarrIndexedTarStore(IndexedTarStore(self.path)), mode="r") - with self.assertRaises(zarr.errors.ReadOnlyError): + _write(self._store("itar", "w")) + root = open_store(IndexedTarStore(self.itar_path)) + with self.assertRaises(Exception): root.create_group("new") -def unpack(file: Optional[Any]) -> bytes: +class TestNodes(unittest.TestCase): + """Typed node API""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = create_root_group( + open_directory_store(self._tmp.name), {"s": "x", "i": 3, "b": True, "l": [1], "d": {"k": 1}} + ) + + def tearDown(self): + self._tmp.cleanup() + + def test_typed_attributes(self): + self.assertEqual(self.root.attr_str("s"), "x") + self.assertEqual(self.root.attr_int("i"), 3) + self.assertEqual(self.root.attr_list("l"), [1]) + self.assertEqual(self.root.attr_dict("d"), {"k": 1}) + with self.assertRaises(TypeError): + self.root.attr_int("b") # (booleans aren't integers) + with self.assertRaises(TypeError): + self.root.attr_str("i") + with self.assertRaises(KeyError): + self.root.attr_str("missing") + + self.root.update_attrs({"i": 4, "new": None}) + self.assertEqual(self.root.attrs, {"s": "x", "i": 4, "b": True, "l": [1], "d": {"k": 1}, "new": None}) + + def test_arrays(self): + array = self.root.create_array("a", [1.5, 2.5], dtype=np.float32, attributes={"unit": "m"}) + self.assertEqual((array.shape, array.dtype, array.attrs), ((2,), np.dtype(np.float32), {"unit": "m"})) + self.assertEqual(array.compression, nodes.BloscCompression()) + np.testing.assert_array_equal(array.read((slice(1, 2),)), [2.5]) + self.assertEqual(array.read((0,)).shape, ()) # 0-d arrays for scalar selections + + self.assertIsNone(self.root.create_array("u", np.arange(3), compression=None).compression) + self.assertEqual(self.root.create_array("c", np.zeros((10, 0)), chunks=(4, 0)).zarr.chunks, (4, 1)) + with self.assertRaises(TypeError): + array.read_bytes() + + def test_groups(self): + self.root.create_group("b") + self.root.create_group("a").create_group("nested") + self.assertEqual([name for name, _ in self.root.groups()], ["a", "b"]) + self.assertEqual(self.root.require_group("a").group("nested").path, "a/nested") + self.assertEqual(self.root.group("a/nested").path, "a/nested") + with self.assertRaises(Exception): + self.root.create_group("a") + + def test_nodes(self): + """Members of unknown kind (groups or arrays)""" + self.root.create_group("g").create_array("a", np.arange(3)) + assert isinstance(group := self.root.node("g"), nodes.Group) + self.assertEqual(group.path, "g") + assert isinstance(array := self.root.node("g/a"), nodes.Array) + np.testing.assert_array_equal(array.read(), np.arange(3)) + with self.assertRaises(KeyError): + self.root.node("g/missing") + + +class TestBloscCompression(unittest.TestCase): + @parameterized.parameterized.expand( + [({"cname": "snappy"},), ({"clevel": 10},), ({"clevel": -1},), ({"shuffle": "byte"},), ({"blocksize": -1},)] + ) + def test_invalid_parameters(self, kwargs: Any): + with self.assertRaises(ValueError): + nodes.BloscCompression(**kwargs) + + @parameterized.parameterized.expand([(shuffle,) for shuffle in ("noshuffle", "shuffle", "bitshuffle")]) + def test_numcodecs(self, shuffle: Any): + compression = nodes.BloscCompression(cname="zstd", clevel=3, shuffle=shuffle, blocksize=1024) + self.assertEqual(nodes.BloscCompression.from_numcodecs(compression.to_numcodecs()), compression) + + +def _read_member(tar: tarfile.TarFile, member: tarfile.TarInfo) -> bytes: + file = tar.extractfile(member) assert file is not None return file.read() diff --git a/ncore/impl/data/types.py b/ncore/impl/data/types.py index e4e29cf4..3fc7a697 100644 --- a/ncore/impl/data/types.py +++ b/ncore/impl/data/types.py @@ -49,7 +49,7 @@ import numpy.typing as npt # type: ignore[import-not-found] from ncore.impl.common.transformations import PoseGraphInterpolator, transform_bbox, transform_point_cloud -from ncore.impl.data import util +from ncore.impl.data import _json, util if sys.version_info >= (3, 11): @@ -61,15 +61,7 @@ ## JSON-like structures -JsonLike = Union[ - Dict[str, "JsonLike"], - List["JsonLike"], - str, - int, - float, - bool, - None, -] +JsonLike = _json.JsonLike ## Data classes representing stored data types diff --git a/ncore/impl/data/v4/components.py b/ncore/impl/data/v4/components.py index 9f7fe2f5..30be6037 100644 --- a/ncore/impl/data/v4/components.py +++ b/ncore/impl/data/v4/components.py @@ -44,17 +44,13 @@ import dataclasses_json import numpy as np import PIL.Image as PILImage -import zarr -import zarr.storage -from numcodecs import Blosc from typing_extensions import Concatenate, ParamSpec from upath import UPath -from zarr._storage.store import Store from ncore.impl.common.transformations import HalfClosedInterval from ncore.impl.common.util import MD5Hasher, unpack_optional -from ncore.impl.data import stores, types, util +from ncore.impl.data import nodes, stores, types, util from ncore.impl.data.types import PointCloud @@ -72,32 +68,30 @@ VERSION = "v4" -def _normalize_chunks( - shape: "Tuple[int, ...]", - *, - require_nonzero_dims: "Optional[Tuple[int, ...]]" = None, -) -> "Tuple[int, ...]": - """Normalize a chunk shape by replacing zero dimensions with 1. +def _check_nonzero_dims(shape: Tuple[int, ...], dims: Tuple[int, ...], name: str) -> None: + """Asserts that the given dimensions of the shape of the named data are non-zero. - Zarr v2 rejects chunk tuples containing zeros (division by zero in internal - bookkeeping). This helper ensures that every dimension in the chunk tuple is - at least 1 while the *logical* data shape passed to ``create_dataset`` via - the ``data=`` argument remains unchanged. + Other dimensions are allowed to be zero (arrays with zero-sized dimensions are supported). Args: - shape: The data shape to derive a chunk shape from. - require_nonzero_dims: Optional tuple of dimension indices that MUST be - non-zero (raises AssertionError otherwise). When ``None`` (the - default) all dimensions are allowed to be zero and get clamped to 1 - in the returned chunk shape. - - Returns: - A chunk-safe tuple where any zero dimension is replaced with 1. + shape: The shape of the data. + dims: Indices of the dimensions that MUST be non-zero (raises AssertionError otherwise). + name: Name of the data (for error messages). """ - if require_nonzero_dims is not None: - for d in require_nonzero_dims: - assert shape[d] != 0, f"Dimension {d} must be non-zero, got shape {shape}" - return tuple(max(1, s) for s in shape) + for d in dims: + assert shape[d] != 0, f"Dimension {d} of {name} must be non-zero, got shape {shape}" + + +def _frame_id( + # end-of-frame timestamp, or start-of-frame / end-of-frame timestamps + timestamps_us: Union[int, npt.NDArray[np.uint64]], +) -> int: + """Returns the frame ID (the end-of-frame timestamp) of a frame""" + + if isinstance(timestamps_us, np.ndarray): + return cast(np.ndarray, timestamps_us)[1].item() # end-of-frame timestamp is frame ID + + return timestamps_us @dataclass @@ -180,7 +174,7 @@ def __init__( # Individual stores for each group are initialized lazily on-demand (indexed tar file or zarr directories) self._stores_rootgroups: dict[ - str, Tuple[zarr.Group, UPath] + str, Tuple[nodes.Group, UPath] ] = {} # maps component group names to stores, store path, and base groups self._store_type = store_type @@ -201,7 +195,7 @@ def sequence_timestamp_interval_us(self) -> HalfClosedInterval: def generic_meta_data(self) -> Dict[str, types.JsonLike]: return self._generic_meta_data - def get_base_group(self, component_group_name: Optional[str]) -> zarr.Group: + def get_base_group(self, component_group_name: Optional[str]) -> nodes.Group: """Lazily initializes ncore base-groups and underlying stores on demand""" if component_group_name is None: @@ -209,7 +203,7 @@ def get_base_group(self, component_group_name: Optional[str]) -> zarr.Group: component_group_name = "" if (store_rootgroup := self._stores_rootgroups.get(component_group_name)) is not None: - # Store already initialized, return it's root group + # Store already initialized, return its root group return store_rootgroup[0] # Store doesn't exist yet, create it @@ -221,23 +215,21 @@ def get_base_group(self, component_group_name: Optional[str]) -> zarr.Group: # append group name as suffix to store name if given store_name += f"-{component_group_name}" - store: Store + store: stores.StoreLike if self._store_type == "itar": # container-based zarr stores ..zarr.itar store_path = self._output_dir_path / f"{self._store_base_name}.{store_name}.zarr.itar" - store = stores.ZarrIndexedTarStore(stores.IndexedTarStore(store_path, mode="w")) + store = stores.IndexedTarStore(store_path, mode="w") elif self._store_type == "directory": - # directory-based zarr stores ..zarr.zarr + # directory-based zarr stores ..zarr store_path = self._output_dir_path / f"{self._store_base_name}.{store_name}.zarr" - store = zarr.storage.DirectoryStore(store_path) + store = stores.open_directory_store(store_path) else: raise ValueError(f"Unknown store type {self._store_type}") - # Create root group in store - root_group = zarr.group(store=store) - - # Store dataset associated meta-data to root - root_group.attrs.put( + # Create root group in store, storing dataset associated meta-data to root + root_group = stores.create_root_group( + store, { "sequence_id": self._sequence_id, "sequence_timestamp_interval_us": { @@ -247,7 +239,7 @@ def get_base_group(self, component_group_name: Optional[str]) -> zarr.Group: "generic_meta_data": self._generic_meta_data, "version": VERSION, "component_group_name": component_group_name, - } + }, ) # Create store / base-group mapping @@ -267,32 +259,27 @@ def finalize(self) -> List[UPath]: # Write deferred component metadata and generic data of all writers for component_writer in self._component_writers.values(): - # Merge generic_meta_data with regular component's meta data - (cw_group := component_writer._group).attrs.put( + # Merge generic_meta_data with regular component's meta data (component groups are created without + # attributes by register_component_writer(), so this sets the complete component attributes) + (cw_group := component_writer._group).update_attrs( {**component_writer._component_meta_data, "generic_meta_data": component_writer._generic_meta_data} ) # Write generic data arrays if any if component_writer._generic_data: - compressor = Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE) gd_group = cw_group.require_group("generic_data") for name, array in component_writer._generic_data.items(): - gd_group.create_dataset( - name, - data=array, - chunks=_normalize_chunks(array.shape), - compressor=compressor, - ) + gd_group.create_array(name, array) # Make sure the stores are consolidated and closed ret = [] for root_group, store_path in self._stores_rootgroups.values(): - store = root_group.store + store = stores.group_store(root_group) - stores.consolidate_compressed_metadata(store) + stores.consolidate_store(store) # Finish writing all files - store.close() + stores.close_store(store) ret.append(store_path) @@ -300,7 +287,7 @@ def finalize(self) -> List[UPath]: def register_component_writer( self, - component_writer_type: "Callable[Concatenate[zarr.Group, HalfClosedInterval, P], CW]", + component_writer_type: "Callable[Concatenate[nodes.Group, HalfClosedInterval, P], CW]", component_instance_name: str, group_name: Optional[str] = None, generic_meta_data: Optional[Dict[str, types.JsonLike]] = None, @@ -411,11 +398,11 @@ def __init__( assert len(component_group_upaths), "No component inputs provided" # Load component stores concurrently (to hide latency) and check for sequence consistency - self._component_stores: Dict[str, Tuple[zarr.Group, UPath]] = {} # use str as the generic path / URL type + self._component_stores: Dict[str, Tuple[nodes.Group, UPath]] = {} # use str as the generic path / URL type with concurrent.futures.ThreadPoolExecutor(max_workers=max_threads) as executor: - def thread_load_component_store(component_store_upath: UPath) -> Tuple[zarr.Group, UPath]: + def thread_load_component_store(component_store_upath: UPath) -> Tuple[nodes.Group, UPath]: """Thread-executed shard opening""" # Make sure paths are absolute at this point @@ -423,7 +410,7 @@ def thread_load_component_store(component_store_upath: UPath) -> Tuple[zarr.Grou _logger.info(f"SequenceStoreReader: Loading component store {component_store_upath}") - component_store: Store + component_store: stores.StoreLike if component_store_upath.is_file(): if not component_store_upath.name.endswith(".zarr.itar"): # not a supported file-based store format @@ -431,21 +418,13 @@ def thread_load_component_store(component_store_upath: UPath) -> Tuple[zarr.Grou f"Unsupported file-based store format {component_store_upath}, expected .zarr.itar" ) - component_store = stores.ZarrIndexedTarStore( - stores.IndexedTarStore( - component_store_upath, mode="r", index_tail_read_size=itar_index_tail_read_size - ) + component_store = stores.IndexedTarStore( + component_store_upath, mode="r", index_tail_read_size=itar_index_tail_read_size ) else: - component_store = zarr.storage.DirectoryStore(component_store_upath) + component_store = stores.open_directory_store(component_store_upath) - component_root = ( - stores.open_compressed_consolidated(store=component_store, mode="r") - if open_consolidated - else zarr.open(store=component_store, mode="r") - ) - - return cast(zarr.Group, component_root), component_store_upath + return stores.open_store(component_store, open_consolidated), component_store_upath for future in concurrent.futures.as_completed( [ @@ -456,21 +435,20 @@ def thread_load_component_store(component_store_upath: UPath) -> Tuple[zarr.Grou # Note: thread completion order is not relevant here component_root, component_store_path = future.result() - component_root_attrs = dict(component_root.attrs.items()) - # Check sequence compatibility - component_root_version = component_root_attrs["version"] + component_root_version = component_root.attr_str("version") if component_root_version not in [VERSION]: raise RuntimeError( f"Can't load V4 component store {component_store_path} with incompatible data version {component_root_version}" ) - component_store_sequence_id = component_root_attrs["sequence_id"] + component_store_sequence_id = component_root.attr_str("sequence_id") + sequence_timestamp_interval_us = component_root.attr_dict("sequence_timestamp_interval_us") component_store_sequence_timestamp_interval_us = HalfClosedInterval( - component_root_attrs["sequence_timestamp_interval_us"]["start"], - component_root_attrs["sequence_timestamp_interval_us"]["stop"], + cast(int, sequence_timestamp_interval_us["start"]), + cast(int, sequence_timestamp_interval_us["stop"]), ) - component_store_generic_meta_data = component_root_attrs["generic_meta_data"] + component_store_generic_meta_data = component_root.attr_dict("generic_meta_data") if not self._component_stores: self._sequence_id: str = component_store_sequence_id @@ -487,7 +465,7 @@ def thread_load_component_store(component_store_upath: UPath) -> Tuple[zarr.Grou if not self._version == component_root_version: raise RuntimeError("Can't load shards from different data versions") - component_group_name = component_root_attrs["component_group_name"] + component_group_name = component_root.attr_str("component_group_name") if component_group_name in self._component_stores: raise RuntimeError(f"Component group {component_group_name} loaded multiple times") @@ -500,12 +478,8 @@ def thread_load_component_store(component_store_upath: UPath) -> Tuple[zarr.Grou def reload_resources(self) -> None: """Trigger a reload of each itar store - useful to re-initialize file objects in multi-process settings""" for component_root, _ in self._component_stores.values(): - store = component_root.store - # unwind one layer of possible consolidated metadata store - if isinstance(store, stores.ConsolidatedCompressedMetadataStore): - store = store.store - if isinstance(store, stores.ZarrIndexedTarStore): - store.store.reload_resources() + if isinstance(store := stores.group_store(component_root), stores.IndexedTarStore): + store.reload_resources() @property def sequence_id(self) -> str: @@ -532,17 +506,17 @@ def open_component_readers( ret = {} for component_root_group, _ in self._component_stores.values(): - if (component_group := component_root_group.get(component_reader_type.get_component_name())) is None: + if (component_name := component_reader_type.get_component_name()) not in component_root_group: continue # instantiate a reader for each of the components - for component_instance_name, component_group in component_group.items(): + for component_instance_name, component_group in component_root_group.group(component_name).groups(): assert component_instance_name not in ret, ( f"Component instance {component_instance_name} encountered multiple times" ) # check if the reader supports the component version - if not component_reader_type.supports_component_version(component_group.attrs["component_version"]): + if not component_reader_type.supports_component_version(component_group.attr_str("component_version")): continue ret[component_instance_name] = component_reader_type(component_instance_name, component_group) @@ -556,14 +530,13 @@ def get_sequence_meta(self) -> SequenceMeta: component_stores_info: List[SequenceMeta.ComponentStoreMeta] = [] for component_root_group, component_store_path in self._component_stores.values(): components: Dict[str, Dict[str, SequenceMeta.ComponentInstanceMeta]] = {} - for component_name, component in component_root_group.items(): + for component_name, component in component_root_group.groups(): # collect component names and instances component_instances: Dict[str, SequenceMeta.ComponentInstanceMeta] = {} - for component_instance_name, component_instance in component.items(): - component_instance_attrs = component_instance.attrs + for component_instance_name, component_instance in component.groups(): component_instances[component_instance_name] = SequenceMeta.ComponentInstanceMeta( - version=component_instance_attrs["component_version"], - generic_meta_data=component_instance_attrs["generic_meta_data"], + version=component_instance.attr_str("component_version"), + generic_meta_data=component_instance.attr_dict("generic_meta_data"), ) components[component_name] = component_instances @@ -611,7 +584,7 @@ def get_component_version() -> str: """Returns the version of the current component writer""" ... - def __init__(self, component_group: zarr.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: + def __init__(self, component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: """Initializes a component writer targeting the given component group and sequence time interval""" self._group = component_group @@ -651,7 +624,7 @@ def finalize(self) -> None: Called by :meth:`SequenceComponentGroupsWriter.finalize` before component metadata is persisted. Override to flush buffered data - (e.g., create zarr datasets/groups). + (e.g., create arrays / groups). """ pass @@ -660,7 +633,7 @@ class ComponentReader(ABC): """Base class for V4 component readers. Subclasses must implement :meth:`get_component_name` and - :meth:`supports_component_version`. The underlying zarr group is + :meth:`supports_component_version`. The underlying component group is accessible as ``self._group``; component metadata is available via the :attr:`instance_name`, :attr:`component_version`, and :attr:`generic_meta_data` properties. @@ -676,14 +649,16 @@ def get_component_name() -> str: def supports_component_version(version: str) -> bool: """Returns true if the component version is supported by the reader""" - def __init__(self, component_instance_name: str, component_group: zarr.Group) -> None: + def __init__(self, component_instance_name: str, component_group: nodes.Group) -> None: """Initializes a component reader for a given component instance name and group""" self._instance_name = component_instance_name self._group = component_group # Preload component meta-data and generic data group (if existing) - self._component_meta_data: Dict = dict(self._group.attrs) - self._generic_data_group: Optional[zarr.Group] = self._group.get("generic_data") + self._component_meta_data: Dict[str, types.JsonLike] = self._group.attrs + self._generic_data_group: Optional[nodes.Group] = ( + self._group.group("generic_data") if "generic_data" in self._group else None + ) @property def instance_name(self) -> str: @@ -693,12 +668,12 @@ def instance_name(self) -> str: @property def component_version(self) -> str: """Returns the component version of the loaded component""" - return self._component_meta_data["component_version"] + return cast(str, self._component_meta_data["component_version"]) @property def generic_meta_data(self) -> Dict[str, types.JsonLike]: """Returns the generic meta data of the loaded component""" - return self._component_meta_data["generic_meta_data"] + return cast(Dict[str, types.JsonLike], self._component_meta_data["generic_meta_data"]) def has_generic_data(self, name: str) -> bool: """Returns True if a named generic data array exists on this component""" @@ -706,7 +681,7 @@ def has_generic_data(self, name: str) -> bool: def get_generic_data_names(self) -> List[str]: """Returns the list of all generic data array names on this component""" - return list(self._generic_data_group.keys()) if self._generic_data_group is not None else [] + return self._generic_data_group.members() if self._generic_data_group is not None else [] def get_generic_data(self, name: str) -> np.ndarray: """Returns a named generic data array from this component. @@ -715,8 +690,8 @@ def get_generic_data(self, name: str) -> np.ndarray: """ if self._generic_data_group is None: raise KeyError("Component has no generic_data") - if (generic_data := self._generic_data_group.get(name)) is not None: - return np.array(generic_data) + if name in self._generic_data_group: + return self._generic_data_group.array(name).read() raise KeyError(f"Generic data '{name}' not found. Available: {self.get_generic_data_names()}") @@ -750,7 +725,7 @@ def get_component_version() -> str: """Returns the version of the current component writer""" return "v1" - def __init__(self, component_group: zarr.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: + def __init__(self, component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: """Initializes the current component writer targeting the given component group and sequence time interval""" super().__init__(component_group, sequence_timestamp_interval_us) @@ -759,8 +734,8 @@ def __init__(self, component_group: zarr.Group, sequence_timestamp_interval_us: def finalize(self) -> None: """Actually store the json-encoded pose data""" - self._group.create_group("static_poses").attrs.put(self.data["static_poses"]) - self._group.create_group("dynamic_poses").attrs.put(self.data["dynamic_poses"]) + self._group.create_group("static_poses", attributes=self.data["static_poses"]) + self._group.create_group("dynamic_poses", attributes=self.data["dynamic_poses"]) def store_static_pose( self, @@ -858,10 +833,18 @@ def supports_component_version(version: str) -> bool: """Returns true if the component version is supported by the reader""" return version == "v1" + def _static_poses(self) -> Dict[str, Dict[str, Any]]: + """Returns the json-encoded static poses data""" + return cast(Dict[str, Dict[str, Any]], self._group.group("static_poses").attrs) + + def _dynamic_poses(self) -> Dict[str, Dict[str, Any]]: + """Returns the json-encoded dynamic poses data""" + return cast(Dict[str, Dict[str, Any]], self._group.group("dynamic_poses").attrs) + def get_static_poses(self) -> Generator[Tuple[Tuple[str, str], npt.NDArray[np.floating]]]: """Returns all static poses (rigid transformations) between named coordinate frames, if available""" - for key, static_pose in self._group["static_poses"].attrs.items(): + for key, static_pose in self._static_poses().items(): yield ast.literal_eval(key), np.array(static_pose["pose"], dtype=static_pose["dtype"]) def get_dynamic_poses( @@ -869,7 +852,7 @@ def get_dynamic_poses( ) -> Generator[Tuple[Tuple[str, str], Tuple[npt.NDArray[np.floating], npt.NDArray[np.uint64]]]]: """Returns all dynamic poses (time-dependent rigid transformations) between named coordinate frames, if available""" - for key, dynamic_poses in self._group["dynamic_poses"].attrs.items(): + for key, dynamic_poses in self._dynamic_poses().items(): yield ( ast.literal_eval(key), ( @@ -882,7 +865,7 @@ def get_static_pose(self, source_frame_id: str, target_frame_id: str) -> npt.NDA """Returns static pose (rigid transformation) between two named coordinate frames, if available""" if ( - static_pose := self._group["static_poses"].attrs.get( + static_pose := self._static_poses().get( key := str((validate_frame_name(source_frame_id), validate_frame_name(target_frame_id))) ) ) is None: @@ -896,7 +879,7 @@ def get_dynamic_pose( """Returns dynamic poses (time-dependent rigid transformations) between two named coordinate frames, if available""" if ( - dynamic_poses := self._group["dynamic_poses"].attrs.get( + dynamic_poses := self._dynamic_poses().get( key := str((validate_frame_name(source_frame_id), validate_frame_name(target_frame_id))) ) ) is None: @@ -925,7 +908,7 @@ def get_component_version() -> str: """Returns the version of the current intrinsic calibration component""" return "v1" - def __init__(self, component_group: zarr.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: + def __init__(self, component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: """Initializes the current component writer targeting the given component group and sequence time interval""" super().__init__(component_group, sequence_timestamp_interval_us) @@ -944,7 +927,7 @@ def store_camera_intrinsics( meta_data = types.encode_camera_model_parameters(camera_model_parameters) - self._cameras_group.create_group(camera_id).attrs.put(meta_data) + self._cameras_group.create_group(camera_id, attributes=meta_data) return self @@ -959,7 +942,7 @@ def store_lidar_intrinsics( # Prepare meta-data containing the serialization of the mandatory lidar model meta_data = types.encode_lidar_model_parameters(lidar_model_parameters) - self._lidars_group.create_group(lidar_id).attrs.put(meta_data) + self._lidars_group.create_group(lidar_id, attributes=meta_data) return self @@ -978,16 +961,14 @@ def supports_component_version(version: str) -> bool: def get_camera_model_parameters(self, camera_id: str) -> types.CameraModelParameters: """Returns the camera model associated with the requested camera sensor""" - return types.decode_camera_model_parameters(cast(zarr.Group, self._group["cameras"][camera_id]).attrs) + return types.decode_camera_model_parameters(self._group.group(f"cameras/{camera_id}").attrs) def get_lidar_model_parameters(self, lidar_id: str) -> Optional[types.LidarModelParameters]: """Returns the lidar model associated with the requested lidar sensor""" - lidars_group = self._group["lidars"] - - if lidar_id not in lidars_group: + if lidar_id not in self._group.group("lidars"): return None - return types.decode_lidar_model_parameters(cast(zarr.Group, lidars_group[lidar_id]).attrs) + return types.decode_lidar_model_parameters(self._group.group(f"lidars/{lidar_id}").attrs) class MasksComponent: @@ -1008,7 +989,7 @@ def get_component_version() -> str: """Returns the version of the current sensor masks component""" return "v1" - def __init__(self, component_group: zarr.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: + def __init__(self, component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: """Initializes the current component writer targeting the given component group and sequence time interval""" super().__init__(component_group, sequence_timestamp_interval_us) @@ -1023,8 +1004,8 @@ def store_camera_masks( """Store camera-associated masks""" # Store mask names - (camera_grp := self._cameras_group.create_group(camera_id)).attrs.put( - {"mask_names": list(mask_images.keys())} + camera_grp = self._cameras_group.create_group( + camera_id, attributes={"mask_names": list(mask_images.keys())} ) # Store mask images @@ -1033,9 +1014,7 @@ def store_camera_masks( FORMAT = "png" mask_image.save(buffer, format=FORMAT, optimize=True) # encodes as png # store mask data (uncompressed, as already encoded) - camera_grp.create_dataset(mask_name, data=np.asarray(buffer.getvalue()), compressor=None).attrs[ - "format" - ] = FORMAT + camera_grp.create_bytes_array(mask_name, buffer.getvalue(), attributes={"format": FORMAT}) return self @@ -1055,14 +1034,14 @@ def supports_component_version(version: str) -> bool: def get_camera_mask_names(self, camera_id: str) -> List[str]: """Returns all constant camera mask names""" - return list(cast(zarr.Group, self._group["cameras"][camera_id]).attrs.get("mask_names", [])) + return list(cast(List[str], self._group.group(f"cameras/{camera_id}").attrs.get("mask_names", []))) def get_camera_mask_image(self, camera_id: str, mask_name: str) -> PILImage.Image: """Returns constant named camera mask image""" - mask_dataset = cast(zarr.Array, cast(zarr.Group, self._group["cameras"][camera_id])[mask_name]) + mask_array = self._group.array(f"cameras/{camera_id}/{mask_name}") - return PILImage.open(io.BytesIO(cast(np.bytes_, mask_dataset[()])), formats=[mask_dataset.attrs["format"]]) + return PILImage.open(io.BytesIO(mask_array.read_bytes()), formats=[mask_array.attr_str("format")]) def get_camera_mask_images(self, camera_id: str) -> Generator[Tuple[str, PILImage.Image]]: """Returns all constant named camera mask images""" @@ -1074,7 +1053,7 @@ def get_camera_mask_images(self, camera_id: str) -> Generator[Tuple[str, PILImag class BaseSensorComponentWriter(ComponentWriter): """Base class for all sensor component writers""" - def __init__(self, component_group: zarr.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: + def __init__(self, component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval) -> None: """Initializes the current component writer targeting the given component group and sequence time interval""" super().__init__(component_group, sequence_timestamp_interval_us) @@ -1102,21 +1081,16 @@ def finalize(self) -> None: ) # Store as meta-data of frames group - self._frames_group.attrs.put({"frames_timestamps_us": frames_timestamps_us.tolist()}) + self._frames_group.update_attrs({"frames_timestamps_us": frames_timestamps_us.tolist()}) def _get_frame_group( self, # end-of-frame timestamp, or start-of-frame / end-of-frame timestamps timestamps_us: Union[int, npt.NDArray[np.uint64]], - ) -> zarr.Group: + ) -> nodes.Group: """Returns the group of a frame, initializing it if required""" - if isinstance(timestamps_us, np.ndarray): - frame_id = cast(np.ndarray, timestamps_us)[1].item() # end-of-frame timestamp is frame ID - else: - frame_id = timestamps_us - - return self._frames_group.require_group(str(frame_id)) + return self._frames_group.require_group(str(_frame_id(timestamps_us))) def _store_base_frame( self, @@ -1125,7 +1099,7 @@ def _store_base_frame( # generic per-frame data (key-value pairs, *not* dimension / dtype validated) and meta-data generic_data: Dict[str, npt.NDArray[Any]], generic_meta_data: Dict[str, types.JsonLike], - ) -> zarr.Group: + ) -> nodes.Group: # Sanity / timestamp consistency checks assert np.shape(frame_timestamps_us) == (2,) assert frame_timestamps_us.dtype == np.dtype("uint64") @@ -1148,16 +1122,11 @@ def _store_base_frame( self._frames_timestamps_us[frame_timestamps_us[1].item()] = frame_timestamps_us[0].item() # Store additional generic frame data and meta-data (not dimension / dtype checked) - (frame_generic_data_group := frame_group.create_group("generic_data")).attrs.put(generic_meta_data) + frame_generic_data_group = frame_group.create_group("generic_data", attributes=generic_meta_data) for name, value in generic_data.items(): - frame_generic_data_group.create_dataset( - name, - data=value, - # we are not accessing sub-ranges, so disable chunking - chunks=_normalize_chunks(value.shape), - # use compression that is fast to decode on modern hardware - compressor=Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE), - ) + # we are not accessing sub-ranges, so store a single chunk (default) - use the default compression, + # which is fast to decode on modern hardware + frame_generic_data_group.create_array(name, value) return frame_group @@ -1165,7 +1134,7 @@ def _store_base_frame( class BaseSensorComponentReader(ComponentReader): """Base class for all sensor component readers""" - def __init__(self, component_instance_name: str, component_group: zarr.Group) -> None: + def __init__(self, component_instance_name: str, component_group: nodes.Group) -> None: """Initializes a component reader for a given component instance name and group""" super().__init__(component_instance_name, component_group) @@ -1173,7 +1142,9 @@ def __init__(self, component_instance_name: str, component_group: zarr.Group) -> raise RuntimeError("Sensor component doesn't contain any frames") # preload frame timestamps and create map - self._frames_timestamps_us = np.array(self._group["frames"].attrs["frames_timestamps_us"], dtype=np.uint64) + self._frames_timestamps_us = np.array( + self._group.group("frames").attr_list("frames_timestamps_us"), dtype=np.uint64 + ) self._frame_end_to_frame_timestamps_us = { end: np.array([self._frames_timestamps_us[i, 0], end], dtype=np.uint64) for i, end in enumerate(self._frames_timestamps_us[:, 1]) @@ -1183,19 +1154,32 @@ def _get_frame_group( self, # end-of-frame timestamp, or start-of-frame / end-of-frame timestamps timestamps_us: Union[int, npt.NDArray[np.uint64]], - ) -> zarr.Group: + ) -> nodes.Group: """Returns the group of a frame""" - if isinstance(timestamps_us, np.ndarray): - frame_id = cast(np.ndarray, timestamps_us)[1].item() # end-of-frame timestamp is frame ID - else: - frame_id = timestamps_us + return self._group.group(self._frame_path(timestamps_us)) + + @staticmethod + def _frame_path( + # end-of-frame timestamp, or start-of-frame / end-of-frame timestamps + timestamps_us: Union[int, npt.NDArray[np.uint64]], + # optional name of a frame member + name: Optional[str] = None, + ) -> str: + """Returns the path of a frame group (or of one of its members) within the component group""" + + frame_path = f"frames/{_frame_id(timestamps_us)}" - return cast(zarr.Group, self._group["frames"][str(frame_id)]) + return frame_path if name is None else f"{frame_path}/{name}" + + def _get_frame_generic_data_group(self, timestamp_us: int) -> nodes.Group: + """Returns the generic data group of a frame""" + + return self._group.group(self._frame_path(timestamp_us, "generic_data")) @property def frames_timestamps_us(self) -> npt.NDArray[np.uint64]: - return np.array(self._group["frames"].attrs["frames_timestamps_us"], dtype=np.uint64) + return np.array(self._group.group("frames").attr_list("frames_timestamps_us"), dtype=np.uint64) @property def frames_count(self) -> int: @@ -1208,7 +1192,7 @@ def get_frame_timestamps_us(self, timestamp_us: int) -> npt.NDArray[np.uint64]: def get_frame_generic_data_names(self, timestamp_us: int) -> List[str]: """List of all generic frame-data names""" - return list(cast(zarr.Group, self._get_frame_group(timestamp_us)["generic_data"]).keys()) + return self._get_frame_generic_data_group(timestamp_us).members() def has_frame_generic_data(self, timestamp_us: int, name: str) -> bool: """Signals if named generic frame-data exists""" @@ -1218,12 +1202,12 @@ def has_frame_generic_data(self, timestamp_us: int, name: str) -> bool: def get_frame_generic_data(self, timestamp_us: int, name: str) -> npt.NDArray[Any]: """Returns generic frame-data for a specific frame and name""" - return np.array(self._get_frame_group(timestamp_us)["generic_data"][name]) + return self._get_frame_generic_data_group(timestamp_us).array(name).read() def get_frame_generic_meta_data(self, timestamp_us: int) -> Dict[str, types.JsonLike]: """Returns generic frame meta-data for a specific frame""" - return dict(self._get_frame_group(timestamp_us)["generic_data"].attrs) + return self._get_frame_generic_data_group(timestamp_us).attrs class BaseRayBundleSensorComponentWriter(BaseSensorComponentWriter): @@ -1243,21 +1227,16 @@ def _store_frame_ray_bundle( ) -> None: ## Initialize ray bundle group frame_group = self._get_frame_group(frame_timestamps_us) - (ray_bundle_group := frame_group.create_group("ray_bundle")).attrs.put({"n_rays": n_rays}) + ray_bundle_group = frame_group.create_group("ray_bundle", attributes={"n_rays": n_rays}) - # Store per-ray data + # Store per-ray data (zero-sized chunk dimensions are clamped to 1, default compression is fast to decode on + # modern hardware) for name, (ray_data_data, chunks) in ray_data.items(): assert len(ray_data_data) == n_rays, f"{name} doesn't have required ray count" - ray_bundle_group.create_dataset( - name, - data=ray_data_data, - chunks=_normalize_chunks(chunks), - # use compression that is fast to decode on modern hardware - compressor=Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE), - ) + ray_bundle_group.create_array(name, ray_data_data, chunks=chunks) ## Initialize ray bundle returns group - (ray_bundle_returns_group := frame_group.create_group("ray_bundle_returns")).attrs.put({"n_returns": n_returns}) + ray_bundle_returns_group = frame_group.create_group("ray_bundle_returns", attributes={"n_returns": n_returns}) # Store per-return data absent_mask = None @@ -1290,13 +1269,7 @@ def _store_frame_ray_bundle( # validate absent mask consistency assert np.array_equal(absent_mask, local_absent_mask), f"Inconsistent NaN masks in return data {name}" - ray_bundle_returns_group.create_dataset( - name, - data=return_data_data, - chunks=_normalize_chunks(chunks), - # use compression that is fast to decode on modern hardware - compressor=Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE), - ) + ray_bundle_returns_group.create_array(name, return_data_data, chunks=chunks) if absent_mask is None: # Initialize empty absent mask if no per-return data was provided @@ -1304,34 +1277,32 @@ def _store_frame_ray_bundle( valid_mask_packed = np.packbits(~absent_mask) - frame_group.create_dataset( + # we are not accessing sub-ranges, so store a single chunk (default) + frame_group.create_array( "ray_bundle_returns_valid_mask_packed", - data=valid_mask_packed, - # we are not accessing sub-ranges, so disable chunking - chunks=_normalize_chunks(valid_mask_packed.shape), - # use compression that is fast to decode on modern hardware - compressor=Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE), - ).attrs.put({"n_returns": n_returns, "n_rays": n_rays}) + valid_mask_packed, + attributes={"n_returns": n_returns, "n_rays": n_rays}, + ) class BaseRayBundleSensorComponentReader(BaseSensorComponentReader): """Base class for all ray bundle sensor component readers""" # per-ray data - def _get_ray_bundle_group(self, timestamp_us: int) -> zarr.Group: + def _get_ray_bundle_group(self, timestamp_us: int) -> nodes.Group: """Returns the ray bundle group of a frame""" - return cast(zarr.Group, self._get_frame_group(timestamp_us)["ray_bundle"]) + return self._group.group(self._frame_path(timestamp_us, "ray_bundle")) def get_frame_ray_bundle_count(self, timestamp_us: int) -> int: """Returns the number of rays and ray returns for a specific frame""" - return self._get_ray_bundle_group(timestamp_us).attrs["n_rays"] + return self._get_ray_bundle_group(timestamp_us).attr_int("n_rays") def get_frame_ray_bundle_data_names(self, timestamp_us: int) -> List[str]: """List of all ray bundle data names for a frame""" - return list(self._get_ray_bundle_group(timestamp_us).keys()) + return self._get_ray_bundle_group(timestamp_us).members() def has_frame_ray_bundle_data(self, timestamp_us: int, name: str) -> bool: """Signals if named ray bundle data exists for a frame""" @@ -1341,23 +1312,23 @@ def has_frame_ray_bundle_data(self, timestamp_us: int, name: str) -> bool: def get_frame_ray_bundle_data(self, timestamp_us: int, name: str) -> npt.NDArray[Any]: """Returns named ray bundle data for a frame""" - return np.array(self._get_ray_bundle_group(timestamp_us)[name]) + return self._get_ray_bundle_group(timestamp_us).array(name).read() # per-ray return data - def _get_ray_bundle_returns_group(self, timestamp_us: int) -> zarr.Group: + def _get_ray_bundle_returns_group(self, timestamp_us: int) -> nodes.Group: """Returns the ray bundle returns group of a frame""" - return cast(zarr.Group, self._get_frame_group(timestamp_us)["ray_bundle_returns"]) + return self._group.group(self._frame_path(timestamp_us, "ray_bundle_returns")) def get_frame_ray_bundle_return_count(self, timestamp_us: int) -> int: """Returns the number of ray returns for a specific frame""" - return self._get_ray_bundle_returns_group(timestamp_us).attrs["n_returns"] + return self._get_ray_bundle_returns_group(timestamp_us).attr_int("n_returns") def get_frame_ray_bundle_return_data_names(self, timestamp_us: int) -> List[str]: """List of all ray bundle return data names for a frame""" - return list(self._get_ray_bundle_returns_group(timestamp_us).keys()) + return self._get_ray_bundle_returns_group(timestamp_us).members() def has_frame_ray_bundle_return_data(self, timestamp_us: int, name: str) -> bool: """Signals if named ray bundle return data exists for a frame""" @@ -1367,13 +1338,12 @@ def has_frame_ray_bundle_return_data(self, timestamp_us: int, name: str) -> bool def get_frame_ray_bundle_return_valid_mask(self, timestamp_us: int) -> npt.NDArray[np.bool_]: """Returns the per-ray return valid mask for a frame""" - valid_mask_packed = self._get_frame_group(timestamp_us)["ray_bundle_returns_valid_mask_packed"] + valid_mask_packed = self._group.array(self._frame_path(timestamp_us, "ray_bundle_returns_valid_mask_packed")) - attrs = valid_mask_packed.attrs - n_returns, n_rays = attrs["n_returns"], attrs["n_rays"] + n_returns, n_rays = valid_mask_packed.attr_int("n_returns"), valid_mask_packed.attr_int("n_rays") return ( - np.unpackbits(np.array(valid_mask_packed), count=n_returns * n_rays) + np.unpackbits(valid_mask_packed.read(), count=n_returns * n_rays) .astype(np.bool_) .reshape((n_returns, n_rays)) ) @@ -1383,14 +1353,13 @@ def get_frame_ray_bundle_return_data( ) -> npt.NDArray[np.float32]: """Returns named ray bundle return data for a frame, optionally indexed by return index to accelerate data-retrieval""" - return_array = self._get_ray_bundle_returns_group(timestamp_us)[ - name - ] # only references the underlying Array, don't load it's data yet + # only references the underlying array, don't load its data yet + return_array = self._get_ray_bundle_returns_group(timestamp_us).array(name) if return_index is None: - return np.array(return_array[slice(return_array.shape[0]), ...]) # load all returns + return return_array.read() # load all returns else: - return np.array(return_array[return_index, ...]) # load specific return only + return return_array.read((return_index, Ellipsis)) # load specific return only class CameraSensorComponent: @@ -1426,9 +1395,7 @@ def store_frame( frame_group = self._store_base_frame(frame_timestamps_us, generic_data, generic_meta_data) # Store image data (uncompressed, as already encoded) - frame_group.create_dataset("image", data=np.asarray(image_binary_data), compressor=None).attrs["format"] = ( - image_format - ) + frame_group.create_bytes_array("image", image_binary_data, attributes={"format": image_format}) return self @@ -1448,18 +1415,16 @@ def supports_component_version(version: str) -> bool: class EncodedImageDataHandle: """References encoded image data without loading it""" - def __init__(self, image_dataset: zarr.Array) -> None: - self._image_dataset = image_dataset + def __init__(self, image_array: nodes.Array) -> None: + self._image_array = image_array def get_data(self) -> types.EncodedImageData: """Loads the referenced encoded image data to memory""" - return types.EncodedImageData( - cast(np.bytes_, self._image_dataset[()]), self._image_dataset.attrs["format"] - ) + return types.EncodedImageData(self._image_array.read_bytes(), self._image_array.attr_str("format")) def get_frame_handle(self, timestamp_us: int) -> EncodedImageDataHandle: """Returns the frame's encoded image data""" - return self.EncodedImageDataHandle(cast(zarr.Array, self._get_frame_group(timestamp_us)["image"])) + return self.EncodedImageDataHandle(self._group.array(self._frame_path(timestamp_us, "image"))) def get_frame_data(self, timestamp_us: int) -> types.EncodedImageData: """Returns the frame's encoded image data""" @@ -1705,7 +1670,7 @@ def store_observations( ) obs_dict_list.append(obs.to_dict()) - self._group.create_group("cuboids").attrs.put({"cuboid_track_observations": obs_dict_list}) + self._group.create_group("cuboids", attributes={"cuboid_track_observations": obs_dict_list}) return self @@ -1725,8 +1690,8 @@ def supports_component_version(version: str) -> bool: def get_observations(self) -> Generator[types.CuboidTrackObservation]: """Returns all stored cuboid track observations""" - for obs in self._group["cuboids"].attrs["cuboid_track_observations"]: - yield types.CuboidTrackObservation.from_dict(obs) + for obs in self._group.group("cuboids").attr_list("cuboid_track_observations"): + yield types.CuboidTrackObservation.from_dict(cast(Dict[str, Any], obs)) class PointCloudsComponent: @@ -1770,7 +1735,7 @@ def get_component_version() -> str: def __init__( self, - component_group: zarr.Group, + component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval, coordinate_unit: PointCloud.CoordinateUnit, attribute_schemas: Optional[Dict[str, PointCloudsComponent.AttributeSchema]] = None, @@ -1784,7 +1749,7 @@ def __init__( # Pre-create the pcs group self._pcs_group = self._group.require_group("pcs") - self._pcs_group.attrs.put( + self._pcs_group.update_attrs( { "coordinate_unit": coordinate_unit.name, "attribute_schemas": {name: s.to_dict() for name, s in self._attribute_schemas.items()}, @@ -1811,7 +1776,6 @@ def store_pc( attributes = unpack_optional(attributes, default={}) generic_data = unpack_optional(generic_data, default={}) generic_meta_data = unpack_optional(generic_meta_data, default={}) - compressor = Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE) # -- Validate xyz -- assert xyz.dtype == np.dtype("float32") @@ -1842,7 +1806,7 @@ def store_pc( pc_group = self._pcs_group.require_group(str(len(self._pc_timestamps))) # Store per-pc metadata (timestamp lives in source-level pc_timestamps_us array) - pc_group.attrs.put( + pc_group.update_attrs( { "reference_frame_id": reference_frame_id, "generic_meta_data": generic_meta_data, @@ -1850,43 +1814,23 @@ def store_pc( ) # Store xyz - pc_group.create_dataset( - "xyz", - data=xyz, - chunks=_normalize_chunks(xyz.shape, require_nonzero_dims=(1,)), - compressor=compressor, - ) + _check_nonzero_dims(xyz.shape, (1,), "xyz") + pc_group.create_array("xyz", xyz) # Store schema-declared attributes for attr_name, attr_array in attributes.items(): - pc_group.create_dataset( - attr_name, - data=attr_array, - chunks=_normalize_chunks(attr_array.shape), - compressor=compressor, - ) + pc_group.create_array(attr_name, attr_array) # Store generic data gd_group = pc_group.require_group("generic_data") for gd_name, gd_array in generic_data.items(): - gd_group.create_dataset( - gd_name, - data=gd_array, - chunks=_normalize_chunks(gd_array.shape), - compressor=compressor, - ) + gd_group.create_array(gd_name, gd_array) self._pc_timestamps.append(reference_frame_timestamp_us) def finalize(self) -> None: """Write pc_timestamps_us array (derived from per-pc reference_frame_timestamp_us values).""" - ts_array = np.array(self._pc_timestamps, dtype=np.uint64) - self._group.create_dataset( - "pc_timestamps_us", - data=ts_array, - chunks=_normalize_chunks(ts_array.shape), - compressor=Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE), - ) + self._group.create_array("pc_timestamps_us", np.array(self._pc_timestamps, dtype=np.uint64)) # -------------------------------------------------------------------------- # Reader @@ -1903,16 +1847,16 @@ def get_component_name() -> str: def supports_component_version(version: str) -> bool: return version == "v1" - def __init__(self, component_instance_name: str, component_group: zarr.Group) -> None: + def __init__(self, component_instance_name: str, component_group: nodes.Group) -> None: super().__init__(component_instance_name, component_group) - self._pc_timestamps_us: npt.NDArray[np.uint64] = np.array(self._group["pc_timestamps_us"][:]) + self._pc_timestamps_us: npt.NDArray[np.uint64] = self._group.array("pc_timestamps_us").read() - pcs_attrs = self._group["pcs"].attrs - self._coordinate_unit = PointCloud.CoordinateUnit[pcs_attrs["coordinate_unit"]] + pcs_group = self._group.group("pcs") + self._coordinate_unit = PointCloud.CoordinateUnit[pcs_group.attr_str("coordinate_unit")] self._attribute_schemas: Dict[str, PointCloudsComponent.AttributeSchema] = { - name: PointCloudsComponent.AttributeSchema.from_dict(s) - for name, s in pcs_attrs["attribute_schemas"].items() + name: PointCloudsComponent.AttributeSchema.from_dict(cast(Dict[str, Any], s)) + for name, s in pcs_group.attr_dict("attribute_schemas").items() } # -- properties -------------------------------------------------------- @@ -1941,32 +1885,40 @@ def get_attribute_schema(self, name: str) -> PointCloudsComponent.AttributeSchem # -- per-pc data access ------------------------------------------------ - def _pc_group(self, pc_index: int) -> zarr.Group: - return cast(zarr.Group, self._group["pcs"][str(pc_index)]) + @staticmethod + def _pc_path(pc_index: int, name: Optional[str] = None) -> str: + """Returns the path of a point cloud group (or of one of its members) within the component group""" + return f"pcs/{pc_index}" if name is None else f"pcs/{pc_index}/{name}" + + def _pc_group(self, pc_index: int, name: Optional[str] = None) -> nodes.Group: + return self._group.group(self._pc_path(pc_index, name)) + + def _pc_array(self, pc_index: int, name: str) -> nodes.Array: + return self._group.array(self._pc_path(pc_index, name)) def get_pc_xyz(self, pc_index: int) -> npt.NDArray[np.float32]: - return np.array(self._pc_group(pc_index)["xyz"][:]) + return self._pc_array(pc_index, "xyz").read() def get_pc_attribute(self, pc_index: int, name: str) -> npt.NDArray[Any]: - return np.array(self._pc_group(pc_index)[name][:]) + return self._pc_array(pc_index, name).read() def get_pc_reference_frame_id(self, pc_index: int) -> str: - return str(self._pc_group(pc_index).attrs["reference_frame_id"]) + return self._pc_group(pc_index).attr_str("reference_frame_id") def get_pc_reference_frame_timestamp_us(self, pc_index: int) -> int: - return int(self._group["pc_timestamps_us"][pc_index]) + return int(self._group.array("pc_timestamps_us").read(pc_index)) def get_pc_generic_data_names(self, pc_index: int) -> List[str]: - return list(cast(zarr.Group, self._pc_group(pc_index)["generic_data"]).keys()) + return self._pc_group(pc_index, "generic_data").members() def has_pc_generic_data(self, pc_index: int, name: str) -> bool: - return name in self._pc_group(pc_index)["generic_data"] + return name in self._pc_group(pc_index, "generic_data") def get_pc_generic_data(self, pc_index: int, name: str) -> npt.NDArray[Any]: - return np.array(self._pc_group(pc_index)["generic_data"][name][:]) + return self._pc_array(pc_index, f"generic_data/{name}").read() def get_pc_generic_meta_data(self, pc_index: int) -> Dict[str, types.JsonLike]: - return dict(self._pc_group(pc_index).attrs.get("generic_meta_data", {})) + return cast(Dict[str, types.JsonLike], self._pc_group(pc_index).attrs.get("generic_meta_data", {})) class CameraLabelsComponent: @@ -1991,7 +1943,7 @@ def get_component_version() -> str: def __init__( self, - component_group: zarr.Group, + component_group: nodes.Group, sequence_timestamp_interval_us: HalfClosedInterval, descriptor: types.CameraLabelDescriptor, ) -> None: @@ -2001,7 +1953,7 @@ def __init__( # Initialize labels group and timestamps list self._labels_group = self._group.require_group("labels") - self._labels_group.attrs.put( + self._labels_group.update_attrs( { "descriptor": descriptor.to_dict(), } @@ -2029,7 +1981,6 @@ def store_label( Optional per-label metadata. """ generic_meta_data = unpack_optional(generic_meta_data, default={}) - compressor = Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE) # Sanity checks assert timestamp_us in self._sequence_timestamp_interval_us, ( @@ -2066,38 +2017,32 @@ def store_label( q.quantized_dtype ) - label_group.create_dataset( - "data", - data=stored, - chunks=_normalize_chunks(stored.shape, require_nonzero_dims=(0, 1)), - compressor=compressor, - ) + _check_nonzero_dims(stored.shape, (0, 1), "label data") + label_group.create_array("data", stored) elif self._descriptor.label_schema.encoding == types.LabelEncoding.IMAGE_ENCODED: assert isinstance(data, bytes), "IMAGE_ENCODED encoding requires bytes" - label_group.create_dataset( + # store encoded image data as uncompressed 1-d uint8 array (as already encoded), with chunks + # determined by zarr-python's chunk shape heuristic + label_group.create_array( "data", - data=np.asarray(bytearray(data), dtype=np.uint8), - compressor=None, - ).attrs["format"] = self._descriptor.label_schema.encoded_format + np.frombuffer(data, dtype=np.uint8), + chunks="auto", + compression=None, + attributes={"format": self._descriptor.label_schema.encoded_format}, + ) else: raise ValueError(f"Unsupported label encoding: {self._descriptor.label_schema.encoding}") - label_group.attrs["generic_meta_data"] = generic_meta_data + label_group.update_attrs({"generic_meta_data": generic_meta_data}) self._timestamps.append(timestamp_us) def finalize(self) -> None: """Write sorted timestamps_us array.""" - ts_array = np.array(sorted(self._timestamps), dtype=np.uint64) - self._group.create_dataset( - "timestamps_us", - data=ts_array, - chunks=_normalize_chunks(ts_array.shape), - compressor=Blosc(cname="lz4", clevel=5, shuffle=Blosc.BITSHUFFLE), - ) + self._group.create_array("timestamps_us", np.array(sorted(self._timestamps), dtype=np.uint64)) # -------------------------------------------------------------------------- # Reader @@ -2114,12 +2059,14 @@ def get_component_name() -> str: def supports_component_version(version: str) -> bool: return version == "v1" - def __init__(self, component_instance_name: str, component_group: zarr.Group) -> None: + def __init__(self, component_instance_name: str, component_group: nodes.Group) -> None: super().__init__(component_instance_name, component_group) - self._timestamps_us: npt.NDArray[np.uint64] = np.array(self._group["timestamps_us"][:]) + self._timestamps_us: npt.NDArray[np.uint64] = self._group.array("timestamps_us").read() self._timestamp_to_index: Dict[int, int] = {int(ts): i for i, ts in enumerate(self._timestamps_us)} - self._descriptor = types.CameraLabelDescriptor.from_dict(self._group["labels"].attrs["descriptor"]) + self._descriptor = types.CameraLabelDescriptor.from_dict( + self._group.group("labels").attr_dict("descriptor") + ) # -- properties -------------------------------------------------------- @@ -2137,11 +2084,11 @@ def timestamps_us(self) -> npt.NDArray[np.uint64]: # -- per-label access -------------------------------------------------- - def _label_group(self, timestamp_us: int) -> zarr.Group: + def _label_group(self, timestamp_us: int) -> nodes.Group: assert timestamp_us in self._timestamp_to_index, ( f"Unknown timestamp: {timestamp_us}. Available: {list(self._timestamp_to_index.keys())[:5]}..." ) - return cast(zarr.Group, self._group["labels"][str(timestamp_us)]) + return self._group.group(f"labels/{timestamp_us}") class CameraLabelHandle: """References label data without eagerly loading it. @@ -2152,7 +2099,7 @@ class CameraLabelHandle: def __init__( self, - label_group: zarr.Group, + label_group: nodes.Group, descriptor: types.CameraLabelDescriptor, timestamp_us: int, generic_meta_data: Dict[str, types.JsonLike], @@ -2181,7 +2128,7 @@ def get_data(self) -> npt.NDArray[Any]: For IMAGE_ENCODED encoding, decodes the image bytes via PIL. """ if self._descriptor.label_schema.encoding == types.LabelEncoding.RAW: - arr = np.array(self._label_group["data"][:]) + arr = self._label_group.array("data").read() # De-quantize if configured if (q := self._descriptor.label_schema.quantization) is not None: @@ -2192,7 +2139,7 @@ def get_data(self) -> npt.NDArray[Any]: return arr elif self._descriptor.label_schema.encoding == types.LabelEncoding.IMAGE_ENCODED: - raw_bytes = bytes(self._label_group["data"][:]) + raw_bytes = self._label_group.array("data").read_bytes() image = PILImage.open(io.BytesIO(raw_bytes)) return np.asarray(image, dtype=self._descriptor.label_schema.dtype) @@ -2203,12 +2150,12 @@ def get_data(self) -> npt.NDArray[Any]: def get_encoded_data(self) -> Optional[bytes]: """Return the raw encoded bytes for IMAGE_ENCODED labels, or None for RAW.""" if self._descriptor.label_schema.encoding == types.LabelEncoding.IMAGE_ENCODED: - return bytes(self._label_group["data"][:]) + return self._label_group.array("data").read_bytes() return None def get_label(self, timestamp_us: int) -> CameraLabelHandle: """Return a lazy handle to the label data at the given timestamp.""" label_group = self._label_group(timestamp_us) return self.CameraLabelHandle( - label_group, self._descriptor, timestamp_us, label_group.attrs["generic_meta_data"] + label_group, self._descriptor, timestamp_us, label_group.attr_dict("generic_meta_data") ) diff --git a/ncore/impl/data/v4/components_test.py b/ncore/impl/data/v4/components_test.py index 5ae85ba0..f04e4cc3 100644 --- a/ncore/impl/data/v4/components_test.py +++ b/ncore/impl/data/v4/components_test.py @@ -29,6 +29,7 @@ from ncore.impl.common.transformations import HalfClosedInterval from ncore.impl.common.util import unpack_optional +from ncore.impl.data.nodes import BloscCompression from ncore.impl.data.types import ( BBox3, BivariateWindshieldModelParameters, @@ -1413,21 +1414,23 @@ def store_velocity( return self def finalize(self): - """Write all velocity data to zarr storage""" + """Write all velocity data to the component group""" if self.velocities: velocities_array = np.stack(self.velocities) timestamps_array = np.array(self.timestamps, dtype=np.uint64) - # Store as zarr arrays - self._group.create_dataset( + # Store as arrays + self._group.create_array( "velocities", - data=velocities_array, + velocities_array, dtype=velocities_array.dtype, + compression=BloscCompression(shuffle="shuffle"), ) - self._group.create_dataset( + self._group.create_array( "timestamps_us", - data=timestamps_array, + timestamps_array, dtype=np.uint64, + compression=BloscCompression(shuffle="shuffle"), ) class Reader(ComponentReader): @@ -1444,8 +1447,8 @@ def supports_component_version(version: str) -> bool: def get_velocities(self) -> Tuple[np.ndarray, np.ndarray]: """Returns (velocities, timestamps_us) arrays""" - velocities = np.array(self._group["velocities"][:]) - timestamps_us = np.array(self._group["timestamps_us"][:]) + velocities = self._group.array("velocities").read() + timestamps_us = self._group.array("timestamps_us").read() return velocities, timestamps_us # ============================================================================ @@ -1605,9 +1608,10 @@ def finalize(self): accelerations_array = np.stack(self.accelerations) timestamps_array = np.array(self.timestamps, dtype=np.uint64) - self._group.create_dataset("velocities", data=velocities_array) - self._group.create_dataset("accelerations", data=accelerations_array) # NEW - self._group.create_dataset("timestamps_us", data=timestamps_array) + compression = BloscCompression(shuffle="shuffle") + self._group.create_array("velocities", velocities_array, compression=compression) + self._group.create_array("accelerations", accelerations_array, compression=compression) # NEW + self._group.create_array("timestamps_us", timestamps_array, compression=compression) # Create a backward-compatible reader that can read both v1 and v2 class VelocityComponentBackwardCompatibleReader(ComponentReader): @@ -1624,15 +1628,15 @@ def supports_component_version(version: str) -> bool: def get_velocities(self) -> Tuple[np.ndarray, np.ndarray]: """Returns velocities (works for both v1 and v2)""" - velocities = np.array(self._group["velocities"][:]) - timestamps_us = np.array(self._group["timestamps_us"][:]) + velocities = self._group.array("velocities").read() + timestamps_us = self._group.array("timestamps_us").read() return velocities, timestamps_us def get_accelerations(self) -> np.ndarray: """Returns accelerations (only available in v2)""" if self.component_version == "v1": raise ValueError("Acceleration data not available in v1") - return np.array(self._group["accelerations"][:]) + return self._group.array("accelerations").read() # Test that v1 reader cannot read v2 data v2_store_writer = SequenceComponentGroupsWriter( @@ -2749,7 +2753,7 @@ def test_schema_json_roundtrip(self) -> None: # ============================================================================== -# Tests for zero-dimension array support (_normalize_chunks) +# Tests for zero-dimension array support # ============================================================================== diff --git a/tools/data_converter/argoverse2/converter_test.py b/tools/data_converter/argoverse2/converter_test.py index c70bc25e..19cc1e31 100644 --- a/tools/data_converter/argoverse2/converter_test.py +++ b/tools/data_converter/argoverse2/converter_test.py @@ -284,7 +284,7 @@ def test_lidar_model_reconstructs_directions(self): ts = int(reader.frames_timestamps_us[0, 1]) direction = reader.get_frame_ray_bundle_data(ts, "direction") elem = reader.get_frame_ray_bundle_data(ts, "model_element") - distance = np.asarray(reader._get_ray_bundle_returns_group(ts)["distance_m"])[0] + distance = reader._get_ray_bundle_returns_group(ts).array("distance_m").read()[0] far = np.isfinite(distance) & (distance > 20.0) & (np.linalg.norm(direction, axis=1) > 0) self.assertGreater(int(far.sum()), 100, f"{unit}: too few far returns to validate") @@ -352,7 +352,7 @@ def test_cuboids_align_with_lidar(self): rig_poses, pose_ts = poses_reader.get_dynamic_pose("rig", "world") direction = lidar_reader.get_frame_ray_bundle_data(ts, "direction") - distance = np.asarray(lidar_reader._get_ray_bundle_returns_group(ts)["distance_m"])[0] + distance = lidar_reader._get_ray_bundle_returns_group(ts).array("distance_m").read()[0] point_ts = lidar_reader.get_frame_ray_bundle_data(ts, "timestamp_us").astype(np.int64) valid = np.isfinite(distance) & (distance > 0) pts_sensor = direction[valid] * distance[valid, None] diff --git a/tools/data_converter/nuscenes/converter_test.py b/tools/data_converter/nuscenes/converter_test.py index 6e5edb83..c13e8198 100644 --- a/tools/data_converter/nuscenes/converter_test.py +++ b/tools/data_converter/nuscenes/converter_test.py @@ -294,7 +294,7 @@ def test_lidar_model_reproduces_point_cloud(self): # Native: direction * distance direction = lidar_reader.get_frame_ray_bundle_data(ts, "direction") - distance_2d = np.array(lidar_reader._get_ray_bundle_returns_group(ts)["distance_m"]) # [R, N] + distance_2d = lidar_reader._get_ray_bundle_returns_group(ts).array("distance_m").read() # [R, N] distance = distance_2d[0] # first return # Model element From dcfb70a555069583f35049e5ca298d6b6b219578 Mon Sep 17 00:00:00 2001 From: Janick Martinez Esturo Date: Sun, 11 Oct 2026 14:04:00 +0200 Subject: [PATCH 3/7] perf(data): node cache for component readers Opening a group / array constructs its zarr-python object, which decodes the node's metadata on every access (~0.5 ms per frame lookup with zarr-python 2, dominating reads of small frame arrays). Component readers now cache the nodes they open (with their decoded metadata, but not their data) in a thread-safe nodes.NodeCache shared by all member groups of the component. The cache size is configurable via SequenceComponentGroupsReader(node_cache_size=...): None (default) caches all accessed nodes, N the N most recently used nodes, and 0 disables caching. With the cache, repeated frame lookups take ~2 us instead of ~470 us. Component stores are opened concurrently (hiding the latency of remote archive index reads), their hierarchies are now opened from their consolidated metadata sequentially on the calling thread: opening hierarchies is pure-Python CPU work, which concurrent threads only slow down. --- docs/data/storage_and_access.rst | 8 ++ ncore/impl/data/nodes.py | 85 +++++++++++++++++-- ncore/impl/data/stores_test.py | 65 ++++++++++++++- ncore/impl/data/v4/components.py | 102 +++++++++++++---------- ncore/impl/data/v4/components_test.py | 113 ++++++++++++++++++++++++-- 5 files changed, 314 insertions(+), 59 deletions(-) diff --git a/docs/data/storage_and_access.rst b/docs/data/storage_and_access.rst index 38d9911d..202799ca 100644 --- a/docs/data/storage_and_access.rst +++ b/docs/data/storage_and_access.rst @@ -361,6 +361,14 @@ Performance Recommendations This is especially important for remote stores where each metadata lookup would otherwise be a separate round-trip. +* **Keep the node cache enabled** (the default). Component readers cache the + groups / arrays they open (including their decoded metadata, but not their + data), so repeated accesses (e.g., of frames) skip re-opening them. The + ``node_cache_size`` parameter on + :class:`~ncore.data.v4.SequenceComponentGroupsReader` bounds the number of + cached nodes per component reader for very long sequences (``None`` caches + all accessed nodes, ``0`` disables caching). + * **Tune the block size and cache type** for your workload. The ``default_block_size`` and ``default_cache_type`` parameters on ``UPath`` control how much data is fetched per HTTP request and how it is cached diff --git a/ncore/impl/data/nodes.py b/ncore/impl/data/nodes.py index a7b47db7..9af7f246 100644 --- a/ncore/impl/data/nodes.py +++ b/ncore/impl/data/nodes.py @@ -22,6 +22,9 @@ from __future__ import annotations +import collections +import threading + from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Literal, Optional, Tuple, Type, TypeVar, Union, cast import numpy as np @@ -46,6 +49,53 @@ T = TypeVar("T") +class NodeCache: + """Cache of the opened nodes of a hierarchy opened for reading, by node path. + + Opening a node constructs its zarr-python object (decoding its metadata), which dominates repeated reads of small + arrays (e.g., of frames). Cached nodes reference their metadata only, never array data. ``max_entries=None`` caches + all opened nodes, ``max_entries=N`` the N most recently used nodes (evicted nodes are re-opened on access). + + All methods are thread-safe. + """ + + def __init__(self, max_entries: Optional[int] = None) -> None: + if max_entries is not None and max_entries < 1: + raise ValueError(f"max_entries needs to be positive or None (unbounded), got {max_entries}") + self._max_entries = max_entries + self._nodes: collections.OrderedDict[str, Union[ZarrGroup, ZarrArray]] = collections.OrderedDict() + self._lock = threading.Lock() + + @property + def max_entries(self) -> Optional[int]: + return self._max_entries + + def __len__(self) -> int: + return len(self._nodes) + + def keys(self) -> List[str]: + """The paths of the cached nodes, from least to most recently used""" + with self._lock: + return list(self._nodes) + + def _open(self, group: ZarrGroup, path: str) -> Union[ZarrGroup, ZarrArray]: + key = _join(group.path, path) + with self._lock: + if (node := self._nodes.get(key)) is not None: + self._nodes.move_to_end(key) + return node + + # (concurrent first accesses open nodes redundantly, which is cheaper than serializing all accesses) + node = _backend.open_member(group, path) + + with self._lock: + self._nodes[key] = node + if self._max_entries is not None: + while len(self._nodes) > self._max_entries: + self._nodes.popitem(last=False) + return node + + class _Node: """Common functionality of groups and arrays""" @@ -131,33 +181,51 @@ def read_bytes(self) -> bytes: class Group(_Node): - """A group of an NCore data store""" + """A group of an NCore data store. + + Members of groups using a :class:`NodeCache` are opened via the cache (also members of member groups). + """ - def __init__(self, group: ZarrGroup) -> None: + def __init__(self, group: ZarrGroup, cache: Optional[NodeCache] = None) -> None: super().__init__(group) self._group = group + self._cache = cache @property def zarr(self) -> ZarrGroup: """The wrapped zarr-python group""" return self._group + @property + def cache(self) -> Optional[NodeCache]: + """The node cache used to open members (None if not cached)""" + return self._cache + + def with_cache(self, cache: Optional[NodeCache]) -> Group: + """Returns the group, using the given node cache to open its members""" + return Group(self._group, cache) + # -- reading --------------------------------------------------------------- + def _open(self, path: str) -> Union[ZarrGroup, ZarrArray]: + if self._cache is not None: + return self._cache._open(self._group, path) + return _backend.open_member(self._group, path) + def node(self, path: str) -> Union[Group, Array]: """Returns a (nested) member group or array (raises KeyError if it doesn't exist)""" - node = _backend.open_member(self._group, path) - return Group(node) if isinstance(node, ZarrGroup) else Array(node) + node = self._open(path) + return Group(node, self._cache) if isinstance(node, ZarrGroup) else Array(node) def group(self, path: str) -> Group: """Returns a (nested) member group (raises KeyError if it doesn't exist or isn't a group)""" - if not isinstance(node := _backend.open_member(self._group, path), ZarrGroup): + if not isinstance(node := self._open(path), ZarrGroup): raise KeyError(f"{_join(self.path, path)} is not a group") - return Group(node) + return Group(node, self._cache) def array(self, path: str) -> Array: """Returns a (nested) member array (raises KeyError if it doesn't exist or isn't an array)""" - if not isinstance(node := _backend.open_member(self._group, path), ZarrArray): + if not isinstance(node := self._open(path), ZarrArray): raise KeyError(f"{_join(self.path, path)} is not an array") return Array(node) @@ -172,7 +240,7 @@ def __contains__(self, name: str) -> bool: def groups(self) -> Iterator[Tuple[str, Group]]: """Iterates over the direct member groups (ordered by name)""" for name, group in sorted(_backend.member_groups(self._group), key=lambda member: member[0]): - yield name, Group(group) + yield name, Group(group, self._cache) # -- writing --------------------------------------------------------------- @@ -238,5 +306,6 @@ def _join(parent: str, path: str) -> str: "Array", "BloscCompression", "Group", + "NodeCache", "SUPPORTED_ZARR_FORMATS", ] diff --git a/ncore/impl/data/stores_test.py b/ncore/impl/data/stores_test.py index 090e39d7..44dd7b5e 100644 --- a/ncore/impl/data/stores_test.py +++ b/ncore/impl/data/stores_test.py @@ -22,7 +22,7 @@ import unittest from pathlib import Path -from typing import Any, Optional +from typing import Any, Optional, cast from unittest import mock import numpy as np @@ -394,6 +394,69 @@ def test_nodes(self): self.root.node("g/missing") +class TestNodeCache(unittest.TestCase): + """Node caches of groups (disabled / unbounded / least recently used)""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + path = os.path.join(self._tmp.name, "store.itar") + store = IndexedTarStore(path, mode="w") + sub = create_root_group(store, {}).create_group("g") + for i in range(8): + sub.create_array(f"a{i}", np.full(3, i)) + consolidate_store(store) + close_store(store) + self.root = open_store(IndexedTarStore(path)) + + def tearDown(self): + self._tmp.cleanup() + + def test_uncached(self): + self.assertIsNone(self.root.cache) + self.assertIsNot(self.root.array("g/a1").zarr, self.root.array("g/a1").zarr) + + def test_unbounded(self): + root = self.root.with_cache(cache := nodes.NodeCache()) + for i in range(8): + np.testing.assert_array_equal(root.array(f"g/a{i}").read(), [i] * 3) + self.assertEqual(len(cache), 8) + self.assertIs(root.array("g/a3").zarr, root.array("g/a3").zarr) + + def test_member_groups_share_the_cache(self): + root = self.root.with_cache(cache := nodes.NodeCache()) + group = root.group("g") + self.assertIs(group.cache, cache) + node = root.node("g") + assert isinstance(node, nodes.Group) + self.assertIs(node.cache, cache) + self.assertIs(cast(nodes.Array, root.node("g/a2")).zarr, root.array("g/a2").zarr) + self.assertIs(group.array("a3").zarr, root.array("g/a3").zarr) + self.assertIs(next(root.groups())[1].cache, cache) + self.assertEqual(cache.keys(), ["g", "g/a2", "g/a3"]) + + def test_least_recently_used(self): + root = self.root.with_cache(cache := nodes.NodeCache(max_entries=3)) + a0 = root.array("g/a0").zarr + root.array("g/a1") + root.array("g/a2") + self.assertIs(root.array("g/a0").zarr, a0) # (a1 is the least recently used node now) + root.array("g/a3") + self.assertEqual(cache.keys(), ["g/a2", "g/a0", "g/a3"]) + for i in range(8): # evicted nodes are re-opened + np.testing.assert_array_equal(root.array(f"g/a{i}").read(), [i] * 3) + self.assertEqual(len(cache), 3) + + def test_missing_nodes_are_not_cached(self): + root = self.root.with_cache(cache := nodes.NodeCache()) + with self.assertRaises(KeyError): + root.array("g/missing") + self.assertEqual(len(cache), 0) + + def test_invalid_sizes(self): + with self.assertRaises(ValueError): + nodes.NodeCache(max_entries=0) + + class TestBloscCompression(unittest.TestCase): @parameterized.parameterized.expand( [({"cname": "snappy"},), ({"clevel": 10},), ({"clevel": -1},), ({"shuffle": "byte"},), ({"blocksize": -1},)] diff --git a/ncore/impl/data/v4/components.py b/ncore/impl/data/v4/components.py index 30be6037..833c841c 100644 --- a/ncore/impl/data/v4/components.py +++ b/ncore/impl/data/v4/components.py @@ -374,6 +374,7 @@ def __init__( open_consolidated: bool = True, itar_index_tail_read_size: int = 1 << 20, # 1 MiB default max_threads: int | None = None, + node_cache_size: int | None = None, ): """Initialize a SequenceComponentReader for a virtual sequence represented by a list of components. @@ -389,10 +390,18 @@ def __init__( single I/O call to initialize the tar file index to minimize reads. Default 1 MiB size fits typical use cases, but may be increased if the compressed index size is expected to be larger or decreased if remote access chunk sizes are smaller. - max_threads: The maximum number of threads used to load the different components (if None, - use interpreter-default number of threads for a ThreadPoolExecutor) + max_threads: The maximum number of threads used to open the component stores concurrently (if + None, use interpreter-default number of threads for a ThreadPoolExecutor) + node_cache_size: Number of opened groups / arrays (with their decoded meta-data, but not their data) + each component reader caches for repeated accesses (e.g., of frames). None caches + all accessed nodes for the lifetime of the reader, a positive number the most recently + used nodes only, and 0 disables caching. """ + if node_cache_size is not None and node_cache_size < 0: + raise ValueError(f"node_cache_size needs to be None or non-negative, got {node_cache_size}") + self._node_cache_size = node_cache_size + component_group_upaths: List[UPath] = self.expand_component_group_paths(component_group_paths) assert len(component_group_upaths), "No component inputs provided" @@ -400,10 +409,13 @@ def __init__( # Load component stores concurrently (to hide latency) and check for sequence consistency self._component_stores: Dict[str, Tuple[nodes.Group, UPath]] = {} # use str as the generic path / URL type + # Component stores are opened concurrently (hiding the latency of remote archive index reads), their + # hierarchies are opened from their consolidated metadata sequentially (pure-Python CPU work that threads + # only slow down) with concurrent.futures.ThreadPoolExecutor(max_workers=max_threads) as executor: - def thread_load_component_store(component_store_upath: UPath) -> Tuple[nodes.Group, UPath]: - """Thread-executed shard opening""" + def thread_open_component_store(component_store_upath: UPath) -> Tuple[stores.StoreLike, UPath]: + """Thread-executed component store opening""" # Make sure paths are absolute at this point component_store_upath = component_store_upath.absolute() @@ -424,52 +436,48 @@ def thread_load_component_store(component_store_upath: UPath) -> Tuple[nodes.Gro else: component_store = stores.open_directory_store(component_store_upath) - return stores.open_store(component_store, open_consolidated), component_store_upath - - for future in concurrent.futures.as_completed( - [ - executor.submit(thread_load_component_store, component_store_upath) - for component_store_upath in component_group_upaths - ] - ): - # Note: thread completion order is not relevant here - component_root, component_store_path = future.result() - - # Check sequence compatibility - component_root_version = component_root.attr_str("version") - if component_root_version not in [VERSION]: - raise RuntimeError( - f"Can't load V4 component store {component_store_path} with incompatible data version {component_root_version}" - ) + return component_store, component_store_upath + + opened_component_stores = list(executor.map(thread_open_component_store, component_group_upaths)) + + for component_store, component_store_path in opened_component_stores: + component_root = stores.open_store(component_store, open_consolidated) - component_store_sequence_id = component_root.attr_str("sequence_id") - sequence_timestamp_interval_us = component_root.attr_dict("sequence_timestamp_interval_us") - component_store_sequence_timestamp_interval_us = HalfClosedInterval( - cast(int, sequence_timestamp_interval_us["start"]), - cast(int, sequence_timestamp_interval_us["stop"]), + # Check sequence compatibility + component_root_version = component_root.attr_str("version") + if component_root_version not in [VERSION]: + raise RuntimeError( + f"Can't load V4 component store {component_store_path} with incompatible data version {component_root_version}" ) - component_store_generic_meta_data = component_root.attr_dict("generic_meta_data") - if not self._component_stores: - self._sequence_id: str = component_store_sequence_id - self._sequence_timestamp_interval_us = component_store_sequence_timestamp_interval_us - self._generic_meta_data: Dict[str, types.JsonLike] = component_store_generic_meta_data - self._version: str = component_root_version + component_store_sequence_id = component_root.attr_str("sequence_id") + sequence_timestamp_interval_us = component_root.attr_dict("sequence_timestamp_interval_us") + component_store_sequence_timestamp_interval_us = HalfClosedInterval( + cast(int, sequence_timestamp_interval_us["start"]), + cast(int, sequence_timestamp_interval_us["stop"]), + ) + component_store_generic_meta_data = component_root.attr_dict("generic_meta_data") + + if not self._component_stores: + self._sequence_id: str = component_store_sequence_id + self._sequence_timestamp_interval_us = component_store_sequence_timestamp_interval_us + self._generic_meta_data: Dict[str, types.JsonLike] = component_store_generic_meta_data + self._version: str = component_root_version - if not self._sequence_id == component_store_sequence_id: - raise RuntimeError("Can't load component store from different sequences") - if not self._sequence_timestamp_interval_us == component_store_sequence_timestamp_interval_us: - raise RuntimeError("Can't load component store with different sequence timestamp intervals") - if not self._generic_meta_data == component_store_generic_meta_data: - raise RuntimeError("Can't load component store with different generic meta-data") - if not self._version == component_root_version: - raise RuntimeError("Can't load shards from different data versions") + if not self._sequence_id == component_store_sequence_id: + raise RuntimeError("Can't load component store from different sequences") + if not self._sequence_timestamp_interval_us == component_store_sequence_timestamp_interval_us: + raise RuntimeError("Can't load component store with different sequence timestamp intervals") + if not self._generic_meta_data == component_store_generic_meta_data: + raise RuntimeError("Can't load component store with different generic meta-data") + if not self._version == component_root_version: + raise RuntimeError("Can't load shards from different data versions") - component_group_name = component_root.attr_str("component_group_name") - if component_group_name in self._component_stores: - raise RuntimeError(f"Component group {component_group_name} loaded multiple times") + component_group_name = component_root.attr_str("component_group_name") + if component_group_name in self._component_stores: + raise RuntimeError(f"Component group {component_group_name} loaded multiple times") - self._component_stores[component_group_name] = (component_root, component_store_path) + self._component_stores[component_group_name] = (component_root, component_store_path) # Check version-compatibility if self._version != VERSION: @@ -519,7 +527,11 @@ def open_component_readers( if not component_reader_type.supports_component_version(component_group.attr_str("component_version")): continue - ret[component_instance_name] = component_reader_type(component_instance_name, component_group) + # each reader caches the nodes of its component + cache = None if self._node_cache_size == 0 else nodes.NodeCache(self._node_cache_size) + ret[component_instance_name] = component_reader_type( + component_instance_name, component_group.with_cache(cache) + ) return ret diff --git a/ncore/impl/data/v4/components_test.py b/ncore/impl/data/v4/components_test.py index f04e4cc3..f9f3451a 100644 --- a/ncore/impl/data/v4/components_test.py +++ b/ncore/impl/data/v4/components_test.py @@ -16,9 +16,11 @@ import dataclasses import io import tempfile +import threading import unittest -from typing import Dict, Literal, Tuple, cast +from typing import Any, Dict, Literal, Optional, Tuple, cast +from unittest import mock import numpy as np import PIL.Image as PILImage @@ -29,6 +31,7 @@ from ncore.impl.common.transformations import HalfClosedInterval from ncore.impl.common.util import unpack_optional +from ncore.impl.data import stores from ncore.impl.data.nodes import BloscCompression from ncore.impl.data.types import ( BBox3, @@ -84,11 +87,13 @@ def setUp(self): @parameterized.expand( [ - (False,), - (True,), + (False, None), + (True, None), + (True, 0), # uncached nodes + (True, 1), # nodes evicted on most accesses ] ) - def test_reload(self, open_consolidated: bool): + def test_reload(self, open_consolidated: bool, node_cache_size: Optional[int]): """Test to make sure serialized data is faithfully reloaded""" tempdir = tempfile.TemporaryDirectory() @@ -574,7 +579,9 @@ def normalize_points(vectors: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: store_paths += store_writer.finalize() ## Reload sequence and verify consistency - store_reader = SequenceComponentGroupsReader(store_paths, open_consolidated=open_consolidated) + store_reader = SequenceComponentGroupsReader( + store_paths, open_consolidated=open_consolidated, node_cache_size=node_cache_size + ) # check sequence data self.assertEqual(store_reader.sequence_id, ref_sequence_id) @@ -2752,6 +2759,102 @@ def test_schema_json_roundtrip(self) -> None: self.assertIsNone(rt.quantization) +class TestReaderOpening(unittest.TestCase): + """Component stores are opened concurrently, their hierarchies sequentially on the calling thread""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + writer = SequenceComponentGroupsWriter( + output_dir_path=UPath(self._tmp.name), + store_base_name="seq", + sequence_id="seq", + sequence_timestamp_interval_us=HalfClosedInterval(0, 10_000_001), + generic_meta_data={}, + ) + for name in ("a", "b", "c"): + writer.register_component_writer(PosesComponent.Writer, name, f"poses_{name}").store_static_pose( + "x", "y", np.eye(4, dtype=np.float32) + ) + self.store_paths = writer.finalize() + + def tearDown(self): + self._tmp.cleanup() + + def test_hierarchies_are_opened_on_the_calling_thread(self): + threads = [] + open_store = stores.open_store + + def recording_open_store(*args: Any, **kwargs: Any) -> Any: + threads.append(threading.current_thread()) + return open_store(*args, **kwargs) + + with mock.patch.object(stores, "open_store", side_effect=recording_open_store): + reader = SequenceComponentGroupsReader(self.store_paths, max_threads=4) + self.assertEqual(len(threads), len(self.store_paths)) + self.assertEqual(set(threads), {threading.current_thread()}) + self.assertEqual(sorted(reader.open_component_readers(PosesComponent.Reader)), ["a", "b", "c"]) + + +class TestReaderNodeCaches(unittest.TestCase): + """Component readers cache the opened nodes of their component (sized by node_cache_size)""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + writer = SequenceComponentGroupsWriter( + output_dir_path=UPath(self._tmp.name), + store_base_name="seq", + sequence_id="seq", + sequence_timestamp_interval_us=HalfClosedInterval(0, 10_000_001), + generic_meta_data={}, + ) + lidar_writer = writer.register_component_writer(LidarSensorComponent.Writer, "lidar", "lidar") + for i in range(4): + direction = np.tile(np.array([1.0, 0.0, 0.0], dtype=np.float32), (5, 1)) + lidar_writer.store_frame( + direction=direction, + timestamp_us=np.full(5, 10 + i * 1_000_000, dtype=np.uint64), + model_element=None, + distance_m=np.full((1, 5), i, dtype=np.float32), + intensity=np.ones((1, 5), dtype=np.float32), + frame_timestamps_us=np.array([10 + i * 1_000_000, 20 + i * 1_000_000], dtype=np.uint64), + generic_data={}, + generic_meta_data={}, + ) + self.store_paths = writer.finalize() + + def tearDown(self): + self._tmp.cleanup() + + @parameterized.expand([(None,), (0,), (1,), (3,)]) + def test_node_cache_sizes(self, node_cache_size: Optional[int]): + reader = SequenceComponentGroupsReader(self.store_paths, node_cache_size=node_cache_size) + lidar = reader.open_component_readers(LidarSensorComponent.Reader)["lidar"] + frames = [int(ts) for ts in lidar.frames_timestamps_us[:, 1]] + for _ in range(2): + for i, ts in enumerate(frames): + np.testing.assert_array_equal(lidar.get_frame_ray_bundle_return_data(ts, "distance_m", 0), [i] * 5) + + cache = lidar._group.cache + if node_cache_size == 0: + self.assertIsNone(cache) + return + assert cache is not None + self.assertEqual(cache.max_entries, node_cache_size) + # every frame read opens the frame's returns group and its distance_m array + accessed = ["lidars/lidar/frames"] + [ + f"lidars/lidar/frames/{ts}/ray_bundle_returns{suffix}" for ts in frames for suffix in ("", "/distance_m") + ] + if node_cache_size is None: + self.assertEqual(sorted(cache.keys()), sorted(accessed)) + else: + # the most recently used nodes only + self.assertEqual(cache.keys(), accessed[-node_cache_size:]) + + def test_invalid_node_cache_size(self): + with self.assertRaises(ValueError): + SequenceComponentGroupsReader(self.store_paths, node_cache_size=-1) + + # ============================================================================== # Tests for zero-dimension array support # ============================================================================== From 5d93e0d79fb36b1bef137cc5dbbc1c3d91b92c2a Mon Sep 17 00:00:00 2001 From: Janick Martinez Esturo Date: Sun, 11 Oct 2026 15:13:25 +0200 Subject: [PATCH 4/7] feat(data)!: support zarr-python 3 Support zarr-python 3 (>= 3.1.4) next to zarr-python 2 for zarr format v2 data. The zarr-python version specific operations of nodes and stores are implemented by a zarr-python 3 backend (_zarr3.py), which is selected at import time: - zarr-python's coroutines run on an event loop owned by the calling thread (offloaded codec work runs inline), avoiding thread hand-offs to zarr-python's I/O thread per access. Forked processes create new loops, coroutines within running event loops run on zarr-python's I/O loop - a zarr-python 3 store of indexed tar archives - consolidated hierarchies are opened from the decoded compressed consolidated metadata (no node metadata reads), with node metadata ordered by parent group (zarr-python < 3.4 attaches members of a group to its first run of keys only, zarr-python#4226) Data written with zarr-python 2 and 3 is equivalent: chunk records are byte-identical, node metadata documents are equivalent JSON. zarr-python 3 rewrites node metadata on every update of a node, which the write-once records of indexed tar archives don't support. Node metadata records of archives opened for writing are now kept in memory and appended when closing the archive (parents first), independent of the zarr-python version. Test environments: python 3.11 uses zarr-python 3.1.4 (the oldest supported version), python 3.13 zarr-python 3.4, python 3.8 / 3.12 zarr-python 2. BREAKING CHANGE: node metadata records of indexed tar archives are appended when closing archives (after all chunk records), so archives are no longer byte-identical to archives written by ncore 19.8 (their records are). --- MODULE.bazel.lock | 1862 ++++++++++++++++++++---------- deps/pip/requirements_3_11.in | 1 + deps/pip/requirements_3_11.txt | 131 ++- deps/pip/requirements_3_12.in | 2 +- deps/pip/requirements_3_13.in | 4 +- deps/pip/requirements_3_13.txt | 353 +++--- deps/pip/requirements_ncore.in | 7 +- docs/data/storage_and_access.rst | 36 + ncore/BUILD.bazel | 5 +- ncore/impl/data/BUILD.bazel | 1 + ncore/impl/data/_zarr3.py | 406 +++++++ ncore/impl/data/itar.py | 47 +- ncore/impl/data/nodes.py | 18 +- ncore/impl/data/stores_test.py | 264 ++++- 14 files changed, 2359 insertions(+), 778 deletions(-) create mode 100644 ncore/impl/data/_zarr3.py diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 7d07a1da..63996be3 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1502,9 +1502,9 @@ "bzlTransitiveDigest": "zc0QWNGxYYxPvHvT1CA/9+X0h1JuhXagw7k7q1jOhwg=", "usagesDigest": "rf/YOtMUZwRHGFAE+9knWdjNz4HOysfZIlX5MJbhT+E=", "recordedFileInputs": { - "@@//deps/pip/requirements_3_11.txt": "845ddefb3a79c099a05f39880752241d3c30cefb3774c0611f253d5bff3f774f", + "@@//deps/pip/requirements_3_11.txt": "a13d4223fd7b56af9192342a5af35245f7678050a5d3e33fd5ea686da8e79680", "@@//deps/pip/requirements_3_12.txt": "56ddea672a89523c8b93e54bcbcffc28c91aeff4f0e72c8b95306dc287af0068", - "@@//deps/pip/requirements_3_13.txt": "48cd877f84a1ac6277b6cd9d8f5c211f2f2b49e058dc1f1c0717e3b1bfbfa689", + "@@//deps/pip/requirements_3_13.txt": "81e3dc5cc6fd0ad9c8a3245c6645328adf2bbc40040f84f10b3cb9b43278e9fb", "@@//deps/pip/requirements_3_8.txt": "94c254d645dc1d0555f48e66fa499aa2efecfc5e547500cd4ed409aee28e8880", "@@rules_fuzzing+//fuzzing/requirements.txt": "ab04664be026b632a0d2a2446c4f65982b7654f5b6851d2f9d399a19b7242a5b", "@@rules_python+//tools/publish/requirements_darwin.txt": "095d4a4f3d639dce831cd493367631cd51b53665292ab20194bac2c0c6458fa8", @@ -1733,36 +1733,6 @@ ] } }, - "ncore_pip_deps_311_asciitree_sdist_4aa4b9b6": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cu128/torch" - ], - "filename": "asciitree-0.3.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "ncore_pip_deps_311", - "requirement": "asciitree==0.3.3", - "sha256": "4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e", - "urls": [ - "https://files.pythonhosted.org/packages/2d/6a/885bc91484e1aa8f618f6f0228d76d0e67000b0fdd6090673b777e311913/asciitree-0.3.3.tar.gz" - ] - } - }, "ncore_pip_deps_311_asttokens_py3_none_any_15a3ebc0": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -3545,6 +3515,60 @@ ] } }, + "ncore_pip_deps_311_donfig_py3_none_any_2a3175ce": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "donfig-0.8.1.post1-py3-none-any.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "donfig==0.8.1.post1", + "sha256": "2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", + "urls": [ + "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl" + ] + } + }, + "ncore_pip_deps_311_donfig_sdist_3bef3413": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cu128/torch" + ], + "filename": "donfig-0.8.1.post1.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "donfig==0.8.1.post1", + "sha256": "3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", + "urls": [ + "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz" + ] + } + }, "ncore_pip_deps_311_dracopy_cp311_cp311_macosx_10_9_x86_64_c4131151": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -3791,60 +3815,6 @@ ] } }, - "ncore_pip_deps_311_fasteners_py3_none_any_758819cb": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "filename": "fasteners-0.19-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "ncore_pip_deps_311", - "requirement": "fasteners==0.19", - "sha256": "758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237", - "urls": [ - "https://files.pythonhosted.org/packages/61/bf/fd60001b3abc5222d8eaa4a204cd8c0ae78e75adc688f33ce4bf25b7fafa/fasteners-0.19-py3-none-any.whl" - ] - } - }, - "ncore_pip_deps_311_fasteners_sdist_b4f37c3a": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp311_linux_aarch64", - "cp311_linux_arm", - "cp311_linux_ppc", - "cp311_linux_s390x", - "cp311_linux_x86_64", - "cp311_osx_aarch64", - "cp311_osx_x86_64", - "cp311_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cu128/torch" - ], - "filename": "fasteners-0.19.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", - "repo": "ncore_pip_deps_311", - "requirement": "fasteners==0.19", - "sha256": "b4f37c3ac52d8a445af3a66bce57b33b5e90b97c696b7b984f530cf8f0ded09c", - "urls": [ - "https://files.pythonhosted.org/packages/5f/d4/e834d929be54bfadb1f3e3b931c38e956aaa3b235a46a3c764c26c774902/fasteners-0.19.tar.gz" - ] - } - }, "ncore_pip_deps_311_fastjsonschema_py3_none_any_c9e5b7e9": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -4361,6 +4331,156 @@ ] } }, + "ncore_pip_deps_311_google_crc32c_cp311_cp311_macosx_12_0_arm64_457d0d9a": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp311-cp311-macosx_12_0_arm64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "google-crc32c==1.9.0", + "sha256": "457d0d9a4718fd52b1494eac5c200ad25beeadbdc91843d550a003910838589f", + "urls": [ + "https://files.pythonhosted.org/packages/0e/55/a2f07f15e624f0de79359b1a6c1deb59ec5061bd3b38744b3b2849400662/google_crc32c-1.9.0-cp311-cp311-macosx_12_0_arm64.whl" + ] + } + }, + "ncore_pip_deps_311_google_crc32c_cp311_cp311_macosx_12_0_x86_64_ccfe4002": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp311-cp311-macosx_12_0_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "google-crc32c==1.9.0", + "sha256": "ccfe40021fd6afe23361175cf7551e3cef5fd34dc1ebe319f14993a83579e0eb", + "urls": [ + "https://files.pythonhosted.org/packages/f8/b3/923743597b774bbcf12a7c3e00e48d745e15fd616ad7489a40a63fff8f2f/google_crc32c-1.9.0-cp311-cp311-macosx_12_0_x86_64.whl" + ] + } + }, + "ncore_pip_deps_311_google_crc32c_cp311_cp311_manylinux_2_17_aarch64_fbef61a3": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "google-crc32c==1.9.0", + "sha256": "fbef61a3794e011c65fb4396a196cf123a7f474fe5a443db8e5dd7d751b9e6d4", + "urls": [ + "https://files.pythonhosted.org/packages/df/a6/4d0352fe889663e0d81cea7fc664ec9158727384de4a44ab10e9967a7682/google_crc32c-1.9.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl" + ] + } + }, + "ncore_pip_deps_311_google_crc32c_cp311_cp311_manylinux_2_17_x86_64_86764b99": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "google-crc32c==1.9.0", + "sha256": "86764b99e7a607830d93cb5b75e0ec3ff6cb06d3c274624418473cee701900d4", + "urls": [ + "https://files.pythonhosted.org/packages/aa/e3/26685384e4b66ff0928d9566ef6110a7df76029175a1842329d7e3515f10/google_crc32c-1.9.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" + ] + } + }, + "ncore_pip_deps_311_google_crc32c_cp311_cp311_win_amd64_43a2dc26": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp311-cp311-win_amd64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "google-crc32c==1.9.0", + "sha256": "43a2dc26f9be213fbe0b4fc4a1088c5d45cbfcb3247420ccc820f0fc3edeea86", + "urls": [ + "https://files.pythonhosted.org/packages/cb/ce/4e90102e84880e97d3cf935f2672ecd29191bdeacf57f01740f92debda00/google_crc32c-1.9.0-cp311-cp311-win_amd64.whl" + ] + } + }, + "ncore_pip_deps_311_google_crc32c_sdist_7b8c84c3": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cu128/torch" + ], + "filename": "google_crc32c-1.9.0.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "google-crc32c==1.9.0", + "sha256": "7b8c84c3d159ab6817fe3f74e6e6cef099c3f95dcec3abc0d8afb1404642efbe", + "urls": [ + "https://files.pythonhosted.org/packages/fa/25/9cb0c1c31c45b893eb8f11ae70b3f4309432d59b5acaebca5dbe791729a4/google_crc32c-1.9.0.tar.gz" + ] + } + }, "ncore_pip_deps_311_google_pasta_py3_none_any_b3248279": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -12269,7 +12389,7 @@ ] } }, - "ncore_pip_deps_311_pyzmq_cp311_cp311_macosx_10_15_universal2_c0598838": { + "ncore_pip_deps_311_pyyaml_cp311_cp311_macosx_10_13_x86_64_44edc647": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -12283,17 +12403,17 @@ "cp311_osx_x86_64", "cp311_windows_x86_64" ], - "filename": "pyzmq-26.2.1-cp311-cp311-macosx_10_15_universal2.whl", + "filename": "pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "ncore_pip_deps_311", - "requirement": "pyzmq==26.2.1", - "sha256": "c059883840e634a21c5b31d9b9a0e2b48f991b94d60a811092bc37992715146a", + "requirement": "pyyaml==6.0.3", + "sha256": "44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", "urls": [ - "https://files.pythonhosted.org/packages/b9/03/5ecc46a6ed5971299f5c03e016ca637802d8660e44392bea774fb7797405/pyzmq-26.2.1-cp311-cp311-macosx_10_15_universal2.whl" + "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl" ] } }, - "ncore_pip_deps_311_pyzmq_cp311_cp311_macosx_10_9_x86_64_ed038a92": { + "ncore_pip_deps_311_pyyaml_cp311_cp311_macosx_11_0_arm64_652cb6ed": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -12307,17 +12427,17 @@ "cp311_osx_x86_64", "cp311_windows_x86_64" ], - "filename": "pyzmq-26.2.1-cp311-cp311-macosx_10_9_x86_64.whl", + "filename": "pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "ncore_pip_deps_311", - "requirement": "pyzmq==26.2.1", - "sha256": "ed038a921df836d2f538e509a59cb638df3e70ca0fcd70d0bf389dfcdf784d2a", + "requirement": "pyyaml==6.0.3", + "sha256": "652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", "urls": [ - "https://files.pythonhosted.org/packages/40/51/48fec8f990ee644f461ff14c8fe5caa341b0b9b3a0ad7544f8ef17d6f528/pyzmq-26.2.1-cp311-cp311-macosx_10_9_x86_64.whl" + "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl" ] } }, - "ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_17_aarch64_9027a7fc": { + "ncore_pip_deps_311_pyyaml_cp311_cp311_manylinux_2_17_aarch64_10892704": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -12331,17 +12451,17 @@ "cp311_osx_x86_64", "cp311_windows_x86_64" ], - "filename": "pyzmq-26.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "filename": "pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "ncore_pip_deps_311", - "requirement": "pyzmq==26.2.1", - "sha256": "9027a7fcf690f1a3635dc9e55e38a0d6602dbbc0548935d08d46d2e7ec91f454", + "requirement": "pyyaml==6.0.3", + "sha256": "10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", "urls": [ - "https://files.pythonhosted.org/packages/c1/f4/f322b389727c687845e38470b48d7a43c18a83f26d4d5084603c6c3f79ca/pyzmq-26.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl" ] } }, - "ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_17_x86_64_f0019cc8": { + "ncore_pip_deps_311_pyyaml_cp311_cp311_manylinux_2_17_s390x_850774a7": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -12355,11 +12475,233 @@ "cp311_osx_x86_64", "cp311_windows_x86_64" ], - "filename": "pyzmq-26.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "filename": "pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "ncore_pip_deps_311", - "requirement": "pyzmq==26.2.1", - "sha256": "f0019cc804ac667fb8c8eaecdb66e6d4a68acf2e155d5c7d6381a5645bd93ae4", + "requirement": "pyyaml==6.0.3", + "sha256": "850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", + "urls": [ + "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl" + ] + } + }, + "ncore_pip_deps_311_pyyaml_cp311_cp311_manylinux_2_17_x86_64_b8bb0864": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyyaml==6.0.3", + "sha256": "b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", + "urls": [ + "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" + ] + } + }, + "ncore_pip_deps_311_pyyaml_cp311_cp311_musllinux_1_2_aarch64_1d37d57a": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyyaml==6.0.3", + "sha256": "1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", + "urls": [ + "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl" + ] + } + }, + "ncore_pip_deps_311_pyyaml_cp311_cp311_musllinux_1_2_x86_64_37503bfb": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyyaml==6.0.3", + "sha256": "37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", + "urls": [ + "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl" + ] + } + }, + "ncore_pip_deps_311_pyyaml_cp311_cp311_win_amd64_9f3bfb49": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyyaml-6.0.3-cp311-cp311-win_amd64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyyaml==6.0.3", + "sha256": "9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", + "urls": [ + "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl" + ] + } + }, + "ncore_pip_deps_311_pyyaml_sdist_d7662337": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cu128/torch" + ], + "filename": "pyyaml-6.0.3.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyyaml==6.0.3", + "sha256": "d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", + "urls": [ + "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz" + ] + } + }, + "ncore_pip_deps_311_pyzmq_cp311_cp311_macosx_10_15_universal2_c0598838": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyzmq-26.2.1-cp311-cp311-macosx_10_15_universal2.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyzmq==26.2.1", + "sha256": "c059883840e634a21c5b31d9b9a0e2b48f991b94d60a811092bc37992715146a", + "urls": [ + "https://files.pythonhosted.org/packages/b9/03/5ecc46a6ed5971299f5c03e016ca637802d8660e44392bea774fb7797405/pyzmq-26.2.1-cp311-cp311-macosx_10_15_universal2.whl" + ] + } + }, + "ncore_pip_deps_311_pyzmq_cp311_cp311_macosx_10_9_x86_64_ed038a92": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyzmq-26.2.1-cp311-cp311-macosx_10_9_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyzmq==26.2.1", + "sha256": "ed038a921df836d2f538e509a59cb638df3e70ca0fcd70d0bf389dfcdf784d2a", + "urls": [ + "https://files.pythonhosted.org/packages/40/51/48fec8f990ee644f461ff14c8fe5caa341b0b9b3a0ad7544f8ef17d6f528/pyzmq-26.2.1-cp311-cp311-macosx_10_9_x86_64.whl" + ] + } + }, + "ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_17_aarch64_9027a7fc": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyzmq-26.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyzmq==26.2.1", + "sha256": "9027a7fcf690f1a3635dc9e55e38a0d6602dbbc0548935d08d46d2e7ec91f454", + "urls": [ + "https://files.pythonhosted.org/packages/c1/f4/f322b389727c687845e38470b48d7a43c18a83f26d4d5084603c6c3f79ca/pyzmq-26.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl" + ] + } + }, + "ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_17_x86_64_f0019cc8": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp311_linux_aarch64", + "cp311_linux_arm", + "cp311_linux_ppc", + "cp311_linux_s390x", + "cp311_linux_x86_64", + "cp311_osx_aarch64", + "cp311_osx_x86_64", + "cp311_windows_x86_64" + ], + "filename": "pyzmq-26.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", + "repo": "ncore_pip_deps_311", + "requirement": "pyzmq==26.2.1", + "sha256": "f0019cc804ac667fb8c8eaecdb66e6d4a68acf2e155d5c7d6381a5645bd93ae4", "urls": [ "https://files.pythonhosted.org/packages/b5/e2/45c0f6e122b562cb8c6c45c0dcac1160a4e2207385ef9b13463e74f93031/pyzmq-26.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" ] @@ -16919,7 +17261,7 @@ ] } }, - "ncore_pip_deps_311_zarr_py3_none_any_2795e20a": { + "ncore_pip_deps_311_zarr_py3_none_any_72c3dcee": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -16933,17 +17275,17 @@ "cp311_osx_x86_64", "cp311_windows_x86_64" ], - "filename": "zarr-2.18.4-py3-none-any.whl", + "filename": "zarr-3.1.4-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "ncore_pip_deps_311", - "requirement": "zarr==2.18.4", - "sha256": "2795e20aff91093ce7e4da36ab1a138aededbd8ab66bf01fd01512e61d31e5d1", + "requirement": "zarr==3.1.4", + "sha256": "72c3dcee2d41527d21cd814cbaca1e6e23550701eb4fc30767ee53d925900eec", "urls": [ - "https://files.pythonhosted.org/packages/b4/d1/c84022a44afc7b7ccc442fba3daee56bdd03593d91ee4bc245a08e4fcc55/zarr-2.18.4-py3-none-any.whl" + "https://files.pythonhosted.org/packages/2d/8e/1b05f5b8a9fb1db01d1c09c3020125fc5c278d97843233dd6fa897a5f24f/zarr-3.1.4-py3-none-any.whl" ] } }, - "ncore_pip_deps_311_zarr_sdist_37790ede": { + "ncore_pip_deps_311_zarr_sdist_b0cf4eea": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -16963,13 +17305,13 @@ "--find-links", "https://download.pytorch.org/whl/cu128/torch" ], - "filename": "zarr-2.18.4.tar.gz", + "filename": "zarr-3.1.4.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_11_host//:python", "repo": "ncore_pip_deps_311", - "requirement": "zarr==2.18.4", - "sha256": "37790ededd0683ae1abe6ff90aa16c22543b3436810060f53d72c15e910c24bb", + "requirement": "zarr==3.1.4", + "sha256": "b0cf4eea8367ae444d7d0ea90b98d29ac0b7eb691fac2e0bcc2d2bd659522903", "urls": [ - "https://files.pythonhosted.org/packages/21/d1/764ca5b66d91b20dede66aedc6eb9ede3adbe5c61779e7378a7ecb010e87/zarr-2.18.4.tar.gz" + "https://files.pythonhosted.org/packages/c6/77/170915815b797861e0d74eb8246da2898f4069838d5f638de5d8bd1ab7c7/zarr-3.1.4.tar.gz" ] } }, @@ -20309,36 +20651,6 @@ ] } }, - "ncore_pip_deps_313_asciitree_sdist_4aa4b9b6": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "asciitree-0.3.3.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "asciitree==0.3.3", - "sha256": "4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e", - "urls": [ - "https://files.pythonhosted.org/packages/2d/6a/885bc91484e1aa8f618f6f0228d76d0e67000b0fdd6090673b777e311913/asciitree-0.3.3.tar.gz" - ] - } - }, "ncore_pip_deps_313_cbor2_cp313_cp313_macosx_11_0_arm64_f850860e": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { @@ -20567,7 +20879,7 @@ ] } }, - "ncore_pip_deps_313_deprecated_py3_none_any_58204cf4": { + "ncore_pip_deps_313_donfig_py3_none_any_2a3175ce": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20581,17 +20893,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "deprecated-3.0.0-py3-none-any.whl", + "filename": "donfig-0.8.1.post1-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "deprecated==3.0.0", - "sha256": "58204cf4a7f6270d547af5c278ee7a6bb56045a4b3d8441a1cd11660f41b7939", + "requirement": "donfig==0.8.1.post1", + "sha256": "2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d", "urls": [ - "https://files.pythonhosted.org/packages/83/ae/676feae8e4644a6d7169951a97f61c56f416c73f67bf1761f2461d75cc81/deprecated-3.0.0-py3-none-any.whl" + "https://files.pythonhosted.org/packages/0c/d5/c5db1ea3394c6e1732fb3286b3bd878b59507a8f77d32a2cebda7d7b7cd4/donfig-0.8.1.post1-py3-none-any.whl" ] } }, - "ncore_pip_deps_313_deprecated_sdist_16850204": { + "ncore_pip_deps_313_donfig_sdist_3bef3413": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20611,17 +20923,17 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "deprecated-3.0.0.tar.gz", + "filename": "donfig-0.8.1.post1.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "deprecated==3.0.0", - "sha256": "16850204d3a1e6bb0acd06bff48d96e8b0a0d25d1c52f71705405a0f4894192d", + "requirement": "donfig==0.8.1.post1", + "sha256": "3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52", "urls": [ - "https://files.pythonhosted.org/packages/f7/9c/16649913bf14c73e0a9453782e148362ff2657067deff6aa9c7ebcddcc31/deprecated-3.0.0.tar.gz" + "https://files.pythonhosted.org/packages/25/71/80cc718ff6d7abfbabacb1f57aaa42e9c1552bfdd01e64ddd704e4a03638/donfig-0.8.1.post1.tar.gz" ] } }, - "ncore_pip_deps_313_fasteners_py3_none_any_9422c40d": { + "ncore_pip_deps_313_filelock_py3_none_any_2ce9818e": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20635,17 +20947,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "fasteners-0.20-py3-none-any.whl", + "filename": "filelock-4.1.0-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "fasteners==0.20", - "sha256": "9422c40d1e350e4259f509fb2e608d6bc43c0136f79a00db1b49046029d0b3b7", + "requirement": "filelock==4.1.0", + "sha256": "2ce9818e3e2d8f284c1a964414447ef148d42a5fd5e2a477a7118e574b293ec1", "urls": [ - "https://files.pythonhosted.org/packages/51/ac/e5d886f892666d2d1e5cb8c1a41146e1d79ae8896477b1153a21711d3b44/fasteners-0.20-py3-none-any.whl" + "https://files.pythonhosted.org/packages/ee/86/032133892a5de43b5a98200b01aadcad68cc255e274a762f08b8a76d2912/filelock-4.1.0-py3-none-any.whl" ] } }, - "ncore_pip_deps_313_fasteners_sdist_55dce879": { + "ncore_pip_deps_313_filelock_sdist_ad7f724a": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20665,17 +20977,17 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "fasteners-0.20.tar.gz", + "filename": "filelock-4.1.0.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "fasteners==0.20", - "sha256": "55dce8792a41b56f727ba6e123fcaee77fd87e638a6863cec00007bfea84c8d8", + "requirement": "filelock==4.1.0", + "sha256": "ad7f724afef953e731b1cc39bcd3a09166d72ed7fcdf29e6e88b1c3235c6715d", "urls": [ - "https://files.pythonhosted.org/packages/2d/18/7881a99ba5244bfc82f06017316ffe93217dbbbcfa52b887caa1d4f2a6d3/fasteners-0.20.tar.gz" + "https://files.pythonhosted.org/packages/4c/58/6fd434bec86eff7c38a3168454cb132b762b2bea9b3ac094101a2f7bc32a/filelock-4.1.0.tar.gz" ] } }, - "ncore_pip_deps_313_filelock_py3_none_any_2ce9818e": { + "ncore_pip_deps_313_fsspec_py3_none_any_8dd6e646": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20689,17 +21001,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "filelock-4.1.0-py3-none-any.whl", + "filename": "fsspec-2026.9.0-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "filelock==4.1.0", - "sha256": "2ce9818e3e2d8f284c1a964414447ef148d42a5fd5e2a477a7118e574b293ec1", + "requirement": "fsspec==2026.9.0", + "sha256": "8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f", "urls": [ - "https://files.pythonhosted.org/packages/ee/86/032133892a5de43b5a98200b01aadcad68cc255e274a762f08b8a76d2912/filelock-4.1.0-py3-none-any.whl" + "https://files.pythonhosted.org/packages/6c/c0/a98505f18594f1bce828bb159cec0fcf9860562f1a2c85913409fc8f3d9e/fsspec-2026.9.0-py3-none-any.whl" ] } }, - "ncore_pip_deps_313_filelock_sdist_ad7f724a": { + "ncore_pip_deps_313_fsspec_sdist_0f081479": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20719,17 +21031,17 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "filelock-4.1.0.tar.gz", + "filename": "fsspec-2026.9.0.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "filelock==4.1.0", - "sha256": "ad7f724afef953e731b1cc39bcd3a09166d72ed7fcdf29e6e88b1c3235c6715d", + "requirement": "fsspec==2026.9.0", + "sha256": "0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe", "urls": [ - "https://files.pythonhosted.org/packages/4c/58/6fd434bec86eff7c38a3168454cb132b762b2bea9b3ac094101a2f7bc32a/filelock-4.1.0.tar.gz" + "https://files.pythonhosted.org/packages/77/cd/9be253869fc42e764de7f3dedd6969af7d44ff9c3375214a3442a6f3fc08/fsspec-2026.9.0.tar.gz" ] } }, - "ncore_pip_deps_313_fsspec_py3_none_any_8dd6e646": { + "ncore_pip_deps_313_google_crc32c_cp313_cp313_macosx_12_0_arm64_f1dc17d9": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20743,17 +21055,113 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "fsspec-2026.9.0-py3-none-any.whl", + "filename": "google_crc32c-1.9.0-cp313-cp313-macosx_12_0_arm64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "fsspec==2026.9.0", - "sha256": "8dd6e646e99ea382bd85f97a45e6b526a442d79423a7dc673f1e2756d05fcb5f", + "requirement": "google-crc32c==1.9.0", + "sha256": "f1dc17d987ddcc5eba12a7ce48f0eb93141dea236b170c1101151396edf2f0cf", "urls": [ - "https://files.pythonhosted.org/packages/6c/c0/a98505f18594f1bce828bb159cec0fcf9860562f1a2c85913409fc8f3d9e/fsspec-2026.9.0-py3-none-any.whl" + "https://files.pythonhosted.org/packages/3f/34/cb484e8b6174f130f8c6dc79c733a9dd8869b410ad6511fb6104c46b973a/google_crc32c-1.9.0-cp313-cp313-macosx_12_0_arm64.whl" ] } }, - "ncore_pip_deps_313_fsspec_sdist_0f081479": { + "ncore_pip_deps_313_google_crc32c_cp313_cp313_macosx_12_0_x86_64_f894a287": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp313-cp313-macosx_12_0_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "google-crc32c==1.9.0", + "sha256": "f894a2877650b56201d26a012a257b76d54a68834dc3913a93830ca8a047b075", + "urls": [ + "https://files.pythonhosted.org/packages/af/25/3e8e567bd48448e225ea27318ccf2b94e05124e7b8b97b13eaec9e127199/google_crc32c-1.9.0-cp313-cp313-macosx_12_0_x86_64.whl" + ] + } + }, + "ncore_pip_deps_313_google_crc32c_cp313_cp313_manylinux_2_17_aarch64_4488f155": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "google-crc32c==1.9.0", + "sha256": "4488f1553a9ab7e86cdedc833374a7e904031803b995dc0bd0be48c271fa6556", + "urls": [ + "https://files.pythonhosted.org/packages/f0/18/bee0dd59ae622482dc6463636c79e4bde7c954d061c859c9256362c9931a/google_crc32c-1.9.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl" + ] + } + }, + "ncore_pip_deps_313_google_crc32c_cp313_cp313_manylinux_2_17_x86_64_0568b17e": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "google-crc32c==1.9.0", + "sha256": "0568b17ed90ac596f29400d99e243fd0cc6276766183def888d1bf8d1dc13827", + "urls": [ + "https://files.pythonhosted.org/packages/fd/b6/e76e80fed5f2558273c7839e622f98095c9b36c719c7147e38e3c055cb70/google_crc32c-1.9.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl" + ] + } + }, + "ncore_pip_deps_313_google_crc32c_cp313_cp313_win_amd64_8583ec21": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "google_crc32c-1.9.0-cp313-cp313-win_amd64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "google-crc32c==1.9.0", + "sha256": "8583ec21d56b565d68ab2963cc7e21b3b271247c29b04286068255ef65f221bd", + "urls": [ + "https://files.pythonhosted.org/packages/87/34/165542bfa99dfef91a76471cc48cce74b8ff4e295722896087ab2b8e8611/google_crc32c-1.9.0-cp313-cp313-win_amd64.whl" + ] + } + }, + "ncore_pip_deps_313_google_crc32c_sdist_7b8c84c3": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -20773,13 +21181,13 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "fsspec-2026.9.0.tar.gz", + "filename": "google_crc32c-1.9.0.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "fsspec==2026.9.0", - "sha256": "0f08147951c8cb31d844c3547d631053b127863b60be04cf06e121333ee0e2fe", + "requirement": "google-crc32c==1.9.0", + "sha256": "7b8c84c3d159ab6817fe3f74e6e6cef099c3f95dcec3abc0d8afb1404642efbe", "urls": [ - "https://files.pythonhosted.org/packages/77/cd/9be253869fc42e764de7f3dedd6969af7d44ff9c3375214a3442a6f3fc08/fsspec-2026.9.0.tar.gz" + "https://files.pythonhosted.org/packages/fa/25/9cb0c1c31c45b893eb8f11ae70b3f4309432d59b5acaebca5dbe791729a4/google_crc32c-1.9.0.tar.gz" ] } }, @@ -21293,31 +21701,7 @@ ] } }, - "ncore_pip_deps_313_mypy_extensions_py3_none_any_1be4cccd": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "mypy_extensions-1.1.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "mypy-extensions==1.1.0", - "sha256": "1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", - "urls": [ - "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl" - ] - } - }, - "ncore_pip_deps_313_mypy_extensions_sdist_52e68efc": { + "ncore_pip_deps_313_msgspec_cp313_cp313_macosx_10_13_x86_64_f13c127a": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21331,23 +21715,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "mypy_extensions-1.1.0.tar.gz", + "filename": "msgspec-0.22.0-cp313-cp313-macosx_10_13_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "mypy-extensions==1.1.0", - "sha256": "52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", + "requirement": "msgspec==0.22.0", + "sha256": "f13c127a945479bc9db057eb253b8851075c8e1ae07ffc967bfa1c5676203a86", "urls": [ - "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz" + "https://files.pythonhosted.org/packages/7f/62/5374fba2ede0408f4bd8b9b3a6c8464f8d0ea7ae9a2a064bd81ca492bd1e/msgspec-0.22.0-cp313-cp313-macosx_10_13_x86_64.whl" ] } }, - "ncore_pip_deps_313_networkx_py3_none_any_e3fd2c13": { + "ncore_pip_deps_313_msgspec_cp313_cp313_macosx_11_0_arm64_5aa24eb4": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21361,17 +21739,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "networkx-3.7-py3-none-any.whl", + "filename": "msgspec-0.22.0-cp313-cp313-macosx_11_0_arm64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "networkx==3.7", - "sha256": "e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0", + "requirement": "msgspec==0.22.0", + "sha256": "5aa24eb475d070ecbbe5b21080fc3ce4b0b76c60de25cfe0c9678d8fb44bb42f", "urls": [ - "https://files.pythonhosted.org/packages/7e/cd/fe58041e9011f307c490e3e17dd48cc516448f7c698a3f2d9d9d65d7e6a8/networkx-3.7-py3-none-any.whl" + "https://files.pythonhosted.org/packages/cc/e3/357baa8d2a9164a98dfd7ef9d3a58125df0ed981be909945bdd337be7194/msgspec-0.22.0-cp313-cp313-macosx_11_0_arm64.whl" ] } }, - "ncore_pip_deps_313_networkx_sdist_fd77a511": { + "ncore_pip_deps_313_msgspec_cp313_cp313_manylinux_2_17_aarch64_627bfdfe": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21385,23 +21763,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "networkx-3.7.tar.gz", + "filename": "msgspec-0.22.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "networkx==3.7", - "sha256": "fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a", + "requirement": "msgspec==0.22.0", + "sha256": "627bfdfe5a4b3d916b3360b30f4cddeee3a084f56593e33527c6872fa8322ff9", "urls": [ - "https://files.pythonhosted.org/packages/dc/76/3af777226b63a5e64a6b36b1ec5855c14e2b94a37096d4760e595fc43511/networkx-3.7.tar.gz" + "https://files.pythonhosted.org/packages/fa/1b/9cc07718d1dee8ed5e89a265801d565bc0f15ead435ccb198f9c7bf92574/msgspec-0.22.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl" ] } }, - "ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_10_13_x86_64_e3d82b70": { + "ncore_pip_deps_313_msgspec_cp313_cp313_manylinux_2_17_x86_64_c6c310ef": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21415,17 +21787,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numcodecs-0.15.1-cp313-cp313-macosx_10_13_x86_64.whl", + "filename": "msgspec-0.22.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numcodecs==0.15.1", - "sha256": "e3d82b70500cf61e8d115faa0d0a76be6ecdc24a16477ee3279d711699ad85f3", + "requirement": "msgspec==0.22.0", + "sha256": "c6c310ef83e7e291b01a63298828f848348bb99e84a1098c4b3923c05674d032", "urls": [ - "https://files.pythonhosted.org/packages/78/57/acbc54b3419e5be65015e47177c76c0a73e037fd3ae2cde5808169194d4d/numcodecs-0.15.1-cp313-cp313-macosx_10_13_x86_64.whl" + "https://files.pythonhosted.org/packages/46/64/f33fdfe95aca76601194a7064d14816c7c22c4eccc1b03a5335785895fa3/msgspec-0.22.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" ] } }, - "ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_11_0_arm64_1d471a18": { + "ncore_pip_deps_313_msgspec_cp313_cp313_musllinux_1_2_aarch64_bc374ded": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21439,17 +21811,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numcodecs-0.15.1-cp313-cp313-macosx_11_0_arm64.whl", + "filename": "msgspec-0.22.0-cp313-cp313-musllinux_1_2_aarch64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numcodecs==0.15.1", - "sha256": "1d471a1829ce52d3f365053a2bd1379e32e369517557c4027ddf5ac0d99c591e", + "requirement": "msgspec==0.22.0", + "sha256": "bc374dedd5f85a5f4de2386dc5f737894ccb8c1ac18e9566ce66fd9839e6285d", "urls": [ - "https://files.pythonhosted.org/packages/b6/56/9863fa6dc679f40a31bea5e9713ee5507a31dcd3ee82ea4b1a9268ce52e8/numcodecs-0.15.1-cp313-cp313-macosx_11_0_arm64.whl" + "https://files.pythonhosted.org/packages/88/a6/7b5c4fb39e0bf2dabc8be923c33c39b07ba769a0ce6f0afbbdfaadb1f2f2/msgspec-0.22.0-cp313-cp313-musllinux_1_2_aarch64.whl" ] } }, - "ncore_pip_deps_313_numcodecs_cp313_cp313_manylinux_2_17_x86_64_1dfdea4a": { + "ncore_pip_deps_313_msgspec_cp313_cp313_musllinux_1_2_x86_64_6f48317f": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21463,17 +21835,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numcodecs-0.15.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "filename": "msgspec-0.22.0-cp313-cp313-musllinux_1_2_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numcodecs==0.15.1", - "sha256": "1dfdea4a67108205edfce99c1cb6cd621343bc7abb7e16a041c966776920e7de", + "requirement": "msgspec==0.22.0", + "sha256": "6f48317f05312bfdf78248f53933f830f07ab75cc1c813ac3ca4220cb3b5b019", "urls": [ - "https://files.pythonhosted.org/packages/fa/91/d96999b41e3146b6c0ce6bddc5ad85803cb4d743c95394562c2a4bb8cded/numcodecs-0.15.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl" + "https://files.pythonhosted.org/packages/6c/e5/b4c5323b17ecfce45350695d40fc93e16856db957a53cbcf2f53007d6e12/msgspec-0.22.0-cp313-cp313-musllinux_1_2_x86_64.whl" ] } }, - "ncore_pip_deps_313_numcodecs_cp313_cp313_win_amd64_a4f7bdb2": { + "ncore_pip_deps_313_msgspec_cp313_cp313_win_amd64_0739b068": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21487,17 +21859,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numcodecs-0.15.1-cp313-cp313-win_amd64.whl", + "filename": "msgspec-0.22.0-cp313-cp313-win_amd64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numcodecs==0.15.1", - "sha256": "a4f7bdb26f1b34423cb56d48e75821223be38040907c9b5954eeb7463e7eb03c", + "requirement": "msgspec==0.22.0", + "sha256": "0739b068f31f2004a364f97679ba91f2f5ecd6ec2a5b4b890188ab5c57d20672", "urls": [ - "https://files.pythonhosted.org/packages/c3/32/233e5ede6568bdb044e6f99aaa9fa39827ff3109c6487fc137315f733586/numcodecs-0.15.1-cp313-cp313-win_amd64.whl" + "https://files.pythonhosted.org/packages/01/33/e591f9d3d8d6c9cfc02ae95f3e3c44920f2d18050f3f252c244e0f293a0e/msgspec-0.22.0-cp313-cp313-win_amd64.whl" ] } }, - "ncore_pip_deps_313_numcodecs_sdist_eeed77e4": { + "ncore_pip_deps_313_msgspec_sdist_0a13624a": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21517,17 +21889,17 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "numcodecs-0.15.1.tar.gz", + "filename": "msgspec-0.22.0.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numcodecs==0.15.1", - "sha256": "eeed77e4d6636641a2cc605fbc6078c7a8f2cc40f3dfa2b3f61e52e6091b04ff", + "requirement": "msgspec==0.22.0", + "sha256": "0a13624a4969159fe35d8c2a3d377b2b61bbd8585e327440d5e52725affcce38", "urls": [ - "https://files.pythonhosted.org/packages/63/fc/bb532969eb8236984ba65e4f0079a7da885b8ac0ce1f0835decbb3938a62/numcodecs-0.15.1.tar.gz" + "https://files.pythonhosted.org/packages/d0/e6/6dcf9306ff3c5e486578f3bf29ed11dfbdbbc2a8bf0caf7e07d392887fda/msgspec-0.22.0.tar.gz" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_macosx_10_13_x86_64_2377da2d": { + "ncore_pip_deps_313_mypy_extensions_py3_none_any_1be4cccd": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21541,17 +21913,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-macosx_10_13_x86_64.whl", + "filename": "mypy_extensions-1.1.0-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "2377da2dd3ba2c1200956acbab2a358c83b8e1f8531191672d1cd6ad83250d53", + "requirement": "mypy-extensions==1.1.0", + "sha256": "1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", "urls": [ - "https://files.pythonhosted.org/packages/67/14/1c3ee0118a8fce08565a5d8482631608426a33af10a01077fada5dc7c119/numpy-2.5.4-cp313-cp313-macosx_10_13_x86_64.whl" + "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_macosx_11_0_arm64_7415db95": { + "ncore_pip_deps_313_mypy_extensions_sdist_52e68efc": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21565,17 +21937,23 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-macosx_11_0_arm64.whl", + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "mypy_extensions-1.1.0.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "7415db95818b39ec475a5eea54d9e3b6bc83e3912158e46da3438cdce399804d", + "requirement": "mypy-extensions==1.1.0", + "sha256": "52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", "urls": [ - "https://files.pythonhosted.org/packages/83/8c/b0ea9477fb1f0d4484bbc5cba21678cc9969704d8d7f3f158d1db35f8e14/numpy-2.5.4-cp313-cp313-macosx_11_0_arm64.whl" + "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_macosx_14_0_arm64_6d6a71b9": { + "ncore_pip_deps_313_networkx_py3_none_any_e3fd2c13": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21589,17 +21967,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-macosx_14_0_arm64.whl", + "filename": "networkx-3.7-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "6d6a71b9d9a97c03633aa12565ef2825ffa036cc1d99cfd50dacf0f128af4fe2", + "requirement": "networkx==3.7", + "sha256": "e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0", "urls": [ - "https://files.pythonhosted.org/packages/e2/84/6a3d75b3ba3dfe84ac0053450753d1e6d250a8bf80f66474cc46d1fb643f/numpy-2.5.4-cp313-cp313-macosx_14_0_arm64.whl" + "https://files.pythonhosted.org/packages/7e/cd/fe58041e9011f307c490e3e17dd48cc516448f7c698a3f2d9d9d65d7e6a8/networkx-3.7-py3-none-any.whl" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_macosx_14_0_x86_64_d8200f16": { + "ncore_pip_deps_313_networkx_sdist_fd77a511": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21613,17 +21991,23 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-macosx_14_0_x86_64.whl", + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "networkx-3.7.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "d8200f16437b289a5bb927c6e184eccc3e8389bc0070fea4cd5b9e13c1757959", + "requirement": "networkx==3.7", + "sha256": "fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a", "urls": [ - "https://files.pythonhosted.org/packages/61/18/bb993f267ca20b376e07092a16793a5b31ed3138751e9ba480011a14d742/numpy-2.5.4-cp313-cp313-macosx_14_0_x86_64.whl" + "https://files.pythonhosted.org/packages/dc/76/3af777226b63a5e64a6b36b1ec5855c14e2b94a37096d4760e595fc43511/networkx-3.7.tar.gz" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_manylinux_2_27_aarch64_1c2e71b0": { + "ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_10_13_x86_64_de2c66db": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21637,17 +22021,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", + "filename": "numcodecs-0.17.0-cp313-cp313-macosx_10_13_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "1c2e71b04c6cad90026e544501bbe0ab9290fa8a4d845e7e8c0d124fb429c988", + "requirement": "numcodecs==0.17.0", + "sha256": "de2c66db238e74e66fe9be7e02b7e0129b75d3f812d38e4019eb0102cc2dcdf0", "urls": [ - "https://files.pythonhosted.org/packages/db/b6/135bb0953b61dc21c6cafa14b424ae666944e4899cf140e00c2b322a1a45/numpy-2.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl" + "https://files.pythonhosted.org/packages/11/f1/1d3d2bcb1240e5000f6647b5b0fd465b2b51ecef180bfa797a85df48cf2f/numcodecs-0.17.0-cp313-cp313-macosx_10_13_x86_64.whl" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_manylinux_2_27_x86_64_6ffa0766": { + "ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_11_0_arm64_69b9b468": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21661,17 +22045,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "filename": "numcodecs-0.17.0-cp313-cp313-macosx_11_0_arm64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "6ffa07666f8da0eef81d149934a626d0d95fbd6838432a33e66245423a9062c0", + "requirement": "numcodecs==0.17.0", + "sha256": "69b9b4685097c4d478a0c829debf4470555ec63e92cdd2c6b5f195460f1dc888", "urls": [ - "https://files.pythonhosted.org/packages/da/24/3bd070f3269dc609d8f26b2643f62ef91bb415841c0b294805aaf7fe06da/numpy-2.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl" + "https://files.pythonhosted.org/packages/64/81/64e2472a8b3a9fa26bccfc7d5fa876770a9027bd5cd77e5b4a7b807a0785/numcodecs-0.17.0-cp313-cp313-macosx_11_0_arm64.whl" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_musllinux_1_2_aarch64_2fa3328f": { + "ncore_pip_deps_313_numcodecs_cp313_cp313_manylinux_2_17_aarch64_7065b334": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21685,17 +22069,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-musllinux_1_2_aarch64.whl", + "filename": "numcodecs-0.17.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "2fa3328f784fc8277fc48026f6cad516f5c561c5d8e2e39b3c9e0c8f23223b34", + "requirement": "numcodecs==0.17.0", + "sha256": "7065b3349b73d54785aa89e00d0b97d80f664e9056757929d28151f9208dc04c", "urls": [ - "https://files.pythonhosted.org/packages/c7/8e/9d15bd356b0a019c965312b1a3c6a727cac4cae5bc40045fbc12ce4cff9c/numpy-2.5.4-cp313-cp313-musllinux_1_2_aarch64.whl" + "https://files.pythonhosted.org/packages/25/ea/2ab25f7e674cf1e78f123c5c2689d8a7dc85475554af0619bcce05cb32a9/numcodecs-0.17.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_musllinux_1_2_x86_64_b86966fb": { + "ncore_pip_deps_313_numcodecs_cp313_cp313_manylinux_2_17_x86_64_6c3342d9": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21709,17 +22093,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-musllinux_1_2_x86_64.whl", + "filename": "numcodecs-0.17.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "b86966fbe4ad7de710422175572bcdc75fdedadfb54bc6fab7deabccddd7780b", + "requirement": "numcodecs==0.17.0", + "sha256": "6c3342d91ed7cf59c1be84396edd364e936bb0ec9e366d24bb69689748d19625", "urls": [ - "https://files.pythonhosted.org/packages/dc/fe/9d5b560db964f15871885f2250795d15945f8699e17ef90c0c2ff4c875b2/numpy-2.5.4-cp313-cp313-musllinux_1_2_x86_64.whl" + "https://files.pythonhosted.org/packages/9d/96/b3bf9a31978d936654a73f2bb1036b92b6515164f170092d162419eb771c/numcodecs-0.17.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" ] } }, - "ncore_pip_deps_313_numpy_cp313_cp313_win_amd64_8b4d2fd2": { + "ncore_pip_deps_313_numcodecs_cp313_cp313_win_amd64_a854e9c8": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21733,17 +22117,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "numpy-2.5.4-cp313-cp313-win_amd64.whl", + "filename": "numcodecs-0.17.0-cp313-cp313-win_amd64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "8b4d2fd2d34e5f8c9235ee787de5631a37a28402b15cb80814df973d2be54129", + "requirement": "numcodecs==0.17.0", + "sha256": "a854e9c89f58eeeb2453f3c1637d1916797edb6eaff26bc186a6cdb09d187092", "urls": [ - "https://files.pythonhosted.org/packages/90/8c/140a40398a66b4471211be1affdb6ed24c486d581bd28d07b7f2fcb69540/numpy-2.5.4-cp313-cp313-win_amd64.whl" + "https://files.pythonhosted.org/packages/5c/ec/47515bea31725376aa6f061c335326cc7437f863ab704b8e167e80734bfd/numcodecs-0.17.0-cp313-cp313-win_amd64.whl" ] } }, - "ncore_pip_deps_313_numpy_sdist_9a94cf75": { + "ncore_pip_deps_313_numcodecs_sdist_e8db2e33": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21763,17 +22147,17 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "numpy-2.5.4.tar.gz", + "filename": "numcodecs-0.17.0.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "numpy==2.5.4", - "sha256": "9a94cf751c9ad8ebaa835bcd3d40dacf8534ad086b88c38029b65123c7999d2a", + "requirement": "numcodecs==0.17.0", + "sha256": "e8db2e337bdafd3bb5f891a2543b53b2b36a509ce9d587af2846db3715b6c8b9", "urls": [ - "https://files.pythonhosted.org/packages/95/b0/c7453d0b6e2073c3264468b106ee1563750cecc910965e67357e3698c83e/numpy-2.5.4.tar.gz" + "https://files.pythonhosted.org/packages/dd/ec/260cdb6304868de6db14eb31064bd2735c0200bcb3331d6b4c9e9be02a03/numcodecs-0.17.0.tar.gz" ] } }, - "ncore_pip_deps_313_opencv_python_headless_cp37_abi3_macosx_13_0_arm64_030ca5e0": { + "ncore_pip_deps_313_numpy_cp313_cp313_macosx_10_13_x86_64_2377da2d": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21787,17 +22171,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "opencv_python_headless-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl", + "filename": "numpy-2.5.4-cp313-cp313-macosx_10_13_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "opencv-python-headless==5.0.0.93", - "sha256": "030ca5e0837a2963ab36ef896baa9767eb8d2b83353fb28af5a521e40dd8756f", + "requirement": "numpy==2.5.4", + "sha256": "2377da2dd3ba2c1200956acbab2a358c83b8e1f8531191672d1cd6ad83250d53", "urls": [ - "https://files.pythonhosted.org/packages/53/7c/8c8097891c509d98cd128493835c95631c80be6a8f37ed9d25716c2e16f1/opencv_python_headless-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl" + "https://files.pythonhosted.org/packages/67/14/1c3ee0118a8fce08565a5d8482631608426a33af10a01077fada5dc7c119/numpy-2.5.4-cp313-cp313-macosx_10_13_x86_64.whl" ] } }, - "ncore_pip_deps_313_opencv_python_headless_cp37_abi3_macosx_14_0_x86_64_1e55af3a": { + "ncore_pip_deps_313_numpy_cp313_cp313_macosx_11_0_arm64_7415db95": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -21811,7 +22195,253 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "opencv_python_headless-5.0.0.93-cp37-abi3-macosx_14_0_x86_64.whl", + "filename": "numpy-2.5.4-cp313-cp313-macosx_11_0_arm64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "7415db95818b39ec475a5eea54d9e3b6bc83e3912158e46da3438cdce399804d", + "urls": [ + "https://files.pythonhosted.org/packages/83/8c/b0ea9477fb1f0d4484bbc5cba21678cc9969704d8d7f3f158d1db35f8e14/numpy-2.5.4-cp313-cp313-macosx_11_0_arm64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_macosx_14_0_arm64_6d6a71b9": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-macosx_14_0_arm64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "6d6a71b9d9a97c03633aa12565ef2825ffa036cc1d99cfd50dacf0f128af4fe2", + "urls": [ + "https://files.pythonhosted.org/packages/e2/84/6a3d75b3ba3dfe84ac0053450753d1e6d250a8bf80f66474cc46d1fb643f/numpy-2.5.4-cp313-cp313-macosx_14_0_arm64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_macosx_14_0_x86_64_d8200f16": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-macosx_14_0_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "d8200f16437b289a5bb927c6e184eccc3e8389bc0070fea4cd5b9e13c1757959", + "urls": [ + "https://files.pythonhosted.org/packages/61/18/bb993f267ca20b376e07092a16793a5b31ed3138751e9ba480011a14d742/numpy-2.5.4-cp313-cp313-macosx_14_0_x86_64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_manylinux_2_27_aarch64_1c2e71b0": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "1c2e71b04c6cad90026e544501bbe0ab9290fa8a4d845e7e8c0d124fb429c988", + "urls": [ + "https://files.pythonhosted.org/packages/db/b6/135bb0953b61dc21c6cafa14b424ae666944e4899cf140e00c2b322a1a45/numpy-2.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_manylinux_2_27_x86_64_6ffa0766": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "6ffa07666f8da0eef81d149934a626d0d95fbd6838432a33e66245423a9062c0", + "urls": [ + "https://files.pythonhosted.org/packages/da/24/3bd070f3269dc609d8f26b2643f62ef91bb415841c0b294805aaf7fe06da/numpy-2.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_musllinux_1_2_aarch64_2fa3328f": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-musllinux_1_2_aarch64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "2fa3328f784fc8277fc48026f6cad516f5c561c5d8e2e39b3c9e0c8f23223b34", + "urls": [ + "https://files.pythonhosted.org/packages/c7/8e/9d15bd356b0a019c965312b1a3c6a727cac4cae5bc40045fbc12ce4cff9c/numpy-2.5.4-cp313-cp313-musllinux_1_2_aarch64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_musllinux_1_2_x86_64_b86966fb": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-musllinux_1_2_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "b86966fbe4ad7de710422175572bcdc75fdedadfb54bc6fab7deabccddd7780b", + "urls": [ + "https://files.pythonhosted.org/packages/dc/fe/9d5b560db964f15871885f2250795d15945f8699e17ef90c0c2ff4c875b2/numpy-2.5.4-cp313-cp313-musllinux_1_2_x86_64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_cp313_cp313_win_amd64_8b4d2fd2": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "numpy-2.5.4-cp313-cp313-win_amd64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "8b4d2fd2d34e5f8c9235ee787de5631a37a28402b15cb80814df973d2be54129", + "urls": [ + "https://files.pythonhosted.org/packages/90/8c/140a40398a66b4471211be1affdb6ed24c486d581bd28d07b7f2fcb69540/numpy-2.5.4-cp313-cp313-win_amd64.whl" + ] + } + }, + "ncore_pip_deps_313_numpy_sdist_9a94cf75": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "numpy-2.5.4.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "numpy==2.5.4", + "sha256": "9a94cf751c9ad8ebaa835bcd3d40dacf8534ad086b88c38029b65123c7999d2a", + "urls": [ + "https://files.pythonhosted.org/packages/95/b0/c7453d0b6e2073c3264468b106ee1563750cecc910965e67357e3698c83e/numpy-2.5.4.tar.gz" + ] + } + }, + "ncore_pip_deps_313_opencv_python_headless_cp37_abi3_macosx_13_0_arm64_030ca5e0": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "opencv_python_headless-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "opencv-python-headless==5.0.0.93", + "sha256": "030ca5e0837a2963ab36ef896baa9767eb8d2b83353fb28af5a521e40dd8756f", + "urls": [ + "https://files.pythonhosted.org/packages/53/7c/8c8097891c509d98cd128493835c95631c80be6a8f37ed9d25716c2e16f1/opencv_python_headless-5.0.0.93-cp37-abi3-macosx_13_0_arm64.whl" + ] + } + }, + "ncore_pip_deps_313_opencv_python_headless_cp37_abi3_macosx_14_0_x86_64_1e55af3a": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "opencv_python_headless-5.0.0.93-cp37-abi3-macosx_14_0_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", "requirement": "opencv-python-headless==5.0.0.93", @@ -22294,14 +22924,254 @@ "filename": "pillow-12.3.0-cp313-cp313-win_amd64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pillow==12.3.0", - "sha256": "1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", + "requirement": "pillow==12.3.0", + "sha256": "1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", + "urls": [ + "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl" + ] + } + }, + "ncore_pip_deps_313_pillow_sdist_3b8182a7": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "pillow-12.3.0.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pillow==12.3.0", + "sha256": "3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", + "urls": [ + "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz" + ] + } + }, + "ncore_pip_deps_313_pluggy_py3_none_any_e920276d": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "pluggy-1.6.0-py3-none-any.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pluggy==1.6.0", + "sha256": "e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", + "urls": [ + "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl" + ] + } + }, + "ncore_pip_deps_313_pluggy_sdist_7dcc130b": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "pluggy-1.6.0.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pluggy==1.6.0", + "sha256": "7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", + "urls": [ + "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz" + ] + } + }, + "ncore_pip_deps_313_pygments_py3_none_any_2363c69b": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "pygments-2.21.0-py3-none-any.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pygments==2.21.0", + "sha256": "2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", + "urls": [ + "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl" + ] + } + }, + "ncore_pip_deps_313_pygments_sdist_610ca751": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "pygments-2.21.0.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pygments==2.21.0", + "sha256": "610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", + "urls": [ + "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz" + ] + } + }, + "ncore_pip_deps_313_pytest_py3_none_any_37a86b45": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "pytest-9.1.1-py3-none-any.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pytest==9.1.1", + "sha256": "37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", + "urls": [ + "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl" + ] + } + }, + "ncore_pip_deps_313_pytest_sdist_1088fbde": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "extra_pip_args": [ + "--index-url", + "https://pypi.org/simple", + "--find-links", + "https://download.pytorch.org/whl/cpu/torch" + ], + "filename": "pytest-9.1.1.tar.gz", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pytest==9.1.1", + "sha256": "1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", + "urls": [ + "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz" + ] + } + }, + "ncore_pip_deps_313_pyyaml_cp313_cp313_macosx_10_13_x86_64_8da9669d": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pyyaml==6.0.3", + "sha256": "8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", + "urls": [ + "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl" + ] + } + }, + "ncore_pip_deps_313_pyyaml_cp313_cp313_macosx_11_0_arm64_2283a07e": { + "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", + "attributes": { + "dep_template": "@ncore_pip_deps//{name}:{target}", + "experimental_target_platforms": [ + "cp313_linux_aarch64", + "cp313_linux_arm", + "cp313_linux_ppc", + "cp313_linux_s390x", + "cp313_linux_x86_64", + "cp313_osx_aarch64", + "cp313_osx_x86_64", + "cp313_windows_x86_64" + ], + "filename": "pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", + "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", + "repo": "ncore_pip_deps_313", + "requirement": "pyyaml==6.0.3", + "sha256": "2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", "urls": [ - "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl" + "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl" ] } }, - "ncore_pip_deps_313_pillow_sdist_3b8182a7": { + "ncore_pip_deps_313_pyyaml_cp313_cp313_manylinux_2_17_aarch64_ee292290": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22315,23 +23185,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "pillow-12.3.0.tar.gz", + "filename": "pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pillow==12.3.0", - "sha256": "3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", + "requirement": "pyyaml==6.0.3", + "sha256": "ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", "urls": [ - "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz" + "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl" ] } }, - "ncore_pip_deps_313_pluggy_py3_none_any_e920276d": { + "ncore_pip_deps_313_pyyaml_cp313_cp313_manylinux_2_17_s390x_a33284e2": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22345,17 +23209,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "pluggy-1.6.0-py3-none-any.whl", + "filename": "pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pluggy==1.6.0", - "sha256": "e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", + "requirement": "pyyaml==6.0.3", + "sha256": "a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", "urls": [ - "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl" + "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl" ] } }, - "ncore_pip_deps_313_pluggy_sdist_7dcc130b": { + "ncore_pip_deps_313_pyyaml_cp313_cp313_manylinux_2_17_x86_64_0f29edc4": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22369,23 +23233,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "pluggy-1.6.0.tar.gz", + "filename": "pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pluggy==1.6.0", - "sha256": "7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", + "requirement": "pyyaml==6.0.3", + "sha256": "0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", "urls": [ - "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz" + "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl" ] } }, - "ncore_pip_deps_313_pygments_py3_none_any_2363c69b": { + "ncore_pip_deps_313_pyyaml_cp313_cp313_musllinux_1_2_aarch64_f7057c9a": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22399,17 +23257,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "pygments-2.21.0-py3-none-any.whl", + "filename": "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pygments==2.21.0", - "sha256": "2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", + "requirement": "pyyaml==6.0.3", + "sha256": "f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", "urls": [ - "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl" + "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl" ] } }, - "ncore_pip_deps_313_pygments_sdist_610ca751": { + "ncore_pip_deps_313_pyyaml_cp313_cp313_musllinux_1_2_x86_64_eda16858": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22423,23 +23281,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "pygments-2.21.0.tar.gz", + "filename": "pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pygments==2.21.0", - "sha256": "610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", + "requirement": "pyyaml==6.0.3", + "sha256": "eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", "urls": [ - "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz" + "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl" ] } }, - "ncore_pip_deps_313_pytest_py3_none_any_37a86b45": { + "ncore_pip_deps_313_pyyaml_cp313_cp313_win_amd64_79005a0d": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22453,17 +23305,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "pytest-9.1.1-py3-none-any.whl", + "filename": "pyyaml-6.0.3-cp313-cp313-win_amd64.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pytest==9.1.1", - "sha256": "37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", + "requirement": "pyyaml==6.0.3", + "sha256": "79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", "urls": [ - "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl" + "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl" ] } }, - "ncore_pip_deps_313_pytest_sdist_1088fbde": { + "ncore_pip_deps_313_pyyaml_sdist_d7662337": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -22483,13 +23335,13 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "pytest-9.1.1.tar.gz", + "filename": "pyyaml-6.0.3.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "pytest==9.1.1", - "sha256": "1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", + "requirement": "pyyaml==6.0.3", + "sha256": "d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", "urls": [ - "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz" + "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz" ] } }, @@ -23027,229 +23879,7 @@ ] } }, - "ncore_pip_deps_313_wrapt_cp313_cp313_macosx_10_13_x86_64_b312b3cc": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-macosx_10_13_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "b312b3cc87951faaed3cfef984d768ee8bee7f935d9cc929aaa9946b0b96a98c", - "urls": [ - "https://files.pythonhosted.org/packages/d6/4b/cc7bb5668f7ddc0e73e236e96a0c06cab8fddfca9c53538c9dffac62db6f/wrapt-2.5.0-cp313-cp313-macosx_10_13_x86_64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_cp313_cp313_macosx_11_0_arm64_c57ddae2": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-macosx_11_0_arm64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "c57ddae24cf72eb6bd18112638a987cafe6109d90f2df111e6934362cc03ac1a", - "urls": [ - "https://files.pythonhosted.org/packages/4a/13/5d15ef0e2f42d5f084930dc4863e6e52c160c28301c8780aae170c58421c/wrapt-2.5.0-cp313-cp313-macosx_11_0_arm64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_cp313_cp313_manylinux_2_17_aarch64_6058e12e": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "6058e12e9caa33468f9a36fb88c15a4bb30a479f997b37834b83abdbf062f264", - "urls": [ - "https://files.pythonhosted.org/packages/a4/f9/47ae1d7ef325c3f6c81ae3c1fb4a3fef9d98c8025ed676c0bfc1550903ce/wrapt-2.5.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_cp313_cp313_manylinux_2_5_x86_64_b95a6eca": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "b95a6eca3b927853529eea958310563c83140ae8451dd5dc4399c7da385dc4f3", - "urls": [ - "https://files.pythonhosted.org/packages/1e/02/c7174e78b0c38bb279b2d3c25a6bd7fb9d3b0200c3e5a8fad084e7cc3e85/wrapt-2.5.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_cp313_cp313_musllinux_1_2_aarch64_f4ef4935": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "f4ef4935962f7029b2058a99f1a47ccbffc3be919dddb3becb6c2c48eac3d9f0", - "urls": [ - "https://files.pythonhosted.org/packages/41/45/fc252bda5aa1ca01bc838d3b108778e786a2a13d0c52fd17c5f6179aa246/wrapt-2.5.0-cp313-cp313-musllinux_1_2_aarch64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_cp313_cp313_musllinux_1_2_x86_64_a346408f": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "a346408f19b6d589bf029f25f65c0b4cdeed6302ef8f40da4e5d1552d22dc037", - "urls": [ - "https://files.pythonhosted.org/packages/81/8e/2b823fded8c3b815408c58633929812eacd29d824fe57548b7868c4ee422/wrapt-2.5.0-cp313-cp313-musllinux_1_2_x86_64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_cp313_cp313_win_amd64_77f0a74f": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-cp313-cp313-win_amd64.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "77f0a74ff6f6cf89f5b673a732d5afe1911a6e6b1c017260836fdfdf85518dc1", - "urls": [ - "https://files.pythonhosted.org/packages/ce/9a/51d95640e01d0ebdd04a7223755f076e4936b0c124ce99bb01a12b53e66c/wrapt-2.5.0-cp313-cp313-win_amd64.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_py3_none_any_107eea1a": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "filename": "wrapt-2.5.0-py3-none-any.whl", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "107eea1a511e98a3a5033b0c2cb403fbb37f05dee6ac1fb85c0460d311ec278c", - "urls": [ - "https://files.pythonhosted.org/packages/87/7d/5ed859fad4b5eddd598a846150aaab2703730ed4886c5c5e03b0df0cfdd5/wrapt-2.5.0-py3-none-any.whl" - ] - } - }, - "ncore_pip_deps_313_wrapt_sdist_c48cdb6c": { - "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", - "attributes": { - "dep_template": "@ncore_pip_deps//{name}:{target}", - "experimental_target_platforms": [ - "cp313_linux_aarch64", - "cp313_linux_arm", - "cp313_linux_ppc", - "cp313_linux_s390x", - "cp313_linux_x86_64", - "cp313_osx_aarch64", - "cp313_osx_x86_64", - "cp313_windows_x86_64" - ], - "extra_pip_args": [ - "--index-url", - "https://pypi.org/simple", - "--find-links", - "https://download.pytorch.org/whl/cpu/torch" - ], - "filename": "wrapt-2.5.0.tar.gz", - "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", - "repo": "ncore_pip_deps_313", - "requirement": "wrapt==2.5.0", - "sha256": "c48cdb6c904dca76d9915a579e4a5fab6b0c25f650c1019ce78a78effaf7a345", - "urls": [ - "https://files.pythonhosted.org/packages/3e/d2/a254a26d8ceaea87e0eee2e89fcfe53ddc1858418647493bb2937549ab6f/wrapt-2.5.0.tar.gz" - ] - } - }, - "ncore_pip_deps_313_zarr_py3_none_any_ac3dc403": { + "ncore_pip_deps_313_zarr_py3_none_any_38b54057": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -23263,17 +23893,17 @@ "cp313_osx_x86_64", "cp313_windows_x86_64" ], - "filename": "zarr-2.18.7-py3-none-any.whl", + "filename": "zarr-3.4.1-py3-none-any.whl", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "zarr==2.18.7", - "sha256": "ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223", + "requirement": "zarr==3.4.1", + "sha256": "38b540578a119352bdce02a720d7bfd99846e77f0728c58b1bab3d1f547526a0", "urls": [ - "https://files.pythonhosted.org/packages/5e/d8/9ffd8c237b3559945bb52103cf0eed64ea098f7b7f573f8d2962ef27b4b2/zarr-2.18.7-py3-none-any.whl" + "https://files.pythonhosted.org/packages/a6/82/0dbc9bc77b49cfb9268dc2b2b1dcd04346c8c7ed52e272b76a628a941656/zarr-3.4.1-py3-none-any.whl" ] } }, - "ncore_pip_deps_313_zarr_sdist_b2b8f66f": { + "ncore_pip_deps_313_zarr_sdist_b34bda11": { "repoRuleId": "@@rules_python+//python/private/pypi:whl_library.bzl%whl_library", "attributes": { "dep_template": "@ncore_pip_deps//{name}:{target}", @@ -23293,13 +23923,13 @@ "--find-links", "https://download.pytorch.org/whl/cpu/torch" ], - "filename": "zarr-2.18.7.tar.gz", + "filename": "zarr-3.4.1.tar.gz", "python_interpreter_target": "@@rules_python++python+python_3_13_host//:python", "repo": "ncore_pip_deps_313", - "requirement": "zarr==2.18.7", - "sha256": "b2b8f66f14dac4af66b180d2338819981b981f70e196c9a66e6bfaa9e59572f5", + "requirement": "zarr==3.4.1", + "sha256": "b34bda11ceb199c81ee78ecd42cd02f46c7f67a6d8a1e9bc501cafd5a1795356", "urls": [ - "https://files.pythonhosted.org/packages/da/1d/01cf9e3ab2d85190278efc3fca9f68563de35ae30ee59e7640e3af98abe3/zarr-2.18.7.tar.gz" + "https://files.pythonhosted.org/packages/3e/62/e8a36a4b65f01499c7aefcf103aabba0c37c018bf135fbf179f6ed2f01a0/zarr-3.4.1.tar.gz" ] } }, @@ -27715,7 +28345,7 @@ "accessible_pygments": "{\"ncore_pip_deps_311_accessible_pygments_py3_none_any_88ae3211\":[{\"filename\":\"accessible_pygments-0.0.5-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_accessible_pygments_sdist_40918d3e\":[{\"filename\":\"accessible_pygments-0.0.5.tar.gz\",\"version\":\"3.11\"}]}", "alabaster": "{\"ncore_pip_deps_311_alabaster_py3_none_any_b46733c0\":[{\"filename\":\"alabaster-0.7.16-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_alabaster_sdist_75a8b99c\":[{\"filename\":\"alabaster-0.7.16.tar.gz\",\"version\":\"3.11\"}]}", "anyio": "{\"ncore_pip_deps_311_anyio_py3_none_any_d4058288\":[{\"filename\":\"anyio-4.12.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_anyio_sdist_41cfcc3a\":[{\"filename\":\"anyio-4.12.1.tar.gz\",\"version\":\"3.11\"}]}", - "asciitree": "{\"ncore_pip_deps_311_asciitree_sdist_4aa4b9b6\":[{\"filename\":\"asciitree-0.3.3.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_asciitree_sdist_4aa4b9b6\":[{\"filename\":\"asciitree-0.3.3.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_asciitree_sdist_4aa4b9b6\":[{\"filename\":\"asciitree-0.3.3.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_asciitree_sdist_4aa4b9b6\":[{\"filename\":\"asciitree-0.3.3.tar.gz\",\"version\":\"3.8\"}]}", + "asciitree": "{\"ncore_pip_deps_312_asciitree_sdist_4aa4b9b6\":[{\"filename\":\"asciitree-0.3.3.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_38_asciitree_sdist_4aa4b9b6\":[{\"filename\":\"asciitree-0.3.3.tar.gz\",\"version\":\"3.8\"}]}", "asttokens": "{\"ncore_pip_deps_311_asttokens_py3_none_any_15a3ebc0\":[{\"filename\":\"asttokens-3.0.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_asttokens_sdist_71a4ee5d\":[{\"filename\":\"asttokens-3.0.1.tar.gz\",\"version\":\"3.11\"}]}", "astunparse": "{\"ncore_pip_deps_311_astunparse_py2_none_any_c2652417\":[{\"filename\":\"astunparse-1.6.3-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_astunparse_sdist_5ad93a84\":[{\"filename\":\"astunparse-1.6.3.tar.gz\",\"version\":\"3.11\"}]}", "attrs": "{\"ncore_pip_deps_311_attrs_py3_none_any_c75a69e2\":[{\"filename\":\"attrs-25.1.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_attrs_sdist_1c97078a\":[{\"filename\":\"attrs-25.1.0.tar.gz\",\"version\":\"3.11\"}]}", @@ -27734,15 +28364,16 @@ "debugpy": "{\"ncore_pip_deps_311_debugpy_cp311_cp311_macosx_14_0_universal2_36f48298\":[{\"filename\":\"debugpy-1.8.12-cp311-cp311-macosx_14_0_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_debugpy_cp311_cp311_manylinux_2_5_x86_64_a28ed481\":[{\"filename\":\"debugpy-1.8.12-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_debugpy_cp311_cp311_win_amd64_4703575b\":[{\"filename\":\"debugpy-1.8.12-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_debugpy_py2_none_any_274b6a20\":[{\"filename\":\"debugpy-1.8.12-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_debugpy_sdist_646530b0\":[{\"filename\":\"debugpy-1.8.12.tar.gz\",\"version\":\"3.11\"}]}", "decorator": "{\"ncore_pip_deps_311_decorator_py3_none_any_d316bb41\":[{\"filename\":\"decorator-5.2.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_decorator_sdist_65f26614\":[{\"filename\":\"decorator-5.2.1.tar.gz\",\"version\":\"3.11\"}]}", "defusedxml": "{\"ncore_pip_deps_311_defusedxml_py2_none_any_a352e7e4\":[{\"filename\":\"defusedxml-0.7.1-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_defusedxml_sdist_1bb3032d\":[{\"filename\":\"defusedxml-0.7.1.tar.gz\",\"version\":\"3.11\"}]}", - "deprecated": "{\"ncore_pip_deps_311_deprecated_py2_none_any_bd501178\":[{\"filename\":\"Deprecated-1.2.18-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_deprecated_sdist_422b6f6d\":[{\"filename\":\"deprecated-1.2.18.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_deprecated_py3_none_any_58204cf4\":[{\"filename\":\"deprecated-3.0.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_deprecated_sdist_16850204\":[{\"filename\":\"deprecated-3.0.0.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_deprecated_py3_none_any_58204cf4\":[{\"filename\":\"deprecated-3.0.0-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_deprecated_sdist_16850204\":[{\"filename\":\"deprecated-3.0.0.tar.gz\",\"version\":\"3.13\"}]}", + "deprecated": "{\"ncore_pip_deps_311_deprecated_py2_none_any_bd501178\":[{\"filename\":\"Deprecated-1.2.18-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_deprecated_sdist_422b6f6d\":[{\"filename\":\"deprecated-1.2.18.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_deprecated_py3_none_any_58204cf4\":[{\"filename\":\"deprecated-3.0.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_deprecated_sdist_16850204\":[{\"filename\":\"deprecated-3.0.0.tar.gz\",\"version\":\"3.12\"}]}", "descartes": "{\"ncore_pip_deps_311_descartes_py2_none_any_b7e412e7\":[{\"filename\":\"descartes-1.1.0-py2-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_descartes_sdist_135a5021\":[{\"filename\":\"descartes-1.1.0.tar.gz\",\"version\":\"3.11\"}]}", "docutils": "{\"ncore_pip_deps_311_docutils_py3_none_any_d0013f54\":[{\"filename\":\"docutils-0.22.4-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_docutils_sdist_4db53b1f\":[{\"filename\":\"docutils-0.22.4.tar.gz\",\"version\":\"3.11\"}]}", + "donfig": "{\"ncore_pip_deps_311_donfig_py3_none_any_2a3175ce\":[{\"filename\":\"donfig-0.8.1.post1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_donfig_sdist_3bef3413\":[{\"filename\":\"donfig-0.8.1.post1.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_313_donfig_py3_none_any_2a3175ce\":[{\"filename\":\"donfig-0.8.1.post1-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_donfig_sdist_3bef3413\":[{\"filename\":\"donfig-0.8.1.post1.tar.gz\",\"version\":\"3.13\"}]}", "dracopy": "{\"ncore_pip_deps_311_dracopy_cp311_cp311_macosx_10_9_x86_64_c4131151\":[{\"filename\":\"dracopy-2.0.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_dracopy_cp311_cp311_macosx_11_0_arm64_e16575ab\":[{\"filename\":\"dracopy-2.0.0-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_dracopy_cp311_cp311_manylinux_2_24_aarch64_418964da\":[{\"filename\":\"dracopy-2.0.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_dracopy_cp311_cp311_manylinux_2_24_x86_64_fa9a3215\":[{\"filename\":\"dracopy-2.0.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_dracopy_cp311_cp311_win_amd64_106c968f\":[{\"filename\":\"dracopy-2.0.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}]}", "embreex": "{\"ncore_pip_deps_311_embreex_cp311_cp311_macosx_10_9_x86_64_8154ab34\":[{\"filename\":\"embreex-2.17.7.post7-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_embreex_cp311_cp311_manylinux_2_12_x86_64_c4775a4d\":[{\"filename\":\"embreex-2.17.7.post7-cp311-cp311-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_embreex_cp311_cp311_win_amd64_a1157976\":[{\"filename\":\"embreex-2.17.7.post7-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}]}", "entrypoints": "{\"ncore_pip_deps_38_entrypoints_py3_none_any_f174b5ff\":[{\"filename\":\"entrypoints-0.4-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_entrypoints_sdist_b706edda\":[{\"filename\":\"entrypoints-0.4.tar.gz\",\"version\":\"3.8\"}]}", "exceptiongroup": "{\"ncore_pip_deps_38_exceptiongroup_py3_none_any_34328066\":[{\"filename\":\"exceptiongroup-1.1.3-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_exceptiongroup_sdist_097acd85\":[{\"filename\":\"exceptiongroup-1.1.3.tar.gz\",\"version\":\"3.8\"}]}", "executing": "{\"ncore_pip_deps_311_executing_py2_none_any_11387150\":[{\"filename\":\"executing-2.2.0-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_executing_sdist_5d108c02\":[{\"filename\":\"executing-2.2.0.tar.gz\",\"version\":\"3.11\"}]}", - "fasteners": "{\"ncore_pip_deps_311_fasteners_py3_none_any_758819cb\":[{\"filename\":\"fasteners-0.19-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fasteners_sdist_b4f37c3a\":[{\"filename\":\"fasteners-0.19.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_fasteners_py3_none_any_9422c40d\":[{\"filename\":\"fasteners-0.20-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_fasteners_sdist_55dce879\":[{\"filename\":\"fasteners-0.20.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_fasteners_py3_none_any_9422c40d\":[{\"filename\":\"fasteners-0.20-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_fasteners_sdist_55dce879\":[{\"filename\":\"fasteners-0.20.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_fasteners_py3_none_any_1d4caf5f\":[{\"filename\":\"fasteners-0.18-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_fasteners_sdist_cb7c13ef\":[{\"filename\":\"fasteners-0.18.tar.gz\",\"version\":\"3.8\"}]}", + "fasteners": "{\"ncore_pip_deps_312_fasteners_py3_none_any_9422c40d\":[{\"filename\":\"fasteners-0.20-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_fasteners_sdist_55dce879\":[{\"filename\":\"fasteners-0.20.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_38_fasteners_py3_none_any_1d4caf5f\":[{\"filename\":\"fasteners-0.18-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_fasteners_sdist_cb7c13ef\":[{\"filename\":\"fasteners-0.18.tar.gz\",\"version\":\"3.8\"}]}", "fastjsonschema": "{\"ncore_pip_deps_311_fastjsonschema_py3_none_any_c9e5b7e9\":[{\"filename\":\"fastjsonschema-2.21.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fastjsonschema_sdist_794d4f0a\":[{\"filename\":\"fastjsonschema-2.21.1.tar.gz\",\"version\":\"3.11\"}]}", "filelock": "{\"ncore_pip_deps_311_filelock_py3_none_any_96f5f634\":[{\"filename\":\"filelock-3.29.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_filelock_sdist_69974355\":[{\"filename\":\"filelock-3.29.0.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_filelock_py3_none_any_2ce9818e\":[{\"filename\":\"filelock-4.1.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_filelock_sdist_ad7f724a\":[{\"filename\":\"filelock-4.1.0.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_filelock_py3_none_any_2ce9818e\":[{\"filename\":\"filelock-4.1.0-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_filelock_sdist_ad7f724a\":[{\"filename\":\"filelock-4.1.0.tar.gz\",\"version\":\"3.13\"}]}", "fire": "{\"ncore_pip_deps_311_fire_py3_none_any_e43fd8a5\":[{\"filename\":\"fire-0.7.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fire_sdist_3b208f05\":[{\"filename\":\"fire-0.7.1.tar.gz\",\"version\":\"3.11\"}]}", @@ -27750,6 +28381,7 @@ "fonttools": "{\"ncore_pip_deps_311_fonttools_cp311_cp311_macosx_10_9_universal2_7ef04bc7\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-macosx_10_9_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_cp311_cp311_macosx_10_9_x86_64_ffda9b8c\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_cp311_cp311_manylinux_2_17_aarch64_e2e993e8\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_cp311_cp311_manylinux_2_17_x86_64_003548ea\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_cp311_cp311_musllinux_1_2_aarch64_bd982582\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_cp311_cp311_musllinux_1_2_x86_64_b23d30a2\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_cp311_cp311_win_amd64_14a3e3e6\":[{\"filename\":\"fonttools-4.56.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_py3_none_any_1088182f\":[{\"filename\":\"fonttools-4.56.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fonttools_sdist_a114d156\":[{\"filename\":\"fonttools-4.56.0.tar.gz\",\"version\":\"3.11\"}]}", "fsspec": "{\"ncore_pip_deps_311_fsspec_py3_none_any_7c771235\":[{\"filename\":\"fsspec-2025.10.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_fsspec_sdist_b6789427\":[{\"filename\":\"fsspec-2025.10.0.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_fsspec_py3_none_any_8dd6e646\":[{\"filename\":\"fsspec-2026.9.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_fsspec_sdist_0f081479\":[{\"filename\":\"fsspec-2026.9.0.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_fsspec_py3_none_any_8dd6e646\":[{\"filename\":\"fsspec-2026.9.0-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_fsspec_sdist_0f081479\":[{\"filename\":\"fsspec-2026.9.0.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_fsspec_py3_none_any_efb87af3\":[{\"filename\":\"fsspec-2025.3.0-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_fsspec_sdist_a935fd1e\":[{\"filename\":\"fsspec-2025.3.0.tar.gz\",\"version\":\"3.8\"}]}", "gast": "{\"ncore_pip_deps_311_gast_py3_none_any_52b18231\":[{\"filename\":\"gast-0.6.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_gast_sdist_88fc5300\":[{\"filename\":\"gast-0.6.0.tar.gz\",\"version\":\"3.11\"}]}", + "google_crc32c": "{\"ncore_pip_deps_311_google_crc32c_cp311_cp311_macosx_12_0_arm64_457d0d9a\":[{\"filename\":\"google_crc32c-1.9.0-cp311-cp311-macosx_12_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_google_crc32c_cp311_cp311_macosx_12_0_x86_64_ccfe4002\":[{\"filename\":\"google_crc32c-1.9.0-cp311-cp311-macosx_12_0_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_google_crc32c_cp311_cp311_manylinux_2_17_aarch64_fbef61a3\":[{\"filename\":\"google_crc32c-1.9.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_google_crc32c_cp311_cp311_manylinux_2_17_x86_64_86764b99\":[{\"filename\":\"google_crc32c-1.9.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_google_crc32c_cp311_cp311_win_amd64_43a2dc26\":[{\"filename\":\"google_crc32c-1.9.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_google_crc32c_sdist_7b8c84c3\":[{\"filename\":\"google_crc32c-1.9.0.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_313_google_crc32c_cp313_cp313_macosx_12_0_arm64_f1dc17d9\":[{\"filename\":\"google_crc32c-1.9.0-cp313-cp313-macosx_12_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_google_crc32c_cp313_cp313_macosx_12_0_x86_64_f894a287\":[{\"filename\":\"google_crc32c-1.9.0-cp313-cp313-macosx_12_0_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_google_crc32c_cp313_cp313_manylinux_2_17_aarch64_4488f155\":[{\"filename\":\"google_crc32c-1.9.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_google_crc32c_cp313_cp313_manylinux_2_17_x86_64_0568b17e\":[{\"filename\":\"google_crc32c-1.9.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_google_crc32c_cp313_cp313_win_amd64_8583ec21\":[{\"filename\":\"google_crc32c-1.9.0-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_google_crc32c_sdist_7b8c84c3\":[{\"filename\":\"google_crc32c-1.9.0.tar.gz\",\"version\":\"3.13\"}]}", "google_pasta": "{\"ncore_pip_deps_311_google_pasta_py3_none_any_b3248279\":[{\"filename\":\"google_pasta-0.2.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_google_pasta_sdist_c9f2c8df\":[{\"filename\":\"google-pasta-0.2.0.tar.gz\",\"version\":\"3.11\"}]}", "grpcio": "{\"ncore_pip_deps_311_grpcio_cp311_cp311_linux_armv7l_17325b0b\":[{\"filename\":\"grpcio-1.70.0-cp311-cp311-linux_armv7l.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_grpcio_cp311_cp311_macosx_10_14_universal2_dbe41ad1\":[{\"filename\":\"grpcio-1.70.0-cp311-cp311-macosx_10_14_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_grpcio_cp311_cp311_manylinux_2_17_aarch64_5ea67c72\":[{\"filename\":\"grpcio-1.70.0-cp311-cp311-manylinux_2_17_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_grpcio_cp311_cp311_manylinux_2_17_x86_64_e7831a0f\":[{\"filename\":\"grpcio-1.70.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_grpcio_cp311_cp311_musllinux_1_1_x86_64_d6376496\":[{\"filename\":\"grpcio-1.70.0-cp311-cp311-musllinux_1_1_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_grpcio_cp311_cp311_win_amd64_d24035d4\":[{\"filename\":\"grpcio-1.70.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_grpcio_sdist_8d1584a6\":[{\"filename\":\"grpcio-1.70.0.tar.gz\",\"version\":\"3.11\"}]}", "h11": "{\"ncore_pip_deps_311_h11_py3_none_any_63cf8bbe\":[{\"filename\":\"h11-0.16.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_h11_sdist_4e35b956\":[{\"filename\":\"h11-0.16.0.tar.gz\",\"version\":\"3.11\"}]}", @@ -27786,7 +28418,7 @@ "mistune": "{\"ncore_pip_deps_311_mistune_py2_none_any_bad7f5d4\":[{\"filename\":\"mistune-2.0.5-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_mistune_sdist_0246113c\":[{\"filename\":\"mistune-2.0.5.tar.gz\",\"version\":\"3.11\"}]}", "ml_dtypes": "{\"ncore_pip_deps_311_ml_dtypes_cp311_cp311_macosx_10_9_universal2_6c7ecb74\":[{\"filename\":\"ml_dtypes-0.5.4-cp311-cp311-macosx_10_9_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_ml_dtypes_cp311_cp311_manylinux_2_27_aarch64_bc11d7e8\":[{\"filename\":\"ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_ml_dtypes_cp311_cp311_manylinux_2_27_x86_64_19b9a535\":[{\"filename\":\"ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_ml_dtypes_cp311_cp311_win_amd64_7c23c54a\":[{\"filename\":\"ml_dtypes-0.5.4-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_ml_dtypes_sdist_8ab06a50\":[{\"filename\":\"ml_dtypes-0.5.4.tar.gz\",\"version\":\"3.11\"}]}", "mpmath": "{\"ncore_pip_deps_311_mpmath_py3_none_any_a0b2b9fe\":[{\"filename\":\"mpmath-1.3.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_mpmath_sdist_7a28eb2a\":[{\"filename\":\"mpmath-1.3.0.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_mpmath_py3_none_any_a0b2b9fe\":[{\"filename\":\"mpmath-1.3.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_mpmath_sdist_7a28eb2a\":[{\"filename\":\"mpmath-1.3.0.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_mpmath_py3_none_any_a0b2b9fe\":[{\"filename\":\"mpmath-1.3.0-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_mpmath_sdist_7a28eb2a\":[{\"filename\":\"mpmath-1.3.0.tar.gz\",\"version\":\"3.13\"}]}", - "msgspec": "{\"ncore_pip_deps_311_msgspec_cp311_cp311_macosx_10_9_x86_64_09e0efbf\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_macosx_11_0_arm64_23ee3787\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_manylinux_2_17_aarch64_81f4ac6f\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_manylinux_2_17_x86_64_bb4d873f\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_musllinux_1_2_aarch64_b92b8334\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_musllinux_1_2_x86_64_562c44b0\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_win_amd64_d1dcc93a\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_sdist_692349e5\":[{\"filename\":\"msgspec-0.20.0.tar.gz\",\"version\":\"3.11\"}]}", + "msgspec": "{\"ncore_pip_deps_311_msgspec_cp311_cp311_macosx_10_9_x86_64_09e0efbf\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_macosx_11_0_arm64_23ee3787\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_manylinux_2_17_aarch64_81f4ac6f\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_manylinux_2_17_x86_64_bb4d873f\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_musllinux_1_2_aarch64_b92b8334\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_musllinux_1_2_x86_64_562c44b0\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_cp311_cp311_win_amd64_d1dcc93a\":[{\"filename\":\"msgspec-0.20.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_msgspec_sdist_692349e5\":[{\"filename\":\"msgspec-0.20.0.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_macosx_10_13_x86_64_f13c127a\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-macosx_10_13_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_macosx_11_0_arm64_5aa24eb4\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-macosx_11_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_manylinux_2_17_aarch64_627bfdfe\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_manylinux_2_17_x86_64_c6c310ef\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_musllinux_1_2_aarch64_bc374ded\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-musllinux_1_2_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_musllinux_1_2_x86_64_6f48317f\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-musllinux_1_2_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_cp313_cp313_win_amd64_0739b068\":[{\"filename\":\"msgspec-0.22.0-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_msgspec_sdist_0a13624a\":[{\"filename\":\"msgspec-0.22.0.tar.gz\",\"version\":\"3.13\"}]}", "mypy_extensions": "{\"ncore_pip_deps_311_mypy_extensions_py3_none_any_4392f6c0\":[{\"filename\":\"mypy_extensions-1.0.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_mypy_extensions_sdist_75dbf895\":[{\"filename\":\"mypy_extensions-1.0.0.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_mypy_extensions_py3_none_any_1be4cccd\":[{\"filename\":\"mypy_extensions-1.1.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_mypy_extensions_sdist_52e68efc\":[{\"filename\":\"mypy_extensions-1.1.0.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_mypy_extensions_py3_none_any_1be4cccd\":[{\"filename\":\"mypy_extensions-1.1.0-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_mypy_extensions_sdist_52e68efc\":[{\"filename\":\"mypy_extensions-1.1.0.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_mypy_extensions_py3_none_any_4392f6c0\":[{\"filename\":\"mypy_extensions-1.0.0-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_mypy_extensions_sdist_75dbf895\":[{\"filename\":\"mypy_extensions-1.0.0.tar.gz\",\"version\":\"3.8\"}]}", "namex": "{\"ncore_pip_deps_311_namex_py3_none_any_7ddb6c2b\":[{\"filename\":\"namex-0.0.8-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_namex_sdist_32a50f6c\":[{\"filename\":\"namex-0.0.8.tar.gz\",\"version\":\"3.11\"}]}", "nbclient": "{\"ncore_pip_deps_311_nbclient_py3_none_any_4ffee11e\":[{\"filename\":\"nbclient-0.10.2-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_nbclient_sdist_90b7fc6b\":[{\"filename\":\"nbclient-0.10.2.tar.gz\",\"version\":\"3.11\"}]}", @@ -27794,7 +28426,7 @@ "nbformat": "{\"ncore_pip_deps_311_nbformat_py3_none_any_3b48d6c8\":[{\"filename\":\"nbformat-5.10.4-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_nbformat_sdist_322168b1\":[{\"filename\":\"nbformat-5.10.4.tar.gz\",\"version\":\"3.11\"}]}", "nbsphinx": "{\"ncore_pip_deps_311_nbsphinx_py3_none_any_6e805e96\":[{\"filename\":\"nbsphinx-0.9.3-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_nbsphinx_sdist_ec339c86\":[{\"filename\":\"nbsphinx-0.9.3.tar.gz\",\"version\":\"3.11\"}]}", "networkx": "{\"ncore_pip_deps_311_networkx_py3_none_any_d47fbf30\":[{\"filename\":\"networkx-3.6.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_networkx_sdist_26b7c357\":[{\"filename\":\"networkx-3.6.1.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_networkx_py3_none_any_e3fd2c13\":[{\"filename\":\"networkx-3.7-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_networkx_sdist_fd77a511\":[{\"filename\":\"networkx-3.7.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_networkx_py3_none_any_e3fd2c13\":[{\"filename\":\"networkx-3.7-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_networkx_sdist_fd77a511\":[{\"filename\":\"networkx-3.7.tar.gz\",\"version\":\"3.13\"}]}", - "numcodecs": "{\"ncore_pip_deps_311_numcodecs_cp311_cp311_macosx_10_9_x86_64_698f1d59\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_cp311_cp311_macosx_11_0_arm64_bef8c8e6\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_cp311_cp311_manylinux_2_17_x86_64_cdfaef9f\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_cp311_cp311_win_amd64_e2547fa3\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_sdist_eeed77e4\":[{\"filename\":\"numcodecs-0.15.1.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_macosx_10_13_x86_64_b0a9d9cd\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-macosx_10_13_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_macosx_11_0_arm64_a34f0fe5\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-macosx_11_0_arm64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_manylinux_2_17_x86_64_c3a09e22\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_win_amd64_daed6066\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-win_amd64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_sdist_eeed77e4\":[{\"filename\":\"numcodecs-0.15.1.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_10_13_x86_64_e3d82b70\":[{\"filename\":\"numcodecs-0.15.1-cp313-cp313-macosx_10_13_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_11_0_arm64_1d471a18\":[{\"filename\":\"numcodecs-0.15.1-cp313-cp313-macosx_11_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_manylinux_2_17_x86_64_1dfdea4a\":[{\"filename\":\"numcodecs-0.15.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_win_amd64_a4f7bdb2\":[{\"filename\":\"numcodecs-0.15.1-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_sdist_eeed77e4\":[{\"filename\":\"numcodecs-0.15.1.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_numcodecs_cp38_cp38_macosx_10_9_x86_64_0c240858\":[{\"filename\":\"numcodecs-0.11.0-cp38-cp38-macosx_10_9_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numcodecs_cp38_cp38_manylinux_2_17_x86_64_ee5bda16\":[{\"filename\":\"numcodecs-0.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numcodecs_cp38_cp38_win_amd64_bd05cdb8\":[{\"filename\":\"numcodecs-0.11.0-cp38-cp38-win_amd64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numcodecs_sdist_6c058b32\":[{\"filename\":\"numcodecs-0.11.0.tar.gz\",\"version\":\"3.8\"}]}", + "numcodecs": "{\"ncore_pip_deps_311_numcodecs_cp311_cp311_macosx_10_9_x86_64_698f1d59\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_cp311_cp311_macosx_11_0_arm64_bef8c8e6\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_cp311_cp311_manylinux_2_17_x86_64_cdfaef9f\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_cp311_cp311_win_amd64_e2547fa3\":[{\"filename\":\"numcodecs-0.15.1-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numcodecs_sdist_eeed77e4\":[{\"filename\":\"numcodecs-0.15.1.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_macosx_10_13_x86_64_b0a9d9cd\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-macosx_10_13_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_macosx_11_0_arm64_a34f0fe5\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-macosx_11_0_arm64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_manylinux_2_17_x86_64_c3a09e22\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_cp312_cp312_win_amd64_daed6066\":[{\"filename\":\"numcodecs-0.15.1-cp312-cp312-win_amd64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numcodecs_sdist_eeed77e4\":[{\"filename\":\"numcodecs-0.15.1.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_10_13_x86_64_de2c66db\":[{\"filename\":\"numcodecs-0.17.0-cp313-cp313-macosx_10_13_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_macosx_11_0_arm64_69b9b468\":[{\"filename\":\"numcodecs-0.17.0-cp313-cp313-macosx_11_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_manylinux_2_17_aarch64_7065b334\":[{\"filename\":\"numcodecs-0.17.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_manylinux_2_17_x86_64_6c3342d9\":[{\"filename\":\"numcodecs-0.17.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_cp313_cp313_win_amd64_a854e9c8\":[{\"filename\":\"numcodecs-0.17.0-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numcodecs_sdist_e8db2e33\":[{\"filename\":\"numcodecs-0.17.0.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_numcodecs_cp38_cp38_macosx_10_9_x86_64_0c240858\":[{\"filename\":\"numcodecs-0.11.0-cp38-cp38-macosx_10_9_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numcodecs_cp38_cp38_manylinux_2_17_x86_64_ee5bda16\":[{\"filename\":\"numcodecs-0.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numcodecs_cp38_cp38_win_amd64_bd05cdb8\":[{\"filename\":\"numcodecs-0.11.0-cp38-cp38-win_amd64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numcodecs_sdist_6c058b32\":[{\"filename\":\"numcodecs-0.11.0.tar.gz\",\"version\":\"3.8\"}]}", "numpy": "{\"ncore_pip_deps_311_numpy_cp311_cp311_macosx_10_9_x86_64_4c66707f\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_cp311_cp311_macosx_11_0_arm64_edd8b5fe\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_cp311_cp311_manylinux_2_17_aarch64_7ab55401\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_cp311_cp311_manylinux_2_17_x86_64_666dbfb6\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_cp311_cp311_musllinux_1_1_aarch64_96ff0b2a\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_cp311_cp311_musllinux_1_1_x86_64_60dedbb9\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_cp311_cp311_win_amd64_cd25bcec\":[{\"filename\":\"numpy-1.26.4-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_numpy_sdist_2a02aba9\":[{\"filename\":\"numpy-1.26.4.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_macosx_10_13_x86_64_c6342f54\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-macosx_10_13_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_macosx_11_0_arm64_b11e8fda\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-macosx_11_0_arm64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_macosx_14_0_arm64_9cb18a32\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-macosx_14_0_arm64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_macosx_14_0_x86_64_aec3fc4b\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-macosx_14_0_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_manylinux_2_27_aarch64_fe4d21ab\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_manylinux_2_27_x86_64_fbde6962\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_musllinux_1_2_aarch64_381a7a3d\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-musllinux_1_2_aarch64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_musllinux_1_2_x86_64_b89d0aaa\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-musllinux_1_2_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_cp312_cp312_win_amd64_a7b1b635\":[{\"filename\":\"numpy-2.5.4-cp312-cp312-win_amd64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_numpy_sdist_9a94cf75\":[{\"filename\":\"numpy-2.5.4.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_macosx_10_13_x86_64_2377da2d\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-macosx_10_13_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_macosx_11_0_arm64_7415db95\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-macosx_11_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_macosx_14_0_arm64_6d6a71b9\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-macosx_14_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_macosx_14_0_x86_64_d8200f16\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-macosx_14_0_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_manylinux_2_27_aarch64_1c2e71b0\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_manylinux_2_27_x86_64_6ffa0766\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_musllinux_1_2_aarch64_2fa3328f\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-musllinux_1_2_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_musllinux_1_2_x86_64_b86966fb\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-musllinux_1_2_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_cp313_cp313_win_amd64_8b4d2fd2\":[{\"filename\":\"numpy-2.5.4-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_numpy_sdist_9a94cf75\":[{\"filename\":\"numpy-2.5.4.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_numpy_cp38_cp38_macosx_10_9_x86_64_cf240200\":[{\"filename\":\"numpy-1.19.5-cp38-cp38-macosx_10_9_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numpy_cp38_cp38_manylinux_2_12_x86_64_a9d17f2b\":[{\"filename\":\"numpy-1.19.5-cp38-cp38-manylinux2010_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numpy_cp38_cp38_manylinux_2_17_aarch64_99abf4f3\":[{\"filename\":\"numpy-1.19.5-cp38-cp38-manylinux2014_aarch64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numpy_cp38_cp38_manylinux_2_5_x86_64_012426a4\":[{\"filename\":\"numpy-1.19.5-cp38-cp38-manylinux1_x86_64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numpy_cp38_cp38_win_amd64_811daee3\":[{\"filename\":\"numpy-1.19.5-cp38-cp38-win_amd64.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_numpy_sdist_a76f5024\":[{\"filename\":\"numpy-1.19.5.zip\",\"version\":\"3.8\"}]}", "nuscenes_devkit": "{\"ncore_pip_deps_311_nuscenes_devkit_py3_none_any_76cee0e7\":[{\"filename\":\"nuscenes_devkit-1.2.0-py3-none-any.whl\",\"version\":\"3.11\"}]}", "nvidia_cublas_cu12": "{\"ncore_pip_deps_311_nvidia_cublas_cu12_py3_none_manylinux_2_27_aarch64_93a4e0e3\":[{\"filename\":\"nvidia_cublas_cu12-12.8.3.14-py3-none-manylinux_2_27_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_nvidia_cublas_cu12_py3_none_manylinux_2_27_x86_64_3f0e05e7\":[{\"filename\":\"nvidia_cublas_cu12-12.8.3.14-py3-none-manylinux_2_27_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_nvidia_cublas_cu12_py3_none_win_amd64_9ae5eae5\":[{\"filename\":\"nvidia_cublas_cu12-12.8.3.14-py3-none-win_amd64.whl\",\"version\":\"3.11\"}]}", @@ -27841,6 +28473,7 @@ "pytest": "{\"ncore_pip_deps_311_pytest_py3_none_any_50e16d95\":[{\"filename\":\"pytest-8.3.4-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pytest_sdist_965370d0\":[{\"filename\":\"pytest-8.3.4.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_pytest_py3_none_any_37a86b45\":[{\"filename\":\"pytest-9.1.1-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_pytest_sdist_1088fbde\":[{\"filename\":\"pytest-9.1.1.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_pytest_py3_none_any_37a86b45\":[{\"filename\":\"pytest-9.1.1-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pytest_sdist_1088fbde\":[{\"filename\":\"pytest-9.1.1.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_pytest_py3_none_any_78bf1645\":[{\"filename\":\"pytest-7.4.0-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_pytest_sdist_b4bf8c45\":[{\"filename\":\"pytest-7.4.0.tar.gz\",\"version\":\"3.8\"}]}", "python_dateutil": "{\"ncore_pip_deps_311_python_dateutil_py2_none_any_a8b2bc7b\":[{\"filename\":\"python_dateutil-2.9.0.post0-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_python_dateutil_sdist_37dd5420\":[{\"filename\":\"python-dateutil-2.9.0.post0.tar.gz\",\"version\":\"3.11\"}]}", "pytz": "{\"ncore_pip_deps_311_pytz_py2_none_any_89dd22dc\":[{\"filename\":\"pytz-2025.1-py2.py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pytz_sdist_c2db42be\":[{\"filename\":\"pytz-2025.1.tar.gz\",\"version\":\"3.11\"}]}", + "pyyaml": "{\"ncore_pip_deps_311_pyyaml_cp311_cp311_macosx_10_13_x86_64_44edc647\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_macosx_11_0_arm64_652cb6ed\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_manylinux_2_17_aarch64_10892704\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_manylinux_2_17_s390x_850774a7\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_manylinux_2_17_x86_64_b8bb0864\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_musllinux_1_2_aarch64_1d37d57a\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_musllinux_1_2_x86_64_37503bfb\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_cp311_cp311_win_amd64_9f3bfb49\":[{\"filename\":\"pyyaml-6.0.3-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyyaml_sdist_d7662337\":[{\"filename\":\"pyyaml-6.0.3.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_macosx_10_13_x86_64_8da9669d\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_macosx_11_0_arm64_2283a07e\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_manylinux_2_17_aarch64_ee292290\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_manylinux_2_17_s390x_a33284e2\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_manylinux_2_17_x86_64_0f29edc4\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_musllinux_1_2_aarch64_f7057c9a\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_musllinux_1_2_x86_64_eda16858\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_cp313_cp313_win_amd64_79005a0d\":[{\"filename\":\"pyyaml-6.0.3-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_pyyaml_sdist_d7662337\":[{\"filename\":\"pyyaml-6.0.3.tar.gz\",\"version\":\"3.13\"}]}", "pyzmq": "{\"ncore_pip_deps_311_pyzmq_cp311_cp311_macosx_10_15_universal2_c0598838\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-macosx_10_15_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_macosx_10_9_x86_64_ed038a92\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_17_aarch64_9027a7fc\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_17_x86_64_f0019cc8\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_manylinux_2_28_x86_64_f19dae58\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_musllinux_1_1_aarch64_f5eeeb82\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-musllinux_1_1_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_musllinux_1_1_x86_64_817fcd33\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-musllinux_1_1_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_cp311_cp311_win_amd64_ef29630f\":[{\"filename\":\"pyzmq-26.2.1-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_pyzmq_sdist_17d72a74\":[{\"filename\":\"pyzmq-26.2.1.tar.gz\",\"version\":\"3.11\"}]}", "referencing": "{\"ncore_pip_deps_311_referencing_py3_none_any_d53ae300\":[{\"filename\":\"referencing-0.34.0-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_referencing_sdist_5773bd84\":[{\"filename\":\"referencing-0.34.0.tar.gz\",\"version\":\"3.11\"}]}", "remotezip": "{\"ncore_pip_deps_311_remotezip_py3_none_any_f70a4026\":[{\"filename\":\"remotezip-0.12.3-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_remotezip_sdist_bf1ebe2b\":[{\"filename\":\"remotezip-0.12.3.tar.gz\",\"version\":\"3.11\"}]}", @@ -27892,10 +28525,10 @@ "websockets": "{\"ncore_pip_deps_311_websockets_cp311_cp311_macosx_10_9_universal2_823c248b\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-macosx_10_9_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_macosx_10_9_x86_64_67899970\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_macosx_11_0_arm64_d50fd1ee\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_manylinux_2_17_aarch64_d99e5546\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_manylinux_2_5_x86_64_8dd8327c\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_musllinux_1_2_aarch64_8fdc5105\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_musllinux_1_2_x86_64_54479983\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_cp311_cp311_win_amd64_27ccee00\":[{\"filename\":\"websockets-15.0.1-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_py3_none_any_f7a866fb\":[{\"filename\":\"websockets-15.0.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_websockets_sdist_82544de0\":[{\"filename\":\"websockets-15.0.1.tar.gz\",\"version\":\"3.11\"}]}", "werkzeug": "{\"ncore_pip_deps_311_werkzeug_py3_none_any_63a77fb8\":[{\"filename\":\"werkzeug-3.1.8-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_werkzeug_sdist_9bad61a4\":[{\"filename\":\"werkzeug-3.1.8.tar.gz\",\"version\":\"3.11\"}]}", "wheel": "{\"ncore_pip_deps_311_wheel_py3_none_any_708e7481\":[{\"filename\":\"wheel-0.45.1-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wheel_sdist_661e1abd\":[{\"filename\":\"wheel-0.45.1.tar.gz\",\"version\":\"3.11\"}]}", - "wrapt": "{\"ncore_pip_deps_311_wrapt_cp311_cp311_macosx_10_9_universal2_ff04ef6e\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-macosx_10_9_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_macosx_10_9_x86_64_4db983e7\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_macosx_11_0_arm64_9abc77a4\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_manylinux_2_17_aarch64_0b929ac1\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_manylinux_2_5_x86_64_1a7ed2d9\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_musllinux_1_2_aarch64_129a150f\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_musllinux_1_2_x86_64_9a2bce78\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_win_amd64_acc130bc\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_py3_none_any_b18f2d15\":[{\"filename\":\"wrapt-1.17.2-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_sdist_41388e9d\":[{\"filename\":\"wrapt-1.17.2.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_macosx_10_13_x86_64_b898caea\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-macosx_10_13_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_macosx_11_0_arm64_8837fbe7\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-macosx_11_0_arm64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_manylinux_2_17_aarch64_6761765c\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_manylinux_2_5_x86_64_0cabb9c1\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_musllinux_1_2_aarch64_6a9ee62a\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-musllinux_1_2_aarch64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_musllinux_1_2_x86_64_e716f47c\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-musllinux_1_2_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_win_amd64_ab45839c\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-win_amd64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_py3_none_any_107eea1a\":[{\"filename\":\"wrapt-2.5.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_sdist_c48cdb6c\":[{\"filename\":\"wrapt-2.5.0.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_macosx_10_13_x86_64_b312b3cc\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-macosx_10_13_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_macosx_11_0_arm64_c57ddae2\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-macosx_11_0_arm64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_manylinux_2_17_aarch64_6058e12e\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_manylinux_2_5_x86_64_b95a6eca\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_musllinux_1_2_aarch64_f4ef4935\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-musllinux_1_2_aarch64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_musllinux_1_2_x86_64_a346408f\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-musllinux_1_2_x86_64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_cp313_cp313_win_amd64_77f0a74f\":[{\"filename\":\"wrapt-2.5.0-cp313-cp313-win_amd64.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_py3_none_any_107eea1a\":[{\"filename\":\"wrapt-2.5.0-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_wrapt_sdist_c48cdb6c\":[{\"filename\":\"wrapt-2.5.0.tar.gz\",\"version\":\"3.13\"}]}", + "wrapt": "{\"ncore_pip_deps_311_wrapt_cp311_cp311_macosx_10_9_universal2_ff04ef6e\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-macosx_10_9_universal2.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_macosx_10_9_x86_64_4db983e7\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_macosx_11_0_arm64_9abc77a4\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_manylinux_2_17_aarch64_0b929ac1\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_manylinux_2_5_x86_64_1a7ed2d9\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_musllinux_1_2_aarch64_129a150f\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_musllinux_1_2_x86_64_9a2bce78\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_cp311_cp311_win_amd64_acc130bc\":[{\"filename\":\"wrapt-1.17.2-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_py3_none_any_b18f2d15\":[{\"filename\":\"wrapt-1.17.2-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_wrapt_sdist_41388e9d\":[{\"filename\":\"wrapt-1.17.2.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_macosx_10_13_x86_64_b898caea\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-macosx_10_13_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_macosx_11_0_arm64_8837fbe7\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-macosx_11_0_arm64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_manylinux_2_17_aarch64_6761765c\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_manylinux_2_5_x86_64_0cabb9c1\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_musllinux_1_2_aarch64_6a9ee62a\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-musllinux_1_2_aarch64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_musllinux_1_2_x86_64_e716f47c\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-musllinux_1_2_x86_64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_cp312_cp312_win_amd64_ab45839c\":[{\"filename\":\"wrapt-2.5.0-cp312-cp312-win_amd64.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_py3_none_any_107eea1a\":[{\"filename\":\"wrapt-2.5.0-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_wrapt_sdist_c48cdb6c\":[{\"filename\":\"wrapt-2.5.0.tar.gz\",\"version\":\"3.12\"}]}", "xxhash": "{\"ncore_pip_deps_311_xxhash_cp311_cp311_macosx_10_9_x86_64_b47bbd8c\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_macosx_11_0_arm64_2b6821e9\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_manylinux_2_17_aarch64_8b29ee68\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_manylinux_2_17_ppc64le_6812c25f\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_manylinux_2_17_s390x_4ccbff01\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_manylinux_2_17_x86_64_297b7fbf\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_musllinux_1_2_aarch64_dea26ae1\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_musllinux_1_2_ppc64le_08d45aef\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-musllinux_1_2_ppc64le.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_musllinux_1_2_s390x_92914236\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-musllinux_1_2_s390x.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_musllinux_1_2_x86_64_51312c76\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_cp311_cp311_win_amd64_26734cdc\":[{\"filename\":\"xxhash-3.6.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_xxhash_sdist_f0162a78\":[{\"filename\":\"xxhash-3.6.0.tar.gz\",\"version\":\"3.11\"}]}", "yourdfpy": "{\"ncore_pip_deps_311_yourdfpy_py3_none_any_8a187a8b\":[{\"filename\":\"yourdfpy-0.0.60-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_yourdfpy_sdist_2af2d8bd\":[{\"filename\":\"yourdfpy-0.0.60.tar.gz\",\"version\":\"3.11\"}]}", - "zarr": "{\"ncore_pip_deps_311_zarr_py3_none_any_2795e20a\":[{\"filename\":\"zarr-2.18.4-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zarr_sdist_37790ede\":[{\"filename\":\"zarr-2.18.4.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_zarr_py3_none_any_ac3dc403\":[{\"filename\":\"zarr-2.18.7-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_zarr_sdist_b2b8f66f\":[{\"filename\":\"zarr-2.18.7.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_zarr_py3_none_any_ac3dc403\":[{\"filename\":\"zarr-2.18.7-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_zarr_sdist_b2b8f66f\":[{\"filename\":\"zarr-2.18.7.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_zarr_py3_none_any_883305e8\":[{\"filename\":\"zarr-2.13.3-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_zarr_sdist_db24b090\":[{\"filename\":\"zarr-2.13.3.tar.gz\",\"version\":\"3.8\"}]}", + "zarr": "{\"ncore_pip_deps_311_zarr_py3_none_any_72c3dcee\":[{\"filename\":\"zarr-3.1.4-py3-none-any.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zarr_sdist_b0cf4eea\":[{\"filename\":\"zarr-3.1.4.tar.gz\",\"version\":\"3.11\"}],\"ncore_pip_deps_312_zarr_py3_none_any_ac3dc403\":[{\"filename\":\"zarr-2.18.7-py3-none-any.whl\",\"version\":\"3.12\"}],\"ncore_pip_deps_312_zarr_sdist_b2b8f66f\":[{\"filename\":\"zarr-2.18.7.tar.gz\",\"version\":\"3.12\"}],\"ncore_pip_deps_313_zarr_py3_none_any_38b54057\":[{\"filename\":\"zarr-3.4.1-py3-none-any.whl\",\"version\":\"3.13\"}],\"ncore_pip_deps_313_zarr_sdist_b34bda11\":[{\"filename\":\"zarr-3.4.1.tar.gz\",\"version\":\"3.13\"}],\"ncore_pip_deps_38_zarr_py3_none_any_883305e8\":[{\"filename\":\"zarr-2.13.3-py3-none-any.whl\",\"version\":\"3.8\"}],\"ncore_pip_deps_38_zarr_sdist_db24b090\":[{\"filename\":\"zarr-2.13.3.tar.gz\",\"version\":\"3.8\"}]}", "zstandard": "{\"ncore_pip_deps_311_zstandard_cp311_cp311_macosx_10_9_x86_64_933b65d7\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-macosx_10_9_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_macosx_11_0_arm64_a3f79487\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-macosx_11_0_arm64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_manylinux_2_17_aarch64_01582723\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_manylinux_2_17_ppc64le_5f1ad7bf\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_manylinux_2_17_s390x_06acb75e\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_manylinux_2_17_x86_64_9300d02e\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_musllinux_1_1_aarch64_bfd06b1c\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-musllinux_1_1_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_musllinux_1_1_x86_64_f373da2c\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-musllinux_1_1_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_musllinux_1_2_aarch64_6c0e5a65\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-musllinux_1_2_aarch64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_musllinux_1_2_ppc64le_98750a30\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-musllinux_1_2_ppc64le.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_musllinux_1_2_s390x_22a086cf\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-musllinux_1_2_s390x.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_musllinux_1_2_x86_64_72d35d7a\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-musllinux_1_2_x86_64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_cp311_cp311_win_amd64_daab68fa\":[{\"filename\":\"zstandard-0.25.0-cp311-cp311-win_amd64.whl\",\"version\":\"3.11\"}],\"ncore_pip_deps_311_zstandard_sdist_7713e117\":[{\"filename\":\"zstandard-0.25.0.tar.gz\",\"version\":\"3.11\"}]}" }, "packages": [ @@ -27925,6 +28558,7 @@ "deprecated", "descartes", "docutils", + "donfig", "dracopy", "embreex", "entrypoints", @@ -27938,6 +28572,7 @@ "fonttools", "fsspec", "gast", + "google_crc32c", "google_pasta", "grpcio", "h11", @@ -28029,6 +28664,7 @@ "pytest", "python_dateutil", "pytz", + "pyyaml", "pyzmq", "referencing", "remotezip", diff --git a/deps/pip/requirements_3_11.in b/deps/pip/requirements_3_11.in index b31bf737..ba8ade94 100644 --- a/deps/pip/requirements_3_11.in +++ b/deps/pip/requirements_3_11.in @@ -26,6 +26,7 @@ # Public API restrictions for 3.11 # torch 2.7+ supports numpy 2.x natively, no constraint needed +zarr==3.1.4 # oldest supported zarr-python 3 (most recent zarr-python 3 is covered by python 3.13) --find-links https://download.pytorch.org/whl/cu128/torch torch==2.7.0+cu128 diff --git a/deps/pip/requirements_3_11.txt b/deps/pip/requirements_3_11.txt index d620e757..19e1ac35 100644 --- a/deps/pip/requirements_3_11.txt +++ b/deps/pip/requirements_3_11.txt @@ -22,9 +22,6 @@ anyio==4.12.1 \ --hash=sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703 \ --hash=sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c # via httpx -asciitree==0.3.3 \ - --hash=sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e - # via zarr asttokens==3.0.1 \ --hash=sha256:15a3ebc0f43c2d0a50eeafea25e19046c68398e487b9f1f5b517f7c0f40f976a \ --hash=sha256:71a4ee5de0bde6a31d64f6b13f2293ac190344478f081c3d1bccfcf5eacb0cb7 @@ -337,6 +334,10 @@ docutils==0.22.4 \ # nbsphinx # pydata-sphinx-theme # sphinx +donfig==0.8.1.post1 \ + --hash=sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d \ + --hash=sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52 + # via zarr dracopy==2.0.0 \ --hash=sha256:00ed24e138bcfbf2605d495c3a4e6c17b71cbd0b63b78e945d9d3c89d4fb9d69 \ --hash=sha256:05b5d4054402877f50221bc917e2943214a956d8d746e01b21f57ecc2d0adf21 \ @@ -416,10 +417,6 @@ executing==2.2.0 \ --hash=sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa \ --hash=sha256:5d108c028108fe2551d1a7b2e8b713341e2cb4fc0aa7dcf966fa4327a5226755 # via stack-data -fasteners==0.19 \ - --hash=sha256:758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237 \ - --hash=sha256:b4f37c3ac52d8a445af3a66bce57b33b5e90b97c696b7b984f530cf8f0ded09c - # via zarr fastjsonschema==2.21.1 \ --hash=sha256:794d4f0a58f848961ba16af7b9c85a3e88cd360df008c59aac6fc5ae9323b5d4 \ --hash=sha256:c9e5b7e908310918cf494a434eeb31384dd84a98b57a30bcb1f535015b554667 @@ -497,6 +494,39 @@ gast==0.6.0 \ --hash=sha256:52b182313f7330389f72b069ba00f174cfe2a06411099547288839c6cbafbd54 \ --hash=sha256:88fc5300d32c7ac6ca7b515310862f71e6fdf2c029bbec7c66c0f5dd47b6b1fb # via tensorflow +google-crc32c==1.9.0 \ + --hash=sha256:0568b17ed90ac596f29400d99e243fd0cc6276766183def888d1bf8d1dc13827 \ + --hash=sha256:0abe7e202c25909869c35672ab0f2fe748a7acf276eb78577332a7c38999740f \ + --hash=sha256:13179f7e3282617923e957b8e54b8f9c3968030f48640a9f47fd7c5c38c4a215 \ + --hash=sha256:16865b477d7941712cb0e0aad8ad4815e984fb5fc16d3fdaef7d986e26e53c95 \ + --hash=sha256:265233aff33d835f5b909584fe36ab29647b598c271b661a300001099109e53e \ + --hash=sha256:280f3a3e47af0eeba3a3e5aa7d311af77001812b8df80fb8beafcd0b40eaf7f1 \ + --hash=sha256:3abb18297d9ef0ab120531838be0e6d68c9fa876570e11c229c48f2edac23ce7 \ + --hash=sha256:43a2dc26f9be213fbe0b4fc4a1088c5d45cbfcb3247420ccc820f0fc3edeea86 \ + --hash=sha256:4488f1553a9ab7e86cdedc833374a7e904031803b995dc0bd0be48c271fa6556 \ + --hash=sha256:457d0d9a4718fd52b1494eac5c200ad25beeadbdc91843d550a003910838589f \ + --hash=sha256:51cb4e23a38ad4f495f35f87c233ca3ea6b9c4559e7ac383cdef786fab0f7977 \ + --hash=sha256:53fdafef58e230d0c946ab5f8446d123d9f548230a73b29c8b41c9546f268bc1 \ + --hash=sha256:56610f548f1b35c9568b9d1de30423480f505dae4991556072d5802820ff35c4 \ + --hash=sha256:5695c8b9327e040b2aba12c6659b0acb5995314ef0af0192da66e662e011103b \ + --hash=sha256:6a3b2c8a343c570ed8100a7627c20badfd92c6caa2067093a86be45af27f5b1b \ + --hash=sha256:7b8c84c3d159ab6817fe3f74e6e6cef099c3f95dcec3abc0d8afb1404642efbe \ + --hash=sha256:8535e75dfead304f30e9122b9ea2c0a570dbaa52c176a0a591540c7914c1e46d \ + --hash=sha256:8583ec21d56b565d68ab2963cc7e21b3b271247c29b04286068255ef65f221bd \ + --hash=sha256:86764b99e7a607830d93cb5b75e0ec3ff6cb06d3c274624418473cee701900d4 \ + --hash=sha256:8b91f41645b15a720357183fa5716682ada441873e3c462c15f9714be36f146b \ + --hash=sha256:af73200fa9791ccd380f3598235dba8d82b8af0905df045b3dc60b59836e8ddd \ + --hash=sha256:ccfe40021fd6afe23361175cf7551e3cef5fd34dc1ebe319f14993a83579e0eb \ + --hash=sha256:dee799544cae42a42b17a88e38b59cf2c271051dc001da2117a8ff240ffa0548 \ + --hash=sha256:e6b529a6a287104ec79d281c411685231200ce954a29c28ab8e5093cb6e130fb \ + --hash=sha256:e6e8be8a94436079cb5340f6d495d9d7ba30124d8b952703994c739c7c06e236 \ + --hash=sha256:f1dc17d987ddcc5eba12a7ce48f0eb93141dea236b170c1101151396edf2f0cf \ + --hash=sha256:f2b64641bca27497b986b9d87883014035aa904cb4fa333407c6752b3afee9ba \ + --hash=sha256:f894a2877650b56201d26a012a257b76d54a68834dc3913a93830ca8a047b075 \ + --hash=sha256:f97c3806dcea41c29c04965347b0e12481561b75e0045dc7a4f69d75dec5d9b1 \ + --hash=sha256:fb63a8d7fa2e95dcff1ca16af2f4d88b526fa5ff72d1696285884ac2d49b6963 \ + --hash=sha256:fbef61a3794e011c65fb4396a196cf123a7f474fe5a443db8e5dd7d751b9e6d4 + # via zarr google-pasta==0.2.0 \ --hash=sha256:4612951da876b1a10fe3960d7226f0c7682cf901e16ac06e473b267a5afa8954 \ --hash=sha256:b32482794a366b5366a32c92a9a9201b107821889935a02b3e51f6b432ea84ed \ @@ -1563,6 +1593,7 @@ packaging==24.2 \ # sphinx # tensorboard # tensorflow + # zarr pandas==2.2.3 \ --hash=sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a \ --hash=sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d \ @@ -1960,6 +1991,81 @@ pytz==2025.1 \ --hash=sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57 \ --hash=sha256:c2db42be2a2518b28e65f9207c4d05e6ff547d1efa4086469ef855e4ab70178e # via pandas +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via donfig pyzmq==26.2.1 \ --hash=sha256:000760e374d6f9d1a3478a42ed0c98604de68c9e94507e5452951e598ebecfba \ --hash=sha256:004837cb958988c75d8042f5dac19a881f3d9b3b75b2f574055e22573745f841 \ @@ -2545,6 +2651,7 @@ typing-extensions==4.12.2 \ # torch # typing-inspect # viser + # zarr typing-inspect==0.9.0 \ --hash=sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f \ --hash=sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78 @@ -2946,10 +3053,12 @@ yourdfpy==0.0.60 \ --hash=sha256:2af2d8bdeea1b85b642590a3b4236fdb35746d7b3e38ce460a169c18d9c4f868 \ --hash=sha256:8a187a8b18c98db87c76e9a950581b3c875b761e00df83942526c17ea693166c # via viser -zarr==2.18.4 \ - --hash=sha256:2795e20aff91093ce7e4da36ab1a138aededbd8ab66bf01fd01512e61d31e5d1 \ - --hash=sha256:37790ededd0683ae1abe6ff90aa16c22543b3436810060f53d72c15e910c24bb - # via -r deps/pip/requirements_ncore.in +zarr==3.1.4 \ + --hash=sha256:72c3dcee2d41527d21cd814cbaca1e6e23550701eb4fc30767ee53d925900eec \ + --hash=sha256:b0cf4eea8367ae444d7d0ea90b98d29ac0b7eb691fac2e0bcc2d2bd659522903 + # via + # -r deps/pip/requirements_3_11.in + # -r deps/pip/requirements_ncore.in zstandard==0.25.0 \ --hash=sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64 \ --hash=sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a \ diff --git a/deps/pip/requirements_3_12.in b/deps/pip/requirements_3_12.in index 1bb6e8b9..a39491fc 100644 --- a/deps/pip/requirements_3_12.in +++ b/deps/pip/requirements_3_12.in @@ -17,7 +17,7 @@ -r requirements_tests.in -r requirements_ncore.in -# Public API restrictions for 3.12 +# Public API restrictions for 3.12: most recent zarr-python 2 zarr>=2.18.7,<3.0.0 # zarr-python 2.18.7 is the first version with the correct numcodecs<0.16 restriction # torch: same version as the python 3.11 environment diff --git a/deps/pip/requirements_3_13.in b/deps/pip/requirements_3_13.in index 5c2627ea..14d3debc 100644 --- a/deps/pip/requirements_3_13.in +++ b/deps/pip/requirements_3_13.in @@ -17,8 +17,8 @@ -r requirements_tests.in -r requirements_ncore.in -# Public API restrictions for 3.13 -zarr>=2.18.7,<3.0.0 # zarr-python 2.18.7 is the first version with the correct numcodecs<0.16 restriction +# Public API restrictions for 3.13: most recent zarr-python 3 +zarr>=3.4.0 # most recent zarr-python 3 (python>=3.12) # torch: most recent version # CPU-only on purpose: python 3.12 / 3.13 cover the CPU code paths, CUDA is tested with python 3.11 diff --git a/deps/pip/requirements_3_13.txt b/deps/pip/requirements_3_13.txt index 131dc3cd..29ff157c 100644 --- a/deps/pip/requirements_3_13.txt +++ b/deps/pip/requirements_3_13.txt @@ -3,9 +3,6 @@ --index-url https://pypi.org/simple --find-links https://download.pytorch.org/whl/cpu/torch -asciitree==0.3.3 \ - --hash=sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e - # via zarr cbor2==6.1.5 \ --hash=sha256:015ed73f10e1f7b67306d41e36e0d7dc40e4a2100bc5c29b7a7f039ad3dc9061 \ --hash=sha256:040cf628af473fe18cb6f56bdac556d2398102e56852aab5206fbeb3dbde6b52 \ @@ -77,13 +74,9 @@ dataclasses-json==0.6.7 \ --hash=sha256:0dbf33f26c8d5305befd61b39d2b3414e8a407bedc2834dea9b8d642666fb40a \ --hash=sha256:b6b3e528266ea45b9535223bc53ca645f5208833c29229e847b3f26a1cc55fc0 # via -r deps/pip/requirements_ncore.in -deprecated==3.0.0 \ - --hash=sha256:16850204d3a1e6bb0acd06bff48d96e8b0a0d25d1c52f71705405a0f4894192d \ - --hash=sha256:58204cf4a7f6270d547af5c278ee7a6bb56045a4b3d8441a1cd11660f41b7939 - # via numcodecs -fasteners==0.20 \ - --hash=sha256:55dce8792a41b56f727ba6e123fcaee77fd87e638a6863cec00007bfea84c8d8 \ - --hash=sha256:9422c40d1e350e4259f509fb2e608d6bc43c0136f79a00db1b49046029d0b3b7 +donfig==0.8.1.post1 \ + --hash=sha256:2a3175ce74a06109ff9307d90a230f81215cbac9a751f4d1c6194644b8204f9d \ + --hash=sha256:3bef3413a4c1c601b585e8d297256d0c1470ea012afa6e8461dc28bfb7c23f52 # via zarr filelock==4.1.0 \ --hash=sha256:2ce9818e3e2d8f284c1a964414447ef148d42a5fd5e2a477a7118e574b293ec1 \ @@ -95,6 +88,39 @@ fsspec==2026.9.0 \ # via # torch # universal-pathlib +google-crc32c==1.9.0 \ + --hash=sha256:0568b17ed90ac596f29400d99e243fd0cc6276766183def888d1bf8d1dc13827 \ + --hash=sha256:0abe7e202c25909869c35672ab0f2fe748a7acf276eb78577332a7c38999740f \ + --hash=sha256:13179f7e3282617923e957b8e54b8f9c3968030f48640a9f47fd7c5c38c4a215 \ + --hash=sha256:16865b477d7941712cb0e0aad8ad4815e984fb5fc16d3fdaef7d986e26e53c95 \ + --hash=sha256:265233aff33d835f5b909584fe36ab29647b598c271b661a300001099109e53e \ + --hash=sha256:280f3a3e47af0eeba3a3e5aa7d311af77001812b8df80fb8beafcd0b40eaf7f1 \ + --hash=sha256:3abb18297d9ef0ab120531838be0e6d68c9fa876570e11c229c48f2edac23ce7 \ + --hash=sha256:43a2dc26f9be213fbe0b4fc4a1088c5d45cbfcb3247420ccc820f0fc3edeea86 \ + --hash=sha256:4488f1553a9ab7e86cdedc833374a7e904031803b995dc0bd0be48c271fa6556 \ + --hash=sha256:457d0d9a4718fd52b1494eac5c200ad25beeadbdc91843d550a003910838589f \ + --hash=sha256:51cb4e23a38ad4f495f35f87c233ca3ea6b9c4559e7ac383cdef786fab0f7977 \ + --hash=sha256:53fdafef58e230d0c946ab5f8446d123d9f548230a73b29c8b41c9546f268bc1 \ + --hash=sha256:56610f548f1b35c9568b9d1de30423480f505dae4991556072d5802820ff35c4 \ + --hash=sha256:5695c8b9327e040b2aba12c6659b0acb5995314ef0af0192da66e662e011103b \ + --hash=sha256:6a3b2c8a343c570ed8100a7627c20badfd92c6caa2067093a86be45af27f5b1b \ + --hash=sha256:7b8c84c3d159ab6817fe3f74e6e6cef099c3f95dcec3abc0d8afb1404642efbe \ + --hash=sha256:8535e75dfead304f30e9122b9ea2c0a570dbaa52c176a0a591540c7914c1e46d \ + --hash=sha256:8583ec21d56b565d68ab2963cc7e21b3b271247c29b04286068255ef65f221bd \ + --hash=sha256:86764b99e7a607830d93cb5b75e0ec3ff6cb06d3c274624418473cee701900d4 \ + --hash=sha256:8b91f41645b15a720357183fa5716682ada441873e3c462c15f9714be36f146b \ + --hash=sha256:af73200fa9791ccd380f3598235dba8d82b8af0905df045b3dc60b59836e8ddd \ + --hash=sha256:ccfe40021fd6afe23361175cf7551e3cef5fd34dc1ebe319f14993a83579e0eb \ + --hash=sha256:dee799544cae42a42b17a88e38b59cf2c271051dc001da2117a8ff240ffa0548 \ + --hash=sha256:e6b529a6a287104ec79d281c411685231200ce954a29c28ab8e5093cb6e130fb \ + --hash=sha256:e6e8be8a94436079cb5340f6d495d9d7ba30124d8b952703994c739c7c06e236 \ + --hash=sha256:f1dc17d987ddcc5eba12a7ce48f0eb93141dea236b170c1101151396edf2f0cf \ + --hash=sha256:f2b64641bca27497b986b9d87883014035aa904cb4fa333407c6752b3afee9ba \ + --hash=sha256:f894a2877650b56201d26a012a257b76d54a68834dc3913a93830ca8a047b075 \ + --hash=sha256:f97c3806dcea41c29c04965347b0e12481561b75e0045dc7a4f69d75dec5d9b1 \ + --hash=sha256:fb63a8d7fa2e95dcff1ca16af2f4d88b526fa5ff72d1696285884ac2d49b6963 \ + --hash=sha256:fbef61a3794e011c65fb4396a196cf123a7f474fe5a443db8e5dd7d751b9e6d4 + # via zarr iniconfig==2.3.1 \ --hash=sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960 \ --hash=sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7 @@ -264,6 +290,90 @@ mpmath==1.3.0 \ --hash=sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f \ --hash=sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c # via sympy +msgspec==0.22.0 \ + --hash=sha256:0067057df265795f742658b15dbe53f3b6f21d19dcfa53676db11088cfa41e0a \ + --hash=sha256:024138c51afd335d0b4dce401be33902caafac2b64f8c9f2509a378986175d98 \ + --hash=sha256:05dbc8268e50c9232ec72b9af1c7b13049aade4d1197764e38c427048706e046 \ + --hash=sha256:0666a1520cab86796612e794e71107e0fbf5e8ff3ddcdfcfff8f1d94b860d2f1 \ + --hash=sha256:0739b068f31f2004a364f97679ba91f2f5ecd6ec2a5b4b890188ab5c57d20672 \ + --hash=sha256:08826f5e5b0fa2f7a88592c396a243cfcc63d37e19f9d4fbe3b3f1be2fbdc404 \ + --hash=sha256:0922714feff5300aacd8ecd65fa828317ce4bf5212b3139258c0bfc0253cd80e \ + --hash=sha256:0a13624a4969159fe35d8c2a3d377b2b61bbd8585e327440d5e52725affcce38 \ + --hash=sha256:0b25dcbc108783cb72503ed705b9fbb8c3cb02ee5801923f44b5f038c91cc365 \ + --hash=sha256:0b31746da07cba0e330c6433a94a4699ad77d3aeb9638d1a320a7686b69f6249 \ + --hash=sha256:0dfadea8bdcfafc614bd031de55a8ede22b43445cfff6d8b77cc0c07d3edc8a8 \ + --hash=sha256:10d0d1d464960d99a949f7ca01ef8928e51c472433a5f5ab74b2d695fb830652 \ + --hash=sha256:12a887c4c06e4a771a2db32c9a80c7bb21866b12458025f636dcdc2253331c28 \ + --hash=sha256:1e547966017265c0d23342bcf2e027305dde40ea042d16694a9b96b4f696a052 \ + --hash=sha256:21460f54cee9208239b1a8421fdf25bffc77293e1daba88f585711ad839b9758 \ + --hash=sha256:21c887d4de397355f6635c2a037b1c067882dac5d132a1793d63bbf7cf5ca78e \ + --hash=sha256:221cbcbfa4478152b91d37dcfd4830e2be92773e8139e883f43773450ebacef8 \ + --hash=sha256:263e110955ed76fe0af2d79f819903b50a70dc0e7a752eb7aabe79d2e0a084fb \ + --hash=sha256:268594d0bae5510572599a6ab0364dd9de43c867d24a30856cd9f5edb63d8dc6 \ + --hash=sha256:27d9ef46c80884f9c4f323e0b18bec464287e872121e70f2cbe47335780bf597 \ + --hash=sha256:28f53f3604dd3e70225f7563c831628dbb03299b428f8e62aadb4b628e386874 \ + --hash=sha256:38c5b9bd347bc9abbcee40752be3c5117854e891ea7a1881a56d4b3dec58c5e7 \ + --hash=sha256:38f7022fbe91954b31afe3888a0af1b652e0f370fafdeb1d425f4a814d789c9f \ + --hash=sha256:3c789b5ccd07c0a3c09767108ee06e089b2875f2309a4569c2648f30a8d31dfa \ + --hash=sha256:3ca7d4cd69fbb66bd2da6211d3e79d40542d196c16c6d99bf838f76767ad35be \ + --hash=sha256:4600dbec738ed74e4c9bd35503e84701200ea7db344cfdeda80677b3ee53eb64 \ + --hash=sha256:4a663a8d7f6ad56ac1dbcba91e046ba8ebab7773ae72ef3dd3c47f8226919184 \ + --hash=sha256:508278300dd4efbd21cd3a4b2b016160a5feac98bc880d3673f6c06697baaf62 \ + --hash=sha256:57c282f474e17acf6bcf84f393c73afd45d6eba47cccff8b76b79c4fbb8a3b54 \ + --hash=sha256:5aa24eb475d070ecbbe5b21080fc3ce4b0b76c60de25cfe0c9678d8fb44bb42f \ + --hash=sha256:5e4f7e09cceac7dbf4c0761b8ae7df51c55b5df5e9af7aff2c895aac1ebea015 \ + --hash=sha256:614e2c827e0a3f934f3cf0cf4ba65210df8132b75a69a8a1f51bb3b2caf0ac5a \ + --hash=sha256:627bfdfe5a4b3d916b3360b30f4cddeee3a084f56593e33527c6872fa8322ff9 \ + --hash=sha256:65eea14bc65ccfeb8f3af62cb204841871e2961f002d7fa87dbe0f79dacf1c1c \ + --hash=sha256:6ad64f5c260866b0d543f89f50cee43628989c1433c5de7ce820281fa28a2611 \ + --hash=sha256:6ae370f92f3517f0e6f209ba7cc649c957b444868439197e046be07154667551 \ + --hash=sha256:6f48317f05312bfdf78248f53933f830f07ab75cc1c813ac3ca4220cb3b5b019 \ + --hash=sha256:71cbbdb39631064e2f2f9e9ac2b1b69931d72276eb5f9da4ed025726296bdbb6 \ + --hash=sha256:7293dee54de040cfa225c22151cc3d72f17cd674b5ebcb52f38fb9f5701592e6 \ + --hash=sha256:749899563d26b211379f142b8ffd7e2d7da149a51717798f0ce994dce50324f0 \ + --hash=sha256:7c1e76c6bd523141b9c05c2f8a70979cd0efedbd68855a66f292f8892c0b8fc7 \ + --hash=sha256:884c28c80b0a511595b29a9b04a3a230c3797369e4a033e6d5c6d9b5427f8e09 \ + --hash=sha256:885c6e0c89d6103648525fe62aa78d600054dedf7b3713d23b15d7ddb6d66a13 \ + --hash=sha256:8c8e84789918fbc15a503b92a829115ddd7567ecd3e4778bd418c56abbb86c11 \ + --hash=sha256:8d67582478b0eaabb899f2fb255c878ee7de57dff80eb73ab24f1865524ec441 \ + --hash=sha256:8f0a5c25516e2034b2db7767081759ff8996e214def9c43b3055f61e1be1caad \ + --hash=sha256:99c401861c5bb3a57f7d6423ea7ed4352cd57aa3f04f4fbe9f3e3e4564a10f08 \ + --hash=sha256:9a696f23f7c1ffb31fae308502e01a3965c3891d5c400f01d0d1096dbe77519e \ + --hash=sha256:a1dab6a99c759d1391ab2993388c1892746a697254f4b5dc6c059ca6e3bfbc8b \ + --hash=sha256:a52eba5c9528fd181fcec39d22b67aaa1dccc6cfe8e24d3f5d41130e6d04289d \ + --hash=sha256:a66b1766311e42371e509c996c3933b161c7ae0eabdf361af5316dec197e1022 \ + --hash=sha256:a6c8a3f210421e29d8f7e9815f106cf59d758665b7fe5428e61152ce24fe65d7 \ + --hash=sha256:a6db3806b3b76ca78064255eac6fa101a8a64fe6f698d80fbaf81fdfa21217d4 \ + --hash=sha256:a88d939d3fe4b8c7314645ebcd6e86c8c8a512ea7820d6550355973e803bc0f1 \ + --hash=sha256:a8b98ae215a102cbf6635f7df45f5c4af12f77fad1f7b71b9808fcf868a5735d \ + --hash=sha256:ab1e9e7531e353653b906cdd12a0220cc288a1e8e3436aabc65f4508d91b14d9 \ + --hash=sha256:b3113ebcceeb7693a915183c73d92c10bf5c62851dd187cab43bd025fb587419 \ + --hash=sha256:b5a169b5b03f0f2c7a296c002647db1dab75d2cd501bca34e32b71cab0261b56 \ + --hash=sha256:b60b43425a47eb9cfe987f6874e354ca7c760e58e295b4e2273ff03574df28a1 \ + --hash=sha256:b6d3ca19a8ff28d0a67a1824e2bff7ec649ec795c80a265f20ade4caa63080de \ + --hash=sha256:b962000e11dd34fb210a5a2c57a8a62b2d92b381c8cb3b05c075a83e38f8d645 \ + --hash=sha256:bc374dedd5f85a5f4de2386dc5f737894ccb8c1ac18e9566ce66fd9839e6285d \ + --hash=sha256:c3c510aba9015c085e514b75a9b3f1ed7c4591ae5e379655821b8bba51f30cc7 \ + --hash=sha256:c6c310ef83e7e291b01a63298828f848348bb99e84a1098c4b3923c05674d032 \ + --hash=sha256:c6f06576eced70462179a4b4638e84cf69fdbba37f44d13a64a21739c131a830 \ + --hash=sha256:cfc3d9557de9c806318725b702f3e664db33167bb42892079b693c69893fd33b \ + --hash=sha256:d2f950239ff1fc7322c6f9634807310265149cb168270d3ddcdda5b6ada13a28 \ + --hash=sha256:d7a738826936c72348c613061d260446f13c82b6fd7d5d7705b6911ab8dca2f3 \ + --hash=sha256:dce29a04966e31abf9b83b697c6d672486526dc5d03fcd6970cb56d5dc1fbeea \ + --hash=sha256:dd9568695911055440d2bb7099ed9098fc181d335daa772d0eb3fe8f31ba4efb \ + --hash=sha256:e0aa0cc3f18c35bab79bd7b87fde95d6274a9deddeebd1ea541f8066a5073165 \ + --hash=sha256:e79725246291516a7359caad5fb743ddc0ec66ed40d2381fb846325b5031504e \ + --hash=sha256:ebd211d7af79ed8710c64e9e8d4c0d02749bc20170e7ab4e1c5801ca7c99d25b \ + --hash=sha256:ec108e96fdaa8fdbe5bb993ec97a9d1faa69b3a521eecd71a6e5acbe0e29ae69 \ + --hash=sha256:f039ef5207b847f075a0a43020ee6140cd47505f890e47e157f2deb485c2dc96 \ + --hash=sha256:f13c127a945479bc9db057eb253b8851075c8e1ae07ffc967bfa1c5676203a86 \ + --hash=sha256:f2ddea9d78d09460f06c26a7a508adcd049761c3208776162b8eb79b8a032cff \ + --hash=sha256:f3413e3647275f787b21b4dfb4836a59a1a5acf1018ab1d45843b1d7edf15c22 \ + --hash=sha256:f7a923bcde480065c8e25967464cfb2a687ee67000bb43157e2d57e40eca7305 \ + --hash=sha256:fa3689b9dfcc663358ef23ba4299d7460f01108515b041a7d30d05908ac9c32f \ + --hash=sha256:fb1e129b81ac8fcf9ec649b081c6c8da1c7ea6f87cab336d46386abc2cd855c1 \ + --hash=sha256:feafe612034d49e9144340c0b5168ee4e22c2af4aaa2c1db11ae84e1aac9543b + # via zarr mypy-extensions==1.1.0 \ --hash=sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 \ --hash=sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558 @@ -272,20 +382,28 @@ networkx==3.7 \ --hash=sha256:e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0 \ --hash=sha256:fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a # via torch -numcodecs==0.15.1 \ - --hash=sha256:1d471a1829ce52d3f365053a2bd1379e32e369517557c4027ddf5ac0d99c591e \ - --hash=sha256:1dfdea4a67108205edfce99c1cb6cd621343bc7abb7e16a041c966776920e7de \ - --hash=sha256:698f1d59511488b8fe215fadc1e679a4c70d894de2cca6d8bf2ab770eed34dfd \ - --hash=sha256:a34f0fe5e5f3b837bbedbeb98794a6d4a12eeeef8d4697b523905837900b5e1c \ - --hash=sha256:a4f7bdb26f1b34423cb56d48e75821223be38040907c9b5954eeb7463e7eb03c \ - --hash=sha256:b0a9d9cd29a0088220682dda4a9898321f7813ff7802be2bbb545f6e3d2f10ff \ - --hash=sha256:bef8c8e64fab76677324a07672b10c31861775d03fc63ed5012ca384144e4bb9 \ - --hash=sha256:c3a09e22140f2c691f7df26303ff8fa2dadcf26d7d0828398c0bc09b69e5efa3 \ - --hash=sha256:cdfaef9f5f2ed8f65858db801f1953f1007c9613ee490a1c56233cd78b505ed5 \ - --hash=sha256:daed6066ffcf40082da847d318b5ab6123d69ceb433ba603cb87c323a541a8bc \ - --hash=sha256:e2547fa3a7ffc9399cfd2936aecb620a3db285f2630c86c8a678e477741a4b3c \ - --hash=sha256:e3d82b70500cf61e8d115faa0d0a76be6ecdc24a16477ee3279d711699ad85f3 \ - --hash=sha256:eeed77e4d6636641a2cc605fbc6078c7a8f2cc40f3dfa2b3f61e52e6091b04ff +numcodecs==0.17.0 \ + --hash=sha256:0fc125d1c726c1937cde346e109e3662a2b4ff6be073289da7d124d172aceda5 \ + --hash=sha256:1837d4d1d646cecd3ab2d1ba22956295d709edea0bddc952737c647bec1d03c4 \ + --hash=sha256:1cce4bf2278ed74841c2088acfd38e67c3e5aa77e3bc1962ef0fa2931becbb12 \ + --hash=sha256:1ebd63cdb8985c66257bc037fcdff5f38637aff72d7ef62612ec46f2299e8749 \ + --hash=sha256:2e29732c5e3a83663e51b40007819d8fd0aae16a2322f7044ce13a2460a99e23 \ + --hash=sha256:3e4e351566b3ab2f6255a9d91c6c48e1d0f9ec6e2ae409a148e091a8fc0a80b0 \ + --hash=sha256:4a62e5a821ccfbe425bbdd9a079f8b6c41b7e796ff3c99324530561193a53047 \ + --hash=sha256:4c36f6fd14dc22939172145c24d3b3eab2410c34ed807906a5ece5f4541c7c43 \ + --hash=sha256:4d83befe67a51ba6a988c562209bf13836438c1b6dce23049d84ff42854af32d \ + --hash=sha256:4f43ba0d834ce012ed482996a7424df9077a47d5899ede2d1d54fe85e6eb12fa \ + --hash=sha256:657b1f9aa4b1025aa0fa7d4bd8d7492900950a11f636dff622bd208c0b99e35e \ + --hash=sha256:69b9b4685097c4d478a0c829debf4470555ec63e92cdd2c6b5f195460f1dc888 \ + --hash=sha256:6c3342d91ed7cf59c1be84396edd364e936bb0ec9e366d24bb69689748d19625 \ + --hash=sha256:6f1293581326e92293b142bd05b389f6682ed1ce333f36f116344bca340cfd10 \ + --hash=sha256:7065b3349b73d54785aa89e00d0b97d80f664e9056757929d28151f9208dc04c \ + --hash=sha256:8697a4631fedded77a75d333e4926b1eb3a11bc7d3e30213e7e565d6910526d0 \ + --hash=sha256:a854e9c89f58eeeb2453f3c1637d1916797edb6eaff26bc186a6cdb09d187092 \ + --hash=sha256:d30c69b4bdb1755af1022fa913e184eaadc4fc0cd38f736e483e8ad205e130d1 \ + --hash=sha256:de2c66db238e74e66fe9be7e02b7e0129b75d3f812d38e4019eb0102cc2dcdf0 \ + --hash=sha256:e8db2e337bdafd3bb5f891a2543b53b2b36a509ce9d587af2846db3715b6c8b9 \ + --hash=sha256:ecd0f6a10e3f8afbbb16ecc999d2b06aa2a31a2946f1c1a85d15d91a1ebcfef3 # via zarr numpy==2.5.4 \ --hash=sha256:067374eb538c34c745436365cf7b0112595c1d326f21ce4ff340f61230239fbb \ @@ -377,6 +495,7 @@ packaging==26.3 \ # via # marshmallow # pytest + # zarr parameterized==0.9.0 \ --hash=sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b \ --hash=sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1 @@ -486,6 +605,81 @@ pytest==9.1.1 \ --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \ --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c # via -r deps/pip/requirements_tests.in +pyyaml==6.0.3 \ + --hash=sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c \ + --hash=sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a \ + --hash=sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3 \ + --hash=sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956 \ + --hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \ + --hash=sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c \ + --hash=sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65 \ + --hash=sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a \ + --hash=sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0 \ + --hash=sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b \ + --hash=sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1 \ + --hash=sha256:22ba7cfcad58ef3ecddc7ed1db3409af68d023b7f940da23c6c2a1890976eda6 \ + --hash=sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7 \ + --hash=sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e \ + --hash=sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007 \ + --hash=sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310 \ + --hash=sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4 \ + --hash=sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9 \ + --hash=sha256:3ff07ec89bae51176c0549bc4c63aa6202991da2d9a6129d7aef7f1407d3f295 \ + --hash=sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea \ + --hash=sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0 \ + --hash=sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e \ + --hash=sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac \ + --hash=sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9 \ + --hash=sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7 \ + --hash=sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35 \ + --hash=sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb \ + --hash=sha256:5cf4e27da7e3fbed4d6c3d8e797387aaad68102272f8f9752883bc32d61cb87b \ + --hash=sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69 \ + --hash=sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5 \ + --hash=sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b \ + --hash=sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c \ + --hash=sha256:6344df0d5755a2c9a276d4473ae6b90647e216ab4757f8426893b5dd2ac3f369 \ + --hash=sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd \ + --hash=sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824 \ + --hash=sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198 \ + --hash=sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065 \ + --hash=sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c \ + --hash=sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c \ + --hash=sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764 \ + --hash=sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196 \ + --hash=sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b \ + --hash=sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00 \ + --hash=sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac \ + --hash=sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8 \ + --hash=sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e \ + --hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \ + --hash=sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3 \ + --hash=sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5 \ + --hash=sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4 \ + --hash=sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b \ + --hash=sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf \ + --hash=sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5 \ + --hash=sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702 \ + --hash=sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8 \ + --hash=sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788 \ + --hash=sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da \ + --hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d \ + --hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \ + --hash=sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c \ + --hash=sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba \ + --hash=sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f \ + --hash=sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917 \ + --hash=sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5 \ + --hash=sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26 \ + --hash=sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f \ + --hash=sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b \ + --hash=sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be \ + --hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c \ + --hash=sha256:efd7b85f94a6f21e4932043973a7ba2613b059c4a000551892ac9f1d11f5baf3 \ + --hash=sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6 \ + --hash=sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926 \ + --hash=sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0 + # via donfig scipy==1.18.1 \ --hash=sha256:011413b7426b75012840e35649e00fe0a2c3bae89fed433876e3a99251572efc \ --hash=sha256:0ac49ea97594532dd44b7136094d35f5440fa06e6d9c6384a74c01764df388c5 \ @@ -566,8 +760,10 @@ typing-extensions==4.16.0 \ --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 # via # -r deps/pip/requirements_ncore.in + # numcodecs # torch # typing-inspect + # zarr typing-inspect==0.9.0 \ --hash=sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f \ --hash=sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78 @@ -576,112 +772,9 @@ universal-pathlib==0.3.10 \ --hash=sha256:4487cbc90730a48cfb64f811d99e14b6faed6d738420cd5f93f59f48e6930bfb \ --hash=sha256:dfaf2fb35683d2eb1287a3ed7b215e4d6016aa6eaf339c607023d22f90821c66 # via -r deps/pip/requirements_ncore.in -wrapt==2.5.0 \ - --hash=sha256:05246a100da68259af521b88788f131ba005465f1c95d358cc3c03ec5e351b52 \ - --hash=sha256:05b7a271e21598694dcb7a6d6224f04e80ecc24b00a31ec57c658196377b0b29 \ - --hash=sha256:0810e060e58f7960405172ad21080df8e7335841c9fe97417bd7d3f05af24f90 \ - --hash=sha256:0a7a369e7fca9fc8c2c50df634382009b09af416853da3d4515e4bb048a5b9ee \ - --hash=sha256:0cabb9c17ab79b2549d1f23b36f436473ad9253ef53995a817feba26fae69d5b \ - --hash=sha256:0d245ac03f5ae77f1eea6eb19edd9e778c2f772490c20496c2f1cd3a102ee1b6 \ - --hash=sha256:0dfc38cb672af51fc29696ba9c6f05d2315f5e62c4af2564e50f07f81198a163 \ - --hash=sha256:0ea62bc142f4fa8b2e0ab058f50699ccd679ef6199c8fa3cc1c2396c7a659000 \ - --hash=sha256:107eea1a511e98a3a5033b0c2cb403fbb37f05dee6ac1fb85c0460d311ec278c \ - --hash=sha256:1122f4f9e363da804ccba05a9f0f39baa3716eb82452c929258bc3f1420c899b \ - --hash=sha256:148052fc55013930217f531c6978e918ab210a12bd73cc9bd6de661a7adaf620 \ - --hash=sha256:181a45000506a6382eb337354ca7e8525690702f1ccf2eae4f23a210ff339543 \ - --hash=sha256:18baaf966bdc22dbdbe6e4323da8c17c0b7e8ea941dccf5426e741ae398953d9 \ - --hash=sha256:1a3c4035d2026b87ef23dd8d165f1f8d3853ee2bd02791cfd22bd8c6226c41ce \ - --hash=sha256:1b35ef7379323a149a6398f6261248bf48b61666210bd69e2ab24a9a9afdedc0 \ - --hash=sha256:1ccd22ef8690ca302425f26d4c3e8b24f1284d4351d20e49ce21f9c0dd58d64d \ - --hash=sha256:1d4da5f0e9a719471502b0db80d5c97503aeca796b7aeb9ab8f47403b2be76e6 \ - --hash=sha256:1ebc0d09906057ada57a32158a657364ca40b8f86e604da3e7d979069b601502 \ - --hash=sha256:1f17c5a3836397bf59fd57b0e5b0dc42969b1daa70d31aa361c6e13cbf138b5a \ - --hash=sha256:1ffb2823c95dbeb8a47fedfba9636b2afeb0a8ef94b66df97bd081bdfe5a263f \ - --hash=sha256:21cfe343ef9c2deb865ad0d5c57822266447c88dcf6d8805dd8c363fe367f30c \ - --hash=sha256:23a9d6cb6413359b76f030d6bbb75340b7669c69da245dde2919a4c93708993b \ - --hash=sha256:2ac82ef59ee05e259902bc7cf73dee5e6397845e8ccdc376d9d25536b59a877c \ - --hash=sha256:2e3d110d7f99644946f249927c346d9dba507a78815d1bcebdbf0d94c14c5649 \ - --hash=sha256:2fd8a61c31220840c7f52621cf51c961af5058bd009a55bcdf4a6732bdb13b35 \ - --hash=sha256:30cf86d1e35a57e772e4709a137c1731751255439368a41b1a2519561e385f4e \ - --hash=sha256:36703cafc2ec059e118c2175e6cb7ad7299c2924aecdcb1b7a7ebbf7a3e20c19 \ - --hash=sha256:3f5dfb867d3f58fde0d850a302f1164bb83f4a922b037882bcc7e0474aa8312f \ - --hash=sha256:425349a99b8c9540399d36620c376dc26e6aca93071cd6fafa239c2f1b5d53a4 \ - --hash=sha256:4343880acd72e74233baf092285aaaf4306244e31d7601828bd2600316027df0 \ - --hash=sha256:439523506fd0d9af4f76d75f2e46aa4979e164fecba6890859326ff06edbaf9c \ - --hash=sha256:47abb2bb7f15b416e72fbe5e68e49a6f09331dae6af1ca6055f5aa2251d2bd2f \ - --hash=sha256:513dd1f4a1f91030d5656d9f4b3af8aa490d4db1eb53481e6846b8a5ee7acfcd \ - --hash=sha256:5375ff1d2159e2ef847449e3dd2329441423d0a3d895d123273a67095fe7ca0c \ - --hash=sha256:5421acb5c363a9bc959122a8645e3f1f42010c932dc53885b11a5ff5b5a6d730 \ - --hash=sha256:557ebf4ce5568588368675014a2540405db687c2e4c7ad1eb83aa7e857be1864 \ - --hash=sha256:57fa1a3fd1279b3ca7655b943ad61d298f2a2464a4cdca7ff298058e408322f9 \ - --hash=sha256:58b2a87c65cbfb20917ec48ace47f71b1962c1f81dbf18a4052e3037abf72028 \ - --hash=sha256:5df50c6133a63071fb77a0a091ceae36e78467b728d65e711ded02510edd9855 \ - --hash=sha256:6058e12e9caa33468f9a36fb88c15a4bb30a479f997b37834b83abdbf062f264 \ - --hash=sha256:6119e5d5bda268171af1f68ab5e7544c618f02a9bdc7c3929a458b719b88cd03 \ - --hash=sha256:61253431e5a6f0eaeae71e2a101753fbaf4360b3e4dd8e118177d565ae119d40 \ - --hash=sha256:6269637d9a54990430b4a769df15833935a46c4d004d9fe8a153bbadf0b9a097 \ - --hash=sha256:63e58f96849f622ce769dcd705f83c7445cd9829bce1dae00e78bb031aec8096 \ - --hash=sha256:65f2ee406dc592a5b22a7dc6abac13e8a3e8de4b2ecf5dc3c22937865496e4b6 \ - --hash=sha256:672dd1bab4256311db1520b1b50e7a10cbaeaae2b0ac6bc5d858cc387ee605a2 \ - --hash=sha256:6761765cc520ff9616fb035c02a85d1d744f7f70edd4649718fd0d09c589eacf \ - --hash=sha256:691671ea05684f921ffc2e935fd3f9311c1795a10fbbfa006b46269733668f66 \ - --hash=sha256:6a31cc61ad1be4091f5b094c91d53d206609928af13b7b24573cb857a3bc07fc \ - --hash=sha256:6a9ee62a970075738909909bdbef3a7da9f7ae03dfca584db283547a29503b56 \ - --hash=sha256:7138b0e7990e5555a905c519e8dad17c1da1f20e08b283e202c414229065740f \ - --hash=sha256:734ea79e4707751fb7cc4f716123b2115cd16ed0c4926e803540aece22e3ba67 \ - --hash=sha256:76fb341d5a707a4f211631b8c77259b2df149147e9d9c245ae6ba3dd936bfdfb \ - --hash=sha256:77f0a74ff6f6cf89f5b673a732d5afe1911a6e6b1c017260836fdfdf85518dc1 \ - --hash=sha256:78b7bdaa8b27b7f7607c66bdb6ab15c1dcbd9e9a1556a253a347dad511f615d1 \ - --hash=sha256:79e68f0fd7d381b9bbd71776f602a2d5440d4d2077459128e02fd6607465422c \ - --hash=sha256:7e25e9697f60af41fb86b08697e470b1e7eb6cd6ac0eb25e4b1f519839adc271 \ - --hash=sha256:7e2af7ff9553c492f684a41d903ec41e37bf6267ee3206c17518a349201cae46 \ - --hash=sha256:8837fbe708cb9d8a2d32a37dee836d24a531f02560db26418e2b181986fa21cb \ - --hash=sha256:88bb24b9fdccb1d805258d5648533206eb58c58b6554985c47db53a89c11be85 \ - --hash=sha256:94ebe745d1b0ebd3af91e22c38961616d32a228bdf9c326e7d4a6007a623d48e \ - --hash=sha256:99f8ea48f14a71c5e2df8763e9a490e8af63096dcd67755b7bab0a4b74fc7cd7 \ - --hash=sha256:a039b009693b58f7e0cf6121851218c2227223e7601cc24ed4957ba0990fd25b \ - --hash=sha256:a10e9af5d4c5977d2d93e75d1662d076a8e8bf8c499dac7be4e78ebd8e0b59af \ - --hash=sha256:a145a7826eddea3eb5814903f98f93756042b919bb5305544cb1331daa2705b1 \ - --hash=sha256:a346408f19b6d589bf029f25f65c0b4cdeed6302ef8f40da4e5d1552d22dc037 \ - --hash=sha256:a45a5249a6965d91aac9f991fda7c17e6b8b41fe91592a6099f182bf53c82724 \ - --hash=sha256:a5bb346a34499e091e4fa23df58251ad192173c088e5413d893ca1c730c133c7 \ - --hash=sha256:aae2f5f4c77335a39ebe5a1c77d5519f6cefefc7dbff50bd551e3771d927e3fc \ - --hash=sha256:ab45839c912777e2738fed369636589c8b2a6d9c44ca56de0fd0814581d467c2 \ - --hash=sha256:b312b3cc87951faaed3cfef984d768ee8bee7f935d9cc929aaa9946b0b96a98c \ - --hash=sha256:b58138d19f34e32833e62de5e910bc2a8baae43310b921d783bd39b15227c2dd \ - --hash=sha256:b620d7559b6b2197c5730332fab0867ecf1c8cb74d45533ebbcbcad1eacf4616 \ - --hash=sha256:b640460f0ffb346b192686bd6fac5589e35a6c6640501c59a9fb6e82b0dd6bd8 \ - --hash=sha256:b898caea081303006decc562c7fca5126f7c96507e78dd8f1ae3285dfa50ddc7 \ - --hash=sha256:b95a6eca3b927853529eea958310563c83140ae8451dd5dc4399c7da385dc4f3 \ - --hash=sha256:bad63bb4dea3c58e8078a3a173259ac2df5442a437e49c632b4099c0250e803b \ - --hash=sha256:bc5607c1911c92530cb402ea90d818933cffb28bd8de9b453a2542279816d8c7 \ - --hash=sha256:bc6491d3008ecabf685b0746f03ad8241a0950336939b14addb03af39b51a316 \ - --hash=sha256:bcd42e7b69c8c1e33a29b79b28de03bdc08876a49745830f9162a3af860e06d0 \ - --hash=sha256:c3a78a3161b3a9bf07725822fd24d379c1f3f161b6db49166c4131096ea73b4c \ - --hash=sha256:c3dfb16e047c912e1a06bfbc45da752219f474897be85f55b8069f7936b08cc0 \ - --hash=sha256:c48cdb6c904dca76d9915a579e4a5fab6b0c25f650c1019ce78a78effaf7a345 \ - --hash=sha256:c57ddae24cf72eb6bd18112638a987cafe6109d90f2df111e6934362cc03ac1a \ - --hash=sha256:c932273bc43b068538f3874fa5e6c2a60f33fa0b11c1ebc7768652f6a0608943 \ - --hash=sha256:cbd45dfba6b5c1bfbabe1feb3c0f117fbd62416e98268d9a7cd9ad8802875356 \ - --hash=sha256:ce4cab32c37ef71e69cf88f909b7febd0dd79543e5ae3650e2b874e0f3d3b975 \ - --hash=sha256:d75d6366203c8d025c1a74bae0b565952187ddae79bd5c7bf10687652a56f020 \ - --hash=sha256:def66258d97ebf1e4e97def12c5daa542d1cc728a3da83ed3a43933f56df6dab \ - --hash=sha256:dfedf844892bf88f387cdeb9fa9bff1186904417c9f05087c99241bec24a65cc \ - --hash=sha256:e0345d4c1f7aa5a27075d28a7f0e9ed386729198045f1f08b47f8320d6bbda23 \ - --hash=sha256:e3c6fb1c1a516881353186bed9cfcb8899f968c03b3509720c79db0d967acf3b \ - --hash=sha256:e5d4885c5625c9d2dcb49458700851574e9d0ea046c7a265526e990282f8ae8e \ - --hash=sha256:e716f47c7f61e11709d3c0904213c94fc22999abf0c41461276cef886c1e8b4d \ - --hash=sha256:ea4fdc79c0045d6bb1603c109127145245cafe888588888444e1e37fbeadbac3 \ - --hash=sha256:f12e80c3089ebc03727d368f8205b811b5af2cd4a72b5e4cac75e901dd316e39 \ - --hash=sha256:f42239c89430eee2d8a6dec39e34677abdbb67fff63caf2467dd6124ea4d4d58 \ - --hash=sha256:f4ef4935962f7029b2058a99f1a47ccbffc3be919dddb3becb6c2c48eac3d9f0 \ - --hash=sha256:fd91203e156d610ecb28b9ccd7b764af7a7b38662d7c163090babab0d10def0c \ - --hash=sha256:fe09aac4837ec720af606a493e814dcc3f65631984e1b7231b589efcf9917024 \ - --hash=sha256:ff909b934b1958e31784d412abab5cbb0709fdbc01c86f22965e1d15331371ba - # via deprecated -zarr==2.18.7 \ - --hash=sha256:ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223 \ - --hash=sha256:b2b8f66f14dac4af66b180d2338819981b981f70e196c9a66e6bfaa9e59572f5 +zarr==3.4.1 \ + --hash=sha256:38b540578a119352bdce02a720d7bfd99846e77f0728c58b1bab3d1f547526a0 \ + --hash=sha256:b34bda11ceb199c81ee78ecd42cd02f46c7f67a6d8a1e9bc501cafd5a1795356 # via # -r deps/pip/requirements_3_13.in # -r deps/pip/requirements_ncore.in diff --git a/deps/pip/requirements_ncore.in b/deps/pip/requirements_ncore.in index 54bf97e8..d691be4c 100644 --- a/deps/pip/requirements_ncore.in +++ b/deps/pip/requirements_ncore.in @@ -24,10 +24,11 @@ pillow>=12.2.0; python_version>="3.9" pillow; python_version<"3.9" # zarr version rationales: -# - 2.17.0 contains crucial perf improvements ["Cache result of FSStore._fsspec_installed()" https://github.com/zarr-developers/zarr-python/pull/1581] +# - both zarr-python 2 and zarr-python 3 are supported # - 2.12.0 is the last version supporting python3.8 with old numpy versions, which we still want to support for now -# - we can't switch to zarr 3 yet unconditionally as it's not available for python3.8 anymore -zarr>=2.12.0,<3.0.0 +# - 2.17.0 contains crucial perf improvements ["Cache result of FSStore._fsspec_installed()" https://github.com/zarr-developers/zarr-python/pull/1581] +# - 3.1.4 is the oldest supported zarr-python 3 version (Array.async_array, Store ABC without set_partial_values) +zarr>=2.12.0,!=3.0.*,!=3.1.0,!=3.1.1,!=3.1.2,!=3.1.3 # cbor2 version rationale: # - 5.9.0 fixes CWE-674 / CVE-2026-26209 (uncontrolled recursion DoS) diff --git a/docs/data/storage_and_access.rst b/docs/data/storage_and_access.rst index 202799ca..679639f1 100644 --- a/docs/data/storage_and_access.rst +++ b/docs/data/storage_and_access.rst @@ -137,6 +137,42 @@ random access without any index. .. |times| unicode:: U+00D7 .. |rarr| unicode:: U+2192 +.. _zarr_versions: + +zarr-python Versions +-------------------- + +NCore supports both major versions of the `zarr-python `_ +library: ``2.x`` (Python |ge| 3.8) and ``3.x`` (Python |ge| 3.11, version |ge| 3.1.4). +The matching implementation is selected automatically at import time. + +Data written with either version is equivalent (chunk records are byte-identical, +node metadata documents are equivalent JSON) and can be read with either version, +including data written by NCore |le| 19.8. + +Hierarchies opened with consolidated metadata (the default of NCore readers) resolve +all zarr nodes from the compressed consolidated metadata record of each component group, +without reading node metadata records. With zarr-python 3, NCore executes zarr's +asynchronous I/O on the calling thread, avoiding per-access thread hand-offs to +zarr-python's global I/O thread. + +Records of ``.itar`` archives are write-once (archives are append-only): rewriting a +record fails. zarr-python 3 rewrites node metadata on every attribute update, so node +metadata records are kept in memory until the archive is closed, and only their final +versions are appended (parents before children). + +.. note:: + + The groups of component readers / writers (``self._group``) are + ``ncore.impl.data.nodes.Group`` instances, which support both zarr-python + versions with a typed API (e.g. ``create_array``, ``array(path).read()``, + ``attrs``). The wrapped zarr-python objects are accessible via ``.zarr`` for + components implemented with the zarr-python API of the installed version + directly. + +.. |ge| unicode:: U+2265 +.. |le| unicode:: U+2264 + Loading V4 Data --------------- diff --git a/ncore/BUILD.bazel b/ncore/BUILD.bazel index e71fe1f7..e1a5b44f 100644 --- a/ncore/BUILD.bazel +++ b/ncore/BUILD.bazel @@ -56,10 +56,11 @@ py_wheel( "dataclasses_json>=0.2.12", "pillow", # zarr version restrictions / suggestions rationale: + # - both zarr-python 2 and zarr-python 3 are supported # - 2.17.0 contains crucial perf improvements ["Cache result of FSStore._fsspec_installed()" https://github.com/zarr-developers/zarr-python/pull/1581] # - 2.12.0 is the last version supporting python3.8 with old numpy versions, which we still want to support for now - # - we can't switch to zarr 3 yet unconditionally as it's not available for python3.8 anymore - "zarr>=2.12.0,<3.0.0", + # - 3.1.4 is the oldest supported zarr-python 3 version (Array.async_array, Store ABC without set_partial_values) + "zarr>=2.12.0,!=3.0.*,!=3.1.0,!=3.1.1,!=3.1.2,!=3.1.3", "cbor2", "scipy", "torch", diff --git a/ncore/impl/data/BUILD.bazel b/ncore/impl/data/BUILD.bazel index 85aea767..c7206756 100644 --- a/ncore/impl/data/BUILD.bazel +++ b/ncore/impl/data/BUILD.bazel @@ -46,6 +46,7 @@ py_library( srcs = [ "_backend.py", "_zarr2.py", + "_zarr3.py", "encoding.py", "itar.py", "nodes.py", diff --git a/ncore/impl/data/_zarr3.py b/ncore/impl/data/_zarr3.py new file mode 100644 index 00000000..5ea4a451 --- /dev/null +++ b/ncore/impl/data/_zarr3.py @@ -0,0 +1,406 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""zarr-python 3 backend of :mod:`ncore.impl.data.nodes` and :mod:`ncore.impl.data.stores` (only imported if +zarr-python 3 is installed). + +zarr-python 3 executes all I/O as coroutines. Its synchronous API submits each call to a global event loop running on +a dedicated I/O thread, and codecs additionally hand off (de)compression to worker threads. For many small accesses +(as typical for reading individual frames) these thread hand-offs dominate. The backend instead runs zarr-python's +coroutines on an event loop owned by the *calling* thread (see :func:`run`), which executes the same zarr-python code +paths without thread hand-offs, and lets multiple reader threads decode data concurrently. +""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import contextlib +import json +import os +import threading +import warnings + +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterator, + Coroutine, + Iterable, + Iterator, + List, + Literal, + Optional, + Tuple, + TypeVar, + Union, + cast, +) + +import numcodecs +import numpy as np +import zarr +import zarr.api.asynchronous as zarr_async + +from zarr.abc.store import ByteRequest, OffsetByteRequest, RangeByteRequest, Store, SuffixByteRequest +from zarr.core.buffer import Buffer, BufferPrototype, default_buffer_prototype +from zarr.core.group import AsyncGroup +from zarr.core.sync import sync +from zarr.storage import LocalStore, StorePath + +from ncore.impl.data._backend import ( + V2_CONSOLIDATED_METADATA_KEY, + Attributes, + BloscCompression, + Selection, + StoreLike, + ZarrStore, + attributes_dict, + is_v2_metadata_key, +) +from ncore.impl.data.encoding import decode_cbor_xz, encode_cbor_xz +from ncore.impl.data.itar import IndexedTarStore + + +if TYPE_CHECKING: + import numpy.typing as npt # type: ignore[import-not-found] + + +T = TypeVar("T") + + +# ----------------------------------------------------------------------------- +# Execution of coroutines on event loops of the calling threads +# ----------------------------------------------------------------------------- + + +class _InlineExecutor(concurrent.futures.ThreadPoolExecutor): + """Executor running submitted work immediately in the submitting thread (asyncio requires default executors to be + thread pools, which start no threads until work is submitted to them)""" + + def submit(self, fn, /, *args, **kwargs): # type: ignore[override] + future: concurrent.futures.Future = concurrent.futures.Future() + try: + future.set_result(fn(*args, **kwargs)) + except BaseException as e: # forwarded to the awaiting coroutine + future.set_exception(e) + return future + + +class _Loop: + """An event loop of a thread (closed when the thread ends)""" + + def __init__(self) -> None: + self.loop = asyncio.new_event_loop() + self.loop.set_default_executor(_InlineExecutor()) # (codecs offload (de)compression to the default executor) + self.pid = os.getpid() + + def __del__(self) -> None: + # Loops of parent processes aren't closed in forked processes: their selectors (e.g., epoll instances) are + # shared with the parent, closing them would unregister the parent's file descriptors (e.g., the self-pipe of + # the parent's loop, which then misses wake-ups by other threads) + if self.pid == os.getpid() and not self.loop.is_closed() and not self.loop.is_running(): + self.loop.close() + + +class _ThreadLoop(threading.local): + state: Optional[_Loop] = None + + +_thread_loop = _ThreadLoop() + + +def run(coroutine: Coroutine[Any, Any, T]) -> T: + """Runs a zarr-python coroutine to completion on an event loop owned by the calling thread. + + The loop runs offloaded work (e.g., of codecs) inline. Processes forked by a thread with a loop create a new loop. + Within running event loops (e.g., of asynchronous applications), coroutines run on zarr-python's I/O loop. + """ + try: + asyncio.get_running_loop() + except RuntimeError: + if (state := _thread_loop.state) is None or state.pid != os.getpid(): + state = _thread_loop.state = _Loop() # (see _Loop.__del__ for loops of parent processes) + return state.loop.run_until_complete(coroutine) + return sync(coroutine) + + +# ----------------------------------------------------------------------------- +# zarr-python 3 store of indexed tar archives +# ----------------------------------------------------------------------------- + + +def _byte_range(byte_range: Optional[ByteRequest], size: int) -> Tuple[int, int]: + """Maps a zarr-python byte request to the (start, end) range of a record of the given size""" + if byte_range is None: + return 0, size + if isinstance(byte_range, RangeByteRequest): + return min(byte_range.start, size), min(byte_range.end, size) + if isinstance(byte_range, OffsetByteRequest): + return min(byte_range.offset, size), size + if isinstance(byte_range, SuffixByteRequest): + return max(0, size - byte_range.suffix), size + raise TypeError(f"Unexpected byte range {byte_range}") + + +class _TarStore(Store): + """zarr-python 3 store of the records of an :class:`IndexedTarStore` (synchronous I/O)""" + + supports_writes: bool = True + supports_deletes: bool = False + supports_partial_writes: Literal[False] = False + supports_listing: bool = True + + def __init__(self, store: IndexedTarStore) -> None: + super().__init__(read_only=store.mode == "r") + self.store = store + self._is_open = True + + def __eq__(self, value: object) -> bool: + return isinstance(value, _TarStore) and self.store == value.store + + def __hash__(self) -> int: + return hash(self.store) + + def __repr__(self) -> str: + return f"_TarStore({self.store!r})" + + def with_read_only(self, read_only: bool = False) -> _TarStore: + if read_only != self.read_only: + raise NotImplementedError("IndexedTarStore: the access mode of archives can't be changed") + return self + + def close(self) -> None: + self.store.close() + self._is_open = False + + def _get( + self, key: str, prototype: Optional[BufferPrototype], byte_range: Optional[ByteRequest] + ) -> Optional[Buffer]: + try: + value = self.store[key] + except KeyError: + return None + start, end = _byte_range(byte_range, len(value)) + return (prototype or default_buffer_prototype()).buffer.from_bytes(value[start:end]) + + async def get( + self, key: str, prototype: BufferPrototype, byte_range: Optional[ByteRequest] = None + ) -> Optional[Buffer]: + return self._get(key, prototype, byte_range) + + async def get_partial_values( + self, prototype: BufferPrototype, key_ranges: Iterable[Tuple[str, Optional[ByteRequest]]] + ) -> List[Optional[Buffer]]: + return [self._get(key, prototype, byte_range) for key, byte_range in key_ranges] + + async def exists(self, key: str) -> bool: + return key in self.store + + async def set(self, key: str, value: Buffer) -> None: + self._check_writable() + self.store[key] = value.to_bytes() + + async def set_if_not_exists(self, key: str, value: Buffer) -> None: + if key not in self.store: + await self.set(key, value) + + async def delete(self, key: str) -> None: + """No-op for records that don't exist (zarr-python deletes them when creating nodes)""" + self._check_writable() + if key in self.store: + raise NotImplementedError("IndexedTarStore: records can't be deleted") + + async def delete_dir(self, prefix: str) -> None: + """No-op for prefixes without records (zarr-python deletes them when creating nodes)""" + self._check_writable() + prefix = prefix if prefix == "" or prefix.endswith("/") else prefix + "/" + if any(key.startswith(prefix) for key in self.store.keys()): + raise NotImplementedError("IndexedTarStore: records can't be deleted") + + async def list(self) -> AsyncIterator[str]: + for key in self.store.keys(): + yield key + + async def list_prefix(self, prefix: str) -> AsyncIterator[str]: + for key in self.store.keys(): + if key.startswith(prefix): + yield key + + async def list_dir(self, prefix: str) -> AsyncIterator[str]: + prefix = prefix.rstrip("/") + prefix = prefix + "/" if prefix else "" + children = {key[len(prefix) :].split("/", 1)[0]: None for key in self.store.keys() if key.startswith(prefix)} + for child in children: + yield child + + +def _zarr_store(store: StoreLike) -> Store: + """The zarr-python 3 store of a store (wraps indexed tar stores)""" + return _TarStore(store) if isinstance(store, IndexedTarStore) else cast(Store, store) + + +def _keys(store: Store) -> List[str]: + async def keys() -> List[str]: + return [key async for key in store.list()] + + return run(keys()) + + +def _get(store: Store, key: str) -> Optional[bytes]: + value = run(store.get(key, prototype=default_buffer_prototype())) + return None if value is None else value.to_bytes() + + +def _parent_grouped(metadata: dict) -> dict: + """Orders consolidated node metadata (by node metadata key) by node depth, with the members of each group adjacent. + + zarr-python < 3.4 nests flat consolidated metadata assuming this order (zarr-python#4226: the members of a group + are attached to the first run of keys of the group only), which isn't the order of hierarchies created in arbitrary + order (e.g., by multiple component writers). + """ + + def order(key: str) -> Tuple[int, str, str]: + parent, _, name = key.rsplit("/", 1)[0].rpartition("/") if "/" in key else ("", "", "") + return key.count("/"), parent, name + + return {key: metadata[key] for key in sorted(metadata, key=order)} + + +@contextlib.contextmanager +def _ignored_warnings() -> Iterator[None]: + with warnings.catch_warnings(): + # compressed consolidated metadata records are not zarr nodes + warnings.filterwarnings("ignore", message="Object at .* is not recognized as a component") + yield + + +# ----------------------------------------------------------------------------- +# Backend +# ----------------------------------------------------------------------------- + + +class Backend: + """zarr-python 3 implementation of the version specific operations of nodes and stores""" + + supported_zarr_formats: Tuple[int, ...] = (2,) + + # -- nodes ----------------------------------------------------------------- + + def zarr_format(self, node: Union[zarr.Group, zarr.Array]) -> int: + return node.metadata.zarr_format + + def open_member(self, group: zarr.Group, path: str) -> Union[zarr.Group, zarr.Array]: + node = run(group._async_group.getitem(path)) + return zarr.Group(node) if isinstance(node, AsyncGroup) else zarr.Array(node) + + def member_names(self, group: zarr.Group) -> List[str]: + with _ignored_warnings(): # (members of groups with consolidated metadata are listed without I/O) + return list(group.keys()) + + def has_member(self, group: zarr.Group, name: str) -> bool: + return name in self.member_names(group) + + def member_groups(self, group: zarr.Group) -> List[Tuple[str, zarr.Group]]: + with _ignored_warnings(): + return list(group.groups()) + + def create_group(self, group: zarr.Group, name: str, attributes: Optional[Attributes]) -> zarr.Group: + return group.create_group(name, attributes=None if attributes is None else attributes_dict(attributes)) + + def require_group(self, group: zarr.Group, name: str) -> zarr.Group: + return group.require_group(name) + + def create_array( + self, + group: zarr.Group, + name: str, + data: npt.NDArray[np.generic], + chunks: Union[Tuple[int, ...], Literal["auto"]], + compression: Optional[BloscCompression], + attributes: Optional[Attributes], + ) -> zarr.Array: + return group.create_array( + name, + data=data, + chunks=chunks, + compressors=None if compression is None else compression.to_numcodecs(), + attributes=None if attributes is None else attributes_dict(attributes), + # (store chunks equal to the fill value too, as zarr-python 2: identical records of both versions) + config={"write_empty_chunks": True}, + ) + + def read(self, array: zarr.Array, selection: Selection) -> object: + return run(array.async_array.getitem(selection)) + + def compression(self, array: zarr.Array) -> Optional[BloscCompression]: + if not (compressors := array.compressors): + return None + if len(compressors) != 1 or not isinstance(compressor := compressors[0], numcodecs.Blosc): + raise ValueError(f"Unsupported compressors {compressors}") + return BloscCompression.from_numcodecs(compressor) + + # -- stores ---------------------------------------------------------------- + + def open_directory_store(self, path: str) -> ZarrStore: + return cast(ZarrStore, LocalStore(path)) + + def create_root_group(self, store: StoreLike, zarr_format: Literal[2, 3], attributes: Attributes) -> zarr.Group: + node = run( + zarr_async.create_group( + store=_zarr_store(store), zarr_format=zarr_format, attributes=attributes_dict(attributes) + ) + ) + return zarr.Group(node) + + def open_store(self, store: StoreLike, consolidated: bool) -> zarr.Group: + zarr_store = _zarr_store(store) + if not consolidated: + return zarr.Group( + run(zarr_async.open_group(store=zarr_store, zarr_format=2, mode="r", use_consolidated=False)) + ) + + if (compressed := _get(zarr_store, V2_CONSOLIDATED_METADATA_KEY)) is None: + raise KeyError(V2_CONSOLIDATED_METADATA_KEY) + document = decode_cbor_xz(compressed) + if (consolidated_format := document.get("zarr_consolidated_format")) != 1: + raise ValueError(f"Unsupported zarr consolidated metadata format: {consolidated_format}") + metadata = document["metadata"] + + # the root group of the hierarchy, with the node metadata of all members (no node metadata reads) + prototype = default_buffer_prototype() + root = AsyncGroup._from_bytes_v2( + StorePath(zarr_store, ""), + prototype.buffer.from_bytes(json.dumps(metadata[".zgroup"]).encode()), + prototype.buffer.from_bytes(json.dumps(metadata.get(".zattrs", {})).encode()), + prototype.buffer.from_bytes(json.dumps({**document, "metadata": _parent_grouped(metadata)}).encode()), + ) + return zarr.Group(root) + + def consolidate_store(self, store: StoreLike) -> None: + zarr_store = _zarr_store(store) + metadata = {} + for key in _keys(zarr_store): + if is_v2_metadata_key(key) and (value := _get(zarr_store, key)) is not None: + metadata[key] = json.loads(value) + value = encode_cbor_xz({"zarr_consolidated_format": 1, "metadata": metadata}) + run(zarr_store.set(V2_CONSOLIDATED_METADATA_KEY, default_buffer_prototype().buffer.from_bytes(value))) + + def group_store(self, group: zarr.Group) -> StoreLike: + store = group.store + return store.store if isinstance(store, _TarStore) else cast(ZarrStore, store) + + +backend = Backend() diff --git a/ncore/impl/data/itar.py b/ncore/impl/data/itar.py index d49e50c7..26d160e6 100644 --- a/ncore/impl/data/itar.py +++ b/ncore/impl/data/itar.py @@ -20,7 +20,9 @@ archives; zarr-python version specific store adapters (see :mod:`ncore.impl.data.stores`) store zarr hierarchies in them. -Records are *write-once*: archives are append-only, and writing a record twice raises a ``ValueError``. +Records are *write-once*: archives are append-only, and writing a record twice raises a ``ValueError``. The only +exception are zarr node metadata records (see :func:`is_node_metadata_key`), which zarr-python 3 rewrites on every +update of a node: they are kept in memory while writing and are appended (top-down) when closing the archive. """ from __future__ import annotations @@ -45,6 +47,15 @@ _logger = logging.getLogger(__name__) +#: Names of the records of zarr node metadata (zarr format v2 and v3) +NODE_METADATA_NAMES = (".zarray", ".zgroup", ".zattrs", "zarr.json") + + +def is_node_metadata_key(key: str) -> bool: + """True for keys of zarr node metadata records (zarr format v2 and v3)""" + return key.rsplit("/", 1)[-1] in NODE_METADATA_NAMES + + class IndexedTarStore: """The records of an indexed tar archive, opened for reading or (truncating) writing. @@ -65,7 +76,8 @@ class IndexedTarStore: which don't serialize reads of multiple threads. Archives opened for writing need to be closed (``close()``, or by using the store as context manager), which - appends the record index. + appends the node metadata records and the record index. Until then, node metadata records can be rewritten (see + the module documentation), and are served from memory. All methods are thread-safe. """ @@ -121,6 +133,7 @@ def __init__( self._lock = threading.RLock() self._records: Dict[str, IndexedTarStore.Record] = {} + self._node_metadata: Dict[str, bytes] = {} # (write mode, written when closing) self._tail: Optional[IndexedTarStore.TailBuffer] = None self._tar_file: Optional[tarfile.TarFile] = None self._file: IO[bytes] = self._open_file(truncate=True) @@ -157,21 +170,28 @@ def __repr__(self) -> str: # ------------------------------------------------------------------------- def keys(self) -> List[str]: - """Returns the keys of all records (in archive order)""" + """Returns the keys of all records (in archive order, followed by node metadata records that are written when + closing the archive)""" with self._lock: - return list(self._records) + return list(self._records) + list(self._node_metadata) def __iter__(self) -> Iterator[str]: return iter(self.keys()) def __len__(self) -> int: - return len(self._records) + with self._lock: + return len(self._records) + len(self._node_metadata) def __contains__(self, key: object) -> bool: - return key in self._records + with self._lock: + return key in self._records or key in self._node_metadata def __getitem__(self, key: str) -> bytes: """Reads the payload of a record (raises KeyError if it doesn't exist)""" + if self._node_metadata: + with self._lock: + if (value := self._node_metadata.get(key)) is not None: + return value record = self._records[key] # (records end before the index, which ends the tail read: records starting within the tail read are part of it) @@ -191,15 +211,24 @@ def __getitem__(self, key: str) -> bytes: return value def __setitem__(self, key: str, value: Union[bytes, bytearray, memoryview]) -> None: - """Appends a record (requires write mode, raises ValueError if the record exists already)""" + """Appends a record (requires write mode, raises ValueError if the record exists already). Node metadata + records are appended when closing the archive (and can be rewritten until then).""" if self._mode != "w": raise ValueError("IndexedTarStore: archive is opened for reading and doesn't support writing") payload = bytes(value) with self._lock: + if self._file.closed: + raise ValueError("IndexedTarStore: archive is closed") + if is_node_metadata_key(key): + self._node_metadata[key] = payload + return if key in self._records: raise ValueError(f"IndexedTarStore: record {key} already exists, records are write-once") + self._append(key, payload) + def _append(self, key: str, payload: bytes) -> None: + with self._lock: # the header of a record can span multiple tar blocks (long keys), the payload is padded to full blocks info = tarfile.TarInfo(key) info.size = len(payload) @@ -226,6 +255,10 @@ def close(self) -> None: if self._file.closed: return if self._mode == "w": + # (deterministic order independent of the order of node updates: parents before children) + node_metadata, self._node_metadata = self._node_metadata, {} + for key in sorted(node_metadata, key=lambda key: (key.count("/"), key)): + self._append(key, node_metadata[key]) # closing the tar file appends the end-of-archive blocks, but keeps the file object open cast(tarfile.TarFile, self._tar_file).close() self._save_index(self._file, self._records) diff --git a/ncore/impl/data/nodes.py b/ncore/impl/data/nodes.py index 9af7f246..33359f4b 100644 --- a/ncore/impl/data/nodes.py +++ b/ncore/impl/data/nodes.py @@ -28,20 +28,32 @@ from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Literal, Optional, Tuple, Type, TypeVar, Union, cast import numpy as np +import zarr from zarr import Array as ZarrArray from zarr import Group as ZarrGroup from ncore.impl.data._backend import Attributes, Backend, BloscCompression, Selection, attributes_dict from ncore.impl.data._json import JsonLike -from ncore.impl.data._zarr2 import backend as _zarr2_backend if TYPE_CHECKING: import numpy.typing as npt # type: ignore[import-not-found] -_backend: Backend = _zarr2_backend +def _installed_backend() -> Backend: + """The backend of the installed zarr-python version""" + if int(zarr.__version__.split(".", 1)[0]) >= 3: + from ncore.impl.data._zarr3 import backend as zarr3_backend + + return zarr3_backend + + from ncore.impl.data._zarr2 import backend as zarr2_backend + + return zarr2_backend + + +_backend: Backend = _installed_backend() #: zarr formats supported by the installed zarr-python version SUPPORTED_ZARR_FORMATS: Tuple[int, ...] = _backend.supported_zarr_formats @@ -115,7 +127,7 @@ def zarr_format(self) -> int: @property def attrs(self) -> Dict[str, JsonLike]: """A copy of the attributes of the node""" - return dict(self._node.attrs.asdict()) + return cast(Dict[str, JsonLike], dict(self._node.attrs.asdict())) def attr_str(self, name: str) -> str: """Returns a string attribute (raises KeyError if missing, TypeError for other types)""" diff --git a/ncore/impl/data/stores_test.py b/ncore/impl/data/stores_test.py index 44dd7b5e..6716a3ee 100644 --- a/ncore/impl/data/stores_test.py +++ b/ncore/impl/data/stores_test.py @@ -13,23 +13,29 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio +import concurrent.futures import gc import hashlib +import json import os import struct import tarfile import tempfile +import threading import unittest from pathlib import Path -from typing import Any, Optional, cast +from typing import Any, Optional, Tuple, cast from unittest import mock import numpy as np import parameterized +import zarr from ncore.impl.data import nodes from ncore.impl.data.encoding import decode_cbor_xz +from ncore.impl.data.itar import is_node_metadata_key from ncore.impl.data.stores import ( V2_CONSOLIDATED_METADATA_KEY, IndexedTarStore, @@ -48,6 +54,8 @@ _ROOT_ATTRIBUTES = {"some": "thing"} +ZARR_PYTHON_3 = int(zarr.__version__.split(".", 1)[0]) >= 3 + def _fill_reference(root: nodes.Group) -> None: rng = np.random.default_rng(0) @@ -216,6 +224,35 @@ def test_empty(self): pass self.assertEqual(IndexedTarStore(self.path).keys(), []) + def test_node_metadata_records(self): + """Node metadata records can be rewritten while writing, they are appended when closing (parents first)""" + with IndexedTarStore(self.path, mode="w") as store: + store["g/a/zarr.json"] = b"a v1" + store["g/a/c/0"] = b"chunk" + store["zarr.json"] = b"root v1" + store["g/a/zarr.json"] = b"a v2" # rewritten + store[".zattrs"] = b"attrs" + self.assertEqual(store["g/a/zarr.json"], b"a v2") # readable while writing + self.assertEqual(len(store), 4) + self.assertIn("zarr.json", store) + with self.assertRaises(ValueError): # other records are write-once + store["g/a/c/0"] = b"again" + with self.assertRaises(ValueError): # closed + store["zarr.json"] = b"root v2" + + store = IndexedTarStore(self.path) + self.assertEqual(store.keys(), ["g/a/c/0", ".zattrs", "zarr.json", "g/a/zarr.json"]) + self.assertEqual(store["g/a/zarr.json"], b"a v2") + self.assertEqual(store["zarr.json"], b"root v1") + with tarfile.open(self.path) as tar: # (written once each) + self.assertEqual([m.name for m in tar.getmembers()], store.keys()) + + def test_node_metadata_keys(self): + for key in (".zarray", ".zgroup", ".zattrs", "zarr.json", "a/b/.zarray", "a/zarr.json"): + self.assertTrue(is_node_metadata_key(key), key) + for key in ("zarr.json.cbor.zlib", ".zmetadata.cbor.xz", "a/0.0", "a/c/0/0", "a/x.zarray", "a/zarr.json/0"): + self.assertFalse(is_node_metadata_key(key), key) + class TestStores(unittest.TestCase): """zarr hierarchies in indexed tar archives and directories""" @@ -293,9 +330,23 @@ def test_directory_stores_are_not_truncated(self): _write(self._store("directory", "w")) self.assertEqual((Path(self.directory_path) / "other").read_bytes(), b"other") - def test_archives_of_ncore_19_8(self): - """Archives are byte-identical to archives written by ncore 19.8 (sha256 of a fixed hierarchy written by - ncore 19.8)""" + #: sha256 of the records of the hierarchy of :meth:`_write_golden` written by ncore 19.8 (zarr-python 2) + GOLDEN_RECORDS = { + ".zattrs": "0e326aaa71db8ecc9051e106a4a70695515ad95a95ff348445699fd79f907623", + ".zgroup": "2383746e67b4bcc2762b3f100f06c3fa2d5f149ab5a8e5da5d33521464a01959", + ".zmetadata.cbor.xz": "245a85421d308c67ea2e452746f0628fe0b3bb3a337af3fec3b46e75b044a4cc", + "foo/.zarray": "28db182434d2702c2ccc384e0c1d386359f308bd275821655377b09a40c610fa", + "foo/0.0.0": "e9361a616760d41ac367d54b2349f18184e8a226563ca34047e691cf393a9610", + "foo/1.0.0": "8611647425ac52bdfd586f7342bdb768d363ae139aadafa7130c7a4368015614", + "subgroup/.zattrs": "b638e55387a56bef81862ccd44889e0068a67b04cf00a4c4e90783cf2940e160", + "subgroup/.zgroup": "2383746e67b4bcc2762b3f100f06c3fa2d5f149ab5a8e5da5d33521464a01959", + "subgroup/bar/.zarray": "6c83cd89670edfe4f9f12cf48f7de5f55ed9cefbfa38b168f1ab14dfb082c449", + "subgroup/bar/0": "3c7acfa845b57df9e3a46779d4f17c7eb9d697d63dd8b2c30c176c6fec90051b", + "subgroup/blob/.zarray": "dcb9ebabd3bb2d803fd59453b16ba414f6d55d4451ba597170b4a90fae4e749b", + "subgroup/blob/0": "5399a4c8461dd3fd590854ff73ec4e88db9462165e9160375113dfec0c741e0c", + } + + def _write_golden(self) -> IndexedTarStore: store = IndexedTarStore(self.itar_path, mode="w") root = create_root_group(store, {"some": "thing", "nested": {"a": [1, 2.5, None]}}) root.create_array( @@ -309,17 +360,92 @@ def test_archives_of_ncore_19_8(self): sub.create_bytes_array("blob", b"\x89PNG\x00data") consolidate_store(store) close_store(store) + return IndexedTarStore(self.itar_path) + @unittest.skipIf(ZARR_PYTHON_3, "zarr-python 2 metadata serialization") + def test_archives_of_ncore_19_8(self): + """Records are byte-identical to records written by ncore 19.8, archives are deterministic (node metadata + records are appended when closing archives)""" + store = self._write_golden() + self.assertEqual({key: hashlib.sha256(store[key]).hexdigest() for key in store}, self.GOLDEN_RECORDS) + # data records in creation order, followed by node metadata records (parents first) + self.assertEqual( + store.keys(), + [ + "foo/0.0.0", + "foo/1.0.0", + "subgroup/bar/0", + "subgroup/blob/0", + ".zmetadata.cbor.xz", + ".zattrs", + ".zgroup", + "foo/.zarray", + "subgroup/.zattrs", + "subgroup/.zgroup", + "subgroup/bar/.zarray", + "subgroup/blob/.zarray", + ], + ) digest = hashlib.sha256(Path(self.itar_path).read_bytes()).hexdigest() self.assertIn( digest, { # (python's tarfile module encodes the device numbers of tar headers differently before python 3.9) - "98852fe2314dc3c9680cd0997d6f914785f5f7d2e552c102ed71c9b400340226", # python 3.8 - "016af4d3f0d4964c025832a4864b45ffe13cd68e74436da331086d255592a7de", # python >= 3.9 + "f2f98df32a018a2f178f3561e21bb14eb3bba9da51e00956bb09ed53177000ea", # python 3.8 + "10c02f6fdbaad3c048af23c47eb50fcb539d660d1487df6c42c6e14c14b98a13", # python >= 3.9 }, ) + def test_archives_equivalent_to_ncore_19_8(self): + """Chunk records are byte-identical to chunk records written by ncore 19.8, node metadata documents are + equivalent (zarr-python 3 serializes them differently, and stores empty attributes of arrays)""" + store = self._write_golden() + # (zarr-python 3 writes the chunks of an array concurrently, in any order) + chunks = sorted(key for key in store if not is_node_metadata_key(key) and key != ".zmetadata.cbor.xz") + self.assertEqual(chunks, ["foo/0.0.0", "foo/1.0.0", "subgroup/bar/0", "subgroup/blob/0"]) + self.assertEqual( + {key: hashlib.sha256(store[key]).hexdigest() for key in chunks}, + {key: self.GOLDEN_RECORDS[key] for key in chunks}, + ) + + metadata = decode_cbor_xz(store[".zmetadata.cbor.xz"])["metadata"] + self.assertEqual(metadata, {key: json.loads(store[key]) for key in store if is_node_metadata_key(key)}) + self.assertEqual(metadata[".zattrs"], {"some": "thing", "nested": {"a": [1, 2.5, None]}}) + self.assertEqual(metadata["subgroup/.zattrs"], {"other": [1, 2, 3]}) + self.assertEqual(metadata.get("foo/.zattrs", {}), {}) + self.assertEqual( + {key: value for key, value in metadata["foo/.zarray"].items() if key != "dimension_separator"}, + { + "shape": [3, 3, 3], + "chunks": [2, 3, 3], + "dtype": " int: + return threading.get_ident() + + def run_in_thread() -> int: + return _zarr3.run(thread()) + + self.assertEqual(_zarr3.run(thread()), threading.get_ident()) + with concurrent.futures.ThreadPoolExecutor(2) as pool: + for future in [pool.submit(run_in_thread) for _ in range(4)]: + self.assertNotEqual(future.result(), threading.get_ident()) + + # offloaded work (e.g., of codecs) runs inline + async def offloaded() -> int: + return await asyncio.get_running_loop().run_in_executor(None, threading.get_ident) + + self.assertEqual(_zarr3.run(offloaded()), threading.get_ident()) + + def test_coroutines_within_event_loops(self): + from ncore.impl.data import _zarr3 + + async def value() -> int: + return 42 + + async def main() -> int: + return _zarr3.run(value()) # (runs on zarr-python's I/O loop) + + self.assertEqual(asyncio.run(main()), 42) + + @unittest.skipUnless(hasattr(os, "fork"), "requires fork") + def test_forked_processes(self): + """Forked processes don't reuse the event loop of the forking thread (its selector is shared with the parent)""" + from ncore.impl.data import _zarr3 + + async def loop_and_pid() -> Tuple[int, int]: + return id(asyncio.get_running_loop()), os.getpid() + + parent_loop, _ = _zarr3.run(loop_and_pid()) # (creates the loop of this thread) + read_fd, write_fd = os.pipe() + if (pid := os.fork()) == 0: # child (continues on this thread) + try: + child_loop, child_pid = _zarr3.run(loop_and_pid()) + os.write(write_fd, b"ok" if child_loop != parent_loop and child_pid == os.getpid() else b"reused") + finally: + os._exit(0) + os.close(write_fd) + result = os.read(read_fd, 16) + os.close(read_fd) + os.waitpid(pid, 0) + self.assertEqual(result, b"ok") + self.assertEqual(_zarr3.run(loop_and_pid())[0], parent_loop) # (the parent keeps its loop) + + # the parent's loop is still woken by other threads (children don't close the selector shared with the parent) + async def woken_by_thread() -> int: + loop, future = asyncio.get_running_loop(), asyncio.get_running_loop().create_future() + timer = threading.Timer(0.05, lambda: loop.call_soon_threadsafe(future.set_result, 42)) + timer.start() + return await asyncio.wait_for(future, timeout=5) + + self.assertEqual(_zarr3.run(woken_by_thread()), 42) + + def test_byte_ranges(self): + from ncore.impl.data import _zarr3 + from ncore.impl.data._zarr3 import ( + OffsetByteRequest, + RangeByteRequest, + SuffixByteRequest, + default_buffer_prototype, + ) + + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "store.itar") + with IndexedTarStore(path, mode="w") as store: + store["r"] = b"0123456789" + tar_store = _zarr3._TarStore(IndexedTarStore(path)) + + def get(byte_range: Any) -> Optional[bytes]: + value = _zarr3.run(tar_store.get("r", default_buffer_prototype(), byte_range)) + return None if value is None else value.to_bytes() + + self.assertEqual(get(None), b"0123456789") + self.assertEqual(get(RangeByteRequest(2, 5)), b"234") + self.assertEqual(get(RangeByteRequest(8, 20)), b"89") + self.assertEqual(get(OffsetByteRequest(7)), b"789") + self.assertEqual(get(SuffixByteRequest(3)), b"789") + self.assertEqual(get(SuffixByteRequest(30)), b"0123456789") + self.assertIsNone(_zarr3.run(tar_store.get("missing", default_buffer_prototype()))) + + class TestNodes(unittest.TestCase): """Typed node API""" From dcc7242db8a0a8d7066b11261c21eeb5eebd8b54 Mon Sep 17 00:00:00 2001 From: Janick Martinez Esturo Date: Mon, 12 Oct 2026 11:06:39 +0200 Subject: [PATCH 5/7] feat(data)!: zarr format v3 stores Support writing and reading component stores in zarr format v3 with zarr-python 3 (zarr format v2 remains the default, readable with all zarr-python and ncore versions): - SequenceComponentGroupsWriter(zarr_format=...) / from_reader(), and stores.create_root_group(zarr_format=...), validated against the formats supported by the installed zarr-python version (nodes.check_zarr_format, nodes.SUPPORTED_ZARR_FORMATS) - data converters: --zarr-format CLI option (BaseDataConverterConfig. zarr_format) - the consolidated metadata of zarr format v3 hierarchies is stored as CBOR + zlib encoded root zarr.json (zarr.json.cbor.zlib) only, which decodes several times faster than CBOR + LZMA - binary blobs are uint8 arrays (fixed-length bytes are not part of the zarr format v3 specification), compression uses the blosc codec of zarr format v3 Component, store and converter tests run for both zarr formats. The time-trimmed zarr format v2 / v3 serializations of the test sequence (test-data-v4-trimmed 0.2) are compared by formats_test. BREAKING CHANGE: BaseDataConverterConfig requires zarr_format. --- MODULE.bazel | 7 + docs/data/storage_and_access.rst | 70 +++- ncore/impl/data/_backend.py | 22 +- ncore/impl/data/_zarr3.py | 97 +++++- ncore/impl/data/encoding.py | 13 + ncore/impl/data/nodes.py | 25 +- ncore/impl/data/stores.py | 18 +- ncore/impl/data/stores_test.py | 167 +++++++-- ncore/impl/data/v4/BUILD.bazel | 17 + ncore/impl/data/v4/components.py | 12 + ncore/impl/data/v4/components_test.py | 84 +++-- ncore/impl/data/v4/formats_test.py | 323 ++++++++++++++++++ ncore/impl/data_converter/base.py | 8 +- tools/data_converter/argoverse2/BUILD.bazel | 1 + tools/data_converter/argoverse2/converter.py | 1 + .../argoverse2/converter_test.py | 10 +- tools/data_converter/cli.py | 7 + tools/data_converter/colmap/converter.py | 2 + tools/data_converter/kitti/BUILD.bazel | 1 + tools/data_converter/kitti/converter.py | 1 + tools/data_converter/kitti/converter_test.py | 10 +- tools/data_converter/nuscenes/BUILD.bazel | 1 + tools/data_converter/nuscenes/converter.py | 1 + .../data_converter/nuscenes/converter_test.py | 10 +- tools/data_converter/pai/converter.py | 2 + tools/data_converter/waymo/converter.py | 1 + 26 files changed, 797 insertions(+), 114 deletions(-) create mode 100644 ncore/impl/data/v4/formats_test.py diff --git a/MODULE.bazel b/MODULE.bazel index 3eaedcc6..4042e925 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -126,6 +126,13 @@ http_archive( urls = ["https://maven.pkg.github.com/NVIDIA/ncore/com/nvidia/ncore/test-data-v4/0.2/test-data-v4-0.2.tar.gz"], ) +# time-trimmed zarr format v2 / v3 serializations of the test-data-v4 sequence +http_archive( + name = "test-data-v4-trimmed", + sha256 = "a4a3457829431e30cdc5bb55cc00a8e6d5dfdc5d52db2221594807b4c9fff81d", + urls = ["https://maven.pkg.github.com/NVIDIA/ncore/com/nvidia/ncore/test-data-v4-trimmed/0.2/test-data-v4-trimmed-0.2.tar.gz"], +) + # documentation binary data http_archive( name = "ncore_docs_data", diff --git a/docs/data/storage_and_access.rst b/docs/data/storage_and_access.rst index 679639f1..0efebcc3 100644 --- a/docs/data/storage_and_access.rst +++ b/docs/data/storage_and_access.rst @@ -139,22 +139,59 @@ random access without any index. .. _zarr_versions: -zarr-python Versions --------------------- +zarr Versions and Formats +------------------------- NCore supports both major versions of the `zarr-python `_ -library: ``2.x`` (Python |ge| 3.8) and ``3.x`` (Python |ge| 3.11, version |ge| 3.1.4). -The matching implementation is selected automatically at import time. +library, as well as both versions of the zarr on-disk format specification: -Data written with either version is equivalent (chunk records are byte-identical, -node metadata documents are equivalent JSON) and can be read with either version, -including data written by NCore |le| 19.8. +* **zarr-python** (the installed library): ``2.x`` (Python |ge| 3.8) or ``3.x`` + (Python |ge| 3.11, version |ge| 3.1.4). The matching implementation is selected + automatically at import time. +* **zarr format** (the on-disk layout of component stores): format v2 (``.zgroup`` / + ``.zarray`` / ``.zattrs`` node metadata) or format v3 (``zarr.json`` node metadata). -Hierarchies opened with consolidated metadata (the default of NCore readers) resolve -all zarr nodes from the compressed consolidated metadata record of each component group, -without reading node metadata records. With zarr-python 3, NCore executes zarr's -asynchronous I/O on the calling thread, avoiding per-access thread hand-offs to -zarr-python's global I/O thread. +.. list-table:: + :header-rows: 1 + :widths: 30 35 35 + + * - + - zarr-python 2 + - zarr-python 3 + * - read zarr format v2 + - |check| + - |check| + * - read zarr format v3 + - |mdash| + - |check| + * - write zarr format v2 (default) + - |check| + - |check| + * - write zarr format v3 + - |mdash| + - |check| (opt-in) + +zarr format v2 data written with either zarr-python version is equivalent (chunk records +are byte-identical, node metadata documents are equivalent JSON) and can be read by all +NCore V4 versions (NCore |le| 19.8 supports zarr-python 2 and zarr format v2 only). Data is +therefore written in zarr format v2 by default. zarr format v3 can be selected explicitly +when writing, e.g. + +.. code-block:: python + + writer = SequenceComponentGroupsWriter(..., zarr_format=3) + +or via ``--zarr-format 3`` in the data converter CLIs. In zarr format v3, binary blobs +(e.g., encoded images) are stored as ``uint8`` arrays (zarr format v2: fixed-length bytes). + +Both formats store the consolidated metadata of each component group as a single +compressed record (``.zmetadata.cbor.xz`` for zarr format v2, CBOR + LZMA encoded; +``zarr.json.cbor.zlib`` for zarr format v3, the CBOR + zlib encoded root ``zarr.json`` +including the consolidated metadata of the hierarchy, which is only stored in this +compressed form). Hierarchies opened with consolidated metadata (the default of NCore +readers) resolve all zarr nodes from it, without reading node metadata records. With +zarr-python 3, NCore executes zarr's asynchronous I/O on the calling thread, avoiding +per-access thread hand-offs to zarr-python's global I/O thread. Records of ``.itar`` archives are write-once (archives are append-only): rewriting a record fails. zarr-python 3 rewrites node metadata on every attribute update, so node @@ -165,11 +202,12 @@ versions are appended (parents before children). The groups of component readers / writers (``self._group``) are ``ncore.impl.data.nodes.Group`` instances, which support both zarr-python - versions with a typed API (e.g. ``create_array``, ``array(path).read()``, - ``attrs``). The wrapped zarr-python objects are accessible via ``.zarr`` for - components implemented with the zarr-python API of the installed version - directly. + versions and zarr formats with a typed API (e.g. ``create_array``, + ``array(path).read()``, ``create_bytes_array`` / ``read_bytes``, ``attrs``). The + wrapped zarr-python objects are accessible via ``.zarr`` for components implemented + with the zarr-python API of the installed version directly. +.. |check| unicode:: U+2713 .. |ge| unicode:: U+2265 .. |le| unicode:: U+2264 diff --git a/ncore/impl/data/_backend.py b/ncore/impl/data/_backend.py index 2b124940..9c2770bc 100644 --- a/ncore/impl/data/_backend.py +++ b/ncore/impl/data/_backend.py @@ -22,7 +22,7 @@ import sys import typing -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Mapping, Optional, Tuple, Union, cast import numcodecs import zarr @@ -47,6 +47,11 @@ #: Key of the compressed consolidated metadata of zarr format v2 hierarchies (CBOR + LZMA encoded ``.zmetadata``) V2_CONSOLIDATED_METADATA_KEY = ".zmetadata.cbor.xz" +#: Key of the compressed consolidated metadata of zarr format v3 hierarchies (CBOR + zlib encoded root ``zarr.json`` +#: including the consolidated metadata of the hierarchy, which decodes several times faster than CBOR + LZMA). The root +#: ``zarr.json`` of such hierarchies is only stored in this compressed form. +V3_CONSOLIDATED_METADATA_KEY = "zarr.json.cbor.zlib" + #: Basic selections of array data (integers, slices and Ellipsis) Selection = Union[int, slice, EllipsisType, Tuple[Union[int, slice, EllipsisType], ...]] @@ -105,6 +110,20 @@ def __post_init__(self) -> None: if self.blocksize < 0: raise ValueError(f"Blosc block size needs to be non-negative, got {self.blocksize}") + def to_config(self) -> Dict[str, object]: + """The configuration of the zarr format v3 ``blosc`` codec""" + return {"cname": self.cname, "clevel": self.clevel, "shuffle": self.shuffle, "blocksize": self.blocksize} + + @staticmethod + def from_config(config: Mapping[str, object]) -> BloscCompression: + """From the configuration of a zarr format v3 ``blosc`` codec""" + return BloscCompression( + cname=cast(BloscCname, config["cname"]), + clevel=cast(int, config["clevel"]), + shuffle=cast(BloscShuffle, config["shuffle"]), + blocksize=cast(int, config.get("blocksize", 0)), + ) + def to_numcodecs(self) -> numcodecs.Blosc: return numcodecs.Blosc( cname=self.cname, @@ -215,6 +234,7 @@ def group_store(self, group: zarr.Group) -> StoreLike: "Selection", "StoreLike", "V2_CONSOLIDATED_METADATA_KEY", + "V3_CONSOLIDATED_METADATA_KEY", "ZarrStore", "is_v2_metadata_key", ] diff --git a/ncore/impl/data/_zarr3.py b/ncore/impl/data/_zarr3.py index 5ea4a451..d244e0e0 100644 --- a/ncore/impl/data/_zarr3.py +++ b/ncore/impl/data/_zarr3.py @@ -37,7 +37,9 @@ TYPE_CHECKING, Any, AsyncIterator, + Callable, Coroutine, + Dict, Iterable, Iterator, List, @@ -54,7 +56,9 @@ import zarr import zarr.api.asynchronous as zarr_async +from zarr.abc.codec import Codec from zarr.abc.store import ByteRequest, OffsetByteRequest, RangeByteRequest, Store, SuffixByteRequest +from zarr.codecs import BloscCodec from zarr.core.buffer import Buffer, BufferPrototype, default_buffer_prototype from zarr.core.group import AsyncGroup from zarr.core.sync import sync @@ -62,6 +66,7 @@ from ncore.impl.data._backend import ( V2_CONSOLIDATED_METADATA_KEY, + V3_CONSOLIDATED_METADATA_KEY, Attributes, BloscCompression, Selection, @@ -70,7 +75,7 @@ attributes_dict, is_v2_metadata_key, ) -from ncore.impl.data.encoding import decode_cbor_xz, encode_cbor_xz +from ncore.impl.data.encoding import decode_cbor_xz, decode_cbor_zlib, encode_cbor_xz, encode_cbor_zlib from ncore.impl.data.itar import IndexedTarStore @@ -264,21 +269,35 @@ def _get(store: Store, key: str) -> Optional[bytes]: return None if value is None else value.to_bytes() -def _parent_grouped(metadata: dict) -> dict: - """Orders consolidated node metadata (by node metadata key) by node depth, with the members of each group adjacent. +def _parent_grouped(metadata: dict, node_path: Callable[[str], str]) -> dict: + """Orders consolidated node metadata by node depth, with the members of each group adjacent. zarr-python < 3.4 nests flat consolidated metadata assuming this order (zarr-python#4226: the members of a group are attached to the first run of keys of the group only), which isn't the order of hierarchies created in arbitrary - order (e.g., by multiple component writers). + order (e.g., by multiple component writers). Keys are node metadata keys (zarr format v2) or node paths (zarr format + v3), ``node_path`` maps them to node paths. """ def order(key: str) -> Tuple[int, str, str]: - parent, _, name = key.rsplit("/", 1)[0].rpartition("/") if "/" in key else ("", "", "") - return key.count("/"), parent, name + parent, _, name = (path := node_path(key)).rpartition("/") + return path.count("/"), parent, name return {key: metadata[key] for key in sorted(metadata, key=order)} +def _v2_node_path(key: str) -> str: + return key.rpartition("/")[0] + + +def _v3_node_documents(store: Store) -> Dict[str, object]: + """The node metadata documents of all members of a zarr format v3 hierarchy, by node path""" + documents: Dict[str, object] = {} + for key in _keys(store): + if key.endswith("/zarr.json") and (value := _get(store, key)) is not None: + documents[key[: -len("/zarr.json")]] = json.loads(value) + return documents + + @contextlib.contextmanager def _ignored_warnings() -> Iterator[None]: with warnings.catch_warnings(): @@ -295,7 +314,7 @@ def _ignored_warnings() -> Iterator[None]: class Backend: """zarr-python 3 implementation of the version specific operations of nodes and stores""" - supported_zarr_formats: Tuple[int, ...] = (2,) + supported_zarr_formats: Tuple[int, ...] = (2, 3) # -- nodes ----------------------------------------------------------------- @@ -332,11 +351,17 @@ def create_array( compression: Optional[BloscCompression], attributes: Optional[Attributes], ) -> zarr.Array: + compressor: Union[numcodecs.Blosc, Codec, None] = None + if compression is not None: + if group.metadata.zarr_format == 2: + compressor = compression.to_numcodecs() + else: + compressor = BloscCodec(**cast(Dict[str, Any], compression.to_config())) return group.create_array( name, data=data, chunks=chunks, - compressors=None if compression is None else compression.to_numcodecs(), + compressors=compressor, attributes=None if attributes is None else attributes_dict(attributes), # (store chunks equal to the fill value too, as zarr-python 2: identical records of both versions) config={"write_empty_chunks": True}, @@ -348,9 +373,12 @@ def read(self, array: zarr.Array, selection: Selection) -> object: def compression(self, array: zarr.Array) -> Optional[BloscCompression]: if not (compressors := array.compressors): return None - if len(compressors) != 1 or not isinstance(compressor := compressors[0], numcodecs.Blosc): - raise ValueError(f"Unsupported compressors {compressors}") - return BloscCompression.from_numcodecs(compressor) + if len(compressors) == 1: + if isinstance(compressor := compressors[0], numcodecs.Blosc): # (zarr format v2) + return BloscCompression.from_numcodecs(compressor) + if isinstance(compressor, BloscCodec): # (zarr format v3) + return BloscCompression.from_config(cast(Dict[str, object], compressor.to_dict()["configuration"])) + raise ValueError(f"Unsupported compressors {compressors}") # -- stores ---------------------------------------------------------------- @@ -367,13 +395,23 @@ def create_root_group(self, store: StoreLike, zarr_format: Literal[2, 3], attrib def open_store(self, store: StoreLike, consolidated: bool) -> zarr.Group: zarr_store = _zarr_store(store) - if not consolidated: + + if consolidated: + # the compressed consolidated metadata of zarr format v2 / v3 hierarchies (root zarr.json documents of zarr + # format v3 hierarchies with consolidated metadata are only stored compressed) + if (compressed := _get(zarr_store, V3_CONSOLIDATED_METADATA_KEY)) is not None: + return self._open_v3(zarr_store, decode_cbor_zlib(compressed), consolidated=True) + if (compressed := _get(zarr_store, V2_CONSOLIDATED_METADATA_KEY)) is None: + raise KeyError(V2_CONSOLIDATED_METADATA_KEY) + else: + if (compressed := _get(zarr_store, V3_CONSOLIDATED_METADATA_KEY)) is not None: + return self._open_v3(zarr_store, decode_cbor_zlib(compressed), consolidated=False) + if (root := _get(zarr_store, "zarr.json")) is not None: # (zarr format v3 without consolidated metadata) + return self._open_v3(zarr_store, json.loads(root), consolidated=False) return zarr.Group( run(zarr_async.open_group(store=zarr_store, zarr_format=2, mode="r", use_consolidated=False)) ) - if (compressed := _get(zarr_store, V2_CONSOLIDATED_METADATA_KEY)) is None: - raise KeyError(V2_CONSOLIDATED_METADATA_KEY) document = decode_cbor_xz(compressed) if (consolidated_format := document.get("zarr_consolidated_format")) != 1: raise ValueError(f"Unsupported zarr consolidated metadata format: {consolidated_format}") @@ -385,12 +423,41 @@ def open_store(self, store: StoreLike, consolidated: bool) -> zarr.Group: StorePath(zarr_store, ""), prototype.buffer.from_bytes(json.dumps(metadata[".zgroup"]).encode()), prototype.buffer.from_bytes(json.dumps(metadata.get(".zattrs", {})).encode()), - prototype.buffer.from_bytes(json.dumps({**document, "metadata": _parent_grouped(metadata)}).encode()), + prototype.buffer.from_bytes( + json.dumps({**document, "metadata": _parent_grouped(metadata, _v2_node_path)}).encode() + ), ) return zarr.Group(root) + def _open_v3(self, store: Store, document: Dict[str, Any], consolidated: bool) -> zarr.Group: + """Opens the root group of a zarr format v3 hierarchy from its root document""" + if consolidated: + nodes = document["consolidated_metadata"] + nodes = {**nodes, "metadata": _parent_grouped(nodes["metadata"], str)} + document = {**document, "consolidated_metadata": nodes} + else: + # (members are opened from their node metadata records) + document = {key: value for key, value in document.items() if key != "consolidated_metadata"} + with _ignored_warnings(): + return zarr.Group(AsyncGroup.from_dict(StorePath(store, ""), document)) + def consolidate_store(self, store: StoreLike) -> None: zarr_store = _zarr_store(store) + if (root := _get(zarr_store, "zarr.json")) is not None: + # zarr format v3: the root document with the consolidated metadata of the hierarchy (as written by + # zarr-python's consolidate_metadata) + document = { + **json.loads(root), + "consolidated_metadata": { + "kind": "inline", + "must_understand": False, + "metadata": _v3_node_documents(zarr_store), + }, + } + value = encode_cbor_zlib(document) + run(zarr_store.set(V3_CONSOLIDATED_METADATA_KEY, default_buffer_prototype().buffer.from_bytes(value))) + return + metadata = {} for key in _keys(zarr_store): if is_v2_metadata_key(key) and (value := _get(zarr_store, key)) is not None: diff --git a/ncore/impl/data/encoding.py b/ncore/impl/data/encoding.py index 277221c4..bc6b4ce8 100644 --- a/ncore/impl/data/encoding.py +++ b/ncore/impl/data/encoding.py @@ -20,6 +20,7 @@ import io import lzma +import zlib from typing import Any @@ -39,7 +40,19 @@ def decode_cbor_xz(data: bytes) -> Any: return cbor2.loads(lzma.LZMADecompressor().decompress(data)) +def encode_cbor_zlib(document: Any) -> bytes: + """Encodes a document as CBOR + zlib (decodes several times faster than CBOR + LZMA)""" + return zlib.compress(cbor2.dumps(document), 9) + + +def decode_cbor_zlib(data: bytes) -> Any: + """Decodes a CBOR + zlib encoded document""" + return cbor2.loads(zlib.decompress(data)) + + __all__ = [ "decode_cbor_xz", + "decode_cbor_zlib", "encode_cbor_xz", + "encode_cbor_zlib", ] diff --git a/ncore/impl/data/nodes.py b/ncore/impl/data/nodes.py index 33359f4b..f17e9779 100644 --- a/ncore/impl/data/nodes.py +++ b/ncore/impl/data/nodes.py @@ -58,6 +58,17 @@ def _installed_backend() -> Backend: #: zarr formats supported by the installed zarr-python version SUPPORTED_ZARR_FORMATS: Tuple[int, ...] = _backend.supported_zarr_formats + +def check_zarr_format(zarr_format: int) -> Literal[2, 3]: + """Validates that a zarr format is supported by the installed zarr-python version (raises ValueError otherwise)""" + if zarr_format not in SUPPORTED_ZARR_FORMATS: + raise ValueError( + f"zarr format v{zarr_format} is not supported by zarr-python {zarr.__version__} " + f"(supported: {', '.join(f'v{f}' for f in SUPPORTED_ZARR_FORMATS)})" + ) + return 2 if zarr_format == 2 else 3 + + T = TypeVar("T") @@ -187,6 +198,9 @@ def read_bytes(self) -> bytes: value = _backend.read(self._array, ()) if isinstance(value, bytes): # fixed-length bytes scalar (trailing NUL bytes are stripped) return bytes(value) + if isinstance(value, np.ndarray) and value.dtype.kind == "S" and value.ndim == 0: + # (0-d fixed-length bytes arrays of some zarr-python 3 versions, zarr-python#4466) + return value.tobytes().rstrip(b"\x00") if isinstance(value, np.ndarray) and value.dtype == np.uint8: return value.tobytes() raise TypeError(f"Array {self.path!r} doesn't store a binary blob ({self.dtype})") @@ -298,9 +312,13 @@ def create_array( return Array(_backend.create_array(self._group, name, values, chunks, compression, attributes)) def create_bytes_array(self, name: str, data: bytes, *, attributes: Optional[Attributes] = None) -> Array: - """Stores a binary blob (e.g., an encoded image) as uncompressed member array (0-dimensional fixed-length - bytes array for zarr format v2)""" - return self.create_array(name, np.asarray(data), compression=None, attributes=attributes) + """Stores a binary blob (e.g., an encoded image) as uncompressed member array. + + zarr format v2: 0-dimensional fixed-length bytes array (``|S``, the layout of ncore <= 19.8). zarr format v3: + 1-dimensional ``uint8`` array (fixed-length bytes are not part of the zarr format v3 specification). + """ + value = np.asarray(data) if self.zarr_format == 2 else np.frombuffer(data, dtype=np.uint8) + return self.create_array(name, value, compression=None, attributes=attributes) def _typed_attr(node: _Node, name: str, expected: Type[T]) -> T: @@ -320,4 +338,5 @@ def _join(parent: str, path: str) -> str: "Group", "NodeCache", "SUPPORTED_ZARR_FORMATS", + "check_zarr_format", ] diff --git a/ncore/impl/data/stores.py b/ncore/impl/data/stores.py index f77795f8..a48ffd4e 100644 --- a/ncore/impl/data/stores.py +++ b/ncore/impl/data/stores.py @@ -23,9 +23,15 @@ from upath import UPath -from ncore.impl.data._backend import V2_CONSOLIDATED_METADATA_KEY, Attributes, StoreLike, ZarrStore +from ncore.impl.data._backend import ( + V2_CONSOLIDATED_METADATA_KEY, + V3_CONSOLIDATED_METADATA_KEY, + Attributes, + StoreLike, + ZarrStore, +) from ncore.impl.data.itar import IndexedTarStore -from ncore.impl.data.nodes import Group, _backend +from ncore.impl.data.nodes import Group, _backend, check_zarr_format def open_directory_store(path: Union[str, Path, UPath]) -> ZarrStore: @@ -33,9 +39,10 @@ def open_directory_store(path: Union[str, Path, UPath]) -> ZarrStore: return _backend.open_directory_store(str(path)) -def create_root_group(store: StoreLike, attributes: Attributes) -> Group: - """Creates the root group of a hierarchy in a store""" - return Group(_backend.create_root_group(store, 2, attributes)) +def create_root_group(store: StoreLike, attributes: Attributes, *, zarr_format: int = 2) -> Group: + """Creates the root group of a hierarchy in a store, in zarr format v2 (default, readable with all zarr-python + versions) or v3 (requires zarr-python 3 for writing and reading)""" + return Group(_backend.create_root_group(store, check_zarr_format(zarr_format), attributes)) def open_store(store: StoreLike, consolidated: bool = True) -> Group: @@ -64,6 +71,7 @@ def group_store(group: Group) -> StoreLike: "IndexedTarStore", "StoreLike", "V2_CONSOLIDATED_METADATA_KEY", + "V3_CONSOLIDATED_METADATA_KEY", "ZarrStore", "close_store", "consolidate_store", diff --git a/ncore/impl/data/stores_test.py b/ncore/impl/data/stores_test.py index 6716a3ee..27c50923 100644 --- a/ncore/impl/data/stores_test.py +++ b/ncore/impl/data/stores_test.py @@ -24,11 +24,13 @@ import tempfile import threading import unittest +import zlib from pathlib import Path from typing import Any, Optional, Tuple, cast from unittest import mock +import cbor2 import numpy as np import parameterized import zarr @@ -38,6 +40,7 @@ from ncore.impl.data.itar import is_node_metadata_key from ncore.impl.data.stores import ( V2_CONSOLIDATED_METADATA_KEY, + V3_CONSOLIDATED_METADATA_KEY, IndexedTarStore, StoreLike, close_store, @@ -56,6 +59,14 @@ ZARR_PYTHON_3 = int(zarr.__version__.split(".", 1)[0]) >= 3 +#: All zarr formats (tests of zarr formats unsupported by the installed zarr-python version are skipped) +ZARR_FORMATS = (2, 3) + + +def _require(test: unittest.TestCase, zarr_format: int) -> None: + if zarr_format not in nodes.SUPPORTED_ZARR_FORMATS: + test.skipTest(f"zarr format v{zarr_format} requires zarr-python 3") + def _fill_reference(root: nodes.Group) -> None: rng = np.random.default_rng(0) @@ -88,8 +99,12 @@ def _check_reference(test: unittest.TestCase, root: nodes.Group) -> None: root.array("missing") -def _write(store: StoreLike, consolidate: bool = True) -> None: - _fill_reference(create_root_group(store, _ROOT_ATTRIBUTES)) +def _decode_v3_root(value: bytes) -> Any: + return cbor2.loads(zlib.decompress(value)) + + +def _write(store: StoreLike, consolidate: bool = True, zarr_format: int = 2) -> None: + _fill_reference(create_root_group(store, _ROOT_ATTRIBUTES, zarr_format=zarr_format)) if consolidate: consolidate_store(store) close_store(store) @@ -271,14 +286,22 @@ def _store(self, store_type: str, mode: str) -> StoreLike: return open_directory_store(self.directory_path) @parameterized.parameterized.expand( - [(store_type, consolidated) for store_type in ("itar", "directory") for consolidated in (True, False)] + [ + (store_type, consolidated, zarr_format) + for store_type in ("itar", "directory") + for consolidated in (True, False) + for zarr_format in ZARR_FORMATS + ] ) - def test_reserialization(self, store_type: str, consolidated: bool): - _write(self._store(store_type, "w")) + def test_reserialization(self, store_type: str, consolidated: bool, zarr_format: int): + _require(self, zarr_format) + _write(self._store(store_type, "w"), zarr_format=zarr_format) store = self._store(store_type, "r") root = open_store(store, consolidated) _check_reference(self, root) + self.assertEqual(root.zarr_format, zarr_format) + self.assertEqual(root.array("subgroup/foo").zarr_format, zarr_format) if isinstance(store, IndexedTarStore): self.assertIs(group_store(root), store) @@ -286,9 +309,12 @@ def test_reserialization(self, store_type: str, consolidated: bool): store.reload_resources() _check_reference(self, root) - @parameterized.parameterized.expand([(size,) for size in INDEX_TAIL_READ_SIZES]) - def test_consolidated_open_reads_no_node_metadata(self, index_tail_read_size: Optional[int]): - _write(self._store("itar", "w")) + @parameterized.parameterized.expand( + [(size, zarr_format) for size in INDEX_TAIL_READ_SIZES for zarr_format in ZARR_FORMATS] + ) + def test_consolidated_open_reads_no_node_metadata(self, index_tail_read_size: Optional[int], zarr_format: int): + _require(self, zarr_format) + _write(self._store("itar", "w"), zarr_format=zarr_format) store = IndexedTarStore(self.itar_path, index_tail_read_size=index_tail_read_size) with mock.patch.object( @@ -296,20 +322,40 @@ def test_consolidated_open_reads_no_node_metadata(self, index_tail_read_size: Op ) as getitem: _check_reference(self, open_store(store, consolidated=True)) read_keys = [call.args[1] for call in getitem.call_args_list] - self.assertIn(V2_CONSOLIDATED_METADATA_KEY, read_keys) - self.assertEqual([key for key in read_keys if key.endswith((".zarray", ".zgroup", ".zattrs"))], []) + self.assertIn(V2_CONSOLIDATED_METADATA_KEY if zarr_format == 2 else V3_CONSOLIDATED_METADATA_KEY, read_keys) + self.assertEqual([key for key in read_keys if is_node_metadata_key(key)], []) - @parameterized.parameterized.expand([("itar",), ("directory",)]) - def test_missing_consolidated_metadata(self, store_type: str): - _write(self._store(store_type, "w"), consolidate=False) + @parameterized.parameterized.expand( + [(store_type, zarr_format) for store_type in ("itar", "directory") for zarr_format in ZARR_FORMATS] + ) + def test_missing_consolidated_metadata(self, store_type: str, zarr_format: int): + _require(self, zarr_format) + _write(self._store(store_type, "w"), consolidate=False, zarr_format=zarr_format) with self.assertRaises(KeyError): open_store(self._store(store_type, "r"), consolidated=True) _check_reference(self, open_store(self._store(store_type, "r"), consolidated=False)) - def test_stores_are_identical(self): + @parameterized.parameterized.expand([(zarr_format,) for zarr_format in ZARR_FORMATS]) + def test_unconsolidated_open_reads_node_metadata(self, zarr_format: int): + """Hierarchies opened without consolidated metadata are resolved from their node metadata records (e.g., nodes + added after consolidating, as component writers extending existing directory stores)""" + _require(self, zarr_format) + root = create_root_group(store := self._store("directory", "w"), _ROOT_ATTRIBUTES, zarr_format=zarr_format) + root.create_array("before", np.zeros(2)) + consolidate_store(store) + root.create_array("after", np.ones(2)) + + self.assertEqual( + sorted(open_store(self._store("directory", "r"), consolidated=False).members()), ["after", "before"] + ) + self.assertEqual(sorted(open_store(self._store("directory", "r"), consolidated=True).members()), ["before"]) + + @parameterized.parameterized.expand([(zarr_format,) for zarr_format in ZARR_FORMATS]) + def test_stores_are_identical(self, zarr_format: int): """Indexed tar archives and directories store the same records""" - _write(self._store("itar", "w")) - _write(self._store("directory", "w")) + _require(self, zarr_format) + _write(self._store("itar", "w"), zarr_format=zarr_format) + _write(self._store("directory", "w"), zarr_format=zarr_format) archive = IndexedTarStore(self.itar_path) files = { @@ -321,6 +367,8 @@ def test_stores_are_identical(self): for key in archive.keys(): if key == V2_CONSOLIDATED_METADATA_KEY: # (ordered by the record order of the stores) self.assertEqual(decode_cbor_xz(archive[key]), decode_cbor_xz(files[key])) + elif key == V3_CONSOLIDATED_METADATA_KEY: + self.assertEqual(_decode_v3_root(archive[key]), _decode_v3_root(files[key])) else: self.assertEqual(archive[key], files[key], key) @@ -446,6 +494,52 @@ def test_chunks_of_fill_values_are_stored(self, store_type: str): } self.assertLessEqual({"zeros/0.0", "zeros/1.0", "blob/0"}, keys) + @parameterized.parameterized.expand([("itar",), ("directory",)]) + def test_zarr_format_v3_layout(self, store_type: str): + """The root zarr.json of consolidated zarr format v3 hierarchies is stored compressed only (CBOR + zlib, including + the consolidated metadata), all other node metadata records as written by zarr-python""" + _require(self, 3) + _write(self._store(store_type, "w"), zarr_format=3) + if store_type == "itar": + records = {key: IndexedTarStore(self.itar_path)[key] for key in IndexedTarStore(self.itar_path)} + else: + root_path = Path(self.directory_path) + records = {str(p.relative_to(root_path)): p.read_bytes() for p in root_path.rglob("*") if p.is_file()} + + self.assertIn(V3_CONSOLIDATED_METADATA_KEY, records) + self.assertNotIn(V2_CONSOLIDATED_METADATA_KEY, records) + self.assertFalse(any(key.endswith((".zarray", ".zgroup", ".zattrs")) for key in records)) + node_documents = { + key[: -len("/zarr.json")]: json.loads(value) for key, value in records.items() if key.endswith("/zarr.json") + } + self.assertEqual(sorted(node_documents), ["foo", "subgroup", "subgroup/blob", "subgroup/empty", "subgroup/foo"]) + + root = _decode_v3_root(records[V3_CONSOLIDATED_METADATA_KEY]) + self.assertEqual((root["zarr_format"], root["node_type"], root["attributes"]), (3, "group", _ROOT_ATTRIBUTES)) + consolidated = root["consolidated_metadata"] + self.assertEqual((consolidated["kind"], consolidated["must_understand"]), ("inline", False)) + self.assertEqual(consolidated["metadata"], node_documents) + + # binary blobs are uint8 arrays, compression is the blosc codec of zarr format v3 + self.assertEqual( + (node_documents["subgroup/blob"]["data_type"], node_documents["subgroup/blob"]["shape"]), ("uint8", [21]) + ) + self.assertEqual([codec["name"] for codec in node_documents["subgroup/foo"]["codecs"]], ["bytes", "blosc"]) + self.assertEqual( + {k: v for k, v in node_documents["subgroup/foo"]["codecs"][1]["configuration"].items() if k != "typesize"}, + {"cname": "lz4", "clevel": 5, "shuffle": "bitshuffle", "blocksize": 0}, + ) + + def test_zarr_format_v3_requires_zarr_python_3(self): + if ZARR_PYTHON_3: + self.skipTest("zarr-python 2 only") + with self.assertRaisesRegex(ValueError, "zarr format v3 is not supported"): + create_root_group(self._store("itar", "w"), {}, zarr_format=3) + + def test_unsupported_zarr_formats(self): + with self.assertRaisesRegex(ValueError, "zarr format v4 is not supported"): + create_root_group(self._store("directory", "w"), {}, zarr_format=4) + def test_empty(self): """Stores without a root group can be consolidated, but not opened""" store = IndexedTarStore(self.itar_path, mode="w") @@ -454,8 +548,10 @@ def test_empty(self): with self.assertRaises(Exception): # (zarr-python's error of missing groups) open_store(IndexedTarStore(self.itar_path), consolidated=True) - def test_read_only(self): - _write(self._store("itar", "w")) + @parameterized.parameterized.expand([(zarr_format,) for zarr_format in ZARR_FORMATS]) + def test_read_only(self, zarr_format: int): + _require(self, zarr_format) + _write(self._store("itar", "w"), zarr_format=zarr_format) root = open_store(IndexedTarStore(self.itar_path)) with self.assertRaises(Exception): root.create_group("new") @@ -471,10 +567,13 @@ def setUp(self): def tearDown(self): self._tmp.cleanup() - @parameterized.parameterized.expand([(True,), (False,)]) - def test_interleaved_hierarchy(self, consolidated: bool): + @parameterized.parameterized.expand( + [(consolidated, zarr_format) for consolidated in (True, False) for zarr_format in ZARR_FORMATS] + ) + def test_interleaved_hierarchy(self, consolidated: bool, zarr_format: int): + _require(self, zarr_format) store = IndexedTarStore(self.path, mode="w") - root = create_root_group(store, {}) + root = create_root_group(store, {}, zarr_format=zarr_format) a, b = root.create_group("a"), root.create_group("b") for i in range(3): # members of a and b created alternately a.create_array(f"x{i}", np.full(2, i)) @@ -587,13 +686,19 @@ def get(byte_range: Any) -> Optional[bytes]: self.assertIsNone(_zarr3.run(tar_store.get("missing", default_buffer_prototype()))) +@parameterized.parameterized_class(("zarr_format",), [(zarr_format,) for zarr_format in ZARR_FORMATS]) class TestNodes(unittest.TestCase): """Typed node API""" + zarr_format: int + def setUp(self): + _require(self, self.zarr_format) self._tmp = tempfile.TemporaryDirectory() self.root = create_root_group( - open_directory_store(self._tmp.name), {"s": "x", "i": 3, "b": True, "l": [1], "d": {"k": 1}} + open_directory_store(self._tmp.name), + {"s": "x", "i": 3, "b": True, "l": [1], "d": {"k": 1}}, + zarr_format=self.zarr_format, ) def tearDown(self): @@ -626,6 +731,24 @@ def test_arrays(self): with self.assertRaises(TypeError): array.read_bytes() + def test_bytes_arrays(self): + for data in (b"\x89PNG\x00binary", b"", b"\x00"): + array = self.root.create_bytes_array(f"blob{len(data)}", data, attributes={"format": "png"}) + self.assertEqual(array.zarr_format, self.zarr_format) + self.assertEqual(array.read_bytes(), data.rstrip(b"\x00") if self.zarr_format == 2 else data) + self.assertEqual(array.attrs, {"format": "png"}) + self.assertIsNone(array.compression) + # (zarr format v2: fixed-length bytes, as ncore <= 19.8, which strip trailing NUL bytes) + self.assertEqual(self.root.array("blob11").dtype, np.dtype("S11") if self.zarr_format == 2 else np.uint8) + + @parameterized.parameterized.expand( + [(None,), (nodes.BloscCompression(),), (nodes.BloscCompression("zstd", 3, "shuffle", 256),)] + ) + def test_compression(self, compression: Optional[nodes.BloscCompression]): + array = self.root.create_array("a", np.arange(1000, dtype=np.int32), compression=compression) + self.assertEqual(array.compression, compression) + np.testing.assert_array_equal(array.read(), np.arange(1000)) + def test_groups(self): self.root.create_group("b") self.root.create_group("a").create_group("nested") diff --git a/ncore/impl/data/v4/BUILD.bazel b/ncore/impl/data/v4/BUILD.bazel index b8a88a24..5b8e5ae1 100644 --- a/ncore/impl/data/v4/BUILD.bazel +++ b/ncore/impl/data/v4/BUILD.bazel @@ -99,3 +99,20 @@ pytest_test( "@rules_python//python/runfiles", ], ) + +pytest_test( + name = "pytest_formats", + srcs = ["formats_test.py"], + data = [ + "@test-data-v4-trimmed//:zarr_v2", + "@test-data-v4-trimmed//:zarr_v3", + ], + deps = [ + ":pylib_components", + "//ncore/impl/data:pylib_stores", + requirement("numpy"), + requirement("parameterized"), + requirement("universal_pathlib"), + "@rules_python//python/runfiles", + ], +) diff --git a/ncore/impl/data/v4/components.py b/ncore/impl/data/v4/components.py index 833c841c..da60fab8 100644 --- a/ncore/impl/data/v4/components.py +++ b/ncore/impl/data/v4/components.py @@ -136,6 +136,7 @@ def from_reader( store_base_name: str, sequence_reader: SequenceComponentGroupsReader, store_type: Literal["itar", "directory"] = "itar", # valid values: ['itar', 'directory'] + zarr_format: Literal[2, 3] = 2, ) -> SequenceComponentGroupsWriter: """Creates a SequenceComponentGroupsWriter from an existing SequenceComponentGroupsReader instance to share consistent per-sequence meta-data""" @@ -146,6 +147,7 @@ def from_reader( sequence_timestamp_interval_us=sequence_reader.sequence_timestamp_interval_us, generic_meta_data=sequence_reader.generic_meta_data, store_type=store_type, + zarr_format=zarr_format, ) def __init__( @@ -160,6 +162,9 @@ def __init__( generic_meta_data: Dict[str, types.JsonLike], # Zarr store type: either serialize as .itar archive store (default / production) or plain "directory" store (simpler for introspection / asynchronous / external setup) store_type: Literal["itar", "directory"] = "itar", # valid values: ['itar', 'directory'] + # On-disk zarr format of the component stores: format v2 (default) is readable with all zarr-python versions + # (and all ncore versions), format v3 requires zarr-python 3 for writing *and* reading + zarr_format: Literal[2, 3] = 2, ): """ Instantiate sequence component groups writer and initialize the default data groups and file stores for a given sequence and sensor IDs @@ -177,6 +182,7 @@ def __init__( str, Tuple[nodes.Group, UPath] ] = {} # maps component group names to stores, store path, and base groups self._store_type = store_type + self._zarr_format = nodes.check_zarr_format(zarr_format) # registered component writers self._component_writers: dict[ @@ -195,6 +201,11 @@ def sequence_timestamp_interval_us(self) -> HalfClosedInterval: def generic_meta_data(self) -> Dict[str, types.JsonLike]: return self._generic_meta_data + @property + def zarr_format(self) -> int: + """The on-disk zarr format of the written component stores""" + return self._zarr_format + def get_base_group(self, component_group_name: Optional[str]) -> nodes.Group: """Lazily initializes ncore base-groups and underlying stores on demand""" @@ -240,6 +251,7 @@ def get_base_group(self, component_group_name: Optional[str]) -> nodes.Group: "version": VERSION, "component_group_name": component_group_name, }, + zarr_format=self._zarr_format, ) # Create store / base-group mapping diff --git a/ncore/impl/data/v4/components_test.py b/ncore/impl/data/v4/components_test.py index f9f3451a..9187886c 100644 --- a/ncore/impl/data/v4/components_test.py +++ b/ncore/impl/data/v4/components_test.py @@ -31,7 +31,7 @@ from ncore.impl.common.transformations import HalfClosedInterval from ncore.impl.common.util import unpack_optional -from ncore.impl.data import stores +from ncore.impl.data import nodes, stores from ncore.impl.data.nodes import BloscCompression from ncore.impl.data.types import ( BBox3, @@ -69,19 +69,27 @@ ) -@parameterized_class( - ("store_type"), - [ - ("itar",), - ("directory",), - ], -) +#: Store types and zarr formats of the component tests (zarr formats unsupported by the installed zarr-python version are +#: skipped) +STORE_TYPES_AND_ZARR_FORMATS = [ + (store_type, zarr_format) for store_type in ("itar", "directory") for zarr_format in (2, 3) +] + + +def _require_zarr_format(test: unittest.TestCase, zarr_format: int) -> None: + if zarr_format not in nodes.SUPPORTED_ZARR_FORMATS: + test.skipTest(f"zarr format v{zarr_format} requires zarr-python 3") + + +@parameterized_class(("store_type", "zarr_format"), STORE_TYPES_AND_ZARR_FORMATS) class TestData4Reload(unittest.TestCase): """Test to verify functionality of V4 data writer + loader""" store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] def setUp(self): + _require_zarr_format(self, self.zarr_format) # Make printed errors more representable numerically np.set_printoptions(floatmode="unique", linewidth=200, suppress=True) @@ -107,6 +115,7 @@ def test_reload(self, open_consolidated: bool, node_cache_size: Optional[int]): ref_sequence_timestamp_interval_us := HalfClosedInterval(int(0 * 1e6), int(1 * 1e6) + 1) ), store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data=cast(Dict[str, JsonLike], ref_generic_sequence_meta_data := {"some": 1, "key": 1.2}), ) @@ -512,6 +521,7 @@ def normalize_points(vectors: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: store_base_name=store_writer._store_base_name, sequence_reader=SequenceComponentGroupsReader(store_paths, open_consolidated=open_consolidated), store_type=self.store_type, + zarr_format=self.zarr_format, ) # Store cuboids with the new writer @@ -1104,6 +1114,7 @@ def test_component_generic_data_roundtrip(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -1168,6 +1179,7 @@ def test_component_generic_data_backwards_compat(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -1213,6 +1225,7 @@ def test_component_generic_data_meta_overwrite(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -1262,6 +1275,7 @@ def test_component_generic_data_meta_only(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -1302,13 +1316,7 @@ def test_component_generic_data_meta_only(self) -> None: tempdir.cleanup() -@parameterized_class( - ("store_type"), - [ - ("itar",), - ("directory",), - ], -) +@parameterized_class(("store_type", "zarr_format"), STORE_TYPES_AND_ZARR_FORMATS) class TestDataNewComponent(unittest.TestCase): """ Test to demonstrate how to extend an existing dataset with a new custom component. @@ -1320,8 +1328,10 @@ class TestDataNewComponent(unittest.TestCase): """ store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] def setUp(self): + _require_zarr_format(self, self.zarr_format) np.set_printoptions(floatmode="unique", linewidth=200, suppress=True) def test_new_component_extension(self): @@ -1350,6 +1360,7 @@ def test_new_component_extension(self): sequence_id=sequence_id, sequence_timestamp_interval_us=(timestamp_interval := HalfClosedInterval(int(0 * 1e6), int(10 * 1e6) + 1)), store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={"dataset": "test", "version": 1.0}, ) @@ -1481,6 +1492,7 @@ def get_velocities(self) -> Tuple[np.ndarray, np.ndarray]: output_dir_path=UPath(tempdir.name), # Same directory as initial store_base_name=sequence_id + "-extension", store_type=self.store_type, + zarr_format=self.zarr_format, ) # Now add our custom velocity component to the extended dataset @@ -1652,6 +1664,7 @@ def get_accelerations(self) -> np.ndarray: sequence_id="test_v2", sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={"version": "v2_test"}, ) @@ -1715,19 +1728,15 @@ def get_accelerations(self) -> np.ndarray: # Version compatibility tests passed - all tests completed successfully -@parameterized_class( - ("store_type"), - [ - ("itar",), - ("directory",), - ], -) +@parameterized_class(("store_type", "zarr_format"), STORE_TYPES_AND_ZARR_FORMATS) class TestPointCloudsComponent(unittest.TestCase): """Round-trip tests for the PointCloudsComponent Writer/Reader.""" store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] def setUp(self): + _require_zarr_format(self, self.zarr_format) np.set_printoptions(floatmode="unique", linewidth=200, suppress=True) def _make_writer_reader( @@ -1746,6 +1755,7 @@ def _make_writer_reader( sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -2070,19 +2080,15 @@ def test_generic_data_and_metadata(self): tmpdir.cleanup() -@parameterized_class( - ("store_type"), - [ - ("itar",), - ("directory",), - ], -) +@parameterized_class(("store_type", "zarr_format"), STORE_TYPES_AND_ZARR_FORMATS) class TestCameraLabelsComponent(unittest.TestCase): """Round-trip tests for the CameraLabelsComponent Writer/Reader.""" store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] def setUp(self): + _require_zarr_format(self, self.zarr_format) np.set_printoptions(floatmode="unique", linewidth=200, suppress=True) def _make_writer( @@ -2106,6 +2112,7 @@ def _make_writer( sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -2431,6 +2438,7 @@ def test_multiple_label_types_per_camera(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type="directory", + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -2593,6 +2601,7 @@ def test_reject_empty_camera_id(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -2860,19 +2869,15 @@ def test_invalid_node_cache_size(self): # ============================================================================== -@parameterized_class( - ("store_type"), - [ - ("itar",), - ("directory",), - ], -) +@parameterized_class(("store_type", "zarr_format"), STORE_TYPES_AND_ZARR_FORMATS) class TestZeroDimArraySupport(unittest.TestCase): """Tests that arrays with zero-length dimensions can be stored and read back correctly.""" store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] def setUp(self): + _require_zarr_format(self, self.zarr_format) np.set_printoptions(floatmode="unique", linewidth=200, suppress=True) # ------------------------------------------------------------------ @@ -2889,6 +2894,7 @@ def test_point_cloud_zero_points(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -2952,6 +2958,7 @@ def test_point_cloud_zero_points_multidim_generic_data(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -2994,6 +3001,7 @@ def test_lidar_zero_rays(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -3060,6 +3068,7 @@ def test_radar_zero_rays(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -3116,6 +3125,7 @@ def test_camera_label_zero_shape_suffix_dim(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -3169,6 +3179,7 @@ def test_component_generic_data_zero_dim(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) @@ -3243,6 +3254,7 @@ def test_per_frame_generic_data_zero_dim(self) -> None: sequence_id=seq_id, sequence_timestamp_interval_us=timestamp_interval, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) diff --git a/ncore/impl/data/v4/formats_test.py b/ncore/impl/data/v4/formats_test.py new file mode 100644 index 00000000..aa0b539e --- /dev/null +++ b/ncore/impl/data/v4/formats_test.py @@ -0,0 +1,323 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Cross-format / cross-runtime consistency of NCore V4 test data + +Compares the zarr format v2 and zarr format v3 serializations of the same (time-trimmed) test sequence, as well as +re-serializations performed by the zarr-python version under test. +""" + +from __future__ import annotations + +import tempfile +import unittest + +from typing import Any, Dict, List, Literal, Optional, cast + +import numpy as np +import parameterized + +from python.runfiles import Runfiles +from upath import UPath + +from ncore.impl.data import nodes +from ncore.impl.data.v4.components import ( + CameraSensorComponent, + CuboidsComponent, + IntrinsicsComponent, + LidarSensorComponent, + MasksComponent, + PosesComponent, + RadarSensorComponent, + SequenceComponentGroupsReader, + SequenceComponentGroupsWriter, +) + + +_RUNFILES = Runfiles.Create() +assert _RUNFILES is not None +_RUNFILES_: Runfiles = _RUNFILES + +_SEQUENCE = "c9b05cf4-afb9-11ec-b3c2-00044bf65fcb@1648597318700123-1648599151600035" + + +def _dataset(name: str) -> UPath: + path = UPath(_RUNFILES_.Rlocation(f"test-data-v4-trimmed/{name}/{_SEQUENCE}.json") or "") + assert path.exists(), f"test data {name} not found (missing data dependency?)" + return path + + +_TRIMMED_V2 = _dataset("zarr-v2") +_TRIMMED_V3 = _dataset("zarr-v3") + + +#: Frame subsampling of the reserialization test (every N-th frame, plus the last frame of each sensor) +_FRAME_STRIDE = 5 + + +def _subsampled(frames_timestamps_us: np.ndarray) -> np.ndarray: + """Subsampled frames (every _FRAME_STRIDE-th frame, plus the last frame)""" + indices = sorted(set(range(0, len(frames_timestamps_us), _FRAME_STRIDE)) | {len(frames_timestamps_us) - 1}) + return frames_timestamps_us[[i for i in indices if i >= 0]] + + +def _summary(reader: SequenceComponentGroupsReader, subsampled: bool = False, frames: bool = True) -> Dict[str, Any]: + """Collects a comparable summary of all data of a sequence (optionally of the subsampled frames only, or without + masks and sensor frames)""" + + def selected(timestamps_us: np.ndarray) -> np.ndarray: + return _subsampled(timestamps_us) if subsampled else timestamps_us + + out: Dict[str, Any] = { + "sequence_id": reader.sequence_id, + "interval": (reader.sequence_timestamp_interval_us.start, reader.sequence_timestamp_interval_us.stop), + "generic_meta_data": reader.generic_meta_data, + } + + for name, poses in sorted(reader.open_component_readers(PosesComponent.Reader).items()): + out[f"poses/{name}/static"] = {k: v.tolist() for k, v in poses.get_static_poses()} + out[f"poses/{name}/dynamic"] = {k: (p.tolist(), t.tolist()) for k, (p, t) in poses.get_dynamic_poses()} + + for name, intrinsics in sorted(reader.open_component_readers(IntrinsicsComponent.Reader).items()): + out[f"intrinsics/{name}"] = (intrinsics._group.group("cameras").attrs, intrinsics._group.group("lidars").attrs) + + for name, cuboids in sorted(reader.open_component_readers(CuboidsComponent.Reader).items()): + out[f"cuboids/{name}"] = [obs.to_dict() for obs in cuboids.get_observations()] + + if not frames: + return out + + for name, masks in sorted(reader.open_component_readers(MasksComponent.Reader).items()): + out[f"masks/{name}"] = { + camera_id: { + mask_name: np.asarray(image).tobytes() for mask_name, image in masks.get_camera_mask_images(camera_id) + } + for camera_id in masks._group.group("cameras").members() + } + + for name, cameras in sorted(reader.open_component_readers(CameraSensorComponent.Reader).items()): + frames: List[object] = [] + for ts in selected(cameras.frames_timestamps_us)[:, 1]: + ts = int(ts) + data = cameras.get_frame_data(ts) + frames.append( + ( + ts, + data.get_encoded_image_format(), + data.get_encoded_image_data(), + { + n: cameras.get_frame_generic_data(ts, n).tobytes() + for n in sorted(cameras.get_frame_generic_data_names(ts)) + }, + cameras.get_frame_generic_meta_data(ts), + ) + ) + out[f"cameras/{name}"] = frames + + for component in (LidarSensorComponent, RadarSensorComponent): + for name, sensor in sorted(reader.open_component_readers(component.Reader).items()): + sensor_frames = [] + for ts in selected(sensor.frames_timestamps_us)[:, 1]: + ts = int(ts) + sensor_frames.append( + ( + ts, + { + n: sensor.get_frame_ray_bundle_data(ts, n).tobytes() + for n in sorted(sensor.get_frame_ray_bundle_data_names(ts)) + }, + { + n: sensor.get_frame_ray_bundle_return_data(ts, n, None).tobytes() + for n in sorted(sensor.get_frame_ray_bundle_return_data_names(ts)) + }, + sensor.get_frame_ray_bundle_return_valid_mask(ts).tobytes(), + { + n: sensor.get_frame_generic_data(ts, n).tobytes() + for n in sorted(sensor.get_frame_generic_data_names(ts)) + }, + sensor.get_frame_generic_meta_data(ts), + ) + ) + out[f"{component.COMPONENT_NAME}/{name}"] = sensor_frames + + return out + + +class TestZarrFormats(unittest.TestCase): + """zarr format v2 and v3 serializations of a sequence need to be identical in content""" + + reference: Dict[str, Any] # summary of the zarr format v2 test data + reference_subsampled: Dict[str, Any] # summary of the subsampled frames of the zarr format v2 test data + + @classmethod + def setUpClass(cls) -> None: + cls.reference = _summary(SequenceComponentGroupsReader([_TRIMMED_V2])) + cls.reference_subsampled = _summary(SequenceComponentGroupsReader([_TRIMMED_V2]), subsampled=True) + + def test_v2_v3_consistency(self): + if 3 not in nodes.SUPPORTED_ZARR_FORMATS: + # zarr format v3 data can't be read with zarr-python 2 + with self.assertRaises(Exception): + SequenceComponentGroupsReader([_TRIMMED_V3]) + return + + _assert_summaries_equal(self, self.reference, _summary(SequenceComponentGroupsReader([_TRIMMED_V3]))) + + @parameterized.parameterized.expand([(None,), (0,), (1,), (16,)]) + def test_node_cache_sizes(self, node_cache_size: Optional[int]): + """All node cache configurations of the readers return the same data, and caches respect their bounds""" + reader = SequenceComponentGroupsReader([_TRIMMED_V2], node_cache_size=node_cache_size) + _assert_summaries_equal(self, self.reference, _summary(reader)) + + lidar = next(iter(reader.open_component_readers(LidarSensorComponent.Reader).values())) + frames = lidar.frames_timestamps_us[:, 1] + for _ in range(2): + for ts in frames: + lidar.get_frame_ray_bundle_data(int(ts), "direction") + if node_cache_size == 0: + self.assertIsNone(lidar._group.cache) + else: + assert (cache := lidar._group.cache) is not None + self.assertEqual(cache.max_entries, node_cache_size) + self.assertLessEqual(len(cache), node_cache_size or len(cache)) + frame_entries = [key for key in cache.keys() if key.endswith("/ray_bundle/direction")] + self.assertEqual(len(frame_entries), min(len(frames), node_cache_size or len(frames))) + + def test_invalid_node_cache_size(self): + with self.assertRaises(ValueError): + SequenceComponentGroupsReader([_TRIMMED_V2], node_cache_size=-1) + + @parameterized.parameterized.expand([(f,) for f in nodes.SUPPORTED_ZARR_FORMATS]) + def test_reserialization(self, zarr_format: int): + """Re-serializing (subsampled frames of) all components of the test data with the installed zarr-python version + via the public component APIs preserves all content""" + reader = SequenceComponentGroupsReader([_TRIMMED_V2]) + + with tempfile.TemporaryDirectory() as output_dir: + writer = SequenceComponentGroupsWriter.from_reader( + UPath(output_dir), "copy", reader, zarr_format=cast(Literal[2, 3], zarr_format) + ) + _copy_components(reader, writer) + copy = _summary(SequenceComponentGroupsReader(writer.finalize())) # (contains the subsampled frames only) + + self.assertEqual(self.reference_subsampled.keys(), copy.keys()) + for component in ("poses/", "intrinsics/", "masks/", "cameras/", "lidars/", "radars/", "cuboids/"): + self.assertTrue(any(key.startswith(component) for key in copy), f"no {component} data in the test data") + _assert_summaries_equal(self, self.reference_subsampled, copy) + + +def _assert_summaries_equal(test: unittest.TestCase, expected: Dict[str, Any], actual: Dict[str, Any]) -> None: + """Compares sequence summaries per component / frame / field (without expensive diffs of large payloads)""" + test.assertEqual(sorted(expected), sorted(actual)) + for key, expected_value in expected.items(): + actual_value = actual[key] + if not isinstance(expected_value, list): + test.assertTrue(expected_value == actual_value, key) + continue + test.assertEqual(len(expected_value), len(actual_value), key) + for index, (expected_frame, actual_frame) in enumerate(zip(expected_value, actual_value)): + if not isinstance(expected_frame, tuple): + test.assertTrue(expected_frame == actual_frame, f"{key}[{index}]") + continue + for field, (expected_field, actual_field) in enumerate(zip(expected_frame, actual_frame)): + if isinstance(expected_field, dict): + expected_items = cast(Dict[str, object], expected_field) + actual_items = cast(Dict[str, object], actual_field) + test.assertEqual(sorted(expected_items), sorted(actual_items), f"{key}[{index}][{field}]") + for name, expected_item in expected_items.items(): + test.assertTrue(expected_item == actual_items[name], f"{key}[{index}][{field}][{name}] differs") + else: + test.assertTrue(expected_field == actual_field, f"{key}[{index}][{field}] differs") + + +def _copy_components(reader: SequenceComponentGroupsReader, writer: SequenceComponentGroupsWriter) -> None: + """Copies (subsampled frames of) all components of a sequence via the public component APIs""" + + for name, poses in reader.open_component_readers(PosesComponent.Reader).items(): + poses_writer = writer.register_component_writer(PosesComponent.Writer, name, name, poses.generic_meta_data) + for (source, target), pose in poses.get_static_poses(): + poses_writer.store_static_pose(source, target, pose) + for (source, target), (trajectory, timestamps_us) in poses.get_dynamic_poses(): + poses_writer.store_dynamic_pose(source, target, trajectory, timestamps_us) + + for name, intrinsics in reader.open_component_readers(IntrinsicsComponent.Reader).items(): + intrinsics_writer = writer.register_component_writer( + IntrinsicsComponent.Writer, name, name, intrinsics.generic_meta_data + ) + for camera_id in intrinsics._group.group("cameras").members(): + intrinsics_writer.store_camera_intrinsics(camera_id, intrinsics.get_camera_model_parameters(camera_id)) + for lidar_id in intrinsics._group.group("lidars").members(): + if (lidar_model := intrinsics.get_lidar_model_parameters(lidar_id)) is not None: + intrinsics_writer.store_lidar_intrinsics(lidar_id, lidar_model) + + for name, masks in reader.open_component_readers(MasksComponent.Reader).items(): + masks_writer = writer.register_component_writer(MasksComponent.Writer, name, name, masks.generic_meta_data) + for camera_id in masks._group.group("cameras").members(): + masks_writer.store_camera_masks(camera_id, dict(masks.get_camera_mask_images(camera_id))) + + for name, cameras in reader.open_component_readers(CameraSensorComponent.Reader).items(): + camera_writer = writer.register_component_writer( + CameraSensorComponent.Writer, name, name, cameras.generic_meta_data + ) + for frame_timestamps_us in _subsampled(cameras.frames_timestamps_us): + ts = int(frame_timestamps_us[1]) + data = cameras.get_frame_data(ts) + camera_writer.store_frame( + data.get_encoded_image_data(), + data.get_encoded_image_format(), + frame_timestamps_us, + {n: cameras.get_frame_generic_data(ts, n) for n in cameras.get_frame_generic_data_names(ts)}, + cameras.get_frame_generic_meta_data(ts), + ) + + for name, lidar in reader.open_component_readers(LidarSensorComponent.Reader).items(): + lidar_writer = writer.register_component_writer( + LidarSensorComponent.Writer, name, name, lidar.generic_meta_data + ) + for frame_timestamps_us in _subsampled(lidar.frames_timestamps_us): + ts = int(frame_timestamps_us[1]) + ray_names = lidar.get_frame_ray_bundle_data_names(ts) + lidar_writer.store_frame( + direction=lidar.get_frame_ray_bundle_data(ts, "direction"), + timestamp_us=lidar.get_frame_ray_bundle_data(ts, "timestamp_us"), + model_element=( + lidar.get_frame_ray_bundle_data(ts, "model_element") if "model_element" in ray_names else None + ), + distance_m=lidar.get_frame_ray_bundle_return_data(ts, "distance_m", None), + intensity=lidar.get_frame_ray_bundle_return_data(ts, "intensity", None), + frame_timestamps_us=frame_timestamps_us, + generic_data={n: lidar.get_frame_generic_data(ts, n) for n in lidar.get_frame_generic_data_names(ts)}, + generic_meta_data=lidar.get_frame_generic_meta_data(ts), + ) + + for name, radar in reader.open_component_readers(RadarSensorComponent.Reader).items(): + radar_writer = writer.register_component_writer( + RadarSensorComponent.Writer, name, name, radar.generic_meta_data + ) + for frame_timestamps_us in _subsampled(radar.frames_timestamps_us): + ts = int(frame_timestamps_us[1]) + radar_writer.store_frame( + direction=radar.get_frame_ray_bundle_data(ts, "direction"), + timestamp_us=radar.get_frame_ray_bundle_data(ts, "timestamp_us"), + distance_m=radar.get_frame_ray_bundle_return_data(ts, "distance_m", None), + frame_timestamps_us=frame_timestamps_us, + generic_data={n: radar.get_frame_generic_data(ts, n) for n in radar.get_frame_generic_data_names(ts)}, + generic_meta_data=radar.get_frame_generic_meta_data(ts), + ) + + for name, cuboids in reader.open_component_readers(CuboidsComponent.Reader).items(): + writer.register_component_writer( + CuboidsComponent.Writer, name, name, cuboids.generic_meta_data + ).store_observations(list(cuboids.get_observations())) diff --git a/ncore/impl/data_converter/base.py b/ncore/impl/data_converter/base.py index 7a692bf6..347a813b 100644 --- a/ncore/impl/data_converter/base.py +++ b/ncore/impl/data_converter/base.py @@ -20,7 +20,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Optional, Tuple +from typing import Literal, Optional, Tuple from upath import UPath @@ -45,6 +45,9 @@ class BaseDataConverterConfig: debug: bool debug_port: int + ## Output + zarr_format: Literal[2, 3] # On-disk zarr format of written component stores (3 requires zarr-python 3) + class BaseDataConverter(ABC): """ @@ -62,6 +65,9 @@ def __init__(self, config: BaseDataConverterConfig) -> None: self.output_dir = UPath(config.output_dir) + # On-disk zarr format of written component stores + self.zarr_format: Literal[2, 3] = config.zarr_format + # External sensor selection overwrites # Store `None`` for `_active__ids` in case all sensors should be used, as the # actual full list of sensor ids will be passed via `get_active__ids()` diff --git a/tools/data_converter/argoverse2/BUILD.bazel b/tools/data_converter/argoverse2/BUILD.bazel index a645eb51..cf67b628 100644 --- a/tools/data_converter/argoverse2/BUILD.bazel +++ b/tools/data_converter/argoverse2/BUILD.bazel @@ -67,6 +67,7 @@ alias( # Integration test for the Argoverse 2 converter (requires AV2_DIR env var) pytest_test( name = "pytest_converter", + size = "large", # converts the test sequence for all store types / zarr formats srcs = ["converter_test.py"], python_versions = ["3.11"], tags = ["manual"], # Only run when explicitly requested (needs external data) diff --git a/tools/data_converter/argoverse2/converter.py b/tools/data_converter/argoverse2/converter.py index 7c999e1a..ceba5c5a 100644 --- a/tools/data_converter/argoverse2/converter.py +++ b/tools/data_converter/argoverse2/converter.py @@ -245,6 +245,7 @@ def convert_sequence(self, sequence_id: str) -> None: sequence_id=log_id, sequence_timestamp_interval_us=sequence_timestamp_interval_us, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={ "source_dataset": "argoverse2", "argoverse2_split": self._split, diff --git a/tools/data_converter/argoverse2/converter_test.py b/tools/data_converter/argoverse2/converter_test.py index 19cc1e31..e76c7fca 100644 --- a/tools/data_converter/argoverse2/converter_test.py +++ b/tools/data_converter/argoverse2/converter_test.py @@ -33,6 +33,7 @@ from parameterized import parameterized_class from upath import UPath +from ncore.impl.data import nodes from ncore.impl.data.types import ( IdealPinholeCameraModelParameters, RowOffsetStructuredSpinningLidarModelParameters, @@ -52,11 +53,8 @@ @parameterized_class( - ("store_type",), - [ - ("itar",), - ("directory",), - ], + ("store_type", "zarr_format"), + [(store_type, zarr_format) for store_type in ("itar", "directory") for zarr_format in nodes.SUPPORTED_ZARR_FORMATS], ) class TestArgoverse2Converter(unittest.TestCase): """Integration tests for the Argoverse 2 data converter. @@ -66,6 +64,7 @@ class TestArgoverse2Converter(unittest.TestCase): """ store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] @classmethod def setUpClass(cls): @@ -97,6 +96,7 @@ def setUpClass(cls): split=cls.split, log_id=cls.log_id, store_type=cls.store_type, + zarr_format=cls.zarr_format, component_group_profile="separate-sensors", store_sequence_meta=True, ) diff --git a/tools/data_converter/cli.py b/tools/data_converter/cli.py index 2e394587..89eb38e3 100644 --- a/tools/data_converter/cli.py +++ b/tools/data_converter/cli.py @@ -44,6 +44,13 @@ @click.option( "--debug-port", type=int, default=5678, help="The port on which debugpy will wait for a client to connect" ) +@click.option( + "--zarr-format", + type=click.Choice(["2", "3"]), + default="2", + callback=lambda _ctx, _param, value: int(value), + help="On-disk zarr format of written component stores (format 3 requires zarr-python 3 for writing and reading)", +) @click.option("--no-cameras", is_flag=True, default=False, help="Disable exporting any camera sensor") @click.option( "--camera-id", diff --git a/tools/data_converter/colmap/converter.py b/tools/data_converter/colmap/converter.py index 2cb28c7c..1e3e4d26 100644 --- a/tools/data_converter/colmap/converter.py +++ b/tools/data_converter/colmap/converter.py @@ -265,6 +265,7 @@ def convert_sequence(self, sequence_id: str) -> None: sequence_id=self.sequence_name, sequence_timestamp_interval_us=sequence_timestamp_interval_us, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data=self.generic_meta_data, ) @@ -768,6 +769,7 @@ def scannetpp_v4(ctx, **kwargs): lidar_ids=base.lidar_ids, no_radars=base.no_radars, radar_ids=base.radar_ids, + zarr_format=base.zarr_format, colmap_dir="dslr/colmap", images_dir=images_dir, masks_dir=masks_dir, diff --git a/tools/data_converter/kitti/BUILD.bazel b/tools/data_converter/kitti/BUILD.bazel index 04946c07..e9e8a5e3 100644 --- a/tools/data_converter/kitti/BUILD.bazel +++ b/tools/data_converter/kitti/BUILD.bazel @@ -59,6 +59,7 @@ alias( # Integration test for the KITTI converter (requires KITTI_RAW_DIR env var) pytest_test( name = "pytest_converter", + size = "large", # converts the test sequence for all store types / zarr formats srcs = ["converter_test.py"], python_versions = ["3.11"], tags = ["manual"], # Only run when explicitly requested (needs external data) diff --git a/tools/data_converter/kitti/converter.py b/tools/data_converter/kitti/converter.py index 39b464a3..919039af 100644 --- a/tools/data_converter/kitti/converter.py +++ b/tools/data_converter/kitti/converter.py @@ -211,6 +211,7 @@ def convert_sequence(self, sequence_id: str) -> None: sequence_id=sequence_name, sequence_timestamp_interval_us=sequence_timestamp_interval_us, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, ) diff --git a/tools/data_converter/kitti/converter_test.py b/tools/data_converter/kitti/converter_test.py index 17121df6..5c59ae38 100644 --- a/tools/data_converter/kitti/converter_test.py +++ b/tools/data_converter/kitti/converter_test.py @@ -26,6 +26,7 @@ from parameterized import parameterized_class from upath import UPath +from ncore.impl.data import nodes from ncore.impl.data.types import IdealPinholeCameraModelParameters from ncore.impl.data.v4.components import ( CameraSensorComponent, @@ -39,11 +40,8 @@ @parameterized_class( - ("store_type"), - [ - ("itar",), - ("directory",), - ], + ("store_type", "zarr_format"), + [(store_type, zarr_format) for store_type in ("itar", "directory") for zarr_format in nodes.SUPPORTED_ZARR_FORMATS], ) class TestKittiConverter(unittest.TestCase): """Integration tests for KITTI raw data converter. @@ -54,6 +52,7 @@ class TestKittiConverter(unittest.TestCase): """ store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] @classmethod def setUpClass(cls): @@ -78,6 +77,7 @@ def setUpClass(cls): debug=False, debug_port=5678, store_type=cls.store_type, + zarr_format=cls.zarr_format, component_group_profile="separate-sensors", store_sequence_meta=True, ) diff --git a/tools/data_converter/nuscenes/BUILD.bazel b/tools/data_converter/nuscenes/BUILD.bazel index e16cb101..8eb13151 100644 --- a/tools/data_converter/nuscenes/BUILD.bazel +++ b/tools/data_converter/nuscenes/BUILD.bazel @@ -65,6 +65,7 @@ alias( # Integration test for the nuScenes converter (requires NUSCENES_DIR env var) pytest_test( name = "pytest_converter", + size = "large", # converts the test sequence for all store types / zarr formats srcs = ["converter_test.py"], python_versions = ["3.11"], tags = ["manual"], # Only run when explicitly requested (needs external data) diff --git a/tools/data_converter/nuscenes/converter.py b/tools/data_converter/nuscenes/converter.py index e8b31915..0332a22b 100644 --- a/tools/data_converter/nuscenes/converter.py +++ b/tools/data_converter/nuscenes/converter.py @@ -280,6 +280,7 @@ def convert_sequence(self, sequence_id: str) -> None: sequence_id=sequence_output_name, sequence_timestamp_interval_us=sequence_timestamp_interval_us, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={ "source_dataset": "nuscenes", "nuscenes_version": self._version, diff --git a/tools/data_converter/nuscenes/converter_test.py b/tools/data_converter/nuscenes/converter_test.py index c13e8198..649205e9 100644 --- a/tools/data_converter/nuscenes/converter_test.py +++ b/tools/data_converter/nuscenes/converter_test.py @@ -33,6 +33,7 @@ from parameterized import parameterized_class from upath import UPath +from ncore.impl.data import nodes from ncore.impl.data.types import IdealPinholeCameraModelParameters, RowOffsetStructuredSpinningLidarModelParameters from ncore.impl.data.v4.components import ( CameraSensorComponent, @@ -49,11 +50,8 @@ @parameterized_class( - ("store_type",), - [ - ("itar",), - ("directory",), - ], + ("store_type", "zarr_format"), + [(store_type, zarr_format) for store_type in ("itar", "directory") for zarr_format in nodes.SUPPORTED_ZARR_FORMATS], ) class TestNuScenesConverter(unittest.TestCase): """Integration tests for nuScenes data converter. @@ -63,6 +61,7 @@ class TestNuScenesConverter(unittest.TestCase): """ store_type: Literal["itar", "directory"] + zarr_format: Literal[2, 3] @classmethod def setUpClass(cls): @@ -97,6 +96,7 @@ def setUpClass(cls): scene_token=cls.scene_token, scene_name=None, store_type=cls.store_type, + zarr_format=cls.zarr_format, component_group_profile="separate-sensors", store_sequence_meta=True, ) diff --git a/tools/data_converter/pai/converter.py b/tools/data_converter/pai/converter.py index f6bd419c..ba79f23e 100644 --- a/tools/data_converter/pai/converter.py +++ b/tools/data_converter/pai/converter.py @@ -175,6 +175,7 @@ def _init_pai_fields(self, config) -> None: self.seek_sec: float | None = config.seek_sec self.duration_sec: float | None = config.duration_sec self.store_type: Literal["itar", "directory"] = config.store_type + self.zarr_format: Literal[2, 3] = config.zarr_format self.component_group_profile: Literal["default", "separate-sensors", "separate-all"] = ( config.component_group_profile ) @@ -286,6 +287,7 @@ def _convert_clip(self, clip_id: str) -> None: sequence_id=sequence_id, sequence_timestamp_interval_us=self.sequence_timestamp_interval_us, store_type=self.store_type, # can also be "directory" + zarr_format=self.zarr_format, generic_meta_data={**source_generic_meta_data, **generic_meta_data}, ) diff --git a/tools/data_converter/waymo/converter.py b/tools/data_converter/waymo/converter.py index 50196768..2eb47649 100644 --- a/tools/data_converter/waymo/converter.py +++ b/tools/data_converter/waymo/converter.py @@ -243,6 +243,7 @@ def convert_sequence(self, sequence_id: str) -> None: sequence_id=sequence_name, sequence_timestamp_interval_us=sequence_timestamp_interval_us, store_type=self.store_type, + zarr_format=self.zarr_format, generic_meta_data={}, # no generic sequence meta data ) From 34e17431991a7f95e87a4b14afb40099978d40cd Mon Sep 17 00:00:00 2001 From: Janick Martinez Esturo Date: Mon, 12 Oct 2026 11:35:33 +0200 Subject: [PATCH 6/7] perf(data): direct chunk reads with zarr-python 3 zarr-python 3's codec pipeline decodes each chunk into a temporary buffer and copies it into the output array (zarr-python#2904), and its per-read overhead dominates reads of small frame arrays (~550 us per frame read vs ~200 us with zarr-python 2). Basic selections (integers, unit-step slices, Ellipsis) of arrays of indexed tar archives opened for reading are now read directly: chunks are determined by zarr-python's indexer, read from the archive and blosc-decompressed into the output array (in place for complete chunks). Uncompressed single-chunk arrays (e.g., encoded images) are returned without decoding. Large chunks of a read (>= 2 MiB) are decoded concurrently on a thread pool, bounded to two concurrent reads per process. Supported are regular chunk grids, C order, numeric / boolean data of native byte order and fixed-length bytes, without filters, uncompressed or blosc compressed (all arrays written by ncore), all other arrays and selections are read via zarr-python. Read plans are derived per read (sub-microsecond, nothing is cached), results are writeable, missing chunks read as the fill value, and the parallel reads state is re-created in forked processes. NCORE_ZARR3_DIRECT_READS=0 reads all arrays via zarr-python. Frame reads of the test sequence take ~240 us instead of ~550 us. --- docs/data/storage_and_access.rst | 7 ++ ncore/impl/data/_zarr3.py | 197 +++++++++++++++++++++++++++++ ncore/impl/data/stores_test.py | 210 ++++++++++++++++++++++++++++++- 3 files changed, 413 insertions(+), 1 deletion(-) diff --git a/docs/data/storage_and_access.rst b/docs/data/storage_and_access.rst index 0efebcc3..dcdcb92e 100644 --- a/docs/data/storage_and_access.rst +++ b/docs/data/storage_and_access.rst @@ -193,6 +193,13 @@ readers) resolve all zarr nodes from it, without reading node metadata records. zarr-python 3, NCore executes zarr's asynchronous I/O on the calling thread, avoiding per-access thread hand-offs to zarr-python's global I/O thread. +With zarr-python 3, numeric arrays of ``.itar`` archives (blosc compressed or +uncompressed, as written by NCore) are read by decompressing their chunks directly into +the output array (zarr-python's indexing determines the chunks), and large chunks of an +array are decoded concurrently. All other arrays and selections are read via +zarr-python's codec pipeline. Setting the environment variable +``NCORE_ZARR3_DIRECT_READS=0`` reads all arrays via zarr-python's codec pipeline. + Records of ``.itar`` archives are write-once (archives are append-only): rewriting a record fails. zarr-python 3 rewrites node metadata on every attribute update, so node metadata records are kept in memory until the archive is closed, and only their final diff --git a/ncore/impl/data/_zarr3.py b/ncore/impl/data/_zarr3.py index d244e0e0..b7c15da5 100644 --- a/ncore/impl/data/_zarr3.py +++ b/ncore/impl/data/_zarr3.py @@ -21,6 +21,9 @@ (as typical for reading individual frames) these thread hand-offs dominate. The backend instead runs zarr-python's coroutines on an event loop owned by the *calling* thread (see :func:`run`), which executes the same zarr-python code paths without thread hand-offs, and lets multiple reader threads decode data concurrently. + +Arrays of indexed tar archives are read by decompressing their chunks directly into the output array (see +:func:`_direct_read`). """ from __future__ import annotations @@ -30,6 +33,7 @@ import contextlib import json import os +import sys import threading import warnings @@ -44,6 +48,7 @@ Iterator, List, Literal, + NamedTuple, Optional, Tuple, TypeVar, @@ -61,6 +66,7 @@ from zarr.codecs import BloscCodec from zarr.core.buffer import Buffer, BufferPrototype, default_buffer_prototype from zarr.core.group import AsyncGroup +from zarr.core.indexing import BasicIndexer from zarr.core.sync import sync from zarr.storage import LocalStore, StorePath @@ -306,6 +312,195 @@ def _ignored_warnings() -> Iterator[None]: yield +# ----------------------------------------------------------------------------- +# Direct chunk reads of indexed tar archives +# ----------------------------------------------------------------------------- + +#: Environment variable disabling direct reads (all reads via zarr-python's codec pipeline), e.g., to compare against +#: or to benefit from zarr-python improvements +DIRECT_READS_ENV = "NCORE_ZARR3_DIRECT_READS" + +#: Minimum number of decompressed bytes per chunk to decode the chunks of a read concurrently (smaller chunks are +#: cheaper to decode than to hand off to a thread pool) +_PARALLEL_CHUNK_BYTES = 2 << 20 + +#: Maximum number of concurrent reads (e.g., of multiple reader threads) decoding their chunks concurrently, further +#: concurrent reads decode their chunks sequentially (the reads themselves already utilize the cores) +_PARALLEL_READS = 2 + + +class _ParallelReads: + """Bounds the number of reads decoding their chunks concurrently (per process)""" + + def __init__(self) -> None: + self.semaphore = threading.BoundedSemaphore(_PARALLEL_READS) + self.executor: Optional[concurrent.futures.ThreadPoolExecutor] = None + self.lock = threading.Lock() + self.pid = os.getpid() + + def pool(self) -> concurrent.futures.ThreadPoolExecutor: + with self.lock: + if self.executor is None: + self.executor = concurrent.futures.ThreadPoolExecutor( + max_workers=min(8, os.cpu_count() or 1), thread_name_prefix="ncore_zarr3_chunks" + ) + return self.executor + + +_parallel_reads = _ParallelReads() + + +def _parallel() -> _ParallelReads: + """The parallel reads state of this process (re-created in forked processes, whose locks / threads are unusable)""" + global _parallel_reads + if _parallel_reads.pid != os.getpid(): + _parallel_reads = _ParallelReads() + return _parallel_reads + + +class _DirectRead(NamedTuple): + """How to read the chunks of an array directly (see :func:`_direct_read`)""" + + store: IndexedTarStore + path: str + dtype: np.dtype + chunks: Tuple[int, ...] + chunk_key: Callable[[Tuple[int, ...]], str] + compressor: Optional[numcodecs.Blosc] + fill_value: object + + +def _direct_reads_enabled() -> bool: + return os.environ.get(DIRECT_READS_ENV, "1").strip().lower() not in ("0", "false", "no", "off") + + +def _direct_read(array: zarr.Array) -> Optional[_DirectRead]: + """How to read the chunks of an array directly (None if the array is read via zarr-python). + + Supported are arrays of indexed tar archives opened for reading with a regular chunk grid, C order, numeric / + boolean data of native byte order or fixed-length bytes, no filters, and no compression or a single blosc compressor + (all arrays written by ncore). zarr-python determines the chunks and their selections (its indexers), but the + chunks are read and decompressed directly into the output array, instead of via zarr-python's codec pipeline, which + decodes each chunk into a temporary buffer and copies it (see zarr-python#2904). + + Planned per read (sub-microsecond), so nothing outlives the array. + """ + store_path = array.async_array.store_path + if not isinstance(store := store_path.store, _TarStore) or store.store.mode != "r": + return None + + metadata: Any = array.metadata + dtype = np.dtype(metadata.dtype.to_native_dtype()) + if dtype.hasobject or not dtype.isnative or dtype.kind not in "biufS": + return None + if metadata.zarr_format == 2: + if metadata.order != "C" or metadata.filters: + return None + compressor = metadata.compressor + chunks = tuple(metadata.chunks) + else: + codecs = list(metadata.codecs) + if not codecs or type(codecs[0]).__name__ != "BytesCodec": + return None + if dtype.itemsize > 1 and str(getattr(codecs[0].endian, "value", codecs[0].endian)) != sys.byteorder: + return None + if len(codecs) == 1: + compressor = None + elif len(codecs) == 2 and isinstance(codecs[1], BloscCodec): + compressor = codecs[1]._blosc_codec + else: + return None + if not hasattr(metadata.chunk_grid, "chunk_shape"): # (regular chunk grids only) + return None + chunks = tuple(metadata.chunk_grid.chunk_shape) + if compressor is not None and not isinstance(compressor, numcodecs.Blosc): + return None + return _DirectRead( + store.store, store_path.path, dtype, chunks, metadata.encode_chunk_key, compressor, metadata.fill_value + ) + + +def _is_basic_selection(selection: Selection) -> bool: + """True for selections of (tuples of) integers, slices with unit step, and Ellipsis""" + for item in selection if isinstance(selection, tuple) else (selection,): + if item is Ellipsis or (isinstance(item, (int, np.integer)) and not isinstance(item, (bool, np.bool_))): + continue + if isinstance(item, slice) and item.step in (None, 1): + continue + return False + return True + + +def _chunk_grid(array: zarr.Array) -> Any: + """zarr-python's chunk grid of an array (an array attribute since zarr-python 3.2, metadata before)""" + if (grid := getattr(array.async_array, "_chunk_grid", None)) is not None: + return grid + return cast(Any, array.metadata).chunk_grid + + +def _read_direct(plan: _DirectRead, array: zarr.Array, selection: Selection) -> npt.NDArray[Any]: + """Reads a basic selection of an array, decompressing its chunks directly into the output array. + + Reads of multiple large chunks (see :data:`_PARALLEL_CHUNK_BYTES`) read and decode their chunks concurrently on a + thread pool (positional reads and blosc decompression release the GIL), unless :data:`_PARALLEL_READS` reads do so + already. All other reads run on the calling thread. + """ + shape = tuple(array.metadata.shape) + indexer = BasicIndexer(selection, shape, _chunk_grid(array)) + projections = list(indexer) if all(indexer.shape) else [] + + if ( + plan.compressor is None + and len(projections) == 1 + and projections[0].is_complete_chunk + and plan.chunks == shape == tuple(indexer.shape) + ): + # an uncompressed chunk with the complete array (e.g., an encoded image): use its bytes without copies + key = plan.chunk_key(projections[0].chunk_coords) + try: + raw = plan.store[f"{plan.path}/{key}" if plan.path else key] + except KeyError: + pass # (chunk not stored: fill value, see below) + else: + # (bytearray: results are writeable, as results of reads via zarr-python) + return np.frombuffer(bytearray(raw), dtype=plan.dtype).reshape(shape) + + out = np.empty(indexer.shape, dtype=plan.dtype) + + def read_chunk(projection: Any) -> None: + key = plan.chunk_key(projection.chunk_coords) + target = out[projection.out_selection] if projection.out_selection != () else out + try: + raw = plan.store[f"{plan.path}/{key}" if plan.path else key] + except KeyError: # chunk not stored (fill value) + target[...] = np.asarray(plan.fill_value, dtype=plan.dtype) if plan.fill_value is not None else 0 + return + if ( + plan.compressor is not None + and projection.is_complete_chunk + and target.shape == plan.chunks + and target.flags.c_contiguous + ): + plan.compressor.decode(raw, out=target) + return + decoded = plan.compressor.decode(raw) if plan.compressor is not None else raw + selected = np.frombuffer(decoded, dtype=plan.dtype).reshape(plan.chunks)[projection.chunk_selection] + target[...] = selected.reshape(target.shape) # (integer selections drop axes) + + chunk_bytes = plan.dtype.itemsize * int(np.prod(plan.chunks)) + parallel = _parallel() + if len(projections) > 1 and chunk_bytes >= _PARALLEL_CHUNK_BYTES and parallel.semaphore.acquire(blocking=False): + try: + for future in [parallel.pool().submit(read_chunk, projection) for projection in projections]: + future.result() + finally: + parallel.semaphore.release() + else: + for projection in projections: + read_chunk(projection) + return out # (0-d arrays for 0-d results, see nodes.Array.read) + + # ----------------------------------------------------------------------------- # Backend # ----------------------------------------------------------------------------- @@ -368,6 +563,8 @@ def create_array( ) def read(self, array: zarr.Array, selection: Selection) -> object: + if _direct_reads_enabled() and _is_basic_selection(selection) and (plan := _direct_read(array)) is not None: + return _read_direct(plan, array, selection) return run(array.async_array.getitem(selection)) def compression(self, array: zarr.Array) -> Optional[BloscCompression]: diff --git a/ncore/impl/data/stores_test.py b/ncore/impl/data/stores_test.py index 27c50923..2a15f198 100644 --- a/ncore/impl/data/stores_test.py +++ b/ncore/impl/data/stores_test.py @@ -27,7 +27,7 @@ import zlib from pathlib import Path -from typing import Any, Optional, Tuple, cast +from typing import Any, Dict, List, Optional, Tuple, cast from unittest import mock import cbor2 @@ -686,6 +686,214 @@ def get(byte_range: Any) -> Optional[bytes]: self.assertIsNone(_zarr3.run(tar_store.get("missing", default_buffer_prototype()))) +@unittest.skipUnless(ZARR_PYTHON_3, "zarr-python 3 backend") +class TestDirectReads(unittest.TestCase): + """Direct chunk reads of arrays of indexed tar archives (zarr-python 3) are identical to zarr-python's reads""" + + #: Arrays (dtype, shape, chunks, compression, missing chunks) of all supported kinds + ARRAYS = { + "f4_multi": (np.float32, (7, 5, 3), (3, 2, 3), nodes.BloscCompression(), ()), + "u2_single_uncompressed": (np.uint16, (11,), (11,), None, ()), + "i8_multi_uncompressed": (np.int64, (9, 4), (4, 3), None, ()), + "bool": (np.bool_, (6, 6), (4, 4), nodes.BloscCompression(cname="zstd", shuffle="noshuffle"), ()), + "u1_partial_last": (np.uint8, (13,), (5,), nodes.BloscCompression(shuffle="shuffle"), ()), + "f8_missing": (np.float64, (6, 2), (2, 2), nodes.BloscCompression(), ("1.0", "c/1/0")), + "f4_empty": (np.float32, (0, 3), (1, 3), nodes.BloscCompression(), ()), + } + + SELECTIONS = [ + Ellipsis, + (), + 0, + -1, + (slice(1, 4),), + (slice(None), 1), + (2, slice(0, 2)), + (slice(2, 2),), + (Ellipsis, 0), + (slice(-3, None),), + (0, slice(None), slice(1, 2)), # (dropped axes next to selected single elements) + (slice(0, 3), slice(0, 2), 1), # (complete chunks, not complete arrays) + ] + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + + def tearDown(self): + self._tmp.cleanup() + + def _write(self, zarr_format: int, skip_records: Tuple[str, ...] = ()) -> str: + """Writes all arrays (chunks of `missing chunks` dropped from the archive) and returns the archive path""" + rng = np.random.default_rng(1) + path = os.path.join(self._tmp.name, f"v{zarr_format}.itar") + full = os.path.join(self._tmp.name, f"v{zarr_format}.full.itar") + root = create_root_group(store := IndexedTarStore(full, mode="w"), {}, zarr_format=zarr_format) + for name, (dtype, shape, chunks, compression, _) in self.ARRAYS.items(): + data = rng.random(shape) > 0.5 if dtype == np.bool_ else (rng.random(shape) * 1000).astype(dtype) + root.create_array(name, data, chunks=chunks, compression=compression) + root.create_bytes_array("blob", b"\x89PNG\x00binary") + root.create_bytes_array("blob_missing", b"some bytes") + consolidate_store(store) + close_store(store) + + # copy without the missing chunks (stores of hierarchies written partially, e.g. by other writers) + source = IndexedTarStore(full) + missing = {f"{name}/{key}" for name, spec in self.ARRAYS.items() for key in spec[4]} + missing |= {"blob_missing/0", "blob_missing/c/0"} + with IndexedTarStore(path, mode="w") as copy: + for key in source.keys(): + if key not in missing: + copy[key] = source[key] + return path + + def _reads(self, path: str) -> Dict[str, Any]: + root = open_store(IndexedTarStore(path)) + out: Dict[str, Any] = {} + for name, (_, shape, *_rest) in self.ARRAYS.items(): + array = root.array(name) + for selection in self.SELECTIONS: + try: + out[f"{name}[{selection!r}]"] = array.read(selection) + except IndexError as e: # (e.g., integer selections of empty arrays) + out[f"{name}[{selection!r}]"] = type(e) + out["blob"] = root.array("blob").read_bytes() + # (fill value of missing chunks; zarr-python returns empty fixed-length bytes of dtype |S0) + out["blob_missing"] = root.array("blob_missing").read_bytes() + return out + + @parameterized.parameterized.expand([(zarr_format,) for zarr_format in ZARR_FORMATS]) + def test_direct_reads_match_zarr(self, zarr_format: int): + from ncore.impl.data import _zarr3 + + path = self._write(zarr_format) + with mock.patch.object(_zarr3, "_read_direct", wraps=_zarr3._read_direct) as direct: + actual = self._reads(path) + self.assertGreater(direct.call_count, len(actual) - 10) # (direct reads were used) + with mock.patch.dict(os.environ, {_zarr3.DIRECT_READS_ENV: "0"}): + with mock.patch.object(_zarr3, "_read_direct", side_effect=AssertionError("direct read")): + expected = self._reads(path) + + self.assertEqual(sorted(actual), sorted(expected)) + for key, value in expected.items(): + if isinstance(value, type): + self.assertIs(actual[key], value, key) + continue + actual_value = actual[key] + if isinstance(value, bytes): + self.assertEqual(actual_value, value, key) + continue + value = np.asarray(value) + self.assertEqual((actual_value.dtype, actual_value.shape), (value.dtype, value.shape), key) + np.testing.assert_array_equal(actual_value, value, key) + self.assertTrue(actual_value.flags.writeable, key) + + def test_missing_chunks_of_fixed_length_bytes(self): + """Missing chunks of fixed-length bytes read as the fill value (empty bytes), as with zarr-python""" + root = open_store(IndexedTarStore(self._write(2))) + self.assertEqual(root.array("blob_missing").read_bytes(), b"") + + def test_unsupported_selections_use_zarr(self): + from ncore.impl.data import _zarr3 + + root = open_store(IndexedTarStore(self._write(3))) + array = root.array("f4_multi") + with mock.patch.object(_zarr3, "_read_direct", side_effect=AssertionError("direct read")): + expected = array.read((slice(0, 6, 2),)) # (non-unit step) + np.testing.assert_array_equal(expected, array.read()[0:6:2]) + + def test_partial_chunk_selections(self): + """Selections of parts of compressed chunks of the output array (no in-place decompression)""" + root = open_store(IndexedTarStore(self._write(2))) + data = root.array("f4_multi").read() + for selection in [(slice(1, 3), slice(0, 2)), (slice(0, 3), slice(0, 2), slice(0, 2)), (slice(3, 7),)]: + np.testing.assert_array_equal(root.array("f4_multi").read(selection), data[selection], str(selection)) + + def test_stores_opened_for_writing_use_zarr(self): + """(node metadata of archives opened for writing is not final)""" + from ncore.impl.data import _zarr3 + + root = create_root_group(IndexedTarStore(os.path.join(self._tmp.name, "w.itar"), mode="w"), {}) + self.assertIsNone(_zarr3._direct_read(root.create_array("a", np.arange(4)).zarr)) + + def test_non_native_byte_order_uses_zarr(self): + from ncore.impl.data import _zarr3 + + path = os.path.join(self._tmp.name, "be.itar") + data = np.arange(6, dtype=">u4") + root = create_root_group(store := IndexedTarStore(path, mode="w"), {}) + root.create_array("a", data) + consolidate_store(store) + close_store(store) + array = open_store(IndexedTarStore(path)).array("a") + self.assertIsNone(_zarr3._direct_read(array.zarr)) + np.testing.assert_array_equal(array.read(), data) + + def test_directory_stores_use_zarr(self): + from ncore.impl.data import _zarr3 + + root = create_root_group(store := open_directory_store(os.path.join(self._tmp.name, "dir")), {}) + root.create_array("a", np.arange(4)) + consolidate_store(store) + self.assertIsNone( + _zarr3._direct_read(open_store(open_directory_store(os.path.join(self._tmp.name, "dir"))).array("a").zarr) + ) + + def test_large_chunks_are_decoded_concurrently(self): + from ncore.impl.data import _zarr3 + + path = os.path.join(self._tmp.name, "large.itar") + data = np.arange(4 * (1 << 20), dtype=np.float32).reshape(4, -1) # (4 chunks of 4 MiB) + root = create_root_group(store := IndexedTarStore(path, mode="w"), {}) + root.create_array("a", data, chunks=(1, data.shape[1])) + consolidate_store(store) + close_store(store) + + array = open_store(IndexedTarStore(path)).array("a") + threads: List[str] = [] + decode = np.frombuffer + + def record(*args: Any, **kwargs: Any) -> Any: + threads.append(threading.current_thread().name) + return decode(*args, **kwargs) + + with mock.patch.object(_zarr3.np, "frombuffer", side_effect=record): + np.testing.assert_array_equal(array.read((slice(None), slice(0, 10))), data[:, :10]) + self.assertEqual(len(threads), 4) + self.assertTrue(all(name.startswith("ncore_zarr3_chunks") for name in threads), threads) + + # reads beyond the bound decode their chunks on the calling thread + threads.clear() + parallel = _zarr3._parallel() + for _ in range(_zarr3._PARALLEL_READS): + parallel.semaphore.acquire() + try: + with mock.patch.object(_zarr3.np, "frombuffer", side_effect=record): + array.read((slice(None), slice(0, 10))) + finally: + for _ in range(_zarr3._PARALLEL_READS): + parallel.semaphore.release() + self.assertEqual(threads, [threading.current_thread().name] * 4) + + @unittest.skipUnless(hasattr(os, "fork"), "requires fork") + def test_forked_processes(self): + """Forked processes don't inherit the parallel reads state (its locks / threads are unusable in children)""" + from ncore.impl.data import _zarr3 + + parent = _zarr3._parallel() + read_fd, write_fd = os.pipe() + if (pid := os.fork()) == 0: + try: + os.write(write_fd, b"ok" if _zarr3._parallel() is not parent else b"inherited") + finally: + os._exit(0) + os.close(write_fd) + result = os.read(read_fd, 16) + os.close(read_fd) + os.waitpid(pid, 0) + self.assertEqual(result, b"ok") + self.assertIs(_zarr3._parallel(), parent) + + @parameterized.parameterized_class(("zarr_format",), [(zarr_format,) for zarr_format in ZARR_FORMATS]) class TestNodes(unittest.TestCase): """Typed node API""" From 772164ca1a40e6040e624749750395fba7828ae8 Mon Sep 17 00:00:00 2001 From: Janick Martinez Esturo Date: Mon, 12 Oct 2026 12:15:23 +0200 Subject: [PATCH 7/7] feat(data): asynchronous component reader API Add asynchronous variants of all component reader methods reading array data (get_frame_data_async, get_frame_ray_bundle_data_async, get_frame_ray_bundle_return_data_async, get_generic_data_async, get_camera_mask_image_async, get_pc_*_async, labels get_data_async, ...), returning the same data as the synchronous methods, based on nodes.Array.read_async / read_bytes_async. Awaiting multiple reads concurrently overlaps their I/O and decompression, without blocking the running event loop: - zarr-python 3: chunks of remote indexed tar archives (asynchronous fsspec filesystems, e.g. s3fs, gcsfs, http) are read via concurrent range requests on the event loop of their filesystem (IndexedTarStore.read_async / is_remote). On event loops of applications, blocking work (local reads, decompression, zarr-python's codec pipeline) runs on worker threads, on the event loops of the calling threads it runs inline - zarr-python 2: reads run on worker threads Metadata (attributes, names, timestamps) is held in memory and only provided synchronously. --- docs/data/storage_and_access.rst | 44 +++++++ ncore/impl/data/_backend.py | 4 + ncore/impl/data/_zarr2.py | 5 + ncore/impl/data/_zarr3.py | 190 +++++++++++++++++++++-------- ncore/impl/data/itar.py | 32 +++++ ncore/impl/data/nodes.py | 16 ++- ncore/impl/data/stores_test.py | 134 ++++++++++++++++++++ ncore/impl/data/v4/components.py | 124 +++++++++++++++---- ncore/impl/data/v4/formats_test.py | 71 +++++++++++ 9 files changed, 545 insertions(+), 75 deletions(-) diff --git a/docs/data/storage_and_access.rst b/docs/data/storage_and_access.rst index dcdcb92e..1dee9f62 100644 --- a/docs/data/storage_and_access.rst +++ b/docs/data/storage_and_access.rst @@ -238,6 +238,50 @@ V4 sequences are loaded by specifying one or more component store paths: poses_readers = reader.open_component_readers(PosesComponent.Reader) camera_readers = reader.open_component_readers(CameraSensorComponent.Reader) +Asynchronous Reads +~~~~~~~~~~~~~~~~~~ + +All component reader methods reading array data have asynchronous variants +(``*_async``, e.g. ``get_frame_data_async``, ``get_frame_ray_bundle_data_async``, +``get_generic_data_async``), returning the same data. Awaiting multiple reads +concurrently overlaps their I/O and decompression (e.g., range requests to remote +storage), and the running event loop isn't blocked (blocking work, such as +decompression, runs on worker threads): + +.. code-block:: python + + import asyncio + + lidar = reader.open_component_readers(LidarSensorComponent.Reader)["lidar"] + + async def load_frames(timestamps_us): + return await asyncio.gather( + *(lidar.get_frame_ray_bundle_data_async(ts, "direction") for ts in timestamps_us) + ) + + directions = asyncio.run(load_frames(lidar.frames_timestamps_us[:, 1].tolist())) + +Metadata (attributes, names, timestamps) is held in memory and only provided +synchronously. Custom components can implement asynchronous accessors via +``self._group.array(path).read_async()``. + +.. note:: + + Each in-flight read holds its decompressed data in memory. When reading many large + frames, bound the number of concurrent reads (e.g., to the number of CPU cores, or + higher for high-latency remote storage) instead of gathering all of them at once: + + .. code-block:: python + + async def load_frames(timestamps_us, max_concurrent_reads=16): + semaphore = asyncio.Semaphore(max_concurrent_reads) + + async def load(ts): + async with semaphore: + return await lidar.get_frame_ray_bundle_data_async(ts, "direction") + + return await asyncio.gather(*(load(ts) for ts in timestamps_us)) + .. _cloud_storage_access: Cloud and Remote Storage Access diff --git a/ncore/impl/data/_backend.py b/ncore/impl/data/_backend.py index 9c2770bc..88cc4658 100644 --- a/ncore/impl/data/_backend.py +++ b/ncore/impl/data/_backend.py @@ -198,6 +198,10 @@ def read(self, array: zarr.Array, selection: Selection) -> object: """Reads (a selection of) the data of an array (0-dimensional selections may result in numpy scalars)""" ... + async def read_async(self, array: zarr.Array, selection: Selection) -> object: + """Reads (a selection of) the data of an array asynchronously, without blocking the running event loop""" + ... + def compression(self, array: zarr.Array) -> Optional[BloscCompression]: """The blosc compression of an array (None if uncompressed, raises ValueError for other compressors)""" ... diff --git a/ncore/impl/data/_zarr2.py b/ncore/impl/data/_zarr2.py index 41df494c..181a99a9 100644 --- a/ncore/impl/data/_zarr2.py +++ b/ncore/impl/data/_zarr2.py @@ -17,6 +17,7 @@ from __future__ import annotations +import asyncio import json from typing import TYPE_CHECKING, Any, Iterator, List, Literal, Optional, Tuple, Union @@ -167,6 +168,10 @@ def create_array( def read(self, array: zarr.Array, selection: Selection) -> object: return _z(array)[selection] + async def read_async(self, array: zarr.Array, selection: Selection) -> object: + # (zarr-python 2 is synchronous: reads run on worker threads, to not block the running event loop) + return await asyncio.get_running_loop().run_in_executor(None, self.read, array, selection) + def compression(self, array: zarr.Array) -> Optional[BloscCompression]: if (compressor := _z(array).compressor) is None: return None diff --git a/ncore/impl/data/_zarr3.py b/ncore/impl/data/_zarr3.py index b7c15da5..1edcb108 100644 --- a/ncore/impl/data/_zarr3.py +++ b/ncore/impl/data/_zarr3.py @@ -133,6 +133,13 @@ class _ThreadLoop(threading.local): _thread_loop = _ThreadLoop() +def _on_own_loop() -> bool: + """True if running on the event loop of :func:`run` of the calling thread (blocking work runs inline), False on + other event loops (blocking work is offloaded to threads, to not block them)""" + state = _thread_loop.state + return state is not None and state.pid == os.getpid() and asyncio.get_running_loop() is state.loop + + def run(coroutine: Coroutine[Any, Any, T]) -> T: """Runs a zarr-python coroutine to completion on an event loop owned by the calling thread. @@ -438,67 +445,138 @@ def _chunk_grid(array: zarr.Array) -> Any: return cast(Any, array.metadata).chunk_grid -def _read_direct(plan: _DirectRead, array: zarr.Array, selection: Selection) -> npt.NDArray[Any]: +class _DirectReadOp: + """A direct read of a basic selection of an array: the chunks to read (see :meth:`keys`), and their decoding into + the output array (see :meth:`decode`)""" + + def __init__(self, plan: _DirectRead, array: zarr.Array, selection: Selection) -> None: + self.plan = plan + self.shape = tuple(array.metadata.shape) + indexer = BasicIndexer(selection, self.shape, _chunk_grid(array)) + self.out_shape = tuple(indexer.shape) + self.projections: List[Any] = list(indexer) if all(indexer.shape) else [] + self.keys = [self._key(projection) for projection in self.projections] + + def _key(self, projection: Any) -> str: + key = self.plan.chunk_key(projection.chunk_coords) + return f"{self.plan.path}/{key}" if self.plan.path else key + + @property + def is_single_raw_chunk(self) -> bool: + """True for selections of complete uncompressed single-chunk arrays (e.g., encoded images), whose chunk bytes + are the result""" + plan = self.plan + return ( + plan.compressor is None + and len(self.projections) == 1 + and self.projections[0].is_complete_chunk + and plan.chunks == self.shape == self.out_shape + ) + + @property + def decodes_concurrently(self) -> bool: + """True for reads of multiple large chunks (see :data:`_PARALLEL_CHUNK_BYTES`)""" + chunk_bytes = self.plan.dtype.itemsize * int(np.prod(self.plan.chunks)) + return len(self.projections) > 1 and chunk_bytes >= _PARALLEL_CHUNK_BYTES + + def decode(self, raws: List[Optional[bytes]], fetch: Optional[Callable[[str], Optional[bytes]]] = None) -> Any: + """Decodes the chunks (None: not stored, read as the fill value) into the output array. With ``fetch``, chunks + are read by the decoding threads (raws are ignored)""" + plan = self.plan + if self.is_single_raw_chunk and (raw := raws[0] if fetch is None else fetch(self.keys[0])) is not None: + # (bytearray: results are writeable, as results of reads via zarr-python) + return np.frombuffer(bytearray(raw), dtype=plan.dtype).reshape(self.shape) + + out = np.empty(self.out_shape, dtype=plan.dtype) + + def decode_chunk(index: int) -> None: + projection = self.projections[index] + raw = raws[index] if fetch is None else fetch(self.keys[index]) + target = out[projection.out_selection] if projection.out_selection != () else out + if raw is None: # chunk not stored (fill value) + target[...] = np.asarray(plan.fill_value, dtype=plan.dtype) if plan.fill_value is not None else 0 + return + if ( + plan.compressor is not None + and projection.is_complete_chunk + and target.shape == plan.chunks + and target.flags.c_contiguous + ): + plan.compressor.decode(raw, out=target) + return + decoded = plan.compressor.decode(raw) if plan.compressor is not None else raw + selected = np.frombuffer(decoded, dtype=plan.dtype).reshape(plan.chunks)[projection.chunk_selection] + target[...] = selected.reshape(target.shape) # (integer selections drop axes) + + parallel = _parallel() + if self.decodes_concurrently and parallel.semaphore.acquire(blocking=False): + try: + for future in [parallel.pool().submit(decode_chunk, i) for i in range(len(self.projections))]: + future.result() + finally: + parallel.semaphore.release() + else: + for index in range(len(self.projections)): + decode_chunk(index) + return out # (0-d arrays for 0-d results, see nodes.Array.read) + + +def _get_record(store: IndexedTarStore, key: str) -> Optional[bytes]: + try: + return store[key] + except KeyError: + return None + + +async def _get_record_async(store: IndexedTarStore, key: str) -> Optional[bytes]: + try: + return await store.read_async(key) + except KeyError: + return None + + +def _read_direct(plan: _DirectRead, array: zarr.Array, selection: Selection) -> Any: """Reads a basic selection of an array, decompressing its chunks directly into the output array. Reads of multiple large chunks (see :data:`_PARALLEL_CHUNK_BYTES`) read and decode their chunks concurrently on a thread pool (positional reads and blosc decompression release the GIL), unless :data:`_PARALLEL_READS` reads do so already. All other reads run on the calling thread. """ - shape = tuple(array.metadata.shape) - indexer = BasicIndexer(selection, shape, _chunk_grid(array)) - projections = list(indexer) if all(indexer.shape) else [] - - if ( - plan.compressor is None - and len(projections) == 1 - and projections[0].is_complete_chunk - and plan.chunks == shape == tuple(indexer.shape) - ): - # an uncompressed chunk with the complete array (e.g., an encoded image): use its bytes without copies - key = plan.chunk_key(projections[0].chunk_coords) - try: - raw = plan.store[f"{plan.path}/{key}" if plan.path else key] - except KeyError: - pass # (chunk not stored: fill value, see below) - else: - # (bytearray: results are writeable, as results of reads via zarr-python) - return np.frombuffer(bytearray(raw), dtype=plan.dtype).reshape(shape) + op = _DirectReadOp(plan, array, selection) + return op.decode([], fetch=lambda key: _get_record(plan.store, key)) + + +async def _read_direct_async(plan: _DirectRead, array: zarr.Array, selection: Selection) -> Any: + """Asynchronous :func:`_read_direct`: chunks of remote archives are read via concurrent asynchronous range + requests. Blocking work (local reads, decompression) runs on the calling thread on the event loop of :func:`run`, + on worker threads on other event loops (e.g., of asynchronous applications), so they aren't blocked.""" + op = _DirectReadOp(plan, array, selection) + if plan.store.is_remote: + raws = list(await asyncio.gather(*(_get_record_async(plan.store, key) for key in op.keys))) + if _on_own_loop(): + return op.decode(raws) + return await asyncio.get_running_loop().run_in_executor(_read_executor(), op.decode, raws) + if _on_own_loop(): + return op.decode([], fetch=lambda key: _get_record(plan.store, key)) + return await asyncio.get_running_loop().run_in_executor( + _read_executor(), lambda: op.decode([], fetch=lambda key: _get_record(plan.store, key)) + ) - out = np.empty(indexer.shape, dtype=plan.dtype) - def read_chunk(projection: Any) -> None: - key = plan.chunk_key(projection.chunk_coords) - target = out[projection.out_selection] if projection.out_selection != () else out - try: - raw = plan.store[f"{plan.path}/{key}" if plan.path else key] - except KeyError: # chunk not stored (fill value) - target[...] = np.asarray(plan.fill_value, dtype=plan.dtype) if plan.fill_value is not None else 0 - return - if ( - plan.compressor is not None - and projection.is_complete_chunk - and target.shape == plan.chunks - and target.flags.c_contiguous - ): - plan.compressor.decode(raw, out=target) - return - decoded = plan.compressor.decode(raw) if plan.compressor is not None else raw - selected = np.frombuffer(decoded, dtype=plan.dtype).reshape(plan.chunks)[projection.chunk_selection] - target[...] = selected.reshape(target.shape) # (integer selections drop axes) +def _read_executor() -> concurrent.futures.ThreadPoolExecutor: + """Executor of blocking parts of reads awaited on event loops other than the one of :func:`run` (per process)""" + global _read_executor_state + if _read_executor_state is None or _read_executor_state[0] != os.getpid(): + _read_executor_state = ( + os.getpid(), + concurrent.futures.ThreadPoolExecutor( + max_workers=min(32, (os.cpu_count() or 1) + 4), thread_name_prefix="ncore_zarr3_reads" + ), + ) + return _read_executor_state[1] - chunk_bytes = plan.dtype.itemsize * int(np.prod(plan.chunks)) - parallel = _parallel() - if len(projections) > 1 and chunk_bytes >= _PARALLEL_CHUNK_BYTES and parallel.semaphore.acquire(blocking=False): - try: - for future in [parallel.pool().submit(read_chunk, projection) for projection in projections]: - future.result() - finally: - parallel.semaphore.release() - else: - for projection in projections: - read_chunk(projection) - return out # (0-d arrays for 0-d results, see nodes.Array.read) + +_read_executor_state: Optional[Tuple[int, concurrent.futures.ThreadPoolExecutor]] = None # ----------------------------------------------------------------------------- @@ -567,6 +645,14 @@ def read(self, array: zarr.Array, selection: Selection) -> object: return _read_direct(plan, array, selection) return run(array.async_array.getitem(selection)) + async def read_async(self, array: zarr.Array, selection: Selection) -> object: + if _direct_reads_enabled() and _is_basic_selection(selection) and (plan := _direct_read(array)) is not None: + return await _read_direct_async(plan, array, selection) + if _on_own_loop(): + return await array.async_array.getitem(selection) + # (zarr-python's codec pipeline decodes on the calling event loop: run it on a worker thread's own loop) + return await asyncio.get_running_loop().run_in_executor(_read_executor(), self.read, array, selection) + def compression(self, array: zarr.Array) -> Optional[BloscCompression]: if not (compressors := array.compressors): return None diff --git a/ncore/impl/data/itar.py b/ncore/impl/data/itar.py index 26d160e6..855492da 100644 --- a/ncore/impl/data/itar.py +++ b/ncore/impl/data/itar.py @@ -27,6 +27,7 @@ from __future__ import annotations +import asyncio import io import logging import os @@ -210,6 +211,37 @@ def __getitem__(self, key: str) -> bytes: self._file.seek(position) return value + @property + def is_remote(self) -> bool: + """True for archives opened for reading on asynchronous fsspec filesystems (e.g., s3fs, gcsfs, http), whose + records are read via asynchronous range requests by :meth:`read_async`""" + return self._async_fs() is not None + + async def read_async(self, key: str) -> bytes: + """Reads the payload of a record asynchronously (raises KeyError if it doesn't exist). + + Records of remote archives (see :attr:`is_remote`) are read via asynchronous range requests (concurrent reads + overlap), all other records synchronously (see :meth:`__getitem__`). + """ + if (fs := self._async_fs()) is None or key in self._node_metadata: + return self[key] + record = self._records[key] + if ((tail := self._tail) is not None and record.offset >= tail.start) or record.size == 0: + return self[key] + + request = fs._cat_file(self._itar_upath.path, start=record.offset, end=record.offset + record.size) + if (fs_loop := getattr(fs, "loop", None)) is None or fs_loop is asyncio.get_running_loop(): + return await request + # (filesystems are bound to fsspec's I/O event loop, their sessions can't be used on other event loops) + return await asyncio.wrap_future(asyncio.run_coroutine_threadsafe(request, fs_loop)) + + def _async_fs(self) -> Any: + """The asynchronous fsspec filesystem of remote archives opened for reading (None otherwise)""" + if self._mode != "r" or self._pread_fd is not None: + return None + fs = self._itar_upath.fs + return fs if getattr(fs, "async_impl", False) and hasattr(fs, "_cat_file") else None + def __setitem__(self, key: str, value: Union[bytes, bytearray, memoryview]) -> None: """Appends a record (requires write mode, raises ValueError if the record exists already). Node metadata records are appended when closing the archive (and can be rewritten until then).""" diff --git a/ncore/impl/data/nodes.py b/ncore/impl/data/nodes.py index f17e9779..af63fde9 100644 --- a/ncore/impl/data/nodes.py +++ b/ncore/impl/data/nodes.py @@ -193,9 +193,23 @@ def read(self, selection: Selection = Ellipsis) -> npt.NDArray[Any]: """Reads (a selection of) the data of the array (0-dimensional selections result in 0-d arrays)""" return np.asarray(_backend.read(self._array, selection)) + async def read_async(self, selection: Selection = Ellipsis) -> npt.NDArray[Any]: + """Reads (a selection of) the data of the array asynchronously, see :meth:`read`. + + Concurrent reads (e.g., via ``asyncio.gather``) overlap their I/O (e.g., range requests of remote archives) and + decompression, the running event loop isn't blocked (blocking work runs on worker threads). + """ + return np.asarray(await _backend.read_async(self._array, selection)) + def read_bytes(self) -> bytes: """Reads a binary blob stored by :meth:`Group.create_bytes_array`""" - value = _backend.read(self._array, ()) + return self._blob(_backend.read(self._array, ())) + + async def read_bytes_async(self) -> bytes: + """Reads a binary blob asynchronously, see :meth:`read_bytes` and :meth:`read_async`""" + return self._blob(await _backend.read_async(self._array, ())) + + def _blob(self, value: object) -> bytes: if isinstance(value, bytes): # fixed-length bytes scalar (trailing NUL bytes are stripped) return bytes(value) if isinstance(value, np.ndarray) and value.dtype.kind == "S" and value.ndim == 0: diff --git a/ncore/impl/data/stores_test.py b/ncore/impl/data/stores_test.py index 2a15f198..f3029dc0 100644 --- a/ncore/impl/data/stores_test.py +++ b/ncore/impl/data/stores_test.py @@ -894,6 +894,140 @@ def test_forked_processes(self): self.assertIs(_zarr3._parallel(), parent) +class TestAsyncReads(unittest.TestCase): + """Asynchronous reads return the same data as synchronous reads, without blocking the running event loop""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.zarr_format = max(nodes.SUPPORTED_ZARR_FORMATS) + self.path = os.path.join(self._tmp.name, "store.itar") + _write(IndexedTarStore(self.path, mode="w"), zarr_format=self.zarr_format) + + def tearDown(self): + self._tmp.cleanup() + + def _reads(self, root: nodes.Group) -> List[Any]: + return [ + root.array("foo").read(), + root.array("subgroup/foo").read((slice(1, 3),)), + root.array("subgroup/foo").read((2, slice(None), 0)), + root.array("subgroup/empty").read(), + root.array("subgroup/blob").read_bytes(), + ] + + async def _reads_async(self, root: nodes.Group) -> List[Any]: + return list( + await asyncio.gather( + root.array("foo").read_async(), + root.array("subgroup/foo").read_async((slice(1, 3),)), + root.array("subgroup/foo").read_async((2, slice(None), 0)), + root.array("subgroup/empty").read_async(), + root.array("subgroup/blob").read_bytes_async(), + ) + ) + + def _assert_equal(self, actual: List[Any], expected: List[Any]) -> None: + self.assertEqual(len(actual), len(expected)) + for a, e in zip(actual, expected): + if isinstance(e, bytes): + self.assertEqual(a, e) + else: + self.assertEqual((a.dtype, a.shape), (e.dtype, e.shape)) + np.testing.assert_array_equal(a, e) + + @parameterized.parameterized.expand([("direct",), ("zarr",)]) + def test_async_reads(self, mode: str): + root = open_store(IndexedTarStore(self.path)) + expected = self._reads(root) + with mock.patch.dict(os.environ, {"NCORE_ZARR3_DIRECT_READS": "1" if mode == "direct" else "0"}): + self._assert_equal(asyncio.run(self._reads_async(root)), expected) # (an application's event loop) + if ZARR_PYTHON_3: + from ncore.impl.data import _zarr3 + + self._assert_equal(_zarr3.run(self._reads_async(root)), expected) # (the loop of the calling thread) + + @parameterized.parameterized.expand([("direct",), ("zarr",)]) + def test_async_reads_dont_block_event_loops(self, mode: str): + """Blocking parts of reads awaited on event loops of applications (record reads, decompression) run on + worker threads""" + root = open_store(IndexedTarStore(self.path)) + expected = self._reads(root) + threads = set() + getitem = IndexedTarStore.__getitem__ + + def recording_getitem(store: IndexedTarStore, key: str) -> bytes: + threads.add(threading.current_thread().name) + return getitem(store, key) + + with mock.patch.dict(os.environ, {"NCORE_ZARR3_DIRECT_READS": "1" if mode == "direct" else "0"}): + with mock.patch.object(IndexedTarStore, "__getitem__", recording_getitem): + self._assert_equal(asyncio.run(self._reads_async(root)), expected) + self.assertTrue(threads, "no record reads") + self.assertNotIn(threading.current_thread().name, threads) + + @parameterized.parameterized.expand([("direct",), ("zarr",)]) + def test_async_reads_on_the_loop_of_the_calling_thread_run_inline(self, mode: str): + """Reads awaited on the event loop of the calling thread (see _zarr3.run) don't hand off to threads""" + if not ZARR_PYTHON_3: + self.skipTest("event loops of the calling threads (zarr-python 3)") + from ncore.impl.data import _zarr3 + + root = open_store(IndexedTarStore(self.path)) + expected = self._reads(root) + threads = set() + getitem = IndexedTarStore.__getitem__ + + def recording_getitem(store: IndexedTarStore, key: str) -> bytes: + threads.add(threading.current_thread().name) + return getitem(store, key) + + with mock.patch.dict(os.environ, {"NCORE_ZARR3_DIRECT_READS": "1" if mode == "direct" else "0"}): + with mock.patch.object(IndexedTarStore, "__getitem__", recording_getitem): + self._assert_equal(_zarr3.run(self._reads_async(root)), expected) + self.assertEqual(threads, {threading.current_thread().name}) + + def test_sync_reads_within_event_loops(self): + """Synchronous reads also work within running event loops""" + root = open_store(IndexedTarStore(self.path)) + expected = self._reads(root) + + async def main() -> List[Any]: + return self._reads(open_store(IndexedTarStore(self.path))) + + self._assert_equal(asyncio.run(main()), expected) + + @unittest.skipUnless(ZARR_PYTHON_3, "range requests of remote archives (zarr-python 3)") + def test_remote_range_requests(self): + """Chunks of remote archives are read via asynchronous range requests on the event loop of their filesystem, + also when awaited on other event loops (e.g., fsspec's http filesystem binds its sessions to fsspec's I/O + loop)""" + import fsspec.asyn + + class LoopBoundFileSystem(fsspec.asyn.AsyncFileSystem): + """A filesystem whose coroutines only work on its own event loop (as fsspec's http filesystem)""" + + async_impl = True + + async def _cat_file(self, path: str, start: Optional[int] = None, end: Optional[int] = None, **_: Any): + if asyncio.get_running_loop() is not self.loop: + raise RuntimeError("Timeout context manager should be used inside a task") + requests.append((start, end)) + with open(path, "rb") as f: + f.seek(start or 0) + return f.read(None if end is None else end - (start or 0)) + + requests: List[Tuple[Optional[int], Optional[int]]] = [] + fs = LoopBoundFileSystem(loop=fsspec.asyn.get_loop()) + store = IndexedTarStore(self.path, index_tail_read_size=None) # (chunks aren't read with the index) + root = open_store(store) + expected = self._reads(root) + with mock.patch.object(IndexedTarStore, "_async_fs", lambda self: fs): + self.assertTrue(store.is_remote) + self._assert_equal(asyncio.run(self._reads_async(root)), expected) + self.assertGreaterEqual(len(requests), 4) # (range requests of the chunks) + self.assertFalse(IndexedTarStore(self.path).is_remote) # (local archives use positional reads) + + @parameterized.parameterized_class(("zarr_format",), [(zarr_format,) for zarr_format in ZARR_FORMATS]) class TestNodes(unittest.TestCase): """Typed node API""" diff --git a/ncore/impl/data/v4/components.py b/ncore/impl/data/v4/components.py index da60fab8..fe160980 100644 --- a/ncore/impl/data/v4/components.py +++ b/ncore/impl/data/v4/components.py @@ -712,10 +712,17 @@ def get_generic_data(self, name: str) -> np.ndarray: Raises KeyError if the name does not exist. """ + return self._generic_data_array(name).read() + + async def get_generic_data_async(self, name: str) -> np.ndarray: + """Asynchronous :meth:`get_generic_data`""" + return await self._generic_data_array(name).read_async() + + def _generic_data_array(self, name: str) -> nodes.Array: if self._generic_data_group is None: raise KeyError("Component has no generic_data") if name in self._generic_data_group: - return self._generic_data_group.array(name).read() + return self._generic_data_group.array(name) raise KeyError(f"Generic data '{name}' not found. Available: {self.get_generic_data_names()}") @@ -1067,6 +1074,15 @@ def get_camera_mask_image(self, camera_id: str, mask_name: str) -> PILImage.Imag return PILImage.open(io.BytesIO(mask_array.read_bytes()), formats=[mask_array.attr_str("format")]) + async def get_camera_mask_image_async(self, camera_id: str, mask_name: str) -> PILImage.Image: + """Asynchronous :meth:`get_camera_mask_image`""" + + mask_array = self._group.array(f"cameras/{camera_id}/{mask_name}") + + return PILImage.open( + io.BytesIO(await mask_array.read_bytes_async()), formats=[mask_array.attr_str("format")] + ) + def get_camera_mask_images(self, camera_id: str) -> Generator[Tuple[str, PILImage.Image]]: """Returns all constant named camera mask images""" @@ -1228,6 +1244,11 @@ def get_frame_generic_data(self, timestamp_us: int, name: str) -> npt.NDArray[An return self._get_frame_generic_data_group(timestamp_us).array(name).read() + async def get_frame_generic_data_async(self, timestamp_us: int, name: str) -> npt.NDArray[Any]: + """Asynchronous :meth:`get_frame_generic_data`""" + + return await self._get_frame_generic_data_group(timestamp_us).array(name).read_async() + def get_frame_generic_meta_data(self, timestamp_us: int) -> Dict[str, types.JsonLike]: """Returns generic frame meta-data for a specific frame""" @@ -1338,6 +1359,11 @@ def get_frame_ray_bundle_data(self, timestamp_us: int, name: str) -> npt.NDArray return self._get_ray_bundle_group(timestamp_us).array(name).read() + async def get_frame_ray_bundle_data_async(self, timestamp_us: int, name: str) -> npt.NDArray[Any]: + """Asynchronous :meth:`get_frame_ray_bundle_data`""" + + return await self._get_ray_bundle_group(timestamp_us).array(name).read_async() + # per-ray return data def _get_ray_bundle_returns_group(self, timestamp_us: int) -> nodes.Group: """Returns the ray bundle returns group of a frame""" @@ -1362,15 +1388,22 @@ def has_frame_ray_bundle_return_data(self, timestamp_us: int, name: str) -> bool def get_frame_ray_bundle_return_valid_mask(self, timestamp_us: int) -> npt.NDArray[np.bool_]: """Returns the per-ray return valid mask for a frame""" - valid_mask_packed = self._group.array(self._frame_path(timestamp_us, "ray_bundle_returns_valid_mask_packed")) + valid_mask_packed = self._valid_mask_packed_array(timestamp_us) + return self._unpack_valid_mask(valid_mask_packed, valid_mask_packed.read()) - n_returns, n_rays = valid_mask_packed.attr_int("n_returns"), valid_mask_packed.attr_int("n_rays") + async def get_frame_ray_bundle_return_valid_mask_async(self, timestamp_us: int) -> npt.NDArray[np.bool_]: + """Asynchronous :meth:`get_frame_ray_bundle_return_valid_mask`""" - return ( - np.unpackbits(valid_mask_packed.read(), count=n_returns * n_rays) - .astype(np.bool_) - .reshape((n_returns, n_rays)) - ) + valid_mask_packed = self._valid_mask_packed_array(timestamp_us) + return self._unpack_valid_mask(valid_mask_packed, await valid_mask_packed.read_async()) + + def _valid_mask_packed_array(self, timestamp_us: int) -> nodes.Array: + return self._group.array(self._frame_path(timestamp_us, "ray_bundle_returns_valid_mask_packed")) + + @staticmethod + def _unpack_valid_mask(valid_mask_packed: nodes.Array, packed: npt.NDArray[np.uint8]) -> npt.NDArray[np.bool_]: + n_returns, n_rays = valid_mask_packed.attr_int("n_returns"), valid_mask_packed.attr_int("n_rays") + return np.unpackbits(packed, count=n_returns * n_rays).astype(np.bool_).reshape((n_returns, n_rays)) def get_frame_ray_bundle_return_data( self, timestamp_us: int, name: str, return_index: Optional[int] @@ -1385,6 +1418,16 @@ def get_frame_ray_bundle_return_data( else: return return_array.read((return_index, Ellipsis)) # load specific return only + async def get_frame_ray_bundle_return_data_async( + self, timestamp_us: int, name: str, return_index: Optional[int] + ) -> npt.NDArray[np.float32]: + """Asynchronous :meth:`get_frame_ray_bundle_return_data`""" + + return_array = self._get_ray_bundle_returns_group(timestamp_us).array(name) + if return_index is None: + return await return_array.read_async() + return await return_array.read_async((return_index, Ellipsis)) + class CameraSensorComponent: """Camera sensor data component""" @@ -1446,6 +1489,12 @@ def get_data(self) -> types.EncodedImageData: """Loads the referenced encoded image data to memory""" return types.EncodedImageData(self._image_array.read_bytes(), self._image_array.attr_str("format")) + async def get_data_async(self) -> types.EncodedImageData: + """Asynchronous :meth:`get_data`""" + return types.EncodedImageData( + await self._image_array.read_bytes_async(), self._image_array.attr_str("format") + ) + def get_frame_handle(self, timestamp_us: int) -> EncodedImageDataHandle: """Returns the frame's encoded image data""" return self.EncodedImageDataHandle(self._group.array(self._frame_path(timestamp_us, "image"))) @@ -1454,6 +1503,10 @@ def get_frame_data(self, timestamp_us: int) -> types.EncodedImageData: """Returns the frame's encoded image data""" return self.get_frame_handle(timestamp_us).get_data() + async def get_frame_data_async(self, timestamp_us: int) -> types.EncodedImageData: + """Asynchronous :meth:`get_frame_data`""" + return await self.get_frame_handle(timestamp_us).get_data_async() + def get_frame_image(self, timestamp_us: int) -> PILImage.Image: """Returns the frame's decoded image data""" return self.get_frame_data(timestamp_us).get_decoded_image() @@ -1923,9 +1976,17 @@ def _pc_array(self, pc_index: int, name: str) -> nodes.Array: def get_pc_xyz(self, pc_index: int) -> npt.NDArray[np.float32]: return self._pc_array(pc_index, "xyz").read() + async def get_pc_xyz_async(self, pc_index: int) -> npt.NDArray[np.float32]: + """Asynchronous :meth:`get_pc_xyz`""" + return await self._pc_array(pc_index, "xyz").read_async() + def get_pc_attribute(self, pc_index: int, name: str) -> npt.NDArray[Any]: return self._pc_array(pc_index, name).read() + async def get_pc_attribute_async(self, pc_index: int, name: str) -> npt.NDArray[Any]: + """Asynchronous :meth:`get_pc_attribute`""" + return await self._pc_array(pc_index, name).read_async() + def get_pc_reference_frame_id(self, pc_index: int) -> str: return self._pc_group(pc_index).attr_str("reference_frame_id") @@ -1941,6 +2002,10 @@ def has_pc_generic_data(self, pc_index: int, name: str) -> bool: def get_pc_generic_data(self, pc_index: int, name: str) -> npt.NDArray[Any]: return self._pc_array(pc_index, f"generic_data/{name}").read() + async def get_pc_generic_data_async(self, pc_index: int, name: str) -> npt.NDArray[Any]: + """Asynchronous :meth:`get_pc_generic_data`""" + return await self._pc_array(pc_index, f"generic_data/{name}").read_async() + def get_pc_generic_meta_data(self, pc_index: int) -> Dict[str, types.JsonLike]: return cast(Dict[str, types.JsonLike], self._pc_group(pc_index).attrs.get("generic_meta_data", {})) @@ -2151,32 +2216,47 @@ def get_data(self) -> npt.NDArray[Any]: For RAW encoding, applies de-quantization if specified in the schema. For IMAGE_ENCODED encoding, decodes the image bytes via PIL. """ + data = self._label_group.array("data") if self._descriptor.label_schema.encoding == types.LabelEncoding.RAW: - arr = self._label_group.array("data").read() - - # De-quantize if configured - if (q := self._descriptor.label_schema.quantization) is not None: - arr = (arr.astype(q.intermediate_dtype) * q.scale + q.offset).astype( - self._descriptor.label_schema.dtype - ) - - return arr - + return self._decode_raw(data.read()) elif self._descriptor.label_schema.encoding == types.LabelEncoding.IMAGE_ENCODED: - raw_bytes = self._label_group.array("data").read_bytes() - image = PILImage.open(io.BytesIO(raw_bytes)) - - return np.asarray(image, dtype=self._descriptor.label_schema.dtype) + return self._decode_image(data.read_bytes()) + else: + raise ValueError(f"Unsupported label encoding: {self._descriptor.label_schema.encoding}") + async def get_data_async(self) -> npt.NDArray[Any]: + """Asynchronous :meth:`get_data`""" + data = self._label_group.array("data") + if self._descriptor.label_schema.encoding == types.LabelEncoding.RAW: + return self._decode_raw(await data.read_async()) + elif self._descriptor.label_schema.encoding == types.LabelEncoding.IMAGE_ENCODED: + return self._decode_image(await data.read_bytes_async()) else: raise ValueError(f"Unsupported label encoding: {self._descriptor.label_schema.encoding}") + def _decode_raw(self, arr: npt.NDArray[Any]) -> npt.NDArray[Any]: + # De-quantize if configured + if (q := self._descriptor.label_schema.quantization) is not None: + arr = (arr.astype(q.intermediate_dtype) * q.scale + q.offset).astype( + self._descriptor.label_schema.dtype + ) + return arr + + def _decode_image(self, raw_bytes: bytes) -> npt.NDArray[Any]: + return np.asarray(PILImage.open(io.BytesIO(raw_bytes)), dtype=self._descriptor.label_schema.dtype) + def get_encoded_data(self) -> Optional[bytes]: """Return the raw encoded bytes for IMAGE_ENCODED labels, or None for RAW.""" if self._descriptor.label_schema.encoding == types.LabelEncoding.IMAGE_ENCODED: return self._label_group.array("data").read_bytes() return None + async def get_encoded_data_async(self) -> Optional[bytes]: + """Asynchronous :meth:`get_encoded_data`""" + if self._descriptor.label_schema.encoding == types.LabelEncoding.IMAGE_ENCODED: + return await self._label_group.array("data").read_bytes_async() + return None + def get_label(self, timestamp_us: int) -> CameraLabelHandle: """Return a lazy handle to the label data at the given timestamp.""" label_group = self._label_group(timestamp_us) diff --git a/ncore/impl/data/v4/formats_test.py b/ncore/impl/data/v4/formats_test.py index aa0b539e..a109ab26 100644 --- a/ncore/impl/data/v4/formats_test.py +++ b/ncore/impl/data/v4/formats_test.py @@ -21,6 +21,7 @@ from __future__ import annotations +import asyncio import tempfile import unittest @@ -155,6 +156,68 @@ def selected(timestamps_us: np.ndarray) -> np.ndarray: return out +async def _summary_async(reader: SequenceComponentGroupsReader) -> Dict[str, Any]: + """:func:`_summary` via the asynchronous reader APIs, all frames of all sensors read concurrently""" + out = _summary(reader, subsampled=False, frames=False) + + async def masks(masks: MasksComponent.Reader) -> Dict[str, Any]: + out: Dict[str, Any] = {} + for camera_id in masks._group.group("cameras").members(): + out[camera_id] = {} + for mask_name in masks.get_camera_mask_names(camera_id): + image = await masks.get_camera_mask_image_async(camera_id, mask_name) + out[camera_id][mask_name] = np.asarray(image).tobytes() + return out + + async def camera_frame(cameras: CameraSensorComponent.Reader, ts: int) -> object: + data, generic = await asyncio.gather( + cameras.get_frame_data_async(ts), + asyncio.gather( + *(cameras.get_frame_generic_data_async(ts, n) for n in sorted(cameras.get_frame_generic_data_names(ts))) + ), + ) + names = sorted(cameras.get_frame_generic_data_names(ts)) + return ( + ts, + data.get_encoded_image_format(), + data.get_encoded_image_data(), + {n: value.tobytes() for n, value in zip(names, generic)}, + cameras.get_frame_generic_meta_data(ts), + ) + + async def sensor_frame(sensor: Any, ts: int) -> object: + ray_names = sorted(sensor.get_frame_ray_bundle_data_names(ts)) + return_names = sorted(sensor.get_frame_ray_bundle_return_data_names(ts)) + generic_names = sorted(sensor.get_frame_generic_data_names(ts)) + rays, returns, mask, generic = await asyncio.gather( + asyncio.gather(*(sensor.get_frame_ray_bundle_data_async(ts, n) for n in ray_names)), + asyncio.gather(*(sensor.get_frame_ray_bundle_return_data_async(ts, n, None) for n in return_names)), + sensor.get_frame_ray_bundle_return_valid_mask_async(ts), + asyncio.gather(*(sensor.get_frame_generic_data_async(ts, n) for n in generic_names)), + ) + return ( + ts, + {n: v.tobytes() for n, v in zip(ray_names, rays)}, + {n: v.tobytes() for n, v in zip(return_names, returns)}, + mask.tobytes(), + {n: v.tobytes() for n, v in zip(generic_names, generic)}, + sensor.get_frame_generic_meta_data(ts), + ) + + for name, masks_reader in sorted(reader.open_component_readers(MasksComponent.Reader).items()): + out[f"masks/{name}"] = await masks(masks_reader) + for name, cameras in sorted(reader.open_component_readers(CameraSensorComponent.Reader).items()): + out[f"cameras/{name}"] = list( + await asyncio.gather(*(camera_frame(cameras, int(ts)) for ts in cameras.frames_timestamps_us[:, 1])) + ) + for component in (LidarSensorComponent, RadarSensorComponent): + for name, sensor in sorted(reader.open_component_readers(component.Reader).items()): + out[f"{component.COMPONENT_NAME}/{name}"] = list( + await asyncio.gather(*(sensor_frame(sensor, int(ts)) for ts in sensor.frames_timestamps_us[:, 1])) + ) + return out + + class TestZarrFormats(unittest.TestCase): """zarr format v2 and v3 serializations of a sequence need to be identical in content""" @@ -195,6 +258,14 @@ def test_node_cache_sizes(self, node_cache_size: Optional[int]): frame_entries = [key for key in cache.keys() if key.endswith("/ray_bundle/direction")] self.assertEqual(len(frame_entries), min(len(frames), node_cache_size or len(frames))) + @parameterized.parameterized.expand([("v2",), ("v3",)]) + def test_async_reads(self, data: str): + """Asynchronous reader APIs (all frames read concurrently) return the same data as the synchronous APIs""" + if data == "v3" and 3 not in nodes.SUPPORTED_ZARR_FORMATS: + self.skipTest("zarr format v3 data requires zarr-python 3") + reader = SequenceComponentGroupsReader([_TRIMMED_V2 if data == "v2" else _TRIMMED_V3]) + _assert_summaries_equal(self, self.reference, asyncio.run(_summary_async(reader))) + def test_invalid_node_cache_size(self): with self.assertRaises(ValueError): SequenceComponentGroupsReader([_TRIMMED_V2], node_cache_size=-1)