diff --git a/changes/1113.feature.md b/changes/1113.feature.md new file mode 100644 index 0000000000..9c463dab16 --- /dev/null +++ b/changes/1113.feature.md @@ -0,0 +1,2 @@ +Add ``Array.chunk_byte_range`` and ``AsyncArray.chunk_byte_range`` to locate stored +logical chunks in regular unsharded arrays and one-level sharded Zarr v3 arrays. diff --git a/docs/user-guide/arrays.md b/docs/user-guide/arrays.md index 5b025047b6..00ec50ff5b 100644 --- a/docs/user-guide/arrays.md +++ b/docs/user-guide/arrays.md @@ -827,6 +827,39 @@ pairs are accepted: When writing, Zarr automatically compresses repeated values into RLE format. +## Locating encoded chunks + +`Array.chunk_byte_range(coords)` returns the store-relative key, byte offset, and +encoded length of a logical chunk. Coordinates are chunk indexes, not element +indexes. An absent chunk returns `None`, including an absent inner chunk within +a shard that contains other chunks. + +```python test="true" +import zarr + +array = zarr.create_array( + store=zarr.storage.MemoryStore(), + shape=(8, 8), chunks=(2, 2), shards=(4, 4), dtype="int32", +) +array[:2, :2] = 7 +key, offset, length = array.chunk_byte_range((0, 0)) +assert key == "c/0/0" +assert offset >= 0 and length > 0 +assert array.chunk_byte_range((0, 1)) is None +``` + +This supports regular unsharded arrays and one-level sharding with no outer +codecs. Nested sharding and rectilinear grids are not supported. For sharded +arrays the method reads and decodes the shard index, not the chunk payload. +For unsharded arrays it uses the store's size lookup, which may read the entire +object if the backend does not implement an efficient lookup. + +Each call reads its own index. The returned location describes encoded bytes, +not uncompressed array bytes, and can become stale if the store changes. This +is not a snapshot API or a payload-integrity check. + +The asynchronous equivalent is `await async_array.chunk_byte_range(coords)`. + ## Features not yet ported to Zarr-Python 3 The following Zarr-Python 2 features are not yet available in Zarr-Python 3. diff --git a/src/zarr/core/_chunk_location.py b/src/zarr/core/_chunk_location.py new file mode 100644 index 0000000000..0371c41690 --- /dev/null +++ b/src/zarr/core/_chunk_location.py @@ -0,0 +1,72 @@ +"""Stored locations of logical chunks, separate from decoded array indexing.""" + +from __future__ import annotations + +import operator +from typing import TYPE_CHECKING, cast + +import numpy as np + +from zarr.codecs.sharding import MAX_UINT_64, ShardingCodec +from zarr.core.metadata.v3 import ArrayV3Metadata, RegularChunkGridMetadata + +if TYPE_CHECKING: + from typing import SupportsIndex + + from zarr.types import AnyAsyncArray + + +async def chunk_byte_range( + array: AnyAsyncArray, coords: tuple[SupportsIndex, ...] +) -> tuple[str, int, int] | None: + """Resolve an encoded location for a supported layout, without payload decoding.""" + metadata = array.metadata + shard = None + if isinstance(metadata, ArrayV3Metadata): + if not isinstance(metadata.chunk_grid, RegularChunkGridMetadata): + raise NotImplementedError("chunk_byte_range requires a regular chunk grid") + if len(metadata.codecs) == 1 and isinstance(metadata.codecs[0], ShardingCodec): + shard = metadata.codecs[0] + if any(isinstance(codec, ShardingCodec) for codec in shard.codecs): + raise NotImplementedError("chunk_byte_range does not support nested sharding") + elif any(isinstance(codec, ShardingCodec) for codec in metadata.codecs): + raise NotImplementedError("chunk_byte_range requires sharding without outer codecs") + + if len(coords) != len(array.shape): + raise ValueError("Expected one chunk coordinate per array dimension") + if any(isinstance(coord, (bool, np.bool_)) for coord in coords): + raise TypeError("Chunk coordinates must be integers, not booleans") + normalized = tuple(operator.index(coord) for coord in coords) + chunk_shape = shard.chunk_shape if shard is not None else array.chunks + if any( + coord < 0 or coord * width >= size + for coord, width, size in zip(normalized, chunk_shape, array.shape, strict=True) + ): + raise IndexError(f"Chunk coordinates {coords} are out of bounds") + + if shard is None: + key = metadata.encode_chunk_key(normalized) + path = array.store_path / key + try: + size = await path.store.getsize(path.path) + except FileNotFoundError: + return None + return path.path, 0, size + + metadata = cast("ArrayV3Metadata", metadata) + grid = cast("RegularChunkGridMetadata", metadata.chunk_grid) + per_shard = tuple( + size // width for size, width in zip(grid.chunk_shape, chunk_shape, strict=True) + ) + outer = tuple(coord // count for coord, count in zip(normalized, per_shard, strict=True)) + path = array.store_path / metadata.encode_chunk_key(outer) + index = await shard._load_shard_index_maybe(path, per_shard) + if index is None: + return None + local = tuple(coord % count for coord, count in zip(normalized, per_shard, strict=True)) + offset, length = (int(value) for value in index.offsets_and_lengths[local]) + if offset == MAX_UINT_64 and length == MAX_UINT_64: + return None + if offset == MAX_UINT_64 or length == MAX_UINT_64 or offset + length > MAX_UINT_64: + raise ValueError("Invalid byte range in shard index") + return path.path, offset, length diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 3735a4da18..e1be985db5 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -162,7 +162,7 @@ if TYPE_CHECKING: from collections.abc import Iterator - from typing import Self + from typing import Self, SupportsIndex import numpy.typing as npt @@ -1311,6 +1311,38 @@ async def example(): async def nbytes_stored(self) -> int: return await _nbytes_stored(self.store_path) + async def chunk_byte_range( + self, coords: tuple[SupportsIndex, ...] + ) -> tuple[str, int, int] | None: + """Locate the encoded bytes of one logical chunk without decoding its data. + + Parameters + ---------- + coords : tuple of int + Nonnegative logical chunk coordinates, not element or shard coordinates. + + Returns + ------- + tuple of (str, int, int) or None + Store-relative key, byte offset, and encoded byte length. Returns None + for an absent chunk, including an absent inner chunk of a stored shard. + + Notes + ----- + Supports regular chunk grids and one-level sharding with no outer codecs. + For sharded arrays this reads the shard index, not chunk payloads. Each + call reads its own index; there is no cross-call cache. For unsharded + arrays this calls `Store.getsize`, whose default implementation reads + the entire object if the store does not provide an efficient size lookup. + + Locations describe the current stored representation, not decoded byte + counts. They are not a snapshot or an integrity check of the payload. + Concurrent writes can invalidate the returned location. + """ + from zarr.core._chunk_location import chunk_byte_range + + return await chunk_byte_range(self, coords) + def _iter_chunk_coords( self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None ) -> Iterator[tuple[int, ...]]: @@ -2450,6 +2482,28 @@ def nbytes_stored(self) -> int: """ return sync(self.async_array.nbytes_stored()) + def chunk_byte_range(self, coords: tuple[SupportsIndex, ...]) -> tuple[str, int, int] | None: + """Locate the encoded bytes of a logical chunk without decoding its data. + + Parameters + ---------- + coords : tuple of int + Nonnegative logical chunk coordinates, not element or shard coordinates. + + Returns + ------- + tuple of (str, int, int) or None + Store-relative key, byte offset, and encoded byte length, or None if + the chunk is absent. + + Notes + ----- + See [`AsyncArray.chunk_byte_range`][zarr.AsyncArray.chunk_byte_range] for + supported layouts, IO costs, and the requirement that the source not change + while using the location. + """ + return sync(self.async_array.chunk_byte_range(coords)) + def _iter_shard_keys( self, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None ) -> Iterator[str]: diff --git a/tests/test_chunk_byte_range.py b/tests/test_chunk_byte_range.py new file mode 100644 index 0000000000..89ce7f18b8 --- /dev/null +++ b/tests/test_chunk_byte_range.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import numcodecs +import numpy as np +import pytest + +import zarr +from zarr.abc.store import RangeByteRequest, SuffixByteRequest +from zarr.codecs import BytesCodec, ShardingCodec, TransposeCodec, ZstdCodec +from zarr.core.buffer import default_buffer_prototype +from zarr.core.metadata.v3 import ArrayV3Metadata +from zarr.core.sync import sync +from zarr.errors import ZarrUserWarning +from zarr.storage import LocalStore, MemoryStore + +if TYPE_CHECKING: + from pathlib import Path + + from zarr.abc.store import ByteRequest + from zarr.codecs.sharding import IndexLocation + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.types import AnyArray + + +def make_array( + tmp_path: Path, + store_kind: str, + layout: str, + location: IndexLocation = "end", + compressed: bool = False, +) -> AnyArray: + store = MemoryStore() if store_kind == "memory" else LocalStore(tmp_path) + if layout == "sharded": + inner: list[BytesCodec | ZstdCodec] = ( + [BytesCodec(), ZstdCodec()] if compressed else [BytesCodec()] + ) + return zarr.create( + store=store, + path="nested/array", + shape=(5, 5), + chunks=(4, 4), + dtype="int32", + zarr_format=3, + codecs=[ShardingCodec(chunk_shape=(2, 2), codecs=inner, index_location=location)], + ) + if layout == "v2": + compressors = numcodecs.Zstd() if compressed else None + else: + compressors = [ZstdCodec()] if compressed else None + return zarr.create_array( + store=store, + name="nested/array", + shape=(5, 5), + chunks=(2, 2), + dtype="int32", + zarr_format=2 if layout == "v2" else 3, + compressors=compressors, + ) + + +@pytest.mark.parametrize("store_kind", ["memory", "local"]) +@pytest.mark.parametrize( + ("layout", "location"), + [("v2", "end"), ("v3", "end"), ("sharded", "start"), ("sharded", "end")], +) +@pytest.mark.parametrize("compressed", [False, True]) +def test_chunk_byte_range_decodes_actual_payload( + tmp_path: Path, + store_kind: str, + layout: str, + location: IndexLocation, + compressed: bool, +) -> None: + array = make_array(tmp_path, store_kind, layout, location, compressed) + data = np.arange(25, dtype="int32").reshape(5, 5) + 1 + array[:] = data + # The last logical chunk intersects the array but is not a complete 2x2 block. + for coords in ((0, 0), (0, 1), (2, 2)): + result = array.chunk_byte_range(coords) + assert result is not None + key, offset, length = result + assert key.startswith("nested/array/") + encoded = sync( + array.store_path.store.get( + key, default_buffer_prototype(), RangeByteRequest(offset, offset + length) + ) + ) + assert encoded is not None + raw = numcodecs.Zstd().decode(encoded.to_bytes()) if compressed else encoded.to_bytes() + decoded = np.frombuffer(raw, dtype=" None: + array = make_array(tmp_path, store_kind, layout) + assert await array.async_array.chunk_byte_range((0, 0)) is None + array[:2, :2] = 7 + assert await array.async_array.chunk_byte_range((0, 0)) is not None + # In a sharded array this shares an existing object with the written chunk. + assert await array.async_array.chunk_byte_range((0, 1)) is None + assert await array.async_array.chunk_byte_range((2, 2)) is None + array[:2, :2] = 0 + assert await array.async_array.chunk_byte_range((0, 0)) is None + + +@pytest.mark.parametrize( + ("coords", "exception"), + [ + ((-1, 0), IndexError), + ((3, 0), IndexError), + ((0, 3), IndexError), + ((0,), ValueError), + ((0, 0, 0), ValueError), + ((0.0, 0), TypeError), + ((True, 0), TypeError), + ((np.bool_(False), 0), TypeError), + ], +) +def test_invalid_coordinates(tmp_path: Path, coords: Any, exception: type[Exception]) -> None: + array = make_array(tmp_path, "memory", "sharded") + with pytest.raises(exception): + array.chunk_byte_range(coords) + + +def test_numpy_integer_coordinates(tmp_path: Path) -> None: + array = make_array(tmp_path, "memory", "sharded") + array[:2, :2] = 5 + assert array.chunk_byte_range((np.int64(0), np.int32(0))) == array.chunk_byte_range((0, 0)) + + +@pytest.mark.parametrize("location", ["start", "end"]) +async def test_sharded_inspection_reads_only_index(tmp_path: Path, location: IndexLocation) -> None: + class RecordingStore(MemoryStore): + reads: list[ByteRequest | None] + + async def get( + self, + key: str, + prototype: BufferPrototype | None = None, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + self.reads.append(byte_range) + return await super().get(key, prototype, byte_range) + + store = RecordingStore() + store.reads = [] + array = zarr.create( + store=store, + shape=(4, 4), + chunks=(4, 4), + dtype="int32", + zarr_format=3, + codecs=[ShardingCodec(chunk_shape=(2, 2), codecs=[BytesCodec()], index_location=location)], + ) + array[:2, :2] = 3 + store.reads.clear() + assert await array.async_array.chunk_byte_range((0, 0)) is not None + expected = RangeByteRequest(0, 68) if location == "start" else SuffixByteRequest(68) + assert store.reads == [expected] + + +def test_scalar_array(tmp_path: Path) -> None: + array = zarr.create_array( + store=LocalStore(tmp_path), shape=(), chunks=(), dtype="int32", compressors=None + ) + assert array.chunk_byte_range(()) is None + array[...] = 7 + result = array.chunk_byte_range(()) + assert result == ("c", 0, 4) + + +def test_zero_size_array(tmp_path: Path) -> None: + array = zarr.create_array(store=LocalStore(tmp_path), shape=(0,), chunks=(2,), dtype="int32") + with pytest.raises(IndexError): + array.chunk_byte_range((0,)) + + +def test_nested_sharding_rejected(tmp_path: Path) -> None: + array = zarr.create( + store=LocalStore(tmp_path), + shape=(4, 4), + chunks=(4, 4), + dtype="int32", + zarr_format=3, + codecs=[ShardingCodec(chunk_shape=(2, 2), codecs=[ShardingCodec(chunk_shape=(1, 1))])], + ) + with pytest.raises(NotImplementedError, match="nested"): + array.chunk_byte_range((0, 0)) + + +def test_outer_transform_rejected(tmp_path: Path) -> None: + with pytest.warns(ZarrUserWarning, match="Combining"): + array = zarr.create( + store=LocalStore(tmp_path), + shape=(4, 4), + chunks=(4, 4), + dtype="int32", + zarr_format=3, + codecs=[TransposeCodec(order=(1, 0)), ShardingCodec(chunk_shape=(2, 2))], + ) + with pytest.raises(NotImplementedError, match="outer codecs"): + array.chunk_byte_range((0, 0)) + + +@pytest.mark.parametrize(("offset", "length"), [(2**64 - 1, 16), (2**64 - 10, 20)]) +async def test_invalid_index_range(tmp_path: Path, offset: int, length: int) -> None: + array = make_array(tmp_path, "memory", "sharded") + array[:2, :2] = 3 + assert isinstance(array.metadata, ArrayV3Metadata) + codec = array.metadata.codecs[0] + assert isinstance(codec, ShardingCodec) + path = array.store_path / "c/0/0" + original = await path.get(default_buffer_prototype()) + assert original is not None + index = await codec._decode_shard_index(original[-68:], (2, 2)) + index = index._replace(offsets_and_lengths=index.offsets_and_lengths.copy()) + index.offsets_and_lengths[0, 0] = (offset, length) + encoded_index = await codec._encode_shard_index(index) + await path.set( + default_buffer_prototype().buffer.from_bytes( + original[:-68].to_bytes() + encoded_index.to_bytes() + ) + ) + with pytest.raises(ValueError, match="Invalid byte range"): + await array.async_array.chunk_byte_range((0, 0)) + + +def test_rectilinear_grid_rejected(tmp_path: Path) -> None: + with zarr.config.set({"array.rectilinear_chunks": True}): + array = zarr.create_array( + store=LocalStore(tmp_path), shape=(5,), chunks=((2, 3),), dtype="int32" + ) + with pytest.raises(NotImplementedError, match="regular chunk grid"): + array.chunk_byte_range((0,)) + + +async def test_size_lookup_errors_are_not_reported_as_missing(tmp_path: Path) -> None: + class UnavailableStore(MemoryStore): + async def getsize(self, key: str) -> int: + raise PermissionError(key) + + array = zarr.create_array(store=UnavailableStore(), shape=(4,), chunks=(2,), dtype="int32") + with pytest.raises(PermissionError): + await array.async_array.chunk_byte_range((0,))