Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions changes/1113.feature.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Add ``Array.chunk_byte_range`` and ``AsyncArray.chunk_byte_range`` to locate stored
logical chunks in regular unsharded arrays and one-level sharded Zarr v3 arrays.
33 changes: 33 additions & 0 deletions docs/user-guide/arrays.md
Original file line number Diff line number Diff line change
Expand Up @@ -827,6 +827,39 @@ pairs are accepted:

When writing, Zarr automatically compresses repeated values into RLE format.

## Locating encoded chunks

`Array.chunk_byte_range(coords)` returns the store-relative key, byte offset, and
encoded length of a logical chunk. Coordinates are chunk indexes, not element
indexes. An absent chunk returns `None`, including an absent inner chunk within
a shard that contains other chunks.

```python test="true"
import zarr

array = zarr.create_array(
store=zarr.storage.MemoryStore(),
shape=(8, 8), chunks=(2, 2), shards=(4, 4), dtype="int32",
)
array[:2, :2] = 7
key, offset, length = array.chunk_byte_range((0, 0))
assert key == "c/0/0"
assert offset >= 0 and length > 0
assert array.chunk_byte_range((0, 1)) is None
```

This supports regular unsharded arrays and one-level sharding with no outer
codecs. Nested sharding and rectilinear grids are not supported. For sharded
arrays the method reads and decodes the shard index, not the chunk payload.
For unsharded arrays it uses the store's size lookup, which may read the entire
object if the backend does not implement an efficient lookup.

Each call reads its own index. The returned location describes encoded bytes,
not uncompressed array bytes, and can become stale if the store changes. This
is not a snapshot API or a payload-integrity check.

The asynchronous equivalent is `await async_array.chunk_byte_range(coords)`.

## Features not yet ported to Zarr-Python 3

The following Zarr-Python 2 features are not yet available in Zarr-Python 3.
Expand Down
72 changes: 72 additions & 0 deletions src/zarr/core/_chunk_location.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""Stored locations of logical chunks, separate from decoded array indexing."""

from __future__ import annotations

import operator
from typing import TYPE_CHECKING, cast

import numpy as np

from zarr.codecs.sharding import MAX_UINT_64, ShardingCodec
from zarr.core.metadata.v3 import ArrayV3Metadata, RegularChunkGridMetadata

if TYPE_CHECKING:
from typing import SupportsIndex

from zarr.types import AnyAsyncArray


async def chunk_byte_range(
array: AnyAsyncArray, coords: tuple[SupportsIndex, ...]
) -> tuple[str, int, int] | None:
"""Resolve an encoded location for a supported layout, without payload decoding."""
metadata = array.metadata
shard = None
if isinstance(metadata, ArrayV3Metadata):
if not isinstance(metadata.chunk_grid, RegularChunkGridMetadata):
raise NotImplementedError("chunk_byte_range requires a regular chunk grid")
if len(metadata.codecs) == 1 and isinstance(metadata.codecs[0], ShardingCodec):
shard = metadata.codecs[0]
if any(isinstance(codec, ShardingCodec) for codec in shard.codecs):
raise NotImplementedError("chunk_byte_range does not support nested sharding")
elif any(isinstance(codec, ShardingCodec) for codec in metadata.codecs):
raise NotImplementedError("chunk_byte_range requires sharding without outer codecs")

if len(coords) != len(array.shape):
raise ValueError("Expected one chunk coordinate per array dimension")
if any(isinstance(coord, (bool, np.bool_)) for coord in coords):
raise TypeError("Chunk coordinates must be integers, not booleans")
normalized = tuple(operator.index(coord) for coord in coords)
chunk_shape = shard.chunk_shape if shard is not None else array.chunks
if any(
coord < 0 or coord * width >= size
for coord, width, size in zip(normalized, chunk_shape, array.shape, strict=True)
):
raise IndexError(f"Chunk coordinates {coords} are out of bounds")

if shard is None:
key = metadata.encode_chunk_key(normalized)
path = array.store_path / key
try:
size = await path.store.getsize(path.path)
except FileNotFoundError:
return None
return path.path, 0, size

metadata = cast("ArrayV3Metadata", metadata)
grid = cast("RegularChunkGridMetadata", metadata.chunk_grid)
per_shard = tuple(
size // width for size, width in zip(grid.chunk_shape, chunk_shape, strict=True)
)
outer = tuple(coord // count for coord, count in zip(normalized, per_shard, strict=True))
path = array.store_path / metadata.encode_chunk_key(outer)
index = await shard._load_shard_index_maybe(path, per_shard)
if index is None:
return None
local = tuple(coord % count for coord, count in zip(normalized, per_shard, strict=True))
offset, length = (int(value) for value in index.offsets_and_lengths[local])
if offset == MAX_UINT_64 and length == MAX_UINT_64:
return None
if offset == MAX_UINT_64 or length == MAX_UINT_64 or offset + length > MAX_UINT_64:
raise ValueError("Invalid byte range in shard index")
return path.path, offset, length
56 changes: 55 additions & 1 deletion src/zarr/core/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@

if TYPE_CHECKING:
from collections.abc import Iterator
from typing import Self
from typing import Self, SupportsIndex

import numpy.typing as npt

Expand Down Expand Up @@ -1311,6 +1311,38 @@ async def example():
async def nbytes_stored(self) -> int:
return await _nbytes_stored(self.store_path)

async def chunk_byte_range(
self, coords: tuple[SupportsIndex, ...]
) -> tuple[str, int, int] | None:
"""Locate the encoded bytes of one logical chunk without decoding its data.

Parameters
----------
coords : tuple of int
Nonnegative logical chunk coordinates, not element or shard coordinates.

Returns
-------
tuple of (str, int, int) or None
Store-relative key, byte offset, and encoded byte length. Returns None
for an absent chunk, including an absent inner chunk of a stored shard.

Notes
-----
Supports regular chunk grids and one-level sharding with no outer codecs.
For sharded arrays this reads the shard index, not chunk payloads. Each
call reads its own index; there is no cross-call cache. For unsharded
arrays this calls `Store.getsize`, whose default implementation reads
the entire object if the store does not provide an efficient size lookup.

Locations describe the current stored representation, not decoded byte
counts. They are not a snapshot or an integrity check of the payload.
Concurrent writes can invalidate the returned location.
"""
from zarr.core._chunk_location import chunk_byte_range

return await chunk_byte_range(self, coords)

def _iter_chunk_coords(
self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None
) -> Iterator[tuple[int, ...]]:
Expand Down Expand Up @@ -2450,6 +2482,28 @@ def nbytes_stored(self) -> int:
"""
return sync(self.async_array.nbytes_stored())

def chunk_byte_range(self, coords: tuple[SupportsIndex, ...]) -> tuple[str, int, int] | None:
"""Locate the encoded bytes of a logical chunk without decoding its data.

Parameters
----------
coords : tuple of int
Nonnegative logical chunk coordinates, not element or shard coordinates.

Returns
-------
tuple of (str, int, int) or None
Store-relative key, byte offset, and encoded byte length, or None if
the chunk is absent.

Notes
-----
See [`AsyncArray.chunk_byte_range`][zarr.AsyncArray.chunk_byte_range] for
supported layouts, IO costs, and the requirement that the source not change
while using the location.
"""
return sync(self.async_array.chunk_byte_range(coords))

def _iter_shard_keys(
self, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None
) -> Iterator[str]:
Expand Down
Loading
Loading