Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/4646.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`WrapperStore`, and so `LoggingStore`, now forwards `getsize`, `getsize_prefix`, `get_ranges_sync` and `supports_consolidated_metadata` to the store it wraps. Before this change, a wrapper read a whole value to measure its size, and reported support for consolidated metadata even when the wrapped store reported none. `LoggingStore` now also logs range reads (`get_ranges` and `get_ranges_sync`).
61 changes: 60 additions & 1 deletion src/zarr/storage/_logging.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from zarr.storage._wrapper import WrapperStore

if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Generator, Iterable
from collections.abc import AsyncGenerator, AsyncIterator, Generator, Iterable, Sequence

from zarr.abc.store import ByteRequest
from zarr.core.buffer import Buffer, BufferPrototype
Expand Down Expand Up @@ -121,6 +121,11 @@ def supports_listing(self) -> bool:
with self.log():
return self._store.supports_listing

@property
def supports_consolidated_metadata(self) -> bool:
with self.log():
return self._store.supports_consolidated_metadata

@property
def read_only(self) -> bool:
with self.log():
Expand Down Expand Up @@ -255,3 +260,57 @@ async def getsize(self, key: str) -> int:
async def getsize_prefix(self, prefix: str) -> int:
with self.log(prefix):
return await self._store.getsize_prefix(prefix)

async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None:
values = list(values)
with self.log(",".join(key for key, _ in values)):
await self._store._set_many(values)

async def _get_many(
self, requests: Iterable[tuple[str, BufferPrototype, ByteRequest | None]]
) -> AsyncGenerator[tuple[str, Buffer | None], None]:
requests = list(requests)
with self.log(",".join(key for key, _, _ in requests)):
async for item in self._store._get_many(requests):
yield item

async def get_ranges(
self,
key: str,
byte_ranges: Sequence[ByteRequest | None],
*,
prototype: BufferPrototype,
max_concurrency: int | None = None,
max_gap_bytes: int | None = None,
max_coalesced_bytes: int | None = None,
) -> AsyncIterator[Sequence[tuple[int, Buffer | None]]]:
# docstring inherited
with self.log(key):
async for group in super().get_ranges(
key,
byte_ranges,
prototype=prototype,
max_concurrency=max_concurrency,
max_gap_bytes=max_gap_bytes,
max_coalesced_bytes=max_coalesced_bytes,
):
yield group

def get_ranges_sync(
self,
key: str,
byte_ranges: Sequence[ByteRequest | None],
*,
prototype: BufferPrototype,
max_gap_bytes: int | None = None,
max_coalesced_bytes: int | None = None,
) -> Sequence[tuple[int, Buffer | None]]:
# docstring inherited
with self.log(key):
return super().get_ranges_sync(
key,
byte_ranges,
prototype=prototype,
max_gap_bytes=max_gap_bytes,
max_coalesced_bytes=max_coalesced_bytes,
)
31 changes: 31 additions & 0 deletions src/zarr/storage/_wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,33 @@ async def get_ranges(
async for group in self._store.get_ranges(key, byte_ranges, prototype=prototype, **kwargs):
yield group

def get_ranges_sync(
self,
key: str,
byte_ranges: Sequence[ByteRequest | None],
*,
prototype: BufferPrototype,
max_gap_bytes: int | None = None,
max_coalesced_bytes: int | None = None,
) -> Sequence[tuple[int, Buffer | None]]:
"""Forward `get_ranges_sync` to the wrapped store.

As with `get_ranges`, `None` for a coalescing kwarg means "don't
override the wrapped store's default".
"""
kwargs: dict[str, int] = {}
if max_gap_bytes is not None:
kwargs["max_gap_bytes"] = max_gap_bytes
if max_coalesced_bytes is not None:
kwargs["max_coalesced_bytes"] = max_coalesced_bytes
return self._store.get_ranges_sync(key, byte_ranges, prototype=prototype, **kwargs)

async def getsize(self, key: str) -> int:
return await self._store.getsize(key)

async def getsize_prefix(self, prefix: str) -> int:
return await self._store.getsize_prefix(prefix)

async def exists(self, key: str) -> bool:
return await self._store.exists(key)

Expand All @@ -155,6 +182,10 @@ def supports_writes(self) -> bool:
def supports_deletes(self) -> bool:
return self._store.supports_deletes

@property
def supports_consolidated_metadata(self) -> bool:
return self._store.supports_consolidated_metadata

@property
def _supports_sync_io(self) -> bool:
# The delegating `*_sync` methods below make every wrapper structurally
Expand Down
21 changes: 21 additions & 0 deletions src/zarr/testing/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -822,6 +822,27 @@ async def get_ranges(
async for group in Store.get_ranges(self, key, byte_ranges, prototype=prototype, **kwargs):
yield group

def get_ranges_sync(
self,
key: str,
byte_ranges: Sequence[ByteRequest | None],
*,
prototype: BufferPrototype,
max_gap_bytes: int | None = None,
max_coalesced_bytes: int | None = None,
) -> Sequence[tuple[int, Buffer | None]]:
"""Byte-range reads built on `self.get_sync`, so each fetch pays latency.

Routes through the coalescing `Store.get_ranges_sync` default instead of
the `WrapperStore` delegation, for the same reason as `get_ranges`.
"""
kwargs: dict[str, int] = {}
if max_gap_bytes is not None:
kwargs["max_gap_bytes"] = max_gap_bytes
if max_coalesced_bytes is not None:
kwargs["max_coalesced_bytes"] = max_coalesced_bytes
return Store.get_ranges_sync(self, key, byte_ranges, prototype=prototype, **kwargs)

async def get_partial_values(
self,
prototype: BufferPrototype,
Expand Down
Loading
Loading