diff --git a/changes/4646.bugfix.md b/changes/4646.bugfix.md new file mode 100644 index 0000000000..2a24d442e9 --- /dev/null +++ b/changes/4646.bugfix.md @@ -0,0 +1 @@ +`WrapperStore`, and so `LoggingStore`, now forwards `getsize`, `getsize_prefix`, `get_ranges_sync` and `supports_consolidated_metadata` to the store it wraps. Before this change, a wrapper read a whole value to measure its size, and reported support for consolidated metadata even when the wrapped store reported none. `LoggingStore` now also logs range reads (`get_ranges` and `get_ranges_sync`). diff --git a/src/zarr/storage/_logging.py b/src/zarr/storage/_logging.py index cdf0731430..962d189c5a 100644 --- a/src/zarr/storage/_logging.py +++ b/src/zarr/storage/_logging.py @@ -12,7 +12,7 @@ from zarr.storage._wrapper import WrapperStore if TYPE_CHECKING: - from collections.abc import AsyncGenerator, Generator, Iterable + from collections.abc import AsyncGenerator, AsyncIterator, Generator, Iterable, Sequence from zarr.abc.store import ByteRequest from zarr.core.buffer import Buffer, BufferPrototype @@ -121,6 +121,11 @@ def supports_listing(self) -> bool: with self.log(): return self._store.supports_listing + @property + def supports_consolidated_metadata(self) -> bool: + with self.log(): + return self._store.supports_consolidated_metadata + @property def read_only(self) -> bool: with self.log(): @@ -255,3 +260,57 @@ async def getsize(self, key: str) -> int: async def getsize_prefix(self, prefix: str) -> int: with self.log(prefix): return await self._store.getsize_prefix(prefix) + + async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None: + values = list(values) + with self.log(",".join(key for key, _ in values)): + await self._store._set_many(values) + + async def _get_many( + self, requests: Iterable[tuple[str, BufferPrototype, ByteRequest | None]] + ) -> AsyncGenerator[tuple[str, Buffer | None], None]: + requests = list(requests) + with self.log(",".join(key for key, _, _ in requests)): + async for item in self._store._get_many(requests): + yield item + + async def get_ranges( + self, + key: str, + byte_ranges: Sequence[ByteRequest | None], + *, + prototype: BufferPrototype, + max_concurrency: int | None = None, + max_gap_bytes: int | None = None, + max_coalesced_bytes: int | None = None, + ) -> AsyncIterator[Sequence[tuple[int, Buffer | None]]]: + # docstring inherited + with self.log(key): + async for group in super().get_ranges( + key, + byte_ranges, + prototype=prototype, + max_concurrency=max_concurrency, + max_gap_bytes=max_gap_bytes, + max_coalesced_bytes=max_coalesced_bytes, + ): + yield group + + def get_ranges_sync( + self, + key: str, + byte_ranges: Sequence[ByteRequest | None], + *, + prototype: BufferPrototype, + max_gap_bytes: int | None = None, + max_coalesced_bytes: int | None = None, + ) -> Sequence[tuple[int, Buffer | None]]: + # docstring inherited + with self.log(key): + return super().get_ranges_sync( + key, + byte_ranges, + prototype=prototype, + max_gap_bytes=max_gap_bytes, + max_coalesced_bytes=max_coalesced_bytes, + ) diff --git a/src/zarr/storage/_wrapper.py b/src/zarr/storage/_wrapper.py index 6f498a655d..58b98ce00c 100644 --- a/src/zarr/storage/_wrapper.py +++ b/src/zarr/storage/_wrapper.py @@ -135,6 +135,33 @@ async def get_ranges( async for group in self._store.get_ranges(key, byte_ranges, prototype=prototype, **kwargs): yield group + def get_ranges_sync( + self, + key: str, + byte_ranges: Sequence[ByteRequest | None], + *, + prototype: BufferPrototype, + max_gap_bytes: int | None = None, + max_coalesced_bytes: int | None = None, + ) -> Sequence[tuple[int, Buffer | None]]: + """Forward `get_ranges_sync` to the wrapped store. + + As with `get_ranges`, `None` for a coalescing kwarg means "don't + override the wrapped store's default". + """ + kwargs: dict[str, int] = {} + if max_gap_bytes is not None: + kwargs["max_gap_bytes"] = max_gap_bytes + if max_coalesced_bytes is not None: + kwargs["max_coalesced_bytes"] = max_coalesced_bytes + return self._store.get_ranges_sync(key, byte_ranges, prototype=prototype, **kwargs) + + async def getsize(self, key: str) -> int: + return await self._store.getsize(key) + + async def getsize_prefix(self, prefix: str) -> int: + return await self._store.getsize_prefix(prefix) + async def exists(self, key: str) -> bool: return await self._store.exists(key) @@ -155,6 +182,10 @@ def supports_writes(self) -> bool: def supports_deletes(self) -> bool: return self._store.supports_deletes + @property + def supports_consolidated_metadata(self) -> bool: + return self._store.supports_consolidated_metadata + @property def _supports_sync_io(self) -> bool: # The delegating `*_sync` methods below make every wrapper structurally diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 4c948a783c..410e2d2128 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -822,6 +822,27 @@ async def get_ranges( async for group in Store.get_ranges(self, key, byte_ranges, prototype=prototype, **kwargs): yield group + def get_ranges_sync( + self, + key: str, + byte_ranges: Sequence[ByteRequest | None], + *, + prototype: BufferPrototype, + max_gap_bytes: int | None = None, + max_coalesced_bytes: int | None = None, + ) -> Sequence[tuple[int, Buffer | None]]: + """Byte-range reads built on `self.get_sync`, so each fetch pays latency. + + Routes through the coalescing `Store.get_ranges_sync` default instead of + the `WrapperStore` delegation, for the same reason as `get_ranges`. + """ + kwargs: dict[str, int] = {} + if max_gap_bytes is not None: + kwargs["max_gap_bytes"] = max_gap_bytes + if max_coalesced_bytes is not None: + kwargs["max_coalesced_bytes"] = max_coalesced_bytes + return Store.get_ranges_sync(self, key, byte_ranges, prototype=prototype, **kwargs) + async def get_partial_values( self, prototype: BufferPrototype, diff --git a/tests/test_store/test_stateful_conformance.py b/tests/test_store/test_stateful_conformance.py new file mode 100644 index 0000000000..2e324c72a3 --- /dev/null +++ b/tests/test_store/test_stateful_conformance.py @@ -0,0 +1,1001 @@ +"""Differential stateful test: every built-in store must agree with one reference model. + +A Hypothesis state machine applies each operation to all stores at once and compares every result with +the contract below. A store that disagrees fails the test, unless the disagreement is a known divergence: +an entry in `KNOWN_DIVERGENCES` that records what that store does instead. `test_divergence_ledger_is_exact` +runs each entry's witness against every store, so the ledger has to name exactly the stores that diverge: +a fixed store must be removed from its entry, and a newly diverging one must be added. + +The contract (what the `Store` docstrings should promise): + +- Keys use the grammar zarr produces through `StorePath`: `/`-separated segments, no empty, `.` or `..` + segment, no leading or trailing `/`. A key is never also a directory of another key. +- A prefix names a directory: `"a"` and `"a/"` mean the same directory, and `""` means the root. + `list_prefix`, `list_dir`, `getsize_prefix`, `is_empty` and `delete_dir` see only keys inside that + directory: `list_prefix("lat")` never returns `lat_bnds/...`, and `list_prefix("a")` never returns a + key named `a`. +- `list_dir` returns the names of the directory's direct children that contain keys. +- `exists`, `get` and `getsize` concern keys only. For any other name (a directory, a name below a key) + they report a missing key. +- A byte range that reaches past the end of a value returns the bytes that exist, as Python slicing does. + +The state machine never generates a case whose divergence would leave the stores in different states: +`delete` on a directory, `delete_dir` on a key, writing a key where a directory used to be, and keys outside +the grammar. Those divergences are in `UNMODELED_DIVERGENCES`, checked only by their witnesses. + +Paths that leave a store's root (a leading `/`, `..`) are never generated: `LocalStore` joins them as real +paths, so `delete_dir("/")` runs `shutil.rmtree("/")`. `test_delete_stays_in_root` checks escapes against a +sentinel directory inside the test's own temporary directory. +""" + +from __future__ import annotations + +import importlib.util +import shutil +import sys +import tempfile +import uuid +from collections.abc import AsyncIterator, Callable +from dataclasses import dataclass, field +from functools import partial +from itertools import product +from pathlib import Path +from typing import Any + +import pytest +from hypothesis import event, note, settings +from hypothesis import strategies as st +from hypothesis.stateful import ( + RuleBasedStateMachine, + initialize, + invariant, + precondition, + rule, + run_state_machine_as_test, +) + +from zarr.abc.store import ( + ByteRequest, + OffsetByteRequest, + RangeByteRequest, + Store, + SuffixByteRequest, +) +from zarr.core.buffer import default_buffer_prototype +from zarr.core.sync import sync +from zarr.storage import ( + FsspecStore, + LocalStore, + LoggingStore, + ManagedMemoryStore, + MemoryStore, + ObjectStore, + ZipStore, +) +from zarr.testing.store import LatencyStore + +WINDOWS = sys.platform == "win32" + +PROTOTYPE = default_buffer_prototype() + +type Model = dict[str, bytes] +type Outcome = tuple[str, object] +"""`("ok", value)` or `("raised", exception class name)`.""" + +# --- stores ------------------------------------------------------------------------------------- + +type Cleanup = Callable[[], None] + + +@dataclass +class StoreUnderTest: + name: str + store: Store + cleanup: Cleanup + + +def _no_cleanup() -> None: + return None + + +def _remove_fsspec_memory_root(root: str) -> None: + import fsspec + + fs = fsspec.filesystem("memory") + if fs.exists(root): + fs.rm(root, recursive=True) + + +def _make_memory(tmp: Path) -> tuple[Store, Cleanup]: + return MemoryStore(), _no_cleanup + + +def _make_managed_memory(tmp: Path) -> tuple[Store, Cleanup]: + return ManagedMemoryStore(name=uuid.uuid4().hex, path="sub/dir"), _no_cleanup + + +def _make_local(tmp: Path) -> tuple[Store, Cleanup]: + return LocalStore(tmp / "local"), _no_cleanup + + +def _make_zip(tmp: Path) -> tuple[Store, Cleanup]: + store = ZipStore(tmp / "store.zip", mode="w") + return store, store.close + + +def _make_fsspec_local(tmp: Path) -> tuple[Store, Cleanup]: + (tmp / "fsspec").mkdir() + url = f"file://{tmp / 'fsspec'}" + return FsspecStore.from_url(url, storage_options={"auto_mkdir": True}), _no_cleanup + + +def _make_fsspec_memory(tmp: Path) -> tuple[Store, Cleanup]: + root = f"/{uuid.uuid4().hex}" + return FsspecStore.from_url(f"memory://{root}"), partial(_remove_fsspec_memory_root, root) + + +def _make_obstore_local(tmp: Path) -> tuple[Store, Cleanup]: + import obstore.store + + (tmp / "obstore").mkdir() + return ObjectStore(obstore.store.LocalStore(str(tmp / "obstore"))), _no_cleanup + + +def _make_obstore_memory(tmp: Path) -> tuple[Store, Cleanup]: + import obstore.store + + return ObjectStore(obstore.store.MemoryStore()), _no_cleanup + + +def _make_logging_memory(tmp: Path) -> tuple[Store, Cleanup]: + return LoggingStore(MemoryStore(), log_level="WARNING"), _no_cleanup + + +def _make_latency_local(tmp: Path) -> tuple[Store, Cleanup]: + return LatencyStore(LocalStore(tmp / "latency")), _no_cleanup + + +STORE_FACTORIES: dict[str, Callable[[Path], tuple[Store, Cleanup]]] = { + "memory": _make_memory, + "managed-memory": _make_managed_memory, + "local": _make_local, + "zip": _make_zip, + "fsspec-local": _make_fsspec_local, + "fsspec-memory": _make_fsspec_memory, + "obstore-local": _make_obstore_local, + "obstore-memory": _make_obstore_memory, + "logging-memory": _make_logging_memory, + "latency-local": _make_latency_local, +} + + +def _fsspec_unavailable() -> str | None: + if importlib.util.find_spec("fsspec") is None: + return "fsspec is not installed" + import fsspec + from packaging.version import parse as parse_version + + if parse_version(fsspec.__version__) < parse_version("2024.12.0"): + # Wrapping the sync `file` and `memory` filesystems needs `AsyncFileSystemWrapper`. + return "fsspec < 2024.12.0 has no AsyncFileSystemWrapper" + return None + + +def _obstore_unavailable() -> str | None: + return "obstore is not installed" if importlib.util.find_spec("obstore") is None else None + + +UNAVAILABLE: dict[str, str] = { + name: reason + for names, reason in ( + (("fsspec-local", "fsspec-memory"), _fsspec_unavailable()), + (("obstore-local", "obstore-memory"), _obstore_unavailable()), + ) + if reason is not None + for name in names +} +"""Stores this environment can't build, and why.""" + +WRITABLE_STORES = tuple(name for name in STORE_FACTORIES if name not in {"zip", *UNAVAILABLE}) +"""Stores that support every operation, including deletes and overwrites.""" + +APPEND_ONLY_STORES = ("zip",) +"""`ZipStore` can't delete or overwrite a key.""" + + +def _build(name: str, tmp: Path) -> StoreUnderTest: + if name in UNAVAILABLE: + pytest.skip(UNAVAILABLE[name]) + store, cleanup = STORE_FACTORIES[name](tmp) + target = StoreUnderTest(name, store, cleanup) + try: + sync(store._ensure_open()) + except BaseException: + cleanup() + raise + return target + + +# --- the contract ------------------------------------------------------------------------------- + + +def _directory(prefix: str) -> str: + return prefix.strip("/") + + +def _is_under(key: str, directory: str) -> bool: + return directory == "" or key.startswith(directory + "/") + + +def _keys_under(model: Model, prefix: str) -> list[str]: + directory = _directory(prefix) + return sorted(key for key in model if _is_under(key, directory)) + + +def _children(model: Model, prefix: str) -> list[str]: + directory = _directory(prefix) + start = len(directory) + 1 if directory else 0 + return sorted({key[start:].split("/")[0] for key in _keys_under(model, prefix)}) + + +def _directories(model: Model) -> set[str]: + return { + "/".join(key.split("/")[:depth]) for key in model for depth in range(1, key.count("/") + 1) + } + + +@dataclass +class State: + """The reference model, and every directory it has ever had.""" + + model: Model = field(default_factory=dict) + past_directories: set[str] = field(default_factory=set) + """Filesystem stores keep a directory after its last key is deleted.""" + + def record(self, operation: str, args: tuple[Any, ...]) -> None: + """Apply a write as the contract says a store applies it.""" + match operation, args: + case "set", (key, value): + self.model[key] = value + case "set_if_not_exists", (key, value): + self.model.setdefault(key, value) + case "delete", (key,): + self.model.pop(key, None) + case "delete_dir", (prefix,): + for key in _keys_under(self.model, prefix): + del self.model[key] + self.past_directories |= _directories(self.model) + + +def _slice(value: bytes, byte_range: ByteRequest | None) -> bytes: + match byte_range: + case None: + return value + case RangeByteRequest(start, end): + return value[start:end] + case OffsetByteRequest(offset): + return value[offset:] + case SuffixByteRequest(suffix): + return value[max(0, len(value) - suffix) :] + raise TypeError(byte_range) + + +def _expect_list(model: Model) -> Outcome: + return ("ok", sorted(model)) + + +def _expect_list_prefix(model: Model, prefix: str) -> Outcome: + return ("ok", _keys_under(model, prefix)) + + +def _expect_list_dir(model: Model, prefix: str) -> Outcome: + return ("ok", _children(model, prefix)) + + +def _expect_getsize_prefix(model: Model, prefix: str) -> Outcome: + return ("ok", sum(len(model[key]) for key in _keys_under(model, prefix))) + + +def _expect_is_empty(model: Model, prefix: str) -> Outcome: + return ("ok", not _keys_under(model, prefix)) + + +def _expect_exists(model: Model, name: str) -> Outcome: + return ("ok", name in model) + + +def _expect_get(model: Model, key: str, byte_range: ByteRequest | None) -> Outcome: + if key not in model: + return ("ok", None) + return ("ok", _slice(model[key], byte_range)) + + +def _expect_getsize(model: Model, key: str) -> Outcome: + if key not in model: + return ("raised", "FileNotFoundError") + return ("ok", len(model[key])) + + +def _expect_none(model: Model, *args: Any) -> Outcome: + return ("ok", None) + + +# --- operations --------------------------------------------------------------------------------- + + +async def _collect(iterator: AsyncIterator[str]) -> list[str]: + return sorted([item async for item in iterator]) + + +async def _call_list(store: Store) -> list[str]: + return await _collect(store.list()) + + +async def _call_list_prefix(store: Store, prefix: str) -> list[str]: + return await _collect(store.list_prefix(prefix)) + + +async def _call_list_dir(store: Store, prefix: str) -> list[str]: + return await _collect(store.list_dir(prefix)) + + +async def _call_get(store: Store, key: str, byte_range: ByteRequest | None) -> bytes | None: + value = await store.get(key, PROTOTYPE, byte_range=byte_range) + return None if value is None else value.to_bytes() + + +async def _call_set(store: Store, key: str, value: bytes) -> None: + await store.set(key, PROTOTYPE.buffer.from_bytes(value)) + + +async def _call_set_if_not_exists(store: Store, key: str, value: bytes) -> None: + await store.set_if_not_exists(key, PROTOTYPE.buffer.from_bytes(value)) + + +async def _call_method(method: str, store: Store, *args: Any) -> Any: + return await getattr(store, method)(*args) + + +@dataclass(frozen=True) +class Operation: + call: Callable[..., Any] + """A coroutine function taking the store and the operation's arguments.""" + contract: Callable[..., Outcome] + """The contract's outcome, given the model and the operation's arguments.""" + + +OPERATIONS: dict[str, Operation] = { + "list": Operation(_call_list, _expect_list), + "list_prefix": Operation(_call_list_prefix, _expect_list_prefix), + "list_dir": Operation(_call_list_dir, _expect_list_dir), + "getsize_prefix": Operation(partial(_call_method, "getsize_prefix"), _expect_getsize_prefix), + "is_empty": Operation(partial(_call_method, "is_empty"), _expect_is_empty), + "exists": Operation(partial(_call_method, "exists"), _expect_exists), + "get": Operation(_call_get, _expect_get), + "getsize": Operation(partial(_call_method, "getsize"), _expect_getsize), + "set": Operation(_call_set, _expect_none), + "set_if_not_exists": Operation(_call_set_if_not_exists, _expect_none), + "delete": Operation(partial(_call_method, "delete"), _expect_none), + "delete_dir": Operation(partial(_call_method, "delete_dir"), _expect_none), +} + + +async def _outcome(operation: str, store: Store, args: tuple[Any, ...]) -> Outcome: + # Call inside the event loop: `ObjectStore.list_dir` starts a coroutine as soon as it is called. + try: + return ("ok", await OPERATIONS[operation].call(store, *args)) + except Exception as e: # noqa: BLE001 + return ("raised", type(e).__name__) + + +def _run(operation: str, store: Store, args: tuple[Any, ...]) -> Outcome: + return sync(_outcome(operation, store, args)) + + +# --- known divergences -------------------------------------------------------------------------- + +type Step = tuple[str, tuple[Any, ...]] +type Accepts = Callable[[Outcome, State, tuple[Any, ...]], bool] + + +@dataclass(frozen=True) +class Divergence: + stores: frozenset[str] + summary: str + witness: tuple[Step, ...] + """Steps whose last one breaks the contract on exactly the listed stores.""" + + +@dataclass(frozen=True) +class KnownDivergence(Divergence): + accepts: Accepts + """Whether an outcome of the witness's last operation is this store's known behavior.""" + + +def _raw_string_prefix(observed: Outcome, state: State, args: tuple[Any, ...]) -> bool: + (prefix,) = args + return observed == ("ok", sorted(key for key in state.model if key.startswith(prefix))) + + +def _keys_under_or_named(model: Model, prefix: str) -> list[str]: + """Directory semantics, except that a key named by the prefix itself counts as inside it.""" + named = [_directory(prefix)] if _directory(prefix) in model else [] + return sorted(_keys_under(model, prefix) + named) + + +def _list_prefix_with_named_key(observed: Outcome, state: State, args: tuple[Any, ...]) -> bool: + (prefix,) = args + return observed == ("ok", _keys_under_or_named(state.model, prefix)) + + +def _getsize_prefix_with_named_key(observed: Outcome, state: State, args: tuple[Any, ...]) -> bool: + (prefix,) = args + keys = _keys_under_or_named(state.model, prefix) + return observed == ("ok", sum(len(state.model[key]) for key in keys)) + + +def _is_empty_with_named_key(observed: Outcome, state: State, args: tuple[Any, ...]) -> bool: + (prefix,) = args + return observed == ("ok", not _keys_under_or_named(state.model, prefix)) + + +def _lists_a_key_as_its_own_child(observed: Outcome, state: State, args: tuple[Any, ...]) -> bool: + (prefix,) = args + name = _directory(prefix) + return name in state.model and observed == ("ok", [name.rsplit("/", 1)[-1]]) + + +def _lists_empty_directories(observed: Outcome, state: State, args: tuple[Any, ...]) -> bool: + """Every extra child is a directory that held keys before deletes emptied it.""" + (prefix,) = args + directory = _directory(prefix) + match observed: + case ("ok", list(names)): + extra = set(names) - set(_children(state.model, prefix)) + paths = {f"{directory}/{name}" if directory else name for name in extra} + return ( + bool(extra) + and set(_children(state.model, prefix)) <= set(names) + and (paths <= state.past_directories) + ) + return False + + +def _is_a_directory(state: State, name: str) -> bool: + return name not in state.model and name in state.past_directories + + +def _exists_for_a_directory(observed: Outcome, state: State, args: tuple[Any, ...]) -> bool: + (name,) = args + return observed == ("ok", True) and _is_a_directory(state, name) + + +def _size_for_a_directory(observed: Outcome, state: State, args: tuple[Any, ...]) -> bool: + (name,) = args + return observed[0] == "ok" and isinstance(observed[1], int) and _is_a_directory(state, name) + + +def _is_below_a_key(state: State, name: str) -> bool: + return any(_is_under(name, key) for key in state.model) + + +def _raises_below_a_key(error: str) -> Accepts: + return partial(_raised_below_a_key, error) + + +def _raised_below_a_key(error: str, observed: Outcome, state: State, args: tuple[Any, ...]) -> bool: + name = args[0] + return observed == ("raised", error) and _is_below_a_key(state, name) + + +def _raises_past_end(observed: Outcome, state: State, args: tuple[Any, ...]) -> bool: + """A byte range that starts at or after the end of the value raises.""" + key, byte_range = args + if key not in state.model: + return False + match byte_range: + case RangeByteRequest(start, _) | OffsetByteRequest(start): + return observed == ("raised", "GenericError") and start >= len(state.model[key]) + return False + + +def _rejects_empty_range(observed: Outcome, state: State, args: tuple[Any, ...]) -> bool: + """A zero-length `RangeByteRequest` raises, even for a missing key.""" + _, byte_range = args + match byte_range: + case RangeByteRequest(start, end): + return observed == ("raised", "ValueError") and start == end + return False + + +def _below_a_key(stores: frozenset[str], error: str, operation: str) -> KnownDivergence: + """`operation` on `k/x`, where `k` is a key, raises `error`.""" + args: tuple[Any, ...] = ("k/x", None) if operation == "get" else ("k/x",) + return KnownDivergence( + stores=stores, + summary=f"{operation}() on a name below a key raises {error}", + accepts=_raises_below_a_key(error), + witness=(("set", ("k", b"v")), (operation, args)), + ) + + +def _raises_for_a_directory(observed: Outcome, state: State, args: tuple[Any, ...]) -> bool: + name = args[0] + return observed == ("raised", "GenericError") and _is_a_directory(state, name) + + +def _posix_only(stores: frozenset[str]) -> frozenset[str]: + """Stores whose divergence comes from POSIX filesystem errors, which Windows doesn't raise.""" + return frozenset() if WINDOWS else stores + + +def _windows_only(stores: frozenset[str]) -> frozenset[str]: + return stores if WINDOWS else frozenset() + + +FSSPEC = frozenset({"fsspec-local", "fsspec-memory"}) +FILESYSTEMS = frozenset({"local", "latency-local", "fsspec-local", "obstore-local"}) + +KNOWN_DIVERGENCES: tuple[KnownDivergence, ...] = ( + KnownDivergence( + stores=frozenset({"memory", "logging-memory", "zip", "managed-memory"}), + summary="matches keys as strings: list_prefix('lat') returns lat_bnds/... (#3773, #4645)", + accepts=_raw_string_prefix, + witness=( + ("set", ("lat/c", b"v")), + ("set", ("lat_bnds/c", b"v")), + ("list_prefix", ("lat",)), + ), + ), + KnownDivergence( + stores=FSSPEC, + summary="a prefix naming a key returns that key", + accepts=_list_prefix_with_named_key, + witness=(("set", ("d/k", b"v")), ("list_prefix", ("d/k",))), + ), + KnownDivergence( + stores=FSSPEC, + summary="a prefix naming a key counts that key", + accepts=_getsize_prefix_with_named_key, + witness=(("set", ("d/k", b"v")), ("getsize_prefix", ("d/k",))), + ), + KnownDivergence( + stores=FSSPEC, + summary="a prefix naming a key is not empty", + accepts=_is_empty_with_named_key, + witness=(("set", ("d/k", b"v")), ("is_empty", ("d/k",))), + ), + KnownDivergence( + stores=FSSPEC, + summary="list_dir() on a top-level key's name returns the key itself", + accepts=_lists_a_key_as_its_own_child, + witness=(("set", ("k", b"v")), ("list_dir", ("k",))), + ), + KnownDivergence( + stores=FILESYSTEMS, + summary="lists directories left empty by deletes (members() then requests them, #4161)", + accepts=_lists_empty_directories, + witness=(("set", ("d/e/k", b"v")), ("delete", ("d/e/k",)), ("list_dir", ("d",))), + ), + KnownDivergence( + stores=_windows_only(frozenset({"obstore-local"})), + summary="exists() on a directory raises GenericError (Windows)", + accepts=_raises_for_a_directory, + witness=(("set", ("d/k", b"v")), ("exists", ("d",))), + ), + KnownDivergence( + stores=_windows_only(frozenset({"obstore-local"})), + summary="getsize() on a directory raises GenericError (Windows)", + accepts=_raises_for_a_directory, + witness=(("set", ("d/k", b"v")), ("getsize", ("d",))), + ), + KnownDivergence( + stores=FSSPEC, + summary="exists() is true for a directory", + accepts=_exists_for_a_directory, + witness=(("set", ("d/k", b"v")), ("exists", ("d",))), + ), + KnownDivergence( + stores=frozenset({"local", "latency-local"} | FSSPEC), + summary="getsize() returns a number for a directory instead of raising", + accepts=_size_for_a_directory, + witness=(("set", ("d/k", b"v")), ("getsize", ("d",))), + ), + KnownDivergence( + stores=frozenset({"obstore-local", "obstore-memory"}), + summary="a byte range starting at or after the end raises GenericError", + accepts=_raises_past_end, + witness=(("set", ("k", b"v")), ("get", ("k", OffsetByteRequest(1)))), + ), + KnownDivergence( + stores=frozenset({"obstore-local", "obstore-memory"}), + summary="a zero-length RangeByteRequest raises ValueError", + accepts=_rejects_empty_range, + witness=(("set", ("k", b"v")), ("get", ("k", RangeByteRequest(0, 0)))), + ), +) +"""Divergences the state machine meets and accepts. Each witness ends with the divergent operation.""" + +KNOWN_DIVERGENCES += tuple( + _below_a_key(_posix_only(frozenset({"obstore-local"})), "GenericError", operation) + for operation in ( + "exists", + "get", + "getsize", + "list_prefix", + "list_dir", + "getsize_prefix", + "is_empty", + "delete", + "delete_dir", + ) +) + ( + _below_a_key( + _posix_only(frozenset({"local", "latency-local", "fsspec-local"})), + "NotADirectoryError", + "getsize", + ), + _below_a_key( + _posix_only(frozenset({"local", "latency-local"})), "NotADirectoryError", "delete" + ), + _below_a_key(_posix_only(frozenset({"fsspec-local"})), "NotADirectoryError", "list_dir"), +) + +UNMODELED_DIVERGENCES: tuple[Divergence, ...] = ( + Divergence( + stores=frozenset({"local", "latency-local"}), + summary="delete_dir() on a key raises ValueError", + witness=(("set", ("a", b"v")), ("delete_dir", ("a",))), + ), + Divergence( + stores=FSSPEC, + summary="delete_dir('a') deletes the key 'a'", + witness=(("set", ("a", b"v")), ("delete_dir", ("a",)), ("list", ())), + ), + Divergence( + stores=FILESYSTEMS, + summary="a key can't be written where deletes left an empty directory of that name", + witness=(("set", ("d/k", b"v")), ("delete", ("d/k",)), ("set", ("d", b"v"))), + ), +) +"""Divergences that would leave stores in different states, so the state machine avoids them.""" + + +def _known_divergence( + store: str, operation: str, observed: Outcome, state: State, args: tuple[Any, ...] +) -> KnownDivergence | None: + """The ledger entry that explains an outcome breaking the contract, if there is one.""" + for divergence in KNOWN_DIVERGENCES: + if ( + store in divergence.stores + and divergence.witness[-1][0] == operation + and divergence.accepts(observed, state, args) + ): + return divergence + return None + + +def _accepted( + store: str, operation: str, observed: Outcome, state: State, args: tuple[Any, ...] +) -> bool: + if observed == OPERATIONS[operation].contract(state.model, *args): + return True + return _known_divergence(store, operation, observed, state, args) is not None + + +# --- strategies --------------------------------------------------------------------------------- + +SEGMENTS = ("a", "b", "la", "lat", "lat_bnds", "zarr.json", "0") +KEYS_BY_DEPTH = tuple( + tuple("/".join(parts) for parts in product(SEGMENTS, repeat=depth)) for depth in (1, 2, 3) +) +VALUES = st.binary(max_size=12) + + +def _conflicts(key: str, model: Model) -> bool: + """Whether `key` would be a directory of an existing key, or have one as a directory.""" + return any(_is_under(other, key) or _is_under(key, other) for other in model if other != key) + + +def _prefix_candidates(model: Model) -> list[str]: + """Directories, keys, names cut short, names below a key, the root, and a missing name.""" + names = _directories(model) | set(model) + cut_short = {name[:-1] for name in names if len(name) > 1} + below_keys = {key + "/x" for key in model} + trailing = {name + "/" for name in names} + return sorted({"", "missing"} | names | cut_short | below_keys | trailing) + + +def _name_candidates(model: Model) -> list[str]: + """The same names, restricted to the key grammar.""" + return [name for name in _prefix_candidates(model) if name and not name.endswith("/")] + + +def _byte_ranges(size: int) -> st.SearchStrategy[ByteRequest | None]: + starts = st.integers(min_value=0, max_value=size + 3) + lengths = st.integers(min_value=0, max_value=size + 3) + return st.one_of( + st.none(), + st.builds(_range_request, starts, lengths), + st.builds(OffsetByteRequest, starts), + st.builds(SuffixByteRequest, st.integers(min_value=1, max_value=size + 3)), + ) + + +def _range_request(start: int, length: int) -> RangeByteRequest: + return RangeByteRequest(start, start + length) + + +MOSTLY = st.sampled_from([True, True, True, False]) +"""Three times out of four.""" + + +# --- the state machine -------------------------------------------------------------------------- + + +class StoreConformanceMachine(RuleBasedStateMachine): + """Apply every operation to every store and compare each result with the contract.""" + + store_names: tuple[str, ...] + supports_deletes: bool + + def __init__(self) -> None: + super().__init__() + self.tmp = Path(tempfile.mkdtemp()) + self.stores: list[StoreUnderTest] = [] + self.state = State() + + @initialize(data=st.data()) + def open_stores(self, data: st.DataObject) -> None: + self.stores = [_build(name, self.tmp) for name in self.store_names] + # Start from a populated tree, so directory operations meet siblings and nesting early. + self._write_some(data, st.integers(min_value=2, max_value=8)) + + def teardown(self) -> None: + for target in self.stores: + target.cleanup() + shutil.rmtree(self.tmp, ignore_errors=True) + + @property + def model(self) -> Model: + return self.state.model + + def _check(self, operation: str, *args: Any) -> None: + note(f"{operation}{args!r}") + failures = [] + for target in self.stores: + observed = _run(operation, target.store, args) + expected = OPERATIONS[operation].contract(self.model, *args) + if observed == expected: + continue + divergence = _known_divergence(target.name, operation, observed, self.state, args) + if divergence is None: + failures.append(f" {target.name}: {observed!r}, expected {expected!r}") + else: + event(f"known divergence: {target.name}: {divergence.summary}") + assert not failures, f"{operation}{args!r} breaks the contract:\n" + "\n".join(failures) + self.state.record(operation, args) + + def _draw_new_key(self, data: st.DataObject) -> str | None: + """A key that doesn't clash with the tree; depth first, so shallow keys are common.""" + by_depth = [ + [ + key + for key in keys + if not _conflicts(key, self.model) + and key not in self.state.past_directories + and (self.supports_deletes or key not in self.model) + ] + for keys in KEYS_BY_DEPTH + ] + depths = [candidates for candidates in by_depth if candidates] + if not depths: + return None + candidates = data.draw(st.sampled_from(depths), label="depth") + return data.draw(st.sampled_from(candidates), label="key") + + def _write_some(self, data: st.DataObject, count: st.SearchStrategy[int]) -> None: + for _ in range(data.draw(count, label="count")): + key = self._draw_new_key(data) + if key is None: + return + self._check("set", key, data.draw(VALUES, label="value")) + + # writes + + @rule(data=st.data()) + def set(self, data: st.DataObject) -> None: + self._write_some(data, st.integers(min_value=1, max_value=3)) + + @rule(data=st.data(), value=VALUES) + def set_if_not_exists(self, data: st.DataObject, value: bytes) -> None: + existing = data.draw(st.booleans(), label="existing") if self.model else False + if existing: + key: str | None = data.draw(st.sampled_from(sorted(self.model)), label="key") + else: + key = self._draw_new_key(data) + if key is not None: + self._check("set_if_not_exists", key, value) + + @precondition(lambda self: self.supports_deletes) + @rule(data=st.data()) + def delete(self, data: st.DataObject) -> None: + candidates = [*sorted(self.model), "missing"] + self._check("delete", data.draw(st.sampled_from(candidates), label="key")) + + @precondition(lambda self: self.supports_deletes) + @rule(data=st.data()) + def delete_dir(self, data: st.DataObject) -> None: + candidates = [ + prefix + for prefix in _prefix_candidates(self.model) + if prefix and _directory(prefix) not in self.model + ] + self._check("delete_dir", data.draw(st.sampled_from(candidates), label="prefix")) + + @precondition(lambda self: self.supports_deletes and len(self.model) >= 6) + @rule() + def delete_everything(self) -> None: + self._check("delete_dir", "") + + # reads + + @rule( + data=st.data(), + operation=st.sampled_from(["list_prefix", "list_dir", "getsize_prefix", "is_empty"]), + ) + def read_directory(self, data: st.DataObject, operation: str) -> None: + prefix = data.draw(st.sampled_from(_prefix_candidates(self.model)), label="prefix") + self._check(operation, prefix) + + @rule(data=st.data(), operation=st.sampled_from(["exists", "getsize"])) + def read_name(self, data: st.DataObject, operation: str) -> None: + name = data.draw(st.sampled_from(_name_candidates(self.model)), label="name") + self._check(operation, name) + + @rule(data=st.data()) + def get(self, data: st.DataObject) -> None: + existing = data.draw(MOSTLY, label="existing") if self.model else False + if existing: + key = data.draw(st.sampled_from(sorted(self.model)), label="key") + else: + key = data.draw(st.sampled_from(_name_candidates(self.model)), label="key") + size = len(self.model.get(key, b"")) + self._check("get", key, data.draw(_byte_ranges(size), label="byte_range")) + + @invariant() + def keys_agree(self) -> None: + if self.stores: + self._check("list") + + +class WritableStoresMachine(StoreConformanceMachine): + store_names = WRITABLE_STORES + supports_deletes = True + + +class AppendOnlyStoresMachine(StoreConformanceMachine): + store_names = APPEND_ONLY_STORES + supports_deletes = False + + +# --- tests -------------------------------------------------------------------------------------- + +QUICK = settings(max_examples=25, stateful_step_count=25) + + +def test_writable_stores_agree_quick() -> None: + run_state_machine_as_test(WritableStoresMachine, settings=QUICK) + + +def test_append_only_stores_agree_quick() -> None: + run_state_machine_as_test(AppendOnlyStoresMachine, settings=QUICK) + + +@pytest.mark.slow_hypothesis +def test_writable_stores_agree() -> None: + run_state_machine_as_test(WritableStoresMachine) + + +@pytest.mark.slow_hypothesis +def test_append_only_stores_agree() -> None: + run_state_machine_as_test(AppendOnlyStoresMachine) + + +def _explained(store: str, steps: tuple[Step, ...], observed: Outcome, state: State) -> bool: + """Whether the outcome of the last of `steps` is a ledger entry for this store. + + An unmodeled divergence explains it only when `steps` end with that divergence's whole witness. + """ + operation, args = steps[-1] + if _accepted(store, operation, observed, state, args): + return True + return any( + store in divergence.stores and steps[-len(divergence.witness) :] == divergence.witness + for divergence in UNMODELED_DIVERGENCES + ) + + +ALL_DIVERGENCES: tuple[Divergence, ...] = (*KNOWN_DIVERGENCES, *UNMODELED_DIVERGENCES) + + +@pytest.mark.parametrize("store_name", list(STORE_FACTORIES)) +@pytest.mark.parametrize( + "divergence", [pytest.param(d, id=d.summary[:48]) for d in ALL_DIVERGENCES] +) +def test_divergence_ledger_is_exact( + store_name: str, divergence: Divergence, tmp_path: Path +) -> None: + """A witness breaks the contract on exactly the stores its entry lists. + + A listed store that now follows the contract has been fixed: remove it from the entry. An unlisted + store that breaks the contract has a divergence the ledger doesn't record: add it. + """ + if store_name in APPEND_ONLY_STORES and any( + operation.startswith("delete") for operation, _ in divergence.witness + ): + pytest.skip("the store can't delete") + *setup, (operation, args) = divergence.witness + state = State() + target = _build(store_name, tmp_path) + try: + for index, (step, step_args) in enumerate(setup): + observed = _run(step, target.store, step_args) + if not _explained(store_name, divergence.witness[: index + 1], observed, state): + pytest.fail(f"setup step {step}{step_args!r} breaks the contract: {observed!r}") + state.record(step, step_args) + observed = _run(operation, target.store, args) + finally: + target.cleanup() + diverges = observed != OPERATIONS[operation].contract(state.model, *args) + if store_name in divergence.stores: + assert diverges, f"{store_name} now follows the contract; remove it from this entry" + if isinstance(divergence, KnownDivergence): + assert divergence.accepts(observed, state, args), observed + else: + assert not diverges or _explained(store_name, divergence.witness, observed, state), ( + f"{store_name} breaks the contract here too ({observed!r}); add it to this entry" + ) + + +ROOT_ESCAPES = { + ("local", "relative"), + ("local", "absolute"), + ("latency-local", "relative"), + ("latency-local", "absolute"), + ("fsspec-local", "relative"), +} +"""Stores whose `delete_dir` reaches a directory outside their root, by path form.""" + + +@pytest.mark.parametrize( + ("store_name", "form"), + [ + pytest.param( + store_name, + form, + marks=[pytest.mark.xfail(strict=True, raises=AssertionError)] + if (store_name, form) in ROOT_ESCAPES + else [], + ) + for store_name in sorted(FILESYSTEMS) + for form in ("relative", "absolute") + ], +) +def test_delete_stays_in_root(store_name: str, form: str, tmp_path: Path) -> None: + """`delete_dir` must not reach a directory outside the store's root. + + The target is a sentinel directory beside the store, inside this test's temporary directory, so an + escape can only delete the sentinel. + """ + sentinel = tmp_path / "outside" + sentinel.mkdir() + (sentinel / "keep").write_bytes(b"x") + (tmp_path / "stores").mkdir() + target = _build(store_name, tmp_path / "stores") + path = "../../outside" if form == "relative" else str(sentinel) + try: + _run("delete_dir", target.store, (path,)) + finally: + target.cleanup() + assert (sentinel / "keep").exists() diff --git a/tests/test_store/test_wrapper.py b/tests/test_store/test_wrapper.py index e556a108c5..49e5a3ee47 100644 --- a/tests/test_store/test_wrapper.py +++ b/tests/test_store/test_wrapper.py @@ -1,18 +1,24 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, TypedDict +import inspect +from dataclasses import dataclass, field +from types import FunctionType +from typing import TYPE_CHECKING, Any, Literal, Self, TypedDict, cast import pytest -from zarr.abc.store import ByteRequest, Store, _store_supports_sync_io +from zarr.abc.store import ByteRequest, RangeByteRequest, Store, _store_supports_sync_io from zarr.core.buffer import Buffer from zarr.core.buffer.cpu import Buffer as CPUBuffer from zarr.core.buffer.cpu import buffer_prototype -from zarr.storage import LocalStore, MemoryStore, WrapperStore, ZipStore +from zarr.experimental.cache_store import CacheStore +from zarr.storage import LocalStore, LoggingStore, MemoryStore, WrapperStore, ZipStore from zarr.testing.store import LatencyStore, StoreTests if TYPE_CHECKING: + from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable, Sequence from pathlib import Path + from types import TracebackType from zarr.core.buffer.core import BufferPrototype @@ -167,3 +173,381 @@ def test_wrapper_delete_sync_without_inner_sync_raises(tmp_path: Any) -> None: store = WrapperStore(ZipStore(tmp_path / "store.zip", mode="w")) with pytest.raises(TypeError, match="does not support synchronous delete"): store.delete_sync("key") + + +# --- Every wrapper forwards every Store method ------------------------------- +# +# `RecordingStore` overrides every method of `Store` (and the sync protocols) to +# record its arguments and return a distinct value. Wrapping it and calling a +# method on the wrapper must produce exactly one call of the same method, with +# the same arguments, on the wrapped store, and must return what that call +# returned. A method the wrapper doesn't forward falls back to a `Store` +# default, which reaches the wrapped store through some other method (e.g. +# `getsize` through `get`), and the check fails. + + +@dataclass(frozen=True) +class Call: + name: str + arguments: dict[str, Any] + + +def _make_result(name: str) -> Any: + """A value the wrapped store returns from `name`, distinct from every other.""" + member = vars(RecordingStore)[name] + if isinstance(member, property): + return object() + if inspect.signature(member).return_annotation == "None": + return None + if name in {"get", "get_sync"}: + # `CacheStore` caches what `get` returns, so it must be a Buffer. + return CPUBuffer.from_bytes(name.encode()) + if name in {"with_read_only", "__enter__"}: + # Wrappers re-wrap what these return, so it must be a Store. + return MemoryStore() + return object() + + +class RecordingStore(Store): + """A store that records each call and returns a distinct value for each method.""" + + def __init__(self) -> None: + super().__init__() + self.calls: list[Call] = [] + self.results: dict[str, Any] = {} + + def _record(self, name: str, arguments: dict[str, Any]) -> Any: + self.calls.append(Call(name, {k: v for k, v in arguments.items() if k != "self"})) + return self.results.setdefault(name, _make_result(name)) + + def with_read_only(self, read_only: bool = False) -> Store: + return cast(Store, self._record("with_read_only", locals())) + + def __enter__(self) -> Self: + return cast(Self, self._record("__enter__", locals())) + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + return cast(None, self._record("__exit__", locals())) + + async def _open(self) -> None: + return cast(None, self._record("_open", locals())) + + async def _ensure_open(self) -> None: + return cast(None, self._record("_ensure_open", locals())) + + async def is_empty(self, prefix: str) -> bool: + return cast(bool, self._record("is_empty", locals())) + + async def clear(self) -> None: + return cast(None, self._record("clear", locals())) + + @property + def read_only(self) -> bool: + return cast(bool, self._record("read_only", {})) + + def _check_writable(self) -> None: + return cast(None, self._record("_check_writable", locals())) + + def __eq__(self, value: object) -> bool: + return self is value + + async def get( + self, key: str, prototype: BufferPrototype, byte_range: ByteRequest | None = None + ) -> Buffer | None: + return cast(Buffer, self._record("get", locals())) + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: Iterable[tuple[str, ByteRequest | None]], + ) -> list[Buffer | None]: + return cast(list[Buffer | None], self._record("get_partial_values", locals())) + + async def exists(self, key: str) -> bool: + return cast(bool, self._record("exists", locals())) + + @property + def supports_writes(self) -> bool: + return cast(bool, self._record("supports_writes", {})) + + async def set(self, key: str, value: Buffer) -> None: + return cast(None, self._record("set", locals())) + + async def set_if_not_exists(self, key: str, value: Buffer) -> None: + return cast(None, self._record("set_if_not_exists", locals())) + + async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None: + return cast(None, self._record("_set_many", locals())) + + @property + def supports_consolidated_metadata(self) -> bool: + return cast(bool, self._record("supports_consolidated_metadata", {})) + + @property + def supports_deletes(self) -> bool: + return cast(bool, self._record("supports_deletes", {})) + + async def delete(self, key: str) -> None: + return cast(None, self._record("delete", locals())) + + @property + def supports_listing(self) -> bool: + return cast(bool, self._record("supports_listing", {})) + + async def list(self) -> AsyncIterator[str]: + yield self._record("list", locals()) + + async def list_prefix(self, prefix: str) -> AsyncIterator[str]: + yield self._record("list_prefix", locals()) + + async def list_dir(self, prefix: str) -> AsyncIterator[str]: + yield self._record("list_dir", locals()) + + async def delete_dir(self, prefix: str) -> None: + return cast(None, self._record("delete_dir", locals())) + + def close(self) -> None: + return cast(None, self._record("close", locals())) + + async def _get_many( + self, requests: Iterable[tuple[str, BufferPrototype, ByteRequest | None]] + ) -> AsyncGenerator[tuple[str, Buffer | None], None]: + yield self._record("_get_many", locals()) + + async def get_ranges( + self, + key: str, + byte_ranges: Sequence[ByteRequest | None], + *, + prototype: BufferPrototype, + max_concurrency: int = 10, + max_gap_bytes: int = 1 << 20, + max_coalesced_bytes: int = 16 << 20, + ) -> AsyncIterator[Sequence[tuple[int, Buffer | None]]]: + yield self._record("get_ranges", locals()) + + def get_ranges_sync( + self, + key: str, + byte_ranges: Sequence[ByteRequest | None], + *, + prototype: BufferPrototype, + max_gap_bytes: int = 1 << 20, + max_coalesced_bytes: int = 16 << 20, + ) -> Sequence[tuple[int, Buffer | None]]: + return cast( + "Sequence[tuple[int, Buffer | None]]", self._record("get_ranges_sync", locals()) + ) + + async def getsize(self, key: str) -> int: + return cast(int, self._record("getsize", locals())) + + async def getsize_prefix(self, prefix: str) -> int: + return cast(int, self._record("getsize_prefix", locals())) + + def get_sync( + self, + key: str, + *, + prototype: BufferPrototype | None = None, + byte_range: ByteRequest | None = None, + ) -> Buffer | None: + return cast(Buffer, self._record("get_sync", locals())) + + def set_sync(self, key: str, value: Buffer) -> None: + return cast(None, self._record("set_sync", locals())) + + def delete_sync(self, key: str) -> None: + return cast(None, self._record("delete_sync", locals())) + + +@dataclass(frozen=True) +class Recipe: + """How to call a method on a wrapper, and what kind of value it gives back.""" + + kind: Literal["property", "sync", "async", "aiter", "rewrap"] + args: tuple[Any, ...] = () + kwargs: dict[str, Any] = field(default_factory=dict) + + +VALUE = CPUBuffer.from_bytes(b"value") +RANGES = [RangeByteRequest(0, 1), RangeByteRequest(4, 6)] +# Coalescing settings that differ from every default, so a wrapper that drops +# them is caught. +COALESCING = {"max_gap_bytes": 5, "max_coalesced_bytes": 7} + +RECIPES: dict[str, Recipe] = { + "with_read_only": Recipe("rewrap", (True,)), + "__enter__": Recipe("rewrap"), + "__exit__": Recipe("sync", (None, None, None)), + "_open": Recipe("async"), + "_ensure_open": Recipe("async"), + "is_empty": Recipe("async", ("a/b",)), + "clear": Recipe("async"), + "read_only": Recipe("property"), + "_check_writable": Recipe("sync"), + "get": Recipe("async", ("a/b", buffer_prototype, RangeByteRequest(1, 3))), + "get_partial_values": Recipe("async", (buffer_prototype, [("a/b", RangeByteRequest(1, 3))])), + "exists": Recipe("async", ("a/b",)), + "supports_writes": Recipe("property"), + "set": Recipe("async", ("a/b", VALUE)), + "set_if_not_exists": Recipe("async", ("a/b", VALUE)), + "_set_many": Recipe("async", ([("a/b", VALUE)],)), + "supports_consolidated_metadata": Recipe("property"), + "supports_deletes": Recipe("property"), + "delete": Recipe("async", ("a/b",)), + "supports_listing": Recipe("property"), + "list": Recipe("aiter"), + "list_prefix": Recipe("aiter", ("a/",)), + "list_dir": Recipe("aiter", ("a/",)), + "delete_dir": Recipe("async", ("a",)), + "close": Recipe("sync"), + "_get_many": Recipe("aiter", ([("a/b", buffer_prototype, None)],)), + "get_ranges": Recipe( + "aiter", + ("a/b", RANGES), + {"prototype": buffer_prototype, "max_concurrency": 3, **COALESCING}, + ), + "get_ranges_sync": Recipe( + "sync", ("a/b", RANGES), {"prototype": buffer_prototype, **COALESCING} + ), + "getsize": Recipe("async", ("a/b",)), + "getsize_prefix": Recipe("async", ("a/",)), + "get_sync": Recipe( + "sync", ("a/b",), {"prototype": buffer_prototype, "byte_range": RangeByteRequest(1, 3)} + ), + "set_sync": Recipe("sync", ("a/b", VALUE)), + "delete_sync": Recipe("sync", ("a/b",)), +} + +# Every argument with a default is given a different value above, so a wrapper that drops an +# argument (and lets the wrapped store's default apply) is caught. + +# Members of `Store` that are not a call a wrapper passes through. +NOT_FORWARDED: dict[str, str] = { + "__init__": "constructs the wrapper", + "open": "constructs the wrapper", + "__eq__": "compares the wrapped stores of two wrappers, so the argument changes", + "supports_partial_writes": "always False, on every store", +} + +# Methods a wrapper deliberately implements itself instead of forwarding. Each +# entry must hold: the test fails if the wrapper starts forwarding the method. +REROUTED: dict[tuple[str, str], str] = { + ("latency", "get_partial_values"): "reads through the wrapper's own get, to add latency", + ("latency", "get_ranges"): "reads through the wrapper's own get, to add latency", + ("latency", "get_ranges_sync"): "reads through the wrapper's own get_sync, to add latency", + ("cache", "__enter__"): "CacheStore can't re-wrap a store, so it raises NotImplementedError", +} + +WRAPPERS: dict[str, Callable[[Store], WrapperStore[Any]]] = { + "wrapper": WrapperStore, + "logging": LoggingStore, + "latency": LatencyStore, + "cache": lambda store: CacheStore(store, cache_store=MemoryStore()), +} + +# Every member of the `Store` interface, including the sync protocol a wrapper +# presents. +STORE_MEMBERS = { + name + for name, member in vars(Store).items() + if isinstance(member, FunctionType | property | classmethod) +} | {"get_sync", "set_sync", "delete_sync"} + + +def _arguments(name: str, args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]: + """The arguments a call binds to, with defaults filled in.""" + bound = inspect.signature(getattr(RecordingStore, name)).bind(None, *args, **kwargs) + bound.apply_defaults() + return {k: v for k, v in bound.arguments.items() if k != "self"} + + +async def _call(store: Store, name: str) -> Any: + """Call `name` on `store` as its recipe says.""" + recipe = RECIPES[name] + if recipe.kind == "property": + return getattr(store, name) + if recipe.kind == "aiter": + return [item async for item in getattr(store, name)(*recipe.args, **recipe.kwargs)] + if recipe.kind == "async": + return await getattr(store, name)(*recipe.args, **recipe.kwargs) + return getattr(store, name)(*recipe.args, **recipe.kwargs) + + +async def _forwarded(wrapper_name: str, name: str) -> tuple[bool, list[Call]]: + """Whether the wrapper passes one call of `name` straight to the store it + wraps, and the calls the wrapped store saw.""" + inner = RecordingStore() + wrapper = WRAPPERS[wrapper_name](inner) + recipe = RECIPES[name] + try: + result = await _call(wrapper, name) + except NotImplementedError: + return False, inner.calls + expected = inner.results.get(name) + if recipe.kind == "aiter": + returned = len(result) == 1 and result[0] is expected + elif recipe.kind == "rewrap": + returned = type(result) is type(wrapper) and result._store is expected + else: + returned = result is expected + arguments = {} if recipe.kind == "property" else _arguments(name, recipe.args, recipe.kwargs) + return returned and inner.calls == [Call(name, arguments)], inner.calls + + +def test_every_store_member_is_checked() -> None: + """Each `Store` member has a recipe or a reason it isn't forwarded, and + `RecordingStore` records every member with a recipe.""" + assert set(RECIPES) | set(NOT_FORWARDED) == STORE_MEMBERS + assert set(RECIPES) <= set(vars(RecordingStore)) + + +@pytest.mark.parametrize( + "name", sorted(name for name, r in RECIPES.items() if r.kind != "property") +) +def test_recipes_override_every_default(name: str) -> None: + """A recipe gives each argument that has a default some other value, so + forwarding that drops the argument changes what the wrapped store sees.""" + recipe = RECIPES[name] + signature = inspect.signature(getattr(RecordingStore, name)) + given = signature.bind(None, *recipe.args, **recipe.kwargs).arguments + defaults = { + parameter.name: parameter.default + for parameter in signature.parameters.values() + if parameter.default is not inspect.Parameter.empty + } + assert {key for key, default in defaults.items() if given.get(key, default) == default} == set() + + +@pytest.mark.parametrize("name", sorted(RECIPES)) +@pytest.mark.parametrize("wrapper_name", sorted(WRAPPERS)) +async def test_wrappers_forward_every_store_method(wrapper_name: str, name: str) -> None: + """A wrapper forwards each `Store` method to the store it wraps, with the + same arguments, unless `REROUTED` says why it doesn't.""" + forwarded, calls = await _forwarded(wrapper_name, name) + expected = (wrapper_name, name) not in REROUTED + assert forwarded is expected, f"{wrapper_name}.{name}: the wrapped store saw {calls}" + + +NOT_LOGGED: dict[str, str] = { + "with_read_only": "returns a new LoggingStore, which logs its own calls", + "__enter__": "returns a new LoggingStore, which logs its own calls", + "__exit__": "doesn't read or write the wrapped store", + "close": "doesn't read or write the wrapped store", + "_check_writable": "doesn't read or write the wrapped store", +} + + +@pytest.mark.parametrize("name", sorted(RECIPES)) +async def test_logging_store_logs_every_store_method(name: str) -> None: + """`LoggingStore` logs each `Store` method it forwards, unless + `NOT_LOGGED` says why it doesn't.""" + store = LoggingStore(RecordingStore(), log_level="WARNING") + await _call(store, name) + assert (name in store.counter) is (name not in NOT_LOGGED), dict(store.counter)