diff --git a/changes/459.bugfix.md b/changes/459.bugfix.md new file mode 100644 index 0000000000..9ce24b339a --- /dev/null +++ b/changes/459.bugfix.md @@ -0,0 +1 @@ +Fix `from_array` overwriting its source array when the destination store is rooted inside the source's directory: overlapping store paths are now detected across stores, not only within one store. diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index af528ec533..9f09080752 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Literal, Protocol, runtime_checkable if TYPE_CHECKING: - from collections.abc import AsyncGenerator, AsyncIterator, Iterable, Sequence + from collections.abc import AsyncGenerator, AsyncIterator, Hashable, Iterable, Sequence from types import TracebackType from typing import Any, Self @@ -192,6 +192,15 @@ def __eq__(self, value: object) -> bool: """Equality comparison.""" ... + def _data_location(self) -> tuple[Hashable, str] | None: + """The data this store reads and writes, as `(backend, root)`, or `None` if unknown. + + Two stores with equal `backend` values see the same keys, and `root` is the + `/`-separated path of this store's keys within that backend. Stores with + different roots in one backend can overlap, which `__eq__` can't detect. + """ + return None + @abstractmethod async def get( self, diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 3735a4da18..3df58edf78 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -157,8 +157,7 @@ get_pipeline_class, ) from zarr.storage._common import StorePath, make_store_path, resolve_zarr_format -from zarr.storage._memory import MemoryStore -from zarr.storage._utils import _relativize_path +from zarr.storage._utils import _join_paths, _relativize_path if TYPE_CHECKING: from collections.abc import Iterator @@ -4166,19 +4165,30 @@ class ShardsConfigParam(TypedDict): type ShardsLike = ChunksLike | ShardsConfigParam | Literal["auto"] -def _stores_share_data(a: Store, b: Store) -> bool: - """Whether two stores are views onto the same underlying data. +def _store_paths_overlap(a: StorePath, b: StorePath) -> bool: + """Whether the keys under two store paths can overlap. - ``Store.__eq__`` can't answer this directly. Several stores include ``read_only`` in - their equality check, so a read-only and a writable view of the same data are - unequal. ``MemoryStore`` compares its dicts by value, so two distinct stores with - equal contents are equal. + Stores that report a data location are compared by backend and by their root joined + with the path, so two stores rooted at nested directories of one backend overlap. + For a store that can't report a location, this falls back to comparing the stores, + ignoring ``read_only``, and then the paths alone. """ - if type(a) is MemoryStore and type(b) is MemoryStore: - return a._store_dict is b._store_dict - with contextlib.suppress(NotImplementedError): - a = a.with_read_only(b.read_only) - return a == b + location_a, location_b = a.store._data_location(), b.store._data_location() + if location_a is not None and location_b is not None: + (backend_a, root_a), (backend_b, root_b) = location_a, location_b + if backend_a != backend_b: + return False + path_a, path_b = _join_paths([root_a, a.path]), _join_paths([root_b, b.path]) + else: + store_a = a.store + with contextlib.suppress(NotImplementedError): + store_a = store_a.with_read_only(b.store.read_only) + if store_a != b.store: + return False + path_a, path_b = a.path, b.path + prefix_a = f"{path_a}/" if path_a else "" + prefix_b = f"{path_b}/" if path_b else "" + return prefix_a.startswith(prefix_b) or prefix_b.startswith(prefix_a) async def from_array( @@ -4423,16 +4433,12 @@ async def from_array( write_data and overwrite and isinstance(data, Array) - and _stores_share_data(data.store_path.store, store_path.store) + and _store_paths_overlap(data.store_path, store_path) ): - src_prefix = f"{data.store_path.path}/" if data.store_path.path else "" - dest_prefix = f"{store_path.path}/" if store_path.path else "" - if src_prefix.startswith(dest_prefix) or dest_prefix.startswith(src_prefix): - raise ValueError( - f"Cannot overwrite {store_path.path!r} with the array at " - f"{data.store_path.path!r} because the paths overlap. Write to a different " - "path, or pass data=array[...]." - ) + raise ValueError( + f"Cannot overwrite {store_path} with the array at {data.store_path} because " + "the paths overlap. Write to a different path, or pass data=array[...]." + ) result = await init_array( store_path=store_path, diff --git a/src/zarr/storage/_fsspec.py b/src/zarr/storage/_fsspec.py index 137043c901..a46f4b5be5 100644 --- a/src/zarr/storage/_fsspec.py +++ b/src/zarr/storage/_fsspec.py @@ -19,7 +19,7 @@ from zarr.storage._utils import _dereference_path if TYPE_CHECKING: - from collections.abc import AsyncIterator, Iterable + from collections.abc import AsyncIterator, Hashable, Iterable from fsspec import AbstractFileSystem from fsspec.asyn import AsyncFileSystem @@ -286,6 +286,12 @@ def __eq__(self, other: object) -> bool: and self.fs == other.fs ) + def _data_location(self) -> tuple[Hashable, str]: + # docstring inherited + # fsspec caches filesystem instances by this token, so two stores built with the + # same filesystem arguments share a backend. + return ("fsspec", self.fs._fs_token), self.path.rstrip("/") + async def get( self, key: str, diff --git a/src/zarr/storage/_local.py b/src/zarr/storage/_local.py index 20d4b102b1..a504f59548 100644 --- a/src/zarr/storage/_local.py +++ b/src/zarr/storage/_local.py @@ -23,7 +23,7 @@ from zarr.core.common import AccessModeLiteral, concurrent_map if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Iterable, Iterator + from collections.abc import AsyncIterator, Callable, Hashable, Iterable, Iterator from zarr.core.buffer import BufferPrototype @@ -300,6 +300,10 @@ def __repr__(self) -> str: def __eq__(self, other: object) -> bool: return isinstance(other, type(self)) and self.root == other.root + def _data_location(self) -> tuple[Hashable, str]: + # docstring inherited + return ("file",), self.root.resolve().as_posix() + # ------------------------------------------------------------------- # Synchronous store methods # ------------------------------------------------------------------- diff --git a/src/zarr/storage/_memory.py b/src/zarr/storage/_memory.py index 3bece0e841..477aee5a7d 100644 --- a/src/zarr/storage/_memory.py +++ b/src/zarr/storage/_memory.py @@ -18,7 +18,7 @@ ) if TYPE_CHECKING: - from collections.abc import AsyncIterator, Iterable, MutableMapping + from collections.abc import AsyncIterator, Hashable, Iterable, MutableMapping from zarr.core.buffer import BufferPrototype @@ -103,6 +103,10 @@ def __eq__(self, other: object) -> bool: and self.read_only == other.read_only ) + def _data_location(self) -> tuple[Hashable, str]: + # docstring inherited + return ("memory", id(self._store_dict)), "" + # ------------------------------------------------------------------- # Synchronous store methods # ------------------------------------------------------------------- @@ -508,6 +512,10 @@ def __eq__(self, other: object) -> bool: and self.read_only == other.read_only ) + def _data_location(self) -> tuple[Hashable, str]: + # docstring inherited + return ("memory", id(self._store_dict)), self.path + @property def name(self) -> str: """The name of this store, used in the memory:// URL.""" diff --git a/src/zarr/storage/_wrapper.py b/src/zarr/storage/_wrapper.py index 6f498a655d..4824b777dc 100644 --- a/src/zarr/storage/_wrapper.py +++ b/src/zarr/storage/_wrapper.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, cast if TYPE_CHECKING: - from collections.abc import AsyncGenerator, AsyncIterator, Iterable, Sequence + from collections.abc import AsyncGenerator, AsyncIterator, Hashable, Iterable, Sequence from types import TracebackType from typing import Any, Self @@ -91,6 +91,10 @@ def _check_writable(self) -> None: def __eq__(self, value: object) -> bool: return type(self) is type(value) and self._store.__eq__(value._store) + def _data_location(self) -> tuple[Hashable, str] | None: + # docstring inherited + return self._store._data_location() + def __str__(self) -> str: return f"wrapping-{self._store}" diff --git a/src/zarr/storage/_zip.py b/src/zarr/storage/_zip.py index e29d2ce496..67c7c3409b 100644 --- a/src/zarr/storage/_zip.py +++ b/src/zarr/storage/_zip.py @@ -19,7 +19,7 @@ from zarr.core.buffer import Buffer, BufferPrototype if TYPE_CHECKING: - from collections.abc import AsyncIterator, Iterable + from collections.abc import AsyncIterator, Hashable, Iterable ZipStoreAccessModeLiteral = Literal["r", "w", "a", "x"] @@ -336,6 +336,12 @@ def __eq__(self, other: object) -> bool: and self._fileobj is other._fileobj ) + def _data_location(self) -> tuple[Hashable, str]: + # docstring inherited + if self.path is None: + return ("zip", id(self._fileobj)), "" + return ("zip", self.path.resolve().as_posix()), "" + def _get( self, key: str, diff --git a/tests/conftest.py b/tests/conftest.py index 89eb90a1d8..61deca4fb3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -153,12 +153,14 @@ async def store(request: pytest.FixtureRequest, tmp_path: pathlib.Path) -> Store @pytest.fixture -async def store2(request: pytest.FixtureRequest, tmp_path: pathlib.Path) -> Store: - """Fixture to create a second store for testing copy operations between stores""" +async def store2(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory) -> Store: + """Fixture to create a second store for testing copy operations between stores. + + Its directory is separate from `store`'s, so a copy from one to the other never + writes inside the source array's directory. + """ param = request.param - store2_path = tmp_path / "store2" - store2_path.mkdir() - return await parse_store(param, str(store2_path)) + return await parse_store(param, str(tmp_path_factory.mktemp("store2"))) @pytest.fixture(params=["local", "memory", "zip"]) diff --git a/tests/test_array.py b/tests/test_array.py index 3e4835e686..8d3341a0c2 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -7,6 +7,7 @@ import re import sys from itertools import accumulate, starmap +from pathlib import Path from typing import TYPE_CHECKING, Any, Literal from unittest import mock @@ -43,6 +44,7 @@ _parse_chunk_encoding_v2, _parse_chunk_encoding_v3, _shards_initialized, + _store_paths_overlap, create_array, default_filters_v2, default_serializer_v3, @@ -81,7 +83,14 @@ ContainsGroupError, ZarrUserWarning, ) -from zarr.storage import LocalStore, MemoryStore, StorePath +from zarr.storage import ( + FsspecStore, + LocalStore, + ManagedMemoryStore, + MemoryStore, + StorePath, + ZipStore, +) from zarr.storage._logging import LoggingStore from zarr.types import AnyArray, AnyAsyncArray @@ -2010,6 +2019,85 @@ def test_from_array_overwrite_equal_memory_stores() -> None: np.testing.assert_array_equal(result[...], np.arange(4.0)) +def _store_path_pair(case: str, tmp_path: Path) -> tuple[StorePath, StorePath]: + """Two store paths for `test_store_paths_overlap`, built for one named case.""" + managed_name = f"overlap-{tmp_path.name}" + match case: + case "local-nested-root": + return StorePath(LocalStore(tmp_path), "a"), StorePath(LocalStore(tmp_path / "a")) + case "local-sibling-root": + return StorePath(LocalStore(tmp_path), "a"), StorePath(LocalStore(tmp_path / "ab")) + case "local-read-only": + local_store = LocalStore(tmp_path) + return StorePath(local_store, "a"), StorePath(local_store.with_read_only(True), "a/b") + case "memory-same-dict": + memory_store = MemoryStore() + return StorePath(memory_store, "g/a"), StorePath(memory_store, "g") + case "memory-equal-contents": + return StorePath(MemoryStore(), "a"), StorePath(MemoryStore(), "a") + case "managed-memory-path": + return ( + StorePath(ManagedMemoryStore(name=managed_name), "g/a"), + StorePath(ManagedMemoryStore(name=managed_name, path="g"), "a"), + ) + case "managed-memory-other-path": + return ( + StorePath(ManagedMemoryStore(name=managed_name), "g/a"), + StorePath(ManagedMemoryStore(name=managed_name, path="h")), + ) + case "zip-same-file": + return ( + StorePath(ZipStore(tmp_path / "x.zip", mode="w"), "a"), + StorePath(ZipStore(tmp_path / "x.zip", mode="r")), + ) + case "fsspec-nested-root": + return ( + StorePath(FsspecStore.from_url(f"memory://{tmp_path.name}"), "x/a"), + StorePath(FsspecStore.from_url(f"memory://{tmp_path.name}/x"), "a"), + ) + case "wrapper": + return ( + StorePath(LoggingStore(LocalStore(tmp_path)), "a"), + StorePath(LocalStore(tmp_path / "a")), + ) + case "different-backends": + return StorePath(LocalStore(tmp_path), "a"), StorePath(MemoryStore(), "a") + raise ValueError(case) + + +@pytest.mark.parametrize( + ("case", "expected"), + [ + ("local-nested-root", True), + ("local-sibling-root", False), + ("local-read-only", True), + ("memory-same-dict", True), + ("memory-equal-contents", False), + ("managed-memory-path", True), + ("managed-memory-other-path", False), + ("zip-same-file", True), + ("fsspec-nested-root", True), + ("wrapper", True), + ("different-backends", False), + ], +) +def test_store_paths_overlap(case: str, expected: bool, tmp_path: Path) -> None: + """Two store paths overlap when their stores share a backend and the stores' roots + joined with the paths nest, whatever root each store was given.""" + a, b = _store_path_pair(case, tmp_path) + assert _store_paths_overlap(a, b) is expected + assert _store_paths_overlap(b, a) is expected + + +def test_from_array_overwrite_nested_store_root_raises(tmp_path: Path) -> None: + """A destination store rooted inside the source array's directory overlaps the + source, so the overwrite raises instead of deleting the source's chunks.""" + src = zarr.create_array(LocalStore(tmp_path), name="a", data=np.arange(1.0, 5.0)) + with pytest.raises(ValueError, match="paths overlap"): + zarr.from_array(LocalStore(tmp_path / "a"), data=src, overwrite=True) + np.testing.assert_array_equal(src[...], np.arange(1.0, 5.0)) + + @pytest.mark.parametrize("store", ["memory"], indirect=True) def test_from_array_overwrite_non_overlapping_source(store: Store) -> None: """Overlap checks respect path boundaries, and metadata-only copies are allowed."""