Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/459.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix `from_array` overwriting its source array when the destination store is rooted inside the source's directory: overlapping store paths are now detected across stores, not only within one store.
11 changes: 10 additions & 1 deletion src/zarr/abc/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from typing import TYPE_CHECKING, Literal, Protocol, runtime_checkable

if TYPE_CHECKING:
from collections.abc import AsyncGenerator, AsyncIterator, Iterable, Sequence
from collections.abc import AsyncGenerator, AsyncIterator, Hashable, Iterable, Sequence
from types import TracebackType
from typing import Any, Self

Expand Down Expand Up @@ -192,6 +192,15 @@ def __eq__(self, value: object) -> bool:
"""Equality comparison."""
...

def _data_location(self) -> tuple[Hashable, str] | None:
"""The data this store reads and writes, as `(backend, root)`, or `None` if unknown.

Two stores with equal `backend` values see the same keys, and `root` is the
`/`-separated path of this store's keys within that backend. Stores with
different roots in one backend can overlap, which `__eq__` can't detect.
"""
return None

@abstractmethod
async def get(
self,
Expand Down
50 changes: 28 additions & 22 deletions src/zarr/core/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,8 +157,7 @@
get_pipeline_class,
)
from zarr.storage._common import StorePath, make_store_path, resolve_zarr_format
from zarr.storage._memory import MemoryStore
from zarr.storage._utils import _relativize_path
from zarr.storage._utils import _join_paths, _relativize_path

if TYPE_CHECKING:
from collections.abc import Iterator
Expand Down Expand Up @@ -4166,19 +4165,30 @@ class ShardsConfigParam(TypedDict):
type ShardsLike = ChunksLike | ShardsConfigParam | Literal["auto"]


def _stores_share_data(a: Store, b: Store) -> bool:
"""Whether two stores are views onto the same underlying data.
def _store_paths_overlap(a: StorePath, b: StorePath) -> bool:
"""Whether the keys under two store paths can overlap.

``Store.__eq__`` can't answer this directly. Several stores include ``read_only`` in
their equality check, so a read-only and a writable view of the same data are
unequal. ``MemoryStore`` compares its dicts by value, so two distinct stores with
equal contents are equal.
Stores that report a data location are compared by backend and by their root joined
with the path, so two stores rooted at nested directories of one backend overlap.
For a store that can't report a location, this falls back to comparing the stores,
ignoring ``read_only``, and then the paths alone.
"""
if type(a) is MemoryStore and type(b) is MemoryStore:
return a._store_dict is b._store_dict
with contextlib.suppress(NotImplementedError):
a = a.with_read_only(b.read_only)
return a == b
location_a, location_b = a.store._data_location(), b.store._data_location()
if location_a is not None and location_b is not None:
(backend_a, root_a), (backend_b, root_b) = location_a, location_b
if backend_a != backend_b:
return False
path_a, path_b = _join_paths([root_a, a.path]), _join_paths([root_b, b.path])
else:
store_a = a.store
with contextlib.suppress(NotImplementedError):
store_a = store_a.with_read_only(b.store.read_only)
if store_a != b.store:
return False
path_a, path_b = a.path, b.path
prefix_a = f"{path_a}/" if path_a else ""
prefix_b = f"{path_b}/" if path_b else ""
return prefix_a.startswith(prefix_b) or prefix_b.startswith(prefix_a)


async def from_array(
Expand Down Expand Up @@ -4423,16 +4433,12 @@ async def from_array(
write_data
and overwrite
and isinstance(data, Array)
and _stores_share_data(data.store_path.store, store_path.store)
and _store_paths_overlap(data.store_path, store_path)
):
src_prefix = f"{data.store_path.path}/" if data.store_path.path else ""
dest_prefix = f"{store_path.path}/" if store_path.path else ""
if src_prefix.startswith(dest_prefix) or dest_prefix.startswith(src_prefix):
raise ValueError(
f"Cannot overwrite {store_path.path!r} with the array at "
f"{data.store_path.path!r} because the paths overlap. Write to a different "
"path, or pass data=array[...]."
)
raise ValueError(
f"Cannot overwrite {store_path} with the array at {data.store_path} because "
"the paths overlap. Write to a different path, or pass data=array[...]."
)

result = await init_array(
store_path=store_path,
Expand Down
8 changes: 7 additions & 1 deletion src/zarr/storage/_fsspec.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from zarr.storage._utils import _dereference_path

if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterable
from collections.abc import AsyncIterator, Hashable, Iterable

from fsspec import AbstractFileSystem
from fsspec.asyn import AsyncFileSystem
Expand Down Expand Up @@ -286,6 +286,12 @@ def __eq__(self, other: object) -> bool:
and self.fs == other.fs
)

def _data_location(self) -> tuple[Hashable, str]:
# docstring inherited
# fsspec caches filesystem instances by this token, so two stores built with the
# same filesystem arguments share a backend.
return ("fsspec", self.fs._fs_token), self.path.rstrip("/")

async def get(
self,
key: str,
Expand Down
6 changes: 5 additions & 1 deletion src/zarr/storage/_local.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from zarr.core.common import AccessModeLiteral, concurrent_map

if TYPE_CHECKING:
from collections.abc import AsyncIterator, Callable, Iterable, Iterator
from collections.abc import AsyncIterator, Callable, Hashable, Iterable, Iterator

from zarr.core.buffer import BufferPrototype

Expand Down Expand Up @@ -300,6 +300,10 @@ def __repr__(self) -> str:
def __eq__(self, other: object) -> bool:
return isinstance(other, type(self)) and self.root == other.root

def _data_location(self) -> tuple[Hashable, str]:
# docstring inherited
return ("file",), self.root.resolve().as_posix()

# -------------------------------------------------------------------
# Synchronous store methods
# -------------------------------------------------------------------
Expand Down
10 changes: 9 additions & 1 deletion src/zarr/storage/_memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
)

if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterable, MutableMapping
from collections.abc import AsyncIterator, Hashable, Iterable, MutableMapping

from zarr.core.buffer import BufferPrototype

Expand Down Expand Up @@ -103,6 +103,10 @@ def __eq__(self, other: object) -> bool:
and self.read_only == other.read_only
)

def _data_location(self) -> tuple[Hashable, str]:
# docstring inherited
return ("memory", id(self._store_dict)), ""

# -------------------------------------------------------------------
# Synchronous store methods
# -------------------------------------------------------------------
Expand Down Expand Up @@ -508,6 +512,10 @@ def __eq__(self, other: object) -> bool:
and self.read_only == other.read_only
)

def _data_location(self) -> tuple[Hashable, str]:
# docstring inherited
return ("memory", id(self._store_dict)), self.path

@property
def name(self) -> str:
"""The name of this store, used in the memory:// URL."""
Expand Down
6 changes: 5 additions & 1 deletion src/zarr/storage/_wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from typing import TYPE_CHECKING, cast

if TYPE_CHECKING:
from collections.abc import AsyncGenerator, AsyncIterator, Iterable, Sequence
from collections.abc import AsyncGenerator, AsyncIterator, Hashable, Iterable, Sequence
from types import TracebackType
from typing import Any, Self

Expand Down Expand Up @@ -91,6 +91,10 @@ def _check_writable(self) -> None:
def __eq__(self, value: object) -> bool:
return type(self) is type(value) and self._store.__eq__(value._store)

def _data_location(self) -> tuple[Hashable, str] | None:
# docstring inherited
return self._store._data_location()

def __str__(self) -> str:
return f"wrapping-{self._store}"

Expand Down
8 changes: 7 additions & 1 deletion src/zarr/storage/_zip.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from zarr.core.buffer import Buffer, BufferPrototype

if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterable
from collections.abc import AsyncIterator, Hashable, Iterable

ZipStoreAccessModeLiteral = Literal["r", "w", "a", "x"]

Expand Down Expand Up @@ -336,6 +336,12 @@ def __eq__(self, other: object) -> bool:
and self._fileobj is other._fileobj
)

def _data_location(self) -> tuple[Hashable, str]:
# docstring inherited
if self.path is None:
return ("zip", id(self._fileobj)), ""
return ("zip", self.path.resolve().as_posix()), ""

def _get(
self,
key: str,
Expand Down
12 changes: 7 additions & 5 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,12 +153,14 @@ async def store(request: pytest.FixtureRequest, tmp_path: pathlib.Path) -> Store


@pytest.fixture
async def store2(request: pytest.FixtureRequest, tmp_path: pathlib.Path) -> Store:
"""Fixture to create a second store for testing copy operations between stores"""
async def store2(request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory) -> Store:
"""Fixture to create a second store for testing copy operations between stores.

Its directory is separate from `store`'s, so a copy from one to the other never
writes inside the source array's directory.
"""
param = request.param
store2_path = tmp_path / "store2"
store2_path.mkdir()
return await parse_store(param, str(store2_path))
return await parse_store(param, str(tmp_path_factory.mktemp("store2")))


@pytest.fixture(params=["local", "memory", "zip"])
Expand Down
90 changes: 89 additions & 1 deletion tests/test_array.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import re
import sys
from itertools import accumulate, starmap
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal
from unittest import mock

Expand Down Expand Up @@ -43,6 +44,7 @@
_parse_chunk_encoding_v2,
_parse_chunk_encoding_v3,
_shards_initialized,
_store_paths_overlap,
create_array,
default_filters_v2,
default_serializer_v3,
Expand Down Expand Up @@ -81,7 +83,14 @@
ContainsGroupError,
ZarrUserWarning,
)
from zarr.storage import LocalStore, MemoryStore, StorePath
from zarr.storage import (
FsspecStore,
LocalStore,
ManagedMemoryStore,
MemoryStore,
StorePath,
ZipStore,
)
from zarr.storage._logging import LoggingStore
from zarr.types import AnyArray, AnyAsyncArray

Expand Down Expand Up @@ -2010,6 +2019,85 @@ def test_from_array_overwrite_equal_memory_stores() -> None:
np.testing.assert_array_equal(result[...], np.arange(4.0))


def _store_path_pair(case: str, tmp_path: Path) -> tuple[StorePath, StorePath]:
"""Two store paths for `test_store_paths_overlap`, built for one named case."""
managed_name = f"overlap-{tmp_path.name}"
match case:
case "local-nested-root":
return StorePath(LocalStore(tmp_path), "a"), StorePath(LocalStore(tmp_path / "a"))
case "local-sibling-root":
return StorePath(LocalStore(tmp_path), "a"), StorePath(LocalStore(tmp_path / "ab"))
case "local-read-only":
local_store = LocalStore(tmp_path)
return StorePath(local_store, "a"), StorePath(local_store.with_read_only(True), "a/b")
case "memory-same-dict":
memory_store = MemoryStore()
return StorePath(memory_store, "g/a"), StorePath(memory_store, "g")
case "memory-equal-contents":
return StorePath(MemoryStore(), "a"), StorePath(MemoryStore(), "a")
case "managed-memory-path":
return (
StorePath(ManagedMemoryStore(name=managed_name), "g/a"),
StorePath(ManagedMemoryStore(name=managed_name, path="g"), "a"),
)
case "managed-memory-other-path":
return (
StorePath(ManagedMemoryStore(name=managed_name), "g/a"),
StorePath(ManagedMemoryStore(name=managed_name, path="h")),
)
case "zip-same-file":
return (
StorePath(ZipStore(tmp_path / "x.zip", mode="w"), "a"),
StorePath(ZipStore(tmp_path / "x.zip", mode="r")),
)
case "fsspec-nested-root":
return (
StorePath(FsspecStore.from_url(f"memory://{tmp_path.name}"), "x/a"),
StorePath(FsspecStore.from_url(f"memory://{tmp_path.name}/x"), "a"),
)
case "wrapper":
return (
StorePath(LoggingStore(LocalStore(tmp_path)), "a"),
StorePath(LocalStore(tmp_path / "a")),
)
case "different-backends":
return StorePath(LocalStore(tmp_path), "a"), StorePath(MemoryStore(), "a")
raise ValueError(case)


@pytest.mark.parametrize(
("case", "expected"),
[
("local-nested-root", True),
("local-sibling-root", False),
("local-read-only", True),
("memory-same-dict", True),
("memory-equal-contents", False),
("managed-memory-path", True),
("managed-memory-other-path", False),
("zip-same-file", True),
("fsspec-nested-root", True),
("wrapper", True),
("different-backends", False),
],
)
def test_store_paths_overlap(case: str, expected: bool, tmp_path: Path) -> None:
"""Two store paths overlap when their stores share a backend and the stores' roots
joined with the paths nest, whatever root each store was given."""
a, b = _store_path_pair(case, tmp_path)
assert _store_paths_overlap(a, b) is expected
assert _store_paths_overlap(b, a) is expected


def test_from_array_overwrite_nested_store_root_raises(tmp_path: Path) -> None:
"""A destination store rooted inside the source array's directory overlaps the
source, so the overwrite raises instead of deleting the source's chunks."""
src = zarr.create_array(LocalStore(tmp_path), name="a", data=np.arange(1.0, 5.0))
with pytest.raises(ValueError, match="paths overlap"):
zarr.from_array(LocalStore(tmp_path / "a"), data=src, overwrite=True)
np.testing.assert_array_equal(src[...], np.arange(1.0, 5.0))


@pytest.mark.parametrize("store", ["memory"], indirect=True)
def test_from_array_overwrite_non_overlapping_source(store: Store) -> None:
"""Overlap checks respect path boundaries, and metadata-only copies are allowed."""
Expand Down
Loading