Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/4643.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Assigning through `oindex`, `vindex` or `__setitem__` with an index array that repeats a position now stores the last value given for that position, as NumPy does. Previously the stored value depended on the platform, the chunk layout and the number of selected items.
10 changes: 6 additions & 4 deletions src/zarr/core/indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -830,8 +830,9 @@ def __init__(
dim_out_sel = np.arange(nitems - 1, -1, -1)
dim_sel_chunk_sorted = dim_sel_chunk[::-1]
else:
# sort indices to group by chunk
dim_out_sel = np.argsort(dim_sel_chunk)
# sort indices to group by chunk; a stable sort keeps repeated indices in
# the order given, so the last value assigned to an item is the one stored
dim_out_sel = np.argsort(dim_sel_chunk, kind="stable")
dim_sel = np.take(dim_sel, dim_out_sel)
dim_sel_chunk_sorted = dim_sel_chunk[dim_out_sel]

Expand Down Expand Up @@ -1345,8 +1346,9 @@ def __init__(

# group points by chunk
if np.any(np.diff(chunks_raveled_indices) < 0):
# optimisation, only sort if needed
sel_sort = np.argsort(chunks_raveled_indices)
# optimisation, only sort if needed; a stable sort keeps repeated points in
# the order given, so the last value assigned to a point is the one stored
sel_sort = np.argsort(chunks_raveled_indices, kind="stable")
selection_broadcast = tuple(dim_sel[sel_sort] for dim_sel in selection_broadcast)
chunks_raveled_indices = chunks_raveled_indices[sel_sort]
else:
Expand Down
47 changes: 36 additions & 11 deletions tests/test_indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -507,17 +507,42 @@ def test_setitem_zarr_array_as_value() -> None:
assert_array_equal(dst2[2:7], np.arange(2, 7))


@pytest.mark.skip(reason="fails on ubuntu, windows; numpy=2.2; in CI")
def test_setitem_repeated_index():
"""oindex assignment with repeated indices writes the last value for each duplicated index position."""
array = zarr.array(data=np.zeros((4,)), chunks=(1,))
indexer = np.array([-1, -1, 0, 0])
array.oindex[(indexer,)] = [0, 1, 2, 3]
np.testing.assert_array_equal(array[:], np.array([3, 0, 0, 1]))

indexer = np.array([-1, 0, 0, -1])
array.oindex[(indexer,)] = [0, 1, 2, 3]
np.testing.assert_array_equal(array[:], np.array([2, 0, 0, 3]))
@pytest.mark.parametrize("method", ["oindex", "vindex"])
@pytest.mark.parametrize("layout", ["unit_chunks", "partial_chunks", "one_chunk", "sharded"])
@pytest.mark.parametrize(
("shape", "selection"),
[
((24,), ([-1, -1, 0, 0],)),
((24,), ([-1, 0, 0, -1],)),
# long enough that an unstable sort by chunk reorders the repeats
((24,), ([1] + [0] * 23,)),
((6, 6), ([0, 5, 0, -1, 2] * 4, [1, 1, 4, 4, 2] * 4)),
],
ids=["adjacent", "interleaved", "many_repeats", "two_dimensions"],
)
def test_setitem_repeated_index(
method: str, layout: str, shape: tuple[int, ...], selection: tuple[list[int], ...]
) -> None:
"""Assignment with repeated indices stores the last value given for each repeated position, as NumPy does."""
chunks = {
"unit_chunks": (1,) * len(shape),
"partial_chunks": (4,) * len(shape),
"one_chunk": shape,
"sharded": (2,) * len(shape),
}[layout]
shards = (6,) * len(shape) if layout == "sharded" else None
array = zarr.create_array(store={}, shape=shape, chunks=chunks, shards=shards, dtype="i8")
expected = np.zeros(shape, dtype="i8")
if method == "oindex":
np_selection = np.ix_(*selection)
value = np.arange(1, 1 + expected[np_selection].size).reshape(expected[np_selection].shape)
expected[np_selection] = value
array.oindex[selection] = value
else:
value = np.arange(1, 1 + len(selection[0]))
expected[selection] = value
array.vindex[selection] = value
assert_array_equal(array[:], expected)


Index = list[int] | tuple[slice | int | list[int], ...]
Expand Down
24 changes: 4 additions & 20 deletions tests/test_properties.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

import hypothesis.extra.numpy as npst
import hypothesis.strategies as st
from hypothesis import assume, event, given, settings
from hypothesis import event, given, settings

from tests.conftest import declared_chunk_data_sizes
from zarr.abc.store import Store
Expand Down Expand Up @@ -199,12 +199,8 @@ async def test_oindex(data: st.DataObject) -> None:
actual = await async_zarray.oindex.getitem(zindexer)
assert_array_equal(nparray[npindexer], actual)

# sync set
for idxr, size in zip(zindexer, nparray.shape, strict=True):
if isinstance(idxr, np.ndarray) and idxr.size != np.unique(idxr % size).size:
# behaviour of setitem with repeated indices is not guaranteed in practice
# Negative and positive spellings of the same index are duplicates too.
assume(False)
# sync set: an index that repeats a position (in either spelling) keeps the last value
# written to it, as in NumPy, so repeated indices are drawn like any other
# The sharding codec sees a coordinate selection (the GH4284 path) when the
# chunk selection has more than one array axis or drops an integer axis.
n_array_axes = sum(isinstance(idxr, np.ndarray) for idxr in zindexer)
Expand Down Expand Up @@ -248,19 +244,7 @@ async def test_vindex(data: st.DataObject) -> None:
actual = await async_zarray.vindex.getitem(indexer)
assert_array_equal(nparray[indexer], actual)

# sync set
# Reads preserve the supplied indices; normalize negative indices explicitly when
# detecting repeated points rather than relying on a read to mutate the indexer.
points = np.stack(
[
(idxr % size).ravel()
for idxr, size in zip(np.broadcast_arrays(*indexer), nparray.shape, strict=True)
],
axis=-1,
)
if len(np.unique(points, axis=0)) != len(points):
# behaviour of setitem with repeated coordinates is not guaranteed in practice
assume(False)
# sync set: a repeated point keeps the last value written to it, as in NumPy
new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype))
nparray[indexer] = new_data
zarray.vindex[indexer] = new_data
Expand Down
Loading