diff --git a/changes/4643.bugfix.md b/changes/4643.bugfix.md new file mode 100644 index 0000000000..1aec76d22a --- /dev/null +++ b/changes/4643.bugfix.md @@ -0,0 +1 @@ +Assigning through `oindex`, `vindex` or `__setitem__` with an index array that repeats a position now stores the last value given for that position, as NumPy does. Previously the stored value depended on the platform, the chunk layout and the number of selected items. diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index ffd4035bb7..3b9f972961 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -830,8 +830,9 @@ def __init__( dim_out_sel = np.arange(nitems - 1, -1, -1) dim_sel_chunk_sorted = dim_sel_chunk[::-1] else: - # sort indices to group by chunk - dim_out_sel = np.argsort(dim_sel_chunk) + # sort indices to group by chunk; a stable sort keeps repeated indices in + # the order given, so the last value assigned to an item is the one stored + dim_out_sel = np.argsort(dim_sel_chunk, kind="stable") dim_sel = np.take(dim_sel, dim_out_sel) dim_sel_chunk_sorted = dim_sel_chunk[dim_out_sel] @@ -1345,8 +1346,9 @@ def __init__( # group points by chunk if np.any(np.diff(chunks_raveled_indices) < 0): - # optimisation, only sort if needed - sel_sort = np.argsort(chunks_raveled_indices) + # optimisation, only sort if needed; a stable sort keeps repeated points in + # the order given, so the last value assigned to a point is the one stored + sel_sort = np.argsort(chunks_raveled_indices, kind="stable") selection_broadcast = tuple(dim_sel[sel_sort] for dim_sel in selection_broadcast) chunks_raveled_indices = chunks_raveled_indices[sel_sort] else: diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 4ff35563c6..53fe1be8a1 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -507,17 +507,42 @@ def test_setitem_zarr_array_as_value() -> None: assert_array_equal(dst2[2:7], np.arange(2, 7)) -@pytest.mark.skip(reason="fails on ubuntu, windows; numpy=2.2; in CI") -def test_setitem_repeated_index(): - """oindex assignment with repeated indices writes the last value for each duplicated index position.""" - array = zarr.array(data=np.zeros((4,)), chunks=(1,)) - indexer = np.array([-1, -1, 0, 0]) - array.oindex[(indexer,)] = [0, 1, 2, 3] - np.testing.assert_array_equal(array[:], np.array([3, 0, 0, 1])) - - indexer = np.array([-1, 0, 0, -1]) - array.oindex[(indexer,)] = [0, 1, 2, 3] - np.testing.assert_array_equal(array[:], np.array([2, 0, 0, 3])) +@pytest.mark.parametrize("method", ["oindex", "vindex"]) +@pytest.mark.parametrize("layout", ["unit_chunks", "partial_chunks", "one_chunk", "sharded"]) +@pytest.mark.parametrize( + ("shape", "selection"), + [ + ((24,), ([-1, -1, 0, 0],)), + ((24,), ([-1, 0, 0, -1],)), + # long enough that an unstable sort by chunk reorders the repeats + ((24,), ([1] + [0] * 23,)), + ((6, 6), ([0, 5, 0, -1, 2] * 4, [1, 1, 4, 4, 2] * 4)), + ], + ids=["adjacent", "interleaved", "many_repeats", "two_dimensions"], +) +def test_setitem_repeated_index( + method: str, layout: str, shape: tuple[int, ...], selection: tuple[list[int], ...] +) -> None: + """Assignment with repeated indices stores the last value given for each repeated position, as NumPy does.""" + chunks = { + "unit_chunks": (1,) * len(shape), + "partial_chunks": (4,) * len(shape), + "one_chunk": shape, + "sharded": (2,) * len(shape), + }[layout] + shards = (6,) * len(shape) if layout == "sharded" else None + array = zarr.create_array(store={}, shape=shape, chunks=chunks, shards=shards, dtype="i8") + expected = np.zeros(shape, dtype="i8") + if method == "oindex": + np_selection = np.ix_(*selection) + value = np.arange(1, 1 + expected[np_selection].size).reshape(expected[np_selection].shape) + expected[np_selection] = value + array.oindex[selection] = value + else: + value = np.arange(1, 1 + len(selection[0])) + expected[selection] = value + array.vindex[selection] = value + assert_array_equal(array[:], expected) Index = list[int] | tuple[slice | int | list[int], ...] diff --git a/tests/test_properties.py b/tests/test_properties.py index b448a524aa..d5d385accf 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -17,7 +17,7 @@ import hypothesis.extra.numpy as npst import hypothesis.strategies as st -from hypothesis import assume, event, given, settings +from hypothesis import event, given, settings from tests.conftest import declared_chunk_data_sizes from zarr.abc.store import Store @@ -199,12 +199,8 @@ async def test_oindex(data: st.DataObject) -> None: actual = await async_zarray.oindex.getitem(zindexer) assert_array_equal(nparray[npindexer], actual) - # sync set - for idxr, size in zip(zindexer, nparray.shape, strict=True): - if isinstance(idxr, np.ndarray) and idxr.size != np.unique(idxr % size).size: - # behaviour of setitem with repeated indices is not guaranteed in practice - # Negative and positive spellings of the same index are duplicates too. - assume(False) + # sync set: an index that repeats a position (in either spelling) keeps the last value + # written to it, as in NumPy, so repeated indices are drawn like any other # The sharding codec sees a coordinate selection (the GH4284 path) when the # chunk selection has more than one array axis or drops an integer axis. n_array_axes = sum(isinstance(idxr, np.ndarray) for idxr in zindexer) @@ -248,19 +244,7 @@ async def test_vindex(data: st.DataObject) -> None: actual = await async_zarray.vindex.getitem(indexer) assert_array_equal(nparray[indexer], actual) - # sync set - # Reads preserve the supplied indices; normalize negative indices explicitly when - # detecting repeated points rather than relying on a read to mutate the indexer. - points = np.stack( - [ - (idxr % size).ravel() - for idxr, size in zip(np.broadcast_arrays(*indexer), nparray.shape, strict=True) - ], - axis=-1, - ) - if len(np.unique(points, axis=0)) != len(points): - # behaviour of setitem with repeated coordinates is not guaranteed in practice - assume(False) + # sync set: a repeated point keeps the last value written to it, as in NumPy new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) nparray[indexer] = new_data zarray.vindex[indexer] = new_data