From fc58439ab2e336f2fff083c62f5053b07b885ede Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 17:46:10 +0100 Subject: [PATCH 1/3] fix(indexing): keep the last value when setitem gets repeated indices The indexers group selected items by chunk with an unstable argsort, so repeated indices reached the in-chunk assignment in an arbitrary order and the stored value depended on the platform, the chunk layout and the number of selected items. Sorting stably keeps repeats in the order given, and the in-chunk NumPy assignment then stores the last value. Refs zarr-developers/zarr-python#2855 Assisted-by: ClaudeCode:claude-fable-5.2 --- src/zarr/core/indexing.py | 10 +++++---- tests/test_indexing.py | 47 ++++++++++++++++++++++++++++++--------- tests/test_properties.py | 24 ++++---------------- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index 8ea59eb960..3f564023e6 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -828,8 +828,9 @@ def __init__( dim_out_sel = np.arange(nitems - 1, -1, -1) dim_sel_chunk_sorted = dim_sel_chunk[::-1] else: - # sort indices to group by chunk - dim_out_sel = np.argsort(dim_sel_chunk) + # sort indices to group by chunk; a stable sort keeps repeated indices in + # the order given, so the last value assigned to an item is the one stored + dim_out_sel = np.argsort(dim_sel_chunk, kind="stable") dim_sel = np.take(dim_sel, dim_out_sel) dim_sel_chunk_sorted = dim_sel_chunk[dim_out_sel] @@ -1343,8 +1344,9 @@ def __init__( # group points by chunk if np.any(np.diff(chunks_raveled_indices) < 0): - # optimisation, only sort if needed - sel_sort = np.argsort(chunks_raveled_indices) + # optimisation, only sort if needed; a stable sort keeps repeated points in + # the order given, so the last value assigned to a point is the one stored + sel_sort = np.argsort(chunks_raveled_indices, kind="stable") selection_broadcast = tuple(dim_sel[sel_sort] for dim_sel in selection_broadcast) chunks_raveled_indices = chunks_raveled_indices[sel_sort] else: diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 785f089513..a0931d00a8 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -507,17 +507,42 @@ def test_setitem_zarr_array_as_value() -> None: assert_array_equal(dst2[2:7], np.arange(2, 7)) -@pytest.mark.skip(reason="fails on ubuntu, windows; numpy=2.2; in CI") -def test_setitem_repeated_index(): - """oindex assignment with repeated indices writes the last value for each duplicated index position.""" - array = zarr.array(data=np.zeros((4,)), chunks=(1,)) - indexer = np.array([-1, -1, 0, 0]) - array.oindex[(indexer,)] = [0, 1, 2, 3] - np.testing.assert_array_equal(array[:], np.array([3, 0, 0, 1])) - - indexer = np.array([-1, 0, 0, -1]) - array.oindex[(indexer,)] = [0, 1, 2, 3] - np.testing.assert_array_equal(array[:], np.array([2, 0, 0, 3])) +@pytest.mark.parametrize("method", ["oindex", "vindex"]) +@pytest.mark.parametrize("layout", ["unit_chunks", "partial_chunks", "one_chunk", "sharded"]) +@pytest.mark.parametrize( + ("shape", "selection"), + [ + ((24,), ([-1, -1, 0, 0],)), + ((24,), ([-1, 0, 0, -1],)), + # long enough that an unstable sort by chunk reorders the repeats + ((24,), ([1] + [0] * 23,)), + ((6, 6), ([0, 5, 0, -1, 2] * 4, [1, 1, 4, 4, 2] * 4)), + ], + ids=["adjacent", "interleaved", "many_repeats", "two_dimensions"], +) +def test_setitem_repeated_index( + method: str, layout: str, shape: tuple[int, ...], selection: tuple[list[int], ...] +) -> None: + """Assignment with repeated indices stores the last value given for each repeated position, as NumPy does.""" + chunks = { + "unit_chunks": (1,) * len(shape), + "partial_chunks": (4,) * len(shape), + "one_chunk": shape, + "sharded": (2,) * len(shape), + }[layout] + shards = (6,) * len(shape) if layout == "sharded" else None + array = zarr.create_array(store={}, shape=shape, chunks=chunks, shards=shards, dtype="i8") + expected = np.zeros(shape, dtype="i8") + if method == "oindex": + np_selection = np.ix_(*selection) + value = np.arange(1, 1 + expected[np_selection].size).reshape(expected[np_selection].shape) + expected[np_selection] = value + array.oindex[selection] = value + else: + value = np.arange(1, 1 + len(selection[0])) + expected[selection] = value + array.vindex[selection] = value + assert_array_equal(array[:], expected) Index = list[int] | tuple[slice | int | list[int], ...] diff --git a/tests/test_properties.py b/tests/test_properties.py index b448a524aa..d5d385accf 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -17,7 +17,7 @@ import hypothesis.extra.numpy as npst import hypothesis.strategies as st -from hypothesis import assume, event, given, settings +from hypothesis import event, given, settings from tests.conftest import declared_chunk_data_sizes from zarr.abc.store import Store @@ -199,12 +199,8 @@ async def test_oindex(data: st.DataObject) -> None: actual = await async_zarray.oindex.getitem(zindexer) assert_array_equal(nparray[npindexer], actual) - # sync set - for idxr, size in zip(zindexer, nparray.shape, strict=True): - if isinstance(idxr, np.ndarray) and idxr.size != np.unique(idxr % size).size: - # behaviour of setitem with repeated indices is not guaranteed in practice - # Negative and positive spellings of the same index are duplicates too. - assume(False) + # sync set: an index that repeats a position (in either spelling) keeps the last value + # written to it, as in NumPy, so repeated indices are drawn like any other # The sharding codec sees a coordinate selection (the GH4284 path) when the # chunk selection has more than one array axis or drops an integer axis. n_array_axes = sum(isinstance(idxr, np.ndarray) for idxr in zindexer) @@ -248,19 +244,7 @@ async def test_vindex(data: st.DataObject) -> None: actual = await async_zarray.vindex.getitem(indexer) assert_array_equal(nparray[indexer], actual) - # sync set - # Reads preserve the supplied indices; normalize negative indices explicitly when - # detecting repeated points rather than relying on a read to mutate the indexer. - points = np.stack( - [ - (idxr % size).ravel() - for idxr, size in zip(np.broadcast_arrays(*indexer), nparray.shape, strict=True) - ], - axis=-1, - ) - if len(np.unique(points, axis=0)) != len(points): - # behaviour of setitem with repeated coordinates is not guaranteed in practice - assume(False) + # sync set: a repeated point keeps the last value written to it, as in NumPy new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) nparray[indexer] = new_data zarray.vindex[indexer] = new_data From aa58a965e548a30c3eefbed2a2a96ef6b62e80ce Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 17:46:41 +0100 Subject: [PATCH 2/3] docs(changes): add changelog fragment for repeated-index writes Assisted-by: ClaudeCode:claude-fable-5.2 --- changes/453.bugfix.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/453.bugfix.md diff --git a/changes/453.bugfix.md b/changes/453.bugfix.md new file mode 100644 index 0000000000..1aec76d22a --- /dev/null +++ b/changes/453.bugfix.md @@ -0,0 +1 @@ +Assigning through `oindex`, `vindex` or `__setitem__` with an index array that repeats a position now stores the last value given for that position, as NumPy does. Previously the stored value depended on the platform, the chunk layout and the number of selected items. From 91b7c2c35e43a71d7ee7257c5e37e263a8ff77af Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 16 Nov 2026 17:50:08 +0100 Subject: [PATCH 3/3] docs(changes): renumber the changelog fragment to the upstream PR Assisted-by: ClaudeCode:claude-fable-5.2 --- changes/{453.bugfix.md => 4643.bugfix.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{453.bugfix.md => 4643.bugfix.md} (100%) diff --git a/changes/453.bugfix.md b/changes/4643.bugfix.md similarity index 100% rename from changes/453.bugfix.md rename to changes/4643.bugfix.md