diff --git a/changes/2855.bugfix.md b/changes/2855.bugfix.md new file mode 100644 index 0000000000..72b1f6c53b --- /dev/null +++ b/changes/2855.bugfix.md @@ -0,0 +1 @@ +Assigning through `oindex`, `vindex` or `__setitem__` with an index array that repeats a position now stores the last value given for that position, as NumPy does, on every platform and for every chunking. Previously the stored value depended on the chunk layout and on the sort order of the platform's NumPy build. diff --git a/docs/user-guide/arrays.md b/docs/user-guide/arrays.md index 5b025047b6..9e0783d206 100644 --- a/docs/user-guide/arrays.md +++ b/docs/user-guide/arrays.md @@ -481,6 +481,10 @@ Mask indexing is conceptually the same as coordinate indexing, and is implemented internally via the same machinery. Both styles of indexing allow selecting arbitrary items from an array, also known as point selection. +When a coordinate or orthogonal selection names the same item more than once +in an assignment, the last value given for that item is the one stored, as +with NumPy. The result does not depend on the chunking of the array. + ### Orthogonal indexing Zarr arrays also support methods for orthogonal indexing, which allows diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 3735a4da18..02bd98c66e 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -5862,6 +5862,10 @@ async def _set_selection( value = np.array(value, dtype=dtype, order="A") value = cast("NDArrayLike", value) + # A selection that names the same element more than once is reduced to one write per + # element, keeping the last value given for it, as sequential NumPy assignment does. + indexer, value = _collapse_repeated_indices(indexer, value, metadata.shape, chunk_grid) + # We accept any ndarray like object from the user and convert it # to an NDBuffer (or subclass). From this point onwards, we only pass # Buffer and NDBuffer between components. @@ -5908,6 +5912,44 @@ async def _set_selection( ) +def _collapse_repeated_indices( + indexer: Indexer, value: NDArrayLike, shape: tuple[int, ...], chunk_grid: ChunkGrid +) -> tuple[Indexer, NDArrayLike]: + """Make a write with repeated indices touch each element once, keeping the last value. + + Orthogonal and coordinate selections may name an element several times. Writing them + as given leaves the result to the order in which the chunk writes and the in-chunk + scatter happen, which differs between platforms and chunkings. Instead the selection + is reduced to the last occurrence of each element and the value array is reduced to + match, which is the result NumPy's sequential assignment gives. + """ + if not isinstance(indexer, OrthogonalIndexer | CoordinateIndexer): + return indexer, value + collapsed = indexer.without_repeats(shape, chunk_grid) + if collapsed is None: + return indexer, value + new_indexer, kept = collapsed + if len(value.shape) == 0: + # a scalar is the same for every element, nothing to reduce + return new_indexer, value + if value.shape != indexer.shape: + try: + value = np.broadcast_to(value, indexer.shape) + except ValueError: + raise ValueError( + f"shape mismatch: value array of shape {value.shape} could not be broadcast " + f"to indexing result of shape {indexer.shape}" + ) from None + if isinstance(indexer, OrthogonalIndexer): + kept = cast("tuple[tuple[int, npt.NDArray[np.intp]], ...]", kept) + for axis, keep in kept: + value = np.take(value, keep, axis=axis) + else: + kept = cast("npt.NDArray[np.intp]", kept) + value = np.reshape(value, -1)[kept] + return new_indexer, value + + def _iter_chunk_coords_to_delete( old_grid_shape: tuple[int, ...], new_grid_shape: tuple[int, ...] ) -> Iterator[tuple[int, ...]]: diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index ffd4035bb7..d0fc29a153 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -762,6 +762,29 @@ def sorted_run_ends( return a[run_starts].astype(np.intp, copy=False), run_ends +def last_occurrences(a: npt.NDArray[np.intp]) -> npt.NDArray[np.intp] | None: + """Positions of the last occurrence of each value of a 1-D integer array, in increasing order. + + Returns None when no value is repeated. Used to collapse a selection with repeated + indices into one that touches each element once, keeping the last value written to it, + which is what sequential (NumPy) assignment ends up with. + """ + if a.size < 2: + return None + if bool((a[1:] >= a[:-1]).all()): + # sorted: repeats are adjacent + repeated = a[1:] == a[:-1] + if not repeated.any(): + return None + return np.flatnonzero(np.append(~repeated, True)).astype(np.intp, copy=False) + _, first_in_reversed = np.unique(a[::-1], return_index=True) + if first_in_reversed.size == a.size: + return None + keep = (a.size - 1 - first_in_reversed).astype(np.intp, copy=False) + keep.sort() + return keep + + @dataclass(frozen=True) class IntArrayDimIndexer: """Integer array selection against a single dimension.""" @@ -943,6 +966,8 @@ class OrthogonalIndexer(Indexer): shape: tuple[int, ...] is_advanced: bool drop_axes: tuple[int, ...] + # the selection after ellipsis and list normalization, one selector per dimension + selection: SelectionNormalized = field(repr=False) def __init__(self, selection: Selection, shape: tuple[int, ...], chunk_grid: ChunkGrid) -> None: dim_grids = chunk_grid._dimensions @@ -996,6 +1021,38 @@ def __init__(self, selection: Selection, shape: tuple[int, ...], chunk_grid: Chu object.__setattr__(self, "shape", shape) object.__setattr__(self, "is_advanced", is_advanced) object.__setattr__(self, "drop_axes", drop_axes) + object.__setattr__(self, "selection", selection) + + def without_repeats( + self, shape: tuple[int, ...], chunk_grid: ChunkGrid + ) -> tuple[OrthogonalIndexer, tuple[tuple[int, npt.NDArray[np.intp]], ...]] | None: + """Collapse repeated integer indices, keeping the last occurrence along each dimension. + + Returns the collapsed indexer and, for each output axis that shrank, the positions + kept along it, so that a value array can be reduced to match. Returns None when no + dimension repeats an index. Negative and positive spellings of the same index count + as repeats. + """ + selection = list(self.selection) + kept: list[tuple[int, npt.NDArray[np.intp]]] = [] + out_axis = 0 + for dim, (dim_sel, dim_indexer, dim_len) in enumerate( + zip(self.selection, self.dim_indexers, shape, strict=True) + ): + if isinstance(dim_indexer, IntDimIndexer): + continue + if isinstance(dim_indexer, IntArrayDimIndexer): + normalized = wraparound_indices( + np.asarray(dim_sel).astype(np.intp, copy=False), dim_len + ) + keep = last_occurrences(normalized) + if keep is not None: + selection[dim] = normalized[keep] + kept.append((out_axis, keep)) + out_axis += 1 + if not kept: + return None + return OrthogonalIndexer(tuple(selection), shape, chunk_grid), tuple(kept) def __iter__(self) -> Iterator[ChunkProjection]: for dim_projections in itertools.product(*self.dim_indexers): @@ -1372,6 +1429,39 @@ def __init__( object.__setattr__(self, "shape", shape) object.__setattr__(self, "drop_axes", ()) + def without_repeats( + self, shape: tuple[int, ...], chunk_grid: ChunkGrid + ) -> tuple[CoordinateIndexer, npt.NDArray[np.intp]] | None: + """Collapse repeated points, keeping the last occurrence of each. + + Returns the collapsed indexer, whose shape is one-dimensional, and the positions kept + in the flattened selection, so that a value array can be reduced to match. Returns + None when no point is repeated. + """ + if shape == (): + return None + # self.selection is flat and non-negative, grouped by chunk; sel_sort maps each entry + # back to its position in the selection as given, which decides what "last" means. + # Points are compared coordinate by coordinate rather than through a raveled index, + # which could overflow for arrays with more than 2**63 elements. + n = self.selection[0].size + if n < 2: + return None + positions = np.arange(n, dtype=np.intp) if self.sel_sort is None else self.sel_sort + # lexsort uses the last key as the primary one: sort by point, then by position + order = np.lexsort((positions, *reversed(self.selection))) + same_as_next = np.ones(n - 1, dtype=bool) + for dim_sel in self.selection: + ordered = dim_sel[order] + same_as_next &= ordered[1:] == ordered[:-1] + if not same_as_next.any(): + return None + is_last = np.append(~same_as_next, True) + keep = np.sort(order[is_last]) + kept_positions = positions[keep] + selection = tuple(dim_sel[keep] for dim_sel in self.selection) + return CoordinateIndexer(selection, shape, chunk_grid), kept_positions + def __getattr__(self, name: str) -> npt.NDArray[np.intp]: if name != "chunk_nitems_cumsum": raise AttributeError(f"{type(self).__name__!r} object has no attribute {name!r}") @@ -1434,6 +1524,12 @@ def __init__( # delegate the rest to superclass super().__init__(selection_indices, shape, chunk_grid) + def without_repeats( + self, shape: tuple[int, ...], chunk_grid: ChunkGrid + ) -> tuple[CoordinateIndexer, npt.NDArray[np.intp]] | None: + # a mask selects each point at most once + return None + @dataclass(frozen=True) class VIndex: diff --git a/tests/test_indexing.py b/tests/test_indexing.py index 4ff35563c6..b604a5a63e 100644 --- a/tests/test_indexing.py +++ b/tests/test_indexing.py @@ -32,6 +32,7 @@ _ArrayIndexingOrder, _iter_grid, _iter_regions, + last_occurrences, make_slice_selection, normalize_integer_selection, oindex, @@ -507,17 +508,136 @@ def test_setitem_zarr_array_as_value() -> None: assert_array_equal(dst2[2:7], np.arange(2, 7)) -@pytest.mark.skip(reason="fails on ubuntu, windows; numpy=2.2; in CI") -def test_setitem_repeated_index(): - """oindex assignment with repeated indices writes the last value for each duplicated index position.""" - array = zarr.array(data=np.zeros((4,)), chunks=(1,)) - indexer = np.array([-1, -1, 0, 0]) - array.oindex[(indexer,)] = [0, 1, 2, 3] - np.testing.assert_array_equal(array[:], np.array([3, 0, 0, 1])) +@pytest.mark.parametrize("chunks", [(1,), (2,), (4,)]) +@pytest.mark.parametrize("via", ["oindex", "vindex", "getitem"]) +def test_setitem_repeated_index(chunks: tuple[int, ...], via: str) -> None: + """Assignment with repeated indices writes the last value given for each repeated position, as NumPy does.""" + array = zarr.zeros((4,), chunks=chunks, dtype="f8") + expected = np.zeros((4,)) + for indexer, value in [ + (np.array([-1, -1, 0, 0]), [0, 1, 2, 3]), + (np.array([-1, 0, 0, -1]), [0, 1, 2, 3]), + (np.array([3, 3, 3]), [7, 8, 9]), + ]: + if via == "oindex": + array.oindex[(indexer,)] = value + elif via == "vindex": + array.vindex[(indexer,)] = value + else: + array[indexer] = value + expected[indexer] = value + assert_array_equal(array[:], expected) + + +def test_set_orthogonal_selection_repeated_index_2d() -> None: + """Repeats along both dimensions of an orthogonal selection resolve like NumPy's ix_ assignment.""" + rows = np.array([0, 5, 0, -1, 2]) + cols = np.array([1, 1, 4, 4, 2]) + value = np.arange(1, 26).reshape(5, 5) + expected = np.zeros((6, 6), dtype="i8") + expected[np.ix_(rows, cols)] = value + for chunks in [(2, 3), (6, 6), (1, 1)]: + array = zarr.zeros((6, 6), chunks=chunks, dtype="i8") + array.oindex[rows, cols] = value + assert_array_equal(array[:], expected) + + # a scalar is written once per selected element whatever the repeats + array = zarr.zeros((6, 6), chunks=(2, 3), dtype="i8") + array.oindex[rows, cols] = 7 + expected = np.zeros((6, 6), dtype="i8") + expected[np.ix_(rows, cols)] = 7 + assert_array_equal(array[:], expected) + + # repeats combined with an integer, a slice and a Boolean array along other dimensions + for selection, np_selection in [ + ((3, cols), (3, cols)), + ((slice(1, 4), cols), np.ix_(np.arange(1, 4), cols)), + ((np.array([1, 0, 1, 1, 0, 0], dtype=bool), cols), np.ix_([0, 2, 3], cols)), + ]: + array = zarr.zeros((6, 6), chunks=(2, 3), dtype="i8") + expected = np.zeros((6, 6), dtype="i8") + value = np.arange(1, 1 + array.oindex[selection].size).reshape( + array.oindex[selection].shape + ) + array.oindex[selection] = value + expected[np_selection] = value + assert_array_equal(array[:], expected) + + +def test_set_coordinate_selection_repeated_points() -> None: + """Repeated points in a coordinate selection keep the last value, including across chunk boundaries.""" + rows = np.array([0, 5, 0, -1, 2, 0]) + cols = np.array([1, 2, 1, 2, 2, -5]) + value = np.arange(1, 7) + expected = np.zeros((6, 6), dtype="i8") + expected[rows, cols] = value + for chunks in [(2, 3), (6, 6), (1, 1)]: + array = zarr.zeros((6, 6), chunks=chunks, dtype="i8") + array.vindex[rows, cols] = value + assert_array_equal(array[:], expected) + + # multidimensional coordinate arrays and broadcasting between them + rows2 = np.array([[0, 5], [0, 5]]) + cols2 = np.array([[1, 2], [1, 2]]) + value2 = np.array([[1, 2], [3, 4]]) + array = zarr.zeros((6, 6), chunks=(2, 3), dtype="i8") + array.vindex[rows2, cols2] = value2 + expected = np.zeros((6, 6), dtype="i8") + expected[rows2, cols2] = value2 + assert_array_equal(array[:], expected) + + array = zarr.zeros((6, 6), chunks=(2, 3), dtype="i8") + array.vindex[np.array([0, 0, 5]), 2] = np.arange(3) + expected = np.zeros((6, 6), dtype="i8") + expected[np.array([0, 0, 5]), 2] = np.arange(3) + assert_array_equal(array[:], expected) + + # the sorted single-array path + coords = np.repeat(np.arange(0, 200, 7), 3) + value = np.arange(coords.size) + array = zarr.zeros((200,), chunks=(10,), dtype="i8") + array.vindex[coords] = value + expected = np.zeros((200,), dtype="i8") + expected[coords] = value + assert_array_equal(array[:], expected) + + +def test_set_coordinate_selection_repeated_points_huge_array() -> None: + """Repeated points are found without raveling the coordinates, which would overflow + for arrays with more than 2**63 elements.""" + n = 2**33 + z = zarr.create_array(store={}, shape=(n, n), chunks=(256, 256), dtype="i4", fill_value=0) + z.vindex[np.array([5, n - 1, 5]), np.array([7, n - 2, 7])] = np.array([1, 2, 3], dtype="i4") + assert_array_equal(z.vindex[np.array([5, n - 1]), np.array([7, n - 2])], [3, 2]) + + +def test_setitem_repeated_index_value_shape_mismatch() -> None: + """A value that does not fit the selection is rejected rather than written in part.""" + array = zarr.zeros((7,), chunks=(3,), dtype="i8") + with pytest.raises(ValueError, match="shape mismatch"): + array.oindex[np.array([0, 0, 1])] = np.arange(5) + assert_array_equal(array[:], np.zeros((7,))) - indexer = np.array([-1, 0, 0, -1]) - array.oindex[(indexer,)] = [0, 1, 2, 3] - np.testing.assert_array_equal(array[:], np.array([2, 0, 0, 3])) + +@pytest.mark.parametrize( + ("indices", "expected"), + [ + ([], None), + ([3], None), + ([0, 1, 2], None), + ([2, 1, 0], None), + ([0, 0], [1]), + ([0, 0, 3, 3, 3, 6], [1, 4, 5]), + ([6, 6, 3, 1, 1], [1, 2, 4]), + ([2, 0, 2, 1, 0], [2, 3, 4]), + ], +) +def test_last_occurrences(indices: list[int], expected: list[int] | None) -> None: + result = last_occurrences(np.array(indices, dtype=np.intp)) + if expected is None: + assert result is None + else: + assert_array_equal(result, np.array(expected, dtype=np.intp)) Index = list[int] | tuple[slice | int | list[int], ...] diff --git a/tests/test_properties.py b/tests/test_properties.py index b448a524aa..d5d385accf 100644 --- a/tests/test_properties.py +++ b/tests/test_properties.py @@ -17,7 +17,7 @@ import hypothesis.extra.numpy as npst import hypothesis.strategies as st -from hypothesis import assume, event, given, settings +from hypothesis import event, given, settings from tests.conftest import declared_chunk_data_sizes from zarr.abc.store import Store @@ -199,12 +199,8 @@ async def test_oindex(data: st.DataObject) -> None: actual = await async_zarray.oindex.getitem(zindexer) assert_array_equal(nparray[npindexer], actual) - # sync set - for idxr, size in zip(zindexer, nparray.shape, strict=True): - if isinstance(idxr, np.ndarray) and idxr.size != np.unique(idxr % size).size: - # behaviour of setitem with repeated indices is not guaranteed in practice - # Negative and positive spellings of the same index are duplicates too. - assume(False) + # sync set: an index that repeats a position (in either spelling) keeps the last value + # written to it, as in NumPy, so repeated indices are drawn like any other # The sharding codec sees a coordinate selection (the GH4284 path) when the # chunk selection has more than one array axis or drops an integer axis. n_array_axes = sum(isinstance(idxr, np.ndarray) for idxr in zindexer) @@ -248,19 +244,7 @@ async def test_vindex(data: st.DataObject) -> None: actual = await async_zarray.vindex.getitem(indexer) assert_array_equal(nparray[indexer], actual) - # sync set - # Reads preserve the supplied indices; normalize negative indices explicitly when - # detecting repeated points rather than relying on a read to mutate the indexer. - points = np.stack( - [ - (idxr % size).ravel() - for idxr, size in zip(np.broadcast_arrays(*indexer), nparray.shape, strict=True) - ], - axis=-1, - ) - if len(np.unique(points, axis=0)) != len(points): - # behaviour of setitem with repeated coordinates is not guaranteed in practice - assume(False) + # sync set: a repeated point keeps the last value written to it, as in NumPy new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype)) nparray[indexer] = new_data zarray.vindex[indexer] = new_data