Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/2855.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Assigning through `oindex`, `vindex` or `__setitem__` with an index array that repeats a position now stores the last value given for that position, as NumPy does, on every platform and for every chunking. Previously the stored value depended on the chunk layout and on the sort order of the platform's NumPy build.
4 changes: 4 additions & 0 deletions docs/user-guide/arrays.md
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,10 @@ Mask indexing is conceptually the same as coordinate indexing, and is
implemented internally via the same machinery. Both styles of indexing allow
selecting arbitrary items from an array, also known as point selection.

When a coordinate or orthogonal selection names the same item more than once
in an assignment, the last value given for that item is the one stored, as
with NumPy. The result does not depend on the chunking of the array.

### Orthogonal indexing

Zarr arrays also support methods for orthogonal indexing, which allows
Expand Down
42 changes: 42 additions & 0 deletions src/zarr/core/array.py
Original file line number Diff line number Diff line change
Expand Up @@ -5862,6 +5862,10 @@ async def _set_selection(
value = np.array(value, dtype=dtype, order="A")
value = cast("NDArrayLike", value)

# A selection that names the same element more than once is reduced to one write per
# element, keeping the last value given for it, as sequential NumPy assignment does.
indexer, value = _collapse_repeated_indices(indexer, value, metadata.shape, chunk_grid)

# We accept any ndarray like object from the user and convert it
# to an NDBuffer (or subclass). From this point onwards, we only pass
# Buffer and NDBuffer between components.
Expand Down Expand Up @@ -5908,6 +5912,44 @@ async def _set_selection(
)


def _collapse_repeated_indices(
indexer: Indexer, value: NDArrayLike, shape: tuple[int, ...], chunk_grid: ChunkGrid
) -> tuple[Indexer, NDArrayLike]:
"""Make a write with repeated indices touch each element once, keeping the last value.

Orthogonal and coordinate selections may name an element several times. Writing them
as given leaves the result to the order in which the chunk writes and the in-chunk
scatter happen, which differs between platforms and chunkings. Instead the selection
is reduced to the last occurrence of each element and the value array is reduced to
match, which is the result NumPy's sequential assignment gives.
"""
if not isinstance(indexer, OrthogonalIndexer | CoordinateIndexer):
return indexer, value
collapsed = indexer.without_repeats(shape, chunk_grid)
if collapsed is None:
return indexer, value
new_indexer, kept = collapsed
if len(value.shape) == 0:
# a scalar is the same for every element, nothing to reduce
return new_indexer, value
if value.shape != indexer.shape:
try:
value = np.broadcast_to(value, indexer.shape)
except ValueError:
raise ValueError(
f"shape mismatch: value array of shape {value.shape} could not be broadcast "
f"to indexing result of shape {indexer.shape}"
) from None
if isinstance(indexer, OrthogonalIndexer):
kept = cast("tuple[tuple[int, npt.NDArray[np.intp]], ...]", kept)
for axis, keep in kept:
value = np.take(value, keep, axis=axis)
else:
kept = cast("npt.NDArray[np.intp]", kept)
value = np.reshape(value, -1)[kept]
return new_indexer, value


def _iter_chunk_coords_to_delete(
old_grid_shape: tuple[int, ...], new_grid_shape: tuple[int, ...]
) -> Iterator[tuple[int, ...]]:
Expand Down
96 changes: 96 additions & 0 deletions src/zarr/core/indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -762,6 +762,29 @@ def sorted_run_ends(
return a[run_starts].astype(np.intp, copy=False), run_ends


def last_occurrences(a: npt.NDArray[np.intp]) -> npt.NDArray[np.intp] | None:
"""Positions of the last occurrence of each value of a 1-D integer array, in increasing order.

Returns None when no value is repeated. Used to collapse a selection with repeated
indices into one that touches each element once, keeping the last value written to it,
which is what sequential (NumPy) assignment ends up with.
"""
if a.size < 2:
return None
if bool((a[1:] >= a[:-1]).all()):
# sorted: repeats are adjacent
repeated = a[1:] == a[:-1]
if not repeated.any():
return None
return np.flatnonzero(np.append(~repeated, True)).astype(np.intp, copy=False)
_, first_in_reversed = np.unique(a[::-1], return_index=True)
if first_in_reversed.size == a.size:
return None
keep = (a.size - 1 - first_in_reversed).astype(np.intp, copy=False)
keep.sort()
return keep


@dataclass(frozen=True)
class IntArrayDimIndexer:
"""Integer array selection against a single dimension."""
Expand Down Expand Up @@ -943,6 +966,8 @@ class OrthogonalIndexer(Indexer):
shape: tuple[int, ...]
is_advanced: bool
drop_axes: tuple[int, ...]
# the selection after ellipsis and list normalization, one selector per dimension
selection: SelectionNormalized = field(repr=False)

def __init__(self, selection: Selection, shape: tuple[int, ...], chunk_grid: ChunkGrid) -> None:
dim_grids = chunk_grid._dimensions
Expand Down Expand Up @@ -996,6 +1021,38 @@ def __init__(self, selection: Selection, shape: tuple[int, ...], chunk_grid: Chu
object.__setattr__(self, "shape", shape)
object.__setattr__(self, "is_advanced", is_advanced)
object.__setattr__(self, "drop_axes", drop_axes)
object.__setattr__(self, "selection", selection)

def without_repeats(
self, shape: tuple[int, ...], chunk_grid: ChunkGrid
) -> tuple[OrthogonalIndexer, tuple[tuple[int, npt.NDArray[np.intp]], ...]] | None:
"""Collapse repeated integer indices, keeping the last occurrence along each dimension.

Returns the collapsed indexer and, for each output axis that shrank, the positions
kept along it, so that a value array can be reduced to match. Returns None when no
dimension repeats an index. Negative and positive spellings of the same index count
as repeats.
"""
selection = list(self.selection)
kept: list[tuple[int, npt.NDArray[np.intp]]] = []
out_axis = 0
for dim, (dim_sel, dim_indexer, dim_len) in enumerate(
zip(self.selection, self.dim_indexers, shape, strict=True)
):
if isinstance(dim_indexer, IntDimIndexer):
continue
if isinstance(dim_indexer, IntArrayDimIndexer):
normalized = wraparound_indices(
np.asarray(dim_sel).astype(np.intp, copy=False), dim_len
)
keep = last_occurrences(normalized)
if keep is not None:
selection[dim] = normalized[keep]
kept.append((out_axis, keep))
out_axis += 1
if not kept:
return None
return OrthogonalIndexer(tuple(selection), shape, chunk_grid), tuple(kept)

def __iter__(self) -> Iterator[ChunkProjection]:
for dim_projections in itertools.product(*self.dim_indexers):
Expand Down Expand Up @@ -1372,6 +1429,39 @@ def __init__(
object.__setattr__(self, "shape", shape)
object.__setattr__(self, "drop_axes", ())

def without_repeats(
self, shape: tuple[int, ...], chunk_grid: ChunkGrid
) -> tuple[CoordinateIndexer, npt.NDArray[np.intp]] | None:
"""Collapse repeated points, keeping the last occurrence of each.

Returns the collapsed indexer, whose shape is one-dimensional, and the positions kept
in the flattened selection, so that a value array can be reduced to match. Returns
None when no point is repeated.
"""
if shape == ():
return None
# self.selection is flat and non-negative, grouped by chunk; sel_sort maps each entry
# back to its position in the selection as given, which decides what "last" means.
# Points are compared coordinate by coordinate rather than through a raveled index,
# which could overflow for arrays with more than 2**63 elements.
n = self.selection[0].size
if n < 2:
return None
positions = np.arange(n, dtype=np.intp) if self.sel_sort is None else self.sel_sort
# lexsort uses the last key as the primary one: sort by point, then by position
order = np.lexsort((positions, *reversed(self.selection)))
same_as_next = np.ones(n - 1, dtype=bool)
for dim_sel in self.selection:
ordered = dim_sel[order]
same_as_next &= ordered[1:] == ordered[:-1]
if not same_as_next.any():
return None
is_last = np.append(~same_as_next, True)
keep = np.sort(order[is_last])
kept_positions = positions[keep]
selection = tuple(dim_sel[keep] for dim_sel in self.selection)
return CoordinateIndexer(selection, shape, chunk_grid), kept_positions

def __getattr__(self, name: str) -> npt.NDArray[np.intp]:
if name != "chunk_nitems_cumsum":
raise AttributeError(f"{type(self).__name__!r} object has no attribute {name!r}")
Expand Down Expand Up @@ -1434,6 +1524,12 @@ def __init__(
# delegate the rest to superclass
super().__init__(selection_indices, shape, chunk_grid)

def without_repeats(
self, shape: tuple[int, ...], chunk_grid: ChunkGrid
) -> tuple[CoordinateIndexer, npt.NDArray[np.intp]] | None:
# a mask selects each point at most once
return None


@dataclass(frozen=True)
class VIndex:
Expand Down
140 changes: 130 additions & 10 deletions tests/test_indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
_ArrayIndexingOrder,
_iter_grid,
_iter_regions,
last_occurrences,
make_slice_selection,
normalize_integer_selection,
oindex,
Expand Down Expand Up @@ -507,17 +508,136 @@ def test_setitem_zarr_array_as_value() -> None:
assert_array_equal(dst2[2:7], np.arange(2, 7))


@pytest.mark.skip(reason="fails on ubuntu, windows; numpy=2.2; in CI")
def test_setitem_repeated_index():
"""oindex assignment with repeated indices writes the last value for each duplicated index position."""
array = zarr.array(data=np.zeros((4,)), chunks=(1,))
indexer = np.array([-1, -1, 0, 0])
array.oindex[(indexer,)] = [0, 1, 2, 3]
np.testing.assert_array_equal(array[:], np.array([3, 0, 0, 1]))
@pytest.mark.parametrize("chunks", [(1,), (2,), (4,)])
@pytest.mark.parametrize("via", ["oindex", "vindex", "getitem"])
def test_setitem_repeated_index(chunks: tuple[int, ...], via: str) -> None:
"""Assignment with repeated indices writes the last value given for each repeated position, as NumPy does."""
array = zarr.zeros((4,), chunks=chunks, dtype="f8")
expected = np.zeros((4,))
for indexer, value in [
(np.array([-1, -1, 0, 0]), [0, 1, 2, 3]),
(np.array([-1, 0, 0, -1]), [0, 1, 2, 3]),
(np.array([3, 3, 3]), [7, 8, 9]),
]:
if via == "oindex":
array.oindex[(indexer,)] = value
elif via == "vindex":
array.vindex[(indexer,)] = value
else:
array[indexer] = value
expected[indexer] = value
assert_array_equal(array[:], expected)


def test_set_orthogonal_selection_repeated_index_2d() -> None:
"""Repeats along both dimensions of an orthogonal selection resolve like NumPy's ix_ assignment."""
rows = np.array([0, 5, 0, -1, 2])
cols = np.array([1, 1, 4, 4, 2])
value = np.arange(1, 26).reshape(5, 5)
expected = np.zeros((6, 6), dtype="i8")
expected[np.ix_(rows, cols)] = value
for chunks in [(2, 3), (6, 6), (1, 1)]:
array = zarr.zeros((6, 6), chunks=chunks, dtype="i8")
array.oindex[rows, cols] = value
assert_array_equal(array[:], expected)

# a scalar is written once per selected element whatever the repeats
array = zarr.zeros((6, 6), chunks=(2, 3), dtype="i8")
array.oindex[rows, cols] = 7
expected = np.zeros((6, 6), dtype="i8")
expected[np.ix_(rows, cols)] = 7
assert_array_equal(array[:], expected)

# repeats combined with an integer, a slice and a Boolean array along other dimensions
for selection, np_selection in [
((3, cols), (3, cols)),
((slice(1, 4), cols), np.ix_(np.arange(1, 4), cols)),
((np.array([1, 0, 1, 1, 0, 0], dtype=bool), cols), np.ix_([0, 2, 3], cols)),
]:
array = zarr.zeros((6, 6), chunks=(2, 3), dtype="i8")
expected = np.zeros((6, 6), dtype="i8")
value = np.arange(1, 1 + array.oindex[selection].size).reshape(
array.oindex[selection].shape
)
array.oindex[selection] = value
expected[np_selection] = value
assert_array_equal(array[:], expected)


def test_set_coordinate_selection_repeated_points() -> None:
"""Repeated points in a coordinate selection keep the last value, including across chunk boundaries."""
rows = np.array([0, 5, 0, -1, 2, 0])
cols = np.array([1, 2, 1, 2, 2, -5])
value = np.arange(1, 7)
expected = np.zeros((6, 6), dtype="i8")
expected[rows, cols] = value
for chunks in [(2, 3), (6, 6), (1, 1)]:
array = zarr.zeros((6, 6), chunks=chunks, dtype="i8")
array.vindex[rows, cols] = value
assert_array_equal(array[:], expected)

# multidimensional coordinate arrays and broadcasting between them
rows2 = np.array([[0, 5], [0, 5]])
cols2 = np.array([[1, 2], [1, 2]])
value2 = np.array([[1, 2], [3, 4]])
array = zarr.zeros((6, 6), chunks=(2, 3), dtype="i8")
array.vindex[rows2, cols2] = value2
expected = np.zeros((6, 6), dtype="i8")
expected[rows2, cols2] = value2
assert_array_equal(array[:], expected)

array = zarr.zeros((6, 6), chunks=(2, 3), dtype="i8")
array.vindex[np.array([0, 0, 5]), 2] = np.arange(3)
expected = np.zeros((6, 6), dtype="i8")
expected[np.array([0, 0, 5]), 2] = np.arange(3)
assert_array_equal(array[:], expected)

# the sorted single-array path
coords = np.repeat(np.arange(0, 200, 7), 3)
value = np.arange(coords.size)
array = zarr.zeros((200,), chunks=(10,), dtype="i8")
array.vindex[coords] = value
expected = np.zeros((200,), dtype="i8")
expected[coords] = value
assert_array_equal(array[:], expected)


def test_set_coordinate_selection_repeated_points_huge_array() -> None:
"""Repeated points are found without raveling the coordinates, which would overflow
for arrays with more than 2**63 elements."""
n = 2**33
z = zarr.create_array(store={}, shape=(n, n), chunks=(256, 256), dtype="i4", fill_value=0)
z.vindex[np.array([5, n - 1, 5]), np.array([7, n - 2, 7])] = np.array([1, 2, 3], dtype="i4")
assert_array_equal(z.vindex[np.array([5, n - 1]), np.array([7, n - 2])], [3, 2])


def test_setitem_repeated_index_value_shape_mismatch() -> None:
"""A value that does not fit the selection is rejected rather than written in part."""
array = zarr.zeros((7,), chunks=(3,), dtype="i8")
with pytest.raises(ValueError, match="shape mismatch"):
array.oindex[np.array([0, 0, 1])] = np.arange(5)
assert_array_equal(array[:], np.zeros((7,)))

indexer = np.array([-1, 0, 0, -1])
array.oindex[(indexer,)] = [0, 1, 2, 3]
np.testing.assert_array_equal(array[:], np.array([2, 0, 0, 3]))

@pytest.mark.parametrize(
("indices", "expected"),
[
([], None),
([3], None),
([0, 1, 2], None),
([2, 1, 0], None),
([0, 0], [1]),
([0, 0, 3, 3, 3, 6], [1, 4, 5]),
([6, 6, 3, 1, 1], [1, 2, 4]),
([2, 0, 2, 1, 0], [2, 3, 4]),
],
)
def test_last_occurrences(indices: list[int], expected: list[int] | None) -> None:
result = last_occurrences(np.array(indices, dtype=np.intp))
if expected is None:
assert result is None
else:
assert_array_equal(result, np.array(expected, dtype=np.intp))


Index = list[int] | tuple[slice | int | list[int], ...]
Expand Down
24 changes: 4 additions & 20 deletions tests/test_properties.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

import hypothesis.extra.numpy as npst
import hypothesis.strategies as st
from hypothesis import assume, event, given, settings
from hypothesis import event, given, settings

from tests.conftest import declared_chunk_data_sizes
from zarr.abc.store import Store
Expand Down Expand Up @@ -199,12 +199,8 @@ async def test_oindex(data: st.DataObject) -> None:
actual = await async_zarray.oindex.getitem(zindexer)
assert_array_equal(nparray[npindexer], actual)

# sync set
for idxr, size in zip(zindexer, nparray.shape, strict=True):
if isinstance(idxr, np.ndarray) and idxr.size != np.unique(idxr % size).size:
# behaviour of setitem with repeated indices is not guaranteed in practice
# Negative and positive spellings of the same index are duplicates too.
assume(False)
# sync set: an index that repeats a position (in either spelling) keeps the last value
# written to it, as in NumPy, so repeated indices are drawn like any other
# The sharding codec sees a coordinate selection (the GH4284 path) when the
# chunk selection has more than one array axis or drops an integer axis.
n_array_axes = sum(isinstance(idxr, np.ndarray) for idxr in zindexer)
Expand Down Expand Up @@ -248,19 +244,7 @@ async def test_vindex(data: st.DataObject) -> None:
actual = await async_zarray.vindex.getitem(indexer)
assert_array_equal(nparray[indexer], actual)

# sync set
# Reads preserve the supplied indices; normalize negative indices explicitly when
# detecting repeated points rather than relying on a read to mutate the indexer.
points = np.stack(
[
(idxr % size).ravel()
for idxr, size in zip(np.broadcast_arrays(*indexer), nparray.shape, strict=True)
],
axis=-1,
)
if len(np.unique(points, axis=0)) != len(points):
# behaviour of setitem with repeated coordinates is not guaranteed in practice
assume(False)
# sync set: a repeated point keeps the last value written to it, as in NumPy
new_data = data.draw(numpy_arrays(shapes=st.just(actual.shape), dtype=nparray.dtype))
nparray[indexer] = new_data
zarray.vindex[indexer] = new_data
Expand Down
Loading