Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/4637.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Reject inconsistent missing-entry sentinels and overflowing byte ranges in shard indices before reading or updating a shard. Partial writes now raise instead of rewriting shards with these malformed entries.
19 changes: 17 additions & 2 deletions src/zarr/codecs/sharding.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,17 @@ class _ShardIndex(NamedTuple):
# dtype uint64, shape (chunks_per_shard_0, chunks_per_shard_1, ..., 2)
offsets_and_lengths: npt.NDArray[np.uint64]

def validate(self) -> None:
"""Reject inconsistent missing entries and ranges that overflow uint64."""
entries = self.offsets_and_lengths.reshape(-1, 2)
offsets, lengths = entries[:, 0], entries[:, 1]
missing = offsets == MAX_UINT_64
if np.any(missing != (lengths == MAX_UINT_64)):
raise ValueError("Invalid shard index: offset and length missing sentinels disagree.")
# Subtraction checks the range without overflowing unsigned addition.
if np.any((lengths > np.uint64(MAX_UINT_64) - offsets) & ~missing):
raise ValueError("Invalid shard index: offset plus length exceeds uint64.")

def _localize_chunk(self, chunk_coords: tuple[int, ...]) -> tuple[int, ...]:
return tuple(
chunk_i % shard_i
Expand Down Expand Up @@ -700,7 +711,9 @@ def _decode_shard_index_sync(
index_transform = self._get_index_chunk_transform(chunks_per_shard)
index_spec = self._get_index_chunk_spec(chunks_per_shard)
index_array = index_transform.decode_chunk(index_bytes, index_spec)
return _ShardIndex(chunks_per_shard, index_array.as_numpy_array())
index = _ShardIndex(chunks_per_shard, index_array.as_numpy_array())
index.validate()
return index

def _encode_shard_index_sync(self, index: _ShardIndex) -> Buffer:
"""Encode shard index synchronously using ChunkTransform."""
Expand Down Expand Up @@ -1588,7 +1601,9 @@ async def _decode_shard_index(
)
if index_array is None:
raise RuntimeError("Decoding the shard index produced no array.")
return _ShardIndex(chunks_per_shard, index_array.as_numpy_array())
index = _ShardIndex(chunks_per_shard, index_array.as_numpy_array())
index.validate()
return index

async def _encode_shard_index(self, index: _ShardIndex) -> Buffer:
# Pure compute: delegate to the sync implementation, with the same
Expand Down
59 changes: 59 additions & 0 deletions tests/test_codecs/test_sharding.py
Original file line number Diff line number Diff line change
Expand Up @@ -1437,6 +1437,65 @@ def test_sharding_truncated_shard_raises(
ops[op]()


@pytest.mark.parametrize(
"pipeline_path",
[
"zarr.core.codec_pipeline.BatchedCodecPipeline",
"zarr.core.codec_pipeline.FusedCodecPipeline",
],
)
@pytest.mark.parametrize("store", ["memory", "local"], indirect=True)
@pytest.mark.parametrize("index_location", ["start", "end"])
@pytest.mark.parametrize(
"entry",
[(MAX_UINT_64, 8), (0, MAX_UINT_64), (MAX_UINT_64 - 3, 8)],
ids=["missing-offset", "missing-length", "overflow"],
)
@pytest.mark.parametrize("operation", ["full read", "partial read", "partial write"])
def test_sharding_invalid_index_preserves_store(
pipeline_path: str,
store: Store,
index_location: IndexLocation,
entry: tuple[int, int],
operation: str,
) -> None:
"""Reject malformed entries before an unrelated partial write can rewrite the shard."""
from zarr.core.sync import sync

prototype = default_buffer_prototype()
codec = ShardingCodec(chunk_shape=(2,), codecs=[BytesCodec()], index_location=index_location)
with zarr.config.set({"codec_pipeline.path": pipeline_path}):
array = zarr.create_array(
store, shape=(4,), chunks=(4,), dtype="int32", serializer=codec, compressors=None
)
array[:] = [11, 12, 13, 14]
stored = sync(store.get("c/0", prototype))
assert stored is not None
index_size = codec._shard_index_size((2,))
index_start = 0 if index_location == "start" else len(stored) - index_size
index = codec._decode_shard_index_sync(stored[index_start : index_start + index_size], (2,))
index = _ShardIndex((2,), index.offsets_and_lengths.copy())
index.offsets_and_lengths[0] = entry
# Encoding the mutated index also updates its checksum.
corrupt = bytearray(stored.to_bytes())
corrupt[index_start : index_start + index_size] = codec._encode_shard_index_sync(
index
).to_bytes()
sync(store.set("c/0", prototype.buffer.from_bytes(bytes(corrupt))))
operations: dict[str, Callable[[], object]] = {
"full read": lambda: array[:],
"partial read": lambda: array[:2],
"partial write": lambda: array.__setitem__(slice(2, 4), [91, 92]),
}
with warnings.catch_warnings():
warnings.simplefilter("error", RuntimeWarning)
with pytest.raises(ValueError, match="Invalid shard index"):
operations[operation]()
after = sync(store.get("c/0", prototype))
assert after is not None
assert after.to_bytes() == bytes(corrupt)


@pytest.mark.filterwarnings(
"ignore:Combining a `sharding_indexed` codec:zarr.errors.ZarrUserWarning"
)
Expand Down
40 changes: 40 additions & 0 deletions tests/test_codecs/test_sharding_unit.py
Original file line number Diff line number Diff line change
Expand Up @@ -780,6 +780,46 @@ def test_shard_index_async_fallback_for_async_only_index_codec() -> None:
np.testing.assert_array_equal(decoded.offsets_and_lengths, index.offsets_and_lengths)


@pytest.mark.parametrize("decoder", ["sync", "async", "async-only"])
@pytest.mark.parametrize(
"entry",
[(MAX_UINT_64, 8), (0, MAX_UINT_64), (MAX_UINT_64 - 3, 8)],
ids=["missing-offset", "missing-length", "overflow"],
)
def test_shard_index_decode_rejects_invalid_entries(decoder: str, entry: tuple[int, int]) -> None:
serializer = _AsyncOnlyBytesCodec() if decoder == "async-only" else BytesCodec()
codec = ShardingCodec(chunk_shape=(1,), index_codecs=[serializer, Crc32cCodec()])
index = _ShardIndex.create_empty((2,))
index.offsets_and_lengths[1] = entry
encoded = asyncio.run(codec._encode_shard_index(index))

def decode() -> _ShardIndex:
if decoder == "sync":
return codec._decode_shard_index_sync(encoded, (2,))
return asyncio.run(codec._decode_shard_index(encoded, (2,)))

with pytest.raises(ValueError, match="Invalid shard index"):
decode()


@pytest.mark.parametrize("decoder", ["sync", "async", "async-only"])
def test_shard_index_validation_preserves_legal_entries(decoder: str) -> None:
serializer = _AsyncOnlyBytesCodec() if decoder == "async-only" else BytesCodec()
codec = ShardingCodec(chunk_shape=(1,), index_codecs=[serializer, Crc32cCodec()])
# Sparse, unordered and zero-length entries are legal. The last range ends
# at UINT64_MAX without overflowing; validation does not impose object bounds.
values = np.array(
[[MAX_UINT_64, MAX_UINT_64], [32, 4], [8, 0], [MAX_UINT_64 - 1, 1]], dtype="<u8"
)
index = _ShardIndex((4,), values)
encoded = asyncio.run(codec._encode_shard_index(index))
if decoder == "sync":
decoded = codec._decode_shard_index_sync(encoded, (4,))
else:
decoded = asyncio.run(codec._decode_shard_index(encoded, (4,)))
np.testing.assert_array_equal(decoded.offsets_and_lengths, values)


# ============================================================================
# Coalescing config option tests
#
Expand Down
Loading