From f11fb8cf73d4da0706b4ea5a20c33a2c38830451 Mon Sep 17 00:00:00 2001 From: Satyawan Singh <196954366+ss1738@users.noreply.github.com> Date: Sun, 15 Nov 2026 00:19:24 +0530 Subject: [PATCH 1/2] fix(sharding): validate index entries before reads and partial writes --- changes/shard-index-validation.bugfix.md | 1 + src/zarr/codecs/sharding.py | 19 +++++++- tests/test_codecs/test_sharding.py | 59 ++++++++++++++++++++++++ tests/test_codecs/test_sharding_unit.py | 40 ++++++++++++++++ 4 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 changes/shard-index-validation.bugfix.md diff --git a/changes/shard-index-validation.bugfix.md b/changes/shard-index-validation.bugfix.md new file mode 100644 index 0000000000..b9e0a56afe --- /dev/null +++ b/changes/shard-index-validation.bugfix.md @@ -0,0 +1 @@ +Reject inconsistent missing-entry sentinels and overflowing byte ranges in shard indices before reading or updating a shard. Partial writes now raise instead of rewriting shards with these malformed entries. diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index d4ccf71cb9..e9372e3a8c 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -213,6 +213,17 @@ class _ShardIndex(NamedTuple): # dtype uint64, shape (chunks_per_shard_0, chunks_per_shard_1, ..., 2) offsets_and_lengths: npt.NDArray[np.uint64] + def validate(self) -> None: + """Reject inconsistent missing entries and ranges that overflow uint64.""" + entries = self.offsets_and_lengths.reshape(-1, 2) + offsets, lengths = entries[:, 0], entries[:, 1] + missing = offsets == MAX_UINT_64 + if np.any(missing != (lengths == MAX_UINT_64)): + raise ValueError("Invalid shard index: offset and length missing sentinels disagree.") + # Subtraction checks the range without overflowing unsigned addition. + if np.any((lengths > np.uint64(MAX_UINT_64) - offsets) & ~missing): + raise ValueError("Invalid shard index: offset plus length exceeds uint64.") + def _localize_chunk(self, chunk_coords: tuple[int, ...]) -> tuple[int, ...]: return tuple( chunk_i % shard_i @@ -700,7 +711,9 @@ def _decode_shard_index_sync( index_transform = self._get_index_chunk_transform(chunks_per_shard) index_spec = self._get_index_chunk_spec(chunks_per_shard) index_array = index_transform.decode_chunk(index_bytes, index_spec) - return _ShardIndex(chunks_per_shard, index_array.as_numpy_array()) + index = _ShardIndex(chunks_per_shard, index_array.as_numpy_array()) + index.validate() + return index def _encode_shard_index_sync(self, index: _ShardIndex) -> Buffer: """Encode shard index synchronously using ChunkTransform.""" @@ -1588,7 +1601,9 @@ async def _decode_shard_index( ) if index_array is None: raise RuntimeError("Decoding the shard index produced no array.") - return _ShardIndex(chunks_per_shard, index_array.as_numpy_array()) + index = _ShardIndex(chunks_per_shard, index_array.as_numpy_array()) + index.validate() + return index async def _encode_shard_index(self, index: _ShardIndex) -> Buffer: # Pure compute: delegate to the sync implementation, with the same diff --git a/tests/test_codecs/test_sharding.py b/tests/test_codecs/test_sharding.py index 5712c9576d..039a353f34 100644 --- a/tests/test_codecs/test_sharding.py +++ b/tests/test_codecs/test_sharding.py @@ -1437,6 +1437,65 @@ def test_sharding_truncated_shard_raises( ops[op]() +@pytest.mark.parametrize( + "pipeline_path", + [ + "zarr.core.codec_pipeline.BatchedCodecPipeline", + "zarr.core.codec_pipeline.FusedCodecPipeline", + ], +) +@pytest.mark.parametrize("store", ["memory", "local"], indirect=True) +@pytest.mark.parametrize("index_location", ["start", "end"]) +@pytest.mark.parametrize( + "entry", + [(MAX_UINT_64, 8), (0, MAX_UINT_64), (MAX_UINT_64 - 3, 8)], + ids=["missing-offset", "missing-length", "overflow"], +) +@pytest.mark.parametrize("operation", ["full read", "partial read", "partial write"]) +def test_sharding_invalid_index_preserves_store( + pipeline_path: str, + store: Store, + index_location: IndexLocation, + entry: tuple[int, int], + operation: str, +) -> None: + """Reject malformed entries before an unrelated partial write can rewrite the shard.""" + from zarr.core.sync import sync + + prototype = default_buffer_prototype() + codec = ShardingCodec(chunk_shape=(2,), codecs=[BytesCodec()], index_location=index_location) + with zarr.config.set({"codec_pipeline.path": pipeline_path}): + array = zarr.create_array( + store, shape=(4,), chunks=(4,), dtype="int32", serializer=codec, compressors=None + ) + array[:] = [11, 12, 13, 14] + stored = sync(store.get("c/0", prototype)) + assert stored is not None + index_size = codec._shard_index_size((2,)) + index_start = 0 if index_location == "start" else len(stored) - index_size + index = codec._decode_shard_index_sync(stored[index_start : index_start + index_size], (2,)) + index = _ShardIndex((2,), index.offsets_and_lengths.copy()) + index.offsets_and_lengths[0] = entry + # Encoding the mutated index also updates its checksum. + corrupt = bytearray(stored.to_bytes()) + corrupt[index_start : index_start + index_size] = codec._encode_shard_index_sync( + index + ).to_bytes() + sync(store.set("c/0", prototype.buffer.from_bytes(bytes(corrupt)))) + operations: dict[str, Callable[[], object]] = { + "full read": lambda: array[:], + "partial read": lambda: array[:2], + "partial write": lambda: array.__setitem__(slice(2, 4), [91, 92]), + } + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + with pytest.raises(ValueError, match="Invalid shard index"): + operations[operation]() + after = sync(store.get("c/0", prototype)) + assert after is not None + assert after.to_bytes() == bytes(corrupt) + + @pytest.mark.filterwarnings( "ignore:Combining a `sharding_indexed` codec:zarr.errors.ZarrUserWarning" ) diff --git a/tests/test_codecs/test_sharding_unit.py b/tests/test_codecs/test_sharding_unit.py index 978f7882f4..856df85284 100644 --- a/tests/test_codecs/test_sharding_unit.py +++ b/tests/test_codecs/test_sharding_unit.py @@ -780,6 +780,46 @@ def test_shard_index_async_fallback_for_async_only_index_codec() -> None: np.testing.assert_array_equal(decoded.offsets_and_lengths, index.offsets_and_lengths) +@pytest.mark.parametrize("decoder", ["sync", "async", "async-only"]) +@pytest.mark.parametrize( + "entry", + [(MAX_UINT_64, 8), (0, MAX_UINT_64), (MAX_UINT_64 - 3, 8)], + ids=["missing-offset", "missing-length", "overflow"], +) +def test_shard_index_decode_rejects_invalid_entries(decoder: str, entry: tuple[int, int]) -> None: + serializer = _AsyncOnlyBytesCodec() if decoder == "async-only" else BytesCodec() + codec = ShardingCodec(chunk_shape=(1,), index_codecs=[serializer, Crc32cCodec()]) + index = _ShardIndex.create_empty((2,)) + index.offsets_and_lengths[1] = entry + encoded = asyncio.run(codec._encode_shard_index(index)) + + def decode() -> _ShardIndex: + if decoder == "sync": + return codec._decode_shard_index_sync(encoded, (2,)) + return asyncio.run(codec._decode_shard_index(encoded, (2,))) + + with pytest.raises(ValueError, match="Invalid shard index"): + decode() + + +@pytest.mark.parametrize("decoder", ["sync", "async", "async-only"]) +def test_shard_index_validation_preserves_legal_entries(decoder: str) -> None: + serializer = _AsyncOnlyBytesCodec() if decoder == "async-only" else BytesCodec() + codec = ShardingCodec(chunk_shape=(1,), index_codecs=[serializer, Crc32cCodec()]) + # Sparse, unordered and zero-length entries are legal. The last range ends + # at UINT64_MAX without overflowing; validation does not impose object bounds. + values = np.array( + [[MAX_UINT_64, MAX_UINT_64], [32, 4], [8, 0], [MAX_UINT_64 - 1, 1]], dtype=" Date: Sun, 15 Nov 2026 00:20:51 +0530 Subject: [PATCH 2/2] docs: associate shard validation changelog with PR 4637 --- changes/{shard-index-validation.bugfix.md => 4637.bugfix.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changes/{shard-index-validation.bugfix.md => 4637.bugfix.md} (100%) diff --git a/changes/shard-index-validation.bugfix.md b/changes/4637.bugfix.md similarity index 100% rename from changes/shard-index-validation.bugfix.md rename to changes/4637.bugfix.md