Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions changes/4630.bugfix.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
`zarr migrate v3` now migrates Zarr format 2 arrays compressed with
`Blosc(shuffle=Blosc.AUTOSHUFFLE)`. The v3 Blosc codec gets the shuffle that
numcodecs used for the array's item size: `bitshuffle` for 1-byte items and
`shuffle` otherwise. Before, the migration failed with
`ValueError: Value must be between 0 and 2. Got -1.`
9 changes: 7 additions & 2 deletions src/zarr/metadata/migrate_v3.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,11 +242,16 @@ def _convert_compressor(
) -> BytesBytesCodec:
match compressor.codec_id:
case "blosc":
typesize = dtype.to_native_dtype().itemsize
shuffle = compressor.shuffle
if shuffle == numcodecs.Blosc.AUTOSHUFFLE:
# numcodecs bit-shuffles 1-byte items and byte-shuffles larger ones
shuffle = numcodecs.Blosc.BITSHUFFLE if typesize == 1 else numcodecs.Blosc.SHUFFLE
return BloscCodec(
typesize=dtype.to_native_dtype().itemsize,
typesize=typesize,
cname=compressor.cname,
clevel=compressor.clevel,
shuffle=BloscShuffle.from_int(compressor.shuffle),
shuffle=BloscShuffle.from_int(shuffle),
blocksize=compressor.blocksize,
)

Expand Down
37 changes: 37 additions & 0 deletions tests/test_cli/test_migrate_v3.py
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,43 @@ def test_migrate_compressor(
assert np.all(zarr_array[:] == 1)


@pytest.mark.parametrize(
("dtype", "expected_shuffle"),
[("uint8", "bitshuffle"), ("<u2", "shuffle")],
ids=["single_byte", "multi_byte"],
)
def test_migrate_blosc_autoshuffle(
local_store: LocalStore, dtype: str, expected_shuffle: Literal["bitshuffle", "shuffle"]
) -> None:
"""Blosc AUTOSHUFFLE becomes the shuffle that numcodecs uses for the item size"""
zarr_array = zarr.create_array(
store=local_store,
shape=(10, 10),
chunks=(10, 10),
dtype=dtype,
compressors=numcodecs.Blosc(cname="zstd", clevel=3, shuffle=numcodecs.Blosc.AUTOSHUFFLE),
zarr_format=2,
fill_value=0,
)
zarr_array[:] = 1

result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)])
assert result.exit_code == 0
assert (local_store.root / "zarr.json").exists()

zarr_array = zarr.open_array(local_store.root, zarr_format=3)
metadata = zarr_array.metadata
assert metadata.zarr_format == 3
assert metadata.codecs[-1] == BloscCodec(
typesize=np.dtype(dtype).itemsize,
cname="zstd",
clevel=3,
shuffle=expected_shuffle,
blocksize=0,
)
assert np.all(zarr_array[:] == 1)


def test_migrate_numcodecs_compressor(local_store: LocalStore) -> None:
"""Test migration of a numcodecs compressor without a zarr.codecs equivalent."""

Expand Down
Loading