diff --git a/changes/4630.bugfix.md b/changes/4630.bugfix.md new file mode 100644 index 0000000000..6f40942920 --- /dev/null +++ b/changes/4630.bugfix.md @@ -0,0 +1,5 @@ +`zarr migrate v3` now migrates Zarr format 2 arrays compressed with +`Blosc(shuffle=Blosc.AUTOSHUFFLE)`. The v3 Blosc codec gets the shuffle that +numcodecs used for the array's item size: `bitshuffle` for 1-byte items and +`shuffle` otherwise. Before, the migration failed with +`ValueError: Value must be between 0 and 2. Got -1.` diff --git a/src/zarr/metadata/migrate_v3.py b/src/zarr/metadata/migrate_v3.py index cd94f12d84..9bfb84d8d8 100644 --- a/src/zarr/metadata/migrate_v3.py +++ b/src/zarr/metadata/migrate_v3.py @@ -242,11 +242,16 @@ def _convert_compressor( ) -> BytesBytesCodec: match compressor.codec_id: case "blosc": + typesize = dtype.to_native_dtype().itemsize + shuffle = compressor.shuffle + if shuffle == numcodecs.Blosc.AUTOSHUFFLE: + # numcodecs bit-shuffles 1-byte items and byte-shuffles larger ones + shuffle = numcodecs.Blosc.BITSHUFFLE if typesize == 1 else numcodecs.Blosc.SHUFFLE return BloscCodec( - typesize=dtype.to_native_dtype().itemsize, + typesize=typesize, cname=compressor.cname, clevel=compressor.clevel, - shuffle=BloscShuffle.from_int(compressor.shuffle), + shuffle=BloscShuffle.from_int(shuffle), blocksize=compressor.blocksize, ) diff --git a/tests/test_cli/test_migrate_v3.py b/tests/test_cli/test_migrate_v3.py index 2707341e1d..1b2e5c78fc 100644 --- a/tests/test_cli/test_migrate_v3.py +++ b/tests/test_cli/test_migrate_v3.py @@ -313,6 +313,43 @@ def test_migrate_compressor( assert np.all(zarr_array[:] == 1) +@pytest.mark.parametrize( + ("dtype", "expected_shuffle"), + [("uint8", "bitshuffle"), (" None: + """Blosc AUTOSHUFFLE becomes the shuffle that numcodecs uses for the item size""" + zarr_array = zarr.create_array( + store=local_store, + shape=(10, 10), + chunks=(10, 10), + dtype=dtype, + compressors=numcodecs.Blosc(cname="zstd", clevel=3, shuffle=numcodecs.Blosc.AUTOSHUFFLE), + zarr_format=2, + fill_value=0, + ) + zarr_array[:] = 1 + + result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)]) + assert result.exit_code == 0 + assert (local_store.root / "zarr.json").exists() + + zarr_array = zarr.open_array(local_store.root, zarr_format=3) + metadata = zarr_array.metadata + assert metadata.zarr_format == 3 + assert metadata.codecs[-1] == BloscCodec( + typesize=np.dtype(dtype).itemsize, + cname="zstd", + clevel=3, + shuffle=expected_shuffle, + blocksize=0, + ) + assert np.all(zarr_array[:] == 1) + + def test_migrate_numcodecs_compressor(local_store: LocalStore) -> None: """Test migration of a numcodecs compressor without a zarr.codecs equivalent."""