From a30680ed06d643891b9c86598777c5072a3fc304 Mon Sep 17 00:00:00 2001 From: Zihan Dai Date: Wed, 11 Nov 2026 13:59:24 +1100 Subject: [PATCH 1/2] fix: migrate v2 arrays compressed with Blosc AUTOSHUFFLE numcodecs stores Blosc(shuffle=AUTOSHUFFLE) as "shuffle": -1 and picks bit-shuffle for 1-byte items and byte-shuffle otherwise when it compresses a chunk. `zarr migrate v3` passed -1 to BloscShuffle.from_int, which fails with "Value must be between 0 and 2. Got -1." Map AUTOSHUFFLE to the shuffle numcodecs uses for the array's item size. --- src/zarr/metadata/migrate_v3.py | 9 ++++++-- tests/test_cli/test_migrate_v3.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/zarr/metadata/migrate_v3.py b/src/zarr/metadata/migrate_v3.py index ad177e19fb..9a45933b9b 100644 --- a/src/zarr/metadata/migrate_v3.py +++ b/src/zarr/metadata/migrate_v3.py @@ -236,11 +236,16 @@ def _convert_compressor( ) -> BytesBytesCodec: match compressor.codec_id: case "blosc": + typesize = dtype.to_native_dtype().itemsize + shuffle = compressor.shuffle + if shuffle == numcodecs.Blosc.AUTOSHUFFLE: + # numcodecs bit-shuffles 1-byte items and byte-shuffles larger ones + shuffle = numcodecs.Blosc.BITSHUFFLE if typesize == 1 else numcodecs.Blosc.SHUFFLE return BloscCodec( - typesize=dtype.to_native_dtype().itemsize, + typesize=typesize, cname=compressor.cname, clevel=compressor.clevel, - shuffle=BloscShuffle.from_int(compressor.shuffle), + shuffle=BloscShuffle.from_int(shuffle), blocksize=compressor.blocksize, ) diff --git a/tests/test_cli/test_migrate_v3.py b/tests/test_cli/test_migrate_v3.py index 83401487db..83df70a9c2 100644 --- a/tests/test_cli/test_migrate_v3.py +++ b/tests/test_cli/test_migrate_v3.py @@ -313,6 +313,43 @@ def test_migrate_compressor( assert np.all(zarr_array[:] == 1) +@pytest.mark.parametrize( + ("dtype", "expected_shuffle"), + [("uint8", "bitshuffle"), (" None: + """Blosc AUTOSHUFFLE becomes the shuffle that numcodecs uses for the item size""" + zarr_array = zarr.create_array( + store=local_store, + shape=(10, 10), + chunks=(10, 10), + dtype=dtype, + compressors=numcodecs.Blosc(cname="zstd", clevel=3, shuffle=numcodecs.Blosc.AUTOSHUFFLE), + zarr_format=2, + fill_value=0, + ) + zarr_array[:] = 1 + + result = runner.invoke(cli.app, ["migrate", "v3", str(local_store.root)]) + assert result.exit_code == 0 + assert (local_store.root / "zarr.json").exists() + + zarr_array = zarr.open_array(local_store.root, zarr_format=3) + metadata = zarr_array.metadata + assert metadata.zarr_format == 3 + assert metadata.codecs[-1] == BloscCodec( + typesize=np.dtype(dtype).itemsize, + cname="zstd", + clevel=3, + shuffle=expected_shuffle, + blocksize=0, + ) + assert np.all(zarr_array[:] == 1) + + def test_migrate_numcodecs_compressor(local_store: LocalStore) -> None: """Test migration of a numcodecs compressor without a zarr.codecs equivalent.""" From ba8e41f15b469a8ceb324f38314d8678432b660d Mon Sep 17 00:00:00 2001 From: Zihan Dai Date: Mon, 16 Nov 2026 14:59:28 +1100 Subject: [PATCH 2/2] Add changelog entry --- changes/4630.bugfix.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changes/4630.bugfix.md diff --git a/changes/4630.bugfix.md b/changes/4630.bugfix.md new file mode 100644 index 0000000000..6f40942920 --- /dev/null +++ b/changes/4630.bugfix.md @@ -0,0 +1,5 @@ +`zarr migrate v3` now migrates Zarr format 2 arrays compressed with +`Blosc(shuffle=Blosc.AUTOSHUFFLE)`. The v3 Blosc codec gets the shuffle that +numcodecs used for the array's item size: `bitshuffle` for 1-byte items and +`shuffle` otherwise. Before, the migration failed with +`ValueError: Value must be between 0 and 2. Got -1.`