From 118bfa21b273fdea1e8e579c17a3f51267c35cbf Mon Sep 17 00:00:00 2001 From: hahmann Date: Wed, 9 Sep 2026 15:23:34 +0200 Subject: [PATCH 1/2] Support blosc 'shuffle' of -1 in v2 metadata Co-Authored-By: Claude Opus 5 (1M context) --- .../zarrjava/v2/codec/core/BloscCodec.java | 18 +++++++- .../java/dev/zarr/zarrjava/ZarrV2Test.java | 41 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java index 487bb95f..985a1793 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.SerializerProvider; @@ -23,6 +24,11 @@ public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec implements Codec { + /** + * Value of 'shuffle' that lets Blosc pick the shuffle variant at write time. + */ + private static final int AUTO_SHUFFLE = -1; + @JsonIgnore public final String id = "blosc"; @@ -118,7 +124,17 @@ public Blosc.Shuffle deserialize(JsonParser jsonParser, DeserializationContext c throws IOException { int shuffle = jsonParser.getCodec() .readValue(jsonParser, int.class); - return Blosc.Shuffle.values()[shuffle]; + if (shuffle == AUTO_SHUFFLE) { + return Blosc.Shuffle.BYTE_SHUFFLE; + } + Blosc.Shuffle parsedShuffle = Blosc.Shuffle.fromInt(shuffle); + if (parsedShuffle == null) { + throw new JsonParseException( + jsonParser, + String.format("Could not parse the Blosc.Shuffle. Got '%d'", shuffle) + ); + } + return parsedShuffle; } } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index bb0f5a53..2ae491b5 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -2,7 +2,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.core.Attributes; +import dev.zarr.zarrjava.v2.codec.core.BloscCodec; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.MemoryStore; import dev.zarr.zarrjava.store.StoreHandle; @@ -57,6 +59,45 @@ public void testCreateBlosc(String cname, String shuffle, int clevel) throws IOE Assertions.assertEquals(0, outArray.getByte(0)); } + /** + * Blosc may be configured with a 'shuffle' of -1, meaning that it picks the shuffle variant + * itself when the chunks are written. Other implementations write that value to the metadata, + * so it needs to be readable here. + */ + @Test + public void testReadBloscAutoShuffle() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("v2_blosc_autoshuffle"); + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.USHORT, new int[]{16, 16}); + for (int i = 0; i < testData.getSize(); i++) { + testData.setShort(i, (short) i); + } + Array array = Array.create( + storeHandle, + Array.metadataBuilder() + .withShape(16, 16) + .withDataType(DataType.UINT16) + .withChunks(8, 8) + .withBloscCompressor("zstd", "shuffle", 5) + .build() + ); + array.write(testData); + + Path zarrayPath = TESTOUTPUT.resolve("v2_blosc_autoshuffle").resolve(ZARRAY); + String zarray = new String(Files.readAllBytes(zarrayPath)); + String autoShuffleZarray = zarray.replaceFirst("\"shuffle\"\\s*:\\s*1", "\"shuffle\": -1"); + Assertions.assertNotEquals(zarray, autoShuffleZarray, zarray); + Files.write(zarrayPath, autoShuffleZarray.getBytes()); + + Array reopenedArray = Array.open(storeHandle); + Assertions.assertEquals( + Blosc.Shuffle.BYTE_SHUFFLE, + ((BloscCodec) reopenedArray.metadata().compressor).shuffle + ); + ucar.ma2.Array readData = reopenedArray.read(); + Assertions.assertEquals(16 * 16, readData.getSize()); + Assertions.assertEquals(255, readData.getShort(255)); + } + @ParameterizedTest @CsvSource({ "BOOL", "FLOAT64" From 1855e7c53d633f02c07793b2cd3e42eeafc93db3 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Wed, 9 Sep 2026 15:35:12 +0200 Subject: [PATCH 2/2] Derive auto shuffle from the item size Blosc's -1 means bit shuffle for single-byte items and byte shuffle otherwise. Since numcodecs omits 'typesize' from the .zarray metadata, the item size is only known once the data type has been read, so the variant is derived again when the codec is evolved. Co-Authored-By: Claude Opus 5 --- .../zarrjava/v2/codec/core/BloscCodec.java | 92 +++++++++++-------- .../java/dev/zarr/zarrjava/ZarrV2Test.java | 36 +++++--- 2 files changed, 76 insertions(+), 52 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java index 985a1793..7e3992c1 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java @@ -4,13 +4,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParseException; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import com.fasterxml.jackson.databind.ser.std.StdSerializer; import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; @@ -25,7 +21,8 @@ public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec implements Codec { /** - * Value of 'shuffle' that lets Blosc pick the shuffle variant at write time. + * Value of 'shuffle' that lets Blosc pick the shuffle variant at write time: bit shuffle for + * single-byte items, byte shuffle otherwise. */ private static final int AUTO_SHUFFLE = -1; @@ -42,27 +39,71 @@ public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec imp public final int typesize; public final int blocksize; + /** + * True if 'shuffle' was given as {@link #AUTO_SHUFFLE} and the variant still needs to be + * derived from the item size. + */ + @JsonIgnore + public final boolean autoShuffle; + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public BloscCodec( + BloscCodec( @Nonnull @JsonProperty(value = "cname", defaultValue = "zstd") @JsonDeserialize(using = CustomCompressorDeserializer.class) Blosc.Compressor cname, - @Nonnull @JsonProperty(value = "shuffle", defaultValue = "noshuffle") - @JsonDeserialize(using = CustomShuffleDeserializer.class) Blosc.Shuffle shuffle, + @JsonProperty(value = "shuffle", defaultValue = "0") int shuffle, @JsonProperty(value = "clevel", defaultValue = "5") int clevel, @JsonProperty(value = "typesize", defaultValue = "0") int typesize, @JsonProperty(value = "blocksize", defaultValue = "0") int blocksize + ) throws ZarrException { + this(cname, parseShuffle(shuffle, typesize), shuffle == AUTO_SHUFFLE, clevel, typesize, + blocksize + ); + } + + public BloscCodec( + @Nonnull Blosc.Compressor cname, @Nonnull Blosc.Shuffle shuffle, int clevel, + int typesize, int blocksize + ) throws ZarrException { + this(cname, shuffle, false, clevel, typesize, blocksize); + } + + private BloscCodec( + @Nonnull Blosc.Compressor cname, @Nonnull Blosc.Shuffle shuffle, boolean autoShuffle, + int clevel, int typesize, int blocksize ) throws ZarrException { if (clevel < 0 || clevel > 9) { throw new ZarrException("'clevel' needs to be between 0 and 9."); } this.cname = cname; this.shuffle = shuffle; + this.autoShuffle = autoShuffle; this.clevel = clevel; this.typesize = typesize; this.blocksize = blocksize; } + private static Blosc.Shuffle parseShuffle(int shuffle, int typesize) throws ZarrException { + if (shuffle == AUTO_SHUFFLE) { + return autoShuffle(typesize); + } + Blosc.Shuffle parsedShuffle = Blosc.Shuffle.fromInt(shuffle); + if (parsedShuffle == null) { + throw new ZarrException( + String.format("Could not parse the Blosc.Shuffle. Got '%d'", shuffle)); + } + return parsedShuffle; + } + + /** + * Blosc bit-shuffles single-byte items and byte-shuffles everything else. A 'typesize' of 0 + * means that the item size is not known yet, in which case the variant is derived again in + * {@link #evolveFromCoreArrayMetadata}. + */ + private static Blosc.Shuffle autoShuffle(int typesize) { + return typesize == 1 ? Blosc.Shuffle.BIT_SHUFFLE : Blosc.Shuffle.BYTE_SHUFFLE; + } + @Override public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { @@ -80,11 +121,13 @@ public ByteBuffer encode(ByteBuffer chunkBytes) @Override public BloscCodec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { if (typesize == 0) { + int evolvedTypesize = arrayMetadata.dataType.getByteCount(); return new BloscCodec( this.cname, - this.shuffle, + this.autoShuffle ? autoShuffle(evolvedTypesize) : this.shuffle, + this.autoShuffle, this.clevel, - arrayMetadata.dataType.getByteCount(), + evolvedTypesize, this.blocksize ); } @@ -108,33 +151,4 @@ public void serialize(Blosc.Shuffle shuffle, JsonGenerator generator, generator.writeNumber(shuffle.ordinal()); } } - - public static final class CustomShuffleDeserializer extends StdDeserializer { - - public CustomShuffleDeserializer() { - this(null); - } - - public CustomShuffleDeserializer(Class vc) { - super(vc); - } - - @Override - public Blosc.Shuffle deserialize(JsonParser jsonParser, DeserializationContext ctxt) - throws IOException { - int shuffle = jsonParser.getCodec() - .readValue(jsonParser, int.class); - if (shuffle == AUTO_SHUFFLE) { - return Blosc.Shuffle.BYTE_SHUFFLE; - } - Blosc.Shuffle parsedShuffle = Blosc.Shuffle.fromInt(shuffle); - if (parsedShuffle == null) { - throw new JsonParseException( - jsonParser, - String.format("Could not parse the Blosc.Shuffle. Got '%d'", shuffle) - ); - } - return parsedShuffle; - } - } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 2ae491b5..2d7c4bcc 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -61,41 +61,51 @@ public void testCreateBlosc(String cname, String shuffle, int clevel) throws IOE /** * Blosc may be configured with a 'shuffle' of -1, meaning that it picks the shuffle variant - * itself when the chunks are written. Other implementations write that value to the metadata, - * so it needs to be readable here. + * itself when the chunks are written: bit shuffle for single-byte items, byte shuffle + * otherwise. Other implementations write that value to the metadata, so it needs to be + * readable here. They also tend to omit 'typesize', so the item size is only known once the + * data type has been read. */ - @Test - public void testReadBloscAutoShuffle() throws IOException, ZarrException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("v2_blosc_autoshuffle"); - ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.USHORT, new int[]{16, 16}); + @ParameterizedTest + @CsvSource({"UINT8,BIT_SHUFFLE", "UINT16,BYTE_SHUFFLE"}) + public void testReadBloscAutoShuffle(String dataType, String expectedShuffle) + throws IOException, ZarrException { + String arrayName = "v2_blosc_autoshuffle_" + dataType; + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve(arrayName); + DataType zarrDataType = DataType.valueOf(dataType); + ucar.ma2.Array testData = + ucar.ma2.Array.factory(zarrDataType.getMA2DataType(), new int[]{16, 16}); for (int i = 0; i < testData.getSize(); i++) { - testData.setShort(i, (short) i); + testData.setInt(i, i % 128); } Array array = Array.create( storeHandle, Array.metadataBuilder() .withShape(16, 16) - .withDataType(DataType.UINT16) + .withDataType(zarrDataType) .withChunks(8, 8) .withBloscCompressor("zstd", "shuffle", 5) .build() ); array.write(testData); - Path zarrayPath = TESTOUTPUT.resolve("v2_blosc_autoshuffle").resolve(ZARRAY); + Path zarrayPath = TESTOUTPUT.resolve(arrayName).resolve(ZARRAY); String zarray = new String(Files.readAllBytes(zarrayPath)); - String autoShuffleZarray = zarray.replaceFirst("\"shuffle\"\\s*:\\s*1", "\"shuffle\": -1"); - Assertions.assertNotEquals(zarray, autoShuffleZarray, zarray); + String autoShuffleZarray = zarray + .replaceFirst("\"shuffle\"\\s*:\\s*1", "\"shuffle\": -1") + .replaceFirst("\"typesize\"\\s*:\\s*\\d+\\s*,\\s*", ""); + Assertions.assertTrue(autoShuffleZarray.contains("\"shuffle\": -1"), zarray); + Assertions.assertFalse(autoShuffleZarray.contains("typesize"), zarray); Files.write(zarrayPath, autoShuffleZarray.getBytes()); Array reopenedArray = Array.open(storeHandle); Assertions.assertEquals( - Blosc.Shuffle.BYTE_SHUFFLE, + Blosc.Shuffle.valueOf(expectedShuffle), ((BloscCodec) reopenedArray.metadata().compressor).shuffle ); ucar.ma2.Array readData = reopenedArray.read(); Assertions.assertEquals(16 * 16, readData.getSize()); - Assertions.assertEquals(255, readData.getShort(255)); + Assertions.assertEquals(127, readData.getInt(127)); } @ParameterizedTest