diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java index 487bb95..7e3992c 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java @@ -4,12 +4,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import com.fasterxml.jackson.databind.ser.std.StdSerializer; import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; @@ -23,6 +20,12 @@ public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec implements Codec { + /** + * Value of 'shuffle' that lets Blosc pick the shuffle variant at write time: bit shuffle for + * single-byte items, byte shuffle otherwise. + */ + private static final int AUTO_SHUFFLE = -1; + @JsonIgnore public final String id = "blosc"; @@ -36,27 +39,71 @@ public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec imp public final int typesize; public final int blocksize; + /** + * True if 'shuffle' was given as {@link #AUTO_SHUFFLE} and the variant still needs to be + * derived from the item size. + */ + @JsonIgnore + public final boolean autoShuffle; + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public BloscCodec( + BloscCodec( @Nonnull @JsonProperty(value = "cname", defaultValue = "zstd") @JsonDeserialize(using = CustomCompressorDeserializer.class) Blosc.Compressor cname, - @Nonnull @JsonProperty(value = "shuffle", defaultValue = "noshuffle") - @JsonDeserialize(using = CustomShuffleDeserializer.class) Blosc.Shuffle shuffle, + @JsonProperty(value = "shuffle", defaultValue = "0") int shuffle, @JsonProperty(value = "clevel", defaultValue = "5") int clevel, @JsonProperty(value = "typesize", defaultValue = "0") int typesize, @JsonProperty(value = "blocksize", defaultValue = "0") int blocksize + ) throws ZarrException { + this(cname, parseShuffle(shuffle, typesize), shuffle == AUTO_SHUFFLE, clevel, typesize, + blocksize + ); + } + + public BloscCodec( + @Nonnull Blosc.Compressor cname, @Nonnull Blosc.Shuffle shuffle, int clevel, + int typesize, int blocksize + ) throws ZarrException { + this(cname, shuffle, false, clevel, typesize, blocksize); + } + + private BloscCodec( + @Nonnull Blosc.Compressor cname, @Nonnull Blosc.Shuffle shuffle, boolean autoShuffle, + int clevel, int typesize, int blocksize ) throws ZarrException { if (clevel < 0 || clevel > 9) { throw new ZarrException("'clevel' needs to be between 0 and 9."); } this.cname = cname; this.shuffle = shuffle; + this.autoShuffle = autoShuffle; this.clevel = clevel; this.typesize = typesize; this.blocksize = blocksize; } + private static Blosc.Shuffle parseShuffle(int shuffle, int typesize) throws ZarrException { + if (shuffle == AUTO_SHUFFLE) { + return autoShuffle(typesize); + } + Blosc.Shuffle parsedShuffle = Blosc.Shuffle.fromInt(shuffle); + if (parsedShuffle == null) { + throw new ZarrException( + String.format("Could not parse the Blosc.Shuffle. Got '%d'", shuffle)); + } + return parsedShuffle; + } + + /** + * Blosc bit-shuffles single-byte items and byte-shuffles everything else. A 'typesize' of 0 + * means that the item size is not known yet, in which case the variant is derived again in + * {@link #evolveFromCoreArrayMetadata}. + */ + private static Blosc.Shuffle autoShuffle(int typesize) { + return typesize == 1 ? Blosc.Shuffle.BIT_SHUFFLE : Blosc.Shuffle.BYTE_SHUFFLE; + } + @Override public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { @@ -74,11 +121,13 @@ public ByteBuffer encode(ByteBuffer chunkBytes) @Override public BloscCodec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { if (typesize == 0) { + int evolvedTypesize = arrayMetadata.dataType.getByteCount(); return new BloscCodec( this.cname, - this.shuffle, + this.autoShuffle ? autoShuffle(evolvedTypesize) : this.shuffle, + this.autoShuffle, this.clevel, - arrayMetadata.dataType.getByteCount(), + evolvedTypesize, this.blocksize ); } @@ -102,23 +151,4 @@ public void serialize(Blosc.Shuffle shuffle, JsonGenerator generator, generator.writeNumber(shuffle.ordinal()); } } - - public static final class CustomShuffleDeserializer extends StdDeserializer { - - public CustomShuffleDeserializer() { - this(null); - } - - public CustomShuffleDeserializer(Class vc) { - super(vc); - } - - @Override - public Blosc.Shuffle deserialize(JsonParser jsonParser, DeserializationContext ctxt) - throws IOException { - int shuffle = jsonParser.getCodec() - .readValue(jsonParser, int.class); - return Blosc.Shuffle.values()[shuffle]; - } - } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index bb0f5a5..2d7c4bc 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -2,7 +2,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.core.Attributes; +import dev.zarr.zarrjava.v2.codec.core.BloscCodec; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.MemoryStore; import dev.zarr.zarrjava.store.StoreHandle; @@ -57,6 +59,55 @@ public void testCreateBlosc(String cname, String shuffle, int clevel) throws IOE Assertions.assertEquals(0, outArray.getByte(0)); } + /** + * Blosc may be configured with a 'shuffle' of -1, meaning that it picks the shuffle variant + * itself when the chunks are written: bit shuffle for single-byte items, byte shuffle + * otherwise. Other implementations write that value to the metadata, so it needs to be + * readable here. They also tend to omit 'typesize', so the item size is only known once the + * data type has been read. + */ + @ParameterizedTest + @CsvSource({"UINT8,BIT_SHUFFLE", "UINT16,BYTE_SHUFFLE"}) + public void testReadBloscAutoShuffle(String dataType, String expectedShuffle) + throws IOException, ZarrException { + String arrayName = "v2_blosc_autoshuffle_" + dataType; + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve(arrayName); + DataType zarrDataType = DataType.valueOf(dataType); + ucar.ma2.Array testData = + ucar.ma2.Array.factory(zarrDataType.getMA2DataType(), new int[]{16, 16}); + for (int i = 0; i < testData.getSize(); i++) { + testData.setInt(i, i % 128); + } + Array array = Array.create( + storeHandle, + Array.metadataBuilder() + .withShape(16, 16) + .withDataType(zarrDataType) + .withChunks(8, 8) + .withBloscCompressor("zstd", "shuffle", 5) + .build() + ); + array.write(testData); + + Path zarrayPath = TESTOUTPUT.resolve(arrayName).resolve(ZARRAY); + String zarray = new String(Files.readAllBytes(zarrayPath)); + String autoShuffleZarray = zarray + .replaceFirst("\"shuffle\"\\s*:\\s*1", "\"shuffle\": -1") + .replaceFirst("\"typesize\"\\s*:\\s*\\d+\\s*,\\s*", ""); + Assertions.assertTrue(autoShuffleZarray.contains("\"shuffle\": -1"), zarray); + Assertions.assertFalse(autoShuffleZarray.contains("typesize"), zarray); + Files.write(zarrayPath, autoShuffleZarray.getBytes()); + + Array reopenedArray = Array.open(storeHandle); + Assertions.assertEquals( + Blosc.Shuffle.valueOf(expectedShuffle), + ((BloscCodec) reopenedArray.metadata().compressor).shuffle + ); + ucar.ma2.Array readData = reopenedArray.read(); + Assertions.assertEquals(16 * 16, readData.getSize()); + Assertions.assertEquals(127, readData.getInt(127)); + } + @ParameterizedTest @CsvSource({ "BOOL", "FLOAT64"