Skip to content

Support blosc 'shuffle' of -1 in v2 metadata #102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
normanrz merged 2 commits into from
Sep 9, 2026
Merged

Support blosc 'shuffle' of -1 in v2 metadata #102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
118bfa2
Support blosc 'shuffle' of -1 in v2 metadata
stefanhahmann Sep 9, 2026
1855e7c
Derive auto shuffle from the item size
normanrz Sep 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 57 additions & 27 deletions src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,9 @@
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import com.scalableminds.bloscjava.Blosc;
import dev.zarr.zarrjava.ZarrException;
Expand All @@ -23,6 +20,12 @@

public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec implements Codec {

/**
* Value of 'shuffle' that lets Blosc pick the shuffle variant at write time: bit shuffle for
* single-byte items, byte shuffle otherwise.
*/
private static final int AUTO_SHUFFLE = -1;

@JsonIgnore
public final String id = "blosc";

Expand All @@ -36,27 +39,71 @@ public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec imp
public final int typesize;
public final int blocksize;

/**
* True if 'shuffle' was given as {@link #AUTO_SHUFFLE} and the variant still needs to be
* derived from the item size.
*/
@JsonIgnore
public final boolean autoShuffle;

@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
public BloscCodec(
BloscCodec(
@Nonnull @JsonProperty(value = "cname", defaultValue = "zstd")
@JsonDeserialize(using = CustomCompressorDeserializer.class)
Blosc.Compressor cname,
@Nonnull @JsonProperty(value = "shuffle", defaultValue = "noshuffle")
@JsonDeserialize(using = CustomShuffleDeserializer.class) Blosc.Shuffle shuffle,
@JsonProperty(value = "shuffle", defaultValue = "0") int shuffle,
@JsonProperty(value = "clevel", defaultValue = "5") int clevel,
@JsonProperty(value = "typesize", defaultValue = "0") int typesize,
@JsonProperty(value = "blocksize", defaultValue = "0") int blocksize
) throws ZarrException {
this(cname, parseShuffle(shuffle, typesize), shuffle == AUTO_SHUFFLE, clevel, typesize,
blocksize
);
}

public BloscCodec(
@Nonnull Blosc.Compressor cname, @Nonnull Blosc.Shuffle shuffle, int clevel,
int typesize, int blocksize
) throws ZarrException {
this(cname, shuffle, false, clevel, typesize, blocksize);
}

private BloscCodec(
@Nonnull Blosc.Compressor cname, @Nonnull Blosc.Shuffle shuffle, boolean autoShuffle,
int clevel, int typesize, int blocksize
) throws ZarrException {
if (clevel < 0 || clevel > 9) {
throw new ZarrException("'clevel' needs to be between 0 and 9.");
}
this.cname = cname;
this.shuffle = shuffle;
this.autoShuffle = autoShuffle;
this.clevel = clevel;
this.typesize = typesize;
this.blocksize = blocksize;
}

private static Blosc.Shuffle parseShuffle(int shuffle, int typesize) throws ZarrException {
if (shuffle == AUTO_SHUFFLE) {
return autoShuffle(typesize);
}
Blosc.Shuffle parsedShuffle = Blosc.Shuffle.fromInt(shuffle);
if (parsedShuffle == null) {
throw new ZarrException(
String.format("Could not parse the Blosc.Shuffle. Got '%d'", shuffle));
}
return parsedShuffle;
}

/**
* Blosc bit-shuffles single-byte items and byte-shuffles everything else. A 'typesize' of 0
* means that the item size is not known yet, in which case the variant is derived again in
* {@link #evolveFromCoreArrayMetadata}.
*/
private static Blosc.Shuffle autoShuffle(int typesize) {
return typesize == 1 ? Blosc.Shuffle.BIT_SHUFFLE : Blosc.Shuffle.BYTE_SHUFFLE;
}

@Override
public ByteBuffer encode(ByteBuffer chunkBytes)
throws ZarrException {
Expand All @@ -74,11 +121,13 @@ public ByteBuffer encode(ByteBuffer chunkBytes)
@Override
public BloscCodec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException {
if (typesize == 0) {
int evolvedTypesize = arrayMetadata.dataType.getByteCount();
return new BloscCodec(
this.cname,
this.shuffle,
this.autoShuffle ? autoShuffle(evolvedTypesize) : this.shuffle,
this.autoShuffle,
this.clevel,
arrayMetadata.dataType.getByteCount(),
evolvedTypesize,
this.blocksize
);
}
Expand All @@ -102,23 +151,4 @@ public void serialize(Blosc.Shuffle shuffle, JsonGenerator generator,
generator.writeNumber(shuffle.ordinal());
}
}

public static final class CustomShuffleDeserializer extends StdDeserializer<Blosc.Shuffle> {

public CustomShuffleDeserializer() {
this(null);
}

public CustomShuffleDeserializer(Class<?> vc) {
super(vc);
}

@Override
public Blosc.Shuffle deserialize(JsonParser jsonParser, DeserializationContext ctxt)
throws IOException {
int shuffle = jsonParser.getCodec()
.readValue(jsonParser, int.class);
return Blosc.Shuffle.values()[shuffle];
}
}
}
51 changes: 51 additions & 0 deletions src/test/java/dev/zarr/zarrjava/ZarrV2Test.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.scalableminds.bloscjava.Blosc;
import dev.zarr.zarrjava.core.Attributes;
import dev.zarr.zarrjava.v2.codec.core.BloscCodec;
import dev.zarr.zarrjava.store.FilesystemStore;
import dev.zarr.zarrjava.store.MemoryStore;
import dev.zarr.zarrjava.store.StoreHandle;
Expand Down Expand Up @@ -57,6 +59,55 @@ public void testCreateBlosc(String cname, String shuffle, int clevel) throws IOE
Assertions.assertEquals(0, outArray.getByte(0));
}

/**
* Blosc may be configured with a 'shuffle' of -1, meaning that it picks the shuffle variant
* itself when the chunks are written: bit shuffle for single-byte items, byte shuffle
* otherwise. Other implementations write that value to the metadata, so it needs to be
* readable here. They also tend to omit 'typesize', so the item size is only known once the
* data type has been read.
*/
@ParameterizedTest
@CsvSource({"UINT8,BIT_SHUFFLE", "UINT16,BYTE_SHUFFLE"})
public void testReadBloscAutoShuffle(String dataType, String expectedShuffle)
throws IOException, ZarrException {
String arrayName = "v2_blosc_autoshuffle_" + dataType;
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve(arrayName);
DataType zarrDataType = DataType.valueOf(dataType);
ucar.ma2.Array testData =
ucar.ma2.Array.factory(zarrDataType.getMA2DataType(), new int[]{16, 16});
for (int i = 0; i < testData.getSize(); i++) {
testData.setInt(i, i % 128);
}
Array array = Array.create(
storeHandle,
Array.metadataBuilder()
.withShape(16, 16)
.withDataType(zarrDataType)
.withChunks(8, 8)
.withBloscCompressor("zstd", "shuffle", 5)
.build()
);
array.write(testData);

Path zarrayPath = TESTOUTPUT.resolve(arrayName).resolve(ZARRAY);
String zarray = new String(Files.readAllBytes(zarrayPath));
String autoShuffleZarray = zarray
.replaceFirst("\"shuffle\"\\s*:\\s*1", "\"shuffle\": -1")
.replaceFirst("\"typesize\"\\s*:\\s*\\d+\\s*,\\s*", "");
Assertions.assertTrue(autoShuffleZarray.contains("\"shuffle\": -1"), zarray);
Assertions.assertFalse(autoShuffleZarray.contains("typesize"), zarray);
Files.write(zarrayPath, autoShuffleZarray.getBytes());

Array reopenedArray = Array.open(storeHandle);
Assertions.assertEquals(
Blosc.Shuffle.valueOf(expectedShuffle),
((BloscCodec) reopenedArray.metadata().compressor).shuffle
);
ucar.ma2.Array readData = reopenedArray.read();
Assertions.assertEquals(16 * 16, readData.getSize());
Assertions.assertEquals(127, readData.getInt(127));
}

@ParameterizedTest
@CsvSource({
"BOOL", "FLOAT64"
Expand Down
Loading