diff --git a/USERGUIDE.md b/USERGUIDE.md index 65bf9944..f150afb8 100644 --- a/USERGUIDE.md +++ b/USERGUIDE.md @@ -610,6 +610,25 @@ ucar.ma2.Array chunk = array.readChunk(chunkCoords); // Write single chunk array.writeChunk(chunkCoords, chunk); ``` +### Chunk Key Encodings (v3) +The chunk key encoding decides where a chunk is stored. Besides `default` (`c/0/1/2`) and `v2` (`0.1.2`), +zarr-java supports the [`template`](https://github.com/zarr-developers/zarr-extensions/tree/main/chunk-key-encodings/template) +encoding, which builds keys from a format string: +```java +Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(1024, 1024, 100) + .withDataType(DataType.UINT8) + .withChunkShape(256, 256, 1) + // chunk (1, 2, 7) is stored at "images/7/0001_0002.jpeg" + .withTemplateChunkKeyEncoding("images/{-1}/{*:04}.jpeg", "_") + .build()); +``` +- `{0}`, `{1}`, … insert a chunk coordinate; negative indices count from the end (`{-1}` is the last). +- `{*}` inserts all coordinates not used elsewhere, joined by the separator (required when `{*}` is used). +- `:0N` pads a value with zeros to `N` digits, e.g. `{0:04}`. +- `{{` and `}}` are literal braces. + +The format must fit the number of dimensions of the array, otherwise creating or opening the array fails. ### Exception Handling zarr-java uses two main exception types for error reporting: diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index cb880cf0..f8cc7f27 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -167,6 +167,7 @@ public ArrayMetadata( this.parsedFillValue ); + chunkKeyEncoding.validate(shape.length); this.chunkKeyEncoding = chunkKeyEncoding; this.codecs = codecs; this.dimensionNames = dimensionNames; diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index fd6b5c37..a9b1204c 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -9,6 +9,7 @@ import dev.zarr.zarrjava.v3.chunkgrid.RegularChunkGrid; import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.v3.chunkkeyencoding.DefaultChunkKeyEncoding; +import dev.zarr.zarrjava.v3.chunkkeyencoding.TemplateChunkKeyEncoding; import dev.zarr.zarrjava.v3.chunkkeyencoding.V2ChunkKeyEncoding; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; @@ -113,6 +114,21 @@ public ArrayMetadataBuilder withV2ChunkKeyEncoding(String separator) { return this; } + /** + * Uses the template chunk key encoding, e.g. {@code withTemplateChunkKeyEncoding("images/{-1}/{*}.jpeg", "_")}. + * + * @param separator joins the values of {@code {*}}; may be {@code null} if the format has no {@code {*}} + */ + public ArrayMetadataBuilder withTemplateChunkKeyEncoding(String format, String separator) throws ZarrException { + this.chunkKeyEncoding = new TemplateChunkKeyEncoding(format, separator); + return this; + } + + public ArrayMetadataBuilder withChunkKeyEncoding(ChunkKeyEncoding chunkKeyEncoding) { + this.chunkKeyEncoding = chunkKeyEncoding; + return this; + } + public ArrayMetadataBuilder withFillValue(Object fillValue) { this.fillValue = fillValue; return this; diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java index f599e72d..10052733 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java +++ b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java @@ -2,16 +2,24 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import dev.zarr.zarrjava.ZarrException; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "name") @JsonSubTypes({ @JsonSubTypes.Type(value = DefaultChunkKeyEncoding.class, name = "default"), - @JsonSubTypes.Type(value = V2ChunkKeyEncoding.class, name = "v2") + @JsonSubTypes.Type(value = V2ChunkKeyEncoding.class, name = "v2"), + @JsonSubTypes.Type(value = TemplateChunkKeyEncoding.class, name = "template") }) public abstract class ChunkKeyEncoding implements dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding { public abstract String[] encodeChunkKey(long[] chunkCoords); + /** + * Checks that this encoding can encode chunk coordinates of an array with {@code ndim} dimensions. + */ + public void validate(int ndim) throws ZarrException { + } + } diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/TemplateChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/TemplateChunkKeyEncoding.java new file mode 100644 index 00000000..3eeb2236 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/TemplateChunkKeyEncoding.java @@ -0,0 +1,264 @@ +package dev.zarr.zarrjava.v3.chunkkeyencoding; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import dev.zarr.zarrjava.ZarrException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Chunk key encoding that builds keys from a format string such as {@code "images/{-1}/{*:04}.jpeg"}. + * + * @see template chunk key encoding + */ +public class TemplateChunkKeyEncoding extends ChunkKeyEncoding { + + private static final Pattern EXPRESSION = Pattern.compile("(\\*|-?\\d+)(?::0(\\d+))?"); + private static final String[] FORBIDDEN_LITERALS = {"..", "\\", "\0", "//", "zarr.json"}; + + @JsonIgnore + public final String name = "template"; + @Nonnull + public final Configuration configuration; + + @JsonIgnore + private final List parts; + @JsonIgnore + private final boolean hasCatchAll; + + @JsonCreator + public TemplateChunkKeyEncoding( + @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration + ) throws ZarrException { + this.configuration = configuration; + this.parts = parse(configuration.format); + + int catchAlls = 0; + for (Part part : parts) { + if (part instanceof Expression && ((Expression) part).index == null) { + catchAlls++; + } + } + if (catchAlls > 1) { + throw new ZarrException("Template '" + configuration.format + "' contains more than one '{*}' expression."); + } + this.hasCatchAll = catchAlls == 1; + if (hasCatchAll && configuration.separator == null) { + throw new ZarrException("Template '" + configuration.format + "' contains '{*}' but no separator is configured."); + } + } + + public TemplateChunkKeyEncoding(@Nonnull String format, @Nullable String separator) throws ZarrException { + this(new Configuration(format, separator)); + } + + public TemplateChunkKeyEncoding(@Nonnull String format) throws ZarrException { + this(format, null); + } + + private static List parse(String format) throws ZarrException { + if (format == null) { + throw new ZarrException("Template chunk key encoding requires a 'format'."); + } + List parts = new ArrayList<>(); + StringBuilder literal = new StringBuilder(); + int i = 0; + while (i < format.length()) { + char c = format.charAt(i); + boolean doubled = i + 1 < format.length() && format.charAt(i + 1) == c; + if (c == '{' && doubled || c == '}' && doubled) { + literal.append(c); + i += 2; + } else if (c == '{') { + int end = format.indexOf('}', i); + if (end < 0) { + throw new ZarrException("Template '" + format + "' has an unmatched '{'."); + } + Matcher matcher = EXPRESSION.matcher(format.substring(i + 1, end)); + if (!matcher.matches()) { + throw new ZarrException("Template '" + format + "' has an invalid expression '" + + format.substring(i, end + 1) + "'."); + } + if (literal.length() > 0) { + parts.add(new Literal(literal.toString())); + literal.setLength(0); + } + Integer index = matcher.group(1).equals("*") ? null : Integer.valueOf(matcher.group(1)); + int width = matcher.group(2) == null ? 0 : Integer.parseInt(matcher.group(2)); + parts.add(new Expression(index, width)); + i = end + 1; + } else if (c == '}') { + throw new ZarrException("Template '" + format + "' has an unmatched '}'."); + } else { + literal.append(c); + i++; + } + } + if (literal.length() > 0) { + parts.add(new Literal(literal.toString())); + } + + if (parts.stream().noneMatch(p -> p instanceof Expression)) { + throw new ZarrException("Template '" + format + "' needs to contain at least one expression."); + } + if (parts.get(0) instanceof Literal && ((Literal) parts.get(0)).value.startsWith("/")) { + throw new ZarrException("Template '" + format + "' must not begin with '/'."); + } + for (Part part : parts) { + if (part instanceof Literal) { + for (String forbidden : FORBIDDEN_LITERALS) { + if (((Literal) part).value.contains(forbidden)) { + throw new ZarrException("Template '" + format + "' must not contain '" + forbidden + "'."); + } + } + } + } + return Collections.unmodifiableList(parts); + } + + private static String pad(long value, int width) { + String digits = Long.toString(Math.abs(value)); + String sign = value < 0 ? "-" : ""; + StringBuilder sb = new StringBuilder(sign); + for (int i = sign.length() + digits.length(); i < width; i++) { + sb.append('0'); + } + return sb.append(digits).toString(); + } + + /** + * Resolves each expression to a position in a chunk coordinate of length {@code ndim}. + * The catch-all expression resolves to {@code -1}. + */ + private int[] resolve(int ndim) throws ZarrException { + int[] positions = new int[parts.size()]; + boolean[] used = new boolean[ndim]; + int explicit = 0; + for (int p = 0; p < parts.size(); p++) { + if (!(parts.get(p) instanceof Expression)) { + continue; + } + Integer index = ((Expression) parts.get(p)).index; + if (index == null) { + positions[p] = -1; + continue; + } + int position = index < 0 ? ndim + index : index; + if (position < 0 || position >= ndim) { + throw new ZarrException("Index " + index + " of template '" + configuration.format + + "' is out of range for " + ndim + " dimensions."); + } + if (used[position]) { + throw new ZarrException("Template '" + configuration.format + "' refers to dimension " + + position + " more than once."); + } + used[position] = true; + positions[p] = position; + explicit++; + } + if (hasCatchAll && explicit == ndim) { + throw new ZarrException("Template '" + configuration.format + "' leaves no dimensions for '{*}' with " + + ndim + " dimensions."); + } + if (!hasCatchAll && explicit != ndim) { + throw new ZarrException("Template '" + configuration.format + "' has " + explicit + + " expressions, but the array has " + ndim + " dimensions."); + } + return positions; + } + + @Override + public void validate(int ndim) throws ZarrException { + resolve(ndim); + } + + public String encode(long[] chunkCoords) throws ZarrException { + int[] positions = resolve(chunkCoords.length); + boolean[] used = new boolean[chunkCoords.length]; + for (int p = 0; p < parts.size(); p++) { + if (parts.get(p) instanceof Expression && positions[p] >= 0) { + used[positions[p]] = true; + } + } + StringBuilder key = new StringBuilder(); + for (int p = 0; p < parts.size(); p++) { + Part part = parts.get(p); + if (part instanceof Literal) { + key.append(((Literal) part).value); + continue; + } + int width = ((Expression) part).width; + if (positions[p] >= 0) { + key.append(pad(chunkCoords[positions[p]], width)); + continue; + } + boolean first = true; + for (int d = 0; d < chunkCoords.length; d++) { + if (!used[d]) { + if (!first) { + key.append(configuration.separator); + } + key.append(pad(chunkCoords[d], width)); + first = false; + } + } + } + return key.toString(); + } + + @Override + public String[] encodeChunkKey(long[] chunkCoords) { + try { + return encode(chunkCoords).split("/"); + } catch (ZarrException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + } + + private interface Part { + } + + private static final class Literal implements Part { + final String value; + + Literal(String value) { + this.value = value; + } + } + + private static final class Expression implements Part { + /** Index into the chunk coordinate, or {@code null} for the catch-all {@code {*}}. */ + final Integer index; + final int width; + + Expression(Integer index, int width) { + this.index = index; + this.width = width; + } + } + + public static final class Configuration { + + @Nonnull + public final String format; + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + public final String separator; + + @JsonCreator + public Configuration( + @Nonnull @JsonProperty(value = "format", required = true) String format, + @Nullable @JsonProperty(value = "separator") String separator) { + this.format = format; + this.separator = separator; + } + } +} diff --git a/src/test/java/dev/zarr/zarrjava/TemplateChunkKeyEncodingTest.java b/src/test/java/dev/zarr/zarrjava/TemplateChunkKeyEncodingTest.java new file mode 100644 index 00000000..23259ada --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/TemplateChunkKeyEncodingTest.java @@ -0,0 +1,140 @@ +package dev.zarr.zarrjava; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.chunkkeyencoding.TemplateChunkKeyEncoding; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class TemplateChunkKeyEncodingTest extends ZarrTest { + + private static long[] coords(String s) { + return Arrays.stream(s.split(",")).mapToLong(Long::parseLong).toArray(); + } + + // Test cases from examples.tsv of the extension + @ParameterizedTest + @CsvSource(delimiter = '|', quoteCharacter = '"', nullValues = "NULL", value = { + "c/{*} | / | 11,22,33 | c/11/22/33", + "{*} | . | 11,22,33 | 11.22.33", + "prefix/{*} | / | 11,22,33 | prefix/11/22/33", + "{*}.suffix | _ | 11,22,33 | 11_22_33.suffix", + "{2}/{0}/{1} | . | 11,22,33 | 33/11/22", + "{-1}/{*} | / | 11,22,33 | 33/11/22", + "{*:04} | / | 11,22,33 | 0011/0022/0033", + "{*:02} | / | 111,2,33 | 111/02/33", + "a{{b{*}}}c | . | 11,22,33 | a{b11.22.33}c", + "images/{-1}/{*:04}.jpeg | _ | 1,2,3 | images/3/0001_0002.jpeg", + "{2}-{0}_{1} | NULL | 1,2,3 | 3-1_2", + "{0:03} | NULL | -5 | -05", + }) + public void testEncode(String format, String separator, String chunkCoords, String expected) throws ZarrException { + TemplateChunkKeyEncoding encoding = new TemplateChunkKeyEncoding(format, separator); + assertEquals(expected, encoding.encode(coords(chunkCoords))); + assertArrayEquals(expected.split("/"), encoding.encodeChunkKey(coords(chunkCoords))); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', nullValues = "NULL", value = { + "c/{*}/{*} | /", + "c/{*} | NULL", + "c/0 | NULL", + "/{0} | NULL", + "a/../{0} | NULL", + "a//{0} | NULL", + "a\\{0} | NULL", + "zarr.json{0}| NULL", + "{0 | NULL", + "{0}} | NULL", + "{x} | NULL", + "{0:4} | NULL", + "{0:04:1} | NULL", + }) + public void testInvalidFormat(String format, String separator) { + assertThrows(ZarrException.class, () -> new TemplateChunkKeyEncoding(format, separator)); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', nullValues = "NULL", value = { + "{0}/{1} | NULL | 1,2,3", + "{0}/{1} | NULL | 1", + "{3}/{*} | / | 1,2,3", + "{-4}/{*} | / | 1,2,3", + "{0}/{-3}/{*}| / | 1,2,3", + "{0}/{1}/{*} | / | 1,2", + }) + public void testIncongruentCoordinate(String format, String separator, String chunkCoords) throws ZarrException { + TemplateChunkKeyEncoding encoding = new TemplateChunkKeyEncoding(format, separator); + assertThrows(ZarrException.class, () -> encoding.encode(coords(chunkCoords))); + assertThrows(ZarrException.class, () -> encoding.validate(coords(chunkCoords).length)); + } + + @Test + public void testMetadataRejectsIncongruentDimensions() { + assertThrows(ZarrException.class, () -> Array.metadataBuilder() + .withShape(10, 10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5, 5) + .withTemplateChunkKeyEncoding("{0}/{1}", null) + .build()); + } + + @Test + public void testWriteReadAndMetadata() throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("template_chunk_key_encoding"); + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(4, 6, 8) + .withDataType(DataType.INT32) + .withChunkShape(2, 3, 4) + .withTemplateChunkKeyEncoding("images/{-1}/{*:02}.bin", "_") + .withFillValue(0) + .build()); + + ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{4, 6, 8}); + for (int i = 0; i < data.getSize(); i++) { + data.setInt(i, i); + } + array.write(data); + + Path root = TESTOUTPUT.resolve("template_chunk_key_encoding"); + Assertions.assertTrue(Files.exists(root.resolve("images/0/00_00.bin"))); + Assertions.assertTrue(Files.exists(root.resolve("images/1/01_01.bin"))); + Assertions.assertFalse(Files.exists(root.resolve("c"))); + + JsonNode zarrJson = new ObjectMapper().readTree(root.resolve("zarr.json").toFile()); + assertEquals("{\"name\":\"template\",\"configuration\":{\"format\":\"images/{-1}/{*:02}.bin\",\"separator\":\"_\"}}", + zarrJson.get("chunk_key_encoding").toString()); + + Array reopened = Array.open(storeHandle); + Assertions.assertInstanceOf(TemplateChunkKeyEncoding.class, reopened.metadata().chunkKeyEncoding); + assertArrayEquals((int[]) data.get1DJavaArray(ucar.ma2.DataType.INT), + (int[]) reopened.read().get1DJavaArray(ucar.ma2.DataType.INT)); + } + + @Test + public void testSeparatorOmittedWhenNull() throws Exception { + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(4, 4) + .withDataType(DataType.UINT8) + .withChunkShape(2, 2) + .withTemplateChunkKeyEncoding("{1}-{0}", null) + .build(); + String json = new ObjectMapper().writeValueAsString(metadata.chunkKeyEncoding); + assertEquals("{\"name\":\"template\",\"configuration\":{\"format\":\"{1}-{0}\"}}", json); + } +}