diff --git a/USERGUIDE.md b/USERGUIDE.md
index 65bf9944..f150afb8 100644
--- a/USERGUIDE.md
+++ b/USERGUIDE.md
@@ -610,6 +610,25 @@ ucar.ma2.Array chunk = array.readChunk(chunkCoords);
// Write single chunk
array.writeChunk(chunkCoords, chunk);
```
+### Chunk Key Encodings (v3)
+The chunk key encoding decides where a chunk is stored. Besides `default` (`c/0/1/2`) and `v2` (`0.1.2`),
+zarr-java supports the [`template`](https://github.com/zarr-developers/zarr-extensions/tree/main/chunk-key-encodings/template)
+encoding, which builds keys from a format string:
+```java
+Array array = Array.create(storeHandle, Array.metadataBuilder()
+ .withShape(1024, 1024, 100)
+ .withDataType(DataType.UINT8)
+ .withChunkShape(256, 256, 1)
+ // chunk (1, 2, 7) is stored at "images/7/0001_0002.jpeg"
+ .withTemplateChunkKeyEncoding("images/{-1}/{*:04}.jpeg", "_")
+ .build());
+```
+- `{0}`, `{1}`, … insert a chunk coordinate; negative indices count from the end (`{-1}` is the last).
+- `{*}` inserts all coordinates not used elsewhere, joined by the separator (required when `{*}` is used).
+- `:0N` pads a value with zeros to `N` digits, e.g. `{0:04}`.
+- `{{` and `}}` are literal braces.
+
+The format must fit the number of dimensions of the array, otherwise creating or opening the array fails.
### Exception Handling
zarr-java uses two main exception types for error reporting:
diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java
index cb880cf0..f8cc7f27 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java
@@ -167,6 +167,7 @@ public ArrayMetadata(
this.parsedFillValue
);
+ chunkKeyEncoding.validate(shape.length);
this.chunkKeyEncoding = chunkKeyEncoding;
this.codecs = codecs;
this.dimensionNames = dimensionNames;
diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java
index fd6b5c37..a9b1204c 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java
@@ -9,6 +9,7 @@
import dev.zarr.zarrjava.v3.chunkgrid.RegularChunkGrid;
import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding;
import dev.zarr.zarrjava.v3.chunkkeyencoding.DefaultChunkKeyEncoding;
+import dev.zarr.zarrjava.v3.chunkkeyencoding.TemplateChunkKeyEncoding;
import dev.zarr.zarrjava.v3.chunkkeyencoding.V2ChunkKeyEncoding;
import dev.zarr.zarrjava.v3.codec.Codec;
import dev.zarr.zarrjava.v3.codec.CodecBuilder;
@@ -113,6 +114,21 @@ public ArrayMetadataBuilder withV2ChunkKeyEncoding(String separator) {
return this;
}
+ /**
+ * Uses the template chunk key encoding, e.g. {@code withTemplateChunkKeyEncoding("images/{-1}/{*}.jpeg", "_")}.
+ *
+ * @param separator joins the values of {@code {*}}; may be {@code null} if the format has no {@code {*}}
+ */
+ public ArrayMetadataBuilder withTemplateChunkKeyEncoding(String format, String separator) throws ZarrException {
+ this.chunkKeyEncoding = new TemplateChunkKeyEncoding(format, separator);
+ return this;
+ }
+
+ public ArrayMetadataBuilder withChunkKeyEncoding(ChunkKeyEncoding chunkKeyEncoding) {
+ this.chunkKeyEncoding = chunkKeyEncoding;
+ return this;
+ }
+
public ArrayMetadataBuilder withFillValue(Object fillValue) {
this.fillValue = fillValue;
return this;
diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java
index f599e72d..10052733 100644
--- a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java
+++ b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java
@@ -2,16 +2,24 @@
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import dev.zarr.zarrjava.ZarrException;
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "name")
@JsonSubTypes({
@JsonSubTypes.Type(value = DefaultChunkKeyEncoding.class, name = "default"),
- @JsonSubTypes.Type(value = V2ChunkKeyEncoding.class, name = "v2")
+ @JsonSubTypes.Type(value = V2ChunkKeyEncoding.class, name = "v2"),
+ @JsonSubTypes.Type(value = TemplateChunkKeyEncoding.class, name = "template")
})
public abstract class ChunkKeyEncoding implements dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding {
public abstract String[] encodeChunkKey(long[] chunkCoords);
+ /**
+ * Checks that this encoding can encode chunk coordinates of an array with {@code ndim} dimensions.
+ */
+ public void validate(int ndim) throws ZarrException {
+ }
+
}
diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/TemplateChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/TemplateChunkKeyEncoding.java
new file mode 100644
index 00000000..3eeb2236
--- /dev/null
+++ b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/TemplateChunkKeyEncoding.java
@@ -0,0 +1,264 @@
+package dev.zarr.zarrjava.v3.chunkkeyencoding;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import dev.zarr.zarrjava.ZarrException;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Chunk key encoding that builds keys from a format string such as {@code "images/{-1}/{*:04}.jpeg"}.
+ *
+ * @see template chunk key encoding
+ */
+public class TemplateChunkKeyEncoding extends ChunkKeyEncoding {
+
+ private static final Pattern EXPRESSION = Pattern.compile("(\\*|-?\\d+)(?::0(\\d+))?");
+ private static final String[] FORBIDDEN_LITERALS = {"..", "\\", "\0", "//", "zarr.json"};
+
+ @JsonIgnore
+ public final String name = "template";
+ @Nonnull
+ public final Configuration configuration;
+
+ @JsonIgnore
+ private final List parts;
+ @JsonIgnore
+ private final boolean hasCatchAll;
+
+ @JsonCreator
+ public TemplateChunkKeyEncoding(
+ @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration
+ ) throws ZarrException {
+ this.configuration = configuration;
+ this.parts = parse(configuration.format);
+
+ int catchAlls = 0;
+ for (Part part : parts) {
+ if (part instanceof Expression && ((Expression) part).index == null) {
+ catchAlls++;
+ }
+ }
+ if (catchAlls > 1) {
+ throw new ZarrException("Template '" + configuration.format + "' contains more than one '{*}' expression.");
+ }
+ this.hasCatchAll = catchAlls == 1;
+ if (hasCatchAll && configuration.separator == null) {
+ throw new ZarrException("Template '" + configuration.format + "' contains '{*}' but no separator is configured.");
+ }
+ }
+
+ public TemplateChunkKeyEncoding(@Nonnull String format, @Nullable String separator) throws ZarrException {
+ this(new Configuration(format, separator));
+ }
+
+ public TemplateChunkKeyEncoding(@Nonnull String format) throws ZarrException {
+ this(format, null);
+ }
+
+ private static List parse(String format) throws ZarrException {
+ if (format == null) {
+ throw new ZarrException("Template chunk key encoding requires a 'format'.");
+ }
+ List parts = new ArrayList<>();
+ StringBuilder literal = new StringBuilder();
+ int i = 0;
+ while (i < format.length()) {
+ char c = format.charAt(i);
+ boolean doubled = i + 1 < format.length() && format.charAt(i + 1) == c;
+ if (c == '{' && doubled || c == '}' && doubled) {
+ literal.append(c);
+ i += 2;
+ } else if (c == '{') {
+ int end = format.indexOf('}', i);
+ if (end < 0) {
+ throw new ZarrException("Template '" + format + "' has an unmatched '{'.");
+ }
+ Matcher matcher = EXPRESSION.matcher(format.substring(i + 1, end));
+ if (!matcher.matches()) {
+ throw new ZarrException("Template '" + format + "' has an invalid expression '" +
+ format.substring(i, end + 1) + "'.");
+ }
+ if (literal.length() > 0) {
+ parts.add(new Literal(literal.toString()));
+ literal.setLength(0);
+ }
+ Integer index = matcher.group(1).equals("*") ? null : Integer.valueOf(matcher.group(1));
+ int width = matcher.group(2) == null ? 0 : Integer.parseInt(matcher.group(2));
+ parts.add(new Expression(index, width));
+ i = end + 1;
+ } else if (c == '}') {
+ throw new ZarrException("Template '" + format + "' has an unmatched '}'.");
+ } else {
+ literal.append(c);
+ i++;
+ }
+ }
+ if (literal.length() > 0) {
+ parts.add(new Literal(literal.toString()));
+ }
+
+ if (parts.stream().noneMatch(p -> p instanceof Expression)) {
+ throw new ZarrException("Template '" + format + "' needs to contain at least one expression.");
+ }
+ if (parts.get(0) instanceof Literal && ((Literal) parts.get(0)).value.startsWith("/")) {
+ throw new ZarrException("Template '" + format + "' must not begin with '/'.");
+ }
+ for (Part part : parts) {
+ if (part instanceof Literal) {
+ for (String forbidden : FORBIDDEN_LITERALS) {
+ if (((Literal) part).value.contains(forbidden)) {
+ throw new ZarrException("Template '" + format + "' must not contain '" + forbidden + "'.");
+ }
+ }
+ }
+ }
+ return Collections.unmodifiableList(parts);
+ }
+
+ private static String pad(long value, int width) {
+ String digits = Long.toString(Math.abs(value));
+ String sign = value < 0 ? "-" : "";
+ StringBuilder sb = new StringBuilder(sign);
+ for (int i = sign.length() + digits.length(); i < width; i++) {
+ sb.append('0');
+ }
+ return sb.append(digits).toString();
+ }
+
+ /**
+ * Resolves each expression to a position in a chunk coordinate of length {@code ndim}.
+ * The catch-all expression resolves to {@code -1}.
+ */
+ private int[] resolve(int ndim) throws ZarrException {
+ int[] positions = new int[parts.size()];
+ boolean[] used = new boolean[ndim];
+ int explicit = 0;
+ for (int p = 0; p < parts.size(); p++) {
+ if (!(parts.get(p) instanceof Expression)) {
+ continue;
+ }
+ Integer index = ((Expression) parts.get(p)).index;
+ if (index == null) {
+ positions[p] = -1;
+ continue;
+ }
+ int position = index < 0 ? ndim + index : index;
+ if (position < 0 || position >= ndim) {
+ throw new ZarrException("Index " + index + " of template '" + configuration.format +
+ "' is out of range for " + ndim + " dimensions.");
+ }
+ if (used[position]) {
+ throw new ZarrException("Template '" + configuration.format + "' refers to dimension " +
+ position + " more than once.");
+ }
+ used[position] = true;
+ positions[p] = position;
+ explicit++;
+ }
+ if (hasCatchAll && explicit == ndim) {
+ throw new ZarrException("Template '" + configuration.format + "' leaves no dimensions for '{*}' with " +
+ ndim + " dimensions.");
+ }
+ if (!hasCatchAll && explicit != ndim) {
+ throw new ZarrException("Template '" + configuration.format + "' has " + explicit +
+ " expressions, but the array has " + ndim + " dimensions.");
+ }
+ return positions;
+ }
+
+ @Override
+ public void validate(int ndim) throws ZarrException {
+ resolve(ndim);
+ }
+
+ public String encode(long[] chunkCoords) throws ZarrException {
+ int[] positions = resolve(chunkCoords.length);
+ boolean[] used = new boolean[chunkCoords.length];
+ for (int p = 0; p < parts.size(); p++) {
+ if (parts.get(p) instanceof Expression && positions[p] >= 0) {
+ used[positions[p]] = true;
+ }
+ }
+ StringBuilder key = new StringBuilder();
+ for (int p = 0; p < parts.size(); p++) {
+ Part part = parts.get(p);
+ if (part instanceof Literal) {
+ key.append(((Literal) part).value);
+ continue;
+ }
+ int width = ((Expression) part).width;
+ if (positions[p] >= 0) {
+ key.append(pad(chunkCoords[positions[p]], width));
+ continue;
+ }
+ boolean first = true;
+ for (int d = 0; d < chunkCoords.length; d++) {
+ if (!used[d]) {
+ if (!first) {
+ key.append(configuration.separator);
+ }
+ key.append(pad(chunkCoords[d], width));
+ first = false;
+ }
+ }
+ }
+ return key.toString();
+ }
+
+ @Override
+ public String[] encodeChunkKey(long[] chunkCoords) {
+ try {
+ return encode(chunkCoords).split("/");
+ } catch (ZarrException e) {
+ throw new IllegalArgumentException(e.getMessage(), e);
+ }
+ }
+
+ private interface Part {
+ }
+
+ private static final class Literal implements Part {
+ final String value;
+
+ Literal(String value) {
+ this.value = value;
+ }
+ }
+
+ private static final class Expression implements Part {
+ /** Index into the chunk coordinate, or {@code null} for the catch-all {@code {*}}. */
+ final Integer index;
+ final int width;
+
+ Expression(Integer index, int width) {
+ this.index = index;
+ this.width = width;
+ }
+ }
+
+ public static final class Configuration {
+
+ @Nonnull
+ public final String format;
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ public final String separator;
+
+ @JsonCreator
+ public Configuration(
+ @Nonnull @JsonProperty(value = "format", required = true) String format,
+ @Nullable @JsonProperty(value = "separator") String separator) {
+ this.format = format;
+ this.separator = separator;
+ }
+ }
+}
diff --git a/src/test/java/dev/zarr/zarrjava/TemplateChunkKeyEncodingTest.java b/src/test/java/dev/zarr/zarrjava/TemplateChunkKeyEncodingTest.java
new file mode 100644
index 00000000..23259ada
--- /dev/null
+++ b/src/test/java/dev/zarr/zarrjava/TemplateChunkKeyEncodingTest.java
@@ -0,0 +1,140 @@
+package dev.zarr.zarrjava;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import dev.zarr.zarrjava.store.FilesystemStore;
+import dev.zarr.zarrjava.store.StoreHandle;
+import dev.zarr.zarrjava.v3.Array;
+import dev.zarr.zarrjava.v3.ArrayMetadata;
+import dev.zarr.zarrjava.v3.DataType;
+import dev.zarr.zarrjava.v3.chunkkeyencoding.TemplateChunkKeyEncoding;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class TemplateChunkKeyEncodingTest extends ZarrTest {
+
+ private static long[] coords(String s) {
+ return Arrays.stream(s.split(",")).mapToLong(Long::parseLong).toArray();
+ }
+
+ // Test cases from examples.tsv of the extension
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', quoteCharacter = '"', nullValues = "NULL", value = {
+ "c/{*} | / | 11,22,33 | c/11/22/33",
+ "{*} | . | 11,22,33 | 11.22.33",
+ "prefix/{*} | / | 11,22,33 | prefix/11/22/33",
+ "{*}.suffix | _ | 11,22,33 | 11_22_33.suffix",
+ "{2}/{0}/{1} | . | 11,22,33 | 33/11/22",
+ "{-1}/{*} | / | 11,22,33 | 33/11/22",
+ "{*:04} | / | 11,22,33 | 0011/0022/0033",
+ "{*:02} | / | 111,2,33 | 111/02/33",
+ "a{{b{*}}}c | . | 11,22,33 | a{b11.22.33}c",
+ "images/{-1}/{*:04}.jpeg | _ | 1,2,3 | images/3/0001_0002.jpeg",
+ "{2}-{0}_{1} | NULL | 1,2,3 | 3-1_2",
+ "{0:03} | NULL | -5 | -05",
+ })
+ public void testEncode(String format, String separator, String chunkCoords, String expected) throws ZarrException {
+ TemplateChunkKeyEncoding encoding = new TemplateChunkKeyEncoding(format, separator);
+ assertEquals(expected, encoding.encode(coords(chunkCoords)));
+ assertArrayEquals(expected.split("/"), encoding.encodeChunkKey(coords(chunkCoords)));
+ }
+
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', nullValues = "NULL", value = {
+ "c/{*}/{*} | /",
+ "c/{*} | NULL",
+ "c/0 | NULL",
+ "/{0} | NULL",
+ "a/../{0} | NULL",
+ "a//{0} | NULL",
+ "a\\{0} | NULL",
+ "zarr.json{0}| NULL",
+ "{0 | NULL",
+ "{0}} | NULL",
+ "{x} | NULL",
+ "{0:4} | NULL",
+ "{0:04:1} | NULL",
+ })
+ public void testInvalidFormat(String format, String separator) {
+ assertThrows(ZarrException.class, () -> new TemplateChunkKeyEncoding(format, separator));
+ }
+
+ @ParameterizedTest
+ @CsvSource(delimiter = '|', nullValues = "NULL", value = {
+ "{0}/{1} | NULL | 1,2,3",
+ "{0}/{1} | NULL | 1",
+ "{3}/{*} | / | 1,2,3",
+ "{-4}/{*} | / | 1,2,3",
+ "{0}/{-3}/{*}| / | 1,2,3",
+ "{0}/{1}/{*} | / | 1,2",
+ })
+ public void testIncongruentCoordinate(String format, String separator, String chunkCoords) throws ZarrException {
+ TemplateChunkKeyEncoding encoding = new TemplateChunkKeyEncoding(format, separator);
+ assertThrows(ZarrException.class, () -> encoding.encode(coords(chunkCoords)));
+ assertThrows(ZarrException.class, () -> encoding.validate(coords(chunkCoords).length));
+ }
+
+ @Test
+ public void testMetadataRejectsIncongruentDimensions() {
+ assertThrows(ZarrException.class, () -> Array.metadataBuilder()
+ .withShape(10, 10, 10)
+ .withDataType(DataType.UINT8)
+ .withChunkShape(5, 5, 5)
+ .withTemplateChunkKeyEncoding("{0}/{1}", null)
+ .build());
+ }
+
+ @Test
+ public void testWriteReadAndMetadata() throws Exception {
+ StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("template_chunk_key_encoding");
+ Array array = Array.create(storeHandle, Array.metadataBuilder()
+ .withShape(4, 6, 8)
+ .withDataType(DataType.INT32)
+ .withChunkShape(2, 3, 4)
+ .withTemplateChunkKeyEncoding("images/{-1}/{*:02}.bin", "_")
+ .withFillValue(0)
+ .build());
+
+ ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{4, 6, 8});
+ for (int i = 0; i < data.getSize(); i++) {
+ data.setInt(i, i);
+ }
+ array.write(data);
+
+ Path root = TESTOUTPUT.resolve("template_chunk_key_encoding");
+ Assertions.assertTrue(Files.exists(root.resolve("images/0/00_00.bin")));
+ Assertions.assertTrue(Files.exists(root.resolve("images/1/01_01.bin")));
+ Assertions.assertFalse(Files.exists(root.resolve("c")));
+
+ JsonNode zarrJson = new ObjectMapper().readTree(root.resolve("zarr.json").toFile());
+ assertEquals("{\"name\":\"template\",\"configuration\":{\"format\":\"images/{-1}/{*:02}.bin\",\"separator\":\"_\"}}",
+ zarrJson.get("chunk_key_encoding").toString());
+
+ Array reopened = Array.open(storeHandle);
+ Assertions.assertInstanceOf(TemplateChunkKeyEncoding.class, reopened.metadata().chunkKeyEncoding);
+ assertArrayEquals((int[]) data.get1DJavaArray(ucar.ma2.DataType.INT),
+ (int[]) reopened.read().get1DJavaArray(ucar.ma2.DataType.INT));
+ }
+
+ @Test
+ public void testSeparatorOmittedWhenNull() throws Exception {
+ ArrayMetadata metadata = Array.metadataBuilder()
+ .withShape(4, 4)
+ .withDataType(DataType.UINT8)
+ .withChunkShape(2, 2)
+ .withTemplateChunkKeyEncoding("{1}-{0}", null)
+ .build();
+ String json = new ObjectMapper().writeValueAsString(metadata.chunkKeyEncoding);
+ assertEquals("{\"name\":\"template\",\"configuration\":{\"format\":\"{1}-{0}\"}}", json);
+ }
+}