From 2d29a3b69890ce798c341628820aa22314114451 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sun, 16 Nov 2025 20:04:21 +0100 Subject: [PATCH 01/12] add MemoryStore --- .../dev/zarr/zarrjava/store/MemoryStore.java | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 src/main/java/dev/zarr/zarrjava/store/MemoryStore.java diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java new file mode 100644 index 0000000..e99f3e2 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -0,0 +1,85 @@ +package dev.zarr.zarrjava.store; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +public class MemoryStore implements Store, Store.ListableStore { + private final Map map = new HashMap<>(); + + @Override + public boolean exists(String[] keys) { + return map.containsKey(keys); + } + + @Nullable + @Override + public ByteBuffer get(String[] keys) { + return get(keys, 0); + } + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start) { + return get(keys, start, -1); + } + + @Nullable + @Override + public ByteBuffer get(String[] keys, long start, long end) { + if(end>Integer.MAX_VALUE) throw new RuntimeException("TODO"); + if(!map.containsKey(keys)) return null; //TODO: necessary? + if (end < 0) end = map.get(keys).length - end; + return ByteBuffer.wrap(map.get(keys), (int) start, (int) end); + } + + + @Override + public void set(String[] keys, ByteBuffer bytes) { + map.put(keys, bytes.array()); + } + + @Override + public void delete(String[] keys) { + map.remove(keys); + } + + public Stream list(String[] keys) { + Set allKeys = new HashSet<>(); + for(String[] k: map.keySet()){ + if (!equalFirstKeys(k, keys, keys.length)); + String key = ""; + for (String s : k) { + key += s + "/"; + allKeys.add(key); + } + } + return allKeys.stream(); + } + + @Nonnull + @Override + public StoreHandle resolve(String... keys) { + return new StoreHandle(this, keys); + } + + @Override + public String toString() { + return String.format("", hashCode()); + } + + private boolean equalFirstKeys(String[] k1, String[] k2, int n){ + if (k1.length < n || k2.length < n) return false; + for (int i = 0; i < n; i++) { + if(!k1[i].equals(k2[i])) + return false; + } + return true; + } + +} From 218f0792b7f57e1be6afbf83f6d121ac65150e8a Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 17 Nov 2025 15:53:49 +0100 Subject: [PATCH 02/12] memorystore map string keys --- .../java/dev/zarr/zarrjava/core/Group.java | 4 +- .../dev/zarr/zarrjava/store/MemoryStore.java | 60 +++++++++++-------- src/main/java/dev/zarr/zarrjava/v2/Group.java | 5 +- src/main/java/dev/zarr/zarrjava/v3/Group.java | 5 +- src/main/java/dev/zarr/zarrjava/v3/Node.java | 1 + .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 27 +++++++-- .../java/dev/zarr/zarrjava/ZarrV2Test.java | 18 ++++++ 7 files changed, 85 insertions(+), 35 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/Group.java b/src/main/java/dev/zarr/zarrjava/core/Group.java index 5cff477..671fe1a 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Group.java +++ b/src/main/java/dev/zarr/zarrjava/core/Group.java @@ -64,14 +64,14 @@ public static Group open(String path) throws IOException, ZarrException { } @Nullable - public abstract Node get(String key) throws ZarrException; + public abstract Node get(String key) throws ZarrException, IOException; public Stream list() { return storeHandle.list() .map(key -> { try { return get(key); - } catch (ZarrException e) { + } catch (Exception e) { throw new RuntimeException(e); } }) diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index e99f3e2..a638c39 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -1,8 +1,11 @@ package dev.zarr.zarrjava.store; +import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; + import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.nio.ByteBuffer; +import java.nio.file.Path; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -10,11 +13,24 @@ import java.util.stream.Stream; public class MemoryStore implements Store, Store.ListableStore { - private final Map map = new HashMap<>(); + private final Map map = new HashMap<>(); + Separator separator; + + public MemoryStore(Separator separator){ + this.separator = separator; + } + + public MemoryStore(){ + this(Separator.SLASH); + } + + String resolveKeys(String[] keys) { + return String.join(separator.getValue(), keys); + } @Override public boolean exists(String[] keys) { - return map.containsKey(keys); + return map.containsKey(resolveKeys(keys)); } @Nullable @@ -32,32 +48,36 @@ public ByteBuffer get(String[] keys, long start) { @Nullable @Override public ByteBuffer get(String[] keys, long start, long end) { - if(end>Integer.MAX_VALUE) throw new RuntimeException("TODO"); - if(!map.containsKey(keys)) return null; //TODO: necessary? - if (end < 0) end = map.get(keys).length - end; - return ByteBuffer.wrap(map.get(keys), (int) start, (int) end); + byte[] bytes = map.get(resolveKeys(keys)); + if (bytes == null) return null; + if (end < 0) end = bytes.length; + if (end > Integer.MAX_VALUE) throw new RuntimeException("TODO"); //TODO + return ByteBuffer.wrap(bytes, (int) start, (int) end); } @Override public void set(String[] keys, ByteBuffer bytes) { - map.put(keys, bytes.array()); + map.put(resolveKeys(keys), bytes.array()); } @Override public void delete(String[] keys) { - map.remove(keys); + map.remove(resolveKeys(keys)); } public Stream list(String[] keys) { + String prefix = resolveKeys(keys); Set allKeys = new HashSet<>(); - for(String[] k: map.keySet()){ - if (!equalFirstKeys(k, keys, keys.length)); - String key = ""; - for (String s : k) { - key += s + "/"; - allKeys.add(key); - } + + for (String k : map.keySet()) { + if (!k.startsWith(prefix)) continue; + String current = ""; + for (String s : k.split(separator.getValue())) { + current += s; + allKeys.add(current); + current += separator.getValue(); + } } return allKeys.stream(); } @@ -72,14 +92,4 @@ public StoreHandle resolve(String... keys) { public String toString() { return String.format("", hashCode()); } - - private boolean equalFirstKeys(String[] k1, String[] k2, int n){ - if (k1.length < n || k2.length < n) return false; - for (int i = 0; i < n; i++) { - if(!k1[i].equals(k2[i])) - return false; - } - return true; - } - } diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index 38c0b94..b519aed 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -10,6 +10,7 @@ import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.function.Function; @@ -60,11 +61,11 @@ public static Group create(String path) throws IOException, ZarrException { } @Nullable - public Node get(String key) throws ZarrException { + public Node get(String key) throws ZarrException, IOException { StoreHandle keyHandle = storeHandle.resolve(key); try { return Node.open(keyHandle); - } catch (IOException e) { + } catch (NoSuchFileException e) { return null; } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 9c56f25..278cab5 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -7,6 +7,7 @@ import dev.zarr.zarrjava.utils.Utils; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Map; @@ -79,11 +80,11 @@ public static Group create(String path) throws IOException, ZarrException { } @Nullable - public Node get(String key) throws ZarrException { + public Node get(String key) throws ZarrException, IOException{ StoreHandle keyHandle = storeHandle.resolve(key); try { return Node.open(keyHandle); - } catch (IOException e) { + } catch (NoSuchFileException e) { return null; } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Node.java b/src/main/java/dev/zarr/zarrjava/v3/Node.java index 235e37f..54a0762 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Node.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Node.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index f788d06..97ab3b0 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -1,12 +1,13 @@ package dev.zarr.zarrjava; import com.fasterxml.jackson.databind.ObjectMapper; -import dev.zarr.zarrjava.store.FilesystemStore; -import dev.zarr.zarrjava.store.HttpStore; -import dev.zarr.zarrjava.store.S3Store; +import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; +import dev.zarr.zarrjava.store.*; import dev.zarr.zarrjava.v3.*; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; @@ -17,7 +18,7 @@ import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; public class ZarrStoreTest extends ZarrTest { - @Test + @Test public void testFileSystemStores() throws IOException, ZarrException { FilesystemStore fsStore = new FilesystemStore(TESTDATA); ObjectMapper objectMapper = makeObjectMapper(); @@ -78,4 +79,22 @@ public void testHttpStore() throws IOException, ZarrException { Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata().shape); } + + @ParameterizedTest + @CsvSource({ + "DOT", "SLASH" + }) + public void testMemoryStore(Separator separator) throws ZarrException, IOException { + StoreHandle storeHandle = new MemoryStore(separator).resolve(); + Group group = Group.create(storeHandle); + Array array = group.createArray("array", b -> b + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + ); + group.createGroup("subgroup"); + Assertions.assertEquals(2, group.list().count()); + for(String s: storeHandle.list().toArray(String[]::new)) + System.out.println(s); + } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 765c51a..4a7612b 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -1,6 +1,8 @@ package dev.zarr.zarrjava; +import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.MemoryStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.v2.Array; import dev.zarr.zarrjava.v2.ArrayMetadata; @@ -255,4 +257,20 @@ public void testCreateGroup() throws ZarrException, IOException { Group.create(storeHandleString); Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); } + + @Test + public void testMemoryStore() throws ZarrException, IOException { + StoreHandle storeHandle = new MemoryStore().resolve(); + Group group = Group.create(storeHandle); + Array array = group.createArray("array", b -> b + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + ); + group.createGroup("subgroup"); + Assertions.assertEquals(2, group.list().count()); + for(String s: storeHandle.list().toArray(String[]::new)) + System.out.println(s); + } + } \ No newline at end of file From fa2626b9b31562f8a01f58d0a941264234bc2a46 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 21 Nov 2025 12:06:24 +0100 Subject: [PATCH 03/12] fix memorystore with List keys --- .../dev/zarr/zarrjava/store/MemoryStore.java | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index a638c39..67eb038 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -5,15 +5,11 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Stream; public class MemoryStore implements Store, Store.ListableStore { - private final Map map = new HashMap<>(); + private final Map, byte[]> map = new HashMap<>(); Separator separator; public MemoryStore(Separator separator){ @@ -24,8 +20,15 @@ public MemoryStore(){ this(Separator.SLASH); } - String resolveKeys(String[] keys) { - return String.join(separator.getValue(), keys); + List resolveKeys(String[] keys) { + ArrayList resolvedKeys = new ArrayList<>(); + for(String key:keys){ + if(key.startsWith("/")){ + key = key.substring(1); + } + resolvedKeys.addAll(Arrays.asList(key.split("/"))); + } + return resolvedKeys; } @Override @@ -67,16 +70,15 @@ public void delete(String[] keys) { } public Stream list(String[] keys) { - String prefix = resolveKeys(keys); + List prefix = resolveKeys(keys); Set allKeys = new HashSet<>(); - for (String k : map.keySet()) { - if (!k.startsWith(prefix)) continue; - String current = ""; - for (String s : k.split(separator.getValue())) { - current += s; - allKeys.add(current); - current += separator.getValue(); + for (List k : map.keySet()) { + if (k.size() <= prefix.size() || ! k.subList(0, prefix.size()).equals(prefix)) + continue; + for (int i = 0; i < k.size(); i++) { + List subKey = k.subList(0, i+1); + allKeys.add(String.join("/", subKey)); } } return allKeys.stream(); From 36cb46a300394ff54cb9ea6e9aa58e5308e1894a Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 21 Nov 2025 13:09:28 +0100 Subject: [PATCH 04/12] remove code duplication and add v2.Group.setAttributes --- src/main/java/dev/zarr/zarrjava/v2/Group.java | 41 ++++++++++++++----- src/main/java/dev/zarr/zarrjava/v3/Group.java | 10 ++--- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index 8fc8f2d..970be54 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -52,16 +52,7 @@ public static Group open(String path) throws IOException { public static Group create( @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata ) throws IOException { - ObjectWriter objectWriter = makeObjectWriter(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(groupMetadata)); - storeHandle.resolve(ZGROUP).set(metadataBytes); - if (groupMetadata.attributes != null) { - StoreHandle attrsHandle = storeHandle.resolve(ZATTRS); - ByteBuffer attrsBytes = ByteBuffer.wrap( - objectWriter.writeValueAsBytes(groupMetadata.attributes)); - attrsHandle.set(attrsBytes); - } - return new Group(storeHandle, groupMetadata); + return new Group(storeHandle, groupMetadata).writeMetadata(); } public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, ZarrException { @@ -108,10 +99,38 @@ public Array createArray(String key, ArrayMetadata arrayMetadata) } public Array createArray(String key, Function arrayMetadataBuilderMapper) - throws IOException, ZarrException { + throws IOException, ZarrException { return Array.create(storeHandle.resolve(key), arrayMetadataBuilderMapper, false); } + private Group writeMetadata() throws IOException { + return writeMetadata(this.metadata); + } + + private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { + ObjectWriter objectWriter = makeObjectWriter(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(newGroupMetadata)); + storeHandle.resolve(ZGROUP).set(metadataBytes); + if (newGroupMetadata.attributes != null) { + StoreHandle attrsHandle = storeHandle.resolve(ZATTRS); + ByteBuffer attrsBytes = ByteBuffer.wrap( + objectWriter.writeValueAsBytes(newGroupMetadata.attributes)); + attrsHandle.set(attrsBytes); + } + return new Group(storeHandle, newGroupMetadata); + } + + public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { + GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes); + return writeMetadata(newGroupMetadata); + } + + public Group updateAttributes(Function attributeMapper) + throws ZarrException, IOException { + return setAttributes(attributeMapper.apply(metadata.attributes)); + } + + @Override public String toString() { return String.format("", storeHandle); diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 96dd1f7..d933c78 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -46,11 +46,7 @@ public static Group open(String path) throws IOException { public static Group create( @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata ) throws IOException { - ObjectWriter objectWriter = makeObjectWriter(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(groupMetadata)); - storeHandle.resolve(ZARR_JSON) - .set(metadataBytes); - return new Group(storeHandle, groupMetadata); + return new Group(storeHandle, groupMetadata).writeMetadata(); } public static Group create( @@ -115,6 +111,10 @@ public Array createArray(String key, return Array.create(storeHandle.resolve(key), arrayMetadataBuilderMapper, false); } + private Group writeMetadata() throws IOException { + return writeMetadata(this.metadata); + } + private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { ObjectWriter objectWriter = makeObjectWriter(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(newGroupMetadata)); From 78fa912ffc8967923b1eb7244ac865bc4ace51d6 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 21 Nov 2025 13:09:51 +0100 Subject: [PATCH 05/12] add ConcurrentMemoryStore --- .../zarrjava/store/ConcurrentMemoryStore.java | 17 +++++++++++++++++ .../dev/zarr/zarrjava/store/MemoryStore.java | 14 ++++---------- 2 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/store/ConcurrentMemoryStore.java diff --git a/src/main/java/dev/zarr/zarrjava/store/ConcurrentMemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/ConcurrentMemoryStore.java new file mode 100644 index 0000000..33482d3 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/store/ConcurrentMemoryStore.java @@ -0,0 +1,17 @@ +package dev.zarr.zarrjava.store; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class ConcurrentMemoryStore extends MemoryStore { + @Override + protected Map, byte[]> map() { + return new ConcurrentHashMap<>(); + } + + @Override + public String toString() { + return String.format("", hashCode()); + } +} diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index 67eb038..8126e70 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -1,7 +1,5 @@ package dev.zarr.zarrjava.store; -import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; - import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.nio.ByteBuffer; @@ -9,15 +7,10 @@ import java.util.stream.Stream; public class MemoryStore implements Store, Store.ListableStore { - private final Map, byte[]> map = new HashMap<>(); - Separator separator; - - public MemoryStore(Separator separator){ - this.separator = separator; - } + private final Map, byte[]> map = map(); - public MemoryStore(){ - this(Separator.SLASH); + protected Map, byte[]> map(){ + return new HashMap<>(); } List resolveKeys(String[] keys) { @@ -95,3 +88,4 @@ public String toString() { return String.format("", hashCode()); } } + From bd3bd7cd685a5ebf583a98b7da720f2c87c8b91f Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 21 Nov 2025 13:10:08 +0100 Subject: [PATCH 06/12] add tests --- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 80 ++++++++++++++++--- .../java/dev/zarr/zarrjava/ZarrV3Test.java | 3 +- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 97ab3b0..581f3ff 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -1,7 +1,7 @@ package dev.zarr.zarrjava; import com.fasterxml.jackson.databind.ObjectMapper; -import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.*; import dev.zarr.zarrjava.v3.*; import org.junit.jupiter.api.Assertions; @@ -14,6 +14,8 @@ import java.io.IOException; import java.nio.file.Files; +import java.util.Arrays; +import java.util.stream.Stream; import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; @@ -82,19 +84,79 @@ public void testHttpStore() throws IOException, ZarrException { @ParameterizedTest @CsvSource({ - "DOT", "SLASH" + "false,MemoryStore", + "false,ConcurrentMemoryStore", + "true,ConcurrentMemoryStore", }) - public void testMemoryStore(Separator separator) throws ZarrException, IOException { - StoreHandle storeHandle = new MemoryStore(separator).resolve(); + public void testMemoryStoreV3(boolean useParallel, String storeType) throws ZarrException, IOException { + int[] testData = new int[1024 * 1024]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle; + switch (storeType) { + case "ConcurrentMemoryStore": + storeHandle = new ConcurrentMemoryStore().resolve(); + break; + case "MemoryStore": + storeHandle = new MemoryStore().resolve(); + break; + default: + throw new IllegalArgumentException("Unknown store type: " + storeType); + } + Group group = Group.create(storeHandle); Array array = group.createArray("array", b -> b - .withShape(10, 10) - .withDataType(DataType.UINT8) + .withShape(1024, 1024) + .withDataType(DataType.UINT32) .withChunkShape(5, 5) ); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel); group.createGroup("subgroup"); - Assertions.assertEquals(2, group.list().count()); - for(String s: storeHandle.list().toArray(String[]::new)) - System.out.println(s); + group.setAttributes(new Attributes().set("description", "test group")); + Stream nodes = group.list(); + Assertions.assertEquals(2, nodes.count()); + + ucar.ma2.Array result = array.read(useParallel); + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + Assertions.assertNotNull(group.metadata().attributes); + Assertions.assertEquals("test group", group.metadata().attributes.getString("description")); + } + + @ParameterizedTest + @CsvSource({ + "false,MemoryStore", + "false,ConcurrentMemoryStore", + "true,ConcurrentMemoryStore", + }) + public void testMemoryStoreV2(boolean useParallel, String storeType) throws ZarrException, IOException { + int[] testData = new int[1024 * 1024]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle; + switch (storeType) { + case "ConcurrentMemoryStore": + storeHandle = new ConcurrentMemoryStore().resolve(); + break; + case "MemoryStore": + storeHandle = new MemoryStore().resolve(); + break; + default: + throw new IllegalArgumentException("Unknown store type: " + storeType); + } + + dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(storeHandle); + dev.zarr.zarrjava.v2.Array array = group.createArray("array", b -> b + .withShape(1024, 1024) + .withDataType(dev.zarr.zarrjava.v2.DataType.UINT32) + .withChunks(5, 5) + ); + array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel); + group.createGroup("subgroup"); + Stream nodes = group.list(); + group.setAttributes(new Attributes().set("description", "test group")); + Assertions.assertEquals(2, nodes.count()); + + ucar.ma2.Array result = array.read(useParallel); + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 846f770..54fded2 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -388,7 +388,7 @@ public void testParallel(boolean useParallel) throws IOException, ZarrException int[] testData = new int[512 * 512 * 512]; Arrays.setAll(testData, p -> p); - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testParallelRead"); + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testParallelRead", useParallel ? "parallel" : "serial"); ArrayMetadata metadata = Array.metadataBuilder() .withShape(512, 512, 512) .withDataType(DataType.UINT32) @@ -402,7 +402,6 @@ public void testParallel(boolean useParallel) throws IOException, ZarrException ucar.ma2.Array result = readArray.read(useParallel); Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - clearTestoutputFolder(); } @Test From e1f8e49152982a4f28fd578a4585112ff3e4ac3a Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 21 Nov 2025 13:27:49 +0100 Subject: [PATCH 07/12] fix Group.writeMetadata --- src/main/java/dev/zarr/zarrjava/v2/Group.java | 3 ++- src/main/java/dev/zarr/zarrjava/v3/Group.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index 970be54..80b5a4f 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -117,7 +117,8 @@ private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { objectWriter.writeValueAsBytes(newGroupMetadata.attributes)); attrsHandle.set(attrsBytes); } - return new Group(storeHandle, newGroupMetadata); + this.metadata = newGroupMetadata; + return this; } public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index d933c78..10fc0d3 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -120,7 +120,8 @@ private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(newGroupMetadata)); storeHandle.resolve(ZARR_JSON) .set(metadataBytes); - return new Group(storeHandle, newGroupMetadata); + this.metadata = newGroupMetadata; + return this; } public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { From 50b40f1f75520d1049513699a8cdaaa7ad4a8551 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 21 Nov 2025 13:28:06 +0100 Subject: [PATCH 08/12] add attributes to tests --- src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 581f3ff..3f510c4 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -112,14 +112,15 @@ public void testMemoryStoreV3(boolean useParallel, String storeType) throws Zarr ); array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel); group.createGroup("subgroup"); - group.setAttributes(new Attributes().set("description", "test group")); + group.setAttributes(new Attributes(b -> b.set("some", "value"))); Stream nodes = group.list(); Assertions.assertEquals(2, nodes.count()); ucar.ma2.Array result = array.read(useParallel); Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - Assertions.assertNotNull(group.metadata().attributes); - Assertions.assertEquals("test group", group.metadata().attributes.getString("description")); + Attributes attrs = group.metadata().attributes; + Assertions.assertNotNull(attrs); + Assertions.assertEquals("value", attrs.getString("some")); } @ParameterizedTest @@ -158,5 +159,9 @@ public void testMemoryStoreV2(boolean useParallel, String storeType) throws Zarr ucar.ma2.Array result = array.read(useParallel); Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + Attributes attrs = group.metadata().attributes; + Assertions.assertNotNull(attrs); + Assertions.assertEquals("test group", attrs.getString("description")); + } } From 6094489ac7c1c2d5c9bae3a1e6d640f64fa55056 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 24 Nov 2025 11:05:40 +0100 Subject: [PATCH 09/12] Array.create and Group.create with no store/path arguments should default to a new MemoryStore --- src/main/java/dev/zarr/zarrjava/v2/Array.java | 11 ++ src/main/java/dev/zarr/zarrjava/v2/Group.java | 5 + src/main/java/dev/zarr/zarrjava/v3/Array.java | 13 ++ src/main/java/dev/zarr/zarrjava/v3/Group.java | 145 +++++++++++++++--- .../dev/zarr/zarrjava/v3/GroupMetadata.java | 9 +- 5 files changed, 163 insertions(+), 20 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index e66c20e..421dbc6 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -5,6 +5,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.MemoryStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.core.codec.CodecPipeline; @@ -87,6 +88,16 @@ public static Array open(String path) throws IOException, ZarrException { return open(Paths.get(path)); } + /** + * Creates a new Zarr array with the provided metadata in an in-memory store + * + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(ArrayMetadata arrayMetadata) throws ZarrException, IOException { + return create(new StoreHandle(new MemoryStore()), arrayMetadata); + } /** * Creates a new Zarr array with the provided metadata at a specified storage location. This * method will raise an exception if a Zarr array already exists at the specified storage diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index 80b5a4f..b1ec668 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -5,6 +5,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.MemoryStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; @@ -49,6 +50,10 @@ public static Group open(String path) throws IOException { return open(Paths.get(path)); } + public static Group create(@Nonnull GroupMetadata groupMetadata) throws IOException { + return new Group(new MemoryStore().resolve(), groupMetadata).writeMetadata(); + } + public static Group create( @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata ) throws IOException { diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 1536893..8e263a1 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -4,6 +4,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.MemoryStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.core.codec.CodecPipeline; @@ -72,6 +73,18 @@ public static Array open(String path) throws IOException, ZarrException { return open(Paths.get(path)); } + /** + * Creates a new Zarr array with the provided metadata in an in-memory store + * + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(new MemoryStore().resolve(), arrayMetadata); + } + /** * Creates a new Zarr array with the provided metadata at a specified storage location. This * method will raise an exception if a Zarr array already exists at the specified storage diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 10fc0d3..0c5cf8b 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -4,6 +4,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.MemoryStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import java.io.IOException; @@ -30,25 +31,67 @@ protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMe public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { StoreHandle metadataHandle = storeHandle.resolve(ZARR_JSON); ByteBuffer metadataBytes = metadataHandle.readNonNull(); - return new Group(storeHandle, makeObjectMapper() - .readValue(Utils.toArray(metadataBytes), GroupMetadata.class)); + return new Group(storeHandle, makeObjectMapper().readValue(Utils.toArray(metadataBytes), GroupMetadata.class)); } - - public static Group open(Path path) throws IOException { - return open(new StoreHandle(new FilesystemStore(path))); - } - public static Group open(String path) throws IOException { - return open(Paths.get(path)); - } - - public static Group create( - @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata - ) throws IOException { + public static Group open(Path path) throws IOException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + public static Group open(String path) throws IOException { + return open(Paths.get(path)); + } + + /** + * Creates a new Zarr group with default metadata in an in-memory store. + * + * @throws IOException if the metadata cannot be serialized + */ + public static Group create() throws IOException { + return new Group(new MemoryStore().resolve(), GroupMetadata.defaultValue()).writeMetadata(); + } + + /** + * Creates a new Zarr group with the provided metadata in an in-memory store. + * + * @param attributes the attributes of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the attributes are invalid + */ + public static Group create(@Nonnull Attributes attributes) throws IOException, ZarrException { + return new Group(new MemoryStore().resolve(), new GroupMetadata(attributes)).writeMetadata(); + } + + /** + * Creates a new Zarr group with the provided metadata in an in-memory store. + * + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ + public static Group create(@Nonnull GroupMetadata groupMetadata) throws IOException { + return new Group(new MemoryStore().resolve(), groupMetadata).writeMetadata(); + } + + /** + * Creates a new Zarr group with the provided metadata at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ + public static Group create(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException { return new Group(storeHandle, groupMetadata).writeMetadata(); } + /** + * Creates a new Zarr group with the provided metadata at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @param attributes attributes of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the attributes are invalid + */ public static Group create( @Nonnull StoreHandle storeHandle, @Nonnull Attributes attributes @@ -56,22 +99,54 @@ public static Group create( return create(storeHandle, new GroupMetadata(attributes)); } - public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, ZarrException { + /** + * Creates a new Zarr group with the provided metadata at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ + public static Group create(@Nonnull StoreHandle storeHandle) throws IOException { return create(storeHandle, GroupMetadata.defaultValue()); } + /** + * Creates a new Zarr group with the provided metadata at a specified storage location. + * + * @param path the storage location of the Zarr group + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ public static Group create(Path path, GroupMetadata groupMetadata) throws IOException, ZarrException { return create(new FilesystemStore(path).resolve(), groupMetadata); } + /** + * Creates a new Zarr group with the provided metadata at a specified storage location. + * + * @param path the storage location of the Zarr group + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ public static Group create(String path, GroupMetadata groupMetadata) throws IOException, ZarrException { return create(Paths.get(path), groupMetadata); } + /** + * Creates a new Zarr group with the provided metadata at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ public static Group create(Path path) throws IOException, ZarrException { return create(new FilesystemStore(path).resolve()); } + /** + * Creates a new Zarr group with the provided metadata at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ public static Group create(String path) throws IOException, ZarrException { return create(Paths.get(path)); } @@ -86,25 +161,59 @@ public Node get(String key) throws ZarrException, IOException{ } } + /** + * Creates a new subgroup with the provided metadata at the specified key. + * + * @param key the key of the new Zarr group within the current group + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ public Group createGroup(String key, GroupMetadata groupMetadata) - throws IOException, ZarrException { + throws IOException, ZarrException { return Group.create(storeHandle.resolve(key), groupMetadata); } + /** + * Creates a new subgroup with the provided attributes at the specified key. + * + * @param key the key of the new Zarr group within the current group + * @param attributes attributes of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ public Group createGroup(String key, Attributes attributes) - throws IOException, ZarrException { + throws IOException, ZarrException { return Group.create(storeHandle.resolve(key), new GroupMetadata(attributes)); } - public Group createGroup(String key) throws IOException, ZarrException { + /** + * Creates a new subgroup with default metadata at the specified key. + * + * @param key the key of the new Zarr group within the current group + * @throws IOException if the metadata cannot be serialized + */ + public Group createGroup(String key) throws IOException { return Group.create(storeHandle.resolve(key), GroupMetadata.defaultValue()); } + /** + * Creates a new array with the provided metadata at the specified key. + * + * @param key the key of the new Zarr array within the current group + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + */ public Array createArray(String key, ArrayMetadata arrayMetadata) - throws IOException, ZarrException { + throws IOException, ZarrException { return Array.create(storeHandle.resolve(key), arrayMetadata); } + /** + * Creates a new array with the provided metadata builder mapper at the specified key. + * + * @param key the key of the new Zarr array within the current group + * @param arrayMetadataBuilderMapper a function building the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + */ public Array createArray(String key, Function arrayMetadataBuilderMapper) throws IOException, ZarrException { diff --git a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java index e761873..e8acd93 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java @@ -44,8 +44,13 @@ public GroupMetadata( this.attributes = attributes; } - public static GroupMetadata defaultValue() throws ZarrException { - return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new Attributes()); + public static GroupMetadata defaultValue() { + try { + return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new Attributes()); + } catch (ZarrException e) { + // This should never happen + throw new RuntimeException(e); + } } @Override From 250daf12a3fada389abd4b11e26b8039c509a76d Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 24 Nov 2025 11:25:55 +0100 Subject: [PATCH 10/12] Add Javadoc comments for Group methods --- src/main/java/dev/zarr/zarrjava/v2/Group.java | 128 +++++++++++++++++- src/main/java/dev/zarr/zarrjava/v3/Group.java | 79 +++++++++-- 2 files changed, 191 insertions(+), 16 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index b1ec668..c568294 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -28,6 +28,12 @@ protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMe this.metadata = groupMetadata; } + /** + * Opens an existing Zarr group at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @throws IOException if the metadata cannot be read + */ public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { ObjectMapper mapper = makeObjectMapper(); GroupMetadata metadata = mapper.readValue( @@ -42,48 +48,126 @@ public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { return new Group(storeHandle, metadata); } + /** + * Opens an existing Zarr group at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be read + */ public static Group open(Path path) throws IOException { return open(new StoreHandle(new FilesystemStore(path))); } - public static Group open(String path) throws IOException { + /** + * Opens an existing Zarr group at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be read + */ + public static Group open(String path) throws IOException { return open(Paths.get(path)); } + /** + * Creates a new Zarr group with the provided metadata in an in-memory store. + * + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ public static Group create(@Nonnull GroupMetadata groupMetadata) throws IOException { return new Group(new MemoryStore().resolve(), groupMetadata).writeMetadata(); } + /** + * Creates a new Zarr group with the provided metadata at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @param groupMetadata the metadata of the Zarr group + * @throws IOException if the metadata cannot be serialized + */ public static Group create( @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata ) throws IOException { return new Group(storeHandle, groupMetadata).writeMetadata(); } + /** + * Creates a new Zarr group with default metadata at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid + */ public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, ZarrException { return create(storeHandle, new GroupMetadata()); } + /** + * Creates a new Zarr group with the provided attributes at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @param attributes the attributes of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the attributes are invalid + */ public static Group create(@Nonnull StoreHandle storeHandle, Attributes attributes) throws IOException, ZarrException { return create(storeHandle, new GroupMetadata(attributes)); } + /** + * Creates a new Zarr group with default metadata at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid + */ public static Group create(Path path) throws IOException, ZarrException { return create(new StoreHandle(new FilesystemStore(path))); } + /** + * Creates a new Zarr group with the provided attributes at a specified storage location. + * + * @param path the storage location of the Zarr group + * @param attributes the attributes of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the attributes are invalid + */ public static Group create(Path path, Attributes attributes) throws IOException, ZarrException { return create(new StoreHandle(new FilesystemStore(path)), attributes); } + /** + * Creates a new Zarr group with default metadata at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid + */ public static Group create(String path) throws IOException, ZarrException { return create(Paths.get(path)); } + /** + * Creates a new Zarr group with the provided attributes at a specified storage location. + * + * @param path the storage location of the Zarr group + * @param attributes the attributes of the Zarr group + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the attributes are invalid + */ public static Group create(String path, Attributes attributes) throws IOException, ZarrException { return create(Paths.get(path), attributes); } + /** + * Retrieves a node (group or array) at the specified key within the current group. + * + * @param key the key of the node to retrieve + * @return the node at the specified key, or null if it does not exist + * @throws ZarrException if the node cannot be opened + * @throws IOException if there is an error accessing the storage + */ @Nullable public Node get(String key) throws ZarrException, IOException { StoreHandle keyHandle = storeHandle.resolve(key); @@ -94,15 +178,41 @@ public Node get(String key) throws ZarrException, IOException { } } + /** + * Creates a new subgroup with default metadata at the specified key. + * + * @param key the key of the new Zarr group within the current group + * @return the created subgroup + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the group cannot be created + */ public Group createGroup(String key) throws IOException, ZarrException { return Group.create(storeHandle.resolve(key)); } + /** + * Creates a new array with the provided metadata at the specified key. + * + * @param key the key of the new Zarr array within the current group + * @param arrayMetadata the metadata of the Zarr array + * @return the created array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the array cannot be created + */ public Array createArray(String key, ArrayMetadata arrayMetadata) throws IOException, ZarrException { return Array.create(storeHandle.resolve(key), arrayMetadata); } + /** + * Creates a new array with the provided metadata at the specified key. + * + * @param key the key of the new Zarr array within the current group + * @param arrayMetadataBuilderMapper a function that modifies the array metadata + * @return the created array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the array cannot be created + */ public Array createArray(String key, Function arrayMetadataBuilderMapper) throws IOException, ZarrException { return Array.create(storeHandle.resolve(key), arrayMetadataBuilderMapper, false); @@ -126,11 +236,27 @@ private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { return this; } + /** + * Sets new attributes for the group, replacing any existing attributes. + * + * @param newAttributes the new attributes to set + * @return the updated group + * @throws ZarrException if the new attributes are invalid + * @throws IOException if the metadata cannot be serialized + */ public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes); return writeMetadata(newGroupMetadata); } + /** + * Updates the attributes of the group using a mapper function. + * + * @param attributeMapper a function that takes the current attributes and returns the updated attributes + * @return the updated group + * @throws ZarrException if the new attributes are invalid + * @throws IOException if the metadata cannot be serialized + */ public Group updateAttributes(Function attributeMapper) throws ZarrException, IOException { return setAttributes(attributeMapper.apply(metadata.attributes)); diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 0c5cf8b..d17eb77 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -28,6 +28,12 @@ protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMe this.metadata = groupMetadata; } + /** + * Opens an existing Zarr group at a specified storage location. + * + * @param storeHandle the storage location of the Zarr group + * @throws IOException if the metadata cannot be read + */ public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { StoreHandle metadataHandle = storeHandle.resolve(ZARR_JSON); ByteBuffer metadataBytes = metadataHandle.readNonNull(); @@ -35,10 +41,22 @@ public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { } + /** + * Opens an existing Zarr group at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be read + */ public static Group open(Path path) throws IOException { return open(new StoreHandle(new FilesystemStore(path))); } + /** + * Opens an existing Zarr group at a specified storage location. + * + * @param path the storage location of the Zarr group + * @throws IOException if the metadata cannot be read + */ public static Group open(String path) throws IOException { return open(Paths.get(path)); } @@ -52,8 +70,8 @@ public static Group create() throws IOException { return new Group(new MemoryStore().resolve(), GroupMetadata.defaultValue()).writeMetadata(); } - /** - * Creates a new Zarr group with the provided metadata in an in-memory store. + /** + * Creates a new Zarr group with the provided attributes in an in-memory store. * * @param attributes the attributes of the Zarr group * @throws IOException if the metadata cannot be serialized @@ -63,7 +81,7 @@ public static Group create(@Nonnull Attributes attributes) throws IOException, Z return new Group(new MemoryStore().resolve(), new GroupMetadata(attributes)).writeMetadata(); } - /** + /** * Creates a new Zarr group with the provided metadata in an in-memory store. * * @param groupMetadata the metadata of the Zarr group @@ -85,10 +103,10 @@ public static Group create(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetad } /** - * Creates a new Zarr group with the provided metadata at a specified storage location. + * Creates a new Zarr group with the provided attributes at a specified storage location. * * @param storeHandle the storage location of the Zarr group - * @param attributes attributes of the Zarr group + * @param attributes the attributes of the Zarr group * @throws IOException if the metadata cannot be serialized * @throws ZarrException if the attributes are invalid */ @@ -100,7 +118,7 @@ public static Group create( } /** - * Creates a new Zarr group with the provided metadata at a specified storage location. + * Creates a new Zarr group with default metadata at a specified storage location. * * @param storeHandle the storage location of the Zarr group * @throws IOException if the metadata cannot be serialized @@ -115,6 +133,7 @@ public static Group create(@Nonnull StoreHandle storeHandle) throws IOException * @param path the storage location of the Zarr group * @param groupMetadata the metadata of the Zarr group * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid */ public static Group create(Path path, GroupMetadata groupMetadata) throws IOException, ZarrException { return create(new FilesystemStore(path).resolve(), groupMetadata); @@ -126,31 +145,42 @@ public static Group create(Path path, GroupMetadata groupMetadata) throws IOExce * @param path the storage location of the Zarr group * @param groupMetadata the metadata of the Zarr group * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid */ public static Group create(String path, GroupMetadata groupMetadata) throws IOException, ZarrException { return create(Paths.get(path), groupMetadata); } /** - * Creates a new Zarr group with the provided metadata at a specified storage location. + * Creates a new Zarr group with default metadata at a specified storage location. * * @param path the storage location of the Zarr group * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid */ public static Group create(Path path) throws IOException, ZarrException { return create(new FilesystemStore(path).resolve()); } /** - * Creates a new Zarr group with the provided metadata at a specified storage location. + * Creates a new Zarr group with default metadata at a specified storage location. * * @param path the storage location of the Zarr group * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the metadata is invalid */ public static Group create(String path) throws IOException, ZarrException { return create(Paths.get(path)); } + /** + * Retrieves a node (group or array) at the specified key within the current group. + * + * @param key the key of the node to retrieve + * @return the node at the specified key, or null if it does not exist + * @throws ZarrException if the node cannot be opened + * @throws IOException if there is an error accessing the storage + */ @Nullable public Node get(String key) throws ZarrException, IOException{ StoreHandle keyHandle = storeHandle.resolve(key); @@ -189,18 +219,21 @@ public Group createGroup(String key, Attributes attributes) * Creates a new subgroup with default metadata at the specified key. * * @param key the key of the new Zarr group within the current group + * @return the created subgroup * @throws IOException if the metadata cannot be serialized */ - public Group createGroup(String key) throws IOException { + public Group createGroup(String key) throws IOException{ return Group.create(storeHandle.resolve(key), GroupMetadata.defaultValue()); } /** * Creates a new array with the provided metadata at the specified key. * - * @param key the key of the new Zarr array within the current group + * @param key the key of the new Zarr array within the current group * @param arrayMetadata the metadata of the Zarr array + * @return the created array * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the array cannot be created */ public Array createArray(String key, ArrayMetadata arrayMetadata) throws IOException, ZarrException { @@ -233,16 +266,32 @@ private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { return this; } - public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { - GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes); - return writeMetadata(newGroupMetadata); - } - + /** + * Updates the attributes of the group using a mapper function. + * + * @param attributeMapper a function that takes the current attributes and returns the updated attributes + * @return the updated group + * @throws ZarrException if the new attributes are invalid + * @throws IOException if the metadata cannot be serialized + */ public Group updateAttributes(Function attributeMapper) throws ZarrException, IOException { return setAttributes(attributeMapper.apply(metadata.attributes)); } + /** + * Sets new attributes for the group, replacing any existing attributes. + * + * @param newAttributes the new attributes to set + * @return the updated group + * @throws ZarrException if the new attributes are invalid + * @throws IOException if the metadata cannot be serialized + */ + public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { + GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes); + return writeMetadata(newGroupMetadata); + } + @Override public String toString() { return String.format("", storeHandle); From a6512de5946467a046703e42c79f7c1ed1680997 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sun, 30 Nov 2025 20:52:43 +0100 Subject: [PATCH 11/12] make MemoryStore allways concurrent --- .../zarrjava/store/ConcurrentMemoryStore.java | 17 ------- .../dev/zarr/zarrjava/store/MemoryStore.java | 7 +-- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 44 +++---------------- 3 files changed, 8 insertions(+), 60 deletions(-) delete mode 100644 src/main/java/dev/zarr/zarrjava/store/ConcurrentMemoryStore.java diff --git a/src/main/java/dev/zarr/zarrjava/store/ConcurrentMemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/ConcurrentMemoryStore.java deleted file mode 100644 index 33482d3..0000000 --- a/src/main/java/dev/zarr/zarrjava/store/ConcurrentMemoryStore.java +++ /dev/null @@ -1,17 +0,0 @@ -package dev.zarr.zarrjava.store; - -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -public class ConcurrentMemoryStore extends MemoryStore { - @Override - protected Map, byte[]> map() { - return new ConcurrentHashMap<>(); - } - - @Override - public String toString() { - return String.format("", hashCode()); - } -} diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index 8126e70..a27aafe 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -4,14 +4,11 @@ import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; public class MemoryStore implements Store, Store.ListableStore { - private final Map, byte[]> map = map(); - - protected Map, byte[]> map(){ - return new HashMap<>(); - } + private final Map, byte[]> map = new ConcurrentHashMap<>(); List resolveKeys(String[] keys) { ArrayList resolvedKeys = new ArrayList<>(); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 3f510c4..4a369c9 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -83,28 +83,12 @@ public void testHttpStore() throws IOException, ZarrException { } @ParameterizedTest - @CsvSource({ - "false,MemoryStore", - "false,ConcurrentMemoryStore", - "true,ConcurrentMemoryStore", - }) - public void testMemoryStoreV3(boolean useParallel, String storeType) throws ZarrException, IOException { + @CsvSource({"false", "true",}) + public void testMemoryStoreV3(boolean useParallel) throws ZarrException, IOException { int[] testData = new int[1024 * 1024]; Arrays.setAll(testData, p -> p); - StoreHandle storeHandle; - switch (storeType) { - case "ConcurrentMemoryStore": - storeHandle = new ConcurrentMemoryStore().resolve(); - break; - case "MemoryStore": - storeHandle = new MemoryStore().resolve(); - break; - default: - throw new IllegalArgumentException("Unknown store type: " + storeType); - } - - Group group = Group.create(storeHandle); + Group group = Group.create(new MemoryStore().resolve()); Array array = group.createArray("array", b -> b .withShape(1024, 1024) .withDataType(DataType.UINT32) @@ -124,28 +108,12 @@ public void testMemoryStoreV3(boolean useParallel, String storeType) throws Zarr } @ParameterizedTest - @CsvSource({ - "false,MemoryStore", - "false,ConcurrentMemoryStore", - "true,ConcurrentMemoryStore", - }) - public void testMemoryStoreV2(boolean useParallel, String storeType) throws ZarrException, IOException { + @CsvSource({"false", "true",}) + public void testMemoryStoreV2(boolean useParallel) throws ZarrException, IOException { int[] testData = new int[1024 * 1024]; Arrays.setAll(testData, p -> p); - StoreHandle storeHandle; - switch (storeType) { - case "ConcurrentMemoryStore": - storeHandle = new ConcurrentMemoryStore().resolve(); - break; - case "MemoryStore": - storeHandle = new MemoryStore().resolve(); - break; - default: - throw new IllegalArgumentException("Unknown store type: " + storeType); - } - - dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(storeHandle); + dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(new MemoryStore().resolve()); dev.zarr.zarrjava.v2.Array array = group.createArray("array", b -> b .withShape(1024, 1024) .withDataType(dev.zarr.zarrjava.v2.DataType.UINT32) From 135ca6d4ba104de901a70220961aeb054bc9ef2b Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 5 Dec 2025 11:58:42 +0100 Subject: [PATCH 12/12] end index validation in MemoryStore --- src/main/java/dev/zarr/zarrjava/store/MemoryStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java index a27aafe..c1bbb9d 100644 --- a/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java +++ b/src/main/java/dev/zarr/zarrjava/store/MemoryStore.java @@ -44,7 +44,7 @@ public ByteBuffer get(String[] keys, long start, long end) { byte[] bytes = map.get(resolveKeys(keys)); if (bytes == null) return null; if (end < 0) end = bytes.length; - if (end > Integer.MAX_VALUE) throw new RuntimeException("TODO"); //TODO + if (end > Integer.MAX_VALUE) throw new IllegalArgumentException("End index too large"); return ByteBuffer.wrap(bytes, (int) start, (int) end); }