From 2f7b6bbd5faa8e2acb1033770dad63f73d5bf01f Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 24 Sep 2025 12:19:22 +0200 Subject: [PATCH 01/20] remove zarrita --- .github/workflows/ci.yml | 1 - src/test/python-scripts/parse_codecs.py | 32 +----------------------- src/test/python-scripts/zarrita_read.py | 29 --------------------- src/test/python-scripts/zarrita_write.py | 23 ----------------- 4 files changed, 1 insertion(+), 84 deletions(-) delete mode 100644 src/test/python-scripts/zarrita_read.py delete mode 100644 src/test/python-scripts/zarrita_write.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 759a84d..a35ab65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,6 @@ jobs: run: | uv venv && uv init uv add zarr - uv add zarrita - name: Download testdata run: | diff --git a/src/test/python-scripts/parse_codecs.py b/src/test/python-scripts/parse_codecs.py index d720706..6cd88ec 100644 --- a/src/test/python-scripts/parse_codecs.py +++ b/src/test/python-scripts/parse_codecs.py @@ -6,7 +6,6 @@ from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation from zarr.codecs.transpose import TransposeCodec from zarr.codecs.zstd import ZstdCodec -import zarrita import numcodecs def parse_codecs_zarr_python(codec_string: str, param_string: str, zarr_version: int = 3): @@ -51,33 +50,4 @@ def parse_codecs_zarr_python(codec_string: str, param_string: str, zarr_version: else: raise ValueError(f"Invalid codec: {codec_string}, zarr_version: {zarr_version}") - return compressor, serializer, filters - -def parse_codecs_zarrita(codec_string: str, param_string: str): - codec = [] - if codec_string == "blosc": - cname, shuffle, clevel = param_string.split("_") - codec = [zarrita.codecs.bytes_codec(), - zarrita.codecs.blosc_codec(typesize=4, cname=cname, shuffle=shuffle, clevel=int(clevel))] - elif codec_string == "gzip": - codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.gzip_codec(level=int(param_string))] - elif codec_string == "zstd": - level, checksum = param_string.split("_") - codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.zstd_codec(checksum=checksum == 'true', level=int(level))] - elif codec_string == "bytes": - codec = [zarrita.codecs.bytes_codec(endian=param_string.lower())] - elif codec_string == "transpose": - codec = [zarrita.codecs.transpose_codec((1, 0, 2)), zarrita.codecs.bytes_codec()] - elif codec_string == "sharding": - codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 2, 4), codecs=[zarrita.codecs.bytes_codec("little")], - index_location=zarrita.metadata.ShardingCodecIndexLocation.start if param_string == "start" - else zarrita.metadata.ShardingCodecIndexLocation.end), - elif codec_string == "sharding_nested": - codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 2, 4), - codecs=[zarrita.codecs.sharding_codec(chunk_shape=(2, 1, 2), codecs=[ - zarrita.codecs.bytes_codec("little")])]), - elif codec_string == "crc32c": - codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.crc32c_codec()] - else: - raise ValueError(f"Invalid codec: {codec_string}") - return codec + return compressor, serializer, filters \ No newline at end of file diff --git a/src/test/python-scripts/zarrita_read.py b/src/test/python-scripts/zarrita_read.py deleted file mode 100644 index 2769694..0000000 --- a/src/test/python-scripts/zarrita_read.py +++ /dev/null @@ -1,29 +0,0 @@ -import sys - -import numpy as np -import zarrita -from zarrita.metadata import ShardingCodecIndexLocation -from parse_codecs import parse_codecs_zarrita - -codec_string = sys.argv[1] -param_string = sys.argv[2] -codec = parse_codecs_zarrita(codec_string, param_string) - -store = zarrita.LocalStore(sys.argv[3]) -expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) - -a = zarrita.Array.open(store / 'write_to_zarrita' / codec_string / param_string) -read_data = a[:, :] -assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" - -b = zarrita.Array.create( - store / 'read_from_zarrita_expected' / codec_string / param_string, - shape=(16, 16, 16), - chunk_shape=(2, 4, 8), - dtype="uint32", - fill_value=0, - attributes={'test_key': 'test_value'}, - codecs=codec -) - -assert a.metadata == b.metadata, f"not equal: \n{a.metadata=}\n{b.metadata=}" diff --git a/src/test/python-scripts/zarrita_write.py b/src/test/python-scripts/zarrita_write.py deleted file mode 100644 index 4000f60..0000000 --- a/src/test/python-scripts/zarrita_write.py +++ /dev/null @@ -1,23 +0,0 @@ -import sys - -import zarrita -import numpy as np -from zarrita.metadata import ShardingCodecIndexLocation -from parse_codecs import parse_codecs_zarrita - -codec_string = sys.argv[1] -param_string = sys.argv[2] -codec = parse_codecs_zarrita(codec_string, param_string) - -store = zarrita.LocalStore(sys.argv[3]) -testdata = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) - -a = zarrita.Array.create( - store / 'read_from_zarrita' / codec_string / param_string, - shape=(16, 16, 16), - chunk_shape=(2, 4, 8), - dtype='int32', - codecs=codec, - attributes={'answer': 42} -) -a[:, :] = testdata From e05996e6c9f62952a46c1e0b19e51b3ec08035dc Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 24 Sep 2025 12:52:25 +0200 Subject: [PATCH 02/20] setup uv in ZarrPythonTests automatically --- .github/workflows/deploy.yml | 1 - README.md | 7 +---- .../dev/zarr/zarrjava/ZarrPythonTests.java | 29 +++++++++++++++---- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 47fbbca..e838449 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -43,7 +43,6 @@ jobs: run: | uv venv && uv init uv add zarr - uv add zarrita - name: Download testdata run: | diff --git a/README.md b/README.md index 0f0010b..7a87b50 100644 --- a/README.md +++ b/README.md @@ -43,12 +43,7 @@ array.write( ## Development Start-Guide ### Run Tests Locally -To be able to run the tests locally, make sure to have `python3.11` installed. -Also, you need to set up a venv for zarrita at the root of the project: -`python3.11 -m venv venv_zarrita`. - -Then install zarrita there with `venv_zarrita/Scripts/pip install zarrita` -for Windows and `venv_zarrita/bin/pip install zarrita` for Linux. +To be able to run the tests locally, make sure to have `python3.11` and `uv` installed. Furthermore, you will need the `l4_sample` test data: diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index c595005..7bc3b3e 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -39,12 +39,9 @@ public static void clearTestoutputFolder() throws IOException { Files.createDirectory(TESTOUTPUT); } - public void run_python_script(String scriptName, String... args) throws IOException, InterruptedException { + public static int runCommand(String... command) throws IOException, InterruptedException { ProcessBuilder pb = new ProcessBuilder(); - pb.command().add("uv"); - pb.command().add("run"); - pb.command().add(PYTHON_TEST_PATH.resolve(scriptName).toString()); - pb.command().addAll(Arrays.asList(args)); + pb.command().addAll(Arrays.asList(command)); Process process = pb.start(); BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); @@ -58,7 +55,27 @@ public void run_python_script(String scriptName, String... args) throws IOExcept System.err.println(line); } - int exitCode = process.waitFor(); + return process.waitFor(); + } + + @BeforeAll + public static void setupUV() { + try { + int exitCode = runCommand("uv", "version"); + if (exitCode != 0) { + //setup uv + assert runCommand("uv", "venv") == 0; + assert runCommand("uv", "init") == 0; + assert runCommand("uv", "add", "zarr") == 0; + } + } catch (IOException | InterruptedException e) { + throw new RuntimeException("uv not installed or not in PATH. See"); + } + } + + public void run_python_script(String scriptName, String... args) throws IOException, InterruptedException { + int exitCode = runCommand(Stream.concat(Stream.of("uv", "run", PYTHON_TEST_PATH.resolve(scriptName) + .toString()), Arrays.stream(args)).toArray(String[]::new)); assert exitCode == 0; } From 3ad5e8854b726388538466ae44c88dfb1a43a59e Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 24 Sep 2025 14:54:02 +0200 Subject: [PATCH 03/20] add core.Node --- .github/workflows/ci.yml | 5 -- .../java/dev/zarr/zarrjava/core/Array.java | 65 +++++++++---------- .../java/dev/zarr/zarrjava/core/Node.java | 25 +++++++ src/main/java/dev/zarr/zarrjava/v2/Array.java | 26 ++------ src/main/java/dev/zarr/zarrjava/v3/Array.java | 25 ++----- src/main/java/dev/zarr/zarrjava/v3/Group.java | 3 + src/main/java/dev/zarr/zarrjava/v3/Node.java | 29 +-------- .../dev/zarr/zarrjava/ZarrPythonTests.java | 44 +++++++++++-- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 40 +----------- 9 files changed, 116 insertions(+), 146 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/core/Node.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a35ab65..0108c47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,11 +36,6 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v6 - - name: Set up zarr-python - run: | - uv venv && uv init - uv add zarr - - name: Download testdata run: | mkdir testoutput diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index 0401b6b..4015c47 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -14,23 +14,24 @@ import java.util.Arrays; import java.util.stream.Stream; -public interface Array { +public abstract class Array extends Node { - ArrayMetadata metadata(); - - StoreHandle storeHandle(); + protected CodecPipeline codecPipeline; + protected abstract ArrayMetadata metadata(); - CodecPipeline codecPipeline(); + protected Array(StoreHandle storeHandle) throws ZarrException { + super(storeHandle); + } /** * Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array * needs be large enough for the write. * - * @param offset the offset where to write the data - * @param array the data to write + * @param offset the offset where to write the data + * @param array the data to write * @param parallel utilizes parallelism if true */ - default void write(long[] offset, ucar.ma2.Array array, boolean parallel) { + public void write(long[] offset, ucar.ma2.Array array, boolean parallel) { ArrayMetadata metadata = metadata(); if (offset.length != metadata.ndim()) { throw new IllegalArgumentException("'offset' needs to have rank '" + metadata.ndim() + "'."); @@ -82,19 +83,19 @@ default void write(long[] offset, ucar.ma2.Array array, boolean parallel) { * * @param chunkCoords The coordinates of the chunk as computed by the offset of the chunk divided * by the chunk shape. - * @param chunkArray The data to write into the chunk + * @param chunkArray The data to write into the chunk * @throws ZarrException throws ZarrException if the write fails */ - default void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws ZarrException { + public void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws ZarrException { ArrayMetadata metadata = metadata(); String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); - StoreHandle chunkHandle = storeHandle().resolve(chunkKeys); + StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); Object parsedFillValue = metadata.parsedFillValue(); if (parsedFillValue != null && MultiArrayUtils.allValuesEqual(chunkArray, parsedFillValue)) { chunkHandle.delete(); } else { - ByteBuffer chunkBytes = codecPipeline().encode(chunkArray); + ByteBuffer chunkBytes = codecPipeline.encode(chunkArray); chunkHandle.set(chunkBytes); } } @@ -108,22 +109,21 @@ default void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws Za * @throws ZarrException throws ZarrException if the requested chunk is outside the array's domain or if the read fails */ @Nonnull - default ucar.ma2.Array readChunk(long[] chunkCoords) - throws ZarrException { + public ucar.ma2.Array readChunk(long[] chunkCoords) throws ZarrException { ArrayMetadata metadata = metadata(); if (!chunkIsInArray(chunkCoords)) { throw new ZarrException("Attempting to read data outside of the array's domain."); } final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); - final StoreHandle chunkHandle = storeHandle().resolve(chunkKeys); + final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); ByteBuffer chunkBytes = chunkHandle.read(); if (chunkBytes == null) { return metadata.allocateFillValueChunk(); } - return codecPipeline().decode(chunkBytes); + return codecPipeline.decode(chunkBytes); } @@ -134,7 +134,7 @@ default ucar.ma2.Array readChunk(long[] chunkCoords) * * @param array the data to write */ - default void write(ucar.ma2.Array array) { + public void write(ucar.ma2.Array array) { write(new long[metadata().ndim()], array); } @@ -144,9 +144,9 @@ default void write(ucar.ma2.Array array) { * Utilizes no parallelism. * * @param offset the offset where to write the data - * @param array the data to write + * @param array the data to write */ - default void write(long[] offset, ucar.ma2.Array array) { + public void write(long[] offset, ucar.ma2.Array array) { write(offset, array, false); } @@ -154,10 +154,10 @@ default void write(long[] offset, ucar.ma2.Array array) { * Writes a ucar.ma2.Array into the Zarr array at the beginning of the Zarr array. The shape of * the Zarr array needs be large enough for the write. * - * @param array the data to write + * @param array the data to write * @param parallel utilizes parallelism if true */ - default void write(ucar.ma2.Array array, boolean parallel) { + public void write(ucar.ma2.Array array, boolean parallel) { write(new long[metadata().ndim()], array, parallel); } @@ -168,7 +168,7 @@ default void write(ucar.ma2.Array array, boolean parallel) { * @throws ZarrException throws ZarrException if the read fails */ @Nonnull - default ucar.ma2.Array read() throws ZarrException { + public ucar.ma2.Array read() throws ZarrException { return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape())); } @@ -177,11 +177,11 @@ default ucar.ma2.Array read() throws ZarrException { * Utilizes no parallelism. * * @param offset the offset where to start reading - * @param shape the shape of the data to read + * @param shape the shape of the data to read * @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails */ @Nonnull - default ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrException { + public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrException { return read(offset, shape, false); } @@ -192,11 +192,11 @@ default ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrE * @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails */ @Nonnull - default ucar.ma2.Array read(final boolean parallel) throws ZarrException { + public ucar.ma2.Array read(final boolean parallel) throws ZarrException { return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape()), parallel); } - default boolean chunkIsInArray(long[] chunkCoords) { + boolean chunkIsInArray(long[] chunkCoords) { final int[] chunkShape = metadata().chunkShape(); for (int dimIdx = 0; dimIdx < metadata().ndim(); dimIdx++) { if (chunkCoords[dimIdx] < 0 @@ -210,15 +210,14 @@ default boolean chunkIsInArray(long[] chunkCoords) { /** * Reads a part of the Zarr array based on a requested offset and shape into an ucar.ma2.Array. * - * @param offset the offset where to start reading - * @param shape the shape of the data to read + * @param offset the offset where to start reading + * @param shape the shape of the data to read * @param parallel utilizes parallelism if true * @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails */ @Nonnull - default ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean parallel) throws ZarrException { + public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean parallel) throws ZarrException { ArrayMetadata metadata = metadata(); - CodecPipeline codecPipeline = codecPipeline(); if (offset.length != metadata.ndim()) { throw new IllegalArgumentException("'offset' needs to have rank '" + metadata.ndim() + "'."); } @@ -258,7 +257,7 @@ default ucar.ma2.Array read(final long[] offset, final int[] shape, final boolea } final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); - final StoreHandle chunkHandle = storeHandle().resolve(chunkKeys); + final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); if (!chunkHandle.exists()) { return; } @@ -281,11 +280,11 @@ default ucar.ma2.Array read(final long[] offset, final int[] shape, final boolea return outputArray; } - default ArrayAccessor access() { + public ArrayAccessor access() { return new ArrayAccessor(this); } - final class ArrayAccessor { + public static final class ArrayAccessor { @Nullable long[] offset; @Nullable diff --git a/src/main/java/dev/zarr/zarrjava/core/Node.java b/src/main/java/dev/zarr/zarrjava/core/Node.java new file mode 100644 index 0000000..6f4b0a4 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/Node.java @@ -0,0 +1,25 @@ +package dev.zarr.zarrjava.core; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.codec.CodecRegistry; +import javax.annotation.Nonnull; + +public abstract class Node { + + @Nonnull + public final StoreHandle storeHandle; + + protected Node(@Nonnull StoreHandle storeHandle) { + this.storeHandle = storeHandle; + } + + public static ObjectMapper makeObjectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new Jdk8Module()); + objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); + return objectMapper; + } + +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 52d2268..8ef42c2 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -17,15 +17,17 @@ import java.util.function.Function; import java.util.stream.Collectors; -public class Array implements dev.zarr.zarrjava.core.Array { +public class Array extends dev.zarr.zarrjava.core.Array { static final String ZARRAY = ".zarray"; public ArrayMetadata metadata; - public StoreHandle storeHandle; - CodecPipeline codecPipeline; + + protected ArrayMetadata metadata() { + return metadata; + } protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOException, ZarrException { - this.storeHandle = storeHandle; + super(storeHandle); this.metadata = arrayMetadata; this.codecPipeline = new CodecPipeline(Utils.concatArrays( new Codec[]{}, @@ -127,20 +129,4 @@ public String toString() { metadata.dataType ); } - - @Override - public ArrayMetadata metadata() { - return metadata; - } - - @Override - public StoreHandle storeHandle() { - return storeHandle; - } - - @Override - public CodecPipeline codecPipeline() { - return codecPipeline; - } - } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index f21b020..f9781e2 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -14,13 +14,15 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; -public class Array extends Node implements dev.zarr.zarrjava.core.Array { +public class Array extends dev.zarr.zarrjava.core.Array implements Node { public ArrayMetadata metadata; - CodecPipeline codecPipeline; - protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) - throws ZarrException { + protected ArrayMetadata metadata(){ + return metadata; + } + + protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws ZarrException { super(storeHandle); this.metadata = arrayMetadata; this.codecPipeline = new CodecPipeline(arrayMetadata.codecs, arrayMetadata.coreArrayMetadata); @@ -113,21 +115,6 @@ public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadat return ArrayMetadataBuilder.fromArrayMetadata(existingMetadata); } - @Override - public CodecPipeline codecPipeline() { - return codecPipeline; - } - - - - - @Override - public ArrayMetadata metadata() { - return metadata; - } - - - private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { ObjectMapper objectMapper = makeObjectMapper(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(newArrayMetadata)); diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 2abcff3..2c7657b 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Node; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import java.io.IOException; @@ -13,6 +14,8 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static dev.zarr.zarrjava.v3.Node.ZARR_JSON; + public class Group extends Node { public GroupMetadata metadata; diff --git a/src/main/java/dev/zarr/zarrjava/v3/Node.java b/src/main/java/dev/zarr/zarrjava/v3/Node.java index 4362999..a019065 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Node.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Node.java @@ -1,31 +1,6 @@ package dev.zarr.zarrjava.v3; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; -import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.v3.codec.CodecRegistry; -import javax.annotation.Nonnull; - -public class Node { - - public final static String ZARR_JSON = "zarr.json"; - - @Nonnull - public final StoreHandle storeHandle; - - protected Node(@Nonnull StoreHandle storeHandle) { - this.storeHandle = storeHandle; - } - - public static ObjectMapper makeObjectMapper() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerModule(new Jdk8Module()); - objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); - return objectMapper; - } - - public StoreHandle storeHandle() { - return storeHandle; - } +public interface Node { + String ZARR_JSON = "zarr.json"; } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 7bc3b3e..414eae0 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -1,5 +1,7 @@ package dev.zarr.zarrjava; +import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdCompressCtx; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.v3.Array; @@ -11,10 +13,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; +import java.io.*; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -66,7 +66,7 @@ public static void setupUV() { //setup uv assert runCommand("uv", "venv") == 0; assert runCommand("uv", "init") == 0; - assert runCommand("uv", "add", "zarr") == 0; + assert runCommand("uv", "add", "zarr", "zstandard") == 0; } } catch (IOException | InterruptedException e) { throw new RuntimeException("uv not installed or not in PATH. See"); @@ -261,4 +261,38 @@ public void testWriteReadWithZarrPythonV2(String compressor, String compressorPa //read in zarr_python run_python_script("zarr_python_read_v2.py", compressor, compressorParam, storeHandle.toPath().toString()); } + + @CsvSource({"0,true", "0,false", "5, true", "10, false"}) + @ParameterizedTest + public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException, InterruptedException { + //compress using ZstdCompressCtx + int number = 123456; + byte[] src = ByteBuffer.allocate(4).putInt(number).array(); + byte[] compressed; + try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { + ctx.setLevel(clevel); + ctx.setChecksum(checksumFlag); + compressed = ctx.compress(src); + } + //decompress with Zstd.decompress + long originalSize = Zstd.decompressedSize(compressed); + byte[] decompressed = Zstd.decompress(compressed, (int) originalSize); + Assertions.assertEquals(number, ByteBuffer.wrap(decompressed).getInt()); + + //write compressed to file + String compressedDataPath = TESTOUTPUT.resolve("compressed" + clevel + checksumFlag + ".bin").toString(); + try (FileOutputStream fos = new FileOutputStream(compressedDataPath)) { + fos.write(compressed); + } + + //decompress in python + int exitCode = ZarrPythonTests.runCommand( + "uv", + "run", + PYTHON_TEST_PATH.resolve("zstd_decompress.py").toString(), + compressedDataPath, + Integer.toString(number) + ); + assert exitCode == 0; + } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 647a3b3..2f2617b 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -5,8 +5,7 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.luben.zstd.Zstd; -import com.github.luben.zstd.ZstdCompressCtx; +import dev.zarr.zarrjava.core.Node; import dev.zarr.zarrjava.store.*; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.v3.*; @@ -53,40 +52,7 @@ public static void clearTestoutputFolder() throws IOException { Files.createDirectory(TESTOUTPUT); } - @CsvSource({"0,true", "0,false", "5, true", "10, false"}) - @ParameterizedTest - public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException, InterruptedException { - //compress using ZstdCompressCtx - int number = 123456; - byte[] src = ByteBuffer.allocate(4).putInt(number).array(); - byte[] compressed; - try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { - ctx.setLevel(clevel); - ctx.setChecksum(checksumFlag); - compressed = ctx.compress(src); - } - //decompress with Zstd.decompress - long originalSize = Zstd.decompressedSize(compressed); - byte[] decompressed = Zstd.decompress(compressed, (int) originalSize); - Assertions.assertEquals(number, ByteBuffer.wrap(decompressed).getInt()); - - //write compressed to file - String compressedDataPath = TESTOUTPUT.resolve("compressed" + clevel + checksumFlag + ".bin").toString(); - try (FileOutputStream fos = new FileOutputStream(compressedDataPath)) { - fos.write(compressed); - } - //decompress in python - Process process = new ProcessBuilder( - "uv", - "run", - PYTHON_TEST_PATH.resolve("zstd_decompress.py").toString(), - compressedDataPath, - Integer.toString(number) - ).start(); - int exitCode = process.waitFor(); - assert exitCode == 0; - } static Stream> invalidCodecBuilder() { @@ -594,7 +560,7 @@ public void testV2noFillValue(dev.zarr.zarrjava.v2.DataType dataType) throws IOE .withChunks(4, 5) .build() ); - Assertions.assertNull(array.metadata().fillValue); + Assertions.assertNull(array.metadata.fillValue); ucar.ma2.Array outArray = array.read(new long[]{0, 0}, new int[]{1, 1}); if (dataType == dev.zarr.zarrjava.v2.DataType.BOOL) { @@ -606,6 +572,6 @@ public void testV2noFillValue(dev.zarr.zarrjava.v2.DataType dataType) throws IOE dev.zarr.zarrjava.v2.Array array2 = dev.zarr.zarrjava.v2.Array.open( storeHandle ); - Assertions.assertNull(array2.metadata().fillValue); + Assertions.assertNull(array2.metadata.fillValue); } } From 0a5930a30273a01bb1d11d218d61dedc7fbf5c0b Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 30 Sep 2025 12:57:34 +0200 Subject: [PATCH 04/20] implement and test v2 Group with metadata handling --- .../java/dev/zarr/zarrjava/core/Array.java | 16 ++-- .../dev/zarr/zarrjava/core/ArrayMetadata.java | 39 ++++++-- .../java/dev/zarr/zarrjava/core/Group.java | 36 +++++++ .../dev/zarr/zarrjava/core/GroupMetadata.java | 3 + .../java/dev/zarr/zarrjava/core/Node.java | 10 -- src/main/java/dev/zarr/zarrjava/v2/Array.java | 13 +-- .../dev/zarr/zarrjava/v2/ArrayMetadata.java | 41 ++------ src/main/java/dev/zarr/zarrjava/v2/Group.java | 94 +++++++++++++++++++ .../dev/zarr/zarrjava/v2/GroupMetadata.java | 27 ++++++ src/main/java/dev/zarr/zarrjava/v2/Node.java | 18 ++++ src/main/java/dev/zarr/zarrjava/v3/Array.java | 1 + .../dev/zarr/zarrjava/v3/ArrayMetadata.java | 36 ++----- src/main/java/dev/zarr/zarrjava/v3/Group.java | 37 ++------ src/main/java/dev/zarr/zarrjava/v3/Node.java | 11 +++ .../dev/zarr/zarrjava/ZarrPythonTests.java | 32 +++++++ src/test/java/dev/zarr/zarrjava/ZarrTest.java | 20 +++- .../python-scripts/zarr_python_group_v2.py | 27 ++++++ 17 files changed, 331 insertions(+), 130 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/core/Group.java create mode 100644 src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java create mode 100644 src/main/java/dev/zarr/zarrjava/v2/Group.java create mode 100644 src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java create mode 100644 src/main/java/dev/zarr/zarrjava/v2/Node.java create mode 100644 src/test/python-scripts/zarr_python_group_v2.py diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index 4015c47..1a8c206 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -43,7 +43,7 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) { int[] shape = array.getShape(); final int[] chunkShape = metadata.chunkShape(); - Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape(), chunkShape, offset, shape)); + Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape)); if (parallel) { chunkStream = chunkStream.parallel(); } @@ -51,7 +51,7 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) { chunkCoords -> { try { final IndexingUtils.ChunkProjection chunkProjection = - IndexingUtils.computeProjection(chunkCoords, metadata.shape(), chunkShape, offset, + IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset, shape ); @@ -169,7 +169,7 @@ public void write(ucar.ma2.Array array, boolean parallel) { */ @Nonnull public ucar.ma2.Array read() throws ZarrException { - return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape())); + return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape)); } /** @@ -193,14 +193,14 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrEx */ @Nonnull public ucar.ma2.Array read(final boolean parallel) throws ZarrException { - return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape()), parallel); + return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape), parallel); } boolean chunkIsInArray(long[] chunkCoords) { final int[] chunkShape = metadata().chunkShape(); for (int dimIdx = 0; dimIdx < metadata().ndim(); dimIdx++) { if (chunkCoords[dimIdx] < 0 - || chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape()[dimIdx]) { + || chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape[dimIdx]) { return false; } } @@ -225,7 +225,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean throw new IllegalArgumentException("'shape' needs to have rank '" + metadata.ndim() + "'."); } for (int dimIdx = 0; dimIdx < metadata.ndim(); dimIdx++) { - if (offset[dimIdx] < 0 || offset[dimIdx] + shape[dimIdx] > metadata.shape()[dimIdx]) { + if (offset[dimIdx] < 0 || offset[dimIdx] + shape[dimIdx] > metadata.shape[dimIdx]) { throw new ZarrException("Requested data is outside of the array's domain."); } } @@ -237,7 +237,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean final ucar.ma2.Array outputArray = ucar.ma2.Array.factory(metadata.dataType().getMA2DataType(), shape); - Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape(), chunkShape, offset, shape)); + Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape)); if (parallel) { chunkStream = chunkStream.parallel(); } @@ -245,7 +245,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean chunkCoords -> { try { final IndexingUtils.ChunkProjection chunkProjection = - IndexingUtils.computeProjection(chunkCoords, metadata.shape(), chunkShape, offset, + IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset, shape ); diff --git a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java index fc42294..aaa004e 100644 --- a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java @@ -1,5 +1,7 @@ package dev.zarr.zarrjava.core; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; @@ -10,23 +12,40 @@ import java.nio.ByteBuffer; import java.util.Arrays; -public interface ArrayMetadata { - int ndim(); +public abstract class ArrayMetadata { - int[] chunkShape(); + public final long[] shape; - long[] shape(); + @JsonProperty("fill_value") + public final Object fillValue; + @JsonIgnore + public final Object parsedFillValue; - DataType dataType(); + public ArrayMetadata(long[] shape, Object fillValue, DataType dataType) throws ZarrException { + this.shape = shape; + this.fillValue = fillValue; + this.parsedFillValue = parseFillValue(fillValue, dataType); + } - Array allocateFillValueChunk(); + public int ndim() { + return shape.length; + } - ChunkKeyEncoding chunkKeyEncoding(); + public abstract int[] chunkShape(); - Object parsedFillValue(); + public abstract DataType dataType(); - static Object parseFillValue(Object fillValue, @Nonnull DataType dataType) + public abstract Array allocateFillValueChunk(); + + public abstract ChunkKeyEncoding chunkKeyEncoding(); + + public abstract Object parsedFillValue(); + + public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType) throws ZarrException { + if (fillValue == null) { + return null; + } boolean dataTypeIsBool = dataType == dev.zarr.zarrjava.v3.DataType.BOOL || dataType == dev.zarr.zarrjava.v2.DataType.BOOL; boolean dataTypeIsByte = dataType == dev.zarr.zarrjava.v3.DataType.INT8 || dataType == dev.zarr.zarrjava.v2.DataType.INT8 || dataType == dev.zarr.zarrjava.v3.DataType.UINT8 || dataType == dev.zarr.zarrjava.v2.DataType.UINT8; boolean dataTypeIsShort = dataType == dev.zarr.zarrjava.v3.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16 || dataType == dev.zarr.zarrjava.v3.DataType.UINT16 || dataType == dev.zarr.zarrjava.v2.DataType.UINT16; @@ -128,7 +147,7 @@ else if (fillValueString.startsWith("0b") || fillValueString.startsWith("0x")) { throw new ZarrException("Invalid fill value '" + fillValue + "'."); } - final class CoreArrayMetadata { + public static final class CoreArrayMetadata { public final long[] shape; public final int[] chunkShape; diff --git a/src/main/java/dev/zarr/zarrjava/core/Group.java b/src/main/java/dev/zarr/zarrjava/core/Group.java new file mode 100644 index 0000000..f3be8b7 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/Group.java @@ -0,0 +1,36 @@ +package dev.zarr.zarrjava.core; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.StoreHandle; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Objects; +import java.util.stream.Stream; + +public abstract class Group extends Node { + + protected Group(@Nonnull StoreHandle storeHandle) { + super(storeHandle); + } + + @Nullable + public abstract Node get(String key) throws ZarrException; + + public Stream list() { + return storeHandle.list() + .map(key -> { + try { + return get(key); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + }) + .filter(Objects::nonNull); + } + + public Node[] listAsArray() { + try (Stream nodeStream = list()) { + return nodeStream.toArray(Node[]::new); + } + } +} diff --git a/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java new file mode 100644 index 0000000..695bfdf --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java @@ -0,0 +1,3 @@ +package dev.zarr.zarrjava.core; + +public abstract class GroupMetadata {} diff --git a/src/main/java/dev/zarr/zarrjava/core/Node.java b/src/main/java/dev/zarr/zarrjava/core/Node.java index 6f4b0a4..1c5fc9f 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Node.java +++ b/src/main/java/dev/zarr/zarrjava/core/Node.java @@ -1,9 +1,6 @@ package dev.zarr.zarrjava.core; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.v3.codec.CodecRegistry; import javax.annotation.Nonnull; public abstract class Node { @@ -15,11 +12,4 @@ protected Node(@Nonnull StoreHandle storeHandle) { this.storeHandle = storeHandle; } - public static ObjectMapper makeObjectMapper() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerModule(new Jdk8Module()); - objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); - return objectMapper; - } - } diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 8ef42c2..9f76e7f 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -1,13 +1,11 @@ package dev.zarr.zarrjava.v2; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.core.codec.CodecPipeline; import dev.zarr.zarrjava.v2.codec.Codec; -import dev.zarr.zarrjava.v2.codec.CodecRegistry; import dev.zarr.zarrjava.v2.codec.core.BytesCodec; import javax.annotation.Nonnull; @@ -16,10 +14,10 @@ import java.util.Arrays; import java.util.function.Function; import java.util.stream.Collectors; +import static dev.zarr.zarrjava.v2.Node.makeObjectMapper; -public class Array extends dev.zarr.zarrjava.core.Array { +public class Array extends dev.zarr.zarrjava.core.Array implements Node { - static final String ZARRAY = ".zarray"; public ArrayMetadata metadata; protected ArrayMetadata metadata() { @@ -55,13 +53,6 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept ); } - public static ObjectMapper makeObjectMapper() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerModule(new Jdk8Module()); - objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); - return objectMapper; - } - /** * Creates a new Zarr array with the provided metadata at a specified storage location. This diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index d0fa0e1..3a21d08 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -13,39 +13,31 @@ import javax.annotation.Nullable; -import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; - -public class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata { +public class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { static final int ZARR_FORMAT = 2; @JsonProperty("zarr_format") public final int zarrFormat = ZARR_FORMAT; - public long[] shape; - public int[] chunks; + public final int[] chunks; @JsonProperty("dtype") - public DataType dataType; + public final DataType dataType; @JsonIgnore public final Endianness endianness; @JsonProperty("order") - public Order order; + public final Order order; @JsonProperty("dimension_separator") - public Separator dimensionSeparator; - - @JsonProperty("fill_value") - public Object fillValue; - @JsonIgnore - public final Object parsedFillValue; + public final Separator dimensionSeparator; @Nullable - public Codec[] filters; + public final Codec[] filters; @Nullable - public Codec compressor; + public final Codec compressor; @JsonIgnore public CoreArrayMetadata coreArrayMetadata; @@ -63,21 +55,14 @@ public ArrayMetadata( @Nullable @JsonProperty(value = "compressor", required = true) Codec compressor, @Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator ) throws ZarrException { - super(); + super(shape, fillValue, dataType); if (zarrFormat != this.zarrFormat) { throw new ZarrException( "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); } - this.shape = shape; this.chunks = chunks; this.dataType = dataType; this.endianness = dataType.getEndianness(); - this.fillValue = fillValue; - if (fillValue == null) { - this.parsedFillValue = null; - } else { - this.parsedFillValue = parseFillValue(fillValue, this.dataType); - } this.order = order; this.dimensionSeparator = dimensionSeparator; this.filters = filters; @@ -85,24 +70,18 @@ public ArrayMetadata( this.coreArrayMetadata = new ArrayMetadata.CoreArrayMetadata(shape, chunks, this.dataType, - parsedFillValue + this.parsedFillValue ); } - public int ndim() { - return shape.length; - } @Override public int[] chunkShape() { return chunks; } - @Override - public long[] shape() { - return shape; - } + @Override public DataType dataType() { diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java new file mode 100644 index 0000000..f3296ce --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -0,0 +1,94 @@ +package dev.zarr.zarrjava.v2; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.utils.Utils; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.function.Function; +import static dev.zarr.zarrjava.v2.Node.makeObjectMapper; + +public class Group extends dev.zarr.zarrjava.core.Group implements Node{ + public GroupMetadata metadata; + + protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException { + super(storeHandle); + this.metadata = groupMetadata; + } + + public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { + StoreHandle metadataHandle = storeHandle.resolve(ZGROUP); + ByteBuffer metadataBytes = metadataHandle.readNonNull(); + return new Group(storeHandle, makeObjectMapper() + .readValue(Utils.toArray(metadataBytes), GroupMetadata.class)); + } + + public static Group create( + @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata + ) throws IOException { + ObjectMapper objectMapper = makeObjectMapper(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(groupMetadata)); + storeHandle.resolve(ZGROUP).set(metadataBytes); + return new Group(storeHandle, groupMetadata); + } + + public static Group create( + @Nonnull StoreHandle storeHandle + ) throws IOException, ZarrException { + return new Group(storeHandle, new GroupMetadata()); + } + + @Nullable + public dev.zarr.zarrjava.core.Node get(String key) throws ZarrException { + StoreHandle keyHandle = storeHandle.resolve(key); + ObjectMapper objectMapper = makeObjectMapper(); + boolean isGroup = keyHandle.resolve(ZGROUP).exists(); + boolean isArray = keyHandle.resolve(ZARRAY).exists(); + if (isGroup && isArray) { + throw new ZarrException("Key '" + key + "' contains both a .zgroup and a .zarray file."); + } else if (isGroup) { + try { + return new Group(keyHandle, objectMapper.readValue( + Utils.toArray(keyHandle.resolve(ZGROUP).readNonNull()), GroupMetadata.class)); + } catch (IOException e) { + return null; + } + } else if (isArray) { + try { + return new dev.zarr.zarrjava.v2.Array(keyHandle, objectMapper.readValue( + Utils.toArray(keyHandle.resolve(ZARRAY).readNonNull()), ArrayMetadata.class)); + } catch (IOException e) { + throw new ZarrException("Failed to read array metadata for key '" + key + "'.", e); + } + } + return null; + } + + public Group createGroup(String key, GroupMetadata groupMetadata) + throws IOException, ZarrException { + return Group.create(storeHandle.resolve(key), groupMetadata); + } + + public Group createGroup(String key) throws IOException, ZarrException { + return Group.create(storeHandle.resolve(key), new GroupMetadata()); + } + + public Array createArray(String key, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(storeHandle.resolve(key), arrayMetadata); + } + + public Array createArray(String key, Function arrayMetadataBuilderMapper) + throws IOException, ZarrException { + return Array.create(storeHandle.resolve(key), arrayMetadataBuilderMapper, false); + } + + @Override + public String toString() { + return String.format("", storeHandle); + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java new file mode 100644 index 0000000..e32f106 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java @@ -0,0 +1,27 @@ +package dev.zarr.zarrjava.v2; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import dev.zarr.zarrjava.ZarrException; + +public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { + + static final int ZARR_FORMAT = 2; + @JsonProperty("zarr_format") + public final int zarrFormat = ZARR_FORMAT; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public GroupMetadata( + @JsonProperty(value = "zarr_format", required = true) int zarrFormat + ) throws ZarrException { + if (zarrFormat != this.zarrFormat) { + throw new ZarrException( + "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); + } + } + + public GroupMetadata() throws ZarrException { + this(ZARR_FORMAT); + } + +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/Node.java b/src/main/java/dev/zarr/zarrjava/v2/Node.java new file mode 100644 index 0000000..ec8f0b2 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/Node.java @@ -0,0 +1,18 @@ +package dev.zarr.zarrjava.v2; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import dev.zarr.zarrjava.v2.codec.CodecRegistry; + +public interface Node { + String ZARRAY = ".zarray"; + String ZGROUP = ".zgroup"; + + static ObjectMapper makeObjectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new Jdk8Module()); + objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); + return objectMapper; + } + +} diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index f9781e2..21743e4 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -13,6 +13,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; +import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; public class Array extends dev.zarr.zarrjava.core.Array implements Node { diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index d50d9d0..c1a8411 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -9,7 +9,6 @@ import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; -import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; import java.util.Arrays; import java.util.Map; @@ -18,9 +17,8 @@ import javax.annotation.Nullable; -public final class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata { - - static final String NODE_TYPE = "array"; +public final class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { + public static final String NODE_TYPE = "array"; static final int ZARR_FORMAT = 3; @JsonProperty("zarr_format") @@ -28,9 +26,6 @@ public final class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata @JsonProperty("node_type") public final String nodeType = NODE_TYPE; - - public final long[] shape; - @JsonProperty("data_type") public final DataType dataType; @@ -40,11 +35,6 @@ public final class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata @JsonProperty("chunk_key_encoding") public final ChunkKeyEncoding chunkKeyEncoding; - @JsonProperty("fill_value") - public final Object fillValue; - @JsonIgnore - public final Object parsedFillValue; - @JsonProperty("codecs") public final Codec[] codecs; @Nullable @@ -52,10 +42,10 @@ public final class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata public final Map attributes; @Nullable @JsonProperty("dimension_names") - public String[] dimensionNames; + public final String[] dimensionNames; @Nullable @JsonProperty("storage_transformers") - public Map[] storageTransformers; + public final Map[] storageTransformers; @JsonIgnore public CoreArrayMetadata coreArrayMetadata; @@ -88,6 +78,7 @@ public ArrayMetadata( @Nullable @JsonProperty(value = "attributes") Map attributes, @Nullable @JsonProperty(value = "storage_transformers") Map[] storageTransformers ) throws ZarrException { + super(shape, fillValue, dataType); if (zarrFormat != this.zarrFormat) { throw new ZarrException( "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); @@ -122,21 +113,17 @@ public ArrayMetadata( shardingCodec = getShardingIndexedCodec(shardingConfig.codecs); } } - - this.shape = shape; this.dataType = dataType; this.chunkGrid = chunkGrid; this.chunkKeyEncoding = chunkKeyEncoding; - this.fillValue = fillValue; - this.parsedFillValue = parseFillValue(fillValue, dataType); this.codecs = codecs; this.dimensionNames = dimensionNames; this.attributes = attributes; this.storageTransformers = storageTransformers; this.coreArrayMetadata = new CoreArrayMetadata(shape, ((RegularChunkGrid) chunkGrid).configuration.chunkShape, - dataType, - parsedFillValue + this.dataType, + this.parsedFillValue ); } @@ -155,10 +142,6 @@ public Object parsedFillValue() { return parsedFillValue; } - public int ndim() { - return shape.length; - } - public static Optional getShardingIndexedCodec(Codec[] codecs) { return Arrays.stream(codecs).filter(codec -> codec instanceof ShardingIndexedCodec).findFirst(); } @@ -167,11 +150,6 @@ public int[] chunkShape() { return ((RegularChunkGrid) this.chunkGrid).configuration.chunkShape; } - @Override - public long[] shape() { - return shape; - } - @Override public DataType dataType() { return dataType; diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 2c7657b..d4c0476 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -2,25 +2,22 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.core.Node; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Map; -import java.util.Objects; import java.util.function.Function; -import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; -import static dev.zarr.zarrjava.v3.Node.ZARR_JSON; -public class Group extends Node { +public class Group extends dev.zarr.zarrjava.core.Group implements Node { public GroupMetadata metadata; - Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException { + protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata) throws IOException { super(storeHandle); this.metadata = groupMetadata; } @@ -28,14 +25,14 @@ public class Group extends Node { public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { StoreHandle metadataHandle = storeHandle.resolve(ZARR_JSON); ByteBuffer metadataBytes = metadataHandle.readNonNull(); - return new Group(storeHandle, Node.makeObjectMapper() + return new Group(storeHandle, makeObjectMapper() .readValue(Utils.toArray(metadataBytes), GroupMetadata.class)); } public static Group create( @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata ) throws IOException { - ObjectMapper objectMapper = Node.makeObjectMapper(); + ObjectMapper objectMapper = makeObjectMapper(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(groupMetadata)); storeHandle.resolve(ZARR_JSON) .set(metadataBytes); @@ -54,9 +51,9 @@ public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, } @Nullable - public Node get(String key) throws ZarrException { + public dev.zarr.zarrjava.core.Node get(String key) throws ZarrException { StoreHandle keyHandle = storeHandle.resolve(key); - ObjectMapper objectMapper = Node.makeObjectMapper(); + ObjectMapper objectMapper = makeObjectMapper(); ByteBuffer metadataBytes = keyHandle.resolve(ZARR_JSON) .read(); if (metadataBytes == null) { @@ -107,26 +104,8 @@ public Array createArray(String key, return Array.create(storeHandle.resolve(key), arrayMetadataBuilderMapper, false); } - public Stream list() { - return storeHandle.list() - .map(key -> { - try { - return get(key); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - }) - .filter(Objects::nonNull); - } - - public Node[] listAsArray() { - try (Stream nodeStream = list()) { - return nodeStream.toArray(Node[]::new); - } - } - private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { - ObjectMapper objectMapper = Node.makeObjectMapper(); + ObjectMapper objectMapper = makeObjectMapper(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(newGroupMetadata)); storeHandle.resolve(ZARR_JSON) .set(metadataBytes); diff --git a/src/main/java/dev/zarr/zarrjava/v3/Node.java b/src/main/java/dev/zarr/zarrjava/v3/Node.java index a019065..662e24b 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Node.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Node.java @@ -1,6 +1,17 @@ package dev.zarr.zarrjava.v3; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import dev.zarr.zarrjava.v3.codec.CodecRegistry; + public interface Node { String ZARR_JSON = "zarr.json"; + static ObjectMapper makeObjectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new Jdk8Module()); + objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); + return objectMapper; + } + } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 414eae0..d624a4d 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -4,12 +4,14 @@ import com.github.luben.zstd.ZstdCompressCtx; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v2.Group; import dev.zarr.zarrjava.v3.Array; import dev.zarr.zarrjava.v3.ArrayMetadataBuilder; import dev.zarr.zarrjava.v3.DataType; import dev.zarr.zarrjava.v3.codec.CodecBuilder; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -295,4 +297,34 @@ public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException ); assert exitCode == 0; } + + @Test + public void testGroupReadWriteV2() throws Exception { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("group_write"); + StoreHandle storeHandle2 = new FilesystemStore(TESTOUTPUT).resolve("group_read"); + Group group = Group.create(storeHandle); + dev.zarr.zarrjava.v2.DataType dataType = dev.zarr.zarrjava.v2.DataType.INT32; + dev.zarr.zarrjava.v2.Array array = group.createGroup("group").createArray("array", arrayMetadataBuilder -> arrayMetadataBuilder + .withShape(16, 16, 16) + .withDataType(dataType) + .withChunks(2, 4, 8) + ); + + array.write(ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{16, 16, 16}, testData)); + + run_python_script("zarr_python_group_v2.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString()); + + Group group2 = Group.open(storeHandle2); + Group subgroup = (Group) group2.get("group2"); + Assertions.assertNotNull(subgroup); + dev.zarr.zarrjava.v2.Array array2 = (dev.zarr.zarrjava.v2.Array) subgroup.get("array2"); + Assertions.assertNotNull(array2); + ucar.ma2.Array result = array2.read(); + Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); + } + } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 2f2617b..d912ff6 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -22,7 +22,6 @@ import ucar.ma2.MAMath; import java.io.*; -import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -34,6 +33,7 @@ import java.util.stream.Stream; import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; +import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; import static org.junit.Assert.assertThrows; public class ZarrTest { @@ -218,7 +218,7 @@ public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exceptio @Test public void testFileSystemStores() throws IOException, ZarrException { FilesystemStore fsStore = new FilesystemStore(TESTDATA); - ObjectMapper objectMapper = Node.makeObjectMapper(); + ObjectMapper objectMapper = makeObjectMapper(); GroupMetadata group = objectMapper.readValue( Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), @@ -574,4 +574,20 @@ public void testV2noFillValue(dev.zarr.zarrjava.v2.DataType dataType) throws IOE ); Assertions.assertNull(array2.metadata.fillValue); } + + @Test + public void testV2Group() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); + + dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(fsStore.resolve("v2_testgroup")); + dev.zarr.zarrjava.v2.Group group2 = group.createGroup("test2"); + dev.zarr.zarrjava.v2.Array array = group2.createArray("array", b -> + b.withShape(10, 10) + .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) + .withChunks(5, 5) + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); + + Assertions.assertArrayEquals(((dev.zarr.zarrjava.v2.Array) ((dev.zarr.zarrjava.v2.Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunks, new int[]{5, 5}); + } } diff --git a/src/test/python-scripts/zarr_python_group_v2.py b/src/test/python-scripts/zarr_python_group_v2.py new file mode 100644 index 0000000..5052ddf --- /dev/null +++ b/src/test/python-scripts/zarr_python_group_v2.py @@ -0,0 +1,27 @@ +import sys +from pathlib import Path + +import numpy as np + +import zarr +from zarr.storage import LocalStore + +store_path_read = Path(sys.argv[1]) + +expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) + +g = zarr.open_group(store=LocalStore(store_path_read), zarr_format=2) +a = g['group']['array'] +read_data = a[:, :] +assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" + +store_path_write = Path(sys.argv[2]) +g2 = zarr.create_group(store=LocalStore(store_path_write), zarr_format=2) +a2 = g2.create_group('group2').create_array( + name='array2', + shape=(16, 16, 16), + chunks=(2, 4, 8), + dtype="int32", + fill_value=0, +) +a2[:] = expected_data \ No newline at end of file From 7f512b3b91ccb19f11d828664cf9976153605f67 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 30 Sep 2025 13:21:30 +0200 Subject: [PATCH 05/20] fix inheritance of v3.codec.Codec --- src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java index d9631a1..afd76e0 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java @@ -5,7 +5,7 @@ import dev.zarr.zarrjava.v3.ArrayMetadata; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name") -public interface Codec extends dev.zarr.zarrjava.v2.codec.Codec { +public interface Codec extends dev.zarr.zarrjava.core.codec.Codec { long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException; } From bf4a1ea0f4a8d6cf1f97f0671b607d4eedcbd567 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 30 Sep 2025 15:12:24 +0200 Subject: [PATCH 06/20] add v2 sample --- testdata/v2_sample/.zattrs | 1 + testdata/v2_sample/.zgroup | 3 +++ testdata/v2_sample/subgroup/.zattrs | 1 + testdata/v2_sample/subgroup/.zgroup | 3 +++ testdata/v2_sample/subgroup/array/.zarray | 25 ++++++++++++++++++++++ testdata/v2_sample/subgroup/array/.zattrs | 1 + testdata/v2_sample/subgroup/array/0.0.0 | Bin 0 -> 84 bytes 7 files changed, 34 insertions(+) create mode 100644 testdata/v2_sample/.zattrs create mode 100644 testdata/v2_sample/.zgroup create mode 100644 testdata/v2_sample/subgroup/.zattrs create mode 100644 testdata/v2_sample/subgroup/.zgroup create mode 100644 testdata/v2_sample/subgroup/array/.zarray create mode 100644 testdata/v2_sample/subgroup/array/.zattrs create mode 100644 testdata/v2_sample/subgroup/array/0.0.0 diff --git a/testdata/v2_sample/.zattrs b/testdata/v2_sample/.zattrs new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/testdata/v2_sample/.zattrs @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/testdata/v2_sample/.zgroup b/testdata/v2_sample/.zgroup new file mode 100644 index 0000000..cab13da --- /dev/null +++ b/testdata/v2_sample/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} \ No newline at end of file diff --git a/testdata/v2_sample/subgroup/.zattrs b/testdata/v2_sample/subgroup/.zattrs new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/testdata/v2_sample/subgroup/.zattrs @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/testdata/v2_sample/subgroup/.zgroup b/testdata/v2_sample/subgroup/.zgroup new file mode 100644 index 0000000..cab13da --- /dev/null +++ b/testdata/v2_sample/subgroup/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} \ No newline at end of file diff --git a/testdata/v2_sample/subgroup/array/.zarray b/testdata/v2_sample/subgroup/array/.zarray new file mode 100644 index 0000000..24b42c6 --- /dev/null +++ b/testdata/v2_sample/subgroup/array/.zarray @@ -0,0 +1,25 @@ +{ + "shape": [ + 16, + 16, + 16 + ], + "chunks": [ + 2, + 4, + 8 + ], + "dtype": " Date: Tue, 30 Sep 2025 15:39:02 +0200 Subject: [PATCH 07/20] add static open function that auto-detects the Zarr version --- .../dev/zarr/zarrjava/core/AbstractNode.java | 15 ++++ .../java/dev/zarr/zarrjava/core/Array.java | 25 ++++++- .../java/dev/zarr/zarrjava/core/Group.java | 75 ++++++++++++------- .../java/dev/zarr/zarrjava/core/Node.java | 36 +++++++-- src/main/java/dev/zarr/zarrjava/v2/Group.java | 24 +----- src/main/java/dev/zarr/zarrjava/v2/Node.java | 33 +++++++- src/main/java/dev/zarr/zarrjava/v3/Group.java | 23 +----- .../dev/zarr/zarrjava/v3/GroupMetadata.java | 4 +- src/main/java/dev/zarr/zarrjava/v3/Node.java | 35 ++++++++- .../dev/zarr/zarrjava/v3/codec/Codec.java | 3 +- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 62 ++++++++++++++- 11 files changed, 251 insertions(+), 84 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/core/AbstractNode.java diff --git a/src/main/java/dev/zarr/zarrjava/core/AbstractNode.java b/src/main/java/dev/zarr/zarrjava/core/AbstractNode.java new file mode 100644 index 0000000..77fa2e9 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/AbstractNode.java @@ -0,0 +1,15 @@ +package dev.zarr.zarrjava.core; + +import dev.zarr.zarrjava.store.StoreHandle; + +import javax.annotation.Nonnull; + +public class AbstractNode implements Node { + + @Nonnull + public final StoreHandle storeHandle; + + protected AbstractNode(@Nonnull StoreHandle storeHandle) { + this.storeHandle = storeHandle; + } +} diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index 1a8c206..bd65850 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -10,11 +10,12 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.stream.Stream; -public abstract class Array extends Node { +public abstract class Array extends AbstractNode { protected CodecPipeline codecPipeline; protected abstract ArrayMetadata metadata(); @@ -23,6 +24,28 @@ protected Array(StoreHandle storeHandle) throws ZarrException { super(storeHandle); } + /** + * Opens an existing Zarr array at a specified storage location. Automatically detects the Zarr version. + * + * @param storeHandle the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(StoreHandle storeHandle) throws IOException, ZarrException { + boolean isV3 = storeHandle.resolve(ZARR_JSON).exists(); + boolean isV2 = storeHandle.resolve(ZARRAY).exists(); + if (isV3 && isV2) { + throw new ZarrException("Both Zarr v2 and v3 arrays found at the specified location."); + } else if (isV3) { + return dev.zarr.zarrjava.v3.Array.open(storeHandle); + } else if (isV2) { + return dev.zarr.zarrjava.v2.Array.open(storeHandle); + } else { + throw new ZarrException("No Zarr array found at the specified location."); + } + } + + /** * Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array * needs be large enough for the write. diff --git a/src/main/java/dev/zarr/zarrjava/core/Group.java b/src/main/java/dev/zarr/zarrjava/core/Group.java index f3be8b7..6f5585b 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Group.java +++ b/src/main/java/dev/zarr/zarrjava/core/Group.java @@ -1,36 +1,61 @@ package dev.zarr.zarrjava.core; + import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.io.IOException; import java.util.Objects; import java.util.stream.Stream; -public abstract class Group extends Node { - - protected Group(@Nonnull StoreHandle storeHandle) { - super(storeHandle); - } - - @Nullable - public abstract Node get(String key) throws ZarrException; - - public Stream list() { - return storeHandle.list() - .map(key -> { - try { - return get(key); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - }) - .filter(Objects::nonNull); - } - - public Node[] listAsArray() { - try (Stream nodeStream = list()) { - return nodeStream.toArray(Node[]::new); +public abstract class Group extends AbstractNode { + + protected Group(@Nonnull StoreHandle storeHandle) { + super(storeHandle); + } + + + /** + * Opens an existing Zarr group at a specified storage location. Automatically detects the Zarr version. + * + * @param storeHandle the storage location of the Zarr group + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr group cannot be opened + */ + public static Group open(StoreHandle storeHandle) throws IOException, ZarrException { + boolean isV3 = storeHandle.resolve(ZARR_JSON).exists(); + boolean isV2 = storeHandle.resolve(ZGROUP).exists(); + if (isV3 && isV2) { + throw new ZarrException("Both Zarr v2 and v3 groups found at " + storeHandle); + } else if (isV3) { + return dev.zarr.zarrjava.v3.Group.open(storeHandle); + } else if (isV2) { + return dev.zarr.zarrjava.v2.Group.open(storeHandle); + } else { + throw new ZarrException("No Zarr group found at " + storeHandle); + } + } + + + @Nullable + public abstract Node get(String key) throws ZarrException; + + public Stream list() { + return storeHandle.list() + .map(key -> { + try { + return get(key); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + }) + .filter(Objects::nonNull); + } + + public Node[] listAsArray() { + try (Stream nodeStream = list()) { + return nodeStream.toArray(Node[]::new); + } } - } } diff --git a/src/main/java/dev/zarr/zarrjava/core/Node.java b/src/main/java/dev/zarr/zarrjava/core/Node.java index 1c5fc9f..76bb4d6 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Node.java +++ b/src/main/java/dev/zarr/zarrjava/core/Node.java @@ -1,15 +1,37 @@ package dev.zarr.zarrjava.core; +import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; -import javax.annotation.Nonnull; -public abstract class Node { +import java.io.IOException; +import java.nio.file.NoSuchFileException; - @Nonnull - public final StoreHandle storeHandle; +public interface Node { - protected Node(@Nonnull StoreHandle storeHandle) { - this.storeHandle = storeHandle; - } + String ZARR_JSON = "zarr.json"; + String ZARRAY = ".zarray"; + String ZGROUP = ".zgroup"; + /** + * Opens an existing Zarr array or group at a specified storage location. Automatically detects the Zarr version. + * + * @param storeHandle the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + static Node open(StoreHandle storeHandle) throws IOException, ZarrException { + boolean isV3 = storeHandle.resolve(ZARR_JSON).exists(); + boolean isV2 = storeHandle.resolve(ZARRAY).exists() || storeHandle.resolve(ZGROUP).exists(); + + if (isV3 && isV2) { + throw new ZarrException("Both Zarr v2 and v3 nodes found at " + storeHandle); + } else if (isV3) { + return dev.zarr.zarrjava.v3.Node.open(storeHandle); + } else if (isV2) { + return dev.zarr.zarrjava.v2.Node.open(storeHandle); + } else { + throw new NoSuchFileException("No Zarr node found at " + storeHandle); + } + } } + diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index f3296ce..791c95c 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -43,29 +43,13 @@ public static Group create( } @Nullable - public dev.zarr.zarrjava.core.Node get(String key) throws ZarrException { + public Node get(String key) throws ZarrException { StoreHandle keyHandle = storeHandle.resolve(key); - ObjectMapper objectMapper = makeObjectMapper(); - boolean isGroup = keyHandle.resolve(ZGROUP).exists(); - boolean isArray = keyHandle.resolve(ZARRAY).exists(); - if (isGroup && isArray) { - throw new ZarrException("Key '" + key + "' contains both a .zgroup and a .zarray file."); - } else if (isGroup) { - try { - return new Group(keyHandle, objectMapper.readValue( - Utils.toArray(keyHandle.resolve(ZGROUP).readNonNull()), GroupMetadata.class)); - } catch (IOException e) { + try { + return Node.open(keyHandle); + } catch (IOException e) { return null; - } - } else if (isArray) { - try { - return new dev.zarr.zarrjava.v2.Array(keyHandle, objectMapper.readValue( - Utils.toArray(keyHandle.resolve(ZARRAY).readNonNull()), ArrayMetadata.class)); - } catch (IOException e) { - throw new ZarrException("Failed to read array metadata for key '" + key + "'.", e); - } } - return null; } public Group createGroup(String key, GroupMetadata groupMetadata) diff --git a/src/main/java/dev/zarr/zarrjava/v2/Node.java b/src/main/java/dev/zarr/zarrjava/v2/Node.java index ec8f0b2..d1cd6ab 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Node.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Node.java @@ -2,11 +2,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.v2.codec.CodecRegistry; -public interface Node { - String ZARRAY = ".zarray"; - String ZGROUP = ".zgroup"; +import java.io.IOException; +import java.nio.file.NoSuchFileException; + +public interface Node extends dev.zarr.zarrjava.core.Node { static ObjectMapper makeObjectMapper() { ObjectMapper objectMapper = new ObjectMapper(); @@ -15,4 +18,28 @@ static ObjectMapper makeObjectMapper() { return objectMapper; } + /** + * Opens an existing Zarr array or group at a specified storage location. + * + * @param storeHandle the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array or group cannot be opened + */ + static Node open(StoreHandle storeHandle) throws IOException, ZarrException { + boolean isGroup = storeHandle.resolve(ZGROUP).exists(); + boolean isArray = storeHandle.resolve(ZARRAY).exists(); + + if (isGroup && isArray) { + throw new ZarrException("Store handle '" + storeHandle + "' contains both a " + ZGROUP + " and a " + ZARRAY + " file."); + } else if (isGroup) { + return Group.open(storeHandle); + } else if (isArray) { + try { + return Array.open(storeHandle); + } catch (IOException e) { + throw new ZarrException("Failed to read array metadata for store handle '" + storeHandle + "'.", e); + } + } + throw new NoSuchFileException("Store handle '" + storeHandle + "' does not contain a " + ZGROUP + " or a " + ZARRAY + " file."); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index d4c0476..028c2b9 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -51,29 +51,10 @@ public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, } @Nullable - public dev.zarr.zarrjava.core.Node get(String key) throws ZarrException { + public Node get(String key) throws ZarrException { StoreHandle keyHandle = storeHandle.resolve(key); - ObjectMapper objectMapper = makeObjectMapper(); - ByteBuffer metadataBytes = keyHandle.resolve(ZARR_JSON) - .read(); - if (metadataBytes == null) { - return null; - } - byte[] metadataBytearray = Utils.toArray(metadataBytes); try { - String nodeType = objectMapper.readTree(metadataBytearray) - .get("node_type") - .asText(); - switch (nodeType) { - case ArrayMetadata.NODE_TYPE: - return new Array(keyHandle, - objectMapper.readValue(metadataBytearray, ArrayMetadata.class)); - case GroupMetadata.NODE_TYPE: - return new Group(keyHandle, - objectMapper.readValue(metadataBytearray, GroupMetadata.class)); - default: - throw new ZarrException("Unsupported node_type '" + nodeType + "' in " + keyHandle); - } + return Node.open(keyHandle); } catch (IOException e) { return null; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java index 3b35c39..df32c04 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java @@ -7,12 +7,12 @@ import java.util.Map; import javax.annotation.Nullable; -public final class GroupMetadata { +public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { static final String NODE_TYPE = "group"; static final int ZARR_FORMAT = 3; @JsonProperty("zarr_format") - public final int zarrFormat = 3; + public final int zarrFormat = ZARR_FORMAT; @JsonProperty("node_type") public final String nodeType = "group"; diff --git a/src/main/java/dev/zarr/zarrjava/v3/Node.java b/src/main/java/dev/zarr/zarrjava/v3/Node.java index 662e24b..ff4e168 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Node.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Node.java @@ -2,10 +2,16 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.codec.CodecRegistry; -public interface Node { - String ZARR_JSON = "zarr.json"; +import java.io.IOException; +import java.nio.ByteBuffer; + + +public interface Node extends dev.zarr.zarrjava.core.Node{ static ObjectMapper makeObjectMapper() { ObjectMapper objectMapper = new ObjectMapper(); @@ -14,4 +20,29 @@ static ObjectMapper makeObjectMapper() { return objectMapper; } + /** + * Opens an existing Zarr array or group at a specified storage location. + * + * @param storeHandle the storage location of the Zarr array or group + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array or group cannot be opened + */ + static Node open(StoreHandle storeHandle) throws IOException, ZarrException { + ObjectMapper objectMapper = makeObjectMapper(); + ByteBuffer metadataBytes = storeHandle.resolve(ZARR_JSON).readNonNull(); + byte[] metadataBytearray = Utils.toArray(metadataBytes); + String nodeType = objectMapper.readTree(metadataBytearray) + .get("node_type") + .asText(); + switch (nodeType) { + case ArrayMetadata.NODE_TYPE: + return new Array(storeHandle, + objectMapper.readValue(metadataBytearray, ArrayMetadata.class)); + case GroupMetadata.NODE_TYPE: + return new Group(storeHandle, + objectMapper.readValue(metadataBytearray, GroupMetadata.class)); + default: + throw new ZarrException("Unsupported node_type '" + nodeType + "' at " + storeHandle); + } + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java index afd76e0..68ee1a5 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java @@ -7,5 +7,4 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name") public interface Codec extends dev.zarr.zarrjava.core.codec.Codec { long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException; -} - +} \ No newline at end of file diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index d912ff6..36e7730 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -23,6 +23,7 @@ import java.io.*; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -40,7 +41,6 @@ public class ZarrTest { final static Path TESTDATA = Paths.get("testdata"); final static Path TESTOUTPUT = Paths.get("testoutput"); - final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); @BeforeAll public static void clearTestoutputFolder() throws IOException { @@ -590,4 +590,64 @@ public void testV2Group() throws IOException, ZarrException { Assertions.assertArrayEquals(((dev.zarr.zarrjava.v2.Array) ((dev.zarr.zarrjava.v2.Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunks, new int[]{5, 5}); } + + @Test + public void testGenericOpenV3() throws ZarrException, IOException { + StoreHandle arrayHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1"); + StoreHandle groupHandle = new FilesystemStore(TESTDATA).resolve("l4_sample"); + StoreHandle v2Handle = new FilesystemStore(TESTDATA).resolve("v2_sample"); + + Array array = (Array) Node.open(arrayHandle); + Assertions.assertEquals(4, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); + Assertions.assertEquals(4, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.v3.Node.open(arrayHandle); + Assertions.assertEquals(4, (array).metadata.shape.length); + + Group group = (Group) Node.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.core.Group.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.v3.Node.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Node.open(v2Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Group.open(v2Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Array.open(v2Handle)); + } + + @Test + public void testGenericOpenV2() throws ZarrException, IOException { + StoreHandle arrayHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", "subgroup", "array"); + StoreHandle groupHandle = new FilesystemStore(TESTDATA).resolve("v2_sample"); + StoreHandle v3Handle = new FilesystemStore(TESTDATA).resolve("l4_sample"); + + dev.zarr.zarrjava.v2.Array array = (dev.zarr.zarrjava.v2.Array) Node.open(arrayHandle); + Assertions.assertEquals(3, (array).metadata.shape.length); + + array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); + Assertions.assertEquals(3, (array).metadata.shape.length); + + array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.v2.Node.open(arrayHandle); + Assertions.assertEquals(3, (array).metadata.shape.length); + + dev.zarr.zarrjava.v2.Group group = (dev.zarr.zarrjava.v2.Group) Node.open(groupHandle); + Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); + + group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.core.Group.open(groupHandle); + Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); + + group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.v2.Node.open(groupHandle); + Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Node.open(v3Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Group.open(v3Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Array.open(v3Handle)); + } } From c03f3c6936cac092d0ee9291d076f38e31b9d099 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 2 Oct 2025 19:56:21 +0200 Subject: [PATCH 08/20] overload open and create for Path and String arguments --- .../java/dev/zarr/zarrjava/core/Array.java | 25 ++++++- .../java/dev/zarr/zarrjava/core/Group.java | 25 +++++++ .../java/dev/zarr/zarrjava/core/Node.java | 25 +++++++ src/main/java/dev/zarr/zarrjava/v2/Array.java | 53 +++++++++++++++ src/main/java/dev/zarr/zarrjava/v2/Group.java | 29 ++++++++- src/main/java/dev/zarr/zarrjava/v2/Node.java | 11 ++++ src/main/java/dev/zarr/zarrjava/v3/Array.java | 65 +++++++++++++++++++ src/main/java/dev/zarr/zarrjava/v3/Group.java | 28 ++++++++ src/main/java/dev/zarr/zarrjava/v3/Node.java | 11 ++++ 9 files changed, 270 insertions(+), 2 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index bd65850..c6cecdc 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.core; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.IndexingUtils; import dev.zarr.zarrjava.utils.MultiArrayUtils; @@ -12,6 +13,8 @@ import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.stream.Stream; @@ -44,7 +47,27 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept throw new ZarrException("No Zarr array found at the specified location."); } } + /** + * Opens an existing Zarr array at a specified storage location. Automatically detects the Zarr version. + * + * @param path the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } + /** + * Opens an existing Zarr array at a specified storage location. Automatically detects the Zarr version. + * + * @param path the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } /** * Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array @@ -357,4 +380,4 @@ public void write(@Nonnull ucar.ma2.Array content) throws ZarrException { } } -} \ No newline at end of file +} diff --git a/src/main/java/dev/zarr/zarrjava/core/Group.java b/src/main/java/dev/zarr/zarrjava/core/Group.java index 6f5585b..5cff477 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Group.java +++ b/src/main/java/dev/zarr/zarrjava/core/Group.java @@ -1,11 +1,14 @@ package dev.zarr.zarrjava.core; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Objects; import java.util.stream.Stream; @@ -38,6 +41,28 @@ public static Group open(StoreHandle storeHandle) throws IOException, ZarrExcept } + /** + * Opens an existing Zarr group at a specified storage location. Automatically detects the Zarr version. + * + * @param path the storage location of the Zarr group + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr group cannot be opened + */ + public static Group open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + /** + * Opens an existing Zarr group at a specified storage location. Automatically detects the Zarr version. + * + * @param path the storage location of the Zarr group + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr group cannot be opened + */ + public static Group open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } + @Nullable public abstract Node get(String key) throws ZarrException; diff --git a/src/main/java/dev/zarr/zarrjava/core/Node.java b/src/main/java/dev/zarr/zarrjava/core/Node.java index 76bb4d6..0b51193 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Node.java +++ b/src/main/java/dev/zarr/zarrjava/core/Node.java @@ -1,10 +1,13 @@ package dev.zarr.zarrjava.core; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import java.io.IOException; import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; public interface Node { @@ -33,5 +36,27 @@ static Node open(StoreHandle storeHandle) throws IOException, ZarrException { throw new NoSuchFileException("No Zarr node found at " + storeHandle); } } + + /** + * Opens an existing Zarr array or group at a specified storage location. Automatically detects the Zarr version. + * + * @param path the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + static Node open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + /** + * Opens an existing Zarr array or group at a specified storage location. Automatically detects the Zarr version. + * + * @param path the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + static Node open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 9f76e7f..50d5ea8 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.core.codec.CodecPipeline; @@ -11,6 +12,8 @@ import javax.annotation.Nonnull; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.function.Function; import java.util.stream.Collectors; @@ -53,7 +56,57 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept ); } + /** + * Opens an existing Zarr array at a specified storage location. + * + * @param path the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + /** + * Opens an existing Zarr array at a specified storage location. + * + * @param path the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } + + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param path the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(Path path, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return create(new StoreHandle(new FilesystemStore(path)), arrayMetadata); + } + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param path the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(String path, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return create(Paths.get(path), arrayMetadata); + } /** * Creates a new Zarr array with the provided metadata at a specified storage location. This * method will raise an exception if a Zarr array already exists at the specified storage diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index 791c95c..9dfd239 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; @@ -9,6 +10,8 @@ import javax.annotation.Nullable; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.function.Function; import static dev.zarr.zarrjava.v2.Node.makeObjectMapper; @@ -27,6 +30,14 @@ public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { .readValue(Utils.toArray(metadataBytes), GroupMetadata.class)); } + public static Group open(Path path) throws IOException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + public static Group open(String path) throws IOException { + return open(Paths.get(path)); + } + public static Group create( @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata ) throws IOException { @@ -39,7 +50,23 @@ public static Group create( public static Group create( @Nonnull StoreHandle storeHandle ) throws IOException, ZarrException { - return new Group(storeHandle, new GroupMetadata()); + return create(storeHandle, new GroupMetadata()); + } + + public static Group create(Path path, GroupMetadata groupMetadata) throws IOException { + return create(new StoreHandle(new FilesystemStore(path)), groupMetadata); + } + + public static Group create(String path, GroupMetadata groupMetadata) throws IOException { + return create(Paths.get(path), groupMetadata); + } + + public static Group create(Path path) throws IOException, ZarrException { + return create(new StoreHandle(new FilesystemStore(path))); + } + + public static Group create(String path) throws IOException, ZarrException { + return create(Paths.get(path)); } @Nullable diff --git a/src/main/java/dev/zarr/zarrjava/v2/Node.java b/src/main/java/dev/zarr/zarrjava/v2/Node.java index d1cd6ab..4623200 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Node.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Node.java @@ -3,11 +3,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.v2.codec.CodecRegistry; import java.io.IOException; import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; public interface Node extends dev.zarr.zarrjava.core.Node { @@ -42,4 +45,12 @@ static Node open(StoreHandle storeHandle) throws IOException, ZarrException { } throw new NoSuchFileException("Store handle '" + storeHandle + "' does not contain a " + ZGROUP + " or a " + ZARRAY + " file."); } + + static Node open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + static Node open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 21743e4..a688504 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -2,11 +2,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.core.codec.CodecPipeline; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -47,6 +50,58 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept ); } + /** + * Opens an existing Zarr array at a specified storage location using a Path. + * + * @param path the storage location of the Zarr array + * @throws IOException if the metadata cannot be read + * @throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(Path path) throws IOException, ZarrException { + return open(new FilesystemStore(path).resolve()); + } + + /** + * Opens an existing Zarr array at a specified storage location using a String path. + * + * @param path the storage location of the Zarr array + * @throws IOException if the metadata cannot be read + * @throws ZarrException if the Zarr array cannot be opened + */ + public static Array open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } + + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param path the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(Path path, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(new FilesystemStore(path).resolve(), arrayMetadata); + } + + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param path the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ + public static Array create(String path, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(Paths.get(path), arrayMetadata); + } + /** * Creates a new Zarr array with the provided metadata at a specified storage location. This * method will raise an exception if a Zarr array already exists at the specified storage @@ -106,6 +161,16 @@ public static Array create(StoreHandle storeHandle, arrayMetadataBuilderMapper.apply(new ArrayMetadataBuilder()).build(), existsOk); } + public static Array create(Path path, Function arrayMetadataBuilderMapper, boolean existsOk) + throws IOException, ZarrException { + return Array.create(new FilesystemStore(path).resolve(), arrayMetadataBuilderMapper, existsOk); + } + + public static Array create(String path, Function arrayMetadataBuilderMapper, boolean existsOk) + throws IOException, ZarrException { + return Array.create(Paths.get(path), arrayMetadataBuilderMapper, existsOk); + } + @Nonnull public static ArrayMetadataBuilder metadataBuilder() { return new ArrayMetadataBuilder(); diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 028c2b9..9c56f25 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -2,10 +2,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Map; import java.util.function.Function; import javax.annotation.Nonnull; @@ -29,6 +32,15 @@ public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { .readValue(Utils.toArray(metadataBytes), GroupMetadata.class)); } + + public static Group open(Path path) throws IOException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + public static Group open(String path) throws IOException { + return open(Paths.get(path)); + } + public static Group create( @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata ) throws IOException { @@ -50,6 +62,22 @@ public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, return create(storeHandle, GroupMetadata.defaultValue()); } + public static Group create(Path path, GroupMetadata groupMetadata) throws IOException, ZarrException { + return create(new FilesystemStore(path).resolve(), groupMetadata); + } + + public static Group create(String path, GroupMetadata groupMetadata) throws IOException, ZarrException { + return create(Paths.get(path), groupMetadata); + } + + public static Group create(Path path) throws IOException, ZarrException { + return create(new FilesystemStore(path).resolve()); + } + + public static Group create(String path) throws IOException, ZarrException { + return create(Paths.get(path)); + } + @Nullable public Node get(String key) throws ZarrException { StoreHandle keyHandle = storeHandle.resolve(key); diff --git a/src/main/java/dev/zarr/zarrjava/v3/Node.java b/src/main/java/dev/zarr/zarrjava/v3/Node.java index ff4e168..235e37f 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Node.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Node.java @@ -3,12 +3,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.codec.CodecRegistry; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; public interface Node extends dev.zarr.zarrjava.core.Node{ @@ -45,4 +48,12 @@ static Node open(StoreHandle storeHandle) throws IOException, ZarrException { throw new ZarrException("Unsupported node_type '" + nodeType + "' at " + storeHandle); } } + + static Node open(Path path) throws IOException, ZarrException { + return open(new StoreHandle(new FilesystemStore(path))); + } + + static Node open(String path) throws IOException, ZarrException { + return open(Paths.get(path)); + } } From 12a0f94308dff101005a68bed66e9f659b65d56c Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 2 Oct 2025 20:10:57 +0200 Subject: [PATCH 09/20] add API tests for open, create and refactor tests structure --- .../java/dev/zarr/zarrjava/ZarrApiTest.java | 253 +++++++ .../dev/zarr/zarrjava/ZarrPythonTests.java | 14 +- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 628 +----------------- .../java/dev/zarr/zarrjava/ZarrV2Test.java | 118 ++++ .../java/dev/zarr/zarrjava/ZarrV3Test.java | 470 +++++++++++++ 5 files changed, 844 insertions(+), 639 deletions(-) create mode 100644 src/test/java/dev/zarr/zarrjava/ZarrApiTest.java create mode 100644 src/test/java/dev/zarr/zarrjava/ZarrV2Test.java create mode 100644 src/test/java/dev/zarr/zarrjava/ZarrV3Test.java diff --git a/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java b/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java new file mode 100644 index 0000000..46b0b1e --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java @@ -0,0 +1,253 @@ +package dev.zarr.zarrjava; + +import dev.zarr.zarrjava.core.Node; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v2.ArrayMetadata; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.Group; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; + + +public class ZarrApiTest extends ZarrTest { + + @Test + public void testGenericOpenV3() throws ZarrException, IOException { + StoreHandle arrayHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1"); + StoreHandle groupHandle = new FilesystemStore(TESTDATA).resolve("l4_sample"); + StoreHandle v2Handle = new FilesystemStore(TESTDATA).resolve("v2_sample"); + + Array array = (Array) Node.open(arrayHandle); + Assertions.assertEquals(4, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); + Assertions.assertEquals(4, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.v3.Node.open(arrayHandle); + Assertions.assertEquals(4, (array).metadata.shape.length); + + Group group = (Group) Node.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.core.Group.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.v3.Node.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Node.open(v2Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Group.open(v2Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Array.open(v2Handle)); + } + + @Test + public void testGenericOpenOverloadsV3() throws ZarrException, IOException { + Path arrayPath = TESTDATA.resolve("l4_sample/color/1"); + Path groupPath = TESTDATA.resolve("l4_sample"); + Path v2GroupPath = TESTDATA.resolve("v2_sample"); + + Array array = (Array) Node.open(arrayPath); + Assertions.assertEquals(4, (array).metadata.shape.length); + array = (Array) Node.open(arrayPath.toString()); + Assertions.assertEquals(4, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath); + Assertions.assertEquals(4, (array).metadata.shape.length); + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath.toString()); + Assertions.assertEquals(4, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.v3.Node.open(arrayPath); + Assertions.assertEquals(4, (array).metadata.shape.length); + array = (Array) dev.zarr.zarrjava.v3.Node.open(arrayPath.toString()); + Assertions.assertEquals(4, (array).metadata.shape.length); + + Group group = (Group) Node.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("color")); + group = (Group) Node.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.core.Group.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("color")); + group = (Group) dev.zarr.zarrjava.core.Group.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.v3.Node.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("color")); + group = (Group) dev.zarr.zarrjava.v3.Node.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing").toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Node.open(v2GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Node.open(v2GroupPath.toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Group.open(v2GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Group.open(v2GroupPath.toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Array.open(v2GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Array.open(v2GroupPath.toString())); + } + + @Test + public void testGenericOpenV2() throws ZarrException, IOException { + StoreHandle arrayHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", "subgroup", "array"); + StoreHandle groupHandle = new FilesystemStore(TESTDATA).resolve("v2_sample"); + StoreHandle v3Handle = new FilesystemStore(TESTDATA).resolve("l4_sample"); + + dev.zarr.zarrjava.v2.Array array = (dev.zarr.zarrjava.v2.Array) Node.open(arrayHandle); + Assertions.assertEquals(3, (array).metadata.shape.length); + + array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); + Assertions.assertEquals(3, (array).metadata.shape.length); + + array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.v2.Node.open(arrayHandle); + Assertions.assertEquals(3, (array).metadata.shape.length); + + dev.zarr.zarrjava.v2.Group group = (dev.zarr.zarrjava.v2.Group) Node.open(groupHandle); + Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); + + group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.core.Group.open(groupHandle); + Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); + + group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.v2.Node.open(groupHandle); + Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Node.open(v3Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Group.open(v3Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Array.open(v3Handle)); + } + + @Test + public void testGenericOpenOverloadsV2() throws ZarrException, IOException { + Path arrayPath = TESTDATA.resolve("v2_sample/subgroup/array"); + Path groupPath = TESTDATA.resolve("v2_sample"); + Path v3GroupPath = TESTDATA.resolve("l4_sample"); + + dev.zarr.zarrjava.v2.Array array = (dev.zarr.zarrjava.v2.Array) Node.open(arrayPath); + Assertions.assertEquals(3, (array).metadata.shape.length); + array = (dev.zarr.zarrjava.v2.Array) Node.open(arrayPath.toString()); + Assertions.assertEquals(3, (array).metadata.shape.length); + + array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.core.Array.open(arrayPath); + Assertions.assertEquals(3, (array).metadata.shape.length); + array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.core.Array.open(arrayPath.toString()); + Assertions.assertEquals(3, (array).metadata.shape.length); + + array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.v2.Node.open(arrayPath); + Assertions.assertEquals(3, (array).metadata.shape.length); + array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.v2.Node.open(arrayPath.toString()); + Assertions.assertEquals(3, (array).metadata.shape.length); + + dev.zarr.zarrjava.v2.Group group = (dev.zarr.zarrjava.v2.Group) Node.open(groupPath); + Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); + group = (dev.zarr.zarrjava.v2.Group) Node.open(groupPath.toString()); + Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); + + group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.core.Group.open(groupPath); + Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); + group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.core.Group.open(groupPath.toString()); + Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); + + group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.v2.Node.open(groupPath); + Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); + group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.v2.Node.open(groupPath.toString()); + Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing").toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Node.open(v3GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Node.open(v3GroupPath.toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Group.open(v3GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Group.open(v3GroupPath.toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Array.open(v3GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Array.open(v3GroupPath.toString())); + } + + @Test + public void testCreateArrayV2() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateArrayV2"); + Path storeHandlePath = TESTOUTPUT.resolve("testCreateArrayV2Path"); + String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateArrayV2String")); + ArrayMetadata arrayMetadata = dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(10, 10) + .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) + .withChunks(5, 5) + .build(); + + dev.zarr.zarrjava.v2.Array.create(storeHandle, arrayMetadata); + Assertions.assertTrue(storeHandle.resolve(".zarray").exists()); + + dev.zarr.zarrjava.v2.Array.create(storeHandlePath, arrayMetadata); + Assertions.assertTrue(Files.exists(storeHandlePath.resolve(".zarray"))); + + dev.zarr.zarrjava.v2.Array.create(storeHandleString, arrayMetadata); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zarray"))); + } + + @Test + public void testCreateGroupV2() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV2"); + Path storeHandlePath = TESTOUTPUT.resolve("testCreateGroupV2Path"); + String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateGroupV2String")); + + dev.zarr.zarrjava.v2.Group.create(storeHandle); + Assertions.assertTrue(storeHandle.resolve(".zgroup").exists()); + + dev.zarr.zarrjava.v2.Group.create(storeHandlePath); + Assertions.assertTrue(Files.exists(storeHandlePath.resolve(".zgroup"))); + + dev.zarr.zarrjava.v2.Group.create(storeHandleString); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); + } + + @Test + public void testCreateArrayV3() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateArrayV3"); + Path storeHandlePath = TESTOUTPUT.resolve("testCreateArrayV3Path"); + String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateArrayV3String")); + dev.zarr.zarrjava.v3.ArrayMetadata arrayMetadata = dev.zarr.zarrjava.v3.Array.metadataBuilder() + .withShape(10, 10) + .withDataType(dev.zarr.zarrjava.v3.DataType.UINT8) + .withChunkShape(5, 5) + .build(); + + dev.zarr.zarrjava.v3.Array.create(storeHandle, arrayMetadata); + Assertions.assertTrue(storeHandle.resolve("zarr.json").exists()); + + dev.zarr.zarrjava.v3.Array.create(storeHandlePath, arrayMetadata); + Assertions.assertTrue(Files.exists(storeHandlePath.resolve("zarr.json"))); + + dev.zarr.zarrjava.v3.Array.create(storeHandleString, arrayMetadata); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); + } + + @Test + public void testCreateGroupV3() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV3"); + Path storeHandlePath = TESTOUTPUT.resolve("testCreateGroupV3Path"); + String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateGroupV3String")); + + dev.zarr.zarrjava.v3.Group.create(storeHandle); + Assertions.assertTrue(storeHandle.resolve("zarr.json").exists()); + + dev.zarr.zarrjava.v3.Group.create(storeHandlePath); + Assertions.assertTrue(Files.exists(storeHandlePath.resolve("zarr.json"))); + + dev.zarr.zarrjava.v3.Group.create(storeHandleString); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); + } + +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index d624a4d..9fcafc1 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -17,29 +17,17 @@ import java.io.*; import java.nio.ByteBuffer; -import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; -import java.util.Comparator; import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; -public class ZarrPythonTests { +public class ZarrPythonTests extends ZarrTest { - final static Path TESTOUTPUT = Paths.get("testoutput"); final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); - @BeforeAll - public static void clearTestoutputFolder() throws IOException { - if (Files.exists(TESTOUTPUT)) { - try (Stream walk = Files.walk(TESTOUTPUT)) { - walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); - } - } - Files.createDirectory(TESTOUTPUT); - } public static int runCommand(String... command) throws IOException, InterruptedException { ProcessBuilder pb = new ProcessBuilder(); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 36e7730..0c0be81 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -1,42 +1,15 @@ package dev.zarr.zarrjava; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.AnonymousAWSCredentials; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import dev.zarr.zarrjava.core.Node; -import dev.zarr.zarrjava.store.*; -import dev.zarr.zarrjava.utils.MultiArrayUtils; -import dev.zarr.zarrjava.v3.*; -import dev.zarr.zarrjava.v3.codec.CodecBuilder; -import dev.zarr.zarrjava.v3.codec.core.BytesCodec; -import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.MethodSource; -import org.junit.jupiter.params.provider.ValueSource; -import ucar.ma2.MAMath; -import java.io.*; +import java.io.File; +import java.io.IOException; import java.nio.file.Files; -import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; import java.util.stream.Stream; -import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; -import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; -import static org.junit.Assert.assertThrows; - public class ZarrTest { final static Path TESTDATA = Paths.get("testdata"); @@ -53,601 +26,4 @@ public static void clearTestoutputFolder() throws IOException { } - - - static Stream> invalidCodecBuilder() { - return Stream.of( - c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE), - c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE), - c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}), - c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}) - ); - } - - @ParameterizedTest - @MethodSource("invalidCodecBuilder") - public void testCheckInvalidCodecConfiguration(Function codecBuilder) throws Exception { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode())); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(new long[]{4, 4}) - .withDataType(DataType.UINT32) - .withChunkShape(new int[]{2, 2}) - .withCodecs(codecBuilder); - - assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build())); - } - - @Test - public void testLargerChunkSizeThanArraySize() throws ZarrException, IOException { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("larger_chunk_size_than_array"); - ArrayMetadata metadata = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(32, 32, 32) - .withFillValue(0) - .build(); - Array writeArray = Array.create(storeHandle, metadata); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); - - //read in zarr-java - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - } - - static Stream invalidChunkSizes() { - return Stream.of( - new int[]{1}, - new int[]{1, 1, 1} - ); - } - - @ParameterizedTest - @MethodSource("invalidChunkSizes") - public void testCheckInvalidChunkDimensions(int[] chunkSize) { - long[] shape = new long[]{4, 4}; - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_chunksize"); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(shape) - .withDataType(DataType.UINT32) - .withChunkShape(chunkSize); - - assertThrows(ZarrException.class, builder::build); - } - - static Stream invalidShardSizes() { - return Stream.of( - new int[]{4}, //wrong dims - new int[]{4, 4, 4}, //wrong dims - new int[]{1, 1}, //smaller than inner chunk shape - new int[]{5, 5}, //no exact multiple of inner chunk shape - new int[]{2, 1}, //smaller than inner chunk shape in 2nd dimension - new int[]{2, 5} //no exact multiple of inner chunk shape in 2nd dimension - ); - } - - @ParameterizedTest - @MethodSource("invalidShardSizes") - public void testCheckShardingBounds(int[] shardSize) throws Exception { - long[] shape = new long[]{10, 10}; - int[] innerChunkSize = new int[]{2, 2}; - - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(shape) - .withDataType(DataType.UINT32).withChunkShape(shardSize); - - if (false) { - int[] nestedChunkSize = new int[]{4, 4}; - builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2}, c1 -> c1.withSharding(nestedChunkSize, c2 -> c2.withBytes("LITTLE")))); - } - builder = builder.withCodecs(c -> c.withSharding(innerChunkSize, c1 -> c1.withBytes("LITTLE"))); - assertThrows(ZarrException.class, builder::build); - } - - @ParameterizedTest - @CsvSource({"0,true", "0,false", "5, true", "5, false"}) - public void testZstdCodecReadWrite(int clevel, boolean checksum) throws ZarrException, IOException { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZstdCodecReadWrite", "checksum_" + checksum, "clevel_" + clevel); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(2, 4, 8) - .withFillValue(0) - .withCodecs(c -> c.withZstd(clevel, checksum)); - Array writeArray = Array.create(storeHandle, builder.build()); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); - - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - } - - @Test - public void testTransposeCodec() throws ZarrException { - ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); - ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{ - 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); - - TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0})); - transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( - new long[]{2, 3, 3}, - new int[]{2, 3, 3}, - DataType.UINT32, - null)); - - assert MAMath.equals(testDataTransposed120, transposeCodec.encode(testData)); - assert MAMath.equals(testData, transposeCodec.decode(testDataTransposed120)); - } - - static Stream invalidTransposeOrder() { - return Stream.of( - new int[]{1, 0, 0}, - new int[]{1, 2, 3}, - new int[]{1, 2, 3, 0}, - new int[]{1, 2} - ); - } - - @ParameterizedTest - @MethodSource("invalidChunkSizes") - public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exception { - int[] shapeInt = new int[]{2, 3, 3}; - long[] shapeLong = new long[]{2, 3, 3}; - - TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(transposeOrder)); - transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( - shapeLong, - shapeInt, - DataType.UINT32, - null)); - - ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, shapeInt); - assertThrows(ZarrException.class, () -> transposeCodec.encode(testData)); - } - - @Test - public void testFileSystemStores() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTDATA); - ObjectMapper objectMapper = makeObjectMapper(); - - GroupMetadata group = objectMapper.readValue( - Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), - GroupMetadata.class - ); - - System.out.println(group); - System.out.println(objectMapper.writeValueAsString(group)); - - ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( - "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), - ArrayMetadata.class); - - System.out.println(arrayMetadata); - System.out.println(objectMapper.writeValueAsString(arrayMetadata)); - - System.out.println( - Array.open(fsStore.resolve("l4_sample", "color", "1"))); - System.out.println( - Arrays.toString(Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new))); - System.out.println( - Arrays.toString(((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list() - .toArray(Node[]::new))); - } - - @Test - public void testS3Store() throws IOException, ZarrException { - S3Store s3Store = new S3Store(AmazonS3ClientBuilder.standard() - .withRegion("eu-west-1") - .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) - .build(), "static.webknossos.org", "data"); - System.out.println(Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1"))); - } - - @Test - public void testV3ShardingReadCutout() throws IOException, ZarrException { - Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); - - ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new int[]{1, 64, 64, 64}); - Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); - Assertions.assertEquals(outArray.getByte(0), -98); - } - - @Test - public void testV3Access() throws IOException, ZarrException { - Array readArray = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); - - ucar.ma2.Array outArray = readArray.access().withOffset(0, 3073, 3073, 513) - .withShape(1, 64, 64, 64) - .read(); - Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); - Assertions.assertEquals(outArray.getByte(0), -98); - - Array writeArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), - readArray.metadata - ); - writeArray.access().withOffset(0, 3073, 3073, 513).write(outArray); - } - - @ParameterizedTest - @ValueSource(strings = {"start", "end"}) - public void testV3ShardingReadWrite(String indexLocation) throws IOException, ZarrException { - Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); - ucar.ma2.Array readArrayContent = readArray.read(); - Array writeArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), - readArray.metadata - ); - writeArray.write(readArrayContent); - ucar.ma2.Array outArray = writeArray.read(); - - assert MultiArrayUtils.allValuesEqual(readArrayContent, outArray); - } - - @Test - public void testV3Codecs() throws IOException, ZarrException { - int[] readShape = new int[]{1, 1, 1024, 1024}; - Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); - ucar.ma2.Array readArrayContent = readArray.read(new long[4], readShape); - { - Array gzipArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withGzip(5)).build() - ); - gzipArray.write(readArrayContent); - ucar.ma2.Array outGzipArray = gzipArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outGzipArray, readArrayContent); - } - { - Array bloscArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withBlosc("zstd", 5)).build() - ); - bloscArray.write(readArrayContent); - ucar.ma2.Array outBloscArray = bloscArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outBloscArray, readArrayContent); - } - { - Array zstdArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withZstd(10)).build() - ); - zstdArray.write(readArrayContent); - ucar.ma2.Array outZstdArray = zstdArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outZstdArray, readArrayContent); - } - } - - @Test - public void testV3ArrayMetadataBuilder() throws ZarrException { - Array.metadataBuilder() - .withShape(1, 4096, 4096, 1536) - .withDataType(DataType.UINT32) - .withChunkShape(1, 1024, 1024, 1024) - .withFillValue(0) - .withCodecs( - c -> c.withSharding(new int[]{1, 32, 32, 32}, CodecBuilder::withBlosc)) - .build(); - } - - @Test - public void testV3FillValue() throws ZarrException { - Assertions.assertEquals((int) parseFillValue(0, DataType.UINT32), 0); - Assertions.assertEquals((int) parseFillValue("0x00010203", DataType.UINT32), 50462976); - Assertions.assertEquals((byte) parseFillValue("0b00000010", DataType.UINT8), 2); - assert Double.isNaN((double) parseFillValue("NaN", DataType.FLOAT64)); - assert Double.isInfinite((double) parseFillValue("-Infinity", DataType.FLOAT64)); - } - - @Test - public void testV3Group() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); - - Map attributes = new HashMap<>(); - attributes.put("hello", "world"); - - Group group = Group.create(fsStore.resolve("testgroup")); - Group group2 = group.createGroup("test2", attributes); - Array array = group2.createArray("array", b -> - b.withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunkShape(5, 5) - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - - Assertions.assertArrayEquals(((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape(), new int[]{5, 5}); - } - - @Test - public void testReadme1() throws IOException, ZarrException { - Group hierarchy = Group.open( - new HttpStore("https://static.webknossos.org/data/zarr_v3") - .resolve("l4_sample") - ); - Group color = (Group) hierarchy.get("color"); - Array array = (Array) color.get("1"); - ucar.ma2.Array outArray = array.read( - new long[]{0, 3073, 3073, 513}, // offset - new int[]{1, 64, 64, 64} // shape - ); - } - - @Test - public void testReadme2() throws IOException, ZarrException { - Array array = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("testoutput", "color", "1"), - Array.metadataBuilder() - .withShape(1, 4096, 4096, 1536) - .withDataType(DataType.UINT32) - .withChunkShape(1, 1024, 1024, 1024) - .withFillValue(0) - .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc())) - .build() - ); - ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1, 2, 2}, new int[]{1, 2, 3, 4}); - array.write( - new long[]{0, 0, 0, 0}, // offset - data - ); - ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new int[]{1, 1, 2, 2}); - assert MultiArrayUtils.allValuesEqual(data, output); - - } - - @ParameterizedTest - @ValueSource(strings = {"1", "2-2-1", "4-4-1", "16-16-4"}) - public void testReadL4Sample(String mag) throws IOException, ZarrException { - StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/").resolve("l4_sample", "color", mag); - StoreHandle localStoreHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", mag); - - Array httpArray = Array.open(httpStoreHandle); - Array localArray = Array.open(localStoreHandle); - System.out.println(httpArray); - System.out.println(localArray); - - ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); - ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); - - assert MultiArrayUtils.allValuesEqual(httpData1, localData1); - - //offset to where l4_sample contains non-zero values - long[] offset = new long[4]; - long[] originalOffset = new long[]{0, 3073, 3073, 513}; - long[] originalShape = new long[]{1, 4096, 4096, 2048}; - long[] arrayShape = httpArray.metadata.shape; - for (int i = 0; i < 4; i++) { - offset[i] = originalOffset[i] / (originalShape[i] / arrayShape[i]); - } - - ucar.ma2.Array httpData2 = httpArray.read(offset, new int[]{1, 64, 64, 64}); - ucar.ma2.Array localData2 = localArray.read(offset, new int[]{1, 64, 64, 64}); - - assert MultiArrayUtils.allValuesEqual(httpData2, localData2); - } - - @ParameterizedTest - @ValueSource(booleans = {false, true}) - public void testParallel(boolean useParallel) throws IOException, ZarrException { - int[] testData = new int[512 * 512 * 512]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testParallelRead"); - ArrayMetadata metadata = Array.metadataBuilder() - .withShape(512, 512, 512) - .withDataType(DataType.UINT32) - .withChunkShape(100, 100, 100) - .withFillValue(0) - .build(); - Array writeArray = Array.create(storeHandle, metadata); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{512, 512, 512}, testData), useParallel); - - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(useParallel); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - clearTestoutputFolder(); - } - - @Test - public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, IOException { - // non-empty storage transformers are currently not supported - - Map[] storageTransformersEmpty = Array.open( - new FilesystemStore(TESTDATA).resolve("storage_transformer", "empty") - ).metadata.storageTransformers; - assert storageTransformersEmpty.length == 0; - - assertThrows(JsonMappingException.class, () -> Array.open( - new FilesystemStore(TESTDATA).resolve("storage_transformer", "exists")) - ); - - ArrayMetadataBuilder builderWithStorageTransformer = Array.metadataBuilder() - .withShape(1) - .withChunkShape(1) - .withDataType(DataType.UINT8) - .withStorageTransformers(new HashMap[]{new HashMap() {{ - put("some", "value"); - }}}); - - assertThrows(ZarrException.class, () -> Array.create( - new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"), - builderWithStorageTransformer.build() - )); - } - - @ParameterizedTest - @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"}) - public void testV2createBlosc(String cname, String shuffle, int clevel) throws IOException, ZarrException { - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( - new FilesystemStore(TESTOUTPUT).resolve("v2_create_blosc", cname + "_" + shuffle + "_" + clevel), - dev.zarr.zarrjava.v2.Array.metadataBuilder() - .withShape(10, 10) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) - .withChunks(5, 5) - .withFillValue(1) - .withBloscCompressor(cname, shuffle, clevel) - .build() - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - - ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); - Assertions.assertEquals(8 * 8, outArray.getSize()); - Assertions.assertEquals(0, outArray.getByte(0)); - } - - @Test - public void testV2create() throws IOException, ZarrException { - dev.zarr.zarrjava.v2.DataType dataType = dev.zarr.zarrjava.v2.DataType.UINT32; - - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( - new FilesystemStore(TESTOUTPUT).resolve("v2_create"), - dev.zarr.zarrjava.v2.Array.metadataBuilder() - .withShape(10, 10) - .withDataType(dataType) - .withChunks(5, 5) - .withFillValue(2) - .build() - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{8, 8})); - - ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); - Assertions.assertEquals(8 * 8, outArray.getSize()); - Assertions.assertEquals(0, outArray.getByte(0)); - } - - @ParameterizedTest - @ValueSource(ints = {0, 1, 5, 9}) - public void testV2createZlib(int level) throws IOException, ZarrException { - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( - new FilesystemStore(TESTOUTPUT).resolve("v2_create_zlib", String.valueOf(level)), - dev.zarr.zarrjava.v2.Array.metadataBuilder() - .withShape(15, 10) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) - .withChunks(4, 5) - .withFillValue(5) - .withZlibCompressor(level) - .build() - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{7, 6})); - - ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{7, 6}); - Assertions.assertEquals(7 * 6, outArray.getSize()); - Assertions.assertEquals(0, outArray.getByte(0)); - } - - @ParameterizedTest - @ValueSource(strings = {"BOOL", "INT8", "UINT8", "INT16", "UINT16", "INT32", "UINT32", "INT64", "UINT64", "FLOAT32", "FLOAT64"}) - public void testV2noFillValue(dev.zarr.zarrjava.v2.DataType dataType) throws IOException, ZarrException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("v2_no_fillvalue", dataType.name()); - - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( - storeHandle, - dev.zarr.zarrjava.v2.Array.metadataBuilder() - .withShape(15, 10) - .withDataType(dataType) - .withChunks(4, 5) - .build() - ); - Assertions.assertNull(array.metadata.fillValue); - - ucar.ma2.Array outArray = array.read(new long[]{0, 0}, new int[]{1, 1}); - if (dataType == dev.zarr.zarrjava.v2.DataType.BOOL) { - Assertions.assertFalse(outArray.getBoolean(0)); - } else { - Assertions.assertEquals(0, outArray.getByte(0)); - } - - dev.zarr.zarrjava.v2.Array array2 = dev.zarr.zarrjava.v2.Array.open( - storeHandle - ); - Assertions.assertNull(array2.metadata.fillValue); - } - - @Test - public void testV2Group() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); - - dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(fsStore.resolve("v2_testgroup")); - dev.zarr.zarrjava.v2.Group group2 = group.createGroup("test2"); - dev.zarr.zarrjava.v2.Array array = group2.createArray("array", b -> - b.withShape(10, 10) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) - .withChunks(5, 5) - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - - Assertions.assertArrayEquals(((dev.zarr.zarrjava.v2.Array) ((dev.zarr.zarrjava.v2.Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunks, new int[]{5, 5}); - } - - @Test - public void testGenericOpenV3() throws ZarrException, IOException { - StoreHandle arrayHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1"); - StoreHandle groupHandle = new FilesystemStore(TESTDATA).resolve("l4_sample"); - StoreHandle v2Handle = new FilesystemStore(TESTDATA).resolve("v2_sample"); - - Array array = (Array) Node.open(arrayHandle); - Assertions.assertEquals(4, (array).metadata.shape.length); - - array = (Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); - Assertions.assertEquals(4, (array).metadata.shape.length); - - array = (Array) dev.zarr.zarrjava.v3.Node.open(arrayHandle); - Assertions.assertEquals(4, (array).metadata.shape.length); - - Group group = (Group) Node.open(groupHandle); - Assertions.assertInstanceOf(Group.class, group.get("color")); - - group = (Group) dev.zarr.zarrjava.core.Group.open(groupHandle); - Assertions.assertInstanceOf(Group.class, group.get("color")); - - group = (Group) dev.zarr.zarrjava.v3.Node.open(groupHandle); - Assertions.assertInstanceOf(Group.class, group.get("color")); - - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing"))); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Node.open(v2Handle)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Group.open(v2Handle)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Array.open(v2Handle)); - } - - @Test - public void testGenericOpenV2() throws ZarrException, IOException { - StoreHandle arrayHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", "subgroup", "array"); - StoreHandle groupHandle = new FilesystemStore(TESTDATA).resolve("v2_sample"); - StoreHandle v3Handle = new FilesystemStore(TESTDATA).resolve("l4_sample"); - - dev.zarr.zarrjava.v2.Array array = (dev.zarr.zarrjava.v2.Array) Node.open(arrayHandle); - Assertions.assertEquals(3, (array).metadata.shape.length); - - array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); - Assertions.assertEquals(3, (array).metadata.shape.length); - - array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.v2.Node.open(arrayHandle); - Assertions.assertEquals(3, (array).metadata.shape.length); - - dev.zarr.zarrjava.v2.Group group = (dev.zarr.zarrjava.v2.Group) Node.open(groupHandle); - Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - - group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.core.Group.open(groupHandle); - Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - - group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.v2.Node.open(groupHandle); - Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing"))); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Node.open(v3Handle)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Group.open(v3Handle)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Array.open(v3Handle)); - } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java new file mode 100644 index 0000000..cd61295 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -0,0 +1,118 @@ +package dev.zarr.zarrjava; + +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; + +public class ZarrV2Test extends ZarrTest { + @ParameterizedTest + @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"}) + public void testV2createBlosc(String cname, String shuffle, int clevel) throws IOException, ZarrException { + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + new FilesystemStore(TESTOUTPUT).resolve("v2_create_blosc", cname + "_" + shuffle + "_" + clevel), + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(10, 10) + .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) + .withChunks(5, 5) + .withFillValue(1) + .withBloscCompressor(cname, shuffle, clevel) + .build() + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); + + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); + Assertions.assertEquals(8 * 8, outArray.getSize()); + Assertions.assertEquals(0, outArray.getByte(0)); + } + + @Test + public void testCreate() throws IOException, ZarrException { + dev.zarr.zarrjava.v2.DataType dataType = dev.zarr.zarrjava.v2.DataType.UINT32; + + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + new FilesystemStore(TESTOUTPUT).resolve("v2_create"), + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(10, 10) + .withDataType(dataType) + .withChunks(5, 5) + .withFillValue(2) + .build() + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{8, 8})); + + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); + Assertions.assertEquals(8 * 8, outArray.getSize()); + Assertions.assertEquals(0, outArray.getByte(0)); + } + + @ParameterizedTest + @ValueSource(ints = {0, 1, 5, 9}) + public void testCreateZlib(int level) throws IOException, ZarrException { + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + new FilesystemStore(TESTOUTPUT).resolve("v2_create_zlib", String.valueOf(level)), + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(15, 10) + .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) + .withChunks(4, 5) + .withFillValue(5) + .withZlibCompressor(level) + .build() + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{7, 6})); + + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{7, 6}); + Assertions.assertEquals(7 * 6, outArray.getSize()); + Assertions.assertEquals(0, outArray.getByte(0)); + } + + @ParameterizedTest + @ValueSource(strings = {"BOOL", "INT8", "UINT8", "INT16", "UINT16", "INT32", "UINT32", "INT64", "UINT64", "FLOAT32", "FLOAT64"}) + public void testNoFillValue(dev.zarr.zarrjava.v2.DataType dataType) throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("v2_no_fillvalue", dataType.name()); + + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + storeHandle, + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(15, 10) + .withDataType(dataType) + .withChunks(4, 5) + .build() + ); + Assertions.assertNull(array.metadata.fillValue); + + ucar.ma2.Array outArray = array.read(new long[]{0, 0}, new int[]{1, 1}); + if (dataType == dev.zarr.zarrjava.v2.DataType.BOOL) { + Assertions.assertFalse(outArray.getBoolean(0)); + } else { + Assertions.assertEquals(0, outArray.getByte(0)); + } + + dev.zarr.zarrjava.v2.Array array2 = dev.zarr.zarrjava.v2.Array.open( + storeHandle + ); + Assertions.assertNull(array2.metadata.fillValue); + } + + @Test + public void testGroup() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); + + dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(fsStore.resolve("v2_testgroup")); + dev.zarr.zarrjava.v2.Group group2 = group.createGroup("test2"); + dev.zarr.zarrjava.v2.Array array = group2.createArray("array", b -> + b.withShape(10, 10) + .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) + .withChunks(5, 5) + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); + + Assertions.assertArrayEquals(((dev.zarr.zarrjava.v2.Array) ((dev.zarr.zarrjava.v2.Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunks, new int[]{5, 5}); + } + +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java new file mode 100644 index 0000000..a32c2f3 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -0,0 +1,470 @@ +package dev.zarr.zarrjava; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.AnonymousAWSCredentials; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.zarr.zarrjava.core.Node; +import dev.zarr.zarrjava.store.*; +import dev.zarr.zarrjava.utils.MultiArrayUtils; +import dev.zarr.zarrjava.v3.*; +import dev.zarr.zarrjava.v3.codec.CodecBuilder; +import dev.zarr.zarrjava.v3.codec.core.BytesCodec; +import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import ucar.ma2.MAMath; + +import java.io.*; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Stream; + +import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; +import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; +import static org.junit.Assert.assertThrows; + +public class ZarrV3Test extends ZarrTest { + + static Stream> invalidCodecBuilder() { + return Stream.of( + c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE), + c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE), + c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}), + c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}) + ); + } + + @ParameterizedTest + @MethodSource("invalidCodecBuilder") + public void testCheckInvalidCodecConfiguration(Function codecBuilder) throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode())); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(new long[]{4, 4}) + .withDataType(DataType.UINT32) + .withChunkShape(new int[]{2, 2}) + .withCodecs(codecBuilder); + + assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build())); + } + + @Test + public void testLargerChunkSizeThanArraySize() throws ZarrException, IOException { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("larger_chunk_size_than_array"); + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(32, 32, 32) + .withFillValue(0) + .build(); + Array writeArray = Array.create(storeHandle, metadata); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + //read in zarr-java + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + } + + static Stream invalidChunkSizes() { + return Stream.of( + new int[]{1}, + new int[]{1, 1, 1} + ); + } + + @ParameterizedTest + @MethodSource("invalidChunkSizes") + public void testCheckInvalidChunkDimensions(int[] chunkSize) { + long[] shape = new long[]{4, 4}; + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_chunksize"); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(shape) + .withDataType(DataType.UINT32) + .withChunkShape(chunkSize); + + assertThrows(ZarrException.class, builder::build); + } + + static Stream invalidShardSizes() { + return Stream.of( + new int[]{4}, //wrong dims + new int[]{4, 4, 4}, //wrong dims + new int[]{1, 1}, //smaller than inner chunk shape + new int[]{5, 5}, //no exact multiple of inner chunk shape + new int[]{2, 1}, //smaller than inner chunk shape in 2nd dimension + new int[]{2, 5} //no exact multiple of inner chunk shape in 2nd dimension + ); + } + + @ParameterizedTest + @MethodSource("invalidShardSizes") + public void testCheckShardingBounds(int[] shardSize) throws Exception { + long[] shape = new long[]{10, 10}; + int[] innerChunkSize = new int[]{2, 2}; + + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(shape) + .withDataType(DataType.UINT32).withChunkShape(shardSize); + + if (false) { + int[] nestedChunkSize = new int[]{4, 4}; + builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2}, c1 -> c1.withSharding(nestedChunkSize, c2 -> c2.withBytes("LITTLE")))); + } + builder = builder.withCodecs(c -> c.withSharding(innerChunkSize, c1 -> c1.withBytes("LITTLE"))); + assertThrows(ZarrException.class, builder::build); + } + + @ParameterizedTest + @CsvSource({"0,true", "0,false", "5, true", "5, false"}) + public void testZstdCodecReadWrite(int clevel, boolean checksum) throws ZarrException, IOException { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZstdCodecReadWrite", "checksum_" + checksum, "clevel_" + clevel); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(2, 4, 8) + .withFillValue(0) + .withCodecs(c -> c.withZstd(clevel, checksum)); + Array writeArray = Array.create(storeHandle, builder.build()); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + } + + @Test + public void testTransposeCodec() throws ZarrException { + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); + ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{ + 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); + + TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0})); + transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + new long[]{2, 3, 3}, + new int[]{2, 3, 3}, + DataType.UINT32, + null)); + + assert MAMath.equals(testDataTransposed120, transposeCodec.encode(testData)); + assert MAMath.equals(testData, transposeCodec.decode(testDataTransposed120)); + } + + static Stream invalidTransposeOrder() { + return Stream.of( + new int[]{1, 0, 0}, + new int[]{1, 2, 3}, + new int[]{1, 2, 3, 0}, + new int[]{1, 2} + ); + } + + @ParameterizedTest + @MethodSource("invalidChunkSizes") + public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exception { + int[] shapeInt = new int[]{2, 3, 3}; + long[] shapeLong = new long[]{2, 3, 3}; + + TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(transposeOrder)); + transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + shapeLong, + shapeInt, + DataType.UINT32, + null)); + + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, shapeInt); + assertThrows(ZarrException.class, () -> transposeCodec.encode(testData)); + } + + @Test + public void testFileSystemStores() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTDATA); + ObjectMapper objectMapper = makeObjectMapper(); + + GroupMetadata group = objectMapper.readValue( + Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), + GroupMetadata.class + ); + + System.out.println(group); + System.out.println(objectMapper.writeValueAsString(group)); + + ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( + "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), + ArrayMetadata.class); + + System.out.println(arrayMetadata); + System.out.println(objectMapper.writeValueAsString(arrayMetadata)); + + System.out.println( + Array.open(fsStore.resolve("l4_sample", "color", "1"))); + System.out.println( + Arrays.toString(Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new))); + System.out.println( + Arrays.toString(((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list() + .toArray(Node[]::new))); + } + + @Test + public void testS3Store() throws IOException, ZarrException { + S3Store s3Store = new S3Store(AmazonS3ClientBuilder.standard() + .withRegion("eu-west-1") + .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) + .build(), "static.webknossos.org", "data"); + System.out.println(Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1"))); + } + + @Test + public void testV3ShardingReadCutout() throws IOException, ZarrException { + Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); + + ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new int[]{1, 64, 64, 64}); + Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); + Assertions.assertEquals(outArray.getByte(0), -98); + } + + @Test + public void testV3Access() throws IOException, ZarrException { + Array readArray = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); + + ucar.ma2.Array outArray = readArray.access().withOffset(0, 3073, 3073, 513) + .withShape(1, 64, 64, 64) + .read(); + Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); + Assertions.assertEquals(outArray.getByte(0), -98); + + Array writeArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), + readArray.metadata + ); + writeArray.access().withOffset(0, 3073, 3073, 513).write(outArray); + } + + @ParameterizedTest + @ValueSource(strings = {"start", "end"}) + public void testV3ShardingReadWrite(String indexLocation) throws IOException, ZarrException { + Array readArray = Array.open( + new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); + ucar.ma2.Array readArrayContent = readArray.read(); + Array writeArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), + readArray.metadata + ); + writeArray.write(readArrayContent); + ucar.ma2.Array outArray = writeArray.read(); + + assert MultiArrayUtils.allValuesEqual(readArrayContent, outArray); + } + + @Test + public void testV3Codecs() throws IOException, ZarrException { + int[] readShape = new int[]{1, 1, 1024, 1024}; + Array readArray = Array.open( + new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); + ucar.ma2.Array readArrayContent = readArray.read(new long[4], readShape); + { + Array gzipArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withGzip(5)).build() + ); + gzipArray.write(readArrayContent); + ucar.ma2.Array outGzipArray = gzipArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outGzipArray, readArrayContent); + } + { + Array bloscArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withBlosc("zstd", 5)).build() + ); + bloscArray.write(readArrayContent); + ucar.ma2.Array outBloscArray = bloscArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outBloscArray, readArrayContent); + } + { + Array zstdArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withZstd(10)).build() + ); + zstdArray.write(readArrayContent); + ucar.ma2.Array outZstdArray = zstdArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outZstdArray, readArrayContent); + } + } + + @Test + public void testV3ArrayMetadataBuilder() throws ZarrException { + Array.metadataBuilder() + .withShape(1, 4096, 4096, 1536) + .withDataType(DataType.UINT32) + .withChunkShape(1, 1024, 1024, 1024) + .withFillValue(0) + .withCodecs( + c -> c.withSharding(new int[]{1, 32, 32, 32}, CodecBuilder::withBlosc)) + .build(); + } + + @Test + public void testV3FillValue() throws ZarrException { + Assertions.assertEquals((int) parseFillValue(0, DataType.UINT32), 0); + Assertions.assertEquals((int) parseFillValue("0x00010203", DataType.UINT32), 50462976); + Assertions.assertEquals((byte) parseFillValue("0b00000010", DataType.UINT8), 2); + assert Double.isNaN((double) parseFillValue("NaN", DataType.FLOAT64)); + assert Double.isInfinite((double) parseFillValue("-Infinity", DataType.FLOAT64)); + } + + @Test + public void testV3Group() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); + + Map attributes = new HashMap<>(); + attributes.put("hello", "world"); + + Group group = Group.create(fsStore.resolve("testgroup")); + Group group2 = group.createGroup("test2", attributes); + Array array = group2.createArray("array", b -> + b.withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); + + Assertions.assertArrayEquals(((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape(), new int[]{5, 5}); + } + + @Test + public void testReadme1() throws IOException, ZarrException { + Group hierarchy = Group.open( + new HttpStore("https://static.webknossos.org/data/zarr_v3") + .resolve("l4_sample") + ); + Group color = (Group) hierarchy.get("color"); + Array array = (Array) color.get("1"); + ucar.ma2.Array outArray = array.read( + new long[]{0, 3073, 3073, 513}, // offset + new int[]{1, 64, 64, 64} // shape + ); + } + + @Test + public void testReadme2() throws IOException, ZarrException { + Array array = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("testoutput", "color", "1"), + Array.metadataBuilder() + .withShape(1, 4096, 4096, 1536) + .withDataType(DataType.UINT32) + .withChunkShape(1, 1024, 1024, 1024) + .withFillValue(0) + .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc())) + .build() + ); + ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1, 2, 2}, new int[]{1, 2, 3, 4}); + array.write( + new long[]{0, 0, 0, 0}, // offset + data + ); + ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new int[]{1, 1, 2, 2}); + assert MultiArrayUtils.allValuesEqual(data, output); + + } + + @ParameterizedTest + @ValueSource(strings = {"1", "2-2-1", "4-4-1", "16-16-4"}) + public void testReadL4Sample(String mag) throws IOException, ZarrException { + StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/").resolve("l4_sample", "color", mag); + StoreHandle localStoreHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", mag); + + Array httpArray = Array.open(httpStoreHandle); + Array localArray = Array.open(localStoreHandle); + System.out.println(httpArray); + System.out.println(localArray); + + ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); + ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); + + assert MultiArrayUtils.allValuesEqual(httpData1, localData1); + + //offset to where l4_sample contains non-zero values + long[] offset = new long[4]; + long[] originalOffset = new long[]{0, 3073, 3073, 513}; + long[] originalShape = new long[]{1, 4096, 4096, 2048}; + long[] arrayShape = httpArray.metadata.shape; + for (int i = 0; i < 4; i++) { + offset[i] = originalOffset[i] / (originalShape[i] / arrayShape[i]); + } + + ucar.ma2.Array httpData2 = httpArray.read(offset, new int[]{1, 64, 64, 64}); + ucar.ma2.Array localData2 = localArray.read(offset, new int[]{1, 64, 64, 64}); + + assert MultiArrayUtils.allValuesEqual(httpData2, localData2); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testParallel(boolean useParallel) throws IOException, ZarrException { + int[] testData = new int[512 * 512 * 512]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testParallelRead"); + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(512, 512, 512) + .withDataType(DataType.UINT32) + .withChunkShape(100, 100, 100) + .withFillValue(0) + .build(); + Array writeArray = Array.create(storeHandle, metadata); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{512, 512, 512}, testData), useParallel); + + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(useParallel); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + clearTestoutputFolder(); + } + + @Test + public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, IOException { + // non-empty storage transformers are currently not supported + + Map[] storageTransformersEmpty = Array.open( + new FilesystemStore(TESTDATA).resolve("storage_transformer", "empty") + ).metadata.storageTransformers; + assert storageTransformersEmpty.length == 0; + + assertThrows(JsonMappingException.class, () -> Array.open( + new FilesystemStore(TESTDATA).resolve("storage_transformer", "exists")) + ); + + ArrayMetadataBuilder builderWithStorageTransformer = Array.metadataBuilder() + .withShape(1) + .withChunkShape(1) + .withDataType(DataType.UINT8) + .withStorageTransformers(new HashMap[]{new HashMap() {{ + put("some", "value"); + }}}); + + assertThrows(ZarrException.class, () -> Array.create( + new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"), + builderWithStorageTransformer.build() + )); + } + + +} From 0fedf6f0a9d9761c5af1a7af94ecb568ba347fe7 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 2 Oct 2025 22:42:13 +0200 Subject: [PATCH 10/20] fix windows paths in tests --- src/test/java/dev/zarr/zarrjava/ZarrApiTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java b/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java index 46b0b1e..cc9d2e6 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java @@ -50,7 +50,7 @@ public void testGenericOpenV3() throws ZarrException, IOException { @Test public void testGenericOpenOverloadsV3() throws ZarrException, IOException { - Path arrayPath = TESTDATA.resolve("l4_sample/color/1"); + Path arrayPath = TESTDATA.resolve("l4_sample").resolve("color").resolve("1"); Path groupPath = TESTDATA.resolve("l4_sample"); Path v2GroupPath = TESTDATA.resolve("v2_sample"); @@ -129,7 +129,7 @@ public void testGenericOpenV2() throws ZarrException, IOException { @Test public void testGenericOpenOverloadsV2() throws ZarrException, IOException { - Path arrayPath = TESTDATA.resolve("v2_sample/subgroup/array"); + Path arrayPath = TESTDATA.resolve("v2_sample").resolve("subgroup").resolve("array"); Path groupPath = TESTDATA.resolve("v2_sample"); Path v3GroupPath = TESTDATA.resolve("l4_sample"); From 8f58d46169a6b2a78d8a81672aed645749ea7f44 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 2 Oct 2025 23:17:57 +0200 Subject: [PATCH 11/20] cleanup create api --- src/main/java/dev/zarr/zarrjava/v2/Group.java | 19 ++------------ src/main/java/dev/zarr/zarrjava/v3/Array.java | 2 ++ .../java/dev/zarr/zarrjava/ZarrApiTest.java | 26 ++++++++++++------- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index 9dfd239..38c0b94 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -47,20 +47,10 @@ public static Group create( return new Group(storeHandle, groupMetadata); } - public static Group create( - @Nonnull StoreHandle storeHandle - ) throws IOException, ZarrException { + public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, ZarrException { return create(storeHandle, new GroupMetadata()); } - public static Group create(Path path, GroupMetadata groupMetadata) throws IOException { - return create(new StoreHandle(new FilesystemStore(path)), groupMetadata); - } - - public static Group create(String path, GroupMetadata groupMetadata) throws IOException { - return create(Paths.get(path), groupMetadata); - } - public static Group create(Path path) throws IOException, ZarrException { return create(new StoreHandle(new FilesystemStore(path))); } @@ -79,13 +69,8 @@ public Node get(String key) throws ZarrException { } } - public Group createGroup(String key, GroupMetadata groupMetadata) - throws IOException, ZarrException { - return Group.create(storeHandle.resolve(key), groupMetadata); - } - public Group createGroup(String key) throws IOException, ZarrException { - return Group.create(storeHandle.resolve(key), new GroupMetadata()); + return Group.create(storeHandle.resolve(key)); } public Array createArray(String key, ArrayMetadata arrayMetadata) diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index a688504..b9181f3 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -193,6 +193,7 @@ private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException * Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or * deleted. This method returns a new instance of the Zarr array class and the old instance * becomes invalid. + * TODO: test * * @param newShape the new shape of the Zarr array * @throws ZarrException if the new metadata is invalid @@ -231,6 +232,7 @@ public Array setAttributes(Map newAttributes) throws ZarrExcepti * attributes as input and needs to return the new set of attributes. The attributes in the * callback may be mutated. This method overwrites and removes any existing attributes. This * method returns a new instance of the Zarr array class and the old instance becomes invalid. + * TODO: test * * @param attributeMapper the callback that is used to construct the new attributes * @throws ZarrException throws ZarrException if the new metadata is invalid diff --git a/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java b/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java index cc9d2e6..dd84580 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java @@ -6,6 +6,7 @@ import dev.zarr.zarrjava.v2.ArrayMetadata; import dev.zarr.zarrjava.v3.Array; import dev.zarr.zarrjava.v3.Group; +import dev.zarr.zarrjava.v3.GroupMetadata; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -14,6 +15,8 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; public class ZarrApiTest extends ZarrTest { @@ -42,7 +45,7 @@ public void testGenericOpenV3() throws ZarrException, IOException { group = (Group) dev.zarr.zarrjava.v3.Node.open(groupHandle); Assertions.assertInstanceOf(Group.class, group.get("color")); - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Node.open(v2Handle)); Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Group.open(v2Handle)); Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Array.open(v2Handle)); @@ -84,8 +87,8 @@ public void testGenericOpenOverloadsV3() throws ZarrException, IOException { group = (Group) dev.zarr.zarrjava.v3.Node.open(groupPath.toString()); Assertions.assertInstanceOf(Group.class, group.get("color")); - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing"))); - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing").toString())); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing").toString())); Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Node.open(v2GroupPath)); Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Node.open(v2GroupPath.toString())); @@ -121,7 +124,7 @@ public void testGenericOpenV2() throws ZarrException, IOException { group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.v2.Node.open(groupHandle); Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Node.open(v3Handle)); Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Group.open(v3Handle)); Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Array.open(v3Handle)); @@ -163,8 +166,8 @@ public void testGenericOpenOverloadsV2() throws ZarrException, IOException { group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.v2.Node.open(groupPath.toString()); Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing"))); - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(new FilesystemStore(TESTDATA).resolve("non_existing").toString())); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing").toString())); Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Node.open(v3GroupPath)); Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Node.open(v3GroupPath.toString())); @@ -239,15 +242,20 @@ public void testCreateGroupV3() throws ZarrException, IOException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV3"); Path storeHandlePath = TESTOUTPUT.resolve("testCreateGroupV3Path"); String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateGroupV3String")); + Map attributes = new HashMap<>(); + attributes.put("hello", "world"); - dev.zarr.zarrjava.v3.Group.create(storeHandle); + Group group = Group.create(storeHandle, new GroupMetadata(attributes)); Assertions.assertTrue(storeHandle.resolve("zarr.json").exists()); + Assertions.assertEquals("world", group.metadata.attributes.get("hello")); - dev.zarr.zarrjava.v3.Group.create(storeHandlePath); + group = Group.create(storeHandlePath, new GroupMetadata(attributes)); Assertions.assertTrue(Files.exists(storeHandlePath.resolve("zarr.json"))); + Assertions.assertEquals("world", group.metadata.attributes.get("hello")); - dev.zarr.zarrjava.v3.Group.create(storeHandleString); + group = Group.create(storeHandleString, new GroupMetadata(attributes)); Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); + Assertions.assertEquals("world", group.metadata.attributes.get("hello")); } } From fbdccd03cb84b57d98656e069b3959528bc41577 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 6 Oct 2025 15:00:32 +0200 Subject: [PATCH 12/20] finish merge update dependency versions --- .../dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java | 2 +- .../java/dev/zarr/zarrjava/ZarrPythonTests.java | 2 +- src/test/java/dev/zarr/zarrjava/ZarrV3Test.java | 13 +++++++------ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java index b79b1ac..75d27f5 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java @@ -28,7 +28,7 @@ public ZstdCodec( public ByteBuffer decode(ByteBuffer compressedBytes) throws ZarrException { byte[] compressedArray = compressedBytes.array(); - long originalSize = Zstd.decompressedSize(compressedArray); + long originalSize = Zstd.getFrameContentSize(compressedArray); if (originalSize == 0) { throw new ZarrException("Failed to get decompressed size"); } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 9fcafc1..676092e 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -265,7 +265,7 @@ public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException compressed = ctx.compress(src); } //decompress with Zstd.decompress - long originalSize = Zstd.decompressedSize(compressed); + long originalSize = Zstd.getFrameContentSize(compressed); byte[] decompressed = Zstd.decompress(compressed, (int) originalSize); Assertions.assertEquals(number, ByteBuffer.wrap(decompressed).getInt()); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index a32c2f3..88ec30b 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -1,8 +1,9 @@ package dev.zarr.zarrjava; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.AnonymousAWSCredentials; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.core.Node; @@ -225,9 +226,9 @@ public void testFileSystemStores() throws IOException, ZarrException { @Test public void testS3Store() throws IOException, ZarrException { - S3Store s3Store = new S3Store(AmazonS3ClientBuilder.standard() - .withRegion("eu-west-1") - .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) + S3Store s3Store = new S3Store(S3Client.builder() + .region(Region.of("eu-west-1")) + .credentialsProvider(AnonymousCredentialsProvider.create()) .build(), "static.webknossos.org", "data"); System.out.println(Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1"))); } From 72393a9a5b1d2fd6d4d647485e27d596461bf23c Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 6 Oct 2025 16:39:11 +0200 Subject: [PATCH 13/20] Refactor ZarrV2 and ZarrV3 tests to improve readability and structure; add new tests for array and group creation --- .../java/dev/zarr/zarrjava/ZarrApiTest.java | 261 ---------------- .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 76 +++++ src/test/java/dev/zarr/zarrjava/ZarrTest.java | 17 +- .../java/dev/zarr/zarrjava/ZarrV2Test.java | 166 ++++++++-- .../java/dev/zarr/zarrjava/ZarrV3Test.java | 289 ++++++++++++------ 5 files changed, 427 insertions(+), 382 deletions(-) delete mode 100644 src/test/java/dev/zarr/zarrjava/ZarrApiTest.java create mode 100644 src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java diff --git a/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java b/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java deleted file mode 100644 index dd84580..0000000 --- a/src/test/java/dev/zarr/zarrjava/ZarrApiTest.java +++ /dev/null @@ -1,261 +0,0 @@ -package dev.zarr.zarrjava; - -import dev.zarr.zarrjava.core.Node; -import dev.zarr.zarrjava.store.FilesystemStore; -import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.v2.ArrayMetadata; -import dev.zarr.zarrjava.v3.Array; -import dev.zarr.zarrjava.v3.Group; -import dev.zarr.zarrjava.v3.GroupMetadata; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Map; - - -public class ZarrApiTest extends ZarrTest { - - @Test - public void testGenericOpenV3() throws ZarrException, IOException { - StoreHandle arrayHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1"); - StoreHandle groupHandle = new FilesystemStore(TESTDATA).resolve("l4_sample"); - StoreHandle v2Handle = new FilesystemStore(TESTDATA).resolve("v2_sample"); - - Array array = (Array) Node.open(arrayHandle); - Assertions.assertEquals(4, (array).metadata.shape.length); - - array = (Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); - Assertions.assertEquals(4, (array).metadata.shape.length); - - array = (Array) dev.zarr.zarrjava.v3.Node.open(arrayHandle); - Assertions.assertEquals(4, (array).metadata.shape.length); - - Group group = (Group) Node.open(groupHandle); - Assertions.assertInstanceOf(Group.class, group.get("color")); - - group = (Group) dev.zarr.zarrjava.core.Group.open(groupHandle); - Assertions.assertInstanceOf(Group.class, group.get("color")); - - group = (Group) dev.zarr.zarrjava.v3.Node.open(groupHandle); - Assertions.assertInstanceOf(Group.class, group.get("color")); - - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Node.open(v2Handle)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Group.open(v2Handle)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Array.open(v2Handle)); - } - - @Test - public void testGenericOpenOverloadsV3() throws ZarrException, IOException { - Path arrayPath = TESTDATA.resolve("l4_sample").resolve("color").resolve("1"); - Path groupPath = TESTDATA.resolve("l4_sample"); - Path v2GroupPath = TESTDATA.resolve("v2_sample"); - - Array array = (Array) Node.open(arrayPath); - Assertions.assertEquals(4, (array).metadata.shape.length); - array = (Array) Node.open(arrayPath.toString()); - Assertions.assertEquals(4, (array).metadata.shape.length); - - array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath); - Assertions.assertEquals(4, (array).metadata.shape.length); - array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath.toString()); - Assertions.assertEquals(4, (array).metadata.shape.length); - - array = (Array) dev.zarr.zarrjava.v3.Node.open(arrayPath); - Assertions.assertEquals(4, (array).metadata.shape.length); - array = (Array) dev.zarr.zarrjava.v3.Node.open(arrayPath.toString()); - Assertions.assertEquals(4, (array).metadata.shape.length); - - Group group = (Group) Node.open(groupPath); - Assertions.assertInstanceOf(Group.class, group.get("color")); - group = (Group) Node.open(groupPath.toString()); - Assertions.assertInstanceOf(Group.class, group.get("color")); - - group = (Group) dev.zarr.zarrjava.core.Group.open(groupPath); - Assertions.assertInstanceOf(Group.class, group.get("color")); - group = (Group) dev.zarr.zarrjava.core.Group.open(groupPath.toString()); - Assertions.assertInstanceOf(Group.class, group.get("color")); - - group = (Group) dev.zarr.zarrjava.v3.Node.open(groupPath); - Assertions.assertInstanceOf(Group.class, group.get("color")); - group = (Group) dev.zarr.zarrjava.v3.Node.open(groupPath.toString()); - Assertions.assertInstanceOf(Group.class, group.get("color")); - - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing").toString())); - - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Node.open(v2GroupPath)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Node.open(v2GroupPath.toString())); - - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Group.open(v2GroupPath)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Group.open(v2GroupPath.toString())); - - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Array.open(v2GroupPath)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v3.Array.open(v2GroupPath.toString())); - } - - @Test - public void testGenericOpenV2() throws ZarrException, IOException { - StoreHandle arrayHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", "subgroup", "array"); - StoreHandle groupHandle = new FilesystemStore(TESTDATA).resolve("v2_sample"); - StoreHandle v3Handle = new FilesystemStore(TESTDATA).resolve("l4_sample"); - - dev.zarr.zarrjava.v2.Array array = (dev.zarr.zarrjava.v2.Array) Node.open(arrayHandle); - Assertions.assertEquals(3, (array).metadata.shape.length); - - array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); - Assertions.assertEquals(3, (array).metadata.shape.length); - - array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.v2.Node.open(arrayHandle); - Assertions.assertEquals(3, (array).metadata.shape.length); - - dev.zarr.zarrjava.v2.Group group = (dev.zarr.zarrjava.v2.Group) Node.open(groupHandle); - Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - - group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.core.Group.open(groupHandle); - Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - - group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.v2.Node.open(groupHandle); - Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Node.open(v3Handle)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Group.open(v3Handle)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Array.open(v3Handle)); - } - - @Test - public void testGenericOpenOverloadsV2() throws ZarrException, IOException { - Path arrayPath = TESTDATA.resolve("v2_sample").resolve("subgroup").resolve("array"); - Path groupPath = TESTDATA.resolve("v2_sample"); - Path v3GroupPath = TESTDATA.resolve("l4_sample"); - - dev.zarr.zarrjava.v2.Array array = (dev.zarr.zarrjava.v2.Array) Node.open(arrayPath); - Assertions.assertEquals(3, (array).metadata.shape.length); - array = (dev.zarr.zarrjava.v2.Array) Node.open(arrayPath.toString()); - Assertions.assertEquals(3, (array).metadata.shape.length); - - array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.core.Array.open(arrayPath); - Assertions.assertEquals(3, (array).metadata.shape.length); - array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.core.Array.open(arrayPath.toString()); - Assertions.assertEquals(3, (array).metadata.shape.length); - - array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.v2.Node.open(arrayPath); - Assertions.assertEquals(3, (array).metadata.shape.length); - array = (dev.zarr.zarrjava.v2.Array) dev.zarr.zarrjava.v2.Node.open(arrayPath.toString()); - Assertions.assertEquals(3, (array).metadata.shape.length); - - dev.zarr.zarrjava.v2.Group group = (dev.zarr.zarrjava.v2.Group) Node.open(groupPath); - Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - group = (dev.zarr.zarrjava.v2.Group) Node.open(groupPath.toString()); - Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - - group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.core.Group.open(groupPath); - Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.core.Group.open(groupPath.toString()); - Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - - group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.v2.Node.open(groupPath); - Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - group = (dev.zarr.zarrjava.v2.Group) dev.zarr.zarrjava.v2.Node.open(groupPath.toString()); - Assertions.assertInstanceOf(dev.zarr.zarrjava.v2.Group.class, group.get("subgroup")); - - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); - Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing").toString())); - - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Node.open(v3GroupPath)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Node.open(v3GroupPath.toString())); - - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Group.open(v3GroupPath)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Group.open(v3GroupPath.toString())); - - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Array.open(v3GroupPath)); - Assertions.assertThrows(NoSuchFileException.class, () -> dev.zarr.zarrjava.v2.Array.open(v3GroupPath.toString())); - } - - @Test - public void testCreateArrayV2() throws ZarrException, IOException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateArrayV2"); - Path storeHandlePath = TESTOUTPUT.resolve("testCreateArrayV2Path"); - String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateArrayV2String")); - ArrayMetadata arrayMetadata = dev.zarr.zarrjava.v2.Array.metadataBuilder() - .withShape(10, 10) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) - .withChunks(5, 5) - .build(); - - dev.zarr.zarrjava.v2.Array.create(storeHandle, arrayMetadata); - Assertions.assertTrue(storeHandle.resolve(".zarray").exists()); - - dev.zarr.zarrjava.v2.Array.create(storeHandlePath, arrayMetadata); - Assertions.assertTrue(Files.exists(storeHandlePath.resolve(".zarray"))); - - dev.zarr.zarrjava.v2.Array.create(storeHandleString, arrayMetadata); - Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zarray"))); - } - - @Test - public void testCreateGroupV2() throws ZarrException, IOException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV2"); - Path storeHandlePath = TESTOUTPUT.resolve("testCreateGroupV2Path"); - String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateGroupV2String")); - - dev.zarr.zarrjava.v2.Group.create(storeHandle); - Assertions.assertTrue(storeHandle.resolve(".zgroup").exists()); - - dev.zarr.zarrjava.v2.Group.create(storeHandlePath); - Assertions.assertTrue(Files.exists(storeHandlePath.resolve(".zgroup"))); - - dev.zarr.zarrjava.v2.Group.create(storeHandleString); - Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); - } - - @Test - public void testCreateArrayV3() throws ZarrException, IOException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateArrayV3"); - Path storeHandlePath = TESTOUTPUT.resolve("testCreateArrayV3Path"); - String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateArrayV3String")); - dev.zarr.zarrjava.v3.ArrayMetadata arrayMetadata = dev.zarr.zarrjava.v3.Array.metadataBuilder() - .withShape(10, 10) - .withDataType(dev.zarr.zarrjava.v3.DataType.UINT8) - .withChunkShape(5, 5) - .build(); - - dev.zarr.zarrjava.v3.Array.create(storeHandle, arrayMetadata); - Assertions.assertTrue(storeHandle.resolve("zarr.json").exists()); - - dev.zarr.zarrjava.v3.Array.create(storeHandlePath, arrayMetadata); - Assertions.assertTrue(Files.exists(storeHandlePath.resolve("zarr.json"))); - - dev.zarr.zarrjava.v3.Array.create(storeHandleString, arrayMetadata); - Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); - } - - @Test - public void testCreateGroupV3() throws ZarrException, IOException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV3"); - Path storeHandlePath = TESTOUTPUT.resolve("testCreateGroupV3Path"); - String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateGroupV3String")); - Map attributes = new HashMap<>(); - attributes.put("hello", "world"); - - Group group = Group.create(storeHandle, new GroupMetadata(attributes)); - Assertions.assertTrue(storeHandle.resolve("zarr.json").exists()); - Assertions.assertEquals("world", group.metadata.attributes.get("hello")); - - group = Group.create(storeHandlePath, new GroupMetadata(attributes)); - Assertions.assertTrue(Files.exists(storeHandlePath.resolve("zarr.json"))); - Assertions.assertEquals("world", group.metadata.attributes.get("hello")); - - group = Group.create(storeHandleString, new GroupMetadata(attributes)); - Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); - Assertions.assertEquals("world", group.metadata.attributes.get("hello")); - } - -} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java new file mode 100644 index 0000000..4611736 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -0,0 +1,76 @@ +package dev.zarr.zarrjava; + +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.HttpStore; +import dev.zarr.zarrjava.store.S3Store; +import dev.zarr.zarrjava.v3.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +import java.io.IOException; +import java.nio.file.Files; + +import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; + +public class ZarrStoreTest extends ZarrTest { + @Test + public void testFileSystemStores() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTDATA); + ObjectMapper objectMapper = makeObjectMapper(); + + GroupMetadata groupMetadata = objectMapper.readValue( + Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), + GroupMetadata.class + ); + + String groupMetadataString = objectMapper.writeValueAsString(groupMetadata); + Assertions.assertTrue(groupMetadataString.contains("\"zarr_format\":3")); + Assertions.assertTrue(groupMetadataString.contains("\"node_type\":\"group\"")); + + ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( + "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), + ArrayMetadata.class); + + String arrayMetadataString = objectMapper.writeValueAsString(arrayMetadata); + Assertions.assertTrue(arrayMetadataString.contains("\"zarr_format\":3")); + Assertions.assertTrue(arrayMetadataString.contains("\"node_type\":\"array\"")); + Assertions.assertTrue(arrayMetadataString.contains("\"shape\":[1,4096,4096,2048]")); + + Assertions.assertInstanceOf(Array.class, Array.open(fsStore.resolve("l4_sample", "color", "1"))); + + Node[] subNodes = Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new); + Assertions.assertEquals(2, subNodes.length); + Assertions.assertInstanceOf(Group.class, subNodes[0]); + + Array[] colorSubNodes = ((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list().toArray(Array[]::new); + + Assertions.assertEquals(5, colorSubNodes.length); + Assertions.assertInstanceOf(Array.class, colorSubNodes[0]); + + Array array = (Array) ((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).get("1"); + Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata.shape); + } + + @Test + public void testS3Store() throws IOException, ZarrException { + S3Store s3Store = new S3Store(S3Client.builder() + .region(Region.of("eu-west-1")) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .build(), "static.webknossos.org", "data"); + Array array = Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1")); + + Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata.shape); + } + + @Test + public void testHttpStore() throws IOException, ZarrException { + HttpStore httpStore = new dev.zarr.zarrjava.store.HttpStore("https://static.webknossos.org/data/zarr_v3/l4_sample"); + Array array = Array.open(httpStore.resolve("color", "1")); + + Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata.shape); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index b3f012a..b40d516 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -1,9 +1,6 @@ package dev.zarr.zarrjava; -import dev.zarr.zarrjava.store.S3Store; -import dev.zarr.zarrjava.v3.Array; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; import java.io.File; import java.io.IOException; @@ -15,18 +12,22 @@ public class ZarrTest { - final static Path TESTDATA = Paths.get("testdata"); - final static Path TESTOUTPUT = Paths.get("testoutput"); + public static final Path TESTDATA = Paths.get("testdata"); + public static final Path TESTOUTPUT = Paths.get("testoutput"); @BeforeAll public static void clearTestoutputFolder() throws IOException { if (Files.exists(TESTOUTPUT)) { try (Stream walk = Files.walk(TESTOUTPUT)) { - walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + walk.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(file -> { + if (!file.delete()) { + throw new RuntimeException("Failed to delete file: " + file.getAbsolutePath()); + } + }); } } Files.createDirectory(TESTOUTPUT); } - - } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index cd61295..bfde3db 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -2,6 +2,11 @@ import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v2.Array; +import dev.zarr.zarrjava.v2.ArrayMetadata; +import dev.zarr.zarrjava.v2.DataType; +import dev.zarr.zarrjava.v2.Group; +import dev.zarr.zarrjava.v2.Node; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -9,16 +14,20 @@ import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; public class ZarrV2Test extends ZarrTest { @ParameterizedTest @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"}) public void testV2createBlosc(String cname, String shuffle, int clevel) throws IOException, ZarrException { - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + Array array = Array.create( new FilesystemStore(TESTOUTPUT).resolve("v2_create_blosc", cname + "_" + shuffle + "_" + clevel), - dev.zarr.zarrjava.v2.Array.metadataBuilder() + Array.metadataBuilder() .withShape(10, 10) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) + .withDataType(DataType.UINT8) .withChunks(5, 5) .withFillValue(1) .withBloscCompressor(cname, shuffle, clevel) @@ -33,11 +42,11 @@ public void testV2createBlosc(String cname, String shuffle, int clevel) throws I @Test public void testCreate() throws IOException, ZarrException { - dev.zarr.zarrjava.v2.DataType dataType = dev.zarr.zarrjava.v2.DataType.UINT32; + DataType dataType = DataType.UINT32; - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + Array array = Array.create( new FilesystemStore(TESTOUTPUT).resolve("v2_create"), - dev.zarr.zarrjava.v2.Array.metadataBuilder() + Array.metadataBuilder() .withShape(10, 10) .withDataType(dataType) .withChunks(5, 5) @@ -54,11 +63,11 @@ public void testCreate() throws IOException, ZarrException { @ParameterizedTest @ValueSource(ints = {0, 1, 5, 9}) public void testCreateZlib(int level) throws IOException, ZarrException { - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + Array array = Array.create( new FilesystemStore(TESTOUTPUT).resolve("v2_create_zlib", String.valueOf(level)), - dev.zarr.zarrjava.v2.Array.metadataBuilder() + Array.metadataBuilder() .withShape(15, 10) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) + .withDataType(DataType.UINT8) .withChunks(4, 5) .withFillValue(5) .withZlibCompressor(level) @@ -73,12 +82,12 @@ public void testCreateZlib(int level) throws IOException, ZarrException { @ParameterizedTest @ValueSource(strings = {"BOOL", "INT8", "UINT8", "INT16", "UINT16", "INT32", "UINT32", "INT64", "UINT64", "FLOAT32", "FLOAT64"}) - public void testNoFillValue(dev.zarr.zarrjava.v2.DataType dataType) throws IOException, ZarrException { + public void testNoFillValue(DataType dataType) throws IOException, ZarrException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("v2_no_fillvalue", dataType.name()); - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + Array array = Array.create( storeHandle, - dev.zarr.zarrjava.v2.Array.metadataBuilder() + Array.metadataBuilder() .withShape(15, 10) .withDataType(dataType) .withChunks(4, 5) @@ -87,32 +96,149 @@ public void testNoFillValue(dev.zarr.zarrjava.v2.DataType dataType) throws IOExc Assertions.assertNull(array.metadata.fillValue); ucar.ma2.Array outArray = array.read(new long[]{0, 0}, new int[]{1, 1}); - if (dataType == dev.zarr.zarrjava.v2.DataType.BOOL) { + if (dataType == DataType.BOOL) { Assertions.assertFalse(outArray.getBoolean(0)); } else { Assertions.assertEquals(0, outArray.getByte(0)); } - dev.zarr.zarrjava.v2.Array array2 = dev.zarr.zarrjava.v2.Array.open( + Array array2 = Array.open( storeHandle ); Assertions.assertNull(array2.metadata.fillValue); } + + @Test + public void testOpen() throws ZarrException, IOException { + StoreHandle arrayHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", "subgroup", "array"); + StoreHandle groupHandle = new FilesystemStore(TESTDATA).resolve("v2_sample"); + StoreHandle v3Handle = new FilesystemStore(TESTDATA).resolve("l4_sample"); + + Array array = (Array) Node.open(arrayHandle); + Assertions.assertEquals(3, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); + Assertions.assertEquals(3, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.core.Node.open(arrayHandle); + Assertions.assertEquals(3, (array).metadata.shape.length); + + Group group = (Group) Node.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + + group = (Group) dev.zarr.zarrjava.core.Group.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + + group = (Group) dev.zarr.zarrjava.core.Node.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(v3Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> Group.open(v3Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> Array.open(v3Handle)); + } + + @Test + public void testOpenOverloads() throws ZarrException, IOException { + Path arrayPath = TESTDATA.resolve("v2_sample").resolve("subgroup").resolve("array"); + Path groupPath = TESTDATA.resolve("v2_sample"); + Path v3GroupPath = TESTDATA.resolve("l4_sample"); + + Array array = (Array) Node.open(arrayPath); + Assertions.assertEquals(3, (array).metadata.shape.length); + array = (Array) Node.open(arrayPath.toString()); + Assertions.assertEquals(3, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath); + Assertions.assertEquals(3, (array).metadata.shape.length); + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath.toString()); + Assertions.assertEquals(3, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.core.Node.open(arrayPath); + Assertions.assertEquals(3, (array).metadata.shape.length); + array = (Array) dev.zarr.zarrjava.core.Node.open(arrayPath.toString()); + Assertions.assertEquals(3, (array).metadata.shape.length); + + Group group = (Group) Node.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + group = (Group) Node.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + + group = (Group) dev.zarr.zarrjava.core.Group.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + group = (Group) dev.zarr.zarrjava.core.Group.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + + group = (Group) dev.zarr.zarrjava.core.Node.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + group = (Group) dev.zarr.zarrjava.core.Node.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("subgroup")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing").toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(v3GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(v3GroupPath.toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> Group.open(v3GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> Group.open(v3GroupPath.toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> Array.open(v3GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> Array.open(v3GroupPath.toString())); + } + + @Test + public void testCreateArray() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateArrayV2"); + Path storeHandlePath = TESTOUTPUT.resolve("testCreateArrayV2Path"); + String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateArrayV2String")); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .build(); + + Array.create(storeHandle, arrayMetadata); + Assertions.assertTrue(storeHandle.resolve(".zarray").exists()); + + Array.create(storeHandlePath, arrayMetadata); + Assertions.assertTrue(Files.exists(storeHandlePath.resolve(".zarray"))); + + Array.create(storeHandleString, arrayMetadata); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zarray"))); + } + @Test public void testGroup() throws IOException, ZarrException { FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); - dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(fsStore.resolve("v2_testgroup")); - dev.zarr.zarrjava.v2.Group group2 = group.createGroup("test2"); - dev.zarr.zarrjava.v2.Array array = group2.createArray("array", b -> + Group group = Group.create(fsStore.resolve("v2_testgroup")); + Group group2 = group.createGroup("test2"); + Array array = group2.createArray("array", b -> b.withShape(10, 10) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) + .withDataType(DataType.UINT8) .withChunks(5, 5) ); array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - Assertions.assertArrayEquals(((dev.zarr.zarrjava.v2.Array) ((dev.zarr.zarrjava.v2.Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunks, new int[]{5, 5}); + Assertions.assertArrayEquals(new int[]{5, 5}, ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunks); } -} + + @Test + public void testCreateGroup() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV2"); + Path storeHandlePath = TESTOUTPUT.resolve("testCreateGroupV2Path"); + String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateGroupV2String")); + + Group.create(storeHandle); + Assertions.assertTrue(storeHandle.resolve(".zgroup").exists()); + + Group.create(storeHandlePath); + Assertions.assertTrue(Files.exists(storeHandlePath.resolve(".zgroup"))); + + Group.create(storeHandleString); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); + } +} \ No newline at end of file diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 88ec30b..07f5bf2 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -1,14 +1,12 @@ package dev.zarr.zarrjava; -import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; +import dev.zarr.zarrjava.v3.codec.core.BloscCodec; +import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; import com.fasterxml.jackson.databind.JsonMappingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import dev.zarr.zarrjava.core.Node; import dev.zarr.zarrjava.store.*; import dev.zarr.zarrjava.utils.MultiArrayUtils; +import dev.zarr.zarrjava.v3.Node; import dev.zarr.zarrjava.v3.*; import dev.zarr.zarrjava.v3.codec.CodecBuilder; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; @@ -16,6 +14,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; @@ -23,6 +22,9 @@ import java.io.*; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -30,7 +32,6 @@ import java.util.stream.Stream; import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; -import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; import static org.junit.Assert.assertThrows; public class ZarrV3Test extends ZarrTest { @@ -46,12 +47,12 @@ static Stream> invalidCodecBuilder() { @ParameterizedTest @MethodSource("invalidCodecBuilder") - public void testCheckInvalidCodecConfiguration(Function codecBuilder) throws Exception { + public void testCheckInvalidCodecConfiguration(Function codecBuilder) { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode())); ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(new long[]{4, 4}) + .withShape(4, 4) .withDataType(DataType.UINT32) - .withChunkShape(new int[]{2, 2}) + .withChunkShape(2, 2) .withCodecs(codecBuilder); assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build())); @@ -91,7 +92,6 @@ static Stream invalidChunkSizes() { public void testCheckInvalidChunkDimensions(int[] chunkSize) { long[] shape = new long[]{4, 4}; - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_chunksize"); ArrayMetadataBuilder builder = Array.metadataBuilder() .withShape(shape) .withDataType(DataType.UINT32) @@ -110,10 +110,14 @@ static Stream invalidShardSizes() { new int[]{2, 5} //no exact multiple of inner chunk shape in 2nd dimension ); } - + static Stream invalidShardSizesWithNested() { + return invalidShardSizes().flatMap(shardSize -> + Stream.of(true, false).map(nested -> Arguments.of(shardSize, nested)) + ); + } @ParameterizedTest - @MethodSource("invalidShardSizes") - public void testCheckShardingBounds(int[] shardSize) throws Exception { + @MethodSource("invalidShardSizesWithNested") + public void testCheckShardingBounds(int[] shardSize, boolean nested) { long[] shape = new long[]{10, 10}; int[] innerChunkSize = new int[]{2, 2}; @@ -121,7 +125,7 @@ public void testCheckShardingBounds(int[] shardSize) throws Exception { .withShape(shape) .withDataType(DataType.UINT32).withChunkShape(shardSize); - if (false) { + if (nested) { int[] nestedChunkSize = new int[]{4, 4}; builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2}, c1 -> c1.withSharding(nestedChunkSize, c2 -> c2.withBytes("LITTLE")))); } @@ -179,7 +183,7 @@ static Stream invalidTransposeOrder() { } @ParameterizedTest - @MethodSource("invalidChunkSizes") + @MethodSource("invalidTransposeOrder") public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exception { int[] shapeInt = new int[]{2, 3, 3}; long[] shapeLong = new long[]{2, 3, 3}; @@ -196,61 +200,23 @@ public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exceptio } @Test - public void testFileSystemStores() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTDATA); - ObjectMapper objectMapper = makeObjectMapper(); - - GroupMetadata group = objectMapper.readValue( - Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), - GroupMetadata.class - ); - - System.out.println(group); - System.out.println(objectMapper.writeValueAsString(group)); - - ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( - "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), - ArrayMetadata.class); - - System.out.println(arrayMetadata); - System.out.println(objectMapper.writeValueAsString(arrayMetadata)); - - System.out.println( - Array.open(fsStore.resolve("l4_sample", "color", "1"))); - System.out.println( - Arrays.toString(Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new))); - System.out.println( - Arrays.toString(((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list() - .toArray(Node[]::new))); - } - - @Test - public void testS3Store() throws IOException, ZarrException { - S3Store s3Store = new S3Store(S3Client.builder() - .region(Region.of("eu-west-1")) - .credentialsProvider(AnonymousCredentialsProvider.create()) - .build(), "static.webknossos.org", "data"); - System.out.println(Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1"))); - } - - @Test - public void testV3ShardingReadCutout() throws IOException, ZarrException { + public void testShardingReadCutout() throws IOException, ZarrException { Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new int[]{1, 64, 64, 64}); - Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); - Assertions.assertEquals(outArray.getByte(0), -98); + Assertions.assertEquals(64 * 64 * 64, outArray.getSize()); + Assertions.assertEquals(-98, outArray.getByte(0)); } @Test - public void testV3Access() throws IOException, ZarrException { + public void testAccess() throws IOException, ZarrException { Array readArray = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); ucar.ma2.Array outArray = readArray.access().withOffset(0, 3073, 3073, 513) .withShape(1, 64, 64, 64) .read(); - Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); - Assertions.assertEquals(outArray.getByte(0), -98); + Assertions.assertEquals(64 * 64 * 64, outArray.getSize()); + Assertions.assertEquals(-98, outArray.getByte(0)); Array writeArray = Array.create( new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), @@ -261,7 +227,7 @@ public void testV3Access() throws IOException, ZarrException { @ParameterizedTest @ValueSource(strings = {"start", "end"}) - public void testV3ShardingReadWrite(String indexLocation) throws IOException, ZarrException { + public void testShardingReadWrite(String indexLocation) throws IOException, ZarrException { Array readArray = Array.open( new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); ucar.ma2.Array readArrayContent = readArray.read(); @@ -276,7 +242,7 @@ public void testV3ShardingReadWrite(String indexLocation) throws IOException, Za } @Test - public void testV3Codecs() throws IOException, ZarrException { + public void testCodecs() throws IOException, ZarrException { int[] readShape = new int[]{1, 1, 1024, 1024}; Array readArray = Array.open( new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); @@ -311,45 +277,40 @@ public void testV3Codecs() throws IOException, ZarrException { } @Test - public void testV3ArrayMetadataBuilder() throws ZarrException { - Array.metadataBuilder() - .withShape(1, 4096, 4096, 1536) - .withDataType(DataType.UINT32) - .withChunkShape(1, 1024, 1024, 1024) - .withFillValue(0) + public void testArrayMetadataBuilder() throws ZarrException { + long[] shape = new long[]{1, 4096, 4096, 1536}; + DataType dataType = DataType.UINT32; + int[] chunkShape = new int[]{1, 1024, 1024, 1024}; + int fillValue = 0; + + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(shape) + .withDataType(dataType) + .withChunkShape(chunkShape) + .withFillValue(fillValue) .withCodecs( c -> c.withSharding(new int[]{1, 32, 32, 32}, CodecBuilder::withBlosc)) .build(); + Assertions.assertArrayEquals(shape, metadata.shape); + Assertions.assertEquals(dataType, metadata.dataType); + Assertions.assertArrayEquals(chunkShape, metadata.chunkShape()); + Assertions.assertEquals(fillValue, metadata.fillValue); + Assertions.assertEquals(1, metadata.codecs.length); + ShardingIndexedCodec shardingCodec = (ShardingIndexedCodec) metadata.codecs[0]; + Assertions.assertInstanceOf(ShardingIndexedCodec.class, shardingCodec); + Assertions.assertInstanceOf(BytesCodec.class, shardingCodec.configuration.codecs[0]); + Assertions.assertInstanceOf(BloscCodec.class, shardingCodec.configuration.codecs[1]); } @Test - public void testV3FillValue() throws ZarrException { - Assertions.assertEquals((int) parseFillValue(0, DataType.UINT32), 0); - Assertions.assertEquals((int) parseFillValue("0x00010203", DataType.UINT32), 50462976); - Assertions.assertEquals((byte) parseFillValue("0b00000010", DataType.UINT8), 2); + public void testFillValue() throws ZarrException { + Assertions.assertEquals(0, (int) parseFillValue(0, DataType.UINT32)); + Assertions.assertEquals(50462976, (int) parseFillValue("0x00010203", DataType.UINT32)); + Assertions.assertEquals(2, (byte) parseFillValue("0b00000010", DataType.UINT8)); assert Double.isNaN((double) parseFillValue("NaN", DataType.FLOAT64)); assert Double.isInfinite((double) parseFillValue("-Infinity", DataType.FLOAT64)); } - @Test - public void testV3Group() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); - - Map attributes = new HashMap<>(); - attributes.put("hello", "world"); - - Group group = Group.create(fsStore.resolve("testgroup")); - Group group2 = group.createGroup("test2", attributes); - Array array = group2.createArray("array", b -> - b.withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunkShape(5, 5) - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - - Assertions.assertArrayEquals(((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape(), new int[]{5, 5}); - } - @Test public void testReadme1() throws IOException, ZarrException { Group hierarchy = Group.open( @@ -362,6 +323,7 @@ public void testReadme1() throws IOException, ZarrException { new long[]{0, 3073, 3073, 513}, // offset new int[]{1, 64, 64, 64} // shape ); + Assertions.assertEquals(64 * 64 * 64, outArray.getSize()); } @Test @@ -383,7 +345,6 @@ public void testReadme2() throws IOException, ZarrException { ); ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new int[]{1, 1, 2, 2}); assert MultiArrayUtils.allValuesEqual(data, output); - } @ParameterizedTest @@ -394,8 +355,9 @@ public void testReadL4Sample(String mag) throws IOException, ZarrException { Array httpArray = Array.open(httpStoreHandle); Array localArray = Array.open(localStoreHandle); - System.out.println(httpArray); - System.out.println(localArray); + + Assertions.assertArrayEquals(httpArray.metadata.shape, localArray.metadata.shape); + Assertions.assertArrayEquals(httpArray.metadata.chunkShape(), localArray.metadata.chunkShape()); ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); @@ -467,5 +429,146 @@ public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, I )); } + @Test + public void testOpen() throws ZarrException, IOException { + StoreHandle arrayHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1"); + StoreHandle groupHandle = new FilesystemStore(TESTDATA).resolve("l4_sample"); + StoreHandle v2Handle = new FilesystemStore(TESTDATA).resolve("v2_sample"); + + Array array = (Array) Node.open(arrayHandle); + Assertions.assertEquals(4, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); + Assertions.assertEquals(4, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.core.Node.open(arrayHandle); + Assertions.assertEquals(4, (array).metadata.shape.length); + + Group group = (Group) Node.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.core.Group.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.core.Node.open(groupHandle); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(v2Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> Group.open(v2Handle)); + Assertions.assertThrows(NoSuchFileException.class, () -> Array.open(v2Handle)); + } + + @Test + public void testOpenOverloads() throws ZarrException, IOException { + Path arrayPath = TESTDATA.resolve("l4_sample").resolve("color").resolve("1"); + Path groupPath = TESTDATA.resolve("l4_sample"); + Path v2GroupPath = TESTDATA.resolve("v2_sample"); + + Array array = (Array) Node.open(arrayPath); + Assertions.assertEquals(4, (array).metadata.shape.length); + array = (Array) Node.open(arrayPath.toString()); + Assertions.assertEquals(4, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath); + Assertions.assertEquals(4, (array).metadata.shape.length); + array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath.toString()); + Assertions.assertEquals(4, (array).metadata.shape.length); + + array = (Array) dev.zarr.zarrjava.core.Node.open(arrayPath); + Assertions.assertEquals(4, (array).metadata.shape.length); + array = (Array) dev.zarr.zarrjava.core.Node.open(arrayPath.toString()); + Assertions.assertEquals(4, (array).metadata.shape.length); + + Group group = (Group) Node.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("color")); + group = (Group) Node.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.core.Group.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("color")); + group = (Group) dev.zarr.zarrjava.core.Group.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + group = (Group) dev.zarr.zarrjava.core.Node.open(groupPath); + Assertions.assertInstanceOf(Group.class, group.get("color")); + group = (Group) dev.zarr.zarrjava.core.Node.open(groupPath.toString()); + Assertions.assertInstanceOf(Group.class, group.get("color")); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing"))); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(TESTDATA.resolve("non_existing").toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(v2GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> Node.open(v2GroupPath.toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> Group.open(v2GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> Group.open(v2GroupPath.toString())); + + Assertions.assertThrows(NoSuchFileException.class, () -> Array.open(v2GroupPath)); + Assertions.assertThrows(NoSuchFileException.class, () -> Array.open(v2GroupPath.toString())); + } + + @Test + public void testGroup() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); + + Map attributes = new HashMap<>(); + attributes.put("hello", "world"); + + Group group = Group.create(fsStore.resolve("testgroup")); + Group group2 = group.createGroup("test2", attributes); + Array array = group2.createArray("array", b -> + b.withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); + + Assertions.assertArrayEquals(new int[]{5, 5}, ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape()); + } + + @Test + public void testCreateArray() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateArrayV3"); + Path storeHandlePath = TESTOUTPUT.resolve("testCreateArrayV3Path"); + String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateArrayV3String")); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .build(); + + Array.create(storeHandle, arrayMetadata); + Assertions.assertTrue(storeHandle.resolve("zarr.json").exists()); + + Array.create(storeHandlePath, arrayMetadata); + Assertions.assertTrue(Files.exists(storeHandlePath.resolve("zarr.json"))); + + Array.create(storeHandleString, arrayMetadata); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); + } + + @Test + public void testCreateGroup() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV3"); + Path storeHandlePath = TESTOUTPUT.resolve("testCreateGroupV3Path"); + String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateGroupV3String")); + Map attributes = new HashMap<>(); + attributes.put("hello", "world"); + + Group group = Group.create(storeHandle, new GroupMetadata(attributes)); + Assertions.assertTrue(storeHandle.resolve("zarr.json").exists()); + Assertions.assertEquals("world", group.metadata.attributes.get("hello")); + + group = Group.create(storeHandlePath, new GroupMetadata(attributes)); + Assertions.assertTrue(Files.exists(storeHandlePath.resolve("zarr.json"))); + Assertions.assertEquals("world", group.metadata.attributes.get("hello")); + + group = Group.create(storeHandleString, new GroupMetadata(attributes)); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); + Assertions.assertEquals("world", group.metadata.attributes.get("hello")); + } + + } From 54175d2f624cc163dd9922927498e8bcb36df5d4 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 13 Oct 2025 15:15:49 +0200 Subject: [PATCH 14/20] detect typesize for blosc with evolve_from_core_array_metadata --- .../dev/zarr/zarrjava/v2/ArrayMetadata.java | 14 +++++++--- .../dev/zarr/zarrjava/v2/codec/Codec.java | 6 ++++- .../zarrjava/v2/codec/core/BloscCodec.java | 19 ++++++++++--- .../zarrjava/v2/codec/core/BytesCodec.java | 7 +++++ .../zarrjava/v2/codec/core/ZlibCodec.java | 6 +++++ src/main/java/dev/zarr/zarrjava/v3/Array.java | 2 -- .../dev/zarr/zarrjava/v3/ArrayMetadata.java | 13 ++++----- .../java/dev/zarr/zarrjava/ZarrV2Test.java | 16 ++++++++++- .../java/dev/zarr/zarrjava/ZarrV3Test.java | 2 +- testdata/v2_sample/double/.zarray | 25 ++++++++++++++++++ testdata/v2_sample/double/.zattrs | 3 +++ testdata/v2_sample/double/0.0.0 | Bin 0 -> 188 bytes 12 files changed, 94 insertions(+), 19 deletions(-) create mode 100644 testdata/v2_sample/double/.zarray create mode 100644 testdata/v2_sample/double/.zattrs create mode 100644 testdata/v2_sample/double/0.0.0 diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index 3a21d08..0c9ac50 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -65,13 +65,19 @@ public ArrayMetadata( this.endianness = dataType.getEndianness(); this.order = order; this.dimensionSeparator = dimensionSeparator; - this.filters = filters; - this.compressor = compressor; this.coreArrayMetadata = - new ArrayMetadata.CoreArrayMetadata(shape, chunks, + new CoreArrayMetadata(shape, chunks, this.dataType, this.parsedFillValue ); + if (filters == null) this.filters = null; + else{ + this.filters = new Codec[filters.length]; + for(int i = 0; i < filters.length; i++) { + this.filters[i] = filters[i].evolve_from_core_array_metadata(this.coreArrayMetadata); + } + } + this.compressor = compressor == null ? null : compressor.evolve_from_core_array_metadata(this.coreArrayMetadata); } @@ -90,7 +96,7 @@ public DataType dataType() { @Override public Array allocateFillValueChunk() { - ucar.ma2.Array outputArray = ucar.ma2.Array.factory(dataType.getMA2DataType(), chunks); + Array outputArray = Array.factory(dataType.getMA2DataType(), chunks); if (parsedFillValue != null) MultiArrayUtils.fill(outputArray, parsedFillValue); return outputArray; } diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java index 43f02f1..864a323 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java @@ -1,7 +1,11 @@ package dev.zarr.zarrjava.v2.codec; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v2.ArrayMetadata; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "id") -public interface Codec extends dev.zarr.zarrjava.core.codec.Codec {} +public interface Codec extends dev.zarr.zarrjava.core.codec.Codec { + Codec evolve_from_core_array_metadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException; +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java index 4838488..79ace52 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.ser.std.StdSerializer; import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.ArrayMetadata; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v2.codec.Codec; @@ -44,9 +45,6 @@ public BloscCodec( @JsonProperty(value = "typesize", defaultValue = "0") int typesize, @JsonProperty(value = "blocksize", defaultValue = "0") int blocksize ) throws ZarrException { - if (typesize < 1 && shuffle != Blosc.Shuffle.NO_SHUFFLE) { - typesize = 4; //todo: in v2 typesize is not a required parameter. default to correct value based on dtype - } if (clevel < 0 || clevel > 9) { throw new ZarrException("'clevel' needs to be between 0 and 9."); } @@ -57,7 +55,6 @@ public BloscCodec( this.blocksize = blocksize; } - @Override public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { @@ -72,6 +69,20 @@ public ByteBuffer encode(ByteBuffer chunkBytes) } } + @Override + public BloscCodec evolve_from_core_array_metadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + if (typesize == 0) { + return new BloscCodec( + this.cname, + this.shuffle, + this.clevel, + arrayMetadata.dataType.getByteCount(), + this.blocksize + ); + } + return this; + } + public static final class CustomShuffleSerializer extends StdSerializer { public CustomShuffleSerializer() { diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java index ac0cd39..1627479 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.ArrayMetadata; import dev.zarr.zarrjava.v2.codec.Codec; import javax.annotation.Nonnull; @@ -22,5 +24,10 @@ public BytesCodec( protected ByteOrder getByteOrder() { return endian.getByteOrder(); } + + @Override + public Codec evolve_from_core_array_metadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + return this; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java index 8b2bed6..cdc3260 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.ArrayMetadata; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v2.codec.Codec; import dev.zarr.zarrjava.core.codec.BytesBytesCodec; @@ -52,4 +53,9 @@ public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { throw new ZarrException("Error in encoding zlib.", ex); } } + + @Override + public Codec evolve_from_core_array_metadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + return this; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index b9181f3..a688504 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -193,7 +193,6 @@ private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException * Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or * deleted. This method returns a new instance of the Zarr array class and the old instance * becomes invalid. - * TODO: test * * @param newShape the new shape of the Zarr array * @throws ZarrException if the new metadata is invalid @@ -232,7 +231,6 @@ public Array setAttributes(Map newAttributes) throws ZarrExcepti * attributes as input and needs to return the new set of attributes. The attributes in the * callback may be mutated. This method overwrites and removes any existing attributes. This * method returns a new instance of the Zarr array class and the old instance becomes invalid. - * TODO: test * * @param attributeMapper the callback that is used to construct the new attributes * @throws ZarrException throws ZarrException if the new metadata is invalid diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index c1a8411..3625348 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -113,18 +113,19 @@ public ArrayMetadata( shardingCodec = getShardingIndexedCodec(shardingConfig.codecs); } } - this.dataType = dataType; this.chunkGrid = chunkGrid; + this.dataType = dataType; + this.coreArrayMetadata = + new CoreArrayMetadata(this.shape, ((RegularChunkGrid) chunkGrid).configuration.chunkShape, + this.dataType, + this.parsedFillValue + ); + this.chunkKeyEncoding = chunkKeyEncoding; this.codecs = codecs; this.dimensionNames = dimensionNames; this.attributes = attributes; this.storageTransformers = storageTransformers; - this.coreArrayMetadata = - new CoreArrayMetadata(shape, ((RegularChunkGrid) chunkGrid).configuration.chunkShape, - this.dataType, - this.parsedFillValue - ); } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index bfde3db..39e57be 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -22,7 +22,7 @@ public class ZarrV2Test extends ZarrTest { @ParameterizedTest @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"}) - public void testV2createBlosc(String cname, String shuffle, int clevel) throws IOException, ZarrException { + public void testCreateBlosc(String cname, String shuffle, int clevel) throws IOException, ZarrException { Array array = Array.create( new FilesystemStore(TESTOUTPUT).resolve("v2_create_blosc", cname + "_" + shuffle + "_" + clevel), Array.metadataBuilder() @@ -40,6 +40,20 @@ public void testV2createBlosc(String cname, String shuffle, int clevel) throws I Assertions.assertEquals(0, outArray.getByte(0)); } + + + @ParameterizedTest + @CsvSource({ + "BOOL", "FLOAT64" + }) + public void testReadBloscDetectTypesize(DataType dt) throws IOException, ZarrException { + String arrayname = dt == DataType.BOOL ? "bool" : "double"; + StoreHandle storeHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", arrayname); + Array array = Array.open(storeHandle); + ucar.ma2.Array output = array.read(new long[]{0, 0, 0}, new int[]{3, 4, 5}); + Assertions.assertEquals(dt, array.metadata.dataType); + } + @Test public void testCreate() throws IOException, ZarrException { DataType dataType = DataType.UINT32; diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 07f5bf2..b6b2d9c 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -527,7 +527,7 @@ public void testGroup() throws IOException, ZarrException { Assertions.assertArrayEquals(new int[]{5, 5}, ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape()); } - @Test + @Test public void testCreateArray() throws ZarrException, IOException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateArrayV3"); Path storeHandlePath = TESTOUTPUT.resolve("testCreateArrayV3Path"); diff --git a/testdata/v2_sample/double/.zarray b/testdata/v2_sample/double/.zarray new file mode 100644 index 0000000..a614af6 --- /dev/null +++ b/testdata/v2_sample/double/.zarray @@ -0,0 +1,25 @@ +{ + "shape": [ + 16, + 16, + 16 + ], + "chunks": [ + 2, + 4, + 8 + ], + "dtype": "+0@QeHEG2 zh~Vd2r6XH%Eb$_?|Mmau#rDFxags7(M&Qbw9G|p}{HB4O{KzJ$lKKqPHD|$+6>BzZ phZ+j(#X%g!Nu0$+WMm5RhFO8v&7Rd+;*l9;qTp$@B=|!g;S0x+7=8c% literal 0 HcmV?d00001 From 9dc2b407ac8cb79a912585e60fa84ded7ba3b3bd Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 13 Oct 2025 15:18:40 +0200 Subject: [PATCH 15/20] fix BytesCodec for boolean --- .../zarrjava/core/codec/core/BytesCodec.java | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java index 7c70b54..0f7fb7c 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java @@ -3,6 +3,8 @@ import com.fasterxml.jackson.annotation.JsonValue; import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.Index; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -11,12 +13,25 @@ public abstract class BytesCodec extends ArrayBytesCodec { protected abstract ByteOrder getByteOrder(); @Override - public Array decode(ByteBuffer chunkBytes) { - chunkBytes.order(getByteOrder()); - return Array.factory(arrayMetadata.dataType.getMA2DataType(), arrayMetadata.chunkShape, - chunkBytes); +public Array decode(ByteBuffer chunkBytes) { + chunkBytes.order(getByteOrder()); + DataType dtype = arrayMetadata.dataType.getMA2DataType(); + int[] shape = arrayMetadata.chunkShape; + + // Array.factory does not support boolean arrays directly from ByteBuffer + if (dtype == DataType.BOOLEAN) { + int size = chunkBytes.remaining(); + boolean[] bools = new boolean[size]; + for (int i = 0; i < size; i++) { + bools[i] = chunkBytes.get(i) != 0; + } + + Index index = Index.factory(shape); + return Array.factory(DataType.BOOLEAN, index, bools); } + return Array.factory(dtype, shape, chunkBytes); +} @Override public ByteBuffer encode(Array chunkArray) { return chunkArray.getDataAsByteBuffer(getByteOrder()); From d37c35540f69c1c425b8c4efdd85f9a0d9eca822 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 13 Oct 2025 15:20:27 +0200 Subject: [PATCH 16/20] add testdata for bool --- testdata/v2_sample/bool/.zarray | 25 +++++++++++++++++++++++++ testdata/v2_sample/bool/.zattrs | 3 +++ testdata/v2_sample/bool/0.0.0 | Bin 0 -> 80 bytes 3 files changed, 28 insertions(+) create mode 100644 testdata/v2_sample/bool/.zarray create mode 100644 testdata/v2_sample/bool/.zattrs create mode 100644 testdata/v2_sample/bool/0.0.0 diff --git a/testdata/v2_sample/bool/.zarray b/testdata/v2_sample/bool/.zarray new file mode 100644 index 0000000..98c3d4c --- /dev/null +++ b/testdata/v2_sample/bool/.zarray @@ -0,0 +1,25 @@ +{ + "shape": [ + 16, + 16, + 16 + ], + "chunks": [ + 2, + 4, + 8 + ], + "dtype": "|b1", + "fill_value": false, + "order": "C", + "filters": null, + "dimension_separator": ".", + "compressor": { + "id": "blosc", + "cname": "blosclz", + "clevel": 3, + "shuffle": 1, + "blocksize": 0 + }, + "zarr_format": 2 +} \ No newline at end of file diff --git a/testdata/v2_sample/bool/.zattrs b/testdata/v2_sample/bool/.zattrs new file mode 100644 index 0000000..d98f0ae --- /dev/null +++ b/testdata/v2_sample/bool/.zattrs @@ -0,0 +1,3 @@ +{ + "answer": 42 +} \ No newline at end of file diff --git a/testdata/v2_sample/bool/0.0.0 b/testdata/v2_sample/bool/0.0.0 new file mode 100644 index 0000000000000000000000000000000000000000..3d2e66b9d07817f84fe4fe343a4c4e5199392c21 GIT binary patch literal 80 XcmZQ#6lQT?U|;~@03c>yqznK6N?ide literal 0 HcmV?d00001 From 88543e6abf9e34f1cdfc775e20bfeb8086bfaf15 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 13 Oct 2025 18:15:17 +0200 Subject: [PATCH 17/20] zarr-python tests for datatypes --- .../dev/zarr/zarrjava/ZarrPythonTests.java | 251 +++++++++++++----- src/test/python-scripts/parse_codecs.py | 2 +- src/test/python-scripts/zarr_python_read.py | 10 +- .../python-scripts/zarr_python_read_v2.py | 10 +- src/test/python-scripts/zarr_python_write.py | 11 +- .../python-scripts/zarr_python_write_v2.py | 10 +- 6 files changed, 210 insertions(+), 84 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 676092e..360ce56 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -14,6 +14,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; import java.io.*; import java.nio.ByteBuffer; @@ -24,6 +25,7 @@ import java.util.Map; import java.util.stream.Stream; + public class ZarrPythonTests extends ZarrTest { final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); @@ -69,56 +71,151 @@ public void run_python_script(String scriptName, String... args) throws IOExcept assert exitCode == 0; } + static ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt){ + ucar.ma2.DataType ma2Type = dt.getMA2DataType(); + ucar.ma2.Array array = ucar.ma2.Array.factory(ma2Type, new int[]{16, 16, 16}); + for (int i = 0; i < array.getSize(); i++) { + switch (ma2Type) { + case BOOLEAN: + array.setBoolean(i, i%2 == 0); + break; + case BYTE: + case UBYTE: + array.setByte(i, (byte) i); + break; + case SHORT: + case USHORT: + array.setShort(i, (short) i); + break; + case INT: + array.setInt(i, i); + break; + case UINT: + array.setLong(i, i & 0xFFFFFFFFL); + break; + case LONG: + case ULONG: + array.setLong(i, (long) i); + break; + case FLOAT: + array.setFloat(i, (float) i); + break; + case DOUBLE: + array.setDouble(i, (double) i); + break; + default: + throw new IllegalArgumentException("Invalid DataType: " + dt); + } + } + return array; + } + + static void assertIsTestdata(ucar.ma2.Array result, dev.zarr.zarrjava.core.DataType dt) { + // expected values are i for index i + ucar.ma2.DataType ma2Type = dt.getMA2DataType(); + for (int i = 0; i < result.getSize(); i++) { + switch (ma2Type) { + case BOOLEAN: + Assertions.assertEquals(i % 2 == 0, result.getBoolean(i)); + break; + case BYTE: + case UBYTE: + Assertions.assertEquals((byte) i, result.getByte(i)); + break; + case SHORT: + case USHORT: + Assertions.assertEquals((short) i, result.getShort(i)); + break; + case INT: + Assertions.assertEquals(i, result.getInt(i)); + break; + case UINT: + Assertions.assertEquals(i & 0xFFFFFFFFL, result.getLong(i)); + break; + case LONG: + case ULONG: + Assertions.assertEquals((long) i, result.getLong(i)); + break; + case FLOAT: + Assertions.assertEquals((float) i, result.getFloat(i), 1e-6); + break; + case DOUBLE: + Assertions.assertEquals((double) i, result.getDouble(i), 1e-12); + break; + default: + throw new IllegalArgumentException("Invalid DataType: " + dt); + } + } + } + + static Stream compressorAndDataTypeProviderV3() { + Stream datatypeTests = Stream.of( +// DataType.BOOL, +// DataType.INT8, +// DataType.UINT8, // -> BUG: see https://github.com/zarr-developers/zarr-java/issues/27 + DataType.INT16, + DataType.UINT16, + DataType.INT32, + DataType.UINT32, + DataType.INT64, + DataType.UINT64, + DataType.FLOAT32, + DataType.FLOAT64 + ).flatMap(dt -> Stream.of( + new Object[]{"sharding", "end", dt}, + new Object[]{"blosc", "blosclz_shuffle_3", dt} + )); + + Stream codecsTests = Stream.of( + new Object[]{"blosc", "blosclz_noshuffle_0", DataType.INT32}, + new Object[]{"blosc", "lz4_shuffle_6", DataType.INT32}, + new Object[]{"blosc", "lz4hc_bitshuffle_3", DataType.INT32}, + new Object[]{"blosc", "zlib_shuffle_5", DataType.INT32}, + new Object[]{"blosc", "zstd_bitshuffle_9", DataType.INT32}, + new Object[]{"gzip", "0", DataType.INT32}, + new Object[]{"gzip", "5", DataType.INT32}, + new Object[]{"zstd", "0_true", DataType.INT32}, + new Object[]{"zstd", "5_true", DataType.INT32}, + new Object[]{"zstd", "0_false", DataType.INT32}, + new Object[]{"zstd", "5_false", DataType.INT32}, + new Object[]{"bytes", "BIG", DataType.INT32}, + new Object[]{"bytes", "LITTLE", DataType.INT32}, + new Object[]{"transpose", "_", DataType.INT32}, + new Object[]{"sharding", "start", DataType.INT32}, + new Object[]{"sharding_nested", "_", DataType.INT32}, + new Object[]{"crc32c", "_", DataType.INT32} + ); + + return Stream.concat(datatypeTests, codecsTests); + } + + @ParameterizedTest - @CsvSource({ - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - "gzip,0", "gzip,5", - "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", - "bytes,BIG", "bytes,LITTLE", - "transpose,_", - "sharding,start", "sharding,end", - "sharding_nested,_", - "crc32c,_", - }) - public void testReadFromZarrPythonV3(String codec, String codecParam) throws IOException, ZarrException, InterruptedException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("read_from_zarr_python", codec, codecParam); - run_python_script("zarr_python_write.py", codec, codecParam, storeHandle.toPath().toString()); + @MethodSource("compressorAndDataTypeProviderV3") + public void testReadV3(String codec, String codecParam, DataType dataType) throws IOException, ZarrException, InterruptedException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testReadV3", codec, codecParam, dataType.name()); + run_python_script("zarr_python_write.py", codec, codecParam, dataType.name().toLowerCase(), storeHandle.toPath().toString()); Array array = Array.open(storeHandle); ucar.ma2.Array result = array.read(); - //for expected values see zarr_python_write.py Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(DataType.INT32, array.metadata.dataType); + Assertions.assertEquals(dataType, array.metadata.dataType); Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata.chunkShape()); Assertions.assertEquals(42, array.metadata.attributes.get("answer")); - int[] expectedData = new int[16 * 16 * 16]; - Arrays.setAll(expectedData, p -> p); - Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); + assertIsTestdata(result, DataType.INT32); } @ParameterizedTest - @CsvSource({ - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - "gzip,0", "gzip,5", - "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", - "bytes,BIG", "bytes,LITTLE", - "transpose,_", - "sharding,start", "sharding,end", - "sharding_nested,_", - "crc32c,_", - }) - public void testWriteReadWithZarrPythonV3(String codec, String codecParam) throws Exception { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - + @MethodSource("compressorAndDataTypeProviderV3") + public void testWriteV3(String codec, String codecParam, DataType dataType) throws Exception { Map attributes = new HashMap<>(); attributes.put("test_key", "test_value"); - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarr_python", codec, codecParam); + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testWriteV3", codec, codecParam, dataType.name()); ArrayMetadataBuilder builder = Array.metadataBuilder() .withShape(16, 16, 16) - .withDataType(DataType.UINT32) + .withDataType(dataType) .withChunkShape(2, 4, 8) .withFillValue(0) .withAttributes(attributes); @@ -158,64 +255,81 @@ public void testWriteReadWithZarrPythonV3(String codec, String codecParam) throw } Array writeArray = Array.create(storeHandle, builder.build()); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + writeArray.write(testdata(dataType)); //read in zarr-java Array readArray = Array.open(storeHandle); ucar.ma2.Array result = readArray.read(); Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(DataType.UINT32, readArray.metadata.dataType); + Assertions.assertEquals(dataType, readArray.metadata.dataType); Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + assertIsTestdata(result, DataType.INT32); //read in zarr_python - run_python_script("zarr_python_read.py", codec, codecParam, storeHandle.toPath().toString()); + run_python_script("zarr_python_read.py", codec, codecParam, dataType.name().toLowerCase(), storeHandle.toPath().toString()); } + static Stream compressorAndDataTypeProviderV2() { + Stream datatypeTests = Stream.of( + dev.zarr.zarrjava.v2.DataType.BOOL, + dev.zarr.zarrjava.v2.DataType.INT8, + dev.zarr.zarrjava.v2.DataType.UINT8, + dev.zarr.zarrjava.v2.DataType.INT16, + dev.zarr.zarrjava.v2.DataType.UINT16, + dev.zarr.zarrjava.v2.DataType.INT32, + dev.zarr.zarrjava.v2.DataType.UINT32, + dev.zarr.zarrjava.v2.DataType.INT64, + dev.zarr.zarrjava.v2.DataType.UINT64, + dev.zarr.zarrjava.v2.DataType.FLOAT32, + dev.zarr.zarrjava.v2.DataType.FLOAT64 + ).flatMap(dt -> Stream.of( + new Object[]{"zlib", "0", dt}, + new Object[]{"blosc", "blosclz_shuffle_3", dt} + )); + + Stream bloscTests = Stream.of( + new Object[]{"blosc", "blosclz_noshuffle_0", dev.zarr.zarrjava.v2.DataType.INT32}, + new Object[]{"blosc", "lz4_shuffle_6", dev.zarr.zarrjava.v2.DataType.INT32}, + new Object[]{"blosc", "lz4hc_bitshuffle_3", dev.zarr.zarrjava.v2.DataType.INT32}, + new Object[]{"blosc", "zlib_shuffle_5", dev.zarr.zarrjava.v2.DataType.INT32}, + new Object[]{"blosc", "zstd_bitshuffle_9", dev.zarr.zarrjava.v2.DataType.INT32} + ); + + return Stream.concat(datatypeTests, bloscTests); + } + @ParameterizedTest - @CsvSource({ - "zlib,0", "zlib,5", - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - }) - public void testReadFromZarrPythonV2(String compressor, String compressorParam) throws IOException, ZarrException, InterruptedException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("read_from_zarr_python_v2", compressor, compressorParam); - run_python_script("zarr_python_write_v2.py", compressor, compressorParam, storeHandle.toPath().toString()); + @MethodSource("compressorAndDataTypeProviderV2") + public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrjava.v2.DataType dt) throws IOException, ZarrException, InterruptedException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testReadV2", compressor, compressorParam, dt.name()); + run_python_script("zarr_python_write_v2.py", compressor, compressorParam, dt.name().toLowerCase(), storeHandle.toPath().toString()); dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.open(storeHandle); ucar.ma2.Array result = array.read(); - //for expected values see zarr_python_write.py Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(dev.zarr.zarrjava.v2.DataType.INT32, array.metadata.dataType); + Assertions.assertEquals(dt, array.metadata.dataType); Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata.chunkShape()); // Assertions.assertEquals(42, array.metadata.attributes.get("answer")); - int[] expectedData = new int[16 * 16 * 16]; - Arrays.setAll(expectedData, p -> p); - Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); + assertIsTestdata(result, dt); } @ParameterizedTest - @CsvSource({ - "zlib,0", "zlib,5", - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - }) - public void testWriteReadWithZarrPythonV2(String compressor, String compressorParam) throws Exception { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - + @MethodSource("compressorAndDataTypeProviderV2") + public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarrjava.v2.DataType dt) throws Exception { // Map attributes = new HashMap<>(); // attributes.put("test_key", "test_value"); - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarr_python_v2", compressor, compressorParam); + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCodecsWriteV2", compressor, compressorParam, dt.name()); dev.zarr.zarrjava.v2.ArrayMetadataBuilder builder = dev.zarr.zarrjava.v2.Array.metadataBuilder() .withShape(16, 16, 16) - .withDataType(dev.zarr.zarrjava.v2.DataType.UINT32) + .withDataType(dt) .withChunks(2, 4, 8) // .withAttributes(attributes) .withFillValue(0); @@ -235,21 +349,20 @@ public void testWriteReadWithZarrPythonV2(String compressor, String compressorPa } dev.zarr.zarrjava.v2.Array writeArray = dev.zarr.zarrjava.v2.Array.create(storeHandle, builder.build()); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + writeArray.write(testdata(dt)); //read in zarr-java dev.zarr.zarrjava.v2.Array readArray = dev.zarr.zarrjava.v2.Array.open(storeHandle); ucar.ma2.Array result = readArray.read(); Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(dev.zarr.zarrjava.v2.DataType.UINT32, readArray.metadata.dataType); + Assertions.assertEquals(dt, readArray.metadata.dataType); Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); // Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + assertIsTestdata(result, dt); //read in zarr_python - run_python_script("zarr_python_read_v2.py", compressor, compressorParam, storeHandle.toPath().toString()); + run_python_script("zarr_python_read_v2.py", compressor, compressorParam, dt.name().toLowerCase(), storeHandle.toPath().toString()); } @CsvSource({"0,true", "0,false", "5, true", "10, false"}) @@ -288,9 +401,6 @@ public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException @Test public void testGroupReadWriteV2() throws Exception { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("group_write"); StoreHandle storeHandle2 = new FilesystemStore(TESTOUTPUT).resolve("group_read"); Group group = Group.create(storeHandle); @@ -301,7 +411,7 @@ public void testGroupReadWriteV2() throws Exception { .withChunks(2, 4, 8) ); - array.write(ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{16, 16, 16}, testData)); + array.write(testdata(dataType)); run_python_script("zarr_python_group_v2.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString()); @@ -312,7 +422,6 @@ public void testGroupReadWriteV2() throws Exception { Assertions.assertNotNull(array2); ucar.ma2.Array result = array2.read(); Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); + assertIsTestdata(result, dataType); } - } diff --git a/src/test/python-scripts/parse_codecs.py b/src/test/python-scripts/parse_codecs.py index 6cd88ec..772b34d 100644 --- a/src/test/python-scripts/parse_codecs.py +++ b/src/test/python-scripts/parse_codecs.py @@ -15,7 +15,7 @@ def parse_codecs_zarr_python(codec_string: str, param_string: str, zarr_version: if codec_string == "blosc" and zarr_version == 3: cname, shuffle, clevel = param_string.split("_") - compressor = BloscCodec(typesize=4, cname=cname, shuffle=shuffle, clevel=int(clevel)) + compressor = BloscCodec(cname=cname, shuffle=shuffle, clevel=int(clevel)) elif codec_string == "blosc" and zarr_version == 2: cname, shuffle, clevel = param_string.split("_") if shuffle == "noshuffle": diff --git a/src/test/python-scripts/zarr_python_read.py b/src/test/python-scripts/zarr_python_read.py index 1e31ff5..d6cee0c 100644 --- a/src/test/python-scripts/zarr_python_read.py +++ b/src/test/python-scripts/zarr_python_read.py @@ -10,9 +10,13 @@ codec_string = sys.argv[1] param_string = sys.argv[2] compressor, serializer, filters = parse_codecs_zarr_python(codec_string, param_string) -store_path = Path(sys.argv[3]) +dtype = sys.argv[3] +store_path = Path(sys.argv[4]) -expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) +if dtype == 'bool': + expected_data = np.arange(16 * 16 * 16, dtype='uint8').reshape(16, 16, 16) % 2 == 0 +else: + expected_data = np.arange(16 * 16 * 16, dtype=dtype).reshape(16, 16, 16) a = zarr.open_array(store=LocalStore(store_path)) read_data = a[:, :] @@ -22,7 +26,7 @@ LocalStore(store_path / "expected"), shape=(16, 16, 16), chunks=(2, 4, 8), - dtype="uint32", + dtype=dtype, fill_value=0, filters=filters, serializer=serializer, diff --git a/src/test/python-scripts/zarr_python_read_v2.py b/src/test/python-scripts/zarr_python_read_v2.py index 5390a23..5ff5e5a 100644 --- a/src/test/python-scripts/zarr_python_read_v2.py +++ b/src/test/python-scripts/zarr_python_read_v2.py @@ -11,9 +11,13 @@ codec_string = sys.argv[1] param_string = sys.argv[2] compressor, serializer, filters = parse_codecs_zarr_python(codec_string, param_string, zarr_version=2) -store_path = Path(sys.argv[3]) +dtype = sys.argv[3] +store_path = Path(sys.argv[4]) -expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) +if dtype == 'bool': + expected_data = np.arange(16 * 16 * 16, dtype='uint8').reshape(16, 16, 16) % 2 == 0 +else: + expected_data = np.arange(16 * 16 * 16, dtype=dtype).reshape(16, 16, 16) a = zarr.open_array(store=LocalStore(store_path)) read_data = a[:, :] @@ -24,7 +28,7 @@ zarr_format=2, shape=(16, 16, 16), chunks=(2, 4, 8), - dtype='uint32', + dtype=dtype, fill_value=0, filters=filters, serializer=serializer, diff --git a/src/test/python-scripts/zarr_python_write.py b/src/test/python-scripts/zarr_python_write.py index 531d356..328400f 100644 --- a/src/test/python-scripts/zarr_python_write.py +++ b/src/test/python-scripts/zarr_python_write.py @@ -10,15 +10,20 @@ codec_string = sys.argv[1] param_string = sys.argv[2] compressor, serializer, filters = parse_codecs_zarr_python(codec_string, param_string) -store_path = Path(sys.argv[3]) +dtype = sys.argv[3] +store_path = Path(sys.argv[4]) + +if dtype == 'bool': + testdata = np.arange(16 * 16 * 16, dtype='uint8').reshape(16, 16, 16) % 2 == 0 +else: + testdata = np.arange(16 * 16 * 16, dtype=dtype).reshape(16, 16, 16) -testdata = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) a = zarr.create_array( LocalStore(store_path), shape=(16, 16, 16), chunks=(2, 4, 8), - dtype='int32', + dtype=dtype, filters=filters, serializer=serializer, compressors=compressor, diff --git a/src/test/python-scripts/zarr_python_write_v2.py b/src/test/python-scripts/zarr_python_write_v2.py index 9c124c1..0068e04 100644 --- a/src/test/python-scripts/zarr_python_write_v2.py +++ b/src/test/python-scripts/zarr_python_write_v2.py @@ -11,16 +11,20 @@ codec_string = sys.argv[1] param_string = sys.argv[2] compressor, serializer, filters = parse_codecs_zarr_python(codec_string, param_string, zarr_version=2) -store_path = Path(sys.argv[3]) +dtype = sys.argv[3] +store_path = Path(sys.argv[4]) -testdata = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) +if dtype == 'bool': + testdata = np.arange(16 * 16 * 16, dtype='uint8').reshape(16, 16, 16) % 2 == 0 +else: + testdata = np.arange(16 * 16 * 16, dtype=dtype).reshape(16, 16, 16) a = zarr.create_array( LocalStore(store_path), zarr_format=2, shape=(16, 16, 16), chunks=(2, 4, 8), - dtype='int32', + dtype=dtype, filters=filters, serializer=serializer, compressors=compressor, From 94e9e37a8aa047478ce389b21336d54003b16dcd Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 13 Oct 2025 18:15:38 +0200 Subject: [PATCH 18/20] fix BytesCodec encode for bool, float, double --- .../zarrjava/core/codec/core/BytesCodec.java | 43 ++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java index 0f7fb7c..4ccc8f9 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java @@ -2,9 +2,7 @@ import com.fasterxml.jackson.annotation.JsonValue; import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; -import ucar.ma2.Array; -import ucar.ma2.DataType; -import ucar.ma2.Index; +import ucar.ma2.*; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -34,9 +32,44 @@ public Array decode(ByteBuffer chunkBytes) { } @Override public ByteBuffer encode(Array chunkArray) { - return chunkArray.getDataAsByteBuffer(getByteOrder()); - } + ByteOrder order = getByteOrder(); + + // Boolean + if (chunkArray instanceof ArrayBoolean) { + boolean[] data = (boolean[]) chunkArray.copyTo1DJavaArray(); + ByteBuffer bb = ByteBuffer.allocate(data.length); + for (boolean b : data) { + bb.put((byte) (b ? 1 : 0)); + } + bb.flip(); + return bb; + } + // Float32 + if (chunkArray instanceof ArrayFloat) { + float[] data = (float[]) chunkArray.copyTo1DJavaArray(); + ByteBuffer bb = ByteBuffer.allocate(data.length * Float.BYTES).order(order); + for (float f : data) { + bb.putFloat(f); + } + bb.flip(); + return bb; + } + + // Float64 + if (chunkArray instanceof ArrayDouble) { + double[] data = (double[]) chunkArray.copyTo1DJavaArray(); + ByteBuffer bb = ByteBuffer.allocate(data.length * Double.BYTES).order(order); + for (double d : data) { + bb.putDouble(d); + } + bb.flip(); + return bb; + } + + // All other primitive types (byte, short, int, long, char) + return chunkArray.getDataAsByteBuffer(order); +} public enum Endian { LITTLE("little"), BIG("big"); From cdd153d4c41530da0c65f75189c1b79d537538f5 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 13 Oct 2025 18:28:33 +0200 Subject: [PATCH 19/20] make metadata() public but field metadata private --- .../java/dev/zarr/zarrjava/core/Array.java | 2 +- src/main/java/dev/zarr/zarrjava/v2/Array.java | 4 +- src/main/java/dev/zarr/zarrjava/v3/Array.java | 4 +- .../dev/zarr/zarrjava/ZarrPythonTests.java | 22 +++++------ .../java/dev/zarr/zarrjava/ZarrStoreTest.java | 6 +-- .../java/dev/zarr/zarrjava/ZarrV2Test.java | 26 ++++++------- .../java/dev/zarr/zarrjava/ZarrV3Test.java | 38 +++++++++---------- 7 files changed, 51 insertions(+), 51 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index c6cecdc..44daa75 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -21,7 +21,7 @@ public abstract class Array extends AbstractNode { protected CodecPipeline codecPipeline; - protected abstract ArrayMetadata metadata(); + public abstract ArrayMetadata metadata(); protected Array(StoreHandle storeHandle) throws ZarrException { super(storeHandle); diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 50d5ea8..638d648 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -21,9 +21,9 @@ public class Array extends dev.zarr.zarrjava.core.Array implements Node { - public ArrayMetadata metadata; + private final ArrayMetadata metadata; - protected ArrayMetadata metadata() { + public ArrayMetadata metadata() { return metadata; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index a688504..4d1434f 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -20,9 +20,9 @@ public class Array extends dev.zarr.zarrjava.core.Array implements Node { - public ArrayMetadata metadata; + private final ArrayMetadata metadata; - protected ArrayMetadata metadata(){ + public ArrayMetadata metadata(){ return metadata; } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 360ce56..e4e763a 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -199,9 +199,9 @@ public void testReadV3(String codec, String codecParam, DataType dataType) throw ucar.ma2.Array result = array.read(); Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(dataType, array.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata.chunkShape()); - Assertions.assertEquals(42, array.metadata.attributes.get("answer")); + Assertions.assertEquals(dataType, array.metadata().dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata().chunkShape()); + Assertions.assertEquals(42, array.metadata().attributes.get("answer")); assertIsTestdata(result, DataType.INT32); } @@ -262,9 +262,9 @@ public void testWriteV3(String codec, String codecParam, DataType dataType) thro ucar.ma2.Array result = readArray.read(); Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(dataType, readArray.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); - Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); + Assertions.assertEquals(dataType, readArray.metadata().dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata().chunkShape()); + Assertions.assertEquals("test_value", readArray.metadata().attributes.get("test_key")); assertIsTestdata(result, DataType.INT32); @@ -312,9 +312,9 @@ public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrj ucar.ma2.Array result = array.read(); Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(dt, array.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata.chunkShape()); -// Assertions.assertEquals(42, array.metadata.attributes.get("answer")); + Assertions.assertEquals(dt, array.metadata().dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata().chunkShape()); +// Assertions.assertEquals(42, array.metadata().attributes.get("answer")); assertIsTestdata(result, dt); } @@ -356,8 +356,8 @@ public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarr ucar.ma2.Array result = readArray.read(); Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(dt, readArray.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); + Assertions.assertEquals(dt, readArray.metadata().dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata().chunkShape()); // Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); assertIsTestdata(result, dt); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java index 4611736..66b0238 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java @@ -52,7 +52,7 @@ public void testFileSystemStores() throws IOException, ZarrException { Assertions.assertInstanceOf(Array.class, colorSubNodes[0]); Array array = (Array) ((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).get("1"); - Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata.shape); + Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata().shape); } @Test @@ -63,7 +63,7 @@ public void testS3Store() throws IOException, ZarrException { .build(), "static.webknossos.org", "data"); Array array = Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1")); - Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata.shape); + Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata().shape); } @Test @@ -71,6 +71,6 @@ public void testHttpStore() throws IOException, ZarrException { HttpStore httpStore = new dev.zarr.zarrjava.store.HttpStore("https://static.webknossos.org/data/zarr_v3/l4_sample"); Array array = Array.open(httpStore.resolve("color", "1")); - Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata.shape); + Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata().shape); } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 39e57be..765c51a 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -51,7 +51,7 @@ public void testReadBloscDetectTypesize(DataType dt) throws IOException, ZarrExc StoreHandle storeHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", arrayname); Array array = Array.open(storeHandle); ucar.ma2.Array output = array.read(new long[]{0, 0, 0}, new int[]{3, 4, 5}); - Assertions.assertEquals(dt, array.metadata.dataType); + Assertions.assertEquals(dt, array.metadata().dataType); } @Test @@ -107,7 +107,7 @@ public void testNoFillValue(DataType dataType) throws IOException, ZarrException .withChunks(4, 5) .build() ); - Assertions.assertNull(array.metadata.fillValue); + Assertions.assertNull(array.metadata().fillValue); ucar.ma2.Array outArray = array.read(new long[]{0, 0}, new int[]{1, 1}); if (dataType == DataType.BOOL) { @@ -119,7 +119,7 @@ public void testNoFillValue(DataType dataType) throws IOException, ZarrException Array array2 = Array.open( storeHandle ); - Assertions.assertNull(array2.metadata.fillValue); + Assertions.assertNull(array2.metadata().fillValue); } @@ -130,13 +130,13 @@ public void testOpen() throws ZarrException, IOException { StoreHandle v3Handle = new FilesystemStore(TESTDATA).resolve("l4_sample"); Array array = (Array) Node.open(arrayHandle); - Assertions.assertEquals(3, (array).metadata.shape.length); + Assertions.assertEquals(3, array.metadata().shape.length); array = (Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); - Assertions.assertEquals(3, (array).metadata.shape.length); + Assertions.assertEquals(3, array.metadata().shape.length); array = (Array) dev.zarr.zarrjava.core.Node.open(arrayHandle); - Assertions.assertEquals(3, (array).metadata.shape.length); + Assertions.assertEquals(3, array.metadata().shape.length); Group group = (Group) Node.open(groupHandle); Assertions.assertInstanceOf(Group.class, group.get("subgroup")); @@ -160,19 +160,19 @@ public void testOpenOverloads() throws ZarrException, IOException { Path v3GroupPath = TESTDATA.resolve("l4_sample"); Array array = (Array) Node.open(arrayPath); - Assertions.assertEquals(3, (array).metadata.shape.length); + Assertions.assertEquals(3, array.metadata().shape.length); array = (Array) Node.open(arrayPath.toString()); - Assertions.assertEquals(3, (array).metadata.shape.length); + Assertions.assertEquals(3, array.metadata().shape.length); array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath); - Assertions.assertEquals(3, (array).metadata.shape.length); + Assertions.assertEquals(3, array.metadata().shape.length); array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath.toString()); - Assertions.assertEquals(3, (array).metadata.shape.length); + Assertions.assertEquals(3, array.metadata().shape.length); array = (Array) dev.zarr.zarrjava.core.Node.open(arrayPath); - Assertions.assertEquals(3, (array).metadata.shape.length); + Assertions.assertEquals(3, array.metadata().shape.length); array = (Array) dev.zarr.zarrjava.core.Node.open(arrayPath.toString()); - Assertions.assertEquals(3, (array).metadata.shape.length); + Assertions.assertEquals(3, array.metadata().shape.length); Group group = (Group) Node.open(groupPath); Assertions.assertInstanceOf(Group.class, group.get("subgroup")); @@ -236,7 +236,7 @@ public void testGroup() throws IOException, ZarrException { ); array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - Assertions.assertArrayEquals(new int[]{5, 5}, ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunks); + Assertions.assertArrayEquals(new int[]{5, 5}, ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata().chunks); } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index b6b2d9c..7f1c15c 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -220,7 +220,7 @@ public void testAccess() throws IOException, ZarrException { Array writeArray = Array.create( new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), - readArray.metadata + readArray.metadata() ); writeArray.access().withOffset(0, 3073, 3073, 513).write(outArray); } @@ -233,7 +233,7 @@ public void testShardingReadWrite(String indexLocation) throws IOException, Zarr ucar.ma2.Array readArrayContent = readArray.read(); Array writeArray = Array.create( new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), - readArray.metadata + readArray.metadata() ); writeArray.write(readArrayContent); ucar.ma2.Array outArray = writeArray.read(); @@ -250,7 +250,7 @@ public void testCodecs() throws IOException, ZarrException { { Array gzipArray = Array.create( new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withGzip(5)).build() + Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withGzip(5)).build() ); gzipArray.write(readArrayContent); ucar.ma2.Array outGzipArray = gzipArray.read(new long[4], readShape); @@ -259,7 +259,7 @@ public void testCodecs() throws IOException, ZarrException { { Array bloscArray = Array.create( new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withBlosc("zstd", 5)).build() + Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withBlosc("zstd", 5)).build() ); bloscArray.write(readArrayContent); ucar.ma2.Array outBloscArray = bloscArray.read(new long[4], readShape); @@ -268,7 +268,7 @@ public void testCodecs() throws IOException, ZarrException { { Array zstdArray = Array.create( new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withZstd(10)).build() + Array.metadataBuilder(readArray.metadata()).withCodecs(c -> c.withZstd(10)).build() ); zstdArray.write(readArrayContent); ucar.ma2.Array outZstdArray = zstdArray.read(new long[4], readShape); @@ -356,8 +356,8 @@ public void testReadL4Sample(String mag) throws IOException, ZarrException { Array httpArray = Array.open(httpStoreHandle); Array localArray = Array.open(localStoreHandle); - Assertions.assertArrayEquals(httpArray.metadata.shape, localArray.metadata.shape); - Assertions.assertArrayEquals(httpArray.metadata.chunkShape(), localArray.metadata.chunkShape()); + Assertions.assertArrayEquals(httpArray.metadata().shape, localArray.metadata().shape); + Assertions.assertArrayEquals(httpArray.metadata().chunkShape(), localArray.metadata().chunkShape()); ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); @@ -368,7 +368,7 @@ public void testReadL4Sample(String mag) throws IOException, ZarrException { long[] offset = new long[4]; long[] originalOffset = new long[]{0, 3073, 3073, 513}; long[] originalShape = new long[]{1, 4096, 4096, 2048}; - long[] arrayShape = httpArray.metadata.shape; + long[] arrayShape = httpArray.metadata().shape; for (int i = 0; i < 4; i++) { offset[i] = originalOffset[i] / (originalShape[i] / arrayShape[i]); } @@ -408,7 +408,7 @@ public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, I Map[] storageTransformersEmpty = Array.open( new FilesystemStore(TESTDATA).resolve("storage_transformer", "empty") - ).metadata.storageTransformers; + ).metadata().storageTransformers; assert storageTransformersEmpty.length == 0; assertThrows(JsonMappingException.class, () -> Array.open( @@ -436,13 +436,13 @@ public void testOpen() throws ZarrException, IOException { StoreHandle v2Handle = new FilesystemStore(TESTDATA).resolve("v2_sample"); Array array = (Array) Node.open(arrayHandle); - Assertions.assertEquals(4, (array).metadata.shape.length); + Assertions.assertEquals(4, array.metadata().shape.length); array = (Array) dev.zarr.zarrjava.core.Array.open(arrayHandle); - Assertions.assertEquals(4, (array).metadata.shape.length); + Assertions.assertEquals(4, array.metadata().shape.length); array = (Array) dev.zarr.zarrjava.core.Node.open(arrayHandle); - Assertions.assertEquals(4, (array).metadata.shape.length); + Assertions.assertEquals(4, array.metadata().shape.length); Group group = (Group) Node.open(groupHandle); Assertions.assertInstanceOf(Group.class, group.get("color")); @@ -466,19 +466,19 @@ public void testOpenOverloads() throws ZarrException, IOException { Path v2GroupPath = TESTDATA.resolve("v2_sample"); Array array = (Array) Node.open(arrayPath); - Assertions.assertEquals(4, (array).metadata.shape.length); + Assertions.assertEquals(4, array.metadata().shape.length); array = (Array) Node.open(arrayPath.toString()); - Assertions.assertEquals(4, (array).metadata.shape.length); + Assertions.assertEquals(4, array.metadata().shape.length); array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath); - Assertions.assertEquals(4, (array).metadata.shape.length); + Assertions.assertEquals(4, array.metadata().shape.length); array = (Array) dev.zarr.zarrjava.core.Array.open(arrayPath.toString()); - Assertions.assertEquals(4, (array).metadata.shape.length); + Assertions.assertEquals(4, array.metadata().shape.length); array = (Array) dev.zarr.zarrjava.core.Node.open(arrayPath); - Assertions.assertEquals(4, (array).metadata.shape.length); + Assertions.assertEquals(4, array.metadata().shape.length); array = (Array) dev.zarr.zarrjava.core.Node.open(arrayPath.toString()); - Assertions.assertEquals(4, (array).metadata.shape.length); + Assertions.assertEquals(4, array.metadata().shape.length); Group group = (Group) Node.open(groupPath); Assertions.assertInstanceOf(Group.class, group.get("color")); @@ -524,7 +524,7 @@ public void testGroup() throws IOException, ZarrException { ); array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - Assertions.assertArrayEquals(new int[]{5, 5}, ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape()); + Assertions.assertArrayEquals(new int[]{5, 5}, ((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata().chunkShape()); } @Test From 31d6aec734f14ecd0890513f0756657fc012ec24 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 23 Oct 2025 11:24:42 +0200 Subject: [PATCH 20/20] camelCase for evolveFromCoreArrayMetadata --- src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java | 4 ++-- src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java | 2 +- src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java | 2 +- src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java | 2 +- src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index 0c9ac50..dabd285 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -74,10 +74,10 @@ public ArrayMetadata( else{ this.filters = new Codec[filters.length]; for(int i = 0; i < filters.length; i++) { - this.filters[i] = filters[i].evolve_from_core_array_metadata(this.coreArrayMetadata); + this.filters[i] = filters[i].evolveFromCoreArrayMetadata(this.coreArrayMetadata); } } - this.compressor = compressor == null ? null : compressor.evolve_from_core_array_metadata(this.coreArrayMetadata); + this.compressor = compressor == null ? null : compressor.evolveFromCoreArrayMetadata(this.coreArrayMetadata); } diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java index 864a323..8c20547 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java @@ -6,6 +6,6 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "id") public interface Codec extends dev.zarr.zarrjava.core.codec.Codec { - Codec evolve_from_core_array_metadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException; + Codec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException; } diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java index 79ace52..0bccc18 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java @@ -70,7 +70,7 @@ public ByteBuffer encode(ByteBuffer chunkBytes) } @Override - public BloscCodec evolve_from_core_array_metadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + public BloscCodec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { if (typesize == 0) { return new BloscCodec( this.cname, diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java index 1627479..773022a 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java @@ -26,7 +26,7 @@ protected ByteOrder getByteOrder() { } @Override - public Codec evolve_from_core_array_metadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + public Codec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) { return this; } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java index cdc3260..5093b58 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java @@ -55,7 +55,7 @@ public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { } @Override - public Codec evolve_from_core_array_metadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + public Codec evolveFromCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) { return this; } }