diff --git a/pom.xml b/pom.xml index 9256ea8db..c2cabe000 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.scijava pom-scijava - 39.0.0 + 45.1.0 @@ -138,10 +138,18 @@ NOTE: To prevent attributes.json read errors the render n5 version should be kept in-sync with the hot-knife n5 version if you need to run concurrent code from both repos on the same n5 group. --> - 3.3.0 - - 7.1.4 - 0.17.2 + 4.0.1 + + + 2.1.3 + + + 3.1.9 2.14.3 diff --git a/render-app/src/test/java/org/janelia/alignment/loader/HDF5SliceLoaderTest.java b/render-app/src/test/java/org/janelia/alignment/loader/HDF5SliceLoaderTest.java index 84bc1158e..fb6c59b4f 100644 --- a/render-app/src/test/java/org/janelia/alignment/loader/HDF5SliceLoaderTest.java +++ b/render-app/src/test/java/org/janelia/alignment/loader/HDF5SliceLoaderTest.java @@ -1,8 +1,9 @@ package org.janelia.alignment.loader; +import com.google.common.base.Throwables; + import ij.process.ImageProcessor; -import org.apache.commons.lang.exception.ExceptionUtils; import org.junit.Assert; import org.junit.Test; @@ -21,7 +22,7 @@ public void testLoad() { try { ip = loader.load(urlString); } catch (final Throwable t) { - final String failureMessage = ExceptionUtils.getStackTrace(t); + final String failureMessage = Throwables.getStackTraceAsString(t); Assert.fail("failed load with following exception: " + failureMessage); } Assert.assertEquals("invalid width", 5000, ip.getWidth()); diff --git a/render-ws-java-client/pom.xml b/render-ws-java-client/pom.xml index 32f4c7cf4..0599c6b3a 100644 --- a/render-ws-java-client/pom.xml +++ b/render-ws-java-client/pom.xml @@ -172,6 +172,15 @@ sc.fiji bigdataviewer-vistools + + 1.0.0-beta-36 + + + + + commons-lang + commons-lang diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java index b1231b0b2..5a37befcf 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java @@ -4,7 +4,6 @@ import com.beust.jcommander.ParametersDelegate; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import com.google.gson.JsonPrimitive; import java.io.IOException; import java.nio.file.Files; @@ -21,6 +20,7 @@ import net.imglib2.RandomAccess; import net.imglib2.RandomAccessibleInterval; import net.imglib2.img.basictypeaccess.AccessFlags; +import net.imglib2.type.NativeType; import net.imglib2.type.numeric.RealType; import net.imglib2.type.numeric.real.FloatType; @@ -34,13 +34,9 @@ import org.janelia.render.client.parameter.CommandLineParameters; import org.janelia.render.client.parameter.RenderWebServiceParameters; import org.janelia.render.client.parameter.ZRangeParameters; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5Reader; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.universe.N5Factory; -import org.janelia.saalfeldlab.n5.zarr.ZarrDatasetAttributes; -import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -417,117 +413,31 @@ private int[] readOptionalIntArray(final N5Reader reader, final String dataset) } /** - * Reads all values of a 1D coordinate array as doubles, or returns null if the array's metadata is not present. + * Reads all values of a 1D coordinate array as doubles, or returns null if the dataset is not present. *

- * n5-zarr 1.3.5 fails to parse a {@code .zarray} whose {@code fill_value} is JSON {@code null} (which is how - * xarray writes coordinate arrays), so {@link N5Utils#open} cannot be used here. Instead the {@code .zarray} - * JSON is read directly, its {@code fill_value} is patched to a parseable value (it is irrelevant for chunks - * that are physically present), the resulting {@link ZarrDatasetAttributes} is built via the reader, and the - * chunks are read with {@link N5Reader#readBlock} (which honors the zarr little-endian byte order). + * Uses the same {@link N5Utils#open} overload as {@link #openHomogenizationArray} (see there for why). + * The imglib2 type handles unsigned promotion, so uint8/uint16 labels come out as their unsigned values. */ - private double[] readCoordinateValues(final N5Reader reader, final String dataset) { - final JsonObject zArray = readZArrayJson(dataset); - if (zArray == null) { + private & NativeType> double[] readCoordinateValues(final N5Reader reader, + final String dataset) { + if (!reader.datasetExists(dataset)) { return null; } - if (!zArray.has("fill_value") || zArray.get("fill_value").isJsonNull()) { - zArray.add("fill_value", new JsonPrimitive("0")); - } - if (!(reader instanceof ZarrKeyValueReader)) { - throw new IllegalArgumentException("expected a zarr reader but got " + reader.getClass().getName()); - } - final ZarrDatasetAttributes attributes = ((ZarrKeyValueReader) reader).createDatasetAttributes(zArray); - if (attributes == null) { - throw new IllegalArgumentException("could not parse .zarray for coordinate dataset " + dataset); - } - if (attributes.getNumDimensions() != 1) { + final Consumer> noMissingBlockHandler = blocks -> { }; + final RandomAccessibleInterval rai = N5Utils.open(reader, dataset, noMissingBlockHandler, AccessFlags.setOf()); + if (rai.numDimensions() != 1) { throw new IllegalArgumentException("coordinate array " + dataset + " is expected to be 1-dimensional but has " - + attributes.getNumDimensions() + " dimensions"); + + rai.numDimensions() + " dimensions"); } - final int length = (int) attributes.getDimensions()[0]; - final int chunkSize = attributes.getBlockSize()[0]; - final DataType dataType = attributes.getDataType(); - final double[] values = new double[length]; - final int numChunks = (int) Math.ceil((double) length / chunkSize); - for (int chunk = 0; chunk < numChunks; chunk++) { - final DataBlock block = reader.readBlock(dataset, attributes, (long) chunk); - if (block == null) { - throw new IllegalArgumentException("missing chunk " + chunk + " of coordinate array " + dataset); - } - copyBlockValues(block.getData(), dataType, values, chunk * chunkSize); + final double[] values = new double[(int) rai.dimension(0)]; + final RandomAccess access = rai.randomAccess(); + for (int i = 0; i < values.length; i++) { + values[i] = access.setPositionAndGet(i).getRealDouble(); } return values; } - /** Reads the raw .zarray JSON for a dataset from the per-array file, falling back to consolidated .zmetadata. */ - private JsonObject readZArrayJson(final String dataset) { - final Path perArray = Paths.get(params.zarrPath, dataset, ".zarray"); - try { - if (Files.isRegularFile(perArray)) { - return JsonParser.parseString(Files.readString(perArray)).getAsJsonObject(); - } - } catch (final Exception e) { - LOG.warn("readZArrayJson: failed to read {} ({})", perArray, e.getMessage()); - } - try { - final Path zMetadata = Paths.get(params.zarrPath, ".zmetadata"); - if (Files.isRegularFile(zMetadata)) { - final JsonObject root = JsonParser.parseString(Files.readString(zMetadata)).getAsJsonObject(); - final JsonObject metadata = root.getAsJsonObject("metadata"); - if (metadata != null) { - final JsonObject zArray = metadata.getAsJsonObject(dataset + "/.zarray"); - if (zArray != null) { - return zArray.deepCopy(); - } - } - } - } catch (final Exception e) { - LOG.warn("readZArrayJson: failed to read .zmetadata for {} ({})", dataset, e.getMessage()); - } - return null; - } - - /** Copies a decoded data block into dst[offset...], applying unsigned promotion based on the data type. */ - private static void copyBlockValues(final Object data, final DataType dataType, final double[] dst, final int offset) { - if (data instanceof byte[]) { - final byte[] a = (byte[]) data; - final boolean unsigned = dataType == DataType.UINT8; - for (int i = 0; i < a.length && offset + i < dst.length; i++) { - dst[offset + i] = unsigned ? (a[i] & 0xFF) : a[i]; - } - } else if (data instanceof short[]) { - final short[] a = (short[]) data; - final boolean unsigned = dataType == DataType.UINT16; - for (int i = 0; i < a.length && offset + i < dst.length; i++) { - dst[offset + i] = unsigned ? (a[i] & 0xFFFF) : a[i]; - } - } else if (data instanceof int[]) { - final int[] a = (int[]) data; - final boolean unsigned = dataType == DataType.UINT32; - for (int i = 0; i < a.length && offset + i < dst.length; i++) { - dst[offset + i] = unsigned ? (a[i] & 0xFFFFFFFFL) : a[i]; - } - } else if (data instanceof long[]) { - final long[] a = (long[]) data; - for (int i = 0; i < a.length && offset + i < dst.length; i++) { - dst[offset + i] = a[i]; - } - } else if (data instanceof float[]) { - final float[] a = (float[]) data; - for (int i = 0; i < a.length && offset + i < dst.length; i++) { - dst[offset + i] = a[i]; - } - } else if (data instanceof double[]) { - final double[] a = (double[]) data; - for (int i = 0; i < a.length && offset + i < dst.length; i++) { - dst[offset + i] = a[i]; - } - } else { - throw new IllegalArgumentException("unsupported coordinate block data type " + data.getClass()); - } - } - /** * Opens the (float32) homogenization array lazily. *

diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/tile/RenderTilesClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/tile/RenderTilesClient.java index 6e5327e1f..fab5bb3c8 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/tile/RenderTilesClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/tile/RenderTilesClient.java @@ -63,6 +63,7 @@ import org.janelia.saalfeldlab.googlecloud.GoogleCloudUtils; import org.janelia.saalfeldlab.n5.KeyValueAccess; import org.janelia.saalfeldlab.n5.LockedChannel; +import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.googlecloud.GoogleCloudStorageKeyValueAccess; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -659,7 +660,7 @@ void ensureWritableDirectory(final URI uri) { if (!keyValueAccess.exists(uri.getPath())) { try { keyValueAccess.createDirectories(uri.getPath()); - } catch (final IOException e) { + } catch (final N5Exception e) { throw new RuntimeException("Could not create directory " + uri, e); } } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/Wafer6061Inpainter.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/Wafer6061Inpainter.java index b05bc3090..54d7628c6 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/Wafer6061Inpainter.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/Wafer6061Inpainter.java @@ -24,6 +24,7 @@ import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.janelia.saalfeldlab.n5.universe.StorageFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,7 +107,7 @@ public void run() { // Read and cache some metadata of the tissue and mask datasets // Assume that the tissue is a multiscale pyramid / mask is a standalone dataset - try (final N5Reader n5 = new N5Factory().openReader(N5Factory.StorageFormat.N5, param.n5Path)) { + try (final N5Reader n5 = new N5Factory().openReader(StorageFormat.N5, param.n5Path)) { LOG.info("Reading metadata from {}", param.n5Path); tissueAttributes = ExtendedAttributes.read(n5, param.fullDataset(), param.dataset); maskAttributes = ExtendedAttributes.read(n5, param.mask, param.mask); @@ -118,7 +119,7 @@ public void run() { throw new IllegalArgumentException("Dataset '" + param.output + "' is different from the input dataset and already exists. Stopping."); } else { LOG.info("Output dataset is '{}'. Creating new dataset.", param.output); - try (final N5Writer n5Writer = new N5Factory().openWriter(N5Factory.StorageFormat.N5, param.n5Path)) { + try (final N5Writer n5Writer = new N5Factory().openWriter(StorageFormat.N5, param.n5Path)) { n5Writer.createDataset(param.output, tissueAttributes.attrs); } } @@ -179,7 +180,7 @@ private static Grid.Block translateAndCheckHomogeneity( // Read the mask block and check if it is homogeneous boolean isHomogeneous = true; - try (final N5Reader n5 = new N5Factory().openReader(N5Factory.StorageFormat.N5, param.n5Path)) { + try (final N5Reader n5 = new N5Factory().openReader(StorageFormat.N5, param.n5Path)) { final Img mask = N5Utils.open(n5, param.mask); final Interval interval = Intervals.intersect(mask, block); final RandomAccessibleInterval maskPixels = Views.interval(mask, interval); @@ -240,7 +241,7 @@ private static void inpaintBlock( // Preallocate the inpainted block final Img inpaintedBlock = ArrayImgs.unsignedBytes(block.dimensions); - try (final N5Reader n5 = new N5Factory().openReader(N5Factory.StorageFormat.N5, param.n5Path)) { + try (final N5Reader n5 = new N5Factory().openReader(StorageFormat.N5, param.n5Path)) { // Load and translate the tissue and mask data LOG.info("Loading data at {}", block.offset); final Img rawTissue = N5Utils.open(n5, param.fullDataset()); @@ -265,7 +266,7 @@ private static void inpaintBlock( LOG.info("Finished inpainting in {} ms", System.currentTimeMillis() - start); } - try (final N5Writer n5Writer = new N5Factory().openWriter(N5Factory.StorageFormat.N5, param.n5Path)) { + try (final N5Writer n5Writer = new N5Factory().openWriter(StorageFormat.N5, param.n5Path)) { N5Utils.saveBlock(inpaintedBlock, n5Writer, param.output, targetAttributes, block.gridPosition); LOG.info("Wrote tissue block to '{}'", param.output); } catch (final Exception e) { diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/DownsampleHelper.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/DownsampleHelper.java index 642fad020..6c2a5eccb 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/DownsampleHelper.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/DownsampleHelper.java @@ -19,6 +19,7 @@ import org.janelia.saalfeldlab.n5.spark.downsample.N5DownsamplerSpark; import org.janelia.saalfeldlab.n5.spark.supplier.N5WriterSupplier; import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.janelia.saalfeldlab.n5.universe.StorageFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,7 +97,7 @@ public void run(final JavaSparkContext sparkContext) basePathOrStorageUrl, sZeroDatasetPath, Arrays.toString(downsampleFactors), requiredSLevel, retryParameters); final N5WriterSupplier n5Supplier = () -> - new N5Factory().openWriter(N5Factory.StorageFormat.N5, basePathOrStorageUrl); + new N5Factory().openWriter(StorageFormat.N5, basePathOrStorageUrl); final N5Writer n5 = n5Supplier.get(); final DatasetAttributes fullScaleAttributes = n5.getDatasetAttributes(sZeroDatasetPath); diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java index 1e420cd34..7afa5bcfd 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/N5Client.java @@ -40,7 +40,7 @@ import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.universe.N5Factory; -import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; +import org.janelia.saalfeldlab.n5.universe.StorageFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java index 026b643ad..aa87e2a62 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/n5/Util.java @@ -5,6 +5,7 @@ import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.spark.supplier.N5WriterSupplier; import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.janelia.saalfeldlab.n5.universe.StorageFormat; /** * Utilities for N5 operations. @@ -22,7 +23,7 @@ public N5PathSupplier(final String path) { @Override public N5Writer get() throws IOException { - return new N5Factory().openWriter(N5Factory.StorageFormat.N5, path); + return new N5Factory().openWriter(StorageFormat.N5, path); } } diff --git a/render-ws/pom.xml b/render-ws/pom.xml index 7cb96b0de..6785ea5db 100644 --- a/render-ws/pom.xml +++ b/render-ws/pom.xml @@ -124,6 +124,12 @@ jakarta.validation jakarta.validation-api + + + org.jboss.spec.javax.annotation + jboss-annotations-api_1.3_spec + @@ -141,6 +147,12 @@ jakarta.validation jakarta.validation-api + + + org.jboss.spec.javax.annotation + jboss-annotations-api_1.3_spec +